FILE
stmmac_main.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
1// SPDX-License-Identifier: GPL-2.0-only 2/******************************************************************************* 3 This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers. 4 ST Ethernet IPs are built around a Synopsys IP Core. 5 6 Copyright(C) 2007-2011 STMicroelectronics Ltd 7 8 9 Author: Giuseppe Cavallaro <peppe.cavallaro@st.com> 10 11 Documentation available at: 12 http://www.stlinux.com 13 Support available at: 14 https://bugzilla.stlinux.com/ 15*******************************************************************************/ 16 17#include <linux/clk.h> 18#include <linux/kernel.h> 19#include <linux/interrupt.h> 20#include <linux/ip.h> 21#include <linux/tcp.h> 22#include <linux/skbuff.h> 23#include <linux/ethtool.h> 24#include <linux/if_ether.h> 25#include <linux/crc32.h> 26#include <linux/mii.h> 27#include <linux/if.h> 28#include <linux/if_vlan.h> 29#include <linux/dma-mapping.h> 30#include <linux/slab.h> 31#include <linux/pm_runtime.h> 32#include <linux/pm_wakeirq.h> 33#include <linux/prefetch.h> 34#include <linux/pinctrl/consumer.h> 35#ifdef CONFIG_DEBUG_FS 36#include <linux/debugfs.h> 37#include <linux/seq_file.h> 38#endif /* CONFIG_DEBUG_FS */ 39#include <linux/net_tstamp.h> 40#include <linux/phylink.h> 41#include <linux/udp.h> 42#include <linux/bpf_trace.h> 43#include <net/devlink.h> 44#include <net/page_pool/helpers.h> 45#include <net/pkt_cls.h> 46#include <net/xdp_sock_drv.h> 47#include "stmmac_ptp.h" 48#include "stmmac_fpe.h" 49#include "stmmac.h" 50#include "stmmac_pcs.h" 51#include "stmmac_xdp.h" 52#include <linux/reset.h> 53#include <linux/of_mdio.h> 54#include "dwmac1000.h" 55#include "dwxgmac2.h" 56#include "hwif.h" 57 58/* As long as the interface is active, we keep the timestamping counter enabled 59 * with fine resolution and binary rollover. This avoid non-monotonic behavior 60 * (clock jumps) when changing timestamping settings at runtime. 61 */ 62#define STMMAC_HWTS_ACTIVE (PTP_TCR_TSENA | PTP_TCR_TSCTRLSSR) 63 64#define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16) 65#define TSO_MAX_BUFF_SIZE (SZ_16K - 1) 66 67/* Module parameters */ 68#define TX_TIMEO 5000 69static int watchdog = TX_TIMEO; 70module_param(watchdog, int, 0644); 71MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)"); 72 73static int debug = -1; 74module_param(debug, int, 0644); 75MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)"); 76 77static int phyaddr = -1; 78module_param(phyaddr, int, 0444); 79MODULE_PARM_DESC(phyaddr, "Physical device address"); 80 81#define STMMAC_TX_THRESH(x) ((x)->dma_conf.dma_tx_size / 4) 82 83/* Limit to make sure XDP TX and slow path can coexist */ 84#define STMMAC_XSK_TX_BUDGET_MAX 256 85#define STMMAC_TX_XSK_AVAIL 16 86#define STMMAC_RX_FILL_BATCH 16 87 88#define STMMAC_XDP_PASS 0 89#define STMMAC_XDP_CONSUMED BIT(0) 90#define STMMAC_XDP_TX BIT(1) 91#define STMMAC_XDP_REDIRECT BIT(2) 92#define STMMAC_XSK_CONSUMED BIT(3) 93 94static int flow_ctrl = 0xdead; 95module_param(flow_ctrl, int, 0644); 96MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off] (obsolete)"); 97 98static int pause = PAUSE_TIME; 99module_param(pause, int, 0644); 100MODULE_PARM_DESC(pause, "Flow Control Pause Time (units of 512 bit times)"); 101 102#define TC_DEFAULT 64 103static int tc = TC_DEFAULT; 104module_param(tc, int, 0644); 105MODULE_PARM_DESC(tc, "DMA threshold control value"); 106 107/* This is unused */ 108#define DEFAULT_BUFSIZE 1536 109static int buf_sz = DEFAULT_BUFSIZE; 110module_param(buf_sz, int, 0644); 111MODULE_PARM_DESC(buf_sz, "DMA buffer size"); 112 113static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE | 114 NETIF_MSG_LINK | NETIF_MSG_IFUP | 115 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER); 116 117#define STMMAC_DEFAULT_LPI_TIMER 1000 118static unsigned int eee_timer = STMMAC_DEFAULT_LPI_TIMER; 119module_param(eee_timer, uint, 0644); 120MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec"); 121#define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x)) 122 123/* By default the driver will use the ring mode to manage tx and rx descriptors, 124 * but allow user to force to use the chain instead of the ring 125 */ 126static unsigned int chain_mode; 127module_param(chain_mode, int, 0444); 128MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode"); 129 130static irqreturn_t stmmac_interrupt(int irq, void *dev_id); 131/* For MSI interrupts handling */ 132static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id); 133static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id); 134static irqreturn_t stmmac_msi_intr_tx(int irq, void *data); 135static irqreturn_t stmmac_msi_intr_rx(int irq, void *data); 136static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue); 137static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue); 138static void stmmac_reset_queues_param(struct stmmac_priv *priv); 139static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue); 140static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue); 141static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 142 u32 rxmode, u32 chan); 143static void stmmac_vlan_restore(struct stmmac_priv *priv); 144 145#ifdef CONFIG_DEBUG_FS 146static const struct net_device_ops stmmac_netdev_ops; 147static void stmmac_init_fs(struct net_device *dev); 148static void stmmac_exit_fs(struct net_device *dev); 149#endif 150 151#define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC)) 152 153struct stmmac_devlink_priv { 154 struct stmmac_priv *stmmac_priv; 155}; 156 157enum stmmac_dl_param_id { 158 STMMAC_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, 159 STMMAC_DEVLINK_PARAM_ID_TS_COARSE, 160}; 161 162/** 163 * stmmac_set_clk_tx_rate() - set the clock rate for the MAC transmit clock 164 * @bsp_priv: BSP private data structure (unused) 165 * @clk_tx_i: the transmit clock 166 * @interface: the selected interface mode 167 * @speed: the speed that the MAC will be operating at 168 * 169 * Set the transmit clock rate for the MAC, normally 2.5MHz for 10Mbps, 170 * 25MHz for 100Mbps and 125MHz for 1Gbps. This is suitable for at least 171 * MII, GMII, RGMII and RMII interface modes. Platforms can hook this into 172 * the plat_data->set_clk_tx_rate method directly, call it via their own 173 * implementation, or implement their own method should they have more 174 * complex requirements. It is intended to only be used in this method. 175 * 176 * plat_data->clk_tx_i must be filled in. 177 */ 178int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, 179 phy_interface_t interface, int speed) 180{ 181 long rate = rgmii_clock(speed); 182 183 /* Silently ignore unsupported speeds as rgmii_clock() only 184 * supports 10, 100 and 1000Mbps. We do not want to spit 185 * errors for 2500 and higher speeds here. 186 */ 187 if (rate < 0) 188 return 0; 189 190 return clk_set_rate(clk_tx_i, rate); 191} 192EXPORT_SYMBOL_GPL(stmmac_set_clk_tx_rate); 193 194/** 195 * stmmac_axi_blen_to_mask() - convert a burst length array to reg value 196 * @regval: pointer to a u32 for the resulting register value 197 * @blen: pointer to an array of u32 containing the burst length values in bytes 198 * @len: the number of entries in the @blen array 199 */ 200void stmmac_axi_blen_to_mask(u32 *regval, const u32 *blen, size_t len) 201{ 202 size_t i; 203 u32 val; 204 205 for (val = i = 0; i < len; i++) { 206 u32 burst = blen[i]; 207 208 /* Burst values of zero must be skipped. */ 209 if (!burst) 210 continue; 211 212 /* The valid range for the burst length is 4 to 256 inclusive, 213 * and it must be a power of two. 214 */ 215 if (burst < 4 || burst > 256 || !is_power_of_2(burst)) { 216 pr_err("stmmac: invalid burst length %u at index %zu\n", 217 burst, i); 218 continue; 219 } 220 221 /* Since burst is a power of two, and the register field starts 222 * with burst = 4, shift right by two bits so bit 0 of the field 223 * corresponds with the minimum value. 224 */ 225 val |= burst >> 2; 226 } 227 228 *regval = FIELD_PREP(DMA_AXI_BLEN_MASK, val); 229} 230EXPORT_SYMBOL_GPL(stmmac_axi_blen_to_mask); 231 232/** 233 * stmmac_verify_args - verify the driver parameters. 234 * Description: it checks the driver parameters and set a default in case of 235 * errors. 236 */ 237static void stmmac_verify_args(void) 238{ 239 if (unlikely(watchdog < 0)) 240 watchdog = TX_TIMEO; 241 if (unlikely((pause < 0) || (pause > 0xffff))) 242 pause = PAUSE_TIME; 243 244 if (flow_ctrl != 0xdead) 245 pr_warn("stmmac: module parameter 'flow_ctrl' is obsolete - please remove from your module configuration\n"); 246} 247 248static void __stmmac_disable_all_queues(struct stmmac_priv *priv) 249{ 250 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 251 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 252 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 253 u32 queue; 254 255 for (queue = 0; queue < maxq; queue++) { 256 struct stmmac_channel *ch = &priv->channel[queue]; 257 258 if (stmmac_xdp_is_enabled(priv) && 259 test_bit(queue, priv->af_xdp_zc_qps)) { 260 napi_disable(&ch->rxtx_napi); 261 continue; 262 } 263 264 if (queue < rx_queues_cnt) 265 napi_disable(&ch->rx_napi); 266 if (queue < tx_queues_cnt) 267 napi_disable(&ch->tx_napi); 268 } 269} 270 271/** 272 * stmmac_disable_all_queues - Disable all queues 273 * @priv: driver private structure 274 */ 275static void stmmac_disable_all_queues(struct stmmac_priv *priv) 276{ 277 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 278 struct stmmac_rx_queue *rx_q; 279 u32 queue; 280 281 /* synchronize_rcu() needed for pending XDP buffers to drain */ 282 for (queue = 0; queue < rx_queues_cnt; queue++) { 283 rx_q = &priv->dma_conf.rx_queue[queue]; 284 if (rx_q->xsk_pool) { 285 synchronize_rcu(); 286 break; 287 } 288 } 289 290 __stmmac_disable_all_queues(priv); 291} 292 293/** 294 * stmmac_enable_all_queues - Enable all queues 295 * @priv: driver private structure 296 */ 297static void stmmac_enable_all_queues(struct stmmac_priv *priv) 298{ 299 u32 rx_queues_cnt = priv->plat->rx_queues_to_use; 300 u32 tx_queues_cnt = priv->plat->tx_queues_to_use; 301 u32 maxq = max(rx_queues_cnt, tx_queues_cnt); 302 u32 queue; 303 304 for (queue = 0; queue < maxq; queue++) { 305 struct stmmac_channel *ch = &priv->channel[queue]; 306 307 if (stmmac_xdp_is_enabled(priv) && 308 test_bit(queue, priv->af_xdp_zc_qps)) { 309 napi_enable(&ch->rxtx_napi); 310 continue; 311 } 312 313 if (queue < rx_queues_cnt) 314 napi_enable(&ch->rx_napi); 315 if (queue < tx_queues_cnt) 316 napi_enable(&ch->tx_napi); 317 } 318} 319 320static void stmmac_service_event_schedule(struct stmmac_priv *priv) 321{ 322 if (!test_bit(STMMAC_DOWN, &priv->state) && 323 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state)) 324 queue_work(priv->wq, &priv->service_task); 325} 326 327static void stmmac_global_err(struct stmmac_priv *priv) 328{ 329 netif_carrier_off(priv->dev); 330 set_bit(STMMAC_RESET_REQUESTED, &priv->state); 331 stmmac_service_event_schedule(priv); 332} 333 334static void print_pkt(unsigned char *buf, int len) 335{ 336 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf); 337 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len); 338} 339 340static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) 341{ 342 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 343 u32 avail; 344 345 if (tx_q->dirty_tx > tx_q->cur_tx) 346 avail = tx_q->dirty_tx - tx_q->cur_tx - 1; 347 else 348 avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; 349 350 return avail; 351} 352 353/** 354 * stmmac_rx_dirty - Get RX queue dirty 355 * @priv: driver private structure 356 * @queue: RX queue index 357 */ 358static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) 359{ 360 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 361 u32 dirty; 362 363 if (rx_q->dirty_rx <= rx_q->cur_rx) 364 dirty = rx_q->cur_rx - rx_q->dirty_rx; 365 else 366 dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; 367 368 return dirty; 369} 370 371static bool stmmac_eee_tx_busy(struct stmmac_priv *priv) 372{ 373 u32 tx_cnt = priv->plat->tx_queues_to_use; 374 u32 queue; 375 376 /* check if all TX queues have the work finished */ 377 for (queue = 0; queue < tx_cnt; queue++) { 378 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 379 380 if (tx_q->dirty_tx != tx_q->cur_tx) 381 return true; /* still unfinished work */ 382 } 383 384 return false; 385} 386 387static void stmmac_restart_sw_lpi_timer(struct stmmac_priv *priv) 388{ 389 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer)); 390} 391 392/** 393 * stmmac_try_to_start_sw_lpi - check and enter in LPI mode 394 * @priv: driver private structure 395 * Description: this function is to verify and enter in LPI mode in case of 396 * EEE. 397 */ 398static void stmmac_try_to_start_sw_lpi(struct stmmac_priv *priv) 399{ 400 if (stmmac_eee_tx_busy(priv)) { 401 stmmac_restart_sw_lpi_timer(priv); 402 return; 403 } 404 405 /* Check and enter in LPI mode */ 406 if (!priv->tx_path_in_lpi_mode) 407 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_FORCED, 408 priv->tx_lpi_clk_stop, 0); 409} 410 411/** 412 * stmmac_stop_sw_lpi - stop transmitting LPI 413 * @priv: driver private structure 414 * Description: When using software-controlled LPI, stop transmitting LPI state. 415 */ 416static void stmmac_stop_sw_lpi(struct stmmac_priv *priv) 417{ 418 timer_delete_sync(&priv->eee_ctrl_timer); 419 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 420 priv->tx_path_in_lpi_mode = false; 421} 422 423/** 424 * stmmac_eee_ctrl_timer - EEE TX SW timer. 425 * @t: timer_list struct containing private info 426 * Description: 427 * if there is no data transfer and if we are not in LPI state, 428 * then MAC Transmitter can be moved to LPI state. 429 */ 430static void stmmac_eee_ctrl_timer(struct timer_list *t) 431{ 432 struct stmmac_priv *priv = timer_container_of(priv, t, eee_ctrl_timer); 433 434 stmmac_try_to_start_sw_lpi(priv); 435} 436 437/* stmmac_get_tx_hwtstamp - get HW TX timestamps 438 * @priv: driver private structure 439 * @p : descriptor pointer 440 * @skb : the socket buffer 441 * Description : 442 * This function will read timestamp from the descriptor & pass it to stack. 443 * and also perform some sanity checks. 444 */ 445static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, 446 struct dma_desc *p, struct sk_buff *skb) 447{ 448 struct skb_shared_hwtstamps shhwtstamp; 449 bool found = false; 450 u64 ns = 0; 451 452 if (!priv->hwts_tx_en) 453 return; 454 455 /* exit if skb doesn't support hw tstamp */ 456 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS))) 457 return; 458 459 /* check tx tstamp status */ 460 if (stmmac_get_tx_timestamp_status(priv, p)) { 461 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns); 462 found = true; 463 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 464 found = true; 465 } 466 467 if (found) { 468 ns -= priv->plat->cdc_error_adj; 469 470 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 471 shhwtstamp.hwtstamp = ns_to_ktime(ns); 472 473 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns); 474 /* pass tstamp to stack */ 475 skb_tstamp_tx(skb, &shhwtstamp); 476 } 477} 478 479/* stmmac_get_rx_hwtstamp - get HW RX timestamps 480 * @priv: driver private structure 481 * @p : descriptor pointer 482 * @np : next descriptor pointer 483 * @skb : the socket buffer 484 * Description : 485 * This function will read received packet's timestamp from the descriptor 486 * and pass it to stack. It also perform some sanity checks. 487 */ 488static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, 489 struct dma_desc *np, struct sk_buff *skb) 490{ 491 struct skb_shared_hwtstamps *shhwtstamp = NULL; 492 struct dma_desc *desc = p; 493 u64 ns = 0; 494 495 if (!priv->hwts_rx_en) 496 return; 497 /* For GMAC4, the valid timestamp is from CTX next desc. */ 498 if (dwmac_is_xmac(priv->plat->core_type)) 499 desc = np; 500 501 /* Check if timestamp is available */ 502 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { 503 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 504 505 ns -= priv->plat->cdc_error_adj; 506 507 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); 508 shhwtstamp = skb_hwtstamps(skb); 509 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); 510 shhwtstamp->hwtstamp = ns_to_ktime(ns); 511 } else { 512 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n"); 513 } 514} 515 516static void stmmac_update_subsecond_increment(struct stmmac_priv *priv) 517{ 518 bool xmac = dwmac_is_xmac(priv->plat->core_type); 519 u32 sec_inc = 0; 520 u64 temp = 0; 521 522 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 523 524 /* program Sub Second Increment reg */ 525 stmmac_config_sub_second_increment(priv, priv->ptpaddr, 526 priv->plat->clk_ptp_rate, 527 xmac, &sec_inc); 528 temp = div_u64(1000000000ULL, sec_inc); 529 530 /* Store sub second increment for later use */ 531 priv->sub_second_inc = sec_inc; 532 533 /* calculate default added value: 534 * formula is : 535 * addend = (2^32)/freq_div_ratio; 536 * where, freq_div_ratio = 1e9ns/sec_inc 537 */ 538 temp = (u64)(temp << 32); 539 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate); 540 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend); 541} 542 543/** 544 * stmmac_hwtstamp_set - control hardware timestamping. 545 * @dev: device pointer. 546 * @config: the timestamping configuration. 547 * @extack: netlink extended ack structure for error reporting. 548 * Description: 549 * This function configures the MAC to enable/disable both outgoing(TX) 550 * and incoming(RX) packets time stamping based on user input. 551 * Return Value: 552 * 0 on success and an appropriate -ve integer on failure. 553 */ 554static int stmmac_hwtstamp_set(struct net_device *dev, 555 struct kernel_hwtstamp_config *config, 556 struct netlink_ext_ack *extack) 557{ 558 struct stmmac_priv *priv = netdev_priv(dev); 559 u32 ptp_v2 = 0; 560 u32 tstamp_all = 0; 561 u32 ptp_over_ipv4_udp = 0; 562 u32 ptp_over_ipv6_udp = 0; 563 u32 ptp_over_ethernet = 0; 564 u32 snap_type_sel = 0; 565 u32 ts_master_en = 0; 566 u32 ts_event_en = 0; 567 568 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) { 569 NL_SET_ERR_MSG_MOD(extack, "No support for HW time stamping"); 570 priv->hwts_tx_en = 0; 571 priv->hwts_rx_en = 0; 572 573 return -EOPNOTSUPP; 574 } 575 576 if (!netif_running(dev)) { 577 NL_SET_ERR_MSG_MOD(extack, 578 "Cannot change timestamping configuration while down"); 579 return -ENODEV; 580 } 581 582 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n", 583 __func__, config->flags, config->tx_type, config->rx_filter); 584 585 if (config->tx_type != HWTSTAMP_TX_OFF && 586 config->tx_type != HWTSTAMP_TX_ON) 587 return -ERANGE; 588 589 if (priv->adv_ts) { 590 switch (config->rx_filter) { 591 case HWTSTAMP_FILTER_NONE: 592 /* time stamp no incoming packet at all */ 593 config->rx_filter = HWTSTAMP_FILTER_NONE; 594 break; 595 596 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 597 /* PTP v1, UDP, any kind of event packet */ 598 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 599 /* 'xmac' hardware can support Sync, Pdelay_Req and 600 * Pdelay_resp by setting bit14 and bits17/16 to 01 601 * This leaves Delay_Req timestamps out. 602 * Enable all events *and* general purpose message 603 * timestamping 604 */ 605 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 606 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 607 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 608 break; 609 610 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 611 /* PTP v1, UDP, Sync packet */ 612 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC; 613 /* take time stamp for SYNC messages only */ 614 ts_event_en = PTP_TCR_TSEVNTENA; 615 616 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 617 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 618 break; 619 620 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 621 /* PTP v1, UDP, Delay_req packet */ 622 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ; 623 /* take time stamp for Delay_Req messages only */ 624 ts_master_en = PTP_TCR_TSMSTRENA; 625 ts_event_en = PTP_TCR_TSEVNTENA; 626 627 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 628 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 629 break; 630 631 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 632 /* PTP v2, UDP, any kind of event packet */ 633 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; 634 ptp_v2 = PTP_TCR_TSVER2ENA; 635 /* take time stamp for all event messages */ 636 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 637 638 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 639 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 640 break; 641 642 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 643 /* PTP v2, UDP, Sync packet */ 644 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC; 645 ptp_v2 = PTP_TCR_TSVER2ENA; 646 /* take time stamp for SYNC messages only */ 647 ts_event_en = PTP_TCR_TSEVNTENA; 648 649 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 650 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 651 break; 652 653 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 654 /* PTP v2, UDP, Delay_req packet */ 655 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ; 656 ptp_v2 = PTP_TCR_TSVER2ENA; 657 /* take time stamp for Delay_Req messages only */ 658 ts_master_en = PTP_TCR_TSMSTRENA; 659 ts_event_en = PTP_TCR_TSEVNTENA; 660 661 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 662 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 663 break; 664 665 case HWTSTAMP_FILTER_PTP_V2_EVENT: 666 /* PTP v2/802.AS1 any layer, any kind of event packet */ 667 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; 668 ptp_v2 = PTP_TCR_TSVER2ENA; 669 snap_type_sel = PTP_TCR_SNAPTYPSEL_1; 670 if (priv->synopsys_id < DWMAC_CORE_4_10) 671 ts_event_en = PTP_TCR_TSEVNTENA; 672 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 673 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 674 ptp_over_ethernet = PTP_TCR_TSIPENA; 675 break; 676 677 case HWTSTAMP_FILTER_PTP_V2_SYNC: 678 /* PTP v2/802.AS1, any layer, Sync packet */ 679 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC; 680 ptp_v2 = PTP_TCR_TSVER2ENA; 681 /* take time stamp for SYNC messages only */ 682 ts_event_en = PTP_TCR_TSEVNTENA; 683 684 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 685 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 686 ptp_over_ethernet = PTP_TCR_TSIPENA; 687 break; 688 689 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 690 /* PTP v2/802.AS1, any layer, Delay_req packet */ 691 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ; 692 ptp_v2 = PTP_TCR_TSVER2ENA; 693 /* take time stamp for Delay_Req messages only */ 694 ts_master_en = PTP_TCR_TSMSTRENA; 695 ts_event_en = PTP_TCR_TSEVNTENA; 696 697 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA; 698 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA; 699 ptp_over_ethernet = PTP_TCR_TSIPENA; 700 break; 701 702 case HWTSTAMP_FILTER_NTP_ALL: 703 case HWTSTAMP_FILTER_ALL: 704 /* time stamp any incoming packet */ 705 config->rx_filter = HWTSTAMP_FILTER_ALL; 706 tstamp_all = PTP_TCR_TSENALL; 707 break; 708 709 default: 710 return -ERANGE; 711 } 712 } else { 713 switch (config->rx_filter) { 714 case HWTSTAMP_FILTER_NONE: 715 config->rx_filter = HWTSTAMP_FILTER_NONE; 716 break; 717 default: 718 /* PTP v1, UDP, any kind of event packet */ 719 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; 720 break; 721 } 722 } 723 priv->hwts_rx_en = config->rx_filter != HWTSTAMP_FILTER_NONE; 724 priv->hwts_tx_en = config->tx_type == HWTSTAMP_TX_ON; 725 726 priv->systime_flags = STMMAC_HWTS_ACTIVE; 727 if (!priv->tsfupdt_coarse) 728 priv->systime_flags |= PTP_TCR_TSCFUPDT; 729 730 if (priv->hwts_tx_en || priv->hwts_rx_en) { 731 priv->systime_flags |= tstamp_all | ptp_v2 | 732 ptp_over_ethernet | ptp_over_ipv6_udp | 733 ptp_over_ipv4_udp | ts_event_en | 734 ts_master_en | snap_type_sel; 735 } 736 737 stmmac_config_hw_tstamping(priv, priv->ptpaddr, priv->systime_flags); 738 739 priv->tstamp_config = *config; 740 741 return 0; 742} 743 744/** 745 * stmmac_hwtstamp_get - read hardware timestamping. 746 * @dev: device pointer. 747 * @config: the timestamping configuration. 748 * Description: 749 * This function obtain the current hardware timestamping settings 750 * as requested. 751 */ 752static int stmmac_hwtstamp_get(struct net_device *dev, 753 struct kernel_hwtstamp_config *config) 754{ 755 struct stmmac_priv *priv = netdev_priv(dev); 756 757 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) 758 return -EOPNOTSUPP; 759 760 *config = priv->tstamp_config; 761 762 return 0; 763} 764 765/** 766 * stmmac_init_tstamp_counter - init hardware timestamping counter 767 * @priv: driver private structure 768 * @systime_flags: timestamping flags 769 * Description: 770 * Initialize hardware counter for packet timestamping. 771 * This is valid as long as the interface is open and not suspended. 772 * Will be rerun after resuming from suspend, case in which the timestamping 773 * flags updated by stmmac_hwtstamp_set() also need to be restored. 774 */ 775static int stmmac_init_tstamp_counter(struct stmmac_priv *priv, 776 u32 systime_flags) 777{ 778 struct timespec64 now; 779 780 if (!priv->plat->clk_ptp_rate) { 781 netdev_err(priv->dev, "Invalid PTP clock rate"); 782 return -EINVAL; 783 } 784 785 stmmac_config_hw_tstamping(priv, priv->ptpaddr, systime_flags); 786 priv->systime_flags = systime_flags; 787 788 stmmac_update_subsecond_increment(priv); 789 790 /* initialize system time */ 791 ktime_get_real_ts64(&now); 792 793 /* lower 32 bits of tv_sec are safe until y2106 */ 794 stmmac_init_systime(priv, priv->ptpaddr, (u32)now.tv_sec, now.tv_nsec); 795 796 return 0; 797} 798 799/** 800 * stmmac_init_timestamping - initialise timestamping 801 * @priv: driver private structure 802 * Description: this is to verify if the HW supports the PTPv1 or PTPv2. 803 * This is done by looking at the HW cap. register. 804 * This function also registers the ptp driver. 805 */ 806static int stmmac_init_timestamping(struct stmmac_priv *priv) 807{ 808 bool xmac = dwmac_is_xmac(priv->plat->core_type); 809 int ret; 810 811 if (priv->plat->ptp_clk_freq_config) 812 priv->plat->ptp_clk_freq_config(priv); 813 814 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp)) { 815 netdev_info(priv->dev, "PTP not supported by HW\n"); 816 return -EOPNOTSUPP; 817 } 818 819 ret = stmmac_init_tstamp_counter(priv, STMMAC_HWTS_ACTIVE | 820 PTP_TCR_TSCFUPDT); 821 if (ret) { 822 netdev_warn(priv->dev, "PTP init failed\n"); 823 return ret; 824 } 825 826 priv->adv_ts = 0; 827 /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */ 828 if (xmac && priv->dma_cap.atime_stamp) 829 priv->adv_ts = 1; 830 /* Dwmac 3.x core with extend_desc can support adv_ts */ 831 else if (priv->extend_desc && priv->dma_cap.atime_stamp) 832 priv->adv_ts = 1; 833 834 if (priv->dma_cap.time_stamp) 835 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n"); 836 837 if (priv->adv_ts) 838 netdev_info(priv->dev, 839 "IEEE 1588-2008 Advanced Timestamp supported\n"); 840 841 priv->hwts_tx_en = 0; 842 priv->hwts_rx_en = 0; 843 844 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 845 stmmac_hwtstamp_correct_latency(priv, priv); 846 847 return 0; 848} 849 850static void stmmac_setup_ptp(struct stmmac_priv *priv) 851{ 852 int ret; 853 854 ret = clk_prepare_enable(priv->plat->clk_ptp_ref); 855 if (ret < 0) 856 netdev_warn(priv->dev, 857 "failed to enable PTP reference clock: %pe\n", 858 ERR_PTR(ret)); 859 860 if (stmmac_init_timestamping(priv) == 0) 861 stmmac_ptp_register(priv); 862} 863 864static void stmmac_release_ptp(struct stmmac_priv *priv) 865{ 866 stmmac_ptp_unregister(priv); 867 clk_disable_unprepare(priv->plat->clk_ptp_ref); 868} 869 870/** 871 * stmmac_mac_flow_ctrl - Configure flow control in all queues 872 * @priv: driver private structure 873 * @duplex: duplex passed to the next function 874 * @flow_ctrl: desired flow control modes 875 * Description: It is used for configuring the flow control in all queues 876 */ 877static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex, 878 unsigned int flow_ctrl) 879{ 880 u32 tx_cnt = priv->plat->tx_queues_to_use; 881 882 stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time, 883 tx_cnt); 884} 885 886static unsigned long stmmac_mac_get_caps(struct phylink_config *config, 887 phy_interface_t interface) 888{ 889 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 890 891 /* Refresh the MAC-specific capabilities */ 892 stmmac_mac_update_caps(priv); 893 894 config->mac_capabilities = priv->hw->link.caps; 895 896 if (priv->plat->max_speed) 897 phylink_limit_mac_speed(config, priv->plat->max_speed); 898 899 return config->mac_capabilities; 900} 901 902static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, 903 phy_interface_t interface) 904{ 905 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 906 struct phylink_pcs *pcs; 907 908 if (priv->plat->select_pcs) { 909 pcs = priv->plat->select_pcs(priv, interface); 910 if (!IS_ERR(pcs)) 911 return pcs; 912 } 913 914 /* The PCS control register is only relevant for SGMII, TBI and RTBI 915 * modes. We no longer support TBI or RTBI, so only configure this 916 * register when operating in SGMII mode with the integrated PCS. 917 */ 918 if (priv->hw->pcs & STMMAC_PCS_SGMII && priv->integrated_pcs) 919 return &priv->integrated_pcs->pcs; 920 921 return NULL; 922} 923 924static void stmmac_mac_config(struct phylink_config *config, unsigned int mode, 925 const struct phylink_link_state *state) 926{ 927 /* Nothing to do, xpcs_config() handles everything */ 928} 929 930static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode, 931 phy_interface_t interface) 932{ 933 struct net_device *ndev = to_net_dev(config->dev); 934 struct stmmac_priv *priv = netdev_priv(ndev); 935 936 if (priv->plat->mac_finish) 937 priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, interface); 938 939 return 0; 940} 941 942static void stmmac_mac_link_down(struct phylink_config *config, 943 unsigned int mode, phy_interface_t interface) 944{ 945 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 946 947 stmmac_mac_set(priv, priv->ioaddr, false); 948 if (priv->dma_cap.eee) 949 stmmac_set_eee_pls(priv, priv->hw, false); 950 951 if (stmmac_fpe_supported(priv)) 952 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, false); 953} 954 955static void stmmac_mac_link_up(struct phylink_config *config, 956 struct phy_device *phy, 957 unsigned int mode, phy_interface_t interface, 958 int speed, int duplex, 959 bool tx_pause, bool rx_pause) 960{ 961 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 962 unsigned int flow_ctrl; 963 u32 old_ctrl, ctrl; 964 int ret; 965 966 if ((priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 967 priv->plat->serdes_powerup) 968 priv->plat->serdes_powerup(priv->dev, priv->plat->bsp_priv); 969 970 old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); 971 ctrl = old_ctrl & ~priv->hw->link.speed_mask; 972 973 if (interface == PHY_INTERFACE_MODE_USXGMII) { 974 switch (speed) { 975 case SPEED_10000: 976 ctrl |= priv->hw->link.xgmii.speed10000; 977 break; 978 case SPEED_5000: 979 ctrl |= priv->hw->link.xgmii.speed5000; 980 break; 981 case SPEED_2500: 982 ctrl |= priv->hw->link.xgmii.speed2500; 983 break; 984 default: 985 return; 986 } 987 } else if (interface == PHY_INTERFACE_MODE_XLGMII) { 988 switch (speed) { 989 case SPEED_100000: 990 ctrl |= priv->hw->link.xlgmii.speed100000; 991 break; 992 case SPEED_50000: 993 ctrl |= priv->hw->link.xlgmii.speed50000; 994 break; 995 case SPEED_40000: 996 ctrl |= priv->hw->link.xlgmii.speed40000; 997 break; 998 case SPEED_25000: 999 ctrl |= priv->hw->link.xlgmii.speed25000; 1000 break; 1001 case SPEED_10000: 1002 ctrl |= priv->hw->link.xgmii.speed10000; 1003 break; 1004 case SPEED_2500: 1005 ctrl |= priv->hw->link.speed2500; 1006 break; 1007 case SPEED_1000: 1008 ctrl |= priv->hw->link.speed1000; 1009 break; 1010 default: 1011 return; 1012 } 1013 } else { 1014 switch (speed) { 1015 case SPEED_2500: 1016 ctrl |= priv->hw->link.speed2500; 1017 break; 1018 case SPEED_1000: 1019 ctrl |= priv->hw->link.speed1000; 1020 break; 1021 case SPEED_100: 1022 ctrl |= priv->hw->link.speed100; 1023 break; 1024 case SPEED_10: 1025 ctrl |= priv->hw->link.speed10; 1026 break; 1027 default: 1028 return; 1029 } 1030 } 1031 1032 if (priv->plat->fix_mac_speed) 1033 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); 1034 1035 if (!duplex) 1036 ctrl &= ~priv->hw->link.duplex; 1037 else 1038 ctrl |= priv->hw->link.duplex; 1039 1040 /* Flow Control operation */ 1041 if (rx_pause && tx_pause) 1042 flow_ctrl = FLOW_AUTO; 1043 else if (rx_pause && !tx_pause) 1044 flow_ctrl = FLOW_RX; 1045 else if (!rx_pause && tx_pause) 1046 flow_ctrl = FLOW_TX; 1047 else 1048 flow_ctrl = FLOW_OFF; 1049 1050 stmmac_mac_flow_ctrl(priv, duplex, flow_ctrl); 1051 1052 if (ctrl != old_ctrl) 1053 writel(ctrl, priv->ioaddr + MAC_CTRL_REG); 1054 1055 if (priv->plat->set_clk_tx_rate) { 1056 ret = priv->plat->set_clk_tx_rate(priv->plat->bsp_priv, 1057 priv->plat->clk_tx_i, 1058 interface, speed); 1059 if (ret < 0) 1060 netdev_err(priv->dev, 1061 "failed to configure %s transmit clock for %dMbps: %pe\n", 1062 phy_modes(interface), speed, ERR_PTR(ret)); 1063 } 1064 1065 stmmac_mac_set(priv, priv->ioaddr, true); 1066 if (priv->dma_cap.eee) 1067 stmmac_set_eee_pls(priv, priv->hw, true); 1068 1069 if (stmmac_fpe_supported(priv)) 1070 ethtool_mmsv_link_state_handle(&priv->fpe_cfg.mmsv, true); 1071 1072 if (priv->plat->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) 1073 stmmac_hwtstamp_correct_latency(priv, priv); 1074} 1075 1076static void stmmac_mac_disable_tx_lpi(struct phylink_config *config) 1077{ 1078 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1079 1080 priv->eee_active = false; 1081 1082 mutex_lock(&priv->lock); 1083 1084 priv->eee_enabled = false; 1085 1086 netdev_dbg(priv->dev, "disable EEE\n"); 1087 priv->eee_sw_timer_en = false; 1088 timer_delete_sync(&priv->eee_ctrl_timer); 1089 stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_DISABLE, false, 0); 1090 priv->tx_path_in_lpi_mode = false; 1091 1092 stmmac_set_eee_timer(priv, priv->hw, 0, STMMAC_DEFAULT_TWT_LS); 1093 mutex_unlock(&priv->lock); 1094} 1095 1096static int stmmac_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, 1097 bool tx_clk_stop) 1098{ 1099 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1100 int ret; 1101 1102 priv->tx_lpi_timer = timer; 1103 priv->eee_active = true; 1104 1105 mutex_lock(&priv->lock); 1106 1107 priv->eee_enabled = true; 1108 1109 /* Update the transmit clock stop according to PHY capability if 1110 * the platform allows 1111 */ 1112 if (priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP) 1113 priv->tx_lpi_clk_stop = tx_clk_stop; 1114 1115 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS, 1116 STMMAC_DEFAULT_TWT_LS); 1117 1118 /* Try to cnfigure the hardware timer. */ 1119 ret = stmmac_set_lpi_mode(priv, priv->hw, STMMAC_LPI_TIMER, 1120 priv->tx_lpi_clk_stop, priv->tx_lpi_timer); 1121 1122 if (ret) { 1123 /* Hardware timer mode not supported, or value out of range. 1124 * Fall back to using software LPI mode 1125 */ 1126 priv->eee_sw_timer_en = true; 1127 stmmac_restart_sw_lpi_timer(priv); 1128 } 1129 1130 mutex_unlock(&priv->lock); 1131 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n"); 1132 1133 return 0; 1134} 1135 1136static int stmmac_mac_wol_set(struct phylink_config *config, u32 wolopts, 1137 const u8 *sopass) 1138{ 1139 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev)); 1140 1141 device_set_wakeup_enable(priv->device, !!wolopts); 1142 1143 mutex_lock(&priv->lock); 1144 priv->wolopts = wolopts; 1145 mutex_unlock(&priv->lock); 1146 1147 return 0; 1148} 1149 1150static const struct phylink_mac_ops stmmac_phylink_mac_ops = { 1151 .mac_get_caps = stmmac_mac_get_caps, 1152 .mac_select_pcs = stmmac_mac_select_pcs, 1153 .mac_config = stmmac_mac_config, 1154 .mac_finish = stmmac_mac_finish, 1155 .mac_link_down = stmmac_mac_link_down, 1156 .mac_link_up = stmmac_mac_link_up, 1157 .mac_disable_tx_lpi = stmmac_mac_disable_tx_lpi, 1158 .mac_enable_tx_lpi = stmmac_mac_enable_tx_lpi, 1159 .mac_wol_set = stmmac_mac_wol_set, 1160}; 1161 1162/** 1163 * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported 1164 * @priv: driver private structure 1165 * Description: this is to verify if the HW supports the PCS. 1166 * Physical Coding Sublayer (PCS) interface that can be used when the MAC is 1167 * configured for the TBI, RTBI, or SGMII PHY interface. 1168 */ 1169static void stmmac_check_pcs_mode(struct stmmac_priv *priv) 1170{ 1171 int interface = priv->plat->phy_interface; 1172 int speed = priv->plat->mac_port_sel_speed; 1173 1174 if (priv->dma_cap.pcs && interface == PHY_INTERFACE_MODE_SGMII) { 1175 netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); 1176 priv->hw->pcs = STMMAC_PCS_SGMII; 1177 1178 switch (speed) { 1179 case SPEED_10: 1180 case SPEED_100: 1181 case SPEED_1000: 1182 priv->hw->reverse_sgmii_enable = true; 1183 break; 1184 1185 default: 1186 dev_warn(priv->device, "invalid port speed\n"); 1187 fallthrough; 1188 case 0: 1189 priv->hw->reverse_sgmii_enable = false; 1190 break; 1191 } 1192 } 1193} 1194 1195/** 1196 * stmmac_init_phy - PHY initialization 1197 * @dev: net device structure 1198 * Description: it initializes the driver's PHY state, and attaches the PHY 1199 * to the mac driver. 1200 * Return value: 1201 * 0 on success 1202 */ 1203static int stmmac_init_phy(struct net_device *dev) 1204{ 1205 struct stmmac_priv *priv = netdev_priv(dev); 1206 int mode = priv->plat->phy_interface; 1207 struct fwnode_handle *phy_fwnode; 1208 struct fwnode_handle *fwnode; 1209 struct ethtool_keee eee; 1210 int ret; 1211 1212 if (!phylink_expects_phy(priv->phylink)) 1213 return 0; 1214 1215 if (priv->hw->xpcs && 1216 xpcs_get_an_mode(priv->hw->xpcs, mode) == DW_AN_C73) 1217 return 0; 1218 1219 fwnode = priv->plat->port_node; 1220 if (!fwnode) 1221 fwnode = dev_fwnode(priv->device); 1222 1223 if (fwnode) 1224 phy_fwnode = fwnode_get_phy_node(fwnode); 1225 else 1226 phy_fwnode = NULL; 1227 1228 /* Some DT bindings do not set-up the PHY handle. Let's try to 1229 * manually parse it 1230 */ 1231 if (!phy_fwnode || IS_ERR(phy_fwnode)) { 1232 int addr = priv->plat->phy_addr; 1233 struct phy_device *phydev; 1234 1235 if (addr < 0) { 1236 netdev_err(priv->dev, "no phy found\n"); 1237 return -ENODEV; 1238 } 1239 1240 phydev = mdiobus_get_phy(priv->mii, addr); 1241 if (!phydev) { 1242 netdev_err(priv->dev, "no phy at addr %d\n", addr); 1243 return -ENODEV; 1244 } 1245 1246 ret = phylink_connect_phy(priv->phylink, phydev); 1247 } else { 1248 fwnode_handle_put(phy_fwnode); 1249 ret = phylink_fwnode_phy_connect(priv->phylink, fwnode, 0); 1250 } 1251 1252 if (ret) { 1253 netdev_err(priv->dev, "cannot attach to PHY (error: %pe)\n", 1254 ERR_PTR(ret)); 1255 return ret; 1256 } 1257 1258 /* Configure phylib's copy of the LPI timer. Normally, 1259 * phylink_config.lpi_timer_default would do this, but there is a 1260 * chance that userspace could change the eee_timer setting via sysfs 1261 * before the first open. Thus, preserve existing behaviour. 1262 */ 1263 if (!phylink_ethtool_get_eee(priv->phylink, &eee)) { 1264 eee.tx_lpi_timer = priv->tx_lpi_timer; 1265 phylink_ethtool_set_eee(priv->phylink, &eee); 1266 } 1267 1268 return 0; 1269} 1270 1271static int stmmac_phylink_setup(struct stmmac_priv *priv) 1272{ 1273 struct stmmac_mdio_bus_data *mdio_bus_data; 1274 struct phylink_config *config; 1275 struct fwnode_handle *fwnode; 1276 struct phylink_pcs *pcs; 1277 struct phylink *phylink; 1278 1279 config = &priv->phylink_config; 1280 1281 config->dev = &priv->dev->dev; 1282 config->type = PHYLINK_NETDEV; 1283 config->mac_managed_pm = true; 1284 1285 /* Stmmac always requires an RX clock for hardware initialization */ 1286 config->mac_requires_rxc = true; 1287 1288 /* Disable EEE RX clock stop to ensure VLAN register access works 1289 * correctly. 1290 */ 1291 if (!(priv->plat->flags & STMMAC_FLAG_RX_CLK_RUNS_IN_LPI) && 1292 !(priv->dev->features & NETIF_F_VLAN_FEATURES)) 1293 config->eee_rx_clk_stop_enable = true; 1294 1295 /* Set the default transmit clock stop bit based on the platform glue */ 1296 priv->tx_lpi_clk_stop = priv->plat->flags & 1297 STMMAC_FLAG_EN_TX_LPI_CLOCKGATING; 1298 1299 mdio_bus_data = priv->plat->mdio_bus_data; 1300 if (mdio_bus_data) 1301 config->default_an_inband = mdio_bus_data->default_an_inband; 1302 1303 /* Get the PHY interface modes (at the PHY end of the link) that 1304 * are supported by the platform. 1305 */ 1306 if (priv->plat->get_interfaces) 1307 priv->plat->get_interfaces(priv, priv->plat->bsp_priv, 1308 config->supported_interfaces); 1309 1310 /* Set the platform/firmware specified interface mode if the 1311 * supported interfaces have not already been provided using 1312 * phy_interface as a last resort. 1313 */ 1314 if (phy_interface_empty(config->supported_interfaces)) 1315 __set_bit(priv->plat->phy_interface, 1316 config->supported_interfaces); 1317 1318 /* If we have an xpcs, it defines which PHY interfaces are supported. */ 1319 if (priv->hw->xpcs) 1320 pcs = xpcs_to_phylink_pcs(priv->hw->xpcs); 1321 else 1322 pcs = priv->hw->phylink_pcs; 1323 1324 if (pcs) 1325 phy_interface_or(config->supported_interfaces, 1326 config->supported_interfaces, 1327 pcs->supported_interfaces); 1328 1329 if (priv->dma_cap.eee) { 1330 /* Assume all supported interfaces also support LPI */ 1331 memcpy(config->lpi_interfaces, config->supported_interfaces, 1332 sizeof(config->lpi_interfaces)); 1333 1334 /* All full duplex speeds above 100Mbps are supported */ 1335 config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD; 1336 config->lpi_timer_default = eee_timer * 1000; 1337 config->eee_enabled_default = true; 1338 } 1339 1340 config->wol_phy_speed_ctrl = true; 1341 if (priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL) { 1342 config->wol_phy_legacy = true; 1343 } else { 1344 if (priv->dma_cap.pmt_remote_wake_up) 1345 config->wol_mac_support |= WAKE_UCAST; 1346 if (priv->dma_cap.pmt_magic_frame) 1347 config->wol_mac_support |= WAKE_MAGIC; 1348 } 1349 1350 fwnode = priv->plat->port_node; 1351 if (!fwnode) 1352 fwnode = dev_fwnode(priv->device); 1353 1354 phylink = phylink_create(config, fwnode, priv->plat->phy_interface, 1355 &stmmac_phylink_mac_ops); 1356 if (IS_ERR(phylink)) 1357 return PTR_ERR(phylink); 1358 1359 priv->phylink = phylink; 1360 return 0; 1361} 1362 1363static void stmmac_display_rx_rings(struct stmmac_priv *priv, 1364 struct stmmac_dma_conf *dma_conf) 1365{ 1366 u32 rx_cnt = priv->plat->rx_queues_to_use; 1367 unsigned int desc_size; 1368 void *head_rx; 1369 u32 queue; 1370 1371 /* Display RX rings */ 1372 for (queue = 0; queue < rx_cnt; queue++) { 1373 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1374 1375 pr_info("\tRX Queue %u rings\n", queue); 1376 1377 if (priv->extend_desc) { 1378 head_rx = (void *)rx_q->dma_erx; 1379 desc_size = sizeof(struct dma_extended_desc); 1380 } else { 1381 head_rx = (void *)rx_q->dma_rx; 1382 desc_size = sizeof(struct dma_desc); 1383 } 1384 1385 /* Display RX ring */ 1386 stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, 1387 rx_q->dma_rx_phy, desc_size); 1388 } 1389} 1390 1391static void stmmac_display_tx_rings(struct stmmac_priv *priv, 1392 struct stmmac_dma_conf *dma_conf) 1393{ 1394 u32 tx_cnt = priv->plat->tx_queues_to_use; 1395 unsigned int desc_size; 1396 void *head_tx; 1397 u32 queue; 1398 1399 /* Display TX rings */ 1400 for (queue = 0; queue < tx_cnt; queue++) { 1401 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1402 1403 pr_info("\tTX Queue %d rings\n", queue); 1404 1405 if (priv->extend_desc) { 1406 head_tx = (void *)tx_q->dma_etx; 1407 desc_size = sizeof(struct dma_extended_desc); 1408 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 1409 head_tx = (void *)tx_q->dma_entx; 1410 desc_size = sizeof(struct dma_edesc); 1411 } else { 1412 head_tx = (void *)tx_q->dma_tx; 1413 desc_size = sizeof(struct dma_desc); 1414 } 1415 1416 stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, 1417 tx_q->dma_tx_phy, desc_size); 1418 } 1419} 1420 1421static void stmmac_display_rings(struct stmmac_priv *priv, 1422 struct stmmac_dma_conf *dma_conf) 1423{ 1424 /* Display RX ring */ 1425 stmmac_display_rx_rings(priv, dma_conf); 1426 1427 /* Display TX ring */ 1428 stmmac_display_tx_rings(priv, dma_conf); 1429} 1430 1431static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) 1432{ 1433 if (stmmac_xdp_is_enabled(priv)) 1434 return XDP_PACKET_HEADROOM; 1435 1436 return NET_SKB_PAD; 1437} 1438 1439static int stmmac_set_bfsize(int mtu) 1440{ 1441 int ret; 1442 1443 if (mtu >= BUF_SIZE_8KiB) 1444 ret = BUF_SIZE_16KiB; 1445 else if (mtu >= BUF_SIZE_4KiB) 1446 ret = BUF_SIZE_8KiB; 1447 else if (mtu >= BUF_SIZE_2KiB) 1448 ret = BUF_SIZE_4KiB; 1449 else if (mtu > DEFAULT_BUFSIZE) 1450 ret = BUF_SIZE_2KiB; 1451 else 1452 ret = DEFAULT_BUFSIZE; 1453 1454 return ret; 1455} 1456 1457/** 1458 * stmmac_clear_rx_descriptors - clear RX descriptors 1459 * @priv: driver private structure 1460 * @dma_conf: structure to take the dma data 1461 * @queue: RX queue index 1462 * Description: this function is called to clear the RX descriptors 1463 * in case of both basic and extended descriptors are used. 1464 */ 1465static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, 1466 struct stmmac_dma_conf *dma_conf, 1467 u32 queue) 1468{ 1469 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1470 int i; 1471 1472 /* Clear the RX descriptors */ 1473 for (i = 0; i < dma_conf->dma_rx_size; i++) 1474 if (priv->extend_desc) 1475 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, 1476 priv->use_riwt, priv->mode, 1477 (i == dma_conf->dma_rx_size - 1), 1478 dma_conf->dma_buf_sz); 1479 else 1480 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], 1481 priv->use_riwt, priv->mode, 1482 (i == dma_conf->dma_rx_size - 1), 1483 dma_conf->dma_buf_sz); 1484} 1485 1486/** 1487 * stmmac_clear_tx_descriptors - clear tx descriptors 1488 * @priv: driver private structure 1489 * @dma_conf: structure to take the dma data 1490 * @queue: TX queue index. 1491 * Description: this function is called to clear the TX descriptors 1492 * in case of both basic and extended descriptors are used. 1493 */ 1494static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, 1495 struct stmmac_dma_conf *dma_conf, 1496 u32 queue) 1497{ 1498 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1499 int i; 1500 1501 /* Clear the TX descriptors */ 1502 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1503 int last = (i == (dma_conf->dma_tx_size - 1)); 1504 struct dma_desc *p; 1505 1506 if (priv->extend_desc) 1507 p = &tx_q->dma_etx[i].basic; 1508 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1509 p = &tx_q->dma_entx[i].basic; 1510 else 1511 p = &tx_q->dma_tx[i]; 1512 1513 stmmac_init_tx_desc(priv, p, priv->mode, last); 1514 } 1515} 1516 1517/** 1518 * stmmac_clear_descriptors - clear descriptors 1519 * @priv: driver private structure 1520 * @dma_conf: structure to take the dma data 1521 * Description: this function is called to clear the TX and RX descriptors 1522 * in case of both basic and extended descriptors are used. 1523 */ 1524static void stmmac_clear_descriptors(struct stmmac_priv *priv, 1525 struct stmmac_dma_conf *dma_conf) 1526{ 1527 u32 rx_queue_cnt = priv->plat->rx_queues_to_use; 1528 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 1529 u32 queue; 1530 1531 /* Clear the RX descriptors */ 1532 for (queue = 0; queue < rx_queue_cnt; queue++) 1533 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1534 1535 /* Clear the TX descriptors */ 1536 for (queue = 0; queue < tx_queue_cnt; queue++) 1537 stmmac_clear_tx_descriptors(priv, dma_conf, queue); 1538} 1539 1540/** 1541 * stmmac_init_rx_buffers - init the RX descriptor buffer. 1542 * @priv: driver private structure 1543 * @dma_conf: structure to take the dma data 1544 * @p: descriptor pointer 1545 * @i: descriptor index 1546 * @flags: gfp flag 1547 * @queue: RX queue index 1548 * Description: this function is called to allocate a receive buffer, perform 1549 * the DMA mapping and init the descriptor. 1550 */ 1551static int stmmac_init_rx_buffers(struct stmmac_priv *priv, 1552 struct stmmac_dma_conf *dma_conf, 1553 struct dma_desc *p, 1554 int i, gfp_t flags, u32 queue) 1555{ 1556 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1557 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1558 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 1559 1560 if (priv->dma_cap.host_dma_width <= 32) 1561 gfp |= GFP_DMA32; 1562 1563 if (!buf->page) { 1564 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1565 if (!buf->page) 1566 return -ENOMEM; 1567 buf->page_offset = stmmac_rx_offset(priv); 1568 } 1569 1570 if (priv->sph_active && !buf->sec_page) { 1571 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 1572 if (!buf->sec_page) 1573 return -ENOMEM; 1574 1575 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 1576 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 1577 } else { 1578 buf->sec_page = NULL; 1579 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 1580 } 1581 1582 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 1583 1584 stmmac_set_desc_addr(priv, p, buf->addr); 1585 if (dma_conf->dma_buf_sz == BUF_SIZE_16KiB) 1586 stmmac_init_desc3(priv, p); 1587 1588 return 0; 1589} 1590 1591/** 1592 * stmmac_free_rx_buffer - free RX dma buffers 1593 * @priv: private structure 1594 * @rx_q: RX queue 1595 * @i: buffer index. 1596 */ 1597static void stmmac_free_rx_buffer(struct stmmac_priv *priv, 1598 struct stmmac_rx_queue *rx_q, 1599 int i) 1600{ 1601 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1602 1603 if (buf->page) 1604 page_pool_put_full_page(rx_q->page_pool, buf->page, false); 1605 buf->page = NULL; 1606 1607 if (buf->sec_page) 1608 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false); 1609 buf->sec_page = NULL; 1610} 1611 1612/** 1613 * stmmac_free_tx_buffer - free RX dma buffers 1614 * @priv: private structure 1615 * @dma_conf: structure to take the dma data 1616 * @queue: RX queue index 1617 * @i: buffer index. 1618 */ 1619static void stmmac_free_tx_buffer(struct stmmac_priv *priv, 1620 struct stmmac_dma_conf *dma_conf, 1621 u32 queue, int i) 1622{ 1623 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1624 1625 if (tx_q->tx_skbuff_dma[i].buf && 1626 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { 1627 if (tx_q->tx_skbuff_dma[i].map_as_page) 1628 dma_unmap_page(priv->device, 1629 tx_q->tx_skbuff_dma[i].buf, 1630 tx_q->tx_skbuff_dma[i].len, 1631 DMA_TO_DEVICE); 1632 else 1633 dma_unmap_single(priv->device, 1634 tx_q->tx_skbuff_dma[i].buf, 1635 tx_q->tx_skbuff_dma[i].len, 1636 DMA_TO_DEVICE); 1637 } 1638 1639 if (tx_q->xdpf[i] && 1640 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX || 1641 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) { 1642 xdp_return_frame(tx_q->xdpf[i]); 1643 tx_q->xdpf[i] = NULL; 1644 } 1645 1646 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX) 1647 tx_q->xsk_frames_done++; 1648 1649 if (tx_q->tx_skbuff[i] && 1650 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { 1651 dev_kfree_skb_any(tx_q->tx_skbuff[i]); 1652 tx_q->tx_skbuff[i] = NULL; 1653 } 1654 1655 tx_q->tx_skbuff_dma[i].buf = 0; 1656 tx_q->tx_skbuff_dma[i].map_as_page = false; 1657} 1658 1659/** 1660 * dma_free_rx_skbufs - free RX dma buffers 1661 * @priv: private structure 1662 * @dma_conf: structure to take the dma data 1663 * @queue: RX queue index 1664 */ 1665static void dma_free_rx_skbufs(struct stmmac_priv *priv, 1666 struct stmmac_dma_conf *dma_conf, 1667 u32 queue) 1668{ 1669 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1670 int i; 1671 1672 for (i = 0; i < dma_conf->dma_rx_size; i++) 1673 stmmac_free_rx_buffer(priv, rx_q, i); 1674} 1675 1676static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, 1677 struct stmmac_dma_conf *dma_conf, 1678 u32 queue, gfp_t flags) 1679{ 1680 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1681 int i; 1682 1683 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1684 struct dma_desc *p; 1685 int ret; 1686 1687 if (priv->extend_desc) 1688 p = &((rx_q->dma_erx + i)->basic); 1689 else 1690 p = rx_q->dma_rx + i; 1691 1692 ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, 1693 queue); 1694 if (ret) 1695 return ret; 1696 1697 rx_q->buf_alloc_num++; 1698 } 1699 1700 return 0; 1701} 1702 1703/** 1704 * dma_free_rx_xskbufs - free RX dma buffers from XSK pool 1705 * @priv: private structure 1706 * @dma_conf: structure to take the dma data 1707 * @queue: RX queue index 1708 */ 1709static void dma_free_rx_xskbufs(struct stmmac_priv *priv, 1710 struct stmmac_dma_conf *dma_conf, 1711 u32 queue) 1712{ 1713 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1714 int i; 1715 1716 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1717 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i]; 1718 1719 if (!buf->xdp) 1720 continue; 1721 1722 xsk_buff_free(buf->xdp); 1723 buf->xdp = NULL; 1724 } 1725} 1726 1727static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, 1728 struct stmmac_dma_conf *dma_conf, 1729 u32 queue) 1730{ 1731 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1732 int i; 1733 1734 /* struct stmmac_xdp_buff is using cb field (maximum size of 24 bytes) 1735 * in struct xdp_buff_xsk to stash driver specific information. Thus, 1736 * use this macro to make sure no size violations. 1737 */ 1738 XSK_CHECK_PRIV_TYPE(struct stmmac_xdp_buff); 1739 1740 for (i = 0; i < dma_conf->dma_rx_size; i++) { 1741 struct stmmac_rx_buffer *buf; 1742 dma_addr_t dma_addr; 1743 struct dma_desc *p; 1744 1745 if (priv->extend_desc) 1746 p = (struct dma_desc *)(rx_q->dma_erx + i); 1747 else 1748 p = rx_q->dma_rx + i; 1749 1750 buf = &rx_q->buf_pool[i]; 1751 1752 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 1753 if (!buf->xdp) 1754 return -ENOMEM; 1755 1756 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 1757 stmmac_set_desc_addr(priv, p, dma_addr); 1758 rx_q->buf_alloc_num++; 1759 } 1760 1761 return 0; 1762} 1763 1764static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue) 1765{ 1766 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps)) 1767 return NULL; 1768 1769 return xsk_get_pool_from_qid(priv->dev, queue); 1770} 1771 1772/** 1773 * __init_dma_rx_desc_rings - init the RX descriptor ring (per queue) 1774 * @priv: driver private structure 1775 * @dma_conf: structure to take the dma data 1776 * @queue: RX queue index 1777 * @flags: gfp flag. 1778 * Description: this function initializes the DMA RX descriptors 1779 * and allocates the socket buffers. It supports the chained and ring 1780 * modes. 1781 */ 1782static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, 1783 struct stmmac_dma_conf *dma_conf, 1784 u32 queue, gfp_t flags) 1785{ 1786 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1787 int ret; 1788 1789 netif_dbg(priv, probe, priv->dev, 1790 "(%s) dma_rx_phy=0x%08x\n", __func__, 1791 (u32)rx_q->dma_rx_phy); 1792 1793 stmmac_clear_rx_descriptors(priv, dma_conf, queue); 1794 1795 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq); 1796 1797 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1798 1799 if (rx_q->xsk_pool) { 1800 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1801 MEM_TYPE_XSK_BUFF_POOL, 1802 NULL)); 1803 netdev_info(priv->dev, 1804 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", 1805 rx_q->queue_index); 1806 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); 1807 } else { 1808 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, 1809 MEM_TYPE_PAGE_POOL, 1810 rx_q->page_pool)); 1811 netdev_info(priv->dev, 1812 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", 1813 rx_q->queue_index); 1814 } 1815 1816 if (rx_q->xsk_pool) { 1817 /* RX XDP ZC buffer pool may not be populated, e.g. 1818 * xdpsock TX-only. 1819 */ 1820 stmmac_alloc_rx_buffers_zc(priv, dma_conf, queue); 1821 } else { 1822 ret = stmmac_alloc_rx_buffers(priv, dma_conf, queue, flags); 1823 if (ret < 0) 1824 return -ENOMEM; 1825 } 1826 1827 /* Setup the chained descriptor addresses */ 1828 if (priv->mode == STMMAC_CHAIN_MODE) { 1829 if (priv->extend_desc) 1830 stmmac_mode_init(priv, rx_q->dma_erx, 1831 rx_q->dma_rx_phy, 1832 dma_conf->dma_rx_size, 1); 1833 else 1834 stmmac_mode_init(priv, rx_q->dma_rx, 1835 rx_q->dma_rx_phy, 1836 dma_conf->dma_rx_size, 0); 1837 } 1838 1839 return 0; 1840} 1841 1842static int init_dma_rx_desc_rings(struct net_device *dev, 1843 struct stmmac_dma_conf *dma_conf, 1844 gfp_t flags) 1845{ 1846 struct stmmac_priv *priv = netdev_priv(dev); 1847 u32 rx_count = priv->plat->rx_queues_to_use; 1848 int queue; 1849 int ret; 1850 1851 /* RX INITIALIZATION */ 1852 netif_dbg(priv, probe, priv->dev, 1853 "SKB addresses:\nskb\t\tskb data\tdma data\n"); 1854 1855 for (queue = 0; queue < rx_count; queue++) { 1856 ret = __init_dma_rx_desc_rings(priv, dma_conf, queue, flags); 1857 if (ret) 1858 goto err_init_rx_buffers; 1859 } 1860 1861 return 0; 1862 1863err_init_rx_buffers: 1864 while (queue >= 0) { 1865 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 1866 1867 if (rx_q->xsk_pool) 1868 dma_free_rx_xskbufs(priv, dma_conf, queue); 1869 else 1870 dma_free_rx_skbufs(priv, dma_conf, queue); 1871 1872 rx_q->buf_alloc_num = 0; 1873 rx_q->xsk_pool = NULL; 1874 1875 queue--; 1876 } 1877 1878 return ret; 1879} 1880 1881/** 1882 * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) 1883 * @priv: driver private structure 1884 * @dma_conf: structure to take the dma data 1885 * @queue: TX queue index 1886 * Description: this function initializes the DMA TX descriptors 1887 * and allocates the socket buffers. It supports the chained and ring 1888 * modes. 1889 */ 1890static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, 1891 struct stmmac_dma_conf *dma_conf, 1892 u32 queue) 1893{ 1894 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1895 int i; 1896 1897 netif_dbg(priv, probe, priv->dev, 1898 "(%s) dma_tx_phy=0x%08x\n", __func__, 1899 (u32)tx_q->dma_tx_phy); 1900 1901 /* Setup the chained descriptor addresses */ 1902 if (priv->mode == STMMAC_CHAIN_MODE) { 1903 if (priv->extend_desc) 1904 stmmac_mode_init(priv, tx_q->dma_etx, 1905 tx_q->dma_tx_phy, 1906 dma_conf->dma_tx_size, 1); 1907 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) 1908 stmmac_mode_init(priv, tx_q->dma_tx, 1909 tx_q->dma_tx_phy, 1910 dma_conf->dma_tx_size, 0); 1911 } 1912 1913 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue); 1914 1915 for (i = 0; i < dma_conf->dma_tx_size; i++) { 1916 struct dma_desc *p; 1917 1918 if (priv->extend_desc) 1919 p = &((tx_q->dma_etx + i)->basic); 1920 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 1921 p = &((tx_q->dma_entx + i)->basic); 1922 else 1923 p = tx_q->dma_tx + i; 1924 1925 stmmac_clear_desc(priv, p); 1926 1927 tx_q->tx_skbuff_dma[i].buf = 0; 1928 tx_q->tx_skbuff_dma[i].map_as_page = false; 1929 tx_q->tx_skbuff_dma[i].len = 0; 1930 tx_q->tx_skbuff_dma[i].last_segment = false; 1931 tx_q->tx_skbuff[i] = NULL; 1932 } 1933 1934 return 0; 1935} 1936 1937static int init_dma_tx_desc_rings(struct net_device *dev, 1938 struct stmmac_dma_conf *dma_conf) 1939{ 1940 struct stmmac_priv *priv = netdev_priv(dev); 1941 u32 tx_queue_cnt; 1942 u32 queue; 1943 1944 tx_queue_cnt = priv->plat->tx_queues_to_use; 1945 1946 for (queue = 0; queue < tx_queue_cnt; queue++) 1947 __init_dma_tx_desc_rings(priv, dma_conf, queue); 1948 1949 return 0; 1950} 1951 1952/** 1953 * init_dma_desc_rings - init the RX/TX descriptor rings 1954 * @dev: net device structure 1955 * @dma_conf: structure to take the dma data 1956 * @flags: gfp flag. 1957 * Description: this function initializes the DMA RX/TX descriptors 1958 * and allocates the socket buffers. It supports the chained and ring 1959 * modes. 1960 */ 1961static int init_dma_desc_rings(struct net_device *dev, 1962 struct stmmac_dma_conf *dma_conf, 1963 gfp_t flags) 1964{ 1965 struct stmmac_priv *priv = netdev_priv(dev); 1966 int ret; 1967 1968 ret = init_dma_rx_desc_rings(dev, dma_conf, flags); 1969 if (ret) 1970 return ret; 1971 1972 ret = init_dma_tx_desc_rings(dev, dma_conf); 1973 1974 stmmac_clear_descriptors(priv, dma_conf); 1975 1976 if (netif_msg_hw(priv)) 1977 stmmac_display_rings(priv, dma_conf); 1978 1979 return ret; 1980} 1981 1982/** 1983 * dma_free_tx_skbufs - free TX dma buffers 1984 * @priv: private structure 1985 * @dma_conf: structure to take the dma data 1986 * @queue: TX queue index 1987 */ 1988static void dma_free_tx_skbufs(struct stmmac_priv *priv, 1989 struct stmmac_dma_conf *dma_conf, 1990 u32 queue) 1991{ 1992 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 1993 int i; 1994 1995 tx_q->xsk_frames_done = 0; 1996 1997 for (i = 0; i < dma_conf->dma_tx_size; i++) 1998 stmmac_free_tx_buffer(priv, dma_conf, queue, i); 1999 2000 if (tx_q->xsk_pool && tx_q->xsk_frames_done) { 2001 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2002 tx_q->xsk_frames_done = 0; 2003 tx_q->xsk_pool = NULL; 2004 } 2005} 2006 2007/** 2008 * stmmac_free_tx_skbufs - free TX skb buffers 2009 * @priv: private structure 2010 */ 2011static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) 2012{ 2013 u32 tx_queue_cnt = priv->plat->tx_queues_to_use; 2014 u32 queue; 2015 2016 for (queue = 0; queue < tx_queue_cnt; queue++) 2017 dma_free_tx_skbufs(priv, &priv->dma_conf, queue); 2018} 2019 2020/** 2021 * __free_dma_rx_desc_resources - free RX dma desc resources (per queue) 2022 * @priv: private structure 2023 * @dma_conf: structure to take the dma data 2024 * @queue: RX queue index 2025 */ 2026static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, 2027 struct stmmac_dma_conf *dma_conf, 2028 u32 queue) 2029{ 2030 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2031 2032 /* Release the DMA RX socket buffers */ 2033 if (rx_q->xsk_pool) 2034 dma_free_rx_xskbufs(priv, dma_conf, queue); 2035 else 2036 dma_free_rx_skbufs(priv, dma_conf, queue); 2037 2038 rx_q->buf_alloc_num = 0; 2039 rx_q->xsk_pool = NULL; 2040 2041 /* Free DMA regions of consistent memory previously allocated */ 2042 if (!priv->extend_desc) 2043 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 2044 sizeof(struct dma_desc), 2045 rx_q->dma_rx, rx_q->dma_rx_phy); 2046 else 2047 dma_free_coherent(priv->device, dma_conf->dma_rx_size * 2048 sizeof(struct dma_extended_desc), 2049 rx_q->dma_erx, rx_q->dma_rx_phy); 2050 2051 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) 2052 xdp_rxq_info_unreg(&rx_q->xdp_rxq); 2053 2054 kfree(rx_q->buf_pool); 2055 if (rx_q->page_pool) 2056 page_pool_destroy(rx_q->page_pool); 2057} 2058 2059static void free_dma_rx_desc_resources(struct stmmac_priv *priv, 2060 struct stmmac_dma_conf *dma_conf) 2061{ 2062 u32 rx_count = priv->plat->rx_queues_to_use; 2063 u32 queue; 2064 2065 /* Free RX queue resources */ 2066 for (queue = 0; queue < rx_count; queue++) 2067 __free_dma_rx_desc_resources(priv, dma_conf, queue); 2068} 2069 2070/** 2071 * __free_dma_tx_desc_resources - free TX dma desc resources (per queue) 2072 * @priv: private structure 2073 * @dma_conf: structure to take the dma data 2074 * @queue: TX queue index 2075 */ 2076static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, 2077 struct stmmac_dma_conf *dma_conf, 2078 u32 queue) 2079{ 2080 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2081 size_t size; 2082 void *addr; 2083 2084 /* Release the DMA TX socket buffers */ 2085 dma_free_tx_skbufs(priv, dma_conf, queue); 2086 2087 if (priv->extend_desc) { 2088 size = sizeof(struct dma_extended_desc); 2089 addr = tx_q->dma_etx; 2090 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { 2091 size = sizeof(struct dma_edesc); 2092 addr = tx_q->dma_entx; 2093 } else { 2094 size = sizeof(struct dma_desc); 2095 addr = tx_q->dma_tx; 2096 } 2097 2098 size *= dma_conf->dma_tx_size; 2099 2100 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); 2101 2102 kfree(tx_q->tx_skbuff_dma); 2103 kfree(tx_q->tx_skbuff); 2104} 2105 2106static void free_dma_tx_desc_resources(struct stmmac_priv *priv, 2107 struct stmmac_dma_conf *dma_conf) 2108{ 2109 u32 tx_count = priv->plat->tx_queues_to_use; 2110 u32 queue; 2111 2112 /* Free TX queue resources */ 2113 for (queue = 0; queue < tx_count; queue++) 2114 __free_dma_tx_desc_resources(priv, dma_conf, queue); 2115} 2116 2117/** 2118 * __alloc_dma_rx_desc_resources - alloc RX resources (per queue). 2119 * @priv: private structure 2120 * @dma_conf: structure to take the dma data 2121 * @queue: RX queue index 2122 * Description: according to which descriptor can be used (extend or basic) 2123 * this function allocates the resources for TX and RX paths. In case of 2124 * reception, for example, it pre-allocated the RX socket buffer in order to 2125 * allow zero-copy mechanism. 2126 */ 2127static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2128 struct stmmac_dma_conf *dma_conf, 2129 u32 queue) 2130{ 2131 struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; 2132 struct stmmac_channel *ch = &priv->channel[queue]; 2133 bool xdp_prog = stmmac_xdp_is_enabled(priv); 2134 struct page_pool_params pp_params = { 0 }; 2135 unsigned int dma_buf_sz_pad, num_pages; 2136 unsigned int napi_id; 2137 int ret; 2138 2139 dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz + 2140 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2141 num_pages = DIV_ROUND_UP(dma_buf_sz_pad, PAGE_SIZE); 2142 2143 rx_q->queue_index = queue; 2144 rx_q->priv_data = priv; 2145 rx_q->napi_skb_frag_size = num_pages * PAGE_SIZE; 2146 2147 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 2148 pp_params.pool_size = dma_conf->dma_rx_size; 2149 pp_params.order = order_base_2(num_pages); 2150 pp_params.nid = dev_to_node(priv->device); 2151 pp_params.dev = priv->device; 2152 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 2153 pp_params.offset = stmmac_rx_offset(priv); 2154 pp_params.max_len = dma_conf->dma_buf_sz; 2155 2156 if (priv->sph_active) { 2157 pp_params.offset = 0; 2158 pp_params.max_len += stmmac_rx_offset(priv); 2159 } 2160 2161 rx_q->page_pool = page_pool_create(&pp_params); 2162 if (IS_ERR(rx_q->page_pool)) { 2163 ret = PTR_ERR(rx_q->page_pool); 2164 rx_q->page_pool = NULL; 2165 return ret; 2166 } 2167 2168 rx_q->buf_pool = kcalloc(dma_conf->dma_rx_size, 2169 sizeof(*rx_q->buf_pool), 2170 GFP_KERNEL); 2171 if (!rx_q->buf_pool) 2172 return -ENOMEM; 2173 2174 if (priv->extend_desc) { 2175 rx_q->dma_erx = dma_alloc_coherent(priv->device, 2176 dma_conf->dma_rx_size * 2177 sizeof(struct dma_extended_desc), 2178 &rx_q->dma_rx_phy, 2179 GFP_KERNEL); 2180 if (!rx_q->dma_erx) 2181 return -ENOMEM; 2182 2183 } else { 2184 rx_q->dma_rx = dma_alloc_coherent(priv->device, 2185 dma_conf->dma_rx_size * 2186 sizeof(struct dma_desc), 2187 &rx_q->dma_rx_phy, 2188 GFP_KERNEL); 2189 if (!rx_q->dma_rx) 2190 return -ENOMEM; 2191 } 2192 2193 if (stmmac_xdp_is_enabled(priv) && 2194 test_bit(queue, priv->af_xdp_zc_qps)) 2195 napi_id = ch->rxtx_napi.napi_id; 2196 else 2197 napi_id = ch->rx_napi.napi_id; 2198 2199 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, 2200 rx_q->queue_index, 2201 napi_id); 2202 if (ret) { 2203 netdev_err(priv->dev, "Failed to register xdp rxq info\n"); 2204 return -EINVAL; 2205 } 2206 2207 return 0; 2208} 2209 2210static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, 2211 struct stmmac_dma_conf *dma_conf) 2212{ 2213 u32 rx_count = priv->plat->rx_queues_to_use; 2214 u32 queue; 2215 int ret; 2216 2217 /* RX queues buffers and DMA */ 2218 for (queue = 0; queue < rx_count; queue++) { 2219 ret = __alloc_dma_rx_desc_resources(priv, dma_conf, queue); 2220 if (ret) 2221 goto err_dma; 2222 } 2223 2224 return 0; 2225 2226err_dma: 2227 free_dma_rx_desc_resources(priv, dma_conf); 2228 2229 return ret; 2230} 2231 2232/** 2233 * __alloc_dma_tx_desc_resources - alloc TX resources (per queue). 2234 * @priv: private structure 2235 * @dma_conf: structure to take the dma data 2236 * @queue: TX queue index 2237 * Description: according to which descriptor can be used (extend or basic) 2238 * this function allocates the resources for TX and RX paths. In case of 2239 * reception, for example, it pre-allocated the RX socket buffer in order to 2240 * allow zero-copy mechanism. 2241 */ 2242static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2243 struct stmmac_dma_conf *dma_conf, 2244 u32 queue) 2245{ 2246 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; 2247 size_t size; 2248 void *addr; 2249 2250 tx_q->queue_index = queue; 2251 tx_q->priv_data = priv; 2252 2253 tx_q->tx_skbuff_dma = kcalloc(dma_conf->dma_tx_size, 2254 sizeof(*tx_q->tx_skbuff_dma), 2255 GFP_KERNEL); 2256 if (!tx_q->tx_skbuff_dma) 2257 return -ENOMEM; 2258 2259 tx_q->tx_skbuff = kcalloc(dma_conf->dma_tx_size, 2260 sizeof(struct sk_buff *), 2261 GFP_KERNEL); 2262 if (!tx_q->tx_skbuff) 2263 return -ENOMEM; 2264 2265 if (priv->extend_desc) 2266 size = sizeof(struct dma_extended_desc); 2267 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2268 size = sizeof(struct dma_edesc); 2269 else 2270 size = sizeof(struct dma_desc); 2271 2272 size *= dma_conf->dma_tx_size; 2273 2274 addr = dma_alloc_coherent(priv->device, size, 2275 &tx_q->dma_tx_phy, GFP_KERNEL); 2276 if (!addr) 2277 return -ENOMEM; 2278 2279 if (priv->extend_desc) 2280 tx_q->dma_etx = addr; 2281 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2282 tx_q->dma_entx = addr; 2283 else 2284 tx_q->dma_tx = addr; 2285 2286 return 0; 2287} 2288 2289static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, 2290 struct stmmac_dma_conf *dma_conf) 2291{ 2292 u32 tx_count = priv->plat->tx_queues_to_use; 2293 u32 queue; 2294 int ret; 2295 2296 /* TX queues buffers and DMA */ 2297 for (queue = 0; queue < tx_count; queue++) { 2298 ret = __alloc_dma_tx_desc_resources(priv, dma_conf, queue); 2299 if (ret) 2300 goto err_dma; 2301 } 2302 2303 return 0; 2304 2305err_dma: 2306 free_dma_tx_desc_resources(priv, dma_conf); 2307 return ret; 2308} 2309 2310/** 2311 * alloc_dma_desc_resources - alloc TX/RX resources. 2312 * @priv: private structure 2313 * @dma_conf: structure to take the dma data 2314 * Description: according to which descriptor can be used (extend or basic) 2315 * this function allocates the resources for TX and RX paths. In case of 2316 * reception, for example, it pre-allocated the RX socket buffer in order to 2317 * allow zero-copy mechanism. 2318 */ 2319static int alloc_dma_desc_resources(struct stmmac_priv *priv, 2320 struct stmmac_dma_conf *dma_conf) 2321{ 2322 /* RX Allocation */ 2323 int ret = alloc_dma_rx_desc_resources(priv, dma_conf); 2324 2325 if (ret) 2326 return ret; 2327 2328 ret = alloc_dma_tx_desc_resources(priv, dma_conf); 2329 2330 return ret; 2331} 2332 2333/** 2334 * free_dma_desc_resources - free dma desc resources 2335 * @priv: private structure 2336 * @dma_conf: structure to take the dma data 2337 */ 2338static void free_dma_desc_resources(struct stmmac_priv *priv, 2339 struct stmmac_dma_conf *dma_conf) 2340{ 2341 /* Release the DMA TX socket buffers */ 2342 free_dma_tx_desc_resources(priv, dma_conf); 2343 2344 /* Release the DMA RX socket buffers later 2345 * to ensure all pending XDP_TX buffers are returned. 2346 */ 2347 free_dma_rx_desc_resources(priv, dma_conf); 2348} 2349 2350/** 2351 * stmmac_mac_enable_rx_queues - Enable MAC rx queues 2352 * @priv: driver private structure 2353 * Description: It is used for enabling the rx queues in the MAC 2354 */ 2355static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) 2356{ 2357 u32 rx_queues_count = priv->plat->rx_queues_to_use; 2358 int queue; 2359 u8 mode; 2360 2361 for (queue = 0; queue < rx_queues_count; queue++) { 2362 mode = priv->plat->rx_queues_cfg[queue].mode_to_use; 2363 stmmac_rx_queue_enable(priv, priv->hw, mode, queue); 2364 } 2365} 2366 2367/** 2368 * stmmac_start_rx_dma - start RX DMA channel 2369 * @priv: driver private structure 2370 * @chan: RX channel index 2371 * Description: 2372 * This starts a RX DMA channel 2373 */ 2374static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan) 2375{ 2376 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan); 2377 stmmac_start_rx(priv, priv->ioaddr, chan); 2378} 2379 2380/** 2381 * stmmac_start_tx_dma - start TX DMA channel 2382 * @priv: driver private structure 2383 * @chan: TX channel index 2384 * Description: 2385 * This starts a TX DMA channel 2386 */ 2387static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan) 2388{ 2389 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan); 2390 stmmac_start_tx(priv, priv->ioaddr, chan); 2391} 2392 2393/** 2394 * stmmac_stop_rx_dma - stop RX DMA channel 2395 * @priv: driver private structure 2396 * @chan: RX channel index 2397 * Description: 2398 * This stops a RX DMA channel 2399 */ 2400static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan) 2401{ 2402 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan); 2403 stmmac_stop_rx(priv, priv->ioaddr, chan); 2404} 2405 2406/** 2407 * stmmac_stop_tx_dma - stop TX DMA channel 2408 * @priv: driver private structure 2409 * @chan: TX channel index 2410 * Description: 2411 * This stops a TX DMA channel 2412 */ 2413static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) 2414{ 2415 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan); 2416 stmmac_stop_tx(priv, priv->ioaddr, chan); 2417} 2418 2419static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) 2420{ 2421 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2422 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2423 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 2424 u32 chan; 2425 2426 for (chan = 0; chan < dma_csr_ch; chan++) { 2427 struct stmmac_channel *ch = &priv->channel[chan]; 2428 unsigned long flags; 2429 2430 spin_lock_irqsave(&ch->lock, flags); 2431 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 2432 spin_unlock_irqrestore(&ch->lock, flags); 2433 } 2434} 2435 2436/** 2437 * stmmac_start_all_dma - start all RX and TX DMA channels 2438 * @priv: driver private structure 2439 * Description: 2440 * This starts all the RX and TX DMA channels 2441 */ 2442static void stmmac_start_all_dma(struct stmmac_priv *priv) 2443{ 2444 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2445 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2446 u32 chan = 0; 2447 2448 for (chan = 0; chan < rx_channels_count; chan++) 2449 stmmac_start_rx_dma(priv, chan); 2450 2451 for (chan = 0; chan < tx_channels_count; chan++) 2452 stmmac_start_tx_dma(priv, chan); 2453} 2454 2455/** 2456 * stmmac_stop_all_dma - stop all RX and TX DMA channels 2457 * @priv: driver private structure 2458 * Description: 2459 * This stops the RX and TX DMA channels 2460 */ 2461static void stmmac_stop_all_dma(struct stmmac_priv *priv) 2462{ 2463 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2464 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2465 u32 chan = 0; 2466 2467 for (chan = 0; chan < rx_channels_count; chan++) 2468 stmmac_stop_rx_dma(priv, chan); 2469 2470 for (chan = 0; chan < tx_channels_count; chan++) 2471 stmmac_stop_tx_dma(priv, chan); 2472} 2473 2474/** 2475 * stmmac_dma_operation_mode - HW DMA operation mode 2476 * @priv: driver private structure 2477 * Description: it is used for configuring the DMA operation mode register in 2478 * order to program the tx/rx DMA thresholds or Store-And-Forward mode. 2479 */ 2480static void stmmac_dma_operation_mode(struct stmmac_priv *priv) 2481{ 2482 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2483 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2484 int rxfifosz = priv->plat->rx_fifo_size; 2485 int txfifosz = priv->plat->tx_fifo_size; 2486 u32 txmode = 0; 2487 u32 rxmode = 0; 2488 u32 chan = 0; 2489 u8 qmode = 0; 2490 2491 if (rxfifosz == 0) 2492 rxfifosz = priv->dma_cap.rx_fifo_size; 2493 if (txfifosz == 0) 2494 txfifosz = priv->dma_cap.tx_fifo_size; 2495 2496 /* Split up the shared Tx/Rx FIFO memory on DW QoS Eth and DW XGMAC */ 2497 if (dwmac_is_xmac(priv->plat->core_type)) { 2498 rxfifosz /= rx_channels_count; 2499 txfifosz /= tx_channels_count; 2500 } 2501 2502 if (priv->plat->force_thresh_dma_mode) { 2503 txmode = tc; 2504 rxmode = tc; 2505 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) { 2506 /* 2507 * In case of GMAC, SF mode can be enabled 2508 * to perform the TX COE in HW. This depends on: 2509 * 1) TX COE if actually supported 2510 * 2) There is no bugged Jumbo frame support 2511 * that needs to not insert csum in the TDES. 2512 */ 2513 txmode = SF_DMA_MODE; 2514 rxmode = SF_DMA_MODE; 2515 priv->xstats.threshold = SF_DMA_MODE; 2516 } else { 2517 txmode = tc; 2518 rxmode = SF_DMA_MODE; 2519 } 2520 2521 /* configure all channels */ 2522 for (chan = 0; chan < rx_channels_count; chan++) { 2523 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 2524 u32 buf_size; 2525 2526 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2527 2528 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, 2529 rxfifosz, qmode); 2530 2531 if (rx_q->xsk_pool) { 2532 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 2533 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2534 buf_size, 2535 chan); 2536 } else { 2537 stmmac_set_dma_bfsize(priv, priv->ioaddr, 2538 priv->dma_conf.dma_buf_sz, 2539 chan); 2540 } 2541 } 2542 2543 for (chan = 0; chan < tx_channels_count; chan++) { 2544 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2545 2546 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, 2547 txfifosz, qmode); 2548 } 2549} 2550 2551static void stmmac_xsk_request_timestamp(void *_priv) 2552{ 2553 struct stmmac_metadata_request *meta_req = _priv; 2554 2555 stmmac_enable_tx_timestamp(meta_req->priv, meta_req->tx_desc); 2556 *meta_req->set_ic = true; 2557} 2558 2559static u64 stmmac_xsk_fill_timestamp(void *_priv) 2560{ 2561 struct stmmac_xsk_tx_complete *tx_compl = _priv; 2562 struct stmmac_priv *priv = tx_compl->priv; 2563 struct dma_desc *desc = tx_compl->desc; 2564 bool found = false; 2565 u64 ns = 0; 2566 2567 if (!priv->hwts_tx_en) 2568 return 0; 2569 2570 /* check tx tstamp status */ 2571 if (stmmac_get_tx_timestamp_status(priv, desc)) { 2572 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); 2573 found = true; 2574 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) { 2575 found = true; 2576 } 2577 2578 if (found) { 2579 ns -= priv->plat->cdc_error_adj; 2580 return ns_to_ktime(ns); 2581 } 2582 2583 return 0; 2584} 2585 2586static void stmmac_xsk_request_launch_time(u64 launch_time, void *_priv) 2587{ 2588 struct timespec64 ts = ns_to_timespec64(launch_time); 2589 struct stmmac_metadata_request *meta_req = _priv; 2590 2591 if (meta_req->tbs & STMMAC_TBS_EN) 2592 stmmac_set_desc_tbs(meta_req->priv, meta_req->edesc, ts.tv_sec, 2593 ts.tv_nsec); 2594} 2595 2596static const struct xsk_tx_metadata_ops stmmac_xsk_tx_metadata_ops = { 2597 .tmo_request_timestamp = stmmac_xsk_request_timestamp, 2598 .tmo_fill_timestamp = stmmac_xsk_fill_timestamp, 2599 .tmo_request_launch_time = stmmac_xsk_request_launch_time, 2600}; 2601 2602static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 2603{ 2604 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue); 2605 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2606 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2607 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 2608 struct xsk_buff_pool *pool = tx_q->xsk_pool; 2609 unsigned int entry = tx_q->cur_tx; 2610 struct dma_desc *tx_desc = NULL; 2611 struct xdp_desc xdp_desc; 2612 bool work_done = true; 2613 u32 tx_set_ic_bit = 0; 2614 2615 /* Avoids TX time-out as we are sharing with slow path */ 2616 txq_trans_cond_update(nq); 2617 2618 budget = min(budget, stmmac_tx_avail(priv, queue)); 2619 2620 for (; budget > 0; budget--) { 2621 struct stmmac_metadata_request meta_req; 2622 struct xsk_tx_metadata *meta = NULL; 2623 dma_addr_t dma_addr; 2624 bool set_ic; 2625 2626 /* We are sharing with slow path and stop XSK TX desc submission when 2627 * available TX ring is less than threshold. 2628 */ 2629 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) || 2630 !netif_carrier_ok(priv->dev)) { 2631 work_done = false; 2632 break; 2633 } 2634 2635 if (!xsk_tx_peek_desc(pool, &xdp_desc)) 2636 break; 2637 2638 if (priv->est && priv->est->enable && 2639 priv->est->max_sdu[queue] && 2640 xdp_desc.len > priv->est->max_sdu[queue]) { 2641 priv->xstats.max_sdu_txq_drop[queue]++; 2642 continue; 2643 } 2644 2645 if (likely(priv->extend_desc)) 2646 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 2647 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2648 tx_desc = &tx_q->dma_entx[entry].basic; 2649 else 2650 tx_desc = tx_q->dma_tx + entry; 2651 2652 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); 2653 meta = xsk_buff_get_metadata(pool, xdp_desc.addr); 2654 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); 2655 2656 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; 2657 2658 /* To return XDP buffer to XSK pool, we simple call 2659 * xsk_tx_completed(), so we don't need to fill up 2660 * 'buf' and 'xdpf'. 2661 */ 2662 tx_q->tx_skbuff_dma[entry].buf = 0; 2663 tx_q->xdpf[entry] = NULL; 2664 2665 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2666 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; 2667 tx_q->tx_skbuff_dma[entry].last_segment = true; 2668 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2669 2670 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 2671 2672 tx_q->tx_count_frames++; 2673 2674 if (!priv->tx_coal_frames[queue]) 2675 set_ic = false; 2676 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 2677 set_ic = true; 2678 else 2679 set_ic = false; 2680 2681 meta_req.priv = priv; 2682 meta_req.tx_desc = tx_desc; 2683 meta_req.set_ic = &set_ic; 2684 meta_req.tbs = tx_q->tbs; 2685 meta_req.edesc = &tx_q->dma_entx[entry]; 2686 xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops, 2687 &meta_req); 2688 if (set_ic) { 2689 tx_q->tx_count_frames = 0; 2690 stmmac_set_tx_ic(priv, tx_desc); 2691 tx_set_ic_bit++; 2692 } 2693 2694 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, 2695 csum, priv->mode, true, true, 2696 xdp_desc.len); 2697 2698 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 2699 2700 xsk_tx_metadata_to_compl(meta, 2701 &tx_q->tx_skbuff_dma[entry].xsk_meta); 2702 2703 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 2704 entry = tx_q->cur_tx; 2705 } 2706 u64_stats_update_begin(&txq_stats->napi_syncp); 2707 u64_stats_add(&txq_stats->napi.tx_set_ic_bit, tx_set_ic_bit); 2708 u64_stats_update_end(&txq_stats->napi_syncp); 2709 2710 if (tx_desc) { 2711 stmmac_flush_tx_descriptors(priv, queue); 2712 xsk_tx_release(pool); 2713 } 2714 2715 /* Return true if all of the 3 conditions are met 2716 * a) TX Budget is still available 2717 * b) work_done = true when XSK TX desc peek is empty (no more 2718 * pending XSK TX for transmission) 2719 */ 2720 return !!budget && work_done; 2721} 2722 2723static void stmmac_bump_dma_threshold(struct stmmac_priv *priv, u32 chan) 2724{ 2725 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) && tc <= 256) { 2726 tc += 64; 2727 2728 if (priv->plat->force_thresh_dma_mode) 2729 stmmac_set_dma_operation_mode(priv, tc, tc, chan); 2730 else 2731 stmmac_set_dma_operation_mode(priv, tc, SF_DMA_MODE, 2732 chan); 2733 2734 priv->xstats.threshold = tc; 2735 } 2736} 2737 2738/** 2739 * stmmac_tx_clean - to manage the transmission completion 2740 * @priv: driver private structure 2741 * @budget: napi budget limiting this functions packet handling 2742 * @queue: TX queue index 2743 * @pending_packets: signal to arm the TX coal timer 2744 * Description: it reclaims the transmit resources after transmission completes. 2745 * If some packets still needs to be handled, due to TX coalesce, set 2746 * pending_packets to true to make NAPI arm the TX coal timer. 2747 */ 2748static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, 2749 bool *pending_packets) 2750{ 2751 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 2752 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 2753 unsigned int bytes_compl = 0, pkts_compl = 0; 2754 unsigned int entry, xmits = 0, count = 0; 2755 u32 tx_packets = 0, tx_errors = 0; 2756 2757 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue)); 2758 2759 tx_q->xsk_frames_done = 0; 2760 2761 entry = tx_q->dirty_tx; 2762 2763 /* Try to clean all TX complete frame in 1 shot */ 2764 while ((entry != tx_q->cur_tx) && count < priv->dma_conf.dma_tx_size) { 2765 struct xdp_frame *xdpf; 2766 struct sk_buff *skb; 2767 struct dma_desc *p; 2768 int status; 2769 2770 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX || 2771 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2772 xdpf = tx_q->xdpf[entry]; 2773 skb = NULL; 2774 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2775 xdpf = NULL; 2776 skb = tx_q->tx_skbuff[entry]; 2777 } else { 2778 xdpf = NULL; 2779 skb = NULL; 2780 } 2781 2782 if (priv->extend_desc) 2783 p = (struct dma_desc *)(tx_q->dma_etx + entry); 2784 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 2785 p = &tx_q->dma_entx[entry].basic; 2786 else 2787 p = tx_q->dma_tx + entry; 2788 2789 status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); 2790 /* Check if the descriptor is owned by the DMA */ 2791 if (unlikely(status & tx_dma_own)) 2792 break; 2793 2794 count++; 2795 2796 /* Make sure descriptor fields are read after reading 2797 * the own bit. 2798 */ 2799 dma_rmb(); 2800 2801 /* Just consider the last segment and ...*/ 2802 if (likely(!(status & tx_not_ls))) { 2803 /* ... verify the status error condition */ 2804 if (unlikely(status & tx_err)) { 2805 tx_errors++; 2806 if (unlikely(status & tx_err_bump_tc)) 2807 stmmac_bump_dma_threshold(priv, queue); 2808 } else { 2809 tx_packets++; 2810 } 2811 if (skb) { 2812 stmmac_get_tx_hwtstamp(priv, p, skb); 2813 } else if (tx_q->xsk_pool && 2814 xp_tx_metadata_enabled(tx_q->xsk_pool)) { 2815 struct stmmac_xsk_tx_complete tx_compl = { 2816 .priv = priv, 2817 .desc = p, 2818 }; 2819 2820 xsk_tx_metadata_complete(&tx_q->tx_skbuff_dma[entry].xsk_meta, 2821 &stmmac_xsk_tx_metadata_ops, 2822 &tx_compl); 2823 } 2824 } 2825 2826 if (likely(tx_q->tx_skbuff_dma[entry].buf && 2827 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { 2828 if (tx_q->tx_skbuff_dma[entry].map_as_page) 2829 dma_unmap_page(priv->device, 2830 tx_q->tx_skbuff_dma[entry].buf, 2831 tx_q->tx_skbuff_dma[entry].len, 2832 DMA_TO_DEVICE); 2833 else 2834 dma_unmap_single(priv->device, 2835 tx_q->tx_skbuff_dma[entry].buf, 2836 tx_q->tx_skbuff_dma[entry].len, 2837 DMA_TO_DEVICE); 2838 tx_q->tx_skbuff_dma[entry].buf = 0; 2839 tx_q->tx_skbuff_dma[entry].len = 0; 2840 tx_q->tx_skbuff_dma[entry].map_as_page = false; 2841 } 2842 2843 stmmac_clean_desc3(priv, tx_q, p); 2844 2845 tx_q->tx_skbuff_dma[entry].last_segment = false; 2846 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 2847 2848 if (xdpf && 2849 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) { 2850 xdp_return_frame_rx_napi(xdpf); 2851 tx_q->xdpf[entry] = NULL; 2852 } 2853 2854 if (xdpf && 2855 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { 2856 xdp_return_frame(xdpf); 2857 tx_q->xdpf[entry] = NULL; 2858 } 2859 2860 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) 2861 tx_q->xsk_frames_done++; 2862 2863 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { 2864 if (likely(skb)) { 2865 pkts_compl++; 2866 bytes_compl += skb->len; 2867 dev_consume_skb_any(skb); 2868 tx_q->tx_skbuff[entry] = NULL; 2869 } 2870 } 2871 2872 stmmac_release_tx_desc(priv, p, priv->mode); 2873 2874 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 2875 } 2876 tx_q->dirty_tx = entry; 2877 2878 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue), 2879 pkts_compl, bytes_compl); 2880 2881 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev, 2882 queue))) && 2883 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) { 2884 2885 netif_dbg(priv, tx_done, priv->dev, 2886 "%s: restart transmit\n", __func__); 2887 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue)); 2888 } 2889 2890 if (tx_q->xsk_pool) { 2891 bool work_done; 2892 2893 if (tx_q->xsk_frames_done) 2894 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done); 2895 2896 if (xsk_uses_need_wakeup(tx_q->xsk_pool)) 2897 xsk_set_tx_need_wakeup(tx_q->xsk_pool); 2898 2899 /* For XSK TX, we try to send as many as possible. 2900 * If XSK work done (XSK TX desc empty and budget still 2901 * available), return "budget - 1" to reenable TX IRQ. 2902 * Else, return "budget" to make NAPI continue polling. 2903 */ 2904 work_done = stmmac_xdp_xmit_zc(priv, queue, 2905 STMMAC_XSK_TX_BUDGET_MAX); 2906 if (work_done) 2907 xmits = budget - 1; 2908 else 2909 xmits = budget; 2910 } 2911 2912 if (priv->eee_sw_timer_en && !priv->tx_path_in_lpi_mode) 2913 stmmac_restart_sw_lpi_timer(priv); 2914 2915 /* We still have pending packets, let's call for a new scheduling */ 2916 if (tx_q->dirty_tx != tx_q->cur_tx) 2917 *pending_packets = true; 2918 2919 u64_stats_update_begin(&txq_stats->napi_syncp); 2920 u64_stats_add(&txq_stats->napi.tx_packets, tx_packets); 2921 u64_stats_add(&txq_stats->napi.tx_pkt_n, tx_packets); 2922 u64_stats_inc(&txq_stats->napi.tx_clean); 2923 u64_stats_update_end(&txq_stats->napi_syncp); 2924 2925 priv->xstats.tx_errors += tx_errors; 2926 2927 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue)); 2928 2929 /* Combine decisions from TX clean and XSK TX */ 2930 return max(count, xmits); 2931} 2932 2933/** 2934 * stmmac_tx_err - to manage the tx error 2935 * @priv: driver private structure 2936 * @chan: channel index 2937 * Description: it cleans the descriptors and restarts the transmission 2938 * in case of transmission errors. 2939 */ 2940static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan) 2941{ 2942 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 2943 2944 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan)); 2945 2946 stmmac_stop_tx_dma(priv, chan); 2947 dma_free_tx_skbufs(priv, &priv->dma_conf, chan); 2948 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, chan); 2949 stmmac_reset_tx_queue(priv, chan); 2950 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 2951 tx_q->dma_tx_phy, chan); 2952 stmmac_start_tx_dma(priv, chan); 2953 2954 priv->xstats.tx_errors++; 2955 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan)); 2956} 2957 2958/** 2959 * stmmac_set_dma_operation_mode - Set DMA operation mode by channel 2960 * @priv: driver private structure 2961 * @txmode: TX operating mode 2962 * @rxmode: RX operating mode 2963 * @chan: channel index 2964 * Description: it is used for configuring of the DMA operation mode in 2965 * runtime in order to program the tx/rx DMA thresholds or Store-And-Forward 2966 * mode. 2967 */ 2968static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, 2969 u32 rxmode, u32 chan) 2970{ 2971 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; 2972 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; 2973 u32 rx_channels_count = priv->plat->rx_queues_to_use; 2974 u32 tx_channels_count = priv->plat->tx_queues_to_use; 2975 int rxfifosz = priv->plat->rx_fifo_size; 2976 int txfifosz = priv->plat->tx_fifo_size; 2977 2978 if (rxfifosz == 0) 2979 rxfifosz = priv->dma_cap.rx_fifo_size; 2980 if (txfifosz == 0) 2981 txfifosz = priv->dma_cap.tx_fifo_size; 2982 2983 /* Adjust for real per queue fifo size */ 2984 rxfifosz /= rx_channels_count; 2985 txfifosz /= tx_channels_count; 2986 2987 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode); 2988 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode); 2989} 2990 2991static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) 2992{ 2993 int ret; 2994 2995 ret = stmmac_safety_feat_irq_status(priv, priv->dev, 2996 priv->ioaddr, priv->dma_cap.asp, &priv->sstats); 2997 if (ret && (ret != -EINVAL)) { 2998 stmmac_global_err(priv); 2999 return true; 3000 } 3001 3002 return false; 3003} 3004 3005static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) 3006{ 3007 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr, 3008 &priv->xstats, chan, dir); 3009 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; 3010 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3011 struct stmmac_channel *ch = &priv->channel[chan]; 3012 struct napi_struct *rx_napi; 3013 struct napi_struct *tx_napi; 3014 unsigned long flags; 3015 3016 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi; 3017 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3018 3019 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) { 3020 if (napi_schedule_prep(rx_napi)) { 3021 spin_lock_irqsave(&ch->lock, flags); 3022 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 3023 spin_unlock_irqrestore(&ch->lock, flags); 3024 __napi_schedule(rx_napi); 3025 } 3026 } 3027 3028 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) { 3029 if (napi_schedule_prep(tx_napi)) { 3030 spin_lock_irqsave(&ch->lock, flags); 3031 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 3032 spin_unlock_irqrestore(&ch->lock, flags); 3033 __napi_schedule(tx_napi); 3034 } 3035 } 3036 3037 return status; 3038} 3039 3040/** 3041 * stmmac_dma_interrupt - DMA ISR 3042 * @priv: driver private structure 3043 * Description: this is the DMA ISR. It is called by the main ISR. 3044 * It calls the dwmac dma routine and schedule poll method in case of some 3045 * work can be done. 3046 */ 3047static void stmmac_dma_interrupt(struct stmmac_priv *priv) 3048{ 3049 u32 tx_channel_count = priv->plat->tx_queues_to_use; 3050 u32 rx_channel_count = priv->plat->rx_queues_to_use; 3051 u32 channels_to_check = tx_channel_count > rx_channel_count ? 3052 tx_channel_count : rx_channel_count; 3053 u32 chan; 3054 int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; 3055 3056 /* Make sure we never check beyond our status buffer. */ 3057 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) 3058 channels_to_check = ARRAY_SIZE(status); 3059 3060 for (chan = 0; chan < channels_to_check; chan++) 3061 status[chan] = stmmac_napi_check(priv, chan, 3062 DMA_DIR_RXTX); 3063 3064 for (chan = 0; chan < tx_channel_count; chan++) { 3065 if (unlikely(status[chan] & tx_hard_error_bump_tc)) { 3066 /* Try to bump up the dma threshold on this failure */ 3067 stmmac_bump_dma_threshold(priv, chan); 3068 } else if (unlikely(status[chan] == tx_hard_error)) { 3069 stmmac_tx_err(priv, chan); 3070 } 3071 } 3072} 3073 3074/** 3075 * stmmac_mmc_setup: setup the Mac Management Counters (MMC) 3076 * @priv: driver private structure 3077 * Description: this masks the MMC irq, in fact, the counters are managed in SW. 3078 */ 3079static void stmmac_mmc_setup(struct stmmac_priv *priv) 3080{ 3081 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET | 3082 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET; 3083 3084 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr); 3085 3086 if (priv->dma_cap.rmon) { 3087 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode); 3088 memset(&priv->mmc, 0, sizeof(struct stmmac_counters)); 3089 } else 3090 netdev_info(priv->dev, "No MAC Management Counters available\n"); 3091} 3092 3093/** 3094 * stmmac_get_hw_features - get MAC capabilities from the HW cap. register. 3095 * @priv: driver private structure 3096 * Description: 3097 * new GMAC chip generations have a new register to indicate the 3098 * presence of the optional feature/functions. 3099 * This can be also used to override the value passed through the 3100 * platform and necessary for old MAC10/100 and GMAC chips. 3101 */ 3102static int stmmac_get_hw_features(struct stmmac_priv *priv) 3103{ 3104 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0; 3105} 3106 3107/** 3108 * stmmac_check_ether_addr - check if the MAC addr is valid 3109 * @priv: driver private structure 3110 * Description: 3111 * it is to verify if the MAC address is valid, in case of failures it 3112 * generates a random MAC address 3113 */ 3114static void stmmac_check_ether_addr(struct stmmac_priv *priv) 3115{ 3116 u8 addr[ETH_ALEN]; 3117 3118 if (!is_valid_ether_addr(priv->dev->dev_addr)) { 3119 stmmac_get_umac_addr(priv, priv->hw, addr, 0); 3120 if (is_valid_ether_addr(addr)) 3121 eth_hw_addr_set(priv->dev, addr); 3122 else 3123 eth_hw_addr_random(priv->dev); 3124 dev_info(priv->device, "device MAC address %pM\n", 3125 priv->dev->dev_addr); 3126 } 3127} 3128 3129int stmmac_get_phy_intf_sel(phy_interface_t interface) 3130{ 3131 int phy_intf_sel = -EINVAL; 3132 3133 if (interface == PHY_INTERFACE_MODE_MII || 3134 interface == PHY_INTERFACE_MODE_GMII) 3135 phy_intf_sel = PHY_INTF_SEL_GMII_MII; 3136 else if (phy_interface_mode_is_rgmii(interface)) 3137 phy_intf_sel = PHY_INTF_SEL_RGMII; 3138 else if (interface == PHY_INTERFACE_MODE_SGMII) 3139 phy_intf_sel = PHY_INTF_SEL_SGMII; 3140 else if (interface == PHY_INTERFACE_MODE_RMII) 3141 phy_intf_sel = PHY_INTF_SEL_RMII; 3142 else if (interface == PHY_INTERFACE_MODE_REVMII) 3143 phy_intf_sel = PHY_INTF_SEL_REVMII; 3144 3145 return phy_intf_sel; 3146} 3147EXPORT_SYMBOL_GPL(stmmac_get_phy_intf_sel); 3148 3149static int stmmac_prereset_configure(struct stmmac_priv *priv) 3150{ 3151 struct plat_stmmacenet_data *plat_dat = priv->plat; 3152 phy_interface_t interface; 3153 int phy_intf_sel, ret; 3154 3155 if (!plat_dat->set_phy_intf_sel) 3156 return 0; 3157 3158 interface = plat_dat->phy_interface; 3159 phy_intf_sel = stmmac_get_phy_intf_sel(interface); 3160 if (phy_intf_sel < 0) { 3161 netdev_err(priv->dev, 3162 "failed to get phy_intf_sel for %s: %pe\n", 3163 phy_modes(interface), ERR_PTR(phy_intf_sel)); 3164 return phy_intf_sel; 3165 } 3166 3167 ret = plat_dat->set_phy_intf_sel(plat_dat->bsp_priv, phy_intf_sel); 3168 if (ret == -EINVAL) 3169 netdev_err(priv->dev, "platform does not support %s\n", 3170 phy_modes(interface)); 3171 else if (ret < 0) 3172 netdev_err(priv->dev, 3173 "platform failed to set interface %s: %pe\n", 3174 phy_modes(interface), ERR_PTR(ret)); 3175 3176 return ret; 3177} 3178 3179/** 3180 * stmmac_init_dma_engine - DMA init. 3181 * @priv: driver private structure 3182 * Description: 3183 * It inits the DMA invoking the specific MAC/GMAC callback. 3184 * Some DMA parameters can be passed from the platform; 3185 * in case of these are not passed a default is kept for the MAC or GMAC. 3186 */ 3187static int stmmac_init_dma_engine(struct stmmac_priv *priv) 3188{ 3189 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3190 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3191 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); 3192 struct stmmac_rx_queue *rx_q; 3193 struct stmmac_tx_queue *tx_q; 3194 u32 chan = 0; 3195 int ret = 0; 3196 3197 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { 3198 netdev_err(priv->dev, "Invalid DMA configuration\n"); 3199 return -EINVAL; 3200 } 3201 3202 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) 3203 priv->plat->dma_cfg->atds = 1; 3204 3205 ret = stmmac_prereset_configure(priv); 3206 if (ret) 3207 return ret; 3208 3209 ret = stmmac_reset(priv); 3210 if (ret) { 3211 netdev_err(priv->dev, "Failed to reset the dma\n"); 3212 return ret; 3213 } 3214 3215 /* DMA Configuration */ 3216 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg); 3217 3218 if (priv->plat->axi) 3219 stmmac_axi(priv, priv->ioaddr, priv->plat->axi); 3220 3221 /* DMA CSR Channel configuration */ 3222 for (chan = 0; chan < dma_csr_ch; chan++) { 3223 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 3224 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 3225 } 3226 3227 /* DMA RX Channel Configuration */ 3228 for (chan = 0; chan < rx_channels_count; chan++) { 3229 rx_q = &priv->dma_conf.rx_queue[chan]; 3230 3231 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3232 rx_q->dma_rx_phy, chan); 3233 3234 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 3235 (rx_q->buf_alloc_num * 3236 sizeof(struct dma_desc)); 3237 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 3238 rx_q->rx_tail_addr, chan); 3239 } 3240 3241 /* DMA TX Channel Configuration */ 3242 for (chan = 0; chan < tx_channels_count; chan++) { 3243 tx_q = &priv->dma_conf.tx_queue[chan]; 3244 3245 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 3246 tx_q->dma_tx_phy, chan); 3247 3248 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 3249 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 3250 tx_q->tx_tail_addr, chan); 3251 } 3252 3253 return ret; 3254} 3255 3256static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) 3257{ 3258 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 3259 u32 tx_coal_timer = priv->tx_coal_timer[queue]; 3260 struct stmmac_channel *ch; 3261 struct napi_struct *napi; 3262 3263 if (!tx_coal_timer) 3264 return; 3265 3266 ch = &priv->channel[tx_q->queue_index]; 3267 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3268 3269 /* Arm timer only if napi is not already scheduled. 3270 * Try to cancel any timer if napi is scheduled, timer will be armed 3271 * again in the next scheduled napi. 3272 */ 3273 if (unlikely(!napi_is_scheduled(napi))) 3274 hrtimer_start(&tx_q->txtimer, 3275 STMMAC_COAL_TIMER(tx_coal_timer), 3276 HRTIMER_MODE_REL); 3277 else 3278 hrtimer_try_to_cancel(&tx_q->txtimer); 3279} 3280 3281/** 3282 * stmmac_tx_timer - mitigation sw timer for tx. 3283 * @t: data pointer 3284 * Description: 3285 * This is the timer handler to directly invoke the stmmac_tx_clean. 3286 */ 3287static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) 3288{ 3289 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer); 3290 struct stmmac_priv *priv = tx_q->priv_data; 3291 struct stmmac_channel *ch; 3292 struct napi_struct *napi; 3293 3294 ch = &priv->channel[tx_q->queue_index]; 3295 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; 3296 3297 if (likely(napi_schedule_prep(napi))) { 3298 unsigned long flags; 3299 3300 spin_lock_irqsave(&ch->lock, flags); 3301 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1); 3302 spin_unlock_irqrestore(&ch->lock, flags); 3303 __napi_schedule(napi); 3304 } 3305 3306 return HRTIMER_NORESTART; 3307} 3308 3309/** 3310 * stmmac_init_coalesce - init mitigation options. 3311 * @priv: driver private structure 3312 * Description: 3313 * This inits the coalesce parameters: i.e. timer rate, 3314 * timer handler and default threshold used for enabling the 3315 * interrupt on completion bit. 3316 */ 3317static void stmmac_init_coalesce(struct stmmac_priv *priv) 3318{ 3319 u32 tx_channel_count = priv->plat->tx_queues_to_use; 3320 u32 rx_channel_count = priv->plat->rx_queues_to_use; 3321 u32 chan; 3322 3323 for (chan = 0; chan < tx_channel_count; chan++) { 3324 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3325 3326 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES; 3327 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER; 3328 3329 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3330 } 3331 3332 for (chan = 0; chan < rx_channel_count; chan++) 3333 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES; 3334} 3335 3336static void stmmac_set_rings_length(struct stmmac_priv *priv) 3337{ 3338 u32 rx_channels_count = priv->plat->rx_queues_to_use; 3339 u32 tx_channels_count = priv->plat->tx_queues_to_use; 3340 u32 chan; 3341 3342 /* set TX ring length */ 3343 for (chan = 0; chan < tx_channels_count; chan++) 3344 stmmac_set_tx_ring_len(priv, priv->ioaddr, 3345 (priv->dma_conf.dma_tx_size - 1), chan); 3346 3347 /* set RX ring length */ 3348 for (chan = 0; chan < rx_channels_count; chan++) 3349 stmmac_set_rx_ring_len(priv, priv->ioaddr, 3350 (priv->dma_conf.dma_rx_size - 1), chan); 3351} 3352 3353/** 3354 * stmmac_set_tx_queue_weight - Set TX queue weight 3355 * @priv: driver private structure 3356 * Description: It is used for setting TX queues weight 3357 */ 3358static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) 3359{ 3360 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3361 u32 weight; 3362 u32 queue; 3363 3364 for (queue = 0; queue < tx_queues_count; queue++) { 3365 weight = priv->plat->tx_queues_cfg[queue].weight; 3366 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue); 3367 } 3368} 3369 3370/** 3371 * stmmac_configure_cbs - Configure CBS in TX queue 3372 * @priv: driver private structure 3373 * Description: It is used for configuring CBS in AVB TX queues 3374 */ 3375static void stmmac_configure_cbs(struct stmmac_priv *priv) 3376{ 3377 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3378 u32 mode_to_use; 3379 u32 queue; 3380 3381 /* queue 0 is reserved for legacy traffic */ 3382 for (queue = 1; queue < tx_queues_count; queue++) { 3383 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use; 3384 if (mode_to_use == MTL_QUEUE_DCB) 3385 continue; 3386 3387 stmmac_config_cbs(priv, priv->hw, 3388 priv->plat->tx_queues_cfg[queue].send_slope, 3389 priv->plat->tx_queues_cfg[queue].idle_slope, 3390 priv->plat->tx_queues_cfg[queue].high_credit, 3391 priv->plat->tx_queues_cfg[queue].low_credit, 3392 queue); 3393 } 3394} 3395 3396/** 3397 * stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel 3398 * @priv: driver private structure 3399 * Description: It is used for mapping RX queues to RX dma channels 3400 */ 3401static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) 3402{ 3403 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3404 u32 queue; 3405 u32 chan; 3406 3407 for (queue = 0; queue < rx_queues_count; queue++) { 3408 chan = priv->plat->rx_queues_cfg[queue].chan; 3409 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan); 3410 } 3411} 3412 3413/** 3414 * stmmac_mac_config_rx_queues_prio - Configure RX Queue priority 3415 * @priv: driver private structure 3416 * Description: It is used for configuring the RX Queue Priority 3417 */ 3418static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) 3419{ 3420 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3421 u32 queue; 3422 u32 prio; 3423 3424 for (queue = 0; queue < rx_queues_count; queue++) { 3425 if (!priv->plat->rx_queues_cfg[queue].use_prio) 3426 continue; 3427 3428 prio = priv->plat->rx_queues_cfg[queue].prio; 3429 stmmac_rx_queue_prio(priv, priv->hw, prio, queue); 3430 } 3431} 3432 3433/** 3434 * stmmac_mac_config_tx_queues_prio - Configure TX Queue priority 3435 * @priv: driver private structure 3436 * Description: It is used for configuring the TX Queue Priority 3437 */ 3438static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) 3439{ 3440 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3441 u32 queue; 3442 u32 prio; 3443 3444 for (queue = 0; queue < tx_queues_count; queue++) { 3445 if (!priv->plat->tx_queues_cfg[queue].use_prio) 3446 continue; 3447 3448 prio = priv->plat->tx_queues_cfg[queue].prio; 3449 stmmac_tx_queue_prio(priv, priv->hw, prio, queue); 3450 } 3451} 3452 3453/** 3454 * stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing 3455 * @priv: driver private structure 3456 * Description: It is used for configuring the RX queue routing 3457 */ 3458static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) 3459{ 3460 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3461 u32 queue; 3462 u8 packet; 3463 3464 for (queue = 0; queue < rx_queues_count; queue++) { 3465 /* no specific packet type routing specified for the queue */ 3466 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0) 3467 continue; 3468 3469 packet = priv->plat->rx_queues_cfg[queue].pkt_route; 3470 stmmac_rx_queue_routing(priv, priv->hw, packet, queue); 3471 } 3472} 3473 3474static void stmmac_mac_config_rss(struct stmmac_priv *priv) 3475{ 3476 if (!priv->dma_cap.rssen || !priv->plat->rss_en) { 3477 priv->rss.enable = false; 3478 return; 3479 } 3480 3481 if (priv->dev->features & NETIF_F_RXHASH) 3482 priv->rss.enable = true; 3483 else 3484 priv->rss.enable = false; 3485 3486 stmmac_rss_configure(priv, priv->hw, &priv->rss, 3487 priv->plat->rx_queues_to_use); 3488} 3489 3490/** 3491 * stmmac_mtl_configuration - Configure MTL 3492 * @priv: driver private structure 3493 * Description: It is used for configurring MTL 3494 */ 3495static void stmmac_mtl_configuration(struct stmmac_priv *priv) 3496{ 3497 u32 rx_queues_count = priv->plat->rx_queues_to_use; 3498 u32 tx_queues_count = priv->plat->tx_queues_to_use; 3499 3500 if (tx_queues_count > 1) 3501 stmmac_set_tx_queue_weight(priv); 3502 3503 /* Configure MTL RX algorithms */ 3504 if (rx_queues_count > 1) 3505 stmmac_prog_mtl_rx_algorithms(priv, priv->hw, 3506 priv->plat->rx_sched_algorithm); 3507 3508 /* Configure MTL TX algorithms */ 3509 if (tx_queues_count > 1) 3510 stmmac_prog_mtl_tx_algorithms(priv, priv->hw, 3511 priv->plat->tx_sched_algorithm); 3512 3513 /* Configure CBS in AVB TX queues */ 3514 if (tx_queues_count > 1) 3515 stmmac_configure_cbs(priv); 3516 3517 /* Map RX MTL to DMA channels */ 3518 stmmac_rx_queue_dma_chan_map(priv); 3519 3520 /* Enable MAC RX Queues */ 3521 stmmac_mac_enable_rx_queues(priv); 3522 3523 /* Set RX priorities */ 3524 if (rx_queues_count > 1) 3525 stmmac_mac_config_rx_queues_prio(priv); 3526 3527 /* Set TX priorities */ 3528 if (tx_queues_count > 1) 3529 stmmac_mac_config_tx_queues_prio(priv); 3530 3531 /* Set RX routing */ 3532 if (rx_queues_count > 1) 3533 stmmac_mac_config_rx_queues_routing(priv); 3534 3535 /* Receive Side Scaling */ 3536 if (rx_queues_count > 1) 3537 stmmac_mac_config_rss(priv); 3538} 3539 3540static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) 3541{ 3542 if (priv->dma_cap.asp) { 3543 netdev_info(priv->dev, "Enabling Safety Features\n"); 3544 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp, 3545 priv->plat->safety_feat_cfg); 3546 } else { 3547 netdev_info(priv->dev, "No Safety Features support found\n"); 3548 } 3549} 3550 3551/** 3552 * stmmac_hw_setup - setup mac in a usable state. 3553 * @dev : pointer to the device structure. 3554 * Description: 3555 * this is the main function to setup the HW in a usable state because the 3556 * dma engine is reset, the core registers are configured (e.g. AXI, 3557 * Checksum features, timers). The DMA is ready to start receiving and 3558 * transmitting. 3559 * Return value: 3560 * 0 on success and an appropriate (-)ve integer as defined in errno.h 3561 * file on failure. 3562 */ 3563static int stmmac_hw_setup(struct net_device *dev) 3564{ 3565 struct stmmac_priv *priv = netdev_priv(dev); 3566 u32 rx_cnt = priv->plat->rx_queues_to_use; 3567 u32 tx_cnt = priv->plat->tx_queues_to_use; 3568 bool sph_en; 3569 u32 chan; 3570 int ret; 3571 3572 /* Make sure RX clock is enabled */ 3573 if (priv->hw->phylink_pcs) 3574 phylink_pcs_pre_init(priv->phylink, priv->hw->phylink_pcs); 3575 3576 /* Note that clk_rx_i must be running for reset to complete. This 3577 * clock may also be required when setting the MAC address. 3578 * 3579 * Block the receive clock stop for LPI mode at the PHY in case 3580 * the link is established with EEE mode active. 3581 */ 3582 phylink_rx_clk_stop_block(priv->phylink); 3583 3584 /* DMA initialization and SW reset */ 3585 ret = stmmac_init_dma_engine(priv); 3586 if (ret < 0) { 3587 phylink_rx_clk_stop_unblock(priv->phylink); 3588 netdev_err(priv->dev, "%s: DMA engine initialization failed\n", 3589 __func__); 3590 return ret; 3591 } 3592 3593 /* Copy the MAC addr into the HW */ 3594 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0); 3595 phylink_rx_clk_stop_unblock(priv->phylink); 3596 3597 /* Initialize the MAC Core */ 3598 stmmac_core_init(priv, priv->hw, dev); 3599 3600 /* Initialize MTL*/ 3601 stmmac_mtl_configuration(priv); 3602 3603 /* Initialize Safety Features */ 3604 stmmac_safety_feat_configuration(priv); 3605 3606 ret = stmmac_rx_ipc(priv, priv->hw); 3607 if (!ret) { 3608 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n"); 3609 priv->plat->rx_coe = STMMAC_RX_COE_NONE; 3610 priv->hw->rx_csum = 0; 3611 } 3612 3613 /* Enable the MAC Rx/Tx */ 3614 stmmac_mac_set(priv, priv->ioaddr, true); 3615 3616 /* Set the HW DMA mode and the COE */ 3617 stmmac_dma_operation_mode(priv); 3618 3619 stmmac_mmc_setup(priv); 3620 3621 if (priv->use_riwt) { 3622 u32 queue; 3623 3624 for (queue = 0; queue < rx_cnt; queue++) { 3625 if (!priv->rx_riwt[queue]) 3626 priv->rx_riwt[queue] = DEF_DMA_RIWT; 3627 3628 stmmac_rx_watchdog(priv, priv->ioaddr, 3629 priv->rx_riwt[queue], queue); 3630 } 3631 } 3632 3633 /* set TX and RX rings length */ 3634 stmmac_set_rings_length(priv); 3635 3636 /* Enable TSO */ 3637 if (priv->tso) { 3638 for (chan = 0; chan < tx_cnt; chan++) { 3639 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3640 3641 /* TSO and TBS cannot co-exist */ 3642 if (tx_q->tbs & STMMAC_TBS_AVAIL) 3643 continue; 3644 3645 stmmac_enable_tso(priv, priv->ioaddr, 1, chan); 3646 } 3647 } 3648 3649 /* Enable Split Header */ 3650 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 3651 for (chan = 0; chan < rx_cnt; chan++) 3652 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 3653 3654 3655 /* VLAN Tag Insertion */ 3656 if (priv->dma_cap.vlins) 3657 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT); 3658 3659 /* TBS */ 3660 for (chan = 0; chan < tx_cnt; chan++) { 3661 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; 3662 int enable = tx_q->tbs & STMMAC_TBS_AVAIL; 3663 3664 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan); 3665 } 3666 3667 /* Configure real RX and TX queues */ 3668 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use); 3669 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use); 3670 3671 /* Start the ball rolling... */ 3672 stmmac_start_all_dma(priv); 3673 3674 phylink_rx_clk_stop_block(priv->phylink); 3675 stmmac_set_hw_vlan_mode(priv, priv->hw); 3676 phylink_rx_clk_stop_unblock(priv->phylink); 3677 3678 return 0; 3679} 3680 3681static void stmmac_free_irq(struct net_device *dev, 3682 enum request_irq_err irq_err, int irq_idx) 3683{ 3684 struct stmmac_priv *priv = netdev_priv(dev); 3685 int j; 3686 3687 switch (irq_err) { 3688 case REQ_IRQ_ERR_ALL: 3689 irq_idx = priv->plat->tx_queues_to_use; 3690 fallthrough; 3691 case REQ_IRQ_ERR_TX: 3692 for (j = irq_idx - 1; j >= 0; j--) { 3693 if (priv->tx_irq[j] > 0) { 3694 irq_set_affinity_hint(priv->tx_irq[j], NULL); 3695 free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); 3696 } 3697 } 3698 irq_idx = priv->plat->rx_queues_to_use; 3699 fallthrough; 3700 case REQ_IRQ_ERR_RX: 3701 for (j = irq_idx - 1; j >= 0; j--) { 3702 if (priv->rx_irq[j] > 0) { 3703 irq_set_affinity_hint(priv->rx_irq[j], NULL); 3704 free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); 3705 } 3706 } 3707 3708 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) 3709 free_irq(priv->sfty_ue_irq, dev); 3710 fallthrough; 3711 case REQ_IRQ_ERR_SFTY_UE: 3712 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) 3713 free_irq(priv->sfty_ce_irq, dev); 3714 fallthrough; 3715 case REQ_IRQ_ERR_SFTY_CE: 3716 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) 3717 free_irq(priv->wol_irq, dev); 3718 fallthrough; 3719 case REQ_IRQ_ERR_SFTY: 3720 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) 3721 free_irq(priv->sfty_irq, dev); 3722 fallthrough; 3723 case REQ_IRQ_ERR_WOL: 3724 free_irq(dev->irq, dev); 3725 fallthrough; 3726 case REQ_IRQ_ERR_MAC: 3727 case REQ_IRQ_ERR_NO: 3728 /* If MAC IRQ request error, no more IRQ to free */ 3729 break; 3730 } 3731} 3732 3733static int stmmac_request_irq_multi_msi(struct net_device *dev) 3734{ 3735 struct stmmac_priv *priv = netdev_priv(dev); 3736 enum request_irq_err irq_err; 3737 int irq_idx = 0; 3738 char *int_name; 3739 int ret; 3740 int i; 3741 3742 /* For common interrupt */ 3743 int_name = priv->int_name_mac; 3744 sprintf(int_name, "%s:%s", dev->name, "mac"); 3745 ret = request_irq(dev->irq, stmmac_mac_interrupt, 3746 0, int_name, dev); 3747 if (unlikely(ret < 0)) { 3748 netdev_err(priv->dev, 3749 "%s: alloc mac MSI %d (error: %d)\n", 3750 __func__, dev->irq, ret); 3751 irq_err = REQ_IRQ_ERR_MAC; 3752 goto irq_error; 3753 } 3754 3755 /* Request the Wake IRQ in case of another line 3756 * is used for WoL 3757 */ 3758 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3759 int_name = priv->int_name_wol; 3760 sprintf(int_name, "%s:%s", dev->name, "wol"); 3761 ret = request_irq(priv->wol_irq, 3762 stmmac_mac_interrupt, 3763 0, int_name, dev); 3764 if (unlikely(ret < 0)) { 3765 netdev_err(priv->dev, 3766 "%s: alloc wol MSI %d (error: %d)\n", 3767 __func__, priv->wol_irq, ret); 3768 irq_err = REQ_IRQ_ERR_WOL; 3769 goto irq_error; 3770 } 3771 } 3772 3773 /* Request the common Safety Feature Correctible/Uncorrectible 3774 * Error line in case of another line is used 3775 */ 3776 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3777 int_name = priv->int_name_sfty; 3778 sprintf(int_name, "%s:%s", dev->name, "safety"); 3779 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3780 0, int_name, dev); 3781 if (unlikely(ret < 0)) { 3782 netdev_err(priv->dev, 3783 "%s: alloc sfty MSI %d (error: %d)\n", 3784 __func__, priv->sfty_irq, ret); 3785 irq_err = REQ_IRQ_ERR_SFTY; 3786 goto irq_error; 3787 } 3788 } 3789 3790 /* Request the Safety Feature Correctible Error line in 3791 * case of another line is used 3792 */ 3793 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { 3794 int_name = priv->int_name_sfty_ce; 3795 sprintf(int_name, "%s:%s", dev->name, "safety-ce"); 3796 ret = request_irq(priv->sfty_ce_irq, 3797 stmmac_safety_interrupt, 3798 0, int_name, dev); 3799 if (unlikely(ret < 0)) { 3800 netdev_err(priv->dev, 3801 "%s: alloc sfty ce MSI %d (error: %d)\n", 3802 __func__, priv->sfty_ce_irq, ret); 3803 irq_err = REQ_IRQ_ERR_SFTY_CE; 3804 goto irq_error; 3805 } 3806 } 3807 3808 /* Request the Safety Feature Uncorrectible Error line in 3809 * case of another line is used 3810 */ 3811 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { 3812 int_name = priv->int_name_sfty_ue; 3813 sprintf(int_name, "%s:%s", dev->name, "safety-ue"); 3814 ret = request_irq(priv->sfty_ue_irq, 3815 stmmac_safety_interrupt, 3816 0, int_name, dev); 3817 if (unlikely(ret < 0)) { 3818 netdev_err(priv->dev, 3819 "%s: alloc sfty ue MSI %d (error: %d)\n", 3820 __func__, priv->sfty_ue_irq, ret); 3821 irq_err = REQ_IRQ_ERR_SFTY_UE; 3822 goto irq_error; 3823 } 3824 } 3825 3826 /* Request Rx MSI irq */ 3827 for (i = 0; i < priv->plat->rx_queues_to_use; i++) { 3828 if (i >= MTL_MAX_RX_QUEUES) 3829 break; 3830 if (priv->rx_irq[i] == 0) 3831 continue; 3832 3833 int_name = priv->int_name_rx_irq[i]; 3834 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); 3835 ret = request_irq(priv->rx_irq[i], 3836 stmmac_msi_intr_rx, 3837 0, int_name, &priv->dma_conf.rx_queue[i]); 3838 if (unlikely(ret < 0)) { 3839 netdev_err(priv->dev, 3840 "%s: alloc rx-%d MSI %d (error: %d)\n", 3841 __func__, i, priv->rx_irq[i], ret); 3842 irq_err = REQ_IRQ_ERR_RX; 3843 irq_idx = i; 3844 goto irq_error; 3845 } 3846 irq_set_affinity_hint(priv->rx_irq[i], 3847 cpumask_of(i % num_online_cpus())); 3848 } 3849 3850 /* Request Tx MSI irq */ 3851 for (i = 0; i < priv->plat->tx_queues_to_use; i++) { 3852 if (i >= MTL_MAX_TX_QUEUES) 3853 break; 3854 if (priv->tx_irq[i] == 0) 3855 continue; 3856 3857 int_name = priv->int_name_tx_irq[i]; 3858 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); 3859 ret = request_irq(priv->tx_irq[i], 3860 stmmac_msi_intr_tx, 3861 0, int_name, &priv->dma_conf.tx_queue[i]); 3862 if (unlikely(ret < 0)) { 3863 netdev_err(priv->dev, 3864 "%s: alloc tx-%d MSI %d (error: %d)\n", 3865 __func__, i, priv->tx_irq[i], ret); 3866 irq_err = REQ_IRQ_ERR_TX; 3867 irq_idx = i; 3868 goto irq_error; 3869 } 3870 irq_set_affinity_hint(priv->tx_irq[i], 3871 cpumask_of(i % num_online_cpus())); 3872 } 3873 3874 return 0; 3875 3876irq_error: 3877 stmmac_free_irq(dev, irq_err, irq_idx); 3878 return ret; 3879} 3880 3881static int stmmac_request_irq_single(struct net_device *dev) 3882{ 3883 struct stmmac_priv *priv = netdev_priv(dev); 3884 enum request_irq_err irq_err; 3885 int ret; 3886 3887 ret = request_irq(dev->irq, stmmac_interrupt, 3888 IRQF_SHARED, dev->name, dev); 3889 if (unlikely(ret < 0)) { 3890 netdev_err(priv->dev, 3891 "%s: ERROR: allocating the IRQ %d (error: %d)\n", 3892 __func__, dev->irq, ret); 3893 irq_err = REQ_IRQ_ERR_MAC; 3894 goto irq_error; 3895 } 3896 3897 /* Request the Wake IRQ in case of another line 3898 * is used for WoL 3899 */ 3900 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { 3901 ret = request_irq(priv->wol_irq, stmmac_interrupt, 3902 IRQF_SHARED, dev->name, dev); 3903 if (unlikely(ret < 0)) { 3904 netdev_err(priv->dev, 3905 "%s: ERROR: allocating the WoL IRQ %d (%d)\n", 3906 __func__, priv->wol_irq, ret); 3907 irq_err = REQ_IRQ_ERR_WOL; 3908 goto irq_error; 3909 } 3910 } 3911 3912 /* Request the common Safety Feature Correctible/Uncorrectible 3913 * Error line in case of another line is used 3914 */ 3915 if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { 3916 ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 3917 IRQF_SHARED, dev->name, dev); 3918 if (unlikely(ret < 0)) { 3919 netdev_err(priv->dev, 3920 "%s: ERROR: allocating the sfty IRQ %d (%d)\n", 3921 __func__, priv->sfty_irq, ret); 3922 irq_err = REQ_IRQ_ERR_SFTY; 3923 goto irq_error; 3924 } 3925 } 3926 3927 return 0; 3928 3929irq_error: 3930 stmmac_free_irq(dev, irq_err, 0); 3931 return ret; 3932} 3933 3934static int stmmac_request_irq(struct net_device *dev) 3935{ 3936 struct stmmac_priv *priv = netdev_priv(dev); 3937 int ret; 3938 3939 /* Request the IRQ lines */ 3940 if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) 3941 ret = stmmac_request_irq_multi_msi(dev); 3942 else 3943 ret = stmmac_request_irq_single(dev); 3944 3945 return ret; 3946} 3947 3948/** 3949 * stmmac_setup_dma_desc - Generate a dma_conf and allocate DMA queue 3950 * @priv: driver private structure 3951 * @mtu: MTU to setup the dma queue and buf with 3952 * Description: Allocate and generate a dma_conf based on the provided MTU. 3953 * Allocate the Tx/Rx DMA queue and init them. 3954 * Return value: 3955 * the dma_conf allocated struct on success and an appropriate ERR_PTR on failure. 3956 */ 3957static struct stmmac_dma_conf * 3958stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) 3959{ 3960 struct stmmac_dma_conf *dma_conf; 3961 int chan, bfsize, ret; 3962 3963 dma_conf = kzalloc(sizeof(*dma_conf), GFP_KERNEL); 3964 if (!dma_conf) { 3965 netdev_err(priv->dev, "%s: DMA conf allocation failed\n", 3966 __func__); 3967 return ERR_PTR(-ENOMEM); 3968 } 3969 3970 /* Returns 0 or BUF_SIZE_16KiB if mtu > 8KiB and dwmac4 or ring mode */ 3971 bfsize = stmmac_set_16kib_bfsize(priv, mtu); 3972 if (bfsize < 0) 3973 bfsize = 0; 3974 3975 if (bfsize < BUF_SIZE_16KiB) 3976 bfsize = stmmac_set_bfsize(mtu); 3977 3978 dma_conf->dma_buf_sz = bfsize; 3979 /* Chose the tx/rx size from the already defined one in the 3980 * priv struct. (if defined) 3981 */ 3982 dma_conf->dma_tx_size = priv->dma_conf.dma_tx_size; 3983 dma_conf->dma_rx_size = priv->dma_conf.dma_rx_size; 3984 3985 if (!dma_conf->dma_tx_size) 3986 dma_conf->dma_tx_size = DMA_DEFAULT_TX_SIZE; 3987 if (!dma_conf->dma_rx_size) 3988 dma_conf->dma_rx_size = DMA_DEFAULT_RX_SIZE; 3989 3990 /* Earlier check for TBS */ 3991 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) { 3992 struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[chan]; 3993 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en; 3994 3995 /* Setup per-TXQ tbs flag before TX descriptor alloc */ 3996 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0; 3997 } 3998 3999 ret = alloc_dma_desc_resources(priv, dma_conf); 4000 if (ret < 0) { 4001 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n", 4002 __func__); 4003 goto alloc_error; 4004 } 4005 4006 ret = init_dma_desc_rings(priv->dev, dma_conf, GFP_KERNEL); 4007 if (ret < 0) { 4008 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n", 4009 __func__); 4010 goto init_error; 4011 } 4012 4013 return dma_conf; 4014 4015init_error: 4016 free_dma_desc_resources(priv, dma_conf); 4017alloc_error: 4018 kfree(dma_conf); 4019 return ERR_PTR(ret); 4020} 4021 4022/** 4023 * __stmmac_open - open entry point of the driver 4024 * @dev : pointer to the device structure. 4025 * @dma_conf : structure to take the dma data 4026 * Description: 4027 * This function is the open entry point of the driver. 4028 * Return value: 4029 * 0 on success and an appropriate (-)ve integer as defined in errno.h 4030 * file on failure. 4031 */ 4032static int __stmmac_open(struct net_device *dev, 4033 struct stmmac_dma_conf *dma_conf) 4034{ 4035 struct stmmac_priv *priv = netdev_priv(dev); 4036 u32 chan; 4037 int ret; 4038 4039 for (int i = 0; i < MTL_MAX_TX_QUEUES; i++) 4040 if (priv->dma_conf.tx_queue[i].tbs & STMMAC_TBS_EN) 4041 dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; 4042 memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); 4043 4044 stmmac_reset_queues_param(priv); 4045 4046 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 4047 priv->plat->serdes_powerup) { 4048 ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv); 4049 if (ret < 0) { 4050 netdev_err(priv->dev, "%s: Serdes powerup failed\n", 4051 __func__); 4052 goto init_error; 4053 } 4054 } 4055 4056 ret = stmmac_hw_setup(dev); 4057 if (ret < 0) { 4058 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 4059 goto init_error; 4060 } 4061 4062 stmmac_setup_ptp(priv); 4063 4064 stmmac_init_coalesce(priv); 4065 4066 phylink_start(priv->phylink); 4067 4068 stmmac_vlan_restore(priv); 4069 4070 ret = stmmac_request_irq(dev); 4071 if (ret) 4072 goto irq_error; 4073 4074 stmmac_enable_all_queues(priv); 4075 netif_tx_start_all_queues(priv->dev); 4076 stmmac_enable_all_dma_irq(priv); 4077 4078 return 0; 4079 4080irq_error: 4081 phylink_stop(priv->phylink); 4082 4083 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4084 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4085 4086 stmmac_release_ptp(priv); 4087init_error: 4088 return ret; 4089} 4090 4091static int stmmac_open(struct net_device *dev) 4092{ 4093 struct stmmac_priv *priv = netdev_priv(dev); 4094 struct stmmac_dma_conf *dma_conf; 4095 int ret; 4096 4097 /* Initialise the tx lpi timer, converting from msec to usec */ 4098 if (!priv->tx_lpi_timer) 4099 priv->tx_lpi_timer = eee_timer * 1000; 4100 4101 dma_conf = stmmac_setup_dma_desc(priv, dev->mtu); 4102 if (IS_ERR(dma_conf)) 4103 return PTR_ERR(dma_conf); 4104 4105 ret = pm_runtime_resume_and_get(priv->device); 4106 if (ret < 0) 4107 goto err_dma_resources; 4108 4109 ret = stmmac_init_phy(dev); 4110 if (ret) 4111 goto err_runtime_pm; 4112 4113 ret = __stmmac_open(dev, dma_conf); 4114 if (ret) 4115 goto err_disconnect_phy; 4116 4117 kfree(dma_conf); 4118 4119 /* We may have called phylink_speed_down before */ 4120 phylink_speed_up(priv->phylink); 4121 4122 return ret; 4123 4124err_disconnect_phy: 4125 phylink_disconnect_phy(priv->phylink); 4126err_runtime_pm: 4127 pm_runtime_put(priv->device); 4128err_dma_resources: 4129 free_dma_desc_resources(priv, dma_conf); 4130 kfree(dma_conf); 4131 return ret; 4132} 4133 4134static void __stmmac_release(struct net_device *dev) 4135{ 4136 struct stmmac_priv *priv = netdev_priv(dev); 4137 u32 chan; 4138 4139 /* Stop and disconnect the PHY */ 4140 phylink_stop(priv->phylink); 4141 4142 stmmac_disable_all_queues(priv); 4143 4144 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 4145 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 4146 4147 netif_tx_disable(dev); 4148 4149 /* Free the IRQ lines */ 4150 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 4151 4152 /* Stop TX/RX DMA and clear the descriptors */ 4153 stmmac_stop_all_dma(priv); 4154 4155 /* Release and free the Rx/Tx resources */ 4156 free_dma_desc_resources(priv, &priv->dma_conf); 4157 4158 /* Powerdown Serdes if there is */ 4159 if (priv->plat->serdes_powerdown) 4160 priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv); 4161 4162 stmmac_release_ptp(priv); 4163 4164 if (stmmac_fpe_supported(priv)) 4165 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 4166} 4167 4168/** 4169 * stmmac_release - close entry point of the driver 4170 * @dev : device pointer. 4171 * Description: 4172 * This is the stop entry point of the driver. 4173 */ 4174static int stmmac_release(struct net_device *dev) 4175{ 4176 struct stmmac_priv *priv = netdev_priv(dev); 4177 4178 /* If the PHY or MAC has WoL enabled, then the PHY will not be 4179 * suspended when phylink_stop() is called below. Set the PHY 4180 * to its slowest speed to save power. 4181 */ 4182 if (device_may_wakeup(priv->device)) 4183 phylink_speed_down(priv->phylink, false); 4184 4185 __stmmac_release(dev); 4186 4187 phylink_disconnect_phy(priv->phylink); 4188 pm_runtime_put(priv->device); 4189 4190 return 0; 4191} 4192 4193static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, 4194 struct stmmac_tx_queue *tx_q) 4195{ 4196 struct dma_desc *p; 4197 u16 tag = 0x0; 4198 4199 if (!priv->dma_cap.vlins || !skb_vlan_tag_present(skb)) 4200 return false; 4201 4202 tag = skb_vlan_tag_get(skb); 4203 4204 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4205 p = &tx_q->dma_entx[tx_q->cur_tx].basic; 4206 else 4207 p = &tx_q->dma_tx[tx_q->cur_tx]; 4208 4209 if (stmmac_set_desc_vlan_tag(priv, p, tag, 0x0, 0x0)) 4210 return false; 4211 4212 stmmac_set_tx_owner(priv, p); 4213 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4214 return true; 4215} 4216 4217/** 4218 * stmmac_tso_allocator - close entry point of the driver 4219 * @priv: driver private structure 4220 * @des: buffer start address 4221 * @total_len: total length to fill in descriptors 4222 * @last_segment: condition for the last descriptor 4223 * @queue: TX queue index 4224 * Description: 4225 * This function fills descriptor and request new descriptors according to 4226 * buffer length to fill 4227 */ 4228static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, 4229 int total_len, bool last_segment, u32 queue) 4230{ 4231 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4232 struct dma_desc *desc; 4233 u32 buff_size; 4234 int tmp_len; 4235 4236 tmp_len = total_len; 4237 4238 while (tmp_len > 0) { 4239 dma_addr_t curr_addr; 4240 4241 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4242 priv->dma_conf.dma_tx_size); 4243 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4244 4245 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4246 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4247 else 4248 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4249 4250 curr_addr = des + (total_len - tmp_len); 4251 stmmac_set_desc_addr(priv, desc, curr_addr); 4252 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ? 4253 TSO_MAX_BUFF_SIZE : tmp_len; 4254 4255 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size, 4256 0, 1, 4257 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE), 4258 0, 0); 4259 4260 tmp_len -= TSO_MAX_BUFF_SIZE; 4261 } 4262} 4263 4264static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) 4265{ 4266 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 4267 int desc_size; 4268 4269 if (likely(priv->extend_desc)) 4270 desc_size = sizeof(struct dma_extended_desc); 4271 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4272 desc_size = sizeof(struct dma_edesc); 4273 else 4274 desc_size = sizeof(struct dma_desc); 4275 4276 /* The own bit must be the latest setting done when prepare the 4277 * descriptor and then barrier is needed to make sure that 4278 * all is coherent before granting the DMA engine. 4279 */ 4280 wmb(); 4281 4282 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); 4283 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); 4284} 4285 4286/** 4287 * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) 4288 * @skb : the socket buffer 4289 * @dev : device pointer 4290 * Description: this is the transmit function that is called on TSO frames 4291 * (support available on GMAC4 and newer chips). 4292 * Diagram below show the ring programming in case of TSO frames: 4293 * 4294 * First Descriptor 4295 * -------- 4296 * | DES0 |---> buffer1 = L2/L3/L4 header 4297 * | DES1 |---> can be used as buffer2 for TCP Payload if the DMA AXI address 4298 * | | width is 32-bit, but we never use it. 4299 * | | Also can be used as the most-significant 8-bits or 16-bits of 4300 * | | buffer1 address pointer if the DMA AXI address width is 40-bit 4301 * | | or 48-bit, and we always use it. 4302 * | DES2 |---> buffer1 len 4303 * | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0] 4304 * -------- 4305 * -------- 4306 * | DES0 |---> buffer1 = TCP Payload (can continue on next descr...) 4307 * | DES1 |---> same as the First Descriptor 4308 * | DES2 |---> buffer1 len 4309 * | DES3 | 4310 * -------- 4311 * | 4312 * ... 4313 * | 4314 * -------- 4315 * | DES0 |---> buffer1 = Split TCP Payload 4316 * | DES1 |---> same as the First Descriptor 4317 * | DES2 |---> buffer1 len 4318 * | DES3 | 4319 * -------- 4320 * 4321 * mss is fixed when enable tso, so w/o programming the TDES3 ctx field. 4322 */ 4323static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) 4324{ 4325 struct dma_desc *desc, *first, *mss_desc = NULL; 4326 struct stmmac_priv *priv = netdev_priv(dev); 4327 unsigned int first_entry, tx_packets; 4328 struct stmmac_txq_stats *txq_stats; 4329 struct stmmac_tx_queue *tx_q; 4330 bool set_ic, is_last_segment; 4331 u32 pay_len, mss, queue; 4332 int i, first_tx, nfrags; 4333 u8 proto_hdr_len, hdr; 4334 dma_addr_t des; 4335 4336 /* Always insert VLAN tag to SKB payload for TSO frames. 4337 * 4338 * Never insert VLAN tag by HW, since segments splited by 4339 * TSO engine will be un-tagged by mistake. 4340 */ 4341 if (skb_vlan_tag_present(skb)) { 4342 skb = __vlan_hwaccel_push_inside(skb); 4343 if (unlikely(!skb)) { 4344 priv->xstats.tx_dropped++; 4345 return NETDEV_TX_OK; 4346 } 4347 } 4348 4349 nfrags = skb_shinfo(skb)->nr_frags; 4350 queue = skb_get_queue_mapping(skb); 4351 4352 tx_q = &priv->dma_conf.tx_queue[queue]; 4353 txq_stats = &priv->xstats.txq_stats[queue]; 4354 first_tx = tx_q->cur_tx; 4355 4356 /* Compute header lengths */ 4357 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 4358 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); 4359 hdr = sizeof(struct udphdr); 4360 } else { 4361 proto_hdr_len = skb_tcp_all_headers(skb); 4362 hdr = tcp_hdrlen(skb); 4363 } 4364 4365 /* Desc availability based on threshold should be enough safe */ 4366 if (unlikely(stmmac_tx_avail(priv, queue) < 4367 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { 4368 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4369 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4370 queue)); 4371 /* This is a hard error, log it. */ 4372 netdev_err(priv->dev, 4373 "%s: Tx Ring full when queue awake\n", 4374 __func__); 4375 } 4376 return NETDEV_TX_BUSY; 4377 } 4378 4379 pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ 4380 4381 mss = skb_shinfo(skb)->gso_size; 4382 4383 /* set new MSS value if needed */ 4384 if (mss != tx_q->mss) { 4385 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4386 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4387 else 4388 mss_desc = &tx_q->dma_tx[tx_q->cur_tx]; 4389 4390 stmmac_set_mss(priv, mss_desc, mss); 4391 tx_q->mss = mss; 4392 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, 4393 priv->dma_conf.dma_tx_size); 4394 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); 4395 } 4396 4397 if (netif_msg_tx_queued(priv)) { 4398 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n", 4399 __func__, hdr, proto_hdr_len, pay_len, mss); 4400 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len, 4401 skb->data_len); 4402 } 4403 4404 first_entry = tx_q->cur_tx; 4405 WARN_ON(tx_q->tx_skbuff[first_entry]); 4406 4407 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4408 desc = &tx_q->dma_entx[first_entry].basic; 4409 else 4410 desc = &tx_q->dma_tx[first_entry]; 4411 first = desc; 4412 4413 /* first descriptor: fill Headers on Buf1 */ 4414 des = dma_map_single(priv->device, skb->data, skb_headlen(skb), 4415 DMA_TO_DEVICE); 4416 if (dma_mapping_error(priv->device, des)) 4417 goto dma_map_err; 4418 4419 stmmac_set_desc_addr(priv, first, des); 4420 stmmac_tso_allocator(priv, des + proto_hdr_len, pay_len, 4421 (nfrags == 0), queue); 4422 4423 /* In case two or more DMA transmit descriptors are allocated for this 4424 * non-paged SKB data, the DMA buffer address should be saved to 4425 * tx_q->tx_skbuff_dma[].buf corresponding to the last descriptor, 4426 * and leave the other tx_q->tx_skbuff_dma[].buf as NULL to guarantee 4427 * that stmmac_tx_clean() does not unmap the entire DMA buffer too early 4428 * since the tail areas of the DMA buffer can be accessed by DMA engine 4429 * sooner or later. 4430 * By saving the DMA buffer address to tx_q->tx_skbuff_dma[].buf 4431 * corresponding to the last descriptor, stmmac_tx_clean() will unmap 4432 * this DMA buffer right after the DMA engine completely finishes the 4433 * full buffer transmission. 4434 */ 4435 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4436 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb); 4437 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false; 4438 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4439 4440 /* Prepare fragments */ 4441 for (i = 0; i < nfrags; i++) { 4442 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4443 4444 des = skb_frag_dma_map(priv->device, frag, 0, 4445 skb_frag_size(frag), 4446 DMA_TO_DEVICE); 4447 if (dma_mapping_error(priv->device, des)) 4448 goto dma_map_err; 4449 4450 stmmac_tso_allocator(priv, des, skb_frag_size(frag), 4451 (i == nfrags - 1), queue); 4452 4453 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; 4454 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); 4455 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; 4456 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4457 } 4458 4459 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; 4460 4461 /* Only the last descriptor gets to point to the skb. */ 4462 tx_q->tx_skbuff[tx_q->cur_tx] = skb; 4463 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; 4464 4465 /* Manage tx mitigation */ 4466 tx_packets = (tx_q->cur_tx + 1) - first_tx; 4467 tx_q->tx_count_frames += tx_packets; 4468 4469 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4470 set_ic = true; 4471 else if (!priv->tx_coal_frames[queue]) 4472 set_ic = false; 4473 else if (tx_packets > priv->tx_coal_frames[queue]) 4474 set_ic = true; 4475 else if ((tx_q->tx_count_frames % 4476 priv->tx_coal_frames[queue]) < tx_packets) 4477 set_ic = true; 4478 else 4479 set_ic = false; 4480 4481 if (set_ic) { 4482 if (tx_q->tbs & STMMAC_TBS_AVAIL) 4483 desc = &tx_q->dma_entx[tx_q->cur_tx].basic; 4484 else 4485 desc = &tx_q->dma_tx[tx_q->cur_tx]; 4486 4487 tx_q->tx_count_frames = 0; 4488 stmmac_set_tx_ic(priv, desc); 4489 } 4490 4491 /* We've used all descriptors we need for this skb, however, 4492 * advance cur_tx so that it references a fresh descriptor. 4493 * ndo_start_xmit will fill this descriptor the next time it's 4494 * called and stmmac_tx_clean may clean up to this descriptor. 4495 */ 4496 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); 4497 4498 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4499 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4500 __func__); 4501 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4502 } 4503 4504 u64_stats_update_begin(&txq_stats->q_syncp); 4505 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4506 u64_stats_inc(&txq_stats->q.tx_tso_frames); 4507 u64_stats_add(&txq_stats->q.tx_tso_nfrags, nfrags); 4508 if (set_ic) 4509 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4510 u64_stats_update_end(&txq_stats->q_syncp); 4511 4512 if (priv->sarc_type) 4513 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4514 4515 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4516 priv->hwts_tx_en)) { 4517 /* declare that device is doing timestamping */ 4518 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4519 stmmac_enable_tx_timestamp(priv, first); 4520 } 4521 4522 /* If we only have one entry used, then the first entry is the last 4523 * segment. 4524 */ 4525 is_last_segment = ((tx_q->cur_tx - first_entry) & 4526 (priv->dma_conf.dma_tx_size - 1)) == 1; 4527 4528 /* Complete the first descriptor before granting the DMA */ 4529 stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1, 4530 is_last_segment, hdr / 4, 4531 skb->len - proto_hdr_len); 4532 4533 /* If context desc is used to change MSS */ 4534 if (mss_desc) { 4535 /* Make sure that first descriptor has been completely 4536 * written, including its own bit. This is because MSS is 4537 * actually before first descriptor, so we need to make 4538 * sure that MSS's own bit is the last thing written. 4539 */ 4540 dma_wmb(); 4541 stmmac_set_tx_owner(priv, mss_desc); 4542 } 4543 4544 if (netif_msg_pktdata(priv)) { 4545 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n", 4546 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4547 tx_q->cur_tx, first, nfrags); 4548 pr_info(">>> frame to be transmitted: "); 4549 print_pkt(skb->data, skb_headlen(skb)); 4550 } 4551 4552 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4553 skb_tx_timestamp(skb); 4554 4555 stmmac_flush_tx_descriptors(priv, queue); 4556 stmmac_tx_timer_arm(priv, queue); 4557 4558 return NETDEV_TX_OK; 4559 4560dma_map_err: 4561 dev_err(priv->device, "Tx dma map failed\n"); 4562 dev_kfree_skb(skb); 4563 priv->xstats.tx_dropped++; 4564 return NETDEV_TX_OK; 4565} 4566 4567/** 4568 * stmmac_has_ip_ethertype() - Check if packet has IP ethertype 4569 * @skb: socket buffer to check 4570 * 4571 * Check if a packet has an ethertype that will trigger the IP header checks 4572 * and IP/TCP checksum engine of the stmmac core. 4573 * 4574 * Return: true if the ethertype can trigger the checksum engine, false 4575 * otherwise 4576 */ 4577static bool stmmac_has_ip_ethertype(struct sk_buff *skb) 4578{ 4579 int depth = 0; 4580 __be16 proto; 4581 4582 proto = __vlan_get_protocol(skb, eth_header_parse_protocol(skb), 4583 &depth); 4584 4585 return (depth <= ETH_HLEN) && 4586 (proto == htons(ETH_P_IP) || proto == htons(ETH_P_IPV6)); 4587} 4588 4589/** 4590 * stmmac_xmit - Tx entry point of the driver 4591 * @skb : the socket buffer 4592 * @dev : device pointer 4593 * Description : this is the tx entry point of the driver. 4594 * It programs the chain or the ring and supports oversized frames 4595 * and SG feature. 4596 */ 4597static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) 4598{ 4599 bool enh_desc, has_vlan, set_ic, is_jumbo = false; 4600 struct stmmac_priv *priv = netdev_priv(dev); 4601 unsigned int nopaged_len = skb_headlen(skb); 4602 u32 queue = skb_get_queue_mapping(skb); 4603 int nfrags = skb_shinfo(skb)->nr_frags; 4604 unsigned int first_entry, tx_packets; 4605 int gso = skb_shinfo(skb)->gso_type; 4606 struct stmmac_txq_stats *txq_stats; 4607 struct dma_edesc *tbs_desc = NULL; 4608 struct dma_desc *desc, *first; 4609 struct stmmac_tx_queue *tx_q; 4610 int i, csum_insertion = 0; 4611 int entry, first_tx; 4612 dma_addr_t des; 4613 u32 sdu_len; 4614 4615 tx_q = &priv->dma_conf.tx_queue[queue]; 4616 txq_stats = &priv->xstats.txq_stats[queue]; 4617 first_tx = tx_q->cur_tx; 4618 4619 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) 4620 stmmac_stop_sw_lpi(priv); 4621 4622 /* Manage oversized TCP frames for GMAC4 device */ 4623 if (skb_is_gso(skb) && priv->tso) { 4624 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) 4625 return stmmac_tso_xmit(skb, dev); 4626 if (priv->plat->core_type == DWMAC_CORE_GMAC4 && 4627 (gso & SKB_GSO_UDP_L4)) 4628 return stmmac_tso_xmit(skb, dev); 4629 } 4630 4631 if (priv->est && priv->est->enable && 4632 priv->est->max_sdu[queue]) { 4633 sdu_len = skb->len; 4634 /* Add VLAN tag length if VLAN tag insertion offload is requested */ 4635 if (priv->dma_cap.vlins && skb_vlan_tag_present(skb)) 4636 sdu_len += VLAN_HLEN; 4637 if (sdu_len > priv->est->max_sdu[queue]) { 4638 priv->xstats.max_sdu_txq_drop[queue]++; 4639 goto max_sdu_err; 4640 } 4641 } 4642 4643 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) { 4644 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { 4645 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, 4646 queue)); 4647 /* This is a hard error, log it. */ 4648 netdev_err(priv->dev, 4649 "%s: Tx Ring full when queue awake\n", 4650 __func__); 4651 } 4652 return NETDEV_TX_BUSY; 4653 } 4654 4655 /* Check if VLAN can be inserted by HW */ 4656 has_vlan = stmmac_vlan_insert(priv, skb, tx_q); 4657 4658 entry = tx_q->cur_tx; 4659 first_entry = entry; 4660 WARN_ON(tx_q->tx_skbuff[first_entry]); 4661 4662 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); 4663 /* DWMAC IPs can be synthesized to support tx coe only for a few tx 4664 * queues. In that case, checksum offloading for those queues that don't 4665 * support tx coe needs to fallback to software checksum calculation. 4666 * 4667 * Packets that won't trigger the COE e.g. most DSA-tagged packets will 4668 * also have to be checksummed in software. 4669 */ 4670 if (csum_insertion && 4671 (priv->plat->tx_queues_cfg[queue].coe_unsupported || 4672 !stmmac_has_ip_ethertype(skb))) { 4673 if (unlikely(skb_checksum_help(skb))) 4674 goto dma_map_err; 4675 csum_insertion = !csum_insertion; 4676 } 4677 4678 if (likely(priv->extend_desc)) 4679 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4680 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4681 desc = &tx_q->dma_entx[entry].basic; 4682 else 4683 desc = tx_q->dma_tx + entry; 4684 4685 first = desc; 4686 4687 if (has_vlan) 4688 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); 4689 4690 enh_desc = priv->plat->enh_desc; 4691 /* To program the descriptors according to the size of the frame */ 4692 if (enh_desc) 4693 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); 4694 4695 if (unlikely(is_jumbo)) { 4696 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); 4697 if (unlikely(entry < 0) && (entry != -EINVAL)) 4698 goto dma_map_err; 4699 } 4700 4701 for (i = 0; i < nfrags; i++) { 4702 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 4703 int len = skb_frag_size(frag); 4704 bool last_segment = (i == (nfrags - 1)); 4705 4706 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4707 WARN_ON(tx_q->tx_skbuff[entry]); 4708 4709 if (likely(priv->extend_desc)) 4710 desc = (struct dma_desc *)(tx_q->dma_etx + entry); 4711 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4712 desc = &tx_q->dma_entx[entry].basic; 4713 else 4714 desc = tx_q->dma_tx + entry; 4715 4716 des = skb_frag_dma_map(priv->device, frag, 0, len, 4717 DMA_TO_DEVICE); 4718 if (dma_mapping_error(priv->device, des)) 4719 goto dma_map_err; /* should reuse desc w/o issues */ 4720 4721 tx_q->tx_skbuff_dma[entry].buf = des; 4722 4723 stmmac_set_desc_addr(priv, desc, des); 4724 4725 tx_q->tx_skbuff_dma[entry].map_as_page = true; 4726 tx_q->tx_skbuff_dma[entry].len = len; 4727 tx_q->tx_skbuff_dma[entry].last_segment = last_segment; 4728 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4729 4730 /* Prepare the descriptor and set the own bit too */ 4731 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, 4732 priv->mode, 1, last_segment, skb->len); 4733 } 4734 4735 /* Only the last descriptor gets to point to the skb. */ 4736 tx_q->tx_skbuff[entry] = skb; 4737 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; 4738 4739 /* According to the coalesce parameter the IC bit for the latest 4740 * segment is reset and the timer re-started to clean the tx status. 4741 * This approach takes care about the fragments: desc is the first 4742 * element in case of no SG. 4743 */ 4744 tx_packets = (entry + 1) - first_tx; 4745 tx_q->tx_count_frames += tx_packets; 4746 4747 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) 4748 set_ic = true; 4749 else if (!priv->tx_coal_frames[queue]) 4750 set_ic = false; 4751 else if (tx_packets > priv->tx_coal_frames[queue]) 4752 set_ic = true; 4753 else if ((tx_q->tx_count_frames % 4754 priv->tx_coal_frames[queue]) < tx_packets) 4755 set_ic = true; 4756 else 4757 set_ic = false; 4758 4759 if (set_ic) { 4760 if (likely(priv->extend_desc)) 4761 desc = &tx_q->dma_etx[entry].basic; 4762 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 4763 desc = &tx_q->dma_entx[entry].basic; 4764 else 4765 desc = &tx_q->dma_tx[entry]; 4766 4767 tx_q->tx_count_frames = 0; 4768 stmmac_set_tx_ic(priv, desc); 4769 } 4770 4771 /* We've used all descriptors we need for this skb, however, 4772 * advance cur_tx so that it references a fresh descriptor. 4773 * ndo_start_xmit will fill this descriptor the next time it's 4774 * called and stmmac_tx_clean may clean up to this descriptor. 4775 */ 4776 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 4777 tx_q->cur_tx = entry; 4778 4779 if (netif_msg_pktdata(priv)) { 4780 netdev_dbg(priv->dev, 4781 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", 4782 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, 4783 entry, first, nfrags); 4784 4785 netdev_dbg(priv->dev, ">>> frame to be transmitted: "); 4786 print_pkt(skb->data, skb->len); 4787 } 4788 4789 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { 4790 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", 4791 __func__); 4792 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); 4793 } 4794 4795 u64_stats_update_begin(&txq_stats->q_syncp); 4796 u64_stats_add(&txq_stats->q.tx_bytes, skb->len); 4797 if (set_ic) 4798 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 4799 u64_stats_update_end(&txq_stats->q_syncp); 4800 4801 if (priv->sarc_type) 4802 stmmac_set_desc_sarc(priv, first, priv->sarc_type); 4803 4804 /* Ready to fill the first descriptor and set the OWN bit w/o any 4805 * problems because all the descriptors are actually ready to be 4806 * passed to the DMA engine. 4807 */ 4808 if (likely(!is_jumbo)) { 4809 bool last_segment = (nfrags == 0); 4810 4811 des = dma_map_single(priv->device, skb->data, 4812 nopaged_len, DMA_TO_DEVICE); 4813 if (dma_mapping_error(priv->device, des)) 4814 goto dma_map_err; 4815 4816 tx_q->tx_skbuff_dma[first_entry].buf = des; 4817 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; 4818 tx_q->tx_skbuff_dma[first_entry].map_as_page = false; 4819 4820 stmmac_set_desc_addr(priv, first, des); 4821 4822 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; 4823 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; 4824 4825 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 4826 priv->hwts_tx_en)) { 4827 /* declare that device is doing timestamping */ 4828 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 4829 stmmac_enable_tx_timestamp(priv, first); 4830 } 4831 4832 /* Prepare the first descriptor setting the OWN bit too */ 4833 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, 4834 csum_insertion, priv->mode, 0, last_segment, 4835 skb->len); 4836 } 4837 4838 if (tx_q->tbs & STMMAC_TBS_EN) { 4839 struct timespec64 ts = ns_to_timespec64(skb->tstamp); 4840 4841 tbs_desc = &tx_q->dma_entx[first_entry]; 4842 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); 4843 } 4844 4845 stmmac_set_tx_owner(priv, first); 4846 4847 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); 4848 4849 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 4850 skb_tx_timestamp(skb); 4851 stmmac_flush_tx_descriptors(priv, queue); 4852 stmmac_tx_timer_arm(priv, queue); 4853 4854 return NETDEV_TX_OK; 4855 4856dma_map_err: 4857 netdev_err(priv->dev, "Tx DMA map failed\n"); 4858max_sdu_err: 4859 dev_kfree_skb(skb); 4860 priv->xstats.tx_dropped++; 4861 return NETDEV_TX_OK; 4862} 4863 4864static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) 4865{ 4866 struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); 4867 __be16 vlan_proto = veth->h_vlan_proto; 4868 u16 vlanid; 4869 4870 if ((vlan_proto == htons(ETH_P_8021Q) && 4871 dev->features & NETIF_F_HW_VLAN_CTAG_RX) || 4872 (vlan_proto == htons(ETH_P_8021AD) && 4873 dev->features & NETIF_F_HW_VLAN_STAG_RX)) { 4874 /* pop the vlan tag */ 4875 vlanid = ntohs(veth->h_vlan_TCI); 4876 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2); 4877 skb_pull(skb, VLAN_HLEN); 4878 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid); 4879 } 4880} 4881 4882/** 4883 * stmmac_rx_refill - refill used skb preallocated buffers 4884 * @priv: driver private structure 4885 * @queue: RX queue index 4886 * Description : this is to reallocate the skb for the reception process 4887 * that is based on zero-copy. 4888 */ 4889static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) 4890{ 4891 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 4892 int dirty = stmmac_rx_dirty(priv, queue); 4893 unsigned int entry = rx_q->dirty_rx; 4894 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); 4895 4896 if (priv->dma_cap.host_dma_width <= 32) 4897 gfp |= GFP_DMA32; 4898 4899 while (dirty-- > 0) { 4900 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 4901 struct dma_desc *p; 4902 bool use_rx_wd; 4903 4904 if (priv->extend_desc) 4905 p = (struct dma_desc *)(rx_q->dma_erx + entry); 4906 else 4907 p = rx_q->dma_rx + entry; 4908 4909 if (!buf->page) { 4910 buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4911 if (!buf->page) 4912 break; 4913 } 4914 4915 if (priv->sph_active && !buf->sec_page) { 4916 buf->sec_page = page_pool_alloc_pages(rx_q->page_pool, gfp); 4917 if (!buf->sec_page) 4918 break; 4919 4920 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page); 4921 } 4922 4923 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset; 4924 4925 stmmac_set_desc_addr(priv, p, buf->addr); 4926 if (priv->sph_active) 4927 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true); 4928 else 4929 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false); 4930 stmmac_refill_desc3(priv, rx_q, p); 4931 4932 rx_q->rx_count_frames++; 4933 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 4934 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 4935 rx_q->rx_count_frames = 0; 4936 4937 use_rx_wd = !priv->rx_coal_frames[queue]; 4938 use_rx_wd |= rx_q->rx_count_frames > 0; 4939 if (!priv->use_riwt) 4940 use_rx_wd = false; 4941 4942 dma_wmb(); 4943 stmmac_set_rx_owner(priv, p, use_rx_wd); 4944 4945 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 4946 } 4947 rx_q->dirty_rx = entry; 4948 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 4949 (rx_q->dirty_rx * sizeof(struct dma_desc)); 4950 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 4951 /* Wake up Rx DMA from the suspend state if required */ 4952 stmmac_enable_dma_reception(priv, priv->ioaddr, queue); 4953} 4954 4955static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv, 4956 struct dma_desc *p, 4957 int status, unsigned int len) 4958{ 4959 unsigned int plen = 0, hlen = 0; 4960 int coe = priv->hw->rx_csum; 4961 4962 /* Not first descriptor, buffer is always zero */ 4963 if (priv->sph_active && len) 4964 return 0; 4965 4966 /* First descriptor, get split header length */ 4967 stmmac_get_rx_header_len(priv, p, &hlen); 4968 if (priv->sph_active && hlen) { 4969 priv->xstats.rx_split_hdr_pkt_n++; 4970 return hlen; 4971 } 4972 4973 /* First descriptor, not last descriptor and not split header */ 4974 if (status & rx_not_ls) 4975 return priv->dma_conf.dma_buf_sz; 4976 4977 plen = stmmac_get_rx_frame_len(priv, p, coe); 4978 4979 /* First descriptor and last descriptor and not split header */ 4980 return min_t(unsigned int, priv->dma_conf.dma_buf_sz, plen); 4981} 4982 4983static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv, 4984 struct dma_desc *p, 4985 int status, unsigned int len) 4986{ 4987 int coe = priv->hw->rx_csum; 4988 unsigned int plen = 0; 4989 4990 /* Not split header, buffer is not available */ 4991 if (!priv->sph_active) 4992 return 0; 4993 4994 /* For GMAC4, when split header is enabled, in some rare cases, the 4995 * hardware does not fill buf2 of the first descriptor with payload. 4996 * Thus we cannot assume buf2 is always fully filled if it is not 4997 * the last descriptor. Otherwise, the length of buf2 of the second 4998 * descriptor will be calculated wrong and cause an oops. 4999 * 5000 * If this is the last descriptor, 'plen' is the length of the 5001 * received packet that was transferred to system memory. 5002 * Otherwise, it is the accumulated number of bytes that have been 5003 * transferred for the current packet. 5004 * 5005 * Thus 'plen - len' always gives the correct length of buf2. 5006 */ 5007 5008 /* Not GMAC4 and not last descriptor */ 5009 if (priv->plat->core_type != DWMAC_CORE_GMAC4 && (status & rx_not_ls)) 5010 return priv->dma_conf.dma_buf_sz; 5011 5012 /* GMAC4 or last descriptor */ 5013 plen = stmmac_get_rx_frame_len(priv, p, coe); 5014 5015 return plen - len; 5016} 5017 5018static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, 5019 struct xdp_frame *xdpf, bool dma_map) 5020{ 5021 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[queue]; 5022 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 5023 bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; 5024 unsigned int entry = tx_q->cur_tx; 5025 struct dma_desc *tx_desc; 5026 dma_addr_t dma_addr; 5027 bool set_ic; 5028 5029 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv)) 5030 return STMMAC_XDP_CONSUMED; 5031 5032 if (priv->est && priv->est->enable && 5033 priv->est->max_sdu[queue] && 5034 xdpf->len > priv->est->max_sdu[queue]) { 5035 priv->xstats.max_sdu_txq_drop[queue]++; 5036 return STMMAC_XDP_CONSUMED; 5037 } 5038 5039 if (likely(priv->extend_desc)) 5040 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); 5041 else if (tx_q->tbs & STMMAC_TBS_AVAIL) 5042 tx_desc = &tx_q->dma_entx[entry].basic; 5043 else 5044 tx_desc = tx_q->dma_tx + entry; 5045 5046 if (dma_map) { 5047 dma_addr = dma_map_single(priv->device, xdpf->data, 5048 xdpf->len, DMA_TO_DEVICE); 5049 if (dma_mapping_error(priv->device, dma_addr)) 5050 return STMMAC_XDP_CONSUMED; 5051 5052 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; 5053 } else { 5054 struct page *page = virt_to_page(xdpf->data); 5055 5056 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) + 5057 xdpf->headroom; 5058 dma_sync_single_for_device(priv->device, dma_addr, 5059 xdpf->len, DMA_BIDIRECTIONAL); 5060 5061 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; 5062 } 5063 5064 tx_q->tx_skbuff_dma[entry].buf = dma_addr; 5065 tx_q->tx_skbuff_dma[entry].map_as_page = false; 5066 tx_q->tx_skbuff_dma[entry].len = xdpf->len; 5067 tx_q->tx_skbuff_dma[entry].last_segment = true; 5068 tx_q->tx_skbuff_dma[entry].is_jumbo = false; 5069 5070 tx_q->xdpf[entry] = xdpf; 5071 5072 stmmac_set_desc_addr(priv, tx_desc, dma_addr); 5073 5074 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, 5075 csum, priv->mode, true, true, 5076 xdpf->len); 5077 5078 tx_q->tx_count_frames++; 5079 5080 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0) 5081 set_ic = true; 5082 else 5083 set_ic = false; 5084 5085 if (set_ic) { 5086 tx_q->tx_count_frames = 0; 5087 stmmac_set_tx_ic(priv, tx_desc); 5088 u64_stats_update_begin(&txq_stats->q_syncp); 5089 u64_stats_inc(&txq_stats->q.tx_set_ic_bit); 5090 u64_stats_update_end(&txq_stats->q_syncp); 5091 } 5092 5093 stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); 5094 5095 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); 5096 tx_q->cur_tx = entry; 5097 5098 return STMMAC_XDP_TX; 5099} 5100 5101static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv, 5102 int cpu) 5103{ 5104 int index = cpu; 5105 5106 if (unlikely(index < 0)) 5107 index = 0; 5108 5109 while (index >= priv->plat->tx_queues_to_use) 5110 index -= priv->plat->tx_queues_to_use; 5111 5112 return index; 5113} 5114 5115static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, 5116 struct xdp_buff *xdp) 5117{ 5118 bool zc = !!(xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL); 5119 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); 5120 int cpu = smp_processor_id(); 5121 struct netdev_queue *nq; 5122 int queue; 5123 int res; 5124 5125 if (unlikely(!xdpf)) 5126 return STMMAC_XDP_CONSUMED; 5127 5128 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5129 nq = netdev_get_tx_queue(priv->dev, queue); 5130 5131 __netif_tx_lock(nq, cpu); 5132 /* Avoids TX time-out as we are sharing with slow path */ 5133 txq_trans_cond_update(nq); 5134 5135 /* For zero copy XDP_TX action, dma_map is true */ 5136 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, zc); 5137 if (res == STMMAC_XDP_TX) { 5138 stmmac_flush_tx_descriptors(priv, queue); 5139 } else if (res == STMMAC_XDP_CONSUMED && zc) { 5140 /* xdp has been freed by xdp_convert_buff_to_frame(), 5141 * no need to call xsk_buff_free() again, so return 5142 * STMMAC_XSK_CONSUMED. 5143 */ 5144 res = STMMAC_XSK_CONSUMED; 5145 xdp_return_frame(xdpf); 5146 } 5147 5148 __netif_tx_unlock(nq); 5149 5150 return res; 5151} 5152 5153static int __stmmac_xdp_run_prog(struct stmmac_priv *priv, 5154 struct bpf_prog *prog, 5155 struct xdp_buff *xdp) 5156{ 5157 u32 act; 5158 int res; 5159 5160 act = bpf_prog_run_xdp(prog, xdp); 5161 switch (act) { 5162 case XDP_PASS: 5163 res = STMMAC_XDP_PASS; 5164 break; 5165 case XDP_TX: 5166 res = stmmac_xdp_xmit_back(priv, xdp); 5167 break; 5168 case XDP_REDIRECT: 5169 if (xdp_do_redirect(priv->dev, xdp, prog) < 0) 5170 res = STMMAC_XDP_CONSUMED; 5171 else 5172 res = STMMAC_XDP_REDIRECT; 5173 break; 5174 default: 5175 bpf_warn_invalid_xdp_action(priv->dev, prog, act); 5176 fallthrough; 5177 case XDP_ABORTED: 5178 trace_xdp_exception(priv->dev, prog, act); 5179 fallthrough; 5180 case XDP_DROP: 5181 res = STMMAC_XDP_CONSUMED; 5182 break; 5183 } 5184 5185 return res; 5186} 5187 5188static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv, 5189 struct xdp_buff *xdp) 5190{ 5191 struct bpf_prog *prog; 5192 int res; 5193 5194 prog = READ_ONCE(priv->xdp_prog); 5195 if (!prog) { 5196 res = STMMAC_XDP_PASS; 5197 goto out; 5198 } 5199 5200 res = __stmmac_xdp_run_prog(priv, prog, xdp); 5201out: 5202 return ERR_PTR(-res); 5203} 5204 5205static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv, 5206 int xdp_status) 5207{ 5208 int cpu = smp_processor_id(); 5209 int queue; 5210 5211 queue = stmmac_xdp_get_tx_queue(priv, cpu); 5212 5213 if (xdp_status & STMMAC_XDP_TX) 5214 stmmac_tx_timer_arm(priv, queue); 5215 5216 if (xdp_status & STMMAC_XDP_REDIRECT) 5217 xdp_do_flush(); 5218} 5219 5220static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch, 5221 struct xdp_buff *xdp) 5222{ 5223 unsigned int metasize = xdp->data - xdp->data_meta; 5224 unsigned int datasize = xdp->data_end - xdp->data; 5225 struct sk_buff *skb; 5226 5227 skb = napi_alloc_skb(&ch->rxtx_napi, 5228 xdp->data_end - xdp->data_hard_start); 5229 if (unlikely(!skb)) 5230 return NULL; 5231 5232 skb_reserve(skb, xdp->data - xdp->data_hard_start); 5233 memcpy(__skb_put(skb, datasize), xdp->data, datasize); 5234 if (metasize) 5235 skb_metadata_set(skb, metasize); 5236 5237 return skb; 5238} 5239 5240static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue, 5241 struct dma_desc *p, struct dma_desc *np, 5242 struct xdp_buff *xdp) 5243{ 5244 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5245 struct stmmac_channel *ch = &priv->channel[queue]; 5246 unsigned int len = xdp->data_end - xdp->data; 5247 enum pkt_hash_types hash_type; 5248 int coe = priv->hw->rx_csum; 5249 struct sk_buff *skb; 5250 u32 hash; 5251 5252 skb = stmmac_construct_skb_zc(ch, xdp); 5253 if (!skb) { 5254 priv->xstats.rx_dropped++; 5255 return; 5256 } 5257 5258 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5259 if (priv->hw->hw_vlan_en) 5260 /* MAC level stripping. */ 5261 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5262 else 5263 /* Driver level stripping. */ 5264 stmmac_rx_vlan(priv->dev, skb); 5265 skb->protocol = eth_type_trans(skb, priv->dev); 5266 5267 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb)) 5268 skb_checksum_none_assert(skb); 5269 else 5270 skb->ip_summed = CHECKSUM_UNNECESSARY; 5271 5272 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5273 skb_set_hash(skb, hash, hash_type); 5274 5275 skb_record_rx_queue(skb, queue); 5276 napi_gro_receive(&ch->rxtx_napi, skb); 5277 5278 u64_stats_update_begin(&rxq_stats->napi_syncp); 5279 u64_stats_inc(&rxq_stats->napi.rx_pkt_n); 5280 u64_stats_add(&rxq_stats->napi.rx_bytes, len); 5281 u64_stats_update_end(&rxq_stats->napi_syncp); 5282} 5283 5284static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) 5285{ 5286 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5287 unsigned int entry = rx_q->dirty_rx; 5288 struct dma_desc *rx_desc = NULL; 5289 bool ret = true; 5290 5291 budget = min(budget, stmmac_rx_dirty(priv, queue)); 5292 5293 while (budget-- > 0 && entry != rx_q->cur_rx) { 5294 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry]; 5295 dma_addr_t dma_addr; 5296 bool use_rx_wd; 5297 5298 if (!buf->xdp) { 5299 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool); 5300 if (!buf->xdp) { 5301 ret = false; 5302 break; 5303 } 5304 } 5305 5306 if (priv->extend_desc) 5307 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); 5308 else 5309 rx_desc = rx_q->dma_rx + entry; 5310 5311 dma_addr = xsk_buff_xdp_get_dma(buf->xdp); 5312 stmmac_set_desc_addr(priv, rx_desc, dma_addr); 5313 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false); 5314 stmmac_refill_desc3(priv, rx_q, rx_desc); 5315 5316 rx_q->rx_count_frames++; 5317 rx_q->rx_count_frames += priv->rx_coal_frames[queue]; 5318 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue]) 5319 rx_q->rx_count_frames = 0; 5320 5321 use_rx_wd = !priv->rx_coal_frames[queue]; 5322 use_rx_wd |= rx_q->rx_count_frames > 0; 5323 if (!priv->use_riwt) 5324 use_rx_wd = false; 5325 5326 dma_wmb(); 5327 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); 5328 5329 entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); 5330 } 5331 5332 if (rx_desc) { 5333 rx_q->dirty_rx = entry; 5334 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 5335 (rx_q->dirty_rx * sizeof(struct dma_desc)); 5336 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); 5337 } 5338 5339 return ret; 5340} 5341 5342static struct stmmac_xdp_buff *xsk_buff_to_stmmac_ctx(struct xdp_buff *xdp) 5343{ 5344 /* In XDP zero copy data path, xdp field in struct xdp_buff_xsk is used 5345 * to represent incoming packet, whereas cb field in the same structure 5346 * is used to store driver specific info. Thus, struct stmmac_xdp_buff 5347 * is laid on top of xdp and cb fields of struct xdp_buff_xsk. 5348 */ 5349 return (struct stmmac_xdp_buff *)xdp; 5350} 5351 5352static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) 5353{ 5354 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5355 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5356 unsigned int count = 0, error = 0, len = 0; 5357 int dirty = stmmac_rx_dirty(priv, queue); 5358 unsigned int next_entry = rx_q->cur_rx; 5359 u32 rx_errors = 0, rx_dropped = 0; 5360 unsigned int desc_size; 5361 struct bpf_prog *prog; 5362 bool failure = false; 5363 int xdp_status = 0; 5364 int status = 0; 5365 5366 if (netif_msg_rx_status(priv)) { 5367 void *rx_head; 5368 5369 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5370 if (priv->extend_desc) { 5371 rx_head = (void *)rx_q->dma_erx; 5372 desc_size = sizeof(struct dma_extended_desc); 5373 } else { 5374 rx_head = (void *)rx_q->dma_rx; 5375 desc_size = sizeof(struct dma_desc); 5376 } 5377 5378 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5379 rx_q->dma_rx_phy, desc_size); 5380 } 5381 while (count < limit) { 5382 struct stmmac_rx_buffer *buf; 5383 struct stmmac_xdp_buff *ctx; 5384 unsigned int buf1_len = 0; 5385 struct dma_desc *np, *p; 5386 int entry; 5387 int res; 5388 5389 if (!count && rx_q->state_saved) { 5390 error = rx_q->state.error; 5391 len = rx_q->state.len; 5392 } else { 5393 rx_q->state_saved = false; 5394 error = 0; 5395 len = 0; 5396 } 5397 5398read_again: 5399 if (count >= limit) 5400 break; 5401 5402 buf1_len = 0; 5403 entry = next_entry; 5404 buf = &rx_q->buf_pool[entry]; 5405 5406 if (dirty >= STMMAC_RX_FILL_BATCH) { 5407 failure = failure || 5408 !stmmac_rx_refill_zc(priv, queue, dirty); 5409 dirty = 0; 5410 } 5411 5412 if (priv->extend_desc) 5413 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5414 else 5415 p = rx_q->dma_rx + entry; 5416 5417 /* read the status of the incoming frame */ 5418 status = stmmac_rx_status(priv, &priv->xstats, p); 5419 /* check if managed by the DMA otherwise go ahead */ 5420 if (unlikely(status & dma_own)) 5421 break; 5422 5423 /* Prefetch the next RX descriptor */ 5424 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5425 priv->dma_conf.dma_rx_size); 5426 next_entry = rx_q->cur_rx; 5427 5428 if (priv->extend_desc) 5429 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5430 else 5431 np = rx_q->dma_rx + next_entry; 5432 5433 prefetch(np); 5434 5435 /* Ensure a valid XSK buffer before proceed */ 5436 if (!buf->xdp) 5437 break; 5438 5439 if (priv->extend_desc) 5440 stmmac_rx_extended_status(priv, &priv->xstats, 5441 rx_q->dma_erx + entry); 5442 if (unlikely(status == discard_frame)) { 5443 xsk_buff_free(buf->xdp); 5444 buf->xdp = NULL; 5445 dirty++; 5446 error = 1; 5447 if (!priv->hwts_rx_en) 5448 rx_errors++; 5449 } 5450 5451 if (unlikely(error && (status & rx_not_ls))) 5452 goto read_again; 5453 if (unlikely(error)) { 5454 count++; 5455 continue; 5456 } 5457 5458 /* XSK pool expects RX frame 1:1 mapped to XSK buffer */ 5459 if (likely(status & rx_not_ls)) { 5460 xsk_buff_free(buf->xdp); 5461 buf->xdp = NULL; 5462 dirty++; 5463 count++; 5464 goto read_again; 5465 } 5466 5467 ctx = xsk_buff_to_stmmac_ctx(buf->xdp); 5468 ctx->priv = priv; 5469 ctx->desc = p; 5470 ctx->ndesc = np; 5471 5472 /* XDP ZC Frame only support primary buffers for now */ 5473 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5474 len += buf1_len; 5475 5476 /* ACS is disabled; strip manually. */ 5477 if (likely(!(status & rx_not_ls))) { 5478 buf1_len -= ETH_FCS_LEN; 5479 len -= ETH_FCS_LEN; 5480 } 5481 5482 /* RX buffer is good and fit into a XSK pool buffer */ 5483 buf->xdp->data_end = buf->xdp->data + buf1_len; 5484 xsk_buff_dma_sync_for_cpu(buf->xdp); 5485 5486 prog = READ_ONCE(priv->xdp_prog); 5487 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp); 5488 5489 switch (res) { 5490 case STMMAC_XDP_PASS: 5491 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp); 5492 xsk_buff_free(buf->xdp); 5493 break; 5494 case STMMAC_XDP_CONSUMED: 5495 xsk_buff_free(buf->xdp); 5496 fallthrough; 5497 case STMMAC_XSK_CONSUMED: 5498 rx_dropped++; 5499 break; 5500 case STMMAC_XDP_TX: 5501 case STMMAC_XDP_REDIRECT: 5502 xdp_status |= res; 5503 break; 5504 } 5505 5506 buf->xdp = NULL; 5507 dirty++; 5508 count++; 5509 } 5510 5511 if (status & rx_not_ls) { 5512 rx_q->state_saved = true; 5513 rx_q->state.error = error; 5514 rx_q->state.len = len; 5515 } 5516 5517 stmmac_finalize_xdp_rx(priv, xdp_status); 5518 5519 u64_stats_update_begin(&rxq_stats->napi_syncp); 5520 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5521 u64_stats_update_end(&rxq_stats->napi_syncp); 5522 5523 priv->xstats.rx_dropped += rx_dropped; 5524 priv->xstats.rx_errors += rx_errors; 5525 5526 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) { 5527 if (failure || stmmac_rx_dirty(priv, queue) > 0) 5528 xsk_set_rx_need_wakeup(rx_q->xsk_pool); 5529 else 5530 xsk_clear_rx_need_wakeup(rx_q->xsk_pool); 5531 5532 return (int)count; 5533 } 5534 5535 return failure ? limit : (int)count; 5536} 5537 5538/** 5539 * stmmac_rx - manage the receive process 5540 * @priv: driver private structure 5541 * @limit: napi bugget 5542 * @queue: RX queue index. 5543 * Description : this the function called by the napi poll method. 5544 * It gets all the frames inside the ring. 5545 */ 5546static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) 5547{ 5548 u32 rx_errors = 0, rx_dropped = 0, rx_bytes = 0, rx_packets = 0; 5549 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[queue]; 5550 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 5551 struct stmmac_channel *ch = &priv->channel[queue]; 5552 unsigned int count = 0, error = 0, len = 0; 5553 int status = 0, coe = priv->hw->rx_csum; 5554 unsigned int next_entry = rx_q->cur_rx; 5555 enum dma_data_direction dma_dir; 5556 unsigned int desc_size; 5557 struct sk_buff *skb = NULL; 5558 struct stmmac_xdp_buff ctx; 5559 int xdp_status = 0; 5560 int bufsz; 5561 5562 dma_dir = page_pool_get_dma_dir(rx_q->page_pool); 5563 bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; 5564 limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); 5565 5566 if (netif_msg_rx_status(priv)) { 5567 void *rx_head; 5568 5569 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); 5570 if (priv->extend_desc) { 5571 rx_head = (void *)rx_q->dma_erx; 5572 desc_size = sizeof(struct dma_extended_desc); 5573 } else { 5574 rx_head = (void *)rx_q->dma_rx; 5575 desc_size = sizeof(struct dma_desc); 5576 } 5577 5578 stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, 5579 rx_q->dma_rx_phy, desc_size); 5580 } 5581 while (count < limit) { 5582 unsigned int buf1_len = 0, buf2_len = 0; 5583 enum pkt_hash_types hash_type; 5584 struct stmmac_rx_buffer *buf; 5585 struct dma_desc *np, *p; 5586 int entry; 5587 u32 hash; 5588 5589 if (!count && rx_q->state_saved) { 5590 skb = rx_q->state.skb; 5591 error = rx_q->state.error; 5592 len = rx_q->state.len; 5593 } else { 5594 rx_q->state_saved = false; 5595 skb = NULL; 5596 error = 0; 5597 len = 0; 5598 } 5599 5600read_again: 5601 if (count >= limit) 5602 break; 5603 5604 buf1_len = 0; 5605 buf2_len = 0; 5606 entry = next_entry; 5607 buf = &rx_q->buf_pool[entry]; 5608 5609 if (priv->extend_desc) 5610 p = (struct dma_desc *)(rx_q->dma_erx + entry); 5611 else 5612 p = rx_q->dma_rx + entry; 5613 5614 /* read the status of the incoming frame */ 5615 status = stmmac_rx_status(priv, &priv->xstats, p); 5616 /* check if managed by the DMA otherwise go ahead */ 5617 if (unlikely(status & dma_own)) 5618 break; 5619 5620 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, 5621 priv->dma_conf.dma_rx_size); 5622 next_entry = rx_q->cur_rx; 5623 5624 if (priv->extend_desc) 5625 np = (struct dma_desc *)(rx_q->dma_erx + next_entry); 5626 else 5627 np = rx_q->dma_rx + next_entry; 5628 5629 prefetch(np); 5630 5631 if (priv->extend_desc) 5632 stmmac_rx_extended_status(priv, &priv->xstats, rx_q->dma_erx + entry); 5633 if (unlikely(status == discard_frame)) { 5634 page_pool_put_page(rx_q->page_pool, buf->page, 0, true); 5635 buf->page = NULL; 5636 error = 1; 5637 if (!priv->hwts_rx_en) 5638 rx_errors++; 5639 } 5640 5641 if (unlikely(error && (status & rx_not_ls))) 5642 goto read_again; 5643 if (unlikely(error)) { 5644 dev_kfree_skb(skb); 5645 skb = NULL; 5646 count++; 5647 continue; 5648 } 5649 5650 /* Buffer is good. Go on. */ 5651 5652 buf1_len = stmmac_rx_buf1_len(priv, p, status, len); 5653 len += buf1_len; 5654 buf2_len = stmmac_rx_buf2_len(priv, p, status, len); 5655 len += buf2_len; 5656 5657 /* ACS is disabled; strip manually. */ 5658 if (likely(!(status & rx_not_ls))) { 5659 if (buf2_len) { 5660 buf2_len -= ETH_FCS_LEN; 5661 len -= ETH_FCS_LEN; 5662 } else if (buf1_len) { 5663 buf1_len -= ETH_FCS_LEN; 5664 len -= ETH_FCS_LEN; 5665 } 5666 } 5667 5668 if (!skb) { 5669 unsigned int pre_len, sync_len; 5670 5671 dma_sync_single_for_cpu(priv->device, buf->addr, 5672 buf1_len, dma_dir); 5673 net_prefetch(page_address(buf->page) + 5674 buf->page_offset); 5675 5676 xdp_init_buff(&ctx.xdp, bufsz, &rx_q->xdp_rxq); 5677 xdp_prepare_buff(&ctx.xdp, page_address(buf->page), 5678 buf->page_offset, buf1_len, true); 5679 5680 pre_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5681 buf->page_offset; 5682 5683 ctx.priv = priv; 5684 ctx.desc = p; 5685 ctx.ndesc = np; 5686 5687 skb = stmmac_xdp_run_prog(priv, &ctx.xdp); 5688 /* Due xdp_adjust_tail: DMA sync for_device 5689 * cover max len CPU touch 5690 */ 5691 sync_len = ctx.xdp.data_end - ctx.xdp.data_hard_start - 5692 buf->page_offset; 5693 sync_len = max(sync_len, pre_len); 5694 5695 /* For Not XDP_PASS verdict */ 5696 if (IS_ERR(skb)) { 5697 unsigned int xdp_res = -PTR_ERR(skb); 5698 5699 if (xdp_res & STMMAC_XDP_CONSUMED) { 5700 page_pool_put_page(rx_q->page_pool, 5701 virt_to_head_page(ctx.xdp.data), 5702 sync_len, true); 5703 buf->page = NULL; 5704 rx_dropped++; 5705 5706 /* Clear skb as it was set as 5707 * status by XDP program. 5708 */ 5709 skb = NULL; 5710 5711 if (unlikely((status & rx_not_ls))) 5712 goto read_again; 5713 5714 count++; 5715 continue; 5716 } else if (xdp_res & (STMMAC_XDP_TX | 5717 STMMAC_XDP_REDIRECT)) { 5718 xdp_status |= xdp_res; 5719 buf->page = NULL; 5720 skb = NULL; 5721 count++; 5722 continue; 5723 } 5724 } 5725 } 5726 5727 if (!skb) { 5728 unsigned int head_pad_len; 5729 5730 /* XDP program may expand or reduce tail */ 5731 buf1_len = ctx.xdp.data_end - ctx.xdp.data; 5732 5733 skb = napi_build_skb(page_address(buf->page), 5734 rx_q->napi_skb_frag_size); 5735 if (!skb) { 5736 page_pool_recycle_direct(rx_q->page_pool, 5737 buf->page); 5738 rx_dropped++; 5739 count++; 5740 goto drain_data; 5741 } 5742 5743 /* XDP program may adjust header */ 5744 head_pad_len = ctx.xdp.data - ctx.xdp.data_hard_start; 5745 skb_reserve(skb, head_pad_len); 5746 skb_put(skb, buf1_len); 5747 skb_mark_for_recycle(skb); 5748 buf->page = NULL; 5749 } else if (buf1_len) { 5750 dma_sync_single_for_cpu(priv->device, buf->addr, 5751 buf1_len, dma_dir); 5752 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5753 buf->page, buf->page_offset, buf1_len, 5754 priv->dma_conf.dma_buf_sz); 5755 buf->page = NULL; 5756 } 5757 5758 if (buf2_len) { 5759 dma_sync_single_for_cpu(priv->device, buf->sec_addr, 5760 buf2_len, dma_dir); 5761 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 5762 buf->sec_page, 0, buf2_len, 5763 priv->dma_conf.dma_buf_sz); 5764 buf->sec_page = NULL; 5765 } 5766 5767drain_data: 5768 if (likely(status & rx_not_ls)) 5769 goto read_again; 5770 if (!skb) 5771 continue; 5772 5773 /* Got entire packet into SKB. Finish it. */ 5774 5775 stmmac_get_rx_hwtstamp(priv, p, np, skb); 5776 5777 if (priv->hw->hw_vlan_en) 5778 /* MAC level stripping. */ 5779 stmmac_rx_hw_vlan(priv, priv->hw, p, skb); 5780 else 5781 /* Driver level stripping. */ 5782 stmmac_rx_vlan(priv->dev, skb); 5783 5784 skb->protocol = eth_type_trans(skb, priv->dev); 5785 5786 if (unlikely(!coe) || !stmmac_has_ip_ethertype(skb) || 5787 (status & csum_none)) 5788 skb_checksum_none_assert(skb); 5789 else 5790 skb->ip_summed = CHECKSUM_UNNECESSARY; 5791 5792 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type)) 5793 skb_set_hash(skb, hash, hash_type); 5794 5795 skb_record_rx_queue(skb, queue); 5796 napi_gro_receive(&ch->rx_napi, skb); 5797 skb = NULL; 5798 5799 rx_packets++; 5800 rx_bytes += len; 5801 count++; 5802 } 5803 5804 if (status & rx_not_ls || skb) { 5805 rx_q->state_saved = true; 5806 rx_q->state.skb = skb; 5807 rx_q->state.error = error; 5808 rx_q->state.len = len; 5809 } 5810 5811 stmmac_finalize_xdp_rx(priv, xdp_status); 5812 5813 stmmac_rx_refill(priv, queue); 5814 5815 u64_stats_update_begin(&rxq_stats->napi_syncp); 5816 u64_stats_add(&rxq_stats->napi.rx_packets, rx_packets); 5817 u64_stats_add(&rxq_stats->napi.rx_bytes, rx_bytes); 5818 u64_stats_add(&rxq_stats->napi.rx_pkt_n, count); 5819 u64_stats_update_end(&rxq_stats->napi_syncp); 5820 5821 priv->xstats.rx_dropped += rx_dropped; 5822 priv->xstats.rx_errors += rx_errors; 5823 5824 return count; 5825} 5826 5827static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget) 5828{ 5829 struct stmmac_channel *ch = 5830 container_of(napi, struct stmmac_channel, rx_napi); 5831 struct stmmac_priv *priv = ch->priv_data; 5832 struct stmmac_rxq_stats *rxq_stats; 5833 u32 chan = ch->index; 5834 int work_done; 5835 5836 rxq_stats = &priv->xstats.rxq_stats[chan]; 5837 u64_stats_update_begin(&rxq_stats->napi_syncp); 5838 u64_stats_inc(&rxq_stats->napi.poll); 5839 u64_stats_update_end(&rxq_stats->napi_syncp); 5840 5841 work_done = stmmac_rx(priv, budget, chan); 5842 if (work_done < budget && napi_complete_done(napi, work_done)) { 5843 unsigned long flags; 5844 5845 spin_lock_irqsave(&ch->lock, flags); 5846 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0); 5847 spin_unlock_irqrestore(&ch->lock, flags); 5848 } 5849 5850 return work_done; 5851} 5852 5853static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget) 5854{ 5855 struct stmmac_channel *ch = 5856 container_of(napi, struct stmmac_channel, tx_napi); 5857 struct stmmac_priv *priv = ch->priv_data; 5858 struct stmmac_txq_stats *txq_stats; 5859 bool pending_packets = false; 5860 u32 chan = ch->index; 5861 int work_done; 5862 5863 txq_stats = &priv->xstats.txq_stats[chan]; 5864 u64_stats_update_begin(&txq_stats->napi_syncp); 5865 u64_stats_inc(&txq_stats->napi.poll); 5866 u64_stats_update_end(&txq_stats->napi_syncp); 5867 5868 work_done = stmmac_tx_clean(priv, budget, chan, &pending_packets); 5869 work_done = min(work_done, budget); 5870 5871 if (work_done < budget && napi_complete_done(napi, work_done)) { 5872 unsigned long flags; 5873 5874 spin_lock_irqsave(&ch->lock, flags); 5875 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1); 5876 spin_unlock_irqrestore(&ch->lock, flags); 5877 } 5878 5879 /* TX still have packet to handle, check if we need to arm tx timer */ 5880 if (pending_packets) 5881 stmmac_tx_timer_arm(priv, chan); 5882 5883 return work_done; 5884} 5885 5886static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget) 5887{ 5888 struct stmmac_channel *ch = 5889 container_of(napi, struct stmmac_channel, rxtx_napi); 5890 struct stmmac_priv *priv = ch->priv_data; 5891 bool tx_pending_packets = false; 5892 int rx_done, tx_done, rxtx_done; 5893 struct stmmac_rxq_stats *rxq_stats; 5894 struct stmmac_txq_stats *txq_stats; 5895 u32 chan = ch->index; 5896 5897 rxq_stats = &priv->xstats.rxq_stats[chan]; 5898 u64_stats_update_begin(&rxq_stats->napi_syncp); 5899 u64_stats_inc(&rxq_stats->napi.poll); 5900 u64_stats_update_end(&rxq_stats->napi_syncp); 5901 5902 txq_stats = &priv->xstats.txq_stats[chan]; 5903 u64_stats_update_begin(&txq_stats->napi_syncp); 5904 u64_stats_inc(&txq_stats->napi.poll); 5905 u64_stats_update_end(&txq_stats->napi_syncp); 5906 5907 tx_done = stmmac_tx_clean(priv, budget, chan, &tx_pending_packets); 5908 tx_done = min(tx_done, budget); 5909 5910 rx_done = stmmac_rx_zc(priv, budget, chan); 5911 5912 rxtx_done = max(tx_done, rx_done); 5913 5914 /* If either TX or RX work is not complete, return budget 5915 * and keep pooling 5916 */ 5917 if (rxtx_done >= budget) 5918 return budget; 5919 5920 /* all work done, exit the polling mode */ 5921 if (napi_complete_done(napi, rxtx_done)) { 5922 unsigned long flags; 5923 5924 spin_lock_irqsave(&ch->lock, flags); 5925 /* Both RX and TX work done are compelte, 5926 * so enable both RX & TX IRQs. 5927 */ 5928 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 5929 spin_unlock_irqrestore(&ch->lock, flags); 5930 } 5931 5932 /* TX still have packet to handle, check if we need to arm tx timer */ 5933 if (tx_pending_packets) 5934 stmmac_tx_timer_arm(priv, chan); 5935 5936 return min(rxtx_done, budget - 1); 5937} 5938 5939/** 5940 * stmmac_tx_timeout 5941 * @dev : Pointer to net device structure 5942 * @txqueue: the index of the hanging transmit queue 5943 * Description: this function is called when a packet transmission fails to 5944 * complete within a reasonable time. The driver will mark the error in the 5945 * netdev structure and arrange for the device to be reset to a sane state 5946 * in order to transmit a new packet. 5947 */ 5948static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue) 5949{ 5950 struct stmmac_priv *priv = netdev_priv(dev); 5951 5952 stmmac_global_err(priv); 5953} 5954 5955/** 5956 * stmmac_set_rx_mode - entry point for multicast addressing 5957 * @dev : pointer to the device structure 5958 * Description: 5959 * This function is a driver entry point which gets called by the kernel 5960 * whenever multicast addresses must be enabled/disabled. 5961 * Return value: 5962 * void. 5963 * 5964 * FIXME: This may need RXC to be running, but it may be called with BH 5965 * disabled, which means we can't call phylink_rx_clk_stop*(). 5966 */ 5967static void stmmac_set_rx_mode(struct net_device *dev) 5968{ 5969 struct stmmac_priv *priv = netdev_priv(dev); 5970 5971 stmmac_set_filter(priv, priv->hw, dev); 5972} 5973 5974/** 5975 * stmmac_change_mtu - entry point to change MTU size for the device. 5976 * @dev : device pointer. 5977 * @new_mtu : the new MTU size for the device. 5978 * Description: the Maximum Transfer Unit (MTU) is used by the network layer 5979 * to drive packet transmission. Ethernet has an MTU of 1500 octets 5980 * (ETH_DATA_LEN). This value can be changed with ifconfig. 5981 * Return value: 5982 * 0 on success and an appropriate (-)ve integer as defined in errno.h 5983 * file on failure. 5984 */ 5985static int stmmac_change_mtu(struct net_device *dev, int new_mtu) 5986{ 5987 struct stmmac_priv *priv = netdev_priv(dev); 5988 int txfifosz = priv->plat->tx_fifo_size; 5989 struct stmmac_dma_conf *dma_conf; 5990 const int mtu = new_mtu; 5991 int ret; 5992 5993 if (txfifosz == 0) 5994 txfifosz = priv->dma_cap.tx_fifo_size; 5995 5996 txfifosz /= priv->plat->tx_queues_to_use; 5997 5998 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) { 5999 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n"); 6000 return -EINVAL; 6001 } 6002 6003 new_mtu = STMMAC_ALIGN(new_mtu); 6004 6005 /* If condition true, FIFO is too small or MTU too large */ 6006 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB)) 6007 return -EINVAL; 6008 6009 if (netif_running(dev)) { 6010 netdev_dbg(priv->dev, "restarting interface to change its MTU\n"); 6011 /* Try to allocate the new DMA conf with the new mtu */ 6012 dma_conf = stmmac_setup_dma_desc(priv, mtu); 6013 if (IS_ERR(dma_conf)) { 6014 netdev_err(priv->dev, "failed allocating new dma conf for new MTU %d\n", 6015 mtu); 6016 return PTR_ERR(dma_conf); 6017 } 6018 6019 __stmmac_release(dev); 6020 6021 ret = __stmmac_open(dev, dma_conf); 6022 if (ret) { 6023 free_dma_desc_resources(priv, dma_conf); 6024 kfree(dma_conf); 6025 netdev_err(priv->dev, "failed reopening the interface after MTU change\n"); 6026 return ret; 6027 } 6028 6029 kfree(dma_conf); 6030 6031 stmmac_set_rx_mode(dev); 6032 } 6033 6034 WRITE_ONCE(dev->mtu, mtu); 6035 netdev_update_features(dev); 6036 6037 return 0; 6038} 6039 6040static netdev_features_t stmmac_fix_features(struct net_device *dev, 6041 netdev_features_t features) 6042{ 6043 struct stmmac_priv *priv = netdev_priv(dev); 6044 6045 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE) 6046 features &= ~NETIF_F_RXCSUM; 6047 6048 if (!priv->plat->tx_coe) 6049 features &= ~NETIF_F_CSUM_MASK; 6050 6051 /* Some GMAC devices have a bugged Jumbo frame support that 6052 * needs to have the Tx COE disabled for oversized frames 6053 * (due to limited buffer sizes). In this case we disable 6054 * the TX csum insertion in the TDES and not use SF. 6055 */ 6056 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) 6057 features &= ~NETIF_F_CSUM_MASK; 6058 6059 /* Disable tso if asked by ethtool */ 6060 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 6061 if (features & NETIF_F_TSO) 6062 priv->tso = true; 6063 else 6064 priv->tso = false; 6065 } 6066 6067 return features; 6068} 6069 6070static int stmmac_set_features(struct net_device *netdev, 6071 netdev_features_t features) 6072{ 6073 struct stmmac_priv *priv = netdev_priv(netdev); 6074 6075 /* Keep the COE Type in case of csum is supporting */ 6076 if (features & NETIF_F_RXCSUM) 6077 priv->hw->rx_csum = priv->plat->rx_coe; 6078 else 6079 priv->hw->rx_csum = 0; 6080 /* No check needed because rx_coe has been set before and it will be 6081 * fixed in case of issue. 6082 */ 6083 stmmac_rx_ipc(priv, priv->hw); 6084 6085 if (priv->sph_capable) { 6086 bool sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 6087 u32 chan; 6088 6089 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) 6090 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 6091 } 6092 6093 if (features & NETIF_F_HW_VLAN_CTAG_RX) 6094 priv->hw->hw_vlan_en = true; 6095 else 6096 priv->hw->hw_vlan_en = false; 6097 6098 phylink_rx_clk_stop_block(priv->phylink); 6099 stmmac_set_hw_vlan_mode(priv, priv->hw); 6100 phylink_rx_clk_stop_unblock(priv->phylink); 6101 6102 return 0; 6103} 6104 6105static void stmmac_common_interrupt(struct stmmac_priv *priv) 6106{ 6107 u32 rx_cnt = priv->plat->rx_queues_to_use; 6108 u32 tx_cnt = priv->plat->tx_queues_to_use; 6109 u32 queues_count; 6110 u32 queue; 6111 bool xmac; 6112 6113 xmac = dwmac_is_xmac(priv->plat->core_type); 6114 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; 6115 6116 if (priv->irq_wake) 6117 pm_wakeup_event(priv->device, 0); 6118 6119 if (priv->dma_cap.estsel) 6120 stmmac_est_irq_status(priv, priv, priv->dev, 6121 &priv->xstats, tx_cnt); 6122 6123 if (stmmac_fpe_supported(priv)) 6124 stmmac_fpe_irq_status(priv); 6125 6126 /* To handle GMAC own interrupts */ 6127 if (priv->plat->core_type == DWMAC_CORE_GMAC || xmac) { 6128 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats); 6129 6130 if (unlikely(status)) { 6131 /* For LPI we need to save the tx status */ 6132 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE) 6133 priv->tx_path_in_lpi_mode = true; 6134 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE) 6135 priv->tx_path_in_lpi_mode = false; 6136 } 6137 6138 for (queue = 0; queue < queues_count; queue++) 6139 stmmac_host_mtl_irq_status(priv, priv->hw, queue); 6140 6141 stmmac_timestamp_interrupt(priv, priv); 6142 } 6143} 6144 6145/** 6146 * stmmac_interrupt - main ISR 6147 * @irq: interrupt number. 6148 * @dev_id: to pass the net device pointer. 6149 * Description: this is the main driver interrupt service routine. 6150 * It can call: 6151 * o DMA service routine (to manage incoming frame reception and transmission 6152 * status) 6153 * o Core interrupts to manage: remote wake-up, management counter, LPI 6154 * interrupts. 6155 */ 6156static irqreturn_t stmmac_interrupt(int irq, void *dev_id) 6157{ 6158 struct net_device *dev = (struct net_device *)dev_id; 6159 struct stmmac_priv *priv = netdev_priv(dev); 6160 6161 /* Check if adapter is up */ 6162 if (test_bit(STMMAC_DOWN, &priv->state)) 6163 return IRQ_HANDLED; 6164 6165 /* Check ASP error if it isn't delivered via an individual IRQ */ 6166 if (priv->sfty_irq <= 0 && stmmac_safety_feat_interrupt(priv)) 6167 return IRQ_HANDLED; 6168 6169 /* To handle Common interrupts */ 6170 stmmac_common_interrupt(priv); 6171 6172 /* To handle DMA interrupts */ 6173 stmmac_dma_interrupt(priv); 6174 6175 return IRQ_HANDLED; 6176} 6177 6178static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id) 6179{ 6180 struct net_device *dev = (struct net_device *)dev_id; 6181 struct stmmac_priv *priv = netdev_priv(dev); 6182 6183 /* Check if adapter is up */ 6184 if (test_bit(STMMAC_DOWN, &priv->state)) 6185 return IRQ_HANDLED; 6186 6187 /* To handle Common interrupts */ 6188 stmmac_common_interrupt(priv); 6189 6190 return IRQ_HANDLED; 6191} 6192 6193static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id) 6194{ 6195 struct net_device *dev = (struct net_device *)dev_id; 6196 struct stmmac_priv *priv = netdev_priv(dev); 6197 6198 /* Check if adapter is up */ 6199 if (test_bit(STMMAC_DOWN, &priv->state)) 6200 return IRQ_HANDLED; 6201 6202 /* Check if a fatal error happened */ 6203 stmmac_safety_feat_interrupt(priv); 6204 6205 return IRQ_HANDLED; 6206} 6207 6208static irqreturn_t stmmac_msi_intr_tx(int irq, void *data) 6209{ 6210 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data; 6211 struct stmmac_dma_conf *dma_conf; 6212 int chan = tx_q->queue_index; 6213 struct stmmac_priv *priv; 6214 int status; 6215 6216 dma_conf = container_of(tx_q, struct stmmac_dma_conf, tx_queue[chan]); 6217 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6218 6219 /* Check if adapter is up */ 6220 if (test_bit(STMMAC_DOWN, &priv->state)) 6221 return IRQ_HANDLED; 6222 6223 status = stmmac_napi_check(priv, chan, DMA_DIR_TX); 6224 6225 if (unlikely(status & tx_hard_error_bump_tc)) { 6226 /* Try to bump up the dma threshold on this failure */ 6227 stmmac_bump_dma_threshold(priv, chan); 6228 } else if (unlikely(status == tx_hard_error)) { 6229 stmmac_tx_err(priv, chan); 6230 } 6231 6232 return IRQ_HANDLED; 6233} 6234 6235static irqreturn_t stmmac_msi_intr_rx(int irq, void *data) 6236{ 6237 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data; 6238 struct stmmac_dma_conf *dma_conf; 6239 int chan = rx_q->queue_index; 6240 struct stmmac_priv *priv; 6241 6242 dma_conf = container_of(rx_q, struct stmmac_dma_conf, rx_queue[chan]); 6243 priv = container_of(dma_conf, struct stmmac_priv, dma_conf); 6244 6245 /* Check if adapter is up */ 6246 if (test_bit(STMMAC_DOWN, &priv->state)) 6247 return IRQ_HANDLED; 6248 6249 stmmac_napi_check(priv, chan, DMA_DIR_RX); 6250 6251 return IRQ_HANDLED; 6252} 6253 6254/** 6255 * stmmac_ioctl - Entry point for the Ioctl 6256 * @dev: Device pointer. 6257 * @rq: An IOCTL specefic structure, that can contain a pointer to 6258 * a proprietary structure used to pass information to the driver. 6259 * @cmd: IOCTL command 6260 * Description: 6261 * Currently it supports the phy_mii_ioctl(...) and HW time stamping. 6262 */ 6263static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 6264{ 6265 struct stmmac_priv *priv = netdev_priv (dev); 6266 int ret = -EOPNOTSUPP; 6267 6268 if (!netif_running(dev)) 6269 return -EINVAL; 6270 6271 switch (cmd) { 6272 case SIOCGMIIPHY: 6273 case SIOCGMIIREG: 6274 case SIOCSMIIREG: 6275 ret = phylink_mii_ioctl(priv->phylink, rq, cmd); 6276 break; 6277 default: 6278 break; 6279 } 6280 6281 return ret; 6282} 6283 6284static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 6285 void *cb_priv) 6286{ 6287 struct stmmac_priv *priv = cb_priv; 6288 int ret = -EOPNOTSUPP; 6289 6290 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data)) 6291 return ret; 6292 6293 __stmmac_disable_all_queues(priv); 6294 6295 switch (type) { 6296 case TC_SETUP_CLSU32: 6297 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data); 6298 break; 6299 case TC_SETUP_CLSFLOWER: 6300 ret = stmmac_tc_setup_cls(priv, priv, type_data); 6301 break; 6302 default: 6303 break; 6304 } 6305 6306 stmmac_enable_all_queues(priv); 6307 return ret; 6308} 6309 6310static LIST_HEAD(stmmac_block_cb_list); 6311 6312static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type, 6313 void *type_data) 6314{ 6315 struct stmmac_priv *priv = netdev_priv(ndev); 6316 6317 switch (type) { 6318 case TC_QUERY_CAPS: 6319 return stmmac_tc_query_caps(priv, priv, type_data); 6320 case TC_SETUP_QDISC_MQPRIO: 6321 return stmmac_tc_setup_mqprio(priv, priv, type_data); 6322 case TC_SETUP_BLOCK: 6323 return flow_block_cb_setup_simple(type_data, 6324 &stmmac_block_cb_list, 6325 stmmac_setup_tc_block_cb, 6326 priv, priv, true); 6327 case TC_SETUP_QDISC_CBS: 6328 return stmmac_tc_setup_cbs(priv, priv, type_data); 6329 case TC_SETUP_QDISC_TAPRIO: 6330 return stmmac_tc_setup_taprio(priv, priv, type_data); 6331 case TC_SETUP_QDISC_ETF: 6332 return stmmac_tc_setup_etf(priv, priv, type_data); 6333 default: 6334 return -EOPNOTSUPP; 6335 } 6336} 6337 6338static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb, 6339 struct net_device *sb_dev) 6340{ 6341 int gso = skb_shinfo(skb)->gso_type; 6342 6343 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) { 6344 /* 6345 * There is no way to determine the number of TSO/USO 6346 * capable Queues. Let's use always the Queue 0 6347 * because if TSO/USO is supported then at least this 6348 * one will be capable. 6349 */ 6350 return 0; 6351 } 6352 6353 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues; 6354} 6355 6356static int stmmac_set_mac_address(struct net_device *ndev, void *addr) 6357{ 6358 struct stmmac_priv *priv = netdev_priv(ndev); 6359 int ret = 0; 6360 6361 ret = pm_runtime_resume_and_get(priv->device); 6362 if (ret < 0) 6363 return ret; 6364 6365 ret = eth_mac_addr(ndev, addr); 6366 if (ret) 6367 goto set_mac_error; 6368 6369 phylink_rx_clk_stop_block(priv->phylink); 6370 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0); 6371 phylink_rx_clk_stop_unblock(priv->phylink); 6372 6373set_mac_error: 6374 pm_runtime_put(priv->device); 6375 6376 return ret; 6377} 6378 6379#ifdef CONFIG_DEBUG_FS 6380static struct dentry *stmmac_fs_dir; 6381 6382static void sysfs_display_ring(void *head, int size, int extend_desc, 6383 struct seq_file *seq, dma_addr_t dma_phy_addr) 6384{ 6385 struct dma_extended_desc *ep = (struct dma_extended_desc *)head; 6386 struct dma_desc *p = (struct dma_desc *)head; 6387 unsigned int desc_size; 6388 dma_addr_t dma_addr; 6389 int i; 6390 6391 desc_size = extend_desc ? sizeof(*ep) : sizeof(*p); 6392 for (i = 0; i < size; i++) { 6393 dma_addr = dma_phy_addr + i * desc_size; 6394 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n", 6395 i, &dma_addr, 6396 le32_to_cpu(p->des0), le32_to_cpu(p->des1), 6397 le32_to_cpu(p->des2), le32_to_cpu(p->des3)); 6398 if (extend_desc) 6399 p = &(++ep)->basic; 6400 else 6401 p++; 6402 } 6403} 6404 6405static int stmmac_rings_status_show(struct seq_file *seq, void *v) 6406{ 6407 struct net_device *dev = seq->private; 6408 struct stmmac_priv *priv = netdev_priv(dev); 6409 u32 rx_count = priv->plat->rx_queues_to_use; 6410 u32 tx_count = priv->plat->tx_queues_to_use; 6411 u32 queue; 6412 6413 if ((dev->flags & IFF_UP) == 0) 6414 return 0; 6415 6416 for (queue = 0; queue < rx_count; queue++) { 6417 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6418 6419 seq_printf(seq, "RX Queue %d:\n", queue); 6420 6421 if (priv->extend_desc) { 6422 seq_printf(seq, "Extended descriptor ring:\n"); 6423 sysfs_display_ring((void *)rx_q->dma_erx, 6424 priv->dma_conf.dma_rx_size, 1, seq, rx_q->dma_rx_phy); 6425 } else { 6426 seq_printf(seq, "Descriptor ring:\n"); 6427 sysfs_display_ring((void *)rx_q->dma_rx, 6428 priv->dma_conf.dma_rx_size, 0, seq, rx_q->dma_rx_phy); 6429 } 6430 } 6431 6432 for (queue = 0; queue < tx_count; queue++) { 6433 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6434 6435 seq_printf(seq, "TX Queue %d:\n", queue); 6436 6437 if (priv->extend_desc) { 6438 seq_printf(seq, "Extended descriptor ring:\n"); 6439 sysfs_display_ring((void *)tx_q->dma_etx, 6440 priv->dma_conf.dma_tx_size, 1, seq, tx_q->dma_tx_phy); 6441 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) { 6442 seq_printf(seq, "Descriptor ring:\n"); 6443 sysfs_display_ring((void *)tx_q->dma_tx, 6444 priv->dma_conf.dma_tx_size, 0, seq, tx_q->dma_tx_phy); 6445 } 6446 } 6447 6448 return 0; 6449} 6450DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status); 6451 6452static int stmmac_dma_cap_show(struct seq_file *seq, void *v) 6453{ 6454 static const char * const dwxgmac_timestamp_source[] = { 6455 "None", 6456 "Internal", 6457 "External", 6458 "Both", 6459 }; 6460 static const char * const dwxgmac_safety_feature_desc[] = { 6461 "No", 6462 "All Safety Features with ECC and Parity", 6463 "All Safety Features without ECC or Parity", 6464 "All Safety Features with Parity Only", 6465 "ECC Only", 6466 "UNDEFINED", 6467 "UNDEFINED", 6468 "UNDEFINED", 6469 }; 6470 struct net_device *dev = seq->private; 6471 struct stmmac_priv *priv = netdev_priv(dev); 6472 6473 if (!priv->hw_cap_support) { 6474 seq_printf(seq, "DMA HW features not supported\n"); 6475 return 0; 6476 } 6477 6478 seq_printf(seq, "==============================\n"); 6479 seq_printf(seq, "\tDMA HW features\n"); 6480 seq_printf(seq, "==============================\n"); 6481 6482 seq_printf(seq, "\t10/100 Mbps: %s\n", 6483 (priv->dma_cap.mbps_10_100) ? "Y" : "N"); 6484 seq_printf(seq, "\t1000 Mbps: %s\n", 6485 (priv->dma_cap.mbps_1000) ? "Y" : "N"); 6486 seq_printf(seq, "\tHalf duplex: %s\n", 6487 (priv->dma_cap.half_duplex) ? "Y" : "N"); 6488 if (priv->plat->core_type == DWMAC_CORE_XGMAC) { 6489 seq_printf(seq, 6490 "\tNumber of Additional MAC address registers: %d\n", 6491 priv->dma_cap.multi_addr); 6492 } else { 6493 seq_printf(seq, "\tHash Filter: %s\n", 6494 (priv->dma_cap.hash_filter) ? "Y" : "N"); 6495 seq_printf(seq, "\tMultiple MAC address registers: %s\n", 6496 (priv->dma_cap.multi_addr) ? "Y" : "N"); 6497 } 6498 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n", 6499 (priv->dma_cap.pcs) ? "Y" : "N"); 6500 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n", 6501 (priv->dma_cap.sma_mdio) ? "Y" : "N"); 6502 seq_printf(seq, "\tPMT Remote wake up: %s\n", 6503 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N"); 6504 seq_printf(seq, "\tPMT Magic Frame: %s\n", 6505 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N"); 6506 seq_printf(seq, "\tRMON module: %s\n", 6507 (priv->dma_cap.rmon) ? "Y" : "N"); 6508 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n", 6509 (priv->dma_cap.time_stamp) ? "Y" : "N"); 6510 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n", 6511 (priv->dma_cap.atime_stamp) ? "Y" : "N"); 6512 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 6513 seq_printf(seq, "\tTimestamp System Time Source: %s\n", 6514 dwxgmac_timestamp_source[priv->dma_cap.tssrc]); 6515 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n", 6516 (priv->dma_cap.eee) ? "Y" : "N"); 6517 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N"); 6518 seq_printf(seq, "\tChecksum Offload in TX: %s\n", 6519 (priv->dma_cap.tx_coe) ? "Y" : "N"); 6520 if (priv->synopsys_id >= DWMAC_CORE_4_00 || 6521 priv->plat->core_type == DWMAC_CORE_XGMAC) { 6522 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n", 6523 (priv->dma_cap.rx_coe) ? "Y" : "N"); 6524 } else { 6525 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n", 6526 (priv->dma_cap.rx_coe_type1) ? "Y" : "N"); 6527 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n", 6528 (priv->dma_cap.rx_coe_type2) ? "Y" : "N"); 6529 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n", 6530 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N"); 6531 } 6532 seq_printf(seq, "\tNumber of Additional RX channel: %d\n", 6533 priv->dma_cap.number_rx_channel); 6534 seq_printf(seq, "\tNumber of Additional TX channel: %d\n", 6535 priv->dma_cap.number_tx_channel); 6536 seq_printf(seq, "\tNumber of Additional RX queues: %d\n", 6537 priv->dma_cap.number_rx_queues); 6538 seq_printf(seq, "\tNumber of Additional TX queues: %d\n", 6539 priv->dma_cap.number_tx_queues); 6540 seq_printf(seq, "\tEnhanced descriptors: %s\n", 6541 (priv->dma_cap.enh_desc) ? "Y" : "N"); 6542 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size); 6543 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size); 6544 seq_printf(seq, "\tHash Table Size: %lu\n", priv->dma_cap.hash_tb_sz ? 6545 (BIT(priv->dma_cap.hash_tb_sz) << 5) : 0); 6546 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N"); 6547 seq_printf(seq, "\tNumber of PPS Outputs: %d\n", 6548 priv->dma_cap.pps_out_num); 6549 seq_printf(seq, "\tSafety Features: %s\n", 6550 dwxgmac_safety_feature_desc[priv->dma_cap.asp]); 6551 seq_printf(seq, "\tFlexible RX Parser: %s\n", 6552 priv->dma_cap.frpsel ? "Y" : "N"); 6553 seq_printf(seq, "\tEnhanced Addressing: %d\n", 6554 priv->dma_cap.host_dma_width); 6555 seq_printf(seq, "\tReceive Side Scaling: %s\n", 6556 priv->dma_cap.rssen ? "Y" : "N"); 6557 seq_printf(seq, "\tVLAN Hash Filtering: %s\n", 6558 priv->dma_cap.vlhash ? "Y" : "N"); 6559 seq_printf(seq, "\tSplit Header: %s\n", 6560 priv->dma_cap.sphen ? "Y" : "N"); 6561 seq_printf(seq, "\tVLAN TX Insertion: %s\n", 6562 priv->dma_cap.vlins ? "Y" : "N"); 6563 seq_printf(seq, "\tDouble VLAN: %s\n", 6564 priv->dma_cap.dvlan ? "Y" : "N"); 6565 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n", 6566 priv->dma_cap.l3l4fnum); 6567 seq_printf(seq, "\tARP Offloading: %s\n", 6568 priv->dma_cap.arpoffsel ? "Y" : "N"); 6569 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n", 6570 priv->dma_cap.estsel ? "Y" : "N"); 6571 seq_printf(seq, "\tFrame Preemption (FPE): %s\n", 6572 priv->dma_cap.fpesel ? "Y" : "N"); 6573 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n", 6574 priv->dma_cap.tbssel ? "Y" : "N"); 6575 seq_printf(seq, "\tNumber of DMA Channels Enabled for TBS: %d\n", 6576 priv->dma_cap.tbs_ch_num); 6577 seq_printf(seq, "\tPer-Stream Filtering: %s\n", 6578 priv->dma_cap.sgfsel ? "Y" : "N"); 6579 seq_printf(seq, "\tTX Timestamp FIFO Depth: %lu\n", 6580 BIT(priv->dma_cap.ttsfd) >> 1); 6581 seq_printf(seq, "\tNumber of Traffic Classes: %d\n", 6582 priv->dma_cap.numtc); 6583 seq_printf(seq, "\tDCB Feature: %s\n", 6584 priv->dma_cap.dcben ? "Y" : "N"); 6585 seq_printf(seq, "\tIEEE 1588 High Word Register: %s\n", 6586 priv->dma_cap.advthword ? "Y" : "N"); 6587 seq_printf(seq, "\tPTP Offload: %s\n", 6588 priv->dma_cap.ptoen ? "Y" : "N"); 6589 seq_printf(seq, "\tOne-Step Timestamping: %s\n", 6590 priv->dma_cap.osten ? "Y" : "N"); 6591 seq_printf(seq, "\tPriority-Based Flow Control: %s\n", 6592 priv->dma_cap.pfcen ? "Y" : "N"); 6593 seq_printf(seq, "\tNumber of Flexible RX Parser Instructions: %lu\n", 6594 BIT(priv->dma_cap.frpes) << 6); 6595 seq_printf(seq, "\tNumber of Flexible RX Parser Parsable Bytes: %lu\n", 6596 BIT(priv->dma_cap.frpbs) << 6); 6597 seq_printf(seq, "\tParallel Instruction Processor Engines: %d\n", 6598 priv->dma_cap.frppipe_num); 6599 seq_printf(seq, "\tNumber of Extended VLAN Tag Filters: %lu\n", 6600 priv->dma_cap.nrvf_num ? 6601 (BIT(priv->dma_cap.nrvf_num) << 1) : 0); 6602 seq_printf(seq, "\tWidth of the Time Interval Field in GCL: %d\n", 6603 priv->dma_cap.estwid ? 4 * priv->dma_cap.estwid + 12 : 0); 6604 seq_printf(seq, "\tDepth of GCL: %lu\n", 6605 priv->dma_cap.estdep ? (BIT(priv->dma_cap.estdep) << 5) : 0); 6606 seq_printf(seq, "\tQueue/Channel-Based VLAN Tag Insertion on TX: %s\n", 6607 priv->dma_cap.cbtisel ? "Y" : "N"); 6608 seq_printf(seq, "\tNumber of Auxiliary Snapshot Inputs: %d\n", 6609 priv->dma_cap.aux_snapshot_n); 6610 seq_printf(seq, "\tOne-Step Timestamping for PTP over UDP/IP: %s\n", 6611 priv->dma_cap.pou_ost_en ? "Y" : "N"); 6612 seq_printf(seq, "\tEnhanced DMA: %s\n", 6613 priv->dma_cap.edma ? "Y" : "N"); 6614 seq_printf(seq, "\tDifferent Descriptor Cache: %s\n", 6615 priv->dma_cap.ediffc ? "Y" : "N"); 6616 seq_printf(seq, "\tVxLAN/NVGRE: %s\n", 6617 priv->dma_cap.vxn ? "Y" : "N"); 6618 seq_printf(seq, "\tDebug Memory Interface: %s\n", 6619 priv->dma_cap.dbgmem ? "Y" : "N"); 6620 seq_printf(seq, "\tNumber of Policing Counters: %lu\n", 6621 priv->dma_cap.pcsel ? BIT(priv->dma_cap.pcsel + 3) : 0); 6622 return 0; 6623} 6624DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap); 6625 6626/* Use network device events to rename debugfs file entries. 6627 */ 6628static int stmmac_device_event(struct notifier_block *unused, 6629 unsigned long event, void *ptr) 6630{ 6631 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 6632 struct stmmac_priv *priv = netdev_priv(dev); 6633 6634 if (dev->netdev_ops != &stmmac_netdev_ops) 6635 goto done; 6636 6637 switch (event) { 6638 case NETDEV_CHANGENAME: 6639 debugfs_change_name(priv->dbgfs_dir, "%s", dev->name); 6640 break; 6641 } 6642done: 6643 return NOTIFY_DONE; 6644} 6645 6646static struct notifier_block stmmac_notifier = { 6647 .notifier_call = stmmac_device_event, 6648}; 6649 6650static void stmmac_init_fs(struct net_device *dev) 6651{ 6652 struct stmmac_priv *priv = netdev_priv(dev); 6653 6654 rtnl_lock(); 6655 6656 /* Create per netdev entries */ 6657 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir); 6658 6659 /* Entry to report DMA RX/TX rings */ 6660 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev, 6661 &stmmac_rings_status_fops); 6662 6663 /* Entry to report the DMA HW features */ 6664 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev, 6665 &stmmac_dma_cap_fops); 6666 6667 rtnl_unlock(); 6668} 6669 6670static void stmmac_exit_fs(struct net_device *dev) 6671{ 6672 struct stmmac_priv *priv = netdev_priv(dev); 6673 6674 debugfs_remove_recursive(priv->dbgfs_dir); 6675} 6676#endif /* CONFIG_DEBUG_FS */ 6677 6678static u32 stmmac_vid_crc32_le(__le16 vid_le) 6679{ 6680 unsigned char *data = (unsigned char *)&vid_le; 6681 unsigned char data_byte = 0; 6682 u32 crc = ~0x0; 6683 u32 temp = 0; 6684 int i, bits; 6685 6686 bits = get_bitmask_order(VLAN_VID_MASK); 6687 for (i = 0; i < bits; i++) { 6688 if ((i % 8) == 0) 6689 data_byte = data[i / 8]; 6690 6691 temp = ((crc & 1) ^ data_byte) & 1; 6692 crc >>= 1; 6693 data_byte >>= 1; 6694 6695 if (temp) 6696 crc ^= 0xedb88320; 6697 } 6698 6699 return crc; 6700} 6701 6702static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double) 6703{ 6704 u32 crc, hash = 0; 6705 u16 pmatch = 0; 6706 int count = 0; 6707 u16 vid = 0; 6708 6709 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) { 6710 __le16 vid_le = cpu_to_le16(vid); 6711 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28; 6712 hash |= (1 << crc); 6713 count++; 6714 } 6715 6716 if (!priv->dma_cap.vlhash) { 6717 if (count > 2) /* VID = 0 always passes filter */ 6718 return -EOPNOTSUPP; 6719 6720 pmatch = vid; 6721 hash = 0; 6722 } 6723 6724 if (!netif_running(priv->dev)) 6725 return 0; 6726 6727 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double); 6728} 6729 6730/* FIXME: This may need RXC to be running, but it may be called with BH 6731 * disabled, which means we can't call phylink_rx_clk_stop*(). 6732 */ 6733static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid) 6734{ 6735 struct stmmac_priv *priv = netdev_priv(ndev); 6736 unsigned int num_double_vlans; 6737 bool is_double = false; 6738 int ret; 6739 6740 ret = pm_runtime_resume_and_get(priv->device); 6741 if (ret < 0) 6742 return ret; 6743 6744 if (be16_to_cpu(proto) == ETH_P_8021AD) 6745 is_double = true; 6746 6747 set_bit(vid, priv->active_vlans); 6748 num_double_vlans = priv->num_double_vlans + is_double; 6749 ret = stmmac_vlan_update(priv, num_double_vlans); 6750 if (ret) { 6751 clear_bit(vid, priv->active_vlans); 6752 goto err_pm_put; 6753 } 6754 6755 if (priv->hw->num_vlan) { 6756 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6757 if (ret) { 6758 clear_bit(vid, priv->active_vlans); 6759 stmmac_vlan_update(priv, priv->num_double_vlans); 6760 goto err_pm_put; 6761 } 6762 } 6763 6764 priv->num_double_vlans = num_double_vlans; 6765 6766err_pm_put: 6767 pm_runtime_put(priv->device); 6768 6769 return ret; 6770} 6771 6772/* FIXME: This may need RXC to be running, but it may be called with BH 6773 * disabled, which means we can't call phylink_rx_clk_stop*(). 6774 */ 6775static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid) 6776{ 6777 struct stmmac_priv *priv = netdev_priv(ndev); 6778 unsigned int num_double_vlans; 6779 bool is_double = false; 6780 int ret; 6781 6782 ret = pm_runtime_resume_and_get(priv->device); 6783 if (ret < 0) 6784 return ret; 6785 6786 if (be16_to_cpu(proto) == ETH_P_8021AD) 6787 is_double = true; 6788 6789 clear_bit(vid, priv->active_vlans); 6790 num_double_vlans = priv->num_double_vlans - is_double; 6791 ret = stmmac_vlan_update(priv, num_double_vlans); 6792 if (ret) { 6793 set_bit(vid, priv->active_vlans); 6794 goto del_vlan_error; 6795 } 6796 6797 if (priv->hw->num_vlan) { 6798 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid); 6799 if (ret) { 6800 set_bit(vid, priv->active_vlans); 6801 stmmac_vlan_update(priv, priv->num_double_vlans); 6802 goto del_vlan_error; 6803 } 6804 } 6805 6806 priv->num_double_vlans = num_double_vlans; 6807 6808del_vlan_error: 6809 pm_runtime_put(priv->device); 6810 6811 return ret; 6812} 6813 6814static void stmmac_vlan_restore(struct stmmac_priv *priv) 6815{ 6816 if (!(priv->dev->features & NETIF_F_VLAN_FEATURES)) 6817 return; 6818 6819 if (priv->hw->num_vlan) 6820 stmmac_restore_hw_vlan_rx_fltr(priv, priv->dev, priv->hw); 6821 6822 stmmac_vlan_update(priv, priv->num_double_vlans); 6823} 6824 6825static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf) 6826{ 6827 struct stmmac_priv *priv = netdev_priv(dev); 6828 6829 switch (bpf->command) { 6830 case XDP_SETUP_PROG: 6831 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack); 6832 case XDP_SETUP_XSK_POOL: 6833 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool, 6834 bpf->xsk.queue_id); 6835 default: 6836 return -EOPNOTSUPP; 6837 } 6838} 6839 6840static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, 6841 struct xdp_frame **frames, u32 flags) 6842{ 6843 struct stmmac_priv *priv = netdev_priv(dev); 6844 int cpu = smp_processor_id(); 6845 struct netdev_queue *nq; 6846 int i, nxmit = 0; 6847 int queue; 6848 6849 if (unlikely(test_bit(STMMAC_DOWN, &priv->state))) 6850 return -ENETDOWN; 6851 6852 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 6853 return -EINVAL; 6854 6855 queue = stmmac_xdp_get_tx_queue(priv, cpu); 6856 nq = netdev_get_tx_queue(priv->dev, queue); 6857 6858 __netif_tx_lock(nq, cpu); 6859 /* Avoids TX time-out as we are sharing with slow path */ 6860 txq_trans_cond_update(nq); 6861 6862 for (i = 0; i < num_frames; i++) { 6863 int res; 6864 6865 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true); 6866 if (res == STMMAC_XDP_CONSUMED) 6867 break; 6868 6869 nxmit++; 6870 } 6871 6872 if (flags & XDP_XMIT_FLUSH) { 6873 stmmac_flush_tx_descriptors(priv, queue); 6874 stmmac_tx_timer_arm(priv, queue); 6875 } 6876 6877 __netif_tx_unlock(nq); 6878 6879 return nxmit; 6880} 6881 6882void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue) 6883{ 6884 struct stmmac_channel *ch = &priv->channel[queue]; 6885 unsigned long flags; 6886 6887 spin_lock_irqsave(&ch->lock, flags); 6888 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6889 spin_unlock_irqrestore(&ch->lock, flags); 6890 6891 stmmac_stop_rx_dma(priv, queue); 6892 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6893} 6894 6895void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) 6896{ 6897 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 6898 struct stmmac_channel *ch = &priv->channel[queue]; 6899 unsigned long flags; 6900 u32 buf_size; 6901 int ret; 6902 6903 ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6904 if (ret) { 6905 netdev_err(priv->dev, "Failed to alloc RX desc.\n"); 6906 return; 6907 } 6908 6909 ret = __init_dma_rx_desc_rings(priv, &priv->dma_conf, queue, GFP_KERNEL); 6910 if (ret) { 6911 __free_dma_rx_desc_resources(priv, &priv->dma_conf, queue); 6912 netdev_err(priv->dev, "Failed to init RX desc.\n"); 6913 return; 6914 } 6915 6916 stmmac_reset_rx_queue(priv, queue); 6917 stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); 6918 6919 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6920 rx_q->dma_rx_phy, rx_q->queue_index); 6921 6922 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * 6923 sizeof(struct dma_desc)); 6924 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 6925 rx_q->rx_tail_addr, rx_q->queue_index); 6926 6927 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 6928 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 6929 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6930 buf_size, 6931 rx_q->queue_index); 6932 } else { 6933 stmmac_set_dma_bfsize(priv, priv->ioaddr, 6934 priv->dma_conf.dma_buf_sz, 6935 rx_q->queue_index); 6936 } 6937 6938 stmmac_start_rx_dma(priv, queue); 6939 6940 spin_lock_irqsave(&ch->lock, flags); 6941 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0); 6942 spin_unlock_irqrestore(&ch->lock, flags); 6943} 6944 6945void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue) 6946{ 6947 struct stmmac_channel *ch = &priv->channel[queue]; 6948 unsigned long flags; 6949 6950 spin_lock_irqsave(&ch->lock, flags); 6951 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6952 spin_unlock_irqrestore(&ch->lock, flags); 6953 6954 stmmac_stop_tx_dma(priv, queue); 6955 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6956} 6957 6958void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) 6959{ 6960 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 6961 struct stmmac_channel *ch = &priv->channel[queue]; 6962 unsigned long flags; 6963 int ret; 6964 6965 ret = __alloc_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6966 if (ret) { 6967 netdev_err(priv->dev, "Failed to alloc TX desc.\n"); 6968 return; 6969 } 6970 6971 ret = __init_dma_tx_desc_rings(priv, &priv->dma_conf, queue); 6972 if (ret) { 6973 __free_dma_tx_desc_resources(priv, &priv->dma_conf, queue); 6974 netdev_err(priv->dev, "Failed to init TX desc.\n"); 6975 return; 6976 } 6977 6978 stmmac_reset_tx_queue(priv, queue); 6979 stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); 6980 6981 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 6982 tx_q->dma_tx_phy, tx_q->queue_index); 6983 6984 if (tx_q->tbs & STMMAC_TBS_AVAIL) 6985 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); 6986 6987 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 6988 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 6989 tx_q->tx_tail_addr, tx_q->queue_index); 6990 6991 stmmac_start_tx_dma(priv, queue); 6992 6993 spin_lock_irqsave(&ch->lock, flags); 6994 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1); 6995 spin_unlock_irqrestore(&ch->lock, flags); 6996} 6997 6998void stmmac_xdp_release(struct net_device *dev) 6999{ 7000 struct stmmac_priv *priv = netdev_priv(dev); 7001 u32 chan; 7002 7003 /* Ensure tx function is not running */ 7004 netif_tx_disable(dev); 7005 7006 /* Disable NAPI process */ 7007 stmmac_disable_all_queues(priv); 7008 7009 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7010 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7011 7012 /* Free the IRQ lines */ 7013 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); 7014 7015 /* Stop TX/RX DMA channels */ 7016 stmmac_stop_all_dma(priv); 7017 7018 /* Release and free the Rx/Tx resources */ 7019 free_dma_desc_resources(priv, &priv->dma_conf); 7020 7021 /* Disable the MAC Rx/Tx */ 7022 stmmac_mac_set(priv, priv->ioaddr, false); 7023 7024 /* set trans_start so we don't get spurious 7025 * watchdogs during reset 7026 */ 7027 netif_trans_update(dev); 7028 netif_carrier_off(dev); 7029} 7030 7031int stmmac_xdp_open(struct net_device *dev) 7032{ 7033 struct stmmac_priv *priv = netdev_priv(dev); 7034 u32 rx_cnt = priv->plat->rx_queues_to_use; 7035 u32 tx_cnt = priv->plat->tx_queues_to_use; 7036 u32 dma_csr_ch = max(rx_cnt, tx_cnt); 7037 struct stmmac_rx_queue *rx_q; 7038 struct stmmac_tx_queue *tx_q; 7039 u32 buf_size; 7040 bool sph_en; 7041 u32 chan; 7042 int ret; 7043 7044 ret = alloc_dma_desc_resources(priv, &priv->dma_conf); 7045 if (ret < 0) { 7046 netdev_err(dev, "%s: DMA descriptors allocation failed\n", 7047 __func__); 7048 goto dma_desc_error; 7049 } 7050 7051 ret = init_dma_desc_rings(dev, &priv->dma_conf, GFP_KERNEL); 7052 if (ret < 0) { 7053 netdev_err(dev, "%s: DMA descriptors initialization failed\n", 7054 __func__); 7055 goto init_error; 7056 } 7057 7058 stmmac_reset_queues_param(priv); 7059 7060 /* DMA CSR Channel configuration */ 7061 for (chan = 0; chan < dma_csr_ch; chan++) { 7062 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); 7063 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 1); 7064 } 7065 7066 /* Adjust Split header */ 7067 sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; 7068 7069 /* DMA RX Channel Configuration */ 7070 for (chan = 0; chan < rx_cnt; chan++) { 7071 rx_q = &priv->dma_conf.rx_queue[chan]; 7072 7073 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7074 rx_q->dma_rx_phy, chan); 7075 7076 rx_q->rx_tail_addr = rx_q->dma_rx_phy + 7077 (rx_q->buf_alloc_num * 7078 sizeof(struct dma_desc)); 7079 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, 7080 rx_q->rx_tail_addr, chan); 7081 7082 if (rx_q->xsk_pool && rx_q->buf_alloc_num) { 7083 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); 7084 stmmac_set_dma_bfsize(priv, priv->ioaddr, 7085 buf_size, 7086 rx_q->queue_index); 7087 } else { 7088 stmmac_set_dma_bfsize(priv, priv->ioaddr, 7089 priv->dma_conf.dma_buf_sz, 7090 rx_q->queue_index); 7091 } 7092 7093 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); 7094 } 7095 7096 /* DMA TX Channel Configuration */ 7097 for (chan = 0; chan < tx_cnt; chan++) { 7098 tx_q = &priv->dma_conf.tx_queue[chan]; 7099 7100 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, 7101 tx_q->dma_tx_phy, chan); 7102 7103 tx_q->tx_tail_addr = tx_q->dma_tx_phy; 7104 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, 7105 tx_q->tx_tail_addr, chan); 7106 7107 hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 7108 } 7109 7110 /* Enable the MAC Rx/Tx */ 7111 stmmac_mac_set(priv, priv->ioaddr, true); 7112 7113 /* Start Rx & Tx DMA Channels */ 7114 stmmac_start_all_dma(priv); 7115 7116 ret = stmmac_request_irq(dev); 7117 if (ret) 7118 goto irq_error; 7119 7120 /* Enable NAPI process*/ 7121 stmmac_enable_all_queues(priv); 7122 netif_carrier_on(dev); 7123 netif_tx_start_all_queues(dev); 7124 stmmac_enable_all_dma_irq(priv); 7125 7126 return 0; 7127 7128irq_error: 7129 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 7130 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 7131 7132init_error: 7133 free_dma_desc_resources(priv, &priv->dma_conf); 7134dma_desc_error: 7135 return ret; 7136} 7137 7138int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) 7139{ 7140 struct stmmac_priv *priv = netdev_priv(dev); 7141 struct stmmac_rx_queue *rx_q; 7142 struct stmmac_tx_queue *tx_q; 7143 struct stmmac_channel *ch; 7144 7145 if (test_bit(STMMAC_DOWN, &priv->state) || 7146 !netif_carrier_ok(priv->dev)) 7147 return -ENETDOWN; 7148 7149 if (!stmmac_xdp_is_enabled(priv)) 7150 return -EINVAL; 7151 7152 if (queue >= priv->plat->rx_queues_to_use || 7153 queue >= priv->plat->tx_queues_to_use) 7154 return -EINVAL; 7155 7156 rx_q = &priv->dma_conf.rx_queue[queue]; 7157 tx_q = &priv->dma_conf.tx_queue[queue]; 7158 ch = &priv->channel[queue]; 7159 7160 if (!rx_q->xsk_pool && !tx_q->xsk_pool) 7161 return -EINVAL; 7162 7163 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) { 7164 /* EQoS does not have per-DMA channel SW interrupt, 7165 * so we schedule RX Napi straight-away. 7166 */ 7167 if (likely(napi_schedule_prep(&ch->rxtx_napi))) 7168 __napi_schedule(&ch->rxtx_napi); 7169 } 7170 7171 return 0; 7172} 7173 7174static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 7175{ 7176 struct stmmac_priv *priv = netdev_priv(dev); 7177 u32 tx_cnt = priv->plat->tx_queues_to_use; 7178 u32 rx_cnt = priv->plat->rx_queues_to_use; 7179 unsigned int start; 7180 int q; 7181 7182 for (q = 0; q < tx_cnt; q++) { 7183 struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; 7184 u64 tx_packets; 7185 u64 tx_bytes; 7186 7187 do { 7188 start = u64_stats_fetch_begin(&txq_stats->q_syncp); 7189 tx_bytes = u64_stats_read(&txq_stats->q.tx_bytes); 7190 } while (u64_stats_fetch_retry(&txq_stats->q_syncp, start)); 7191 do { 7192 start = u64_stats_fetch_begin(&txq_stats->napi_syncp); 7193 tx_packets = u64_stats_read(&txq_stats->napi.tx_packets); 7194 } while (u64_stats_fetch_retry(&txq_stats->napi_syncp, start)); 7195 7196 stats->tx_packets += tx_packets; 7197 stats->tx_bytes += tx_bytes; 7198 } 7199 7200 for (q = 0; q < rx_cnt; q++) { 7201 struct stmmac_rxq_stats *rxq_stats = &priv->xstats.rxq_stats[q]; 7202 u64 rx_packets; 7203 u64 rx_bytes; 7204 7205 do { 7206 start = u64_stats_fetch_begin(&rxq_stats->napi_syncp); 7207 rx_packets = u64_stats_read(&rxq_stats->napi.rx_packets); 7208 rx_bytes = u64_stats_read(&rxq_stats->napi.rx_bytes); 7209 } while (u64_stats_fetch_retry(&rxq_stats->napi_syncp, start)); 7210 7211 stats->rx_packets += rx_packets; 7212 stats->rx_bytes += rx_bytes; 7213 } 7214 7215 stats->rx_dropped = priv->xstats.rx_dropped; 7216 stats->rx_errors = priv->xstats.rx_errors; 7217 stats->tx_dropped = priv->xstats.tx_dropped; 7218 stats->tx_errors = priv->xstats.tx_errors; 7219 stats->tx_carrier_errors = priv->xstats.tx_losscarrier + priv->xstats.tx_carrier; 7220 stats->collisions = priv->xstats.tx_collision + priv->xstats.rx_collision; 7221 stats->rx_length_errors = priv->xstats.rx_length; 7222 stats->rx_crc_errors = priv->xstats.rx_crc_errors; 7223 stats->rx_over_errors = priv->xstats.rx_overflow_cntr; 7224 stats->rx_missed_errors = priv->xstats.rx_missed_cntr; 7225} 7226 7227static const struct net_device_ops stmmac_netdev_ops = { 7228 .ndo_open = stmmac_open, 7229 .ndo_start_xmit = stmmac_xmit, 7230 .ndo_stop = stmmac_release, 7231 .ndo_change_mtu = stmmac_change_mtu, 7232 .ndo_fix_features = stmmac_fix_features, 7233 .ndo_set_features = stmmac_set_features, 7234 .ndo_set_rx_mode = stmmac_set_rx_mode, 7235 .ndo_tx_timeout = stmmac_tx_timeout, 7236 .ndo_eth_ioctl = stmmac_ioctl, 7237 .ndo_get_stats64 = stmmac_get_stats64, 7238 .ndo_setup_tc = stmmac_setup_tc, 7239 .ndo_select_queue = stmmac_select_queue, 7240 .ndo_set_mac_address = stmmac_set_mac_address, 7241 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid, 7242 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid, 7243 .ndo_bpf = stmmac_bpf, 7244 .ndo_xdp_xmit = stmmac_xdp_xmit, 7245 .ndo_xsk_wakeup = stmmac_xsk_wakeup, 7246 .ndo_hwtstamp_get = stmmac_hwtstamp_get, 7247 .ndo_hwtstamp_set = stmmac_hwtstamp_set, 7248}; 7249 7250static void stmmac_reset_subtask(struct stmmac_priv *priv) 7251{ 7252 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state)) 7253 return; 7254 if (test_bit(STMMAC_DOWN, &priv->state)) 7255 return; 7256 7257 netdev_err(priv->dev, "Reset adapter.\n"); 7258 7259 rtnl_lock(); 7260 netif_trans_update(priv->dev); 7261 while (test_and_set_bit(STMMAC_RESETING, &priv->state)) 7262 usleep_range(1000, 2000); 7263 7264 set_bit(STMMAC_DOWN, &priv->state); 7265 dev_close(priv->dev); 7266 dev_open(priv->dev, NULL); 7267 clear_bit(STMMAC_DOWN, &priv->state); 7268 clear_bit(STMMAC_RESETING, &priv->state); 7269 rtnl_unlock(); 7270} 7271 7272static void stmmac_service_task(struct work_struct *work) 7273{ 7274 struct stmmac_priv *priv = container_of(work, struct stmmac_priv, 7275 service_task); 7276 7277 stmmac_reset_subtask(priv); 7278 clear_bit(STMMAC_SERVICE_SCHED, &priv->state); 7279} 7280 7281/** 7282 * stmmac_hw_init - Init the MAC device 7283 * @priv: driver private structure 7284 * Description: this function is to configure the MAC device according to 7285 * some platform parameters or the HW capability register. It prepares the 7286 * driver to use either ring or chain modes and to setup either enhanced or 7287 * normal descriptors. 7288 */ 7289static int stmmac_hw_init(struct stmmac_priv *priv) 7290{ 7291 int ret; 7292 7293 /* dwmac-sun8i only work in chain mode */ 7294 if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) 7295 chain_mode = 1; 7296 priv->chain_mode = chain_mode; 7297 7298 /* Initialize HW Interface */ 7299 ret = stmmac_hwif_init(priv); 7300 if (ret) 7301 return ret; 7302 7303 /* Get the HW capability (new GMAC newer than 3.50a) */ 7304 priv->hw_cap_support = stmmac_get_hw_features(priv); 7305 if (priv->hw_cap_support) { 7306 dev_info(priv->device, "DMA HW capability register supported\n"); 7307 7308 /* We can override some gmac/dma configuration fields: e.g. 7309 * enh_desc, tx_coe (e.g. that are passed through the 7310 * platform) with the values from the HW capability 7311 * register (if supported). 7312 */ 7313 priv->plat->enh_desc = priv->dma_cap.enh_desc; 7314 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up && 7315 !(priv->plat->flags & STMMAC_FLAG_USE_PHY_WOL); 7316 if (priv->dma_cap.hash_tb_sz) { 7317 priv->hw->multicast_filter_bins = 7318 (BIT(priv->dma_cap.hash_tb_sz) << 5); 7319 priv->hw->mcast_bits_log2 = 7320 ilog2(priv->hw->multicast_filter_bins); 7321 } 7322 7323 /* TXCOE doesn't work in thresh DMA mode */ 7324 if (priv->plat->force_thresh_dma_mode) 7325 priv->plat->tx_coe = 0; 7326 else 7327 priv->plat->tx_coe = priv->dma_cap.tx_coe; 7328 7329 /* In case of GMAC4 rx_coe is from HW cap register. */ 7330 priv->plat->rx_coe = priv->dma_cap.rx_coe; 7331 7332 if (priv->dma_cap.rx_coe_type2) 7333 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2; 7334 else if (priv->dma_cap.rx_coe_type1) 7335 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1; 7336 7337 } else { 7338 dev_info(priv->device, "No HW DMA feature register supported\n"); 7339 } 7340 7341 if (priv->plat->rx_coe) { 7342 priv->hw->rx_csum = priv->plat->rx_coe; 7343 dev_info(priv->device, "RX Checksum Offload Engine supported\n"); 7344 if (priv->synopsys_id < DWMAC_CORE_4_00) 7345 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum); 7346 } 7347 if (priv->plat->tx_coe) 7348 dev_info(priv->device, "TX Checksum insertion supported\n"); 7349 7350 if (priv->plat->pmt) { 7351 dev_info(priv->device, "Wake-Up On Lan supported\n"); 7352 device_set_wakeup_capable(priv->device, 1); 7353 devm_pm_set_wake_irq(priv->device, priv->wol_irq); 7354 } 7355 7356 if (priv->dma_cap.tsoen) 7357 dev_info(priv->device, "TSO supported\n"); 7358 7359 if (priv->dma_cap.number_rx_queues && 7360 priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) { 7361 dev_warn(priv->device, 7362 "Number of Rx queues (%u) exceeds dma capability\n", 7363 priv->plat->rx_queues_to_use); 7364 priv->plat->rx_queues_to_use = priv->dma_cap.number_rx_queues; 7365 } 7366 if (priv->dma_cap.number_tx_queues && 7367 priv->plat->tx_queues_to_use > priv->dma_cap.number_tx_queues) { 7368 dev_warn(priv->device, 7369 "Number of Tx queues (%u) exceeds dma capability\n", 7370 priv->plat->tx_queues_to_use); 7371 priv->plat->tx_queues_to_use = priv->dma_cap.number_tx_queues; 7372 } 7373 7374 if (priv->dma_cap.rx_fifo_size && 7375 priv->plat->rx_fifo_size > priv->dma_cap.rx_fifo_size) { 7376 dev_warn(priv->device, 7377 "Rx FIFO size (%u) exceeds dma capability\n", 7378 priv->plat->rx_fifo_size); 7379 priv->plat->rx_fifo_size = priv->dma_cap.rx_fifo_size; 7380 } 7381 if (priv->dma_cap.tx_fifo_size && 7382 priv->plat->tx_fifo_size > priv->dma_cap.tx_fifo_size) { 7383 dev_warn(priv->device, 7384 "Tx FIFO size (%u) exceeds dma capability\n", 7385 priv->plat->tx_fifo_size); 7386 priv->plat->tx_fifo_size = priv->dma_cap.tx_fifo_size; 7387 } 7388 7389 priv->hw->vlan_fail_q_en = 7390 (priv->plat->flags & STMMAC_FLAG_VLAN_FAIL_Q_EN); 7391 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q; 7392 7393 /* Run HW quirks, if any */ 7394 if (priv->hwif_quirks) { 7395 ret = priv->hwif_quirks(priv); 7396 if (ret) 7397 return ret; 7398 } 7399 7400 /* Rx Watchdog is available in the COREs newer than the 3.40. 7401 * In some case, for example on bugged HW this feature 7402 * has to be disable and this can be done by passing the 7403 * riwt_off field from the platform. 7404 */ 7405 if ((priv->synopsys_id >= DWMAC_CORE_3_50 || 7406 priv->plat->core_type == DWMAC_CORE_XGMAC) && 7407 !priv->plat->riwt_off) { 7408 priv->use_riwt = 1; 7409 dev_info(priv->device, 7410 "Enable RX Mitigation via HW Watchdog Timer\n"); 7411 } 7412 7413 /* Unimplemented PCS init (as indicated by stmmac_do_callback() 7414 * perversely returning -EINVAL) is non-fatal. 7415 */ 7416 ret = stmmac_mac_pcs_init(priv); 7417 if (ret != -EINVAL) 7418 return ret; 7419 7420 return 0; 7421} 7422 7423static void stmmac_napi_add(struct net_device *dev) 7424{ 7425 struct stmmac_priv *priv = netdev_priv(dev); 7426 u32 queue, maxq; 7427 7428 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7429 7430 for (queue = 0; queue < maxq; queue++) { 7431 struct stmmac_channel *ch = &priv->channel[queue]; 7432 7433 ch->priv_data = priv; 7434 ch->index = queue; 7435 spin_lock_init(&ch->lock); 7436 7437 if (queue < priv->plat->rx_queues_to_use) { 7438 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx); 7439 } 7440 if (queue < priv->plat->tx_queues_to_use) { 7441 netif_napi_add_tx(dev, &ch->tx_napi, 7442 stmmac_napi_poll_tx); 7443 } 7444 if (queue < priv->plat->rx_queues_to_use && 7445 queue < priv->plat->tx_queues_to_use) { 7446 netif_napi_add(dev, &ch->rxtx_napi, 7447 stmmac_napi_poll_rxtx); 7448 } 7449 } 7450} 7451 7452static void stmmac_napi_del(struct net_device *dev) 7453{ 7454 struct stmmac_priv *priv = netdev_priv(dev); 7455 u32 queue, maxq; 7456 7457 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); 7458 7459 for (queue = 0; queue < maxq; queue++) { 7460 struct stmmac_channel *ch = &priv->channel[queue]; 7461 7462 if (queue < priv->plat->rx_queues_to_use) 7463 netif_napi_del(&ch->rx_napi); 7464 if (queue < priv->plat->tx_queues_to_use) 7465 netif_napi_del(&ch->tx_napi); 7466 if (queue < priv->plat->rx_queues_to_use && 7467 queue < priv->plat->tx_queues_to_use) { 7468 netif_napi_del(&ch->rxtx_napi); 7469 } 7470 } 7471} 7472 7473int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) 7474{ 7475 struct stmmac_priv *priv = netdev_priv(dev); 7476 int ret = 0, i; 7477 7478 if (netif_running(dev)) 7479 stmmac_release(dev); 7480 7481 stmmac_napi_del(dev); 7482 7483 priv->plat->rx_queues_to_use = rx_cnt; 7484 priv->plat->tx_queues_to_use = tx_cnt; 7485 if (!netif_is_rxfh_configured(dev)) 7486 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7487 priv->rss.table[i] = ethtool_rxfh_indir_default(i, 7488 rx_cnt); 7489 7490 stmmac_napi_add(dev); 7491 7492 if (netif_running(dev)) 7493 ret = stmmac_open(dev); 7494 7495 return ret; 7496} 7497 7498int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size) 7499{ 7500 struct stmmac_priv *priv = netdev_priv(dev); 7501 int ret = 0; 7502 7503 if (netif_running(dev)) 7504 stmmac_release(dev); 7505 7506 priv->dma_conf.dma_rx_size = rx_size; 7507 priv->dma_conf.dma_tx_size = tx_size; 7508 7509 if (netif_running(dev)) 7510 ret = stmmac_open(dev); 7511 7512 return ret; 7513} 7514 7515static int stmmac_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp) 7516{ 7517 const struct stmmac_xdp_buff *ctx = (void *)_ctx; 7518 struct dma_desc *desc_contains_ts = ctx->desc; 7519 struct stmmac_priv *priv = ctx->priv; 7520 struct dma_desc *ndesc = ctx->ndesc; 7521 struct dma_desc *desc = ctx->desc; 7522 u64 ns = 0; 7523 7524 if (!priv->hwts_rx_en) 7525 return -ENODATA; 7526 7527 /* For GMAC4, the valid timestamp is from CTX next desc. */ 7528 if (dwmac_is_xmac(priv->plat->core_type)) 7529 desc_contains_ts = ndesc; 7530 7531 /* Check if timestamp is available */ 7532 if (stmmac_get_rx_timestamp_status(priv, desc, ndesc, priv->adv_ts)) { 7533 stmmac_get_timestamp(priv, desc_contains_ts, priv->adv_ts, &ns); 7534 ns -= priv->plat->cdc_error_adj; 7535 *timestamp = ns_to_ktime(ns); 7536 return 0; 7537 } 7538 7539 return -ENODATA; 7540} 7541 7542static const struct xdp_metadata_ops stmmac_xdp_metadata_ops = { 7543 .xmo_rx_timestamp = stmmac_xdp_rx_timestamp, 7544}; 7545 7546static int stmmac_dl_ts_coarse_set(struct devlink *dl, u32 id, 7547 struct devlink_param_gset_ctx *ctx, 7548 struct netlink_ext_ack *extack) 7549{ 7550 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7551 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7552 7553 priv->tsfupdt_coarse = ctx->val.vbool; 7554 7555 if (priv->tsfupdt_coarse) 7556 priv->systime_flags &= ~PTP_TCR_TSCFUPDT; 7557 else 7558 priv->systime_flags |= PTP_TCR_TSCFUPDT; 7559 7560 /* In Coarse mode, we can use a smaller subsecond increment, let's 7561 * reconfigure the systime, subsecond increment and addend. 7562 */ 7563 stmmac_update_subsecond_increment(priv); 7564 7565 return 0; 7566} 7567 7568static int stmmac_dl_ts_coarse_get(struct devlink *dl, u32 id, 7569 struct devlink_param_gset_ctx *ctx, 7570 struct netlink_ext_ack *extack) 7571{ 7572 struct stmmac_devlink_priv *dl_priv = devlink_priv(dl); 7573 struct stmmac_priv *priv = dl_priv->stmmac_priv; 7574 7575 ctx->val.vbool = priv->tsfupdt_coarse; 7576 7577 return 0; 7578} 7579 7580static const struct devlink_param stmmac_devlink_params[] = { 7581 DEVLINK_PARAM_DRIVER(STMMAC_DEVLINK_PARAM_ID_TS_COARSE, "phc_coarse_adj", 7582 DEVLINK_PARAM_TYPE_BOOL, 7583 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 7584 stmmac_dl_ts_coarse_get, 7585 stmmac_dl_ts_coarse_set, NULL), 7586}; 7587 7588/* None of the generic devlink parameters are implemented */ 7589static const struct devlink_ops stmmac_devlink_ops = {}; 7590 7591static int stmmac_register_devlink(struct stmmac_priv *priv) 7592{ 7593 struct stmmac_devlink_priv *dl_priv; 7594 int ret; 7595 7596 /* For now, what is exposed over devlink is only relevant when 7597 * timestamping is available and we have a valid ptp clock rate 7598 */ 7599 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp) || 7600 !priv->plat->clk_ptp_rate) 7601 return 0; 7602 7603 priv->devlink = devlink_alloc(&stmmac_devlink_ops, sizeof(*dl_priv), 7604 priv->device); 7605 if (!priv->devlink) 7606 return -ENOMEM; 7607 7608 dl_priv = devlink_priv(priv->devlink); 7609 dl_priv->stmmac_priv = priv; 7610 7611 ret = devlink_params_register(priv->devlink, stmmac_devlink_params, 7612 ARRAY_SIZE(stmmac_devlink_params)); 7613 if (ret) 7614 goto dl_free; 7615 7616 devlink_register(priv->devlink); 7617 return 0; 7618 7619dl_free: 7620 devlink_free(priv->devlink); 7621 7622 return ret; 7623} 7624 7625static void stmmac_unregister_devlink(struct stmmac_priv *priv) 7626{ 7627 if (!priv->devlink) 7628 return; 7629 7630 devlink_unregister(priv->devlink); 7631 devlink_params_unregister(priv->devlink, stmmac_devlink_params, 7632 ARRAY_SIZE(stmmac_devlink_params)); 7633 devlink_free(priv->devlink); 7634} 7635 7636struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev) 7637{ 7638 struct plat_stmmacenet_data *plat_dat; 7639 int i; 7640 7641 plat_dat = devm_kzalloc(dev, sizeof(*plat_dat), GFP_KERNEL); 7642 if (!plat_dat) 7643 return NULL; 7644 7645 /* Set the defaults: 7646 * - phy autodetection 7647 * - determine GMII_Address CR field from CSR clock 7648 * - allow MTU up to JUMBO_LEN 7649 * - hash table size 7650 * - one unicast filter entry 7651 */ 7652 plat_dat->phy_addr = -1; 7653 plat_dat->clk_csr = -1; 7654 plat_dat->maxmtu = JUMBO_LEN; 7655 plat_dat->multicast_filter_bins = HASH_TABLE_SIZE; 7656 plat_dat->unicast_filter_entries = 1; 7657 7658 /* Set the mtl defaults */ 7659 plat_dat->tx_queues_to_use = 1; 7660 plat_dat->rx_queues_to_use = 1; 7661 7662 /* Setup the default RX queue channel map */ 7663 for (i = 0; i < ARRAY_SIZE(plat_dat->rx_queues_cfg); i++) 7664 plat_dat->rx_queues_cfg[i].chan = i; 7665 7666 return plat_dat; 7667} 7668EXPORT_SYMBOL_GPL(stmmac_plat_dat_alloc); 7669 7670static int __stmmac_dvr_probe(struct device *device, 7671 struct plat_stmmacenet_data *plat_dat, 7672 struct stmmac_resources *res) 7673{ 7674 struct net_device *ndev = NULL; 7675 struct stmmac_priv *priv; 7676 u32 rxq; 7677 int i, ret = 0; 7678 7679 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), 7680 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); 7681 if (!ndev) 7682 return -ENOMEM; 7683 7684 SET_NETDEV_DEV(ndev, device); 7685 7686 priv = netdev_priv(ndev); 7687 priv->device = device; 7688 priv->dev = ndev; 7689 7690 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7691 u64_stats_init(&priv->xstats.rxq_stats[i].napi_syncp); 7692 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) { 7693 u64_stats_init(&priv->xstats.txq_stats[i].q_syncp); 7694 u64_stats_init(&priv->xstats.txq_stats[i].napi_syncp); 7695 } 7696 7697 priv->xstats.pcpu_stats = 7698 devm_netdev_alloc_pcpu_stats(device, struct stmmac_pcpu_stats); 7699 if (!priv->xstats.pcpu_stats) 7700 return -ENOMEM; 7701 7702 stmmac_set_ethtool_ops(ndev); 7703 priv->pause_time = pause; 7704 priv->plat = plat_dat; 7705 priv->ioaddr = res->addr; 7706 priv->dev->base_addr = (unsigned long)res->addr; 7707 priv->plat->dma_cfg->multi_msi_en = 7708 (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN); 7709 7710 priv->dev->irq = res->irq; 7711 priv->wol_irq = res->wol_irq; 7712 priv->sfty_irq = res->sfty_irq; 7713 priv->sfty_ce_irq = res->sfty_ce_irq; 7714 priv->sfty_ue_irq = res->sfty_ue_irq; 7715 for (i = 0; i < MTL_MAX_RX_QUEUES; i++) 7716 priv->rx_irq[i] = res->rx_irq[i]; 7717 for (i = 0; i < MTL_MAX_TX_QUEUES; i++) 7718 priv->tx_irq[i] = res->tx_irq[i]; 7719 7720 if (!is_zero_ether_addr(res->mac)) 7721 eth_hw_addr_set(priv->dev, res->mac); 7722 7723 dev_set_drvdata(device, priv->dev); 7724 7725 /* Verify driver arguments */ 7726 stmmac_verify_args(); 7727 7728 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL); 7729 if (!priv->af_xdp_zc_qps) 7730 return -ENOMEM; 7731 7732 /* Allocate workqueue */ 7733 priv->wq = create_singlethread_workqueue("stmmac_wq"); 7734 if (!priv->wq) { 7735 dev_err(priv->device, "failed to create workqueue\n"); 7736 ret = -ENOMEM; 7737 goto error_wq_init; 7738 } 7739 7740 INIT_WORK(&priv->service_task, stmmac_service_task); 7741 7742 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0); 7743 7744 /* Override with kernel parameters if supplied XXX CRS XXX 7745 * this needs to have multiple instances 7746 */ 7747 if ((phyaddr >= 0) && (phyaddr <= 31)) 7748 priv->plat->phy_addr = phyaddr; 7749 7750 if (priv->plat->stmmac_rst) { 7751 ret = reset_control_assert(priv->plat->stmmac_rst); 7752 reset_control_deassert(priv->plat->stmmac_rst); 7753 /* Some reset controllers have only reset callback instead of 7754 * assert + deassert callbacks pair. 7755 */ 7756 if (ret == -ENOTSUPP) 7757 reset_control_reset(priv->plat->stmmac_rst); 7758 } 7759 7760 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst); 7761 if (ret == -ENOTSUPP) 7762 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n", 7763 ERR_PTR(ret)); 7764 7765 /* Wait a bit for the reset to take effect */ 7766 udelay(10); 7767 7768 /* Init MAC and get the capabilities */ 7769 ret = stmmac_hw_init(priv); 7770 if (ret) 7771 goto error_hw_init; 7772 7773 /* Only DWMAC core version 5.20 onwards supports HW descriptor prefetch. 7774 */ 7775 if (priv->synopsys_id < DWMAC_CORE_5_20) 7776 priv->plat->dma_cfg->dche = false; 7777 7778 stmmac_check_ether_addr(priv); 7779 7780 ndev->netdev_ops = &stmmac_netdev_ops; 7781 7782 ndev->xdp_metadata_ops = &stmmac_xdp_metadata_ops; 7783 ndev->xsk_tx_metadata_ops = &stmmac_xsk_tx_metadata_ops; 7784 7785 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 7786 NETIF_F_RXCSUM; 7787 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 7788 NETDEV_XDP_ACT_XSK_ZEROCOPY; 7789 7790 ret = stmmac_tc_init(priv, priv); 7791 if (!ret) { 7792 ndev->hw_features |= NETIF_F_HW_TC; 7793 } 7794 7795 if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { 7796 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 7797 if (priv->plat->core_type == DWMAC_CORE_GMAC4) 7798 ndev->hw_features |= NETIF_F_GSO_UDP_L4; 7799 priv->tso = true; 7800 dev_info(priv->device, "TSO feature enabled\n"); 7801 } 7802 7803 if (priv->dma_cap.sphen && 7804 !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { 7805 ndev->hw_features |= NETIF_F_GRO; 7806 priv->sph_capable = true; 7807 priv->sph_active = priv->sph_capable; 7808 dev_info(priv->device, "SPH feature enabled\n"); 7809 } 7810 7811 /* Ideally our host DMA address width is the same as for the 7812 * device. However, it may differ and then we have to use our 7813 * host DMA width for allocation and the device DMA width for 7814 * register handling. 7815 */ 7816 if (priv->plat->host_dma_width) 7817 priv->dma_cap.host_dma_width = priv->plat->host_dma_width; 7818 else 7819 priv->dma_cap.host_dma_width = priv->dma_cap.addr64; 7820 7821 if (priv->dma_cap.host_dma_width) { 7822 ret = dma_set_mask_and_coherent(device, 7823 DMA_BIT_MASK(priv->dma_cap.host_dma_width)); 7824 if (!ret) { 7825 dev_info(priv->device, "Using %d/%d bits DMA host/device width\n", 7826 priv->dma_cap.host_dma_width, priv->dma_cap.addr64); 7827 7828 /* 7829 * If more than 32 bits can be addressed, make sure to 7830 * enable enhanced addressing mode. 7831 */ 7832 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) 7833 priv->plat->dma_cfg->eame = true; 7834 } else { 7835 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32)); 7836 if (ret) { 7837 dev_err(priv->device, "Failed to set DMA Mask\n"); 7838 goto error_hw_init; 7839 } 7840 7841 priv->dma_cap.host_dma_width = 32; 7842 } 7843 } 7844 7845 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA; 7846 ndev->watchdog_timeo = msecs_to_jiffies(watchdog); 7847#ifdef STMMAC_VLAN_TAG_USED 7848 /* Both mac100 and gmac support receive VLAN tag detection */ 7849 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; 7850 if (dwmac_is_xmac(priv->plat->core_type)) { 7851 ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 7852 priv->hw->hw_vlan_en = true; 7853 } 7854 if (priv->dma_cap.vlhash) { 7855 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 7856 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 7857 } 7858 if (priv->dma_cap.vlins) 7859 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; 7860#endif 7861 priv->msg_enable = netif_msg_init(debug, default_msg_level); 7862 7863 priv->xstats.threshold = tc; 7864 7865 /* Initialize RSS */ 7866 rxq = priv->plat->rx_queues_to_use; 7867 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key)); 7868 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++) 7869 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq); 7870 7871 if (priv->dma_cap.rssen && priv->plat->rss_en) 7872 ndev->features |= NETIF_F_RXHASH; 7873 7874 ndev->vlan_features |= ndev->features; 7875 7876 /* MTU range: 46 - hw-specific max */ 7877 ndev->min_mtu = ETH_ZLEN - ETH_HLEN; 7878 7879 if (priv->plat->core_type == DWMAC_CORE_XGMAC) 7880 ndev->max_mtu = XGMAC_JUMBO_LEN; 7881 else if (priv->plat->enh_desc || priv->synopsys_id >= DWMAC_CORE_4_00) 7882 ndev->max_mtu = JUMBO_LEN; 7883 else 7884 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN); 7885 7886 /* Warn if the platform's maxmtu is smaller than the minimum MTU, 7887 * otherwise clamp the maximum MTU above to the platform's maxmtu. 7888 */ 7889 if (priv->plat->maxmtu < ndev->min_mtu) 7890 dev_warn(priv->device, 7891 "%s: warning: maxmtu having invalid value (%d)\n", 7892 __func__, priv->plat->maxmtu); 7893 else if (priv->plat->maxmtu < ndev->max_mtu) 7894 ndev->max_mtu = priv->plat->maxmtu; 7895 7896 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 7897 7898 /* Setup channels NAPI */ 7899 stmmac_napi_add(ndev); 7900 7901 mutex_init(&priv->lock); 7902 7903 stmmac_fpe_init(priv); 7904 7905 stmmac_check_pcs_mode(priv); 7906 7907 pm_runtime_get_noresume(device); 7908 pm_runtime_set_active(device); 7909 if (!pm_runtime_enabled(device)) 7910 pm_runtime_enable(device); 7911 7912 ret = stmmac_mdio_register(ndev); 7913 if (ret < 0) { 7914 dev_err_probe(priv->device, ret, 7915 "MDIO bus (id: %d) registration failed\n", 7916 priv->plat->bus_id); 7917 goto error_mdio_register; 7918 } 7919 7920 ret = stmmac_pcs_setup(ndev); 7921 if (ret) 7922 goto error_pcs_setup; 7923 7924 ret = stmmac_phylink_setup(priv); 7925 if (ret) { 7926 netdev_err(ndev, "failed to setup phy (%d)\n", ret); 7927 goto error_phy_setup; 7928 } 7929 7930 ret = stmmac_register_devlink(priv); 7931 if (ret) 7932 goto error_devlink_setup; 7933 7934 ret = register_netdev(ndev); 7935 if (ret) { 7936 dev_err(priv->device, "%s: ERROR %i registering the device\n", 7937 __func__, ret); 7938 goto error_netdev_register; 7939 } 7940 7941#ifdef CONFIG_DEBUG_FS 7942 stmmac_init_fs(ndev); 7943#endif 7944 7945 if (priv->plat->dump_debug_regs) 7946 priv->plat->dump_debug_regs(priv->plat->bsp_priv); 7947 7948 /* Let pm_runtime_put() disable the clocks. 7949 * If CONFIG_PM is not enabled, the clocks will stay powered. 7950 */ 7951 pm_runtime_put(device); 7952 7953 return ret; 7954 7955error_netdev_register: 7956 stmmac_unregister_devlink(priv); 7957error_devlink_setup: 7958 phylink_destroy(priv->phylink); 7959error_phy_setup: 7960 stmmac_pcs_clean(ndev); 7961error_pcs_setup: 7962 stmmac_mdio_unregister(ndev); 7963error_mdio_register: 7964 stmmac_napi_del(ndev); 7965error_hw_init: 7966 destroy_workqueue(priv->wq); 7967error_wq_init: 7968 bitmap_free(priv->af_xdp_zc_qps); 7969 7970 return ret; 7971} 7972 7973/** 7974 * stmmac_dvr_probe 7975 * @dev: device pointer 7976 * @plat_dat: platform data pointer 7977 * @res: stmmac resource pointer 7978 * Description: this is the main probe function used to 7979 * call the alloc_etherdev, allocate the priv structure. 7980 * Return: 7981 * returns 0 on success, otherwise errno. 7982 */ 7983int stmmac_dvr_probe(struct device *dev, struct plat_stmmacenet_data *plat_dat, 7984 struct stmmac_resources *res) 7985{ 7986 int ret; 7987 7988 if (plat_dat->init) { 7989 ret = plat_dat->init(dev, plat_dat->bsp_priv); 7990 if (ret) 7991 return ret; 7992 } 7993 7994 ret = __stmmac_dvr_probe(dev, plat_dat, res); 7995 if (ret && plat_dat->exit) 7996 plat_dat->exit(dev, plat_dat->bsp_priv); 7997 7998 return ret; 7999} 8000EXPORT_SYMBOL_GPL(stmmac_dvr_probe); 8001 8002/** 8003 * stmmac_dvr_remove 8004 * @dev: device pointer 8005 * Description: this function resets the TX/RX processes, disables the MAC RX/TX 8006 * changes the link status, releases the DMA descriptor rings. 8007 */ 8008void stmmac_dvr_remove(struct device *dev) 8009{ 8010 struct net_device *ndev = dev_get_drvdata(dev); 8011 struct stmmac_priv *priv = netdev_priv(ndev); 8012 8013 netdev_info(priv->dev, "%s: removing driver", __func__); 8014 8015 pm_runtime_get_sync(dev); 8016 8017 unregister_netdev(ndev); 8018 8019#ifdef CONFIG_DEBUG_FS 8020 stmmac_exit_fs(ndev); 8021#endif 8022 stmmac_unregister_devlink(priv); 8023 8024 phylink_destroy(priv->phylink); 8025 if (priv->plat->stmmac_rst) 8026 reset_control_assert(priv->plat->stmmac_rst); 8027 reset_control_assert(priv->plat->stmmac_ahb_rst); 8028 8029 stmmac_pcs_clean(ndev); 8030 stmmac_mdio_unregister(ndev); 8031 8032 destroy_workqueue(priv->wq); 8033 mutex_destroy(&priv->lock); 8034 bitmap_free(priv->af_xdp_zc_qps); 8035 8036 pm_runtime_disable(dev); 8037 pm_runtime_put_noidle(dev); 8038 8039 if (priv->plat->exit) 8040 priv->plat->exit(dev, priv->plat->bsp_priv); 8041} 8042EXPORT_SYMBOL_GPL(stmmac_dvr_remove); 8043 8044/** 8045 * stmmac_suspend - suspend callback 8046 * @dev: device pointer 8047 * Description: this is the function to suspend the device and it is called 8048 * by the platform driver to stop the network queue, release the resources, 8049 * program the PMT register (for WoL), clean and release driver resources. 8050 */ 8051int stmmac_suspend(struct device *dev) 8052{ 8053 struct net_device *ndev = dev_get_drvdata(dev); 8054 struct stmmac_priv *priv = netdev_priv(ndev); 8055 u32 chan; 8056 8057 if (!ndev || !netif_running(ndev)) 8058 goto suspend_bsp; 8059 8060 mutex_lock(&priv->lock); 8061 8062 netif_device_detach(ndev); 8063 8064 stmmac_disable_all_queues(priv); 8065 8066 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) 8067 hrtimer_cancel(&priv->dma_conf.tx_queue[chan].txtimer); 8068 8069 if (priv->eee_sw_timer_en) { 8070 priv->tx_path_in_lpi_mode = false; 8071 timer_delete_sync(&priv->eee_ctrl_timer); 8072 } 8073 8074 /* Stop TX/RX DMA */ 8075 stmmac_stop_all_dma(priv); 8076 8077 if (priv->plat->serdes_powerdown) 8078 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv); 8079 8080 /* Enable Power down mode by programming the PMT regs */ 8081 if (priv->wolopts) { 8082 stmmac_pmt(priv, priv->hw, priv->wolopts); 8083 priv->irq_wake = 1; 8084 } else { 8085 stmmac_mac_set(priv, priv->ioaddr, false); 8086 pinctrl_pm_select_sleep_state(priv->device); 8087 } 8088 8089 mutex_unlock(&priv->lock); 8090 8091 rtnl_lock(); 8092 phylink_suspend(priv->phylink, !!priv->wolopts); 8093 rtnl_unlock(); 8094 8095 if (stmmac_fpe_supported(priv)) 8096 ethtool_mmsv_stop(&priv->fpe_cfg.mmsv); 8097 8098suspend_bsp: 8099 if (priv->plat->suspend) 8100 return priv->plat->suspend(dev, priv->plat->bsp_priv); 8101 8102 return 0; 8103} 8104EXPORT_SYMBOL_GPL(stmmac_suspend); 8105 8106static void stmmac_reset_rx_queue(struct stmmac_priv *priv, u32 queue) 8107{ 8108 struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; 8109 8110 rx_q->cur_rx = 0; 8111 rx_q->dirty_rx = 0; 8112} 8113 8114static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) 8115{ 8116 struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; 8117 8118 tx_q->cur_tx = 0; 8119 tx_q->dirty_tx = 0; 8120 tx_q->mss = 0; 8121 8122 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue)); 8123} 8124 8125/** 8126 * stmmac_reset_queues_param - reset queue parameters 8127 * @priv: device pointer 8128 */ 8129static void stmmac_reset_queues_param(struct stmmac_priv *priv) 8130{ 8131 u32 rx_cnt = priv->plat->rx_queues_to_use; 8132 u32 tx_cnt = priv->plat->tx_queues_to_use; 8133 u32 queue; 8134 8135 for (queue = 0; queue < rx_cnt; queue++) 8136 stmmac_reset_rx_queue(priv, queue); 8137 8138 for (queue = 0; queue < tx_cnt; queue++) 8139 stmmac_reset_tx_queue(priv, queue); 8140} 8141 8142/** 8143 * stmmac_resume - resume callback 8144 * @dev: device pointer 8145 * Description: when resume this function is invoked to setup the DMA and CORE 8146 * in a usable state. 8147 */ 8148int stmmac_resume(struct device *dev) 8149{ 8150 struct net_device *ndev = dev_get_drvdata(dev); 8151 struct stmmac_priv *priv = netdev_priv(ndev); 8152 int ret; 8153 8154 if (priv->plat->resume) { 8155 ret = priv->plat->resume(dev, priv->plat->bsp_priv); 8156 if (ret) 8157 return ret; 8158 } 8159 8160 if (!netif_running(ndev)) 8161 return 0; 8162 8163 /* Power Down bit, into the PM register, is cleared 8164 * automatically as soon as a magic packet or a Wake-up frame 8165 * is received. Anyway, it's better to manually clear 8166 * this bit because it can generate problems while resuming 8167 * from another devices (e.g. serial console). 8168 */ 8169 if (priv->wolopts) { 8170 mutex_lock(&priv->lock); 8171 stmmac_pmt(priv, priv->hw, 0); 8172 mutex_unlock(&priv->lock); 8173 priv->irq_wake = 0; 8174 } else { 8175 pinctrl_pm_select_default_state(priv->device); 8176 /* reset the phy so that it's ready */ 8177 if (priv->mii) 8178 stmmac_mdio_reset(priv->mii); 8179 } 8180 8181 if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && 8182 priv->plat->serdes_powerup) { 8183 ret = priv->plat->serdes_powerup(ndev, 8184 priv->plat->bsp_priv); 8185 8186 if (ret < 0) 8187 return ret; 8188 } 8189 8190 rtnl_lock(); 8191 8192 /* Prepare the PHY to resume, ensuring that its clocks which are 8193 * necessary for the MAC DMA reset to complete are running 8194 */ 8195 phylink_prepare_resume(priv->phylink); 8196 8197 mutex_lock(&priv->lock); 8198 8199 stmmac_reset_queues_param(priv); 8200 8201 stmmac_free_tx_skbufs(priv); 8202 stmmac_clear_descriptors(priv, &priv->dma_conf); 8203 8204 ret = stmmac_hw_setup(ndev); 8205 if (ret < 0) { 8206 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__); 8207 mutex_unlock(&priv->lock); 8208 rtnl_unlock(); 8209 return ret; 8210 } 8211 8212 stmmac_init_timestamping(priv); 8213 8214 stmmac_init_coalesce(priv); 8215 phylink_rx_clk_stop_block(priv->phylink); 8216 stmmac_set_rx_mode(ndev); 8217 phylink_rx_clk_stop_unblock(priv->phylink); 8218 8219 stmmac_vlan_restore(priv); 8220 8221 stmmac_enable_all_queues(priv); 8222 stmmac_enable_all_dma_irq(priv); 8223 8224 mutex_unlock(&priv->lock); 8225 8226 /* phylink_resume() must be called after the hardware has been 8227 * initialised because it may bring the link up immediately in a 8228 * workqueue thread, which will race with initialisation. 8229 */ 8230 phylink_resume(priv->phylink); 8231 rtnl_unlock(); 8232 8233 netif_device_attach(ndev); 8234 8235 return 0; 8236} 8237EXPORT_SYMBOL_GPL(stmmac_resume); 8238 8239/* This is not the same as EXPORT_GPL_SIMPLE_DEV_PM_OPS() when CONFIG_PM=n */ 8240DEFINE_SIMPLE_DEV_PM_OPS(stmmac_simple_pm_ops, stmmac_suspend, stmmac_resume); 8241EXPORT_SYMBOL_GPL(stmmac_simple_pm_ops); 8242 8243#ifndef MODULE 8244static int __init stmmac_cmdline_opt(char *str) 8245{ 8246 char *opt; 8247 8248 if (!str || !*str) 8249 return 1; 8250 while ((opt = strsep(&str, ",")) != NULL) { 8251 if (!strncmp(opt, "debug:", 6)) { 8252 if (kstrtoint(opt + 6, 0, &debug)) 8253 goto err; 8254 } else if (!strncmp(opt, "phyaddr:", 8)) { 8255 if (kstrtoint(opt + 8, 0, &phyaddr)) 8256 goto err; 8257 } else if (!strncmp(opt, "tc:", 3)) { 8258 if (kstrtoint(opt + 3, 0, &tc)) 8259 goto err; 8260 } else if (!strncmp(opt, "watchdog:", 9)) { 8261 if (kstrtoint(opt + 9, 0, &watchdog)) 8262 goto err; 8263 } else if (!strncmp(opt, "flow_ctrl:", 10)) { 8264 if (kstrtoint(opt + 10, 0, &flow_ctrl)) 8265 goto err; 8266 } else if (!strncmp(opt, "pause:", 6)) { 8267 if (kstrtoint(opt + 6, 0, &pause)) 8268 goto err; 8269 } else if (!strncmp(opt, "eee_timer:", 10)) { 8270 if (kstrtoint(opt + 10, 0, &eee_timer)) 8271 goto err; 8272 } else if (!strncmp(opt, "chain_mode:", 11)) { 8273 if (kstrtoint(opt + 11, 0, &chain_mode)) 8274 goto err; 8275 } 8276 } 8277 return 1; 8278 8279err: 8280 pr_err("%s: ERROR broken module parameter conversion", __func__); 8281 return 1; 8282} 8283 8284__setup("stmmaceth=", stmmac_cmdline_opt); 8285#endif /* MODULE */ 8286 8287static int __init stmmac_init(void) 8288{ 8289#ifdef CONFIG_DEBUG_FS 8290 /* Create debugfs main directory if it doesn't exist yet */ 8291 if (!stmmac_fs_dir) 8292 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL); 8293 register_netdevice_notifier(&stmmac_notifier); 8294#endif 8295 8296 return 0; 8297} 8298 8299static void __exit stmmac_exit(void) 8300{ 8301#ifdef CONFIG_DEBUG_FS 8302 unregister_netdevice_notifier(&stmmac_notifier); 8303 debugfs_remove_recursive(stmmac_fs_dir); 8304#endif 8305} 8306 8307module_init(stmmac_init) 8308module_exit(stmmac_exit) 8309 8310MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver"); 8311MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>"); 8312MODULE_LICENSE("GPL");
~/
Read-only file preview.