FILE
macb_main.c
File preview with syntax highlighting.
linux-6.19.14/drivers/net/ethernet/cadence/macb_main.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Cadence MACB/GEM Ethernet Controller driver 4 * 5 * Copyright (C) 2004-2006 Atmel Corporation 6 */ 7 8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9#include <linux/circ_buf.h> 10#include <linux/clk-provider.h> 11#include <linux/clk.h> 12#include <linux/crc32.h> 13#include <linux/dma-mapping.h> 14#include <linux/etherdevice.h> 15#include <linux/firmware/xlnx-zynqmp.h> 16#include <linux/inetdevice.h> 17#include <linux/init.h> 18#include <linux/interrupt.h> 19#include <linux/io.h> 20#include <linux/iopoll.h> 21#include <linux/ip.h> 22#include <linux/kernel.h> 23#include <linux/module.h> 24#include <linux/moduleparam.h> 25#include <linux/netdevice.h> 26#include <linux/of.h> 27#include <linux/of_mdio.h> 28#include <linux/of_net.h> 29#include <linux/phy/phy.h> 30#include <linux/phylink.h> 31#include <linux/platform_device.h> 32#include <linux/pm_runtime.h> 33#include <linux/ptp_classify.h> 34#include <linux/reset.h> 35#include <linux/slab.h> 36#include <linux/tcp.h> 37#include <linux/types.h> 38#include <linux/udp.h> 39#include <linux/gcd.h> 40#include <net/pkt_sched.h> 41#include "macb.h" 42 43/* This structure is only used for MACB on SiFive FU540 devices */ 44struct sifive_fu540_macb_mgmt { 45 void __iomem *reg; 46 unsigned long rate; 47 struct clk_hw hw; 48}; 49 50#define MACB_RX_BUFFER_SIZE 128 51#define RX_BUFFER_MULTIPLE 64 /* bytes */ 52 53#define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ 54#define MIN_RX_RING_SIZE 64 55#define MAX_RX_RING_SIZE 8192 56 57#define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */ 58#define MIN_TX_RING_SIZE 64 59#define MAX_TX_RING_SIZE 4096 60 61/* level of occupied TX descriptors under which we wake up TX process */ 62#define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4) 63 64#define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR)) 65#define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \ 66 | MACB_BIT(ISR_RLE) \ 67 | MACB_BIT(TXERR)) 68#define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ 69 | MACB_BIT(TXUBR)) 70 71/* Max length of transmit frame must be a multiple of 8 bytes */ 72#define MACB_TX_LEN_ALIGN 8 73#define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) 74/* Limit maximum TX length as per Cadence TSO errata. This is to avoid a 75 * false amba_error in TX path from the DMA assuming there is not enough 76 * space in the SRAM (16KB) even when there is. 77 */ 78#define GEM_MAX_TX_LEN (unsigned int)(0x3FC0) 79 80#define GEM_MTU_MIN_SIZE ETH_MIN_MTU 81#define MACB_NETIF_LSO NETIF_F_TSO 82 83#define MACB_WOL_ENABLED BIT(0) 84 85#define HS_SPEED_10000M 4 86#define MACB_SERDES_RATE_10G 1 87 88/* Graceful stop timeouts in us. We should allow up to 89 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions) 90 */ 91#define MACB_HALT_TIMEOUT 14000 92#define MACB_PM_TIMEOUT 100 /* ms */ 93 94#define MACB_MDIO_TIMEOUT 1000000 /* in usecs */ 95 96/* DMA buffer descriptor might be different size 97 * depends on hardware configuration: 98 * 99 * 1. dma address width 32 bits: 100 * word 1: 32 bit address of Data Buffer 101 * word 2: control 102 * 103 * 2. dma address width 64 bits: 104 * word 1: 32 bit address of Data Buffer 105 * word 2: control 106 * word 3: upper 32 bit address of Data Buffer 107 * word 4: unused 108 * 109 * 3. dma address width 32 bits with hardware timestamping: 110 * word 1: 32 bit address of Data Buffer 111 * word 2: control 112 * word 3: timestamp word 1 113 * word 4: timestamp word 2 114 * 115 * 4. dma address width 64 bits with hardware timestamping: 116 * word 1: 32 bit address of Data Buffer 117 * word 2: control 118 * word 3: upper 32 bit address of Data Buffer 119 * word 4: unused 120 * word 5: timestamp word 1 121 * word 6: timestamp word 2 122 */ 123static unsigned int macb_dma_desc_get_size(struct macb *bp) 124{ 125 unsigned int desc_size = sizeof(struct macb_dma_desc); 126 127 if (macb_dma64(bp)) 128 desc_size += sizeof(struct macb_dma_desc_64); 129 if (macb_dma_ptp(bp)) 130 desc_size += sizeof(struct macb_dma_desc_ptp); 131 132 return desc_size; 133} 134 135static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx) 136{ 137 return desc_idx * (1 + macb_dma64(bp) + macb_dma_ptp(bp)); 138} 139 140static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc) 141{ 142 return (struct macb_dma_desc_64 *)((void *)desc 143 + sizeof(struct macb_dma_desc)); 144} 145 146/* Ring buffer accessors */ 147static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index) 148{ 149 return index & (bp->tx_ring_size - 1); 150} 151 152static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, 153 unsigned int index) 154{ 155 index = macb_tx_ring_wrap(queue->bp, index); 156 index = macb_adj_dma_desc_idx(queue->bp, index); 157 return &queue->tx_ring[index]; 158} 159 160static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, 161 unsigned int index) 162{ 163 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; 164} 165 166static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) 167{ 168 dma_addr_t offset; 169 170 offset = macb_tx_ring_wrap(queue->bp, index) * 171 macb_dma_desc_get_size(queue->bp); 172 173 return queue->tx_ring_dma + offset; 174} 175 176static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index) 177{ 178 return index & (bp->rx_ring_size - 1); 179} 180 181static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index) 182{ 183 index = macb_rx_ring_wrap(queue->bp, index); 184 index = macb_adj_dma_desc_idx(queue->bp, index); 185 return &queue->rx_ring[index]; 186} 187 188static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index) 189{ 190 return queue->rx_buffers + queue->bp->rx_buffer_size * 191 macb_rx_ring_wrap(queue->bp, index); 192} 193 194/* I/O accessors */ 195static u32 hw_readl_native(struct macb *bp, int offset) 196{ 197 return __raw_readl(bp->regs + offset); 198} 199 200static void hw_writel_native(struct macb *bp, int offset, u32 value) 201{ 202 __raw_writel(value, bp->regs + offset); 203} 204 205static u32 hw_readl(struct macb *bp, int offset) 206{ 207 return readl_relaxed(bp->regs + offset); 208} 209 210static void hw_writel(struct macb *bp, int offset, u32 value) 211{ 212 writel_relaxed(value, bp->regs + offset); 213} 214 215/* Find the CPU endianness by using the loopback bit of NCR register. When the 216 * CPU is in big endian we need to program swapped mode for management 217 * descriptor access. 218 */ 219static bool hw_is_native_io(void __iomem *addr) 220{ 221 u32 value = MACB_BIT(LLB); 222 223 __raw_writel(value, addr + MACB_NCR); 224 value = __raw_readl(addr + MACB_NCR); 225 226 /* Write 0 back to disable everything */ 227 __raw_writel(0, addr + MACB_NCR); 228 229 return value == MACB_BIT(LLB); 230} 231 232static bool hw_is_gem(void __iomem *addr, bool native_io) 233{ 234 u32 id; 235 236 if (native_io) 237 id = __raw_readl(addr + MACB_MID); 238 else 239 id = readl_relaxed(addr + MACB_MID); 240 241 return MACB_BFEXT(IDNUM, id) >= 0x2; 242} 243 244static void macb_set_hwaddr(struct macb *bp) 245{ 246 u32 bottom; 247 u16 top; 248 249 bottom = get_unaligned_le32(bp->dev->dev_addr); 250 macb_or_gem_writel(bp, SA1B, bottom); 251 top = get_unaligned_le16(bp->dev->dev_addr + 4); 252 macb_or_gem_writel(bp, SA1T, top); 253 254 if (gem_has_ptp(bp)) { 255 gem_writel(bp, RXPTPUNI, bottom); 256 gem_writel(bp, TXPTPUNI, bottom); 257 } 258 259 /* Clear unused address register sets */ 260 macb_or_gem_writel(bp, SA2B, 0); 261 macb_or_gem_writel(bp, SA2T, 0); 262 macb_or_gem_writel(bp, SA3B, 0); 263 macb_or_gem_writel(bp, SA3T, 0); 264 macb_or_gem_writel(bp, SA4B, 0); 265 macb_or_gem_writel(bp, SA4T, 0); 266} 267 268static void macb_get_hwaddr(struct macb *bp) 269{ 270 u32 bottom; 271 u16 top; 272 u8 addr[6]; 273 int i; 274 275 /* Check all 4 address register for valid address */ 276 for (i = 0; i < 4; i++) { 277 bottom = macb_or_gem_readl(bp, SA1B + i * 8); 278 top = macb_or_gem_readl(bp, SA1T + i * 8); 279 280 addr[0] = bottom & 0xff; 281 addr[1] = (bottom >> 8) & 0xff; 282 addr[2] = (bottom >> 16) & 0xff; 283 addr[3] = (bottom >> 24) & 0xff; 284 addr[4] = top & 0xff; 285 addr[5] = (top >> 8) & 0xff; 286 287 if (is_valid_ether_addr(addr)) { 288 eth_hw_addr_set(bp->dev, addr); 289 return; 290 } 291 } 292 293 dev_info(&bp->pdev->dev, "invalid hw address, using random\n"); 294 eth_hw_addr_random(bp->dev); 295} 296 297static int macb_mdio_wait_for_idle(struct macb *bp) 298{ 299 u32 val; 300 301 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE), 302 1, MACB_MDIO_TIMEOUT); 303} 304 305static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum) 306{ 307 struct macb *bp = bus->priv; 308 int status; 309 310 status = pm_runtime_resume_and_get(&bp->pdev->dev); 311 if (status < 0) 312 goto mdio_pm_exit; 313 314 status = macb_mdio_wait_for_idle(bp); 315 if (status < 0) 316 goto mdio_read_exit; 317 318 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 319 | MACB_BF(RW, MACB_MAN_C22_READ) 320 | MACB_BF(PHYA, mii_id) 321 | MACB_BF(REGA, regnum) 322 | MACB_BF(CODE, MACB_MAN_C22_CODE))); 323 324 status = macb_mdio_wait_for_idle(bp); 325 if (status < 0) 326 goto mdio_read_exit; 327 328 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 329 330mdio_read_exit: 331 pm_runtime_put_autosuspend(&bp->pdev->dev); 332mdio_pm_exit: 333 return status; 334} 335 336static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad, 337 int regnum) 338{ 339 struct macb *bp = bus->priv; 340 int status; 341 342 status = pm_runtime_get_sync(&bp->pdev->dev); 343 if (status < 0) { 344 pm_runtime_put_noidle(&bp->pdev->dev); 345 goto mdio_pm_exit; 346 } 347 348 status = macb_mdio_wait_for_idle(bp); 349 if (status < 0) 350 goto mdio_read_exit; 351 352 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 353 | MACB_BF(RW, MACB_MAN_C45_ADDR) 354 | MACB_BF(PHYA, mii_id) 355 | MACB_BF(REGA, devad & 0x1F) 356 | MACB_BF(DATA, regnum & 0xFFFF) 357 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 358 359 status = macb_mdio_wait_for_idle(bp); 360 if (status < 0) 361 goto mdio_read_exit; 362 363 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 364 | MACB_BF(RW, MACB_MAN_C45_READ) 365 | MACB_BF(PHYA, mii_id) 366 | MACB_BF(REGA, devad & 0x1F) 367 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 368 369 status = macb_mdio_wait_for_idle(bp); 370 if (status < 0) 371 goto mdio_read_exit; 372 373 status = MACB_BFEXT(DATA, macb_readl(bp, MAN)); 374 375mdio_read_exit: 376 pm_runtime_put_autosuspend(&bp->pdev->dev); 377mdio_pm_exit: 378 return status; 379} 380 381static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum, 382 u16 value) 383{ 384 struct macb *bp = bus->priv; 385 int status; 386 387 status = pm_runtime_resume_and_get(&bp->pdev->dev); 388 if (status < 0) 389 goto mdio_pm_exit; 390 391 status = macb_mdio_wait_for_idle(bp); 392 if (status < 0) 393 goto mdio_write_exit; 394 395 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF) 396 | MACB_BF(RW, MACB_MAN_C22_WRITE) 397 | MACB_BF(PHYA, mii_id) 398 | MACB_BF(REGA, regnum) 399 | MACB_BF(CODE, MACB_MAN_C22_CODE) 400 | MACB_BF(DATA, value))); 401 402 status = macb_mdio_wait_for_idle(bp); 403 if (status < 0) 404 goto mdio_write_exit; 405 406mdio_write_exit: 407 pm_runtime_put_autosuspend(&bp->pdev->dev); 408mdio_pm_exit: 409 return status; 410} 411 412static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id, 413 int devad, int regnum, 414 u16 value) 415{ 416 struct macb *bp = bus->priv; 417 int status; 418 419 status = pm_runtime_get_sync(&bp->pdev->dev); 420 if (status < 0) { 421 pm_runtime_put_noidle(&bp->pdev->dev); 422 goto mdio_pm_exit; 423 } 424 425 status = macb_mdio_wait_for_idle(bp); 426 if (status < 0) 427 goto mdio_write_exit; 428 429 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 430 | MACB_BF(RW, MACB_MAN_C45_ADDR) 431 | MACB_BF(PHYA, mii_id) 432 | MACB_BF(REGA, devad & 0x1F) 433 | MACB_BF(DATA, regnum & 0xFFFF) 434 | MACB_BF(CODE, MACB_MAN_C45_CODE))); 435 436 status = macb_mdio_wait_for_idle(bp); 437 if (status < 0) 438 goto mdio_write_exit; 439 440 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF) 441 | MACB_BF(RW, MACB_MAN_C45_WRITE) 442 | MACB_BF(PHYA, mii_id) 443 | MACB_BF(REGA, devad & 0x1F) 444 | MACB_BF(CODE, MACB_MAN_C45_CODE) 445 | MACB_BF(DATA, value))); 446 447 status = macb_mdio_wait_for_idle(bp); 448 if (status < 0) 449 goto mdio_write_exit; 450 451mdio_write_exit: 452 pm_runtime_put_autosuspend(&bp->pdev->dev); 453mdio_pm_exit: 454 return status; 455} 456 457static void macb_init_buffers(struct macb *bp) 458{ 459 struct macb_queue *queue; 460 unsigned int q; 461 462 /* Single register for all queues' high 32 bits. */ 463 if (macb_dma64(bp)) { 464 macb_writel(bp, RBQPH, 465 upper_32_bits(bp->queues[0].rx_ring_dma)); 466 macb_writel(bp, TBQPH, 467 upper_32_bits(bp->queues[0].tx_ring_dma)); 468 } 469 470 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 471 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma)); 472 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 473 } 474} 475 476/** 477 * macb_set_tx_clk() - Set a clock to a new frequency 478 * @bp: pointer to struct macb 479 * @speed: New frequency in Hz 480 */ 481static void macb_set_tx_clk(struct macb *bp, int speed) 482{ 483 long ferr, rate, rate_rounded; 484 485 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG)) 486 return; 487 488 /* In case of MII the PHY is the clock master */ 489 if (bp->phy_interface == PHY_INTERFACE_MODE_MII) 490 return; 491 492 rate = rgmii_clock(speed); 493 if (rate < 0) 494 return; 495 496 rate_rounded = clk_round_rate(bp->tx_clk, rate); 497 if (rate_rounded < 0) 498 return; 499 500 /* RGMII allows 50 ppm frequency error. Test and warn if this limit 501 * is not satisfied. 502 */ 503 ferr = abs(rate_rounded - rate); 504 ferr = DIV_ROUND_UP(ferr, rate / 100000); 505 if (ferr > 5) 506 netdev_warn(bp->dev, 507 "unable to generate target frequency: %ld Hz\n", 508 rate); 509 510 if (clk_set_rate(bp->tx_clk, rate_rounded)) 511 netdev_err(bp->dev, "adjusting tx_clk failed.\n"); 512} 513 514static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode, 515 phy_interface_t interface, int speed, 516 int duplex) 517{ 518 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 519 u32 config; 520 521 config = gem_readl(bp, USX_CONTROL); 522 config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config); 523 config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config); 524 config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS)); 525 config |= GEM_BIT(TX_EN); 526 gem_writel(bp, USX_CONTROL, config); 527} 528 529static void macb_usx_pcs_get_state(struct phylink_pcs *pcs, 530 unsigned int neg_mode, 531 struct phylink_link_state *state) 532{ 533 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 534 u32 val; 535 536 state->speed = SPEED_10000; 537 state->duplex = 1; 538 state->an_complete = 1; 539 540 val = gem_readl(bp, USX_STATUS); 541 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK)); 542 val = gem_readl(bp, NCFGR); 543 if (val & GEM_BIT(PAE)) 544 state->pause = MLO_PAUSE_RX; 545} 546 547static int macb_usx_pcs_config(struct phylink_pcs *pcs, 548 unsigned int neg_mode, 549 phy_interface_t interface, 550 const unsigned long *advertising, 551 bool permit_pause_to_mac) 552{ 553 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs); 554 555 gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) | 556 GEM_BIT(SIGNAL_OK)); 557 558 return 0; 559} 560 561static void macb_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode, 562 struct phylink_link_state *state) 563{ 564 state->link = 0; 565} 566 567static void macb_pcs_an_restart(struct phylink_pcs *pcs) 568{ 569 /* Not supported */ 570} 571 572static int macb_pcs_config(struct phylink_pcs *pcs, 573 unsigned int neg_mode, 574 phy_interface_t interface, 575 const unsigned long *advertising, 576 bool permit_pause_to_mac) 577{ 578 return 0; 579} 580 581static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { 582 .pcs_get_state = macb_usx_pcs_get_state, 583 .pcs_config = macb_usx_pcs_config, 584 .pcs_link_up = macb_usx_pcs_link_up, 585}; 586 587static const struct phylink_pcs_ops macb_phylink_pcs_ops = { 588 .pcs_get_state = macb_pcs_get_state, 589 .pcs_an_restart = macb_pcs_an_restart, 590 .pcs_config = macb_pcs_config, 591}; 592 593static void macb_mac_config(struct phylink_config *config, unsigned int mode, 594 const struct phylink_link_state *state) 595{ 596 struct net_device *ndev = to_net_dev(config->dev); 597 struct macb *bp = netdev_priv(ndev); 598 unsigned long flags; 599 u32 old_ctrl, ctrl; 600 u32 old_ncr, ncr; 601 602 spin_lock_irqsave(&bp->lock, flags); 603 604 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR); 605 old_ncr = ncr = macb_or_gem_readl(bp, NCR); 606 607 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) { 608 if (state->interface == PHY_INTERFACE_MODE_RMII) 609 ctrl |= MACB_BIT(RM9200_RMII); 610 } else if (macb_is_gem(bp)) { 611 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL)); 612 ncr &= ~GEM_BIT(ENABLE_HS_MAC); 613 614 if (state->interface == PHY_INTERFACE_MODE_SGMII) { 615 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 616 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) { 617 ctrl |= GEM_BIT(PCSSEL); 618 ncr |= GEM_BIT(ENABLE_HS_MAC); 619 } else if (bp->caps & MACB_CAPS_MIIONRGMII && 620 bp->phy_interface == PHY_INTERFACE_MODE_MII) { 621 ncr |= MACB_BIT(MIIONRGMII); 622 } 623 } 624 625 /* Apply the new configuration, if any */ 626 if (old_ctrl ^ ctrl) 627 macb_or_gem_writel(bp, NCFGR, ctrl); 628 629 if (old_ncr ^ ncr) 630 macb_or_gem_writel(bp, NCR, ncr); 631 632 /* Disable AN for SGMII fixed link configuration, enable otherwise. 633 * Must be written after PCSSEL is set in NCFGR, 634 * otherwise writes will not take effect. 635 */ 636 if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) { 637 u32 pcsctrl, old_pcsctrl; 638 639 old_pcsctrl = gem_readl(bp, PCSCNTRL); 640 if (mode == MLO_AN_FIXED) 641 pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); 642 else 643 pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG); 644 if (old_pcsctrl != pcsctrl) 645 gem_writel(bp, PCSCNTRL, pcsctrl); 646 } 647 648 spin_unlock_irqrestore(&bp->lock, flags); 649} 650 651static void macb_mac_link_down(struct phylink_config *config, unsigned int mode, 652 phy_interface_t interface) 653{ 654 struct net_device *ndev = to_net_dev(config->dev); 655 struct macb *bp = netdev_priv(ndev); 656 struct macb_queue *queue; 657 unsigned int q; 658 u32 ctrl; 659 660 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 661 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 662 queue_writel(queue, IDR, 663 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 664 665 /* Disable Rx and Tx */ 666 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE)); 667 macb_writel(bp, NCR, ctrl); 668 669 netif_tx_stop_all_queues(ndev); 670} 671 672/* Use juggling algorithm to left rotate tx ring and tx skb array */ 673static void gem_shuffle_tx_one_ring(struct macb_queue *queue) 674{ 675 unsigned int head, tail, count, ring_size, desc_size; 676 struct macb_tx_skb tx_skb, *skb_curr, *skb_next; 677 struct macb_dma_desc *desc_curr, *desc_next; 678 unsigned int i, cycles, shift, curr, next; 679 struct macb *bp = queue->bp; 680 unsigned char desc[24]; 681 unsigned long flags; 682 683 desc_size = macb_dma_desc_get_size(bp); 684 685 if (WARN_ON_ONCE(desc_size > ARRAY_SIZE(desc))) 686 return; 687 688 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 689 head = queue->tx_head; 690 tail = queue->tx_tail; 691 ring_size = bp->tx_ring_size; 692 count = CIRC_CNT(head, tail, ring_size); 693 694 if (!(tail % ring_size)) 695 goto unlock; 696 697 if (!count) { 698 queue->tx_head = 0; 699 queue->tx_tail = 0; 700 goto unlock; 701 } 702 703 shift = tail % ring_size; 704 cycles = gcd(ring_size, shift); 705 706 for (i = 0; i < cycles; i++) { 707 memcpy(&desc, macb_tx_desc(queue, i), desc_size); 708 memcpy(&tx_skb, macb_tx_skb(queue, i), 709 sizeof(struct macb_tx_skb)); 710 711 curr = i; 712 next = (curr + shift) % ring_size; 713 714 while (next != i) { 715 desc_curr = macb_tx_desc(queue, curr); 716 desc_next = macb_tx_desc(queue, next); 717 718 memcpy(desc_curr, desc_next, desc_size); 719 720 if (next == ring_size - 1) 721 desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); 722 if (curr == ring_size - 1) 723 desc_curr->ctrl |= MACB_BIT(TX_WRAP); 724 725 skb_curr = macb_tx_skb(queue, curr); 726 skb_next = macb_tx_skb(queue, next); 727 memcpy(skb_curr, skb_next, sizeof(struct macb_tx_skb)); 728 729 curr = next; 730 next = (curr + shift) % ring_size; 731 } 732 733 desc_curr = macb_tx_desc(queue, curr); 734 memcpy(desc_curr, &desc, desc_size); 735 if (i == ring_size - 1) 736 desc_curr->ctrl &= ~MACB_BIT(TX_WRAP); 737 if (curr == ring_size - 1) 738 desc_curr->ctrl |= MACB_BIT(TX_WRAP); 739 memcpy(macb_tx_skb(queue, curr), &tx_skb, 740 sizeof(struct macb_tx_skb)); 741 } 742 743 queue->tx_head = count; 744 queue->tx_tail = 0; 745 746 /* Make descriptor updates visible to hardware */ 747 wmb(); 748 749unlock: 750 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 751} 752 753/* Rotate the queue so that the tail is at index 0 */ 754static void gem_shuffle_tx_rings(struct macb *bp) 755{ 756 struct macb_queue *queue; 757 int q; 758 759 for (q = 0, queue = bp->queues; q < bp->num_queues; q++, queue++) 760 gem_shuffle_tx_one_ring(queue); 761} 762 763static void macb_mac_link_up(struct phylink_config *config, 764 struct phy_device *phy, 765 unsigned int mode, phy_interface_t interface, 766 int speed, int duplex, 767 bool tx_pause, bool rx_pause) 768{ 769 struct net_device *ndev = to_net_dev(config->dev); 770 struct macb *bp = netdev_priv(ndev); 771 struct macb_queue *queue; 772 unsigned long flags; 773 unsigned int q; 774 u32 ctrl; 775 776 spin_lock_irqsave(&bp->lock, flags); 777 778 ctrl = macb_or_gem_readl(bp, NCFGR); 779 780 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD)); 781 782 if (speed == SPEED_100) 783 ctrl |= MACB_BIT(SPD); 784 785 if (duplex) 786 ctrl |= MACB_BIT(FD); 787 788 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 789 ctrl &= ~MACB_BIT(PAE); 790 if (macb_is_gem(bp)) { 791 ctrl &= ~GEM_BIT(GBE); 792 793 if (speed == SPEED_1000) 794 ctrl |= GEM_BIT(GBE); 795 } 796 797 if (rx_pause) 798 ctrl |= MACB_BIT(PAE); 799 800 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 801 queue_writel(queue, IER, 802 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP)); 803 } 804 } 805 806 macb_or_gem_writel(bp, NCFGR, ctrl); 807 808 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER) 809 gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M, 810 gem_readl(bp, HS_MAC_CONFIG))); 811 812 spin_unlock_irqrestore(&bp->lock, flags); 813 814 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 815 macb_set_tx_clk(bp, speed); 816 gem_shuffle_tx_rings(bp); 817 } 818 819 /* Enable Rx and Tx; Enable PTP unicast */ 820 ctrl = macb_readl(bp, NCR); 821 if (gem_has_ptp(bp)) 822 ctrl |= MACB_BIT(PTPUNI); 823 824 macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE)); 825 826 netif_tx_wake_all_queues(ndev); 827} 828 829static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config, 830 phy_interface_t interface) 831{ 832 struct net_device *ndev = to_net_dev(config->dev); 833 struct macb *bp = netdev_priv(ndev); 834 835 if (interface == PHY_INTERFACE_MODE_10GBASER) 836 return &bp->phylink_usx_pcs; 837 else if (interface == PHY_INTERFACE_MODE_SGMII) 838 return &bp->phylink_sgmii_pcs; 839 else 840 return NULL; 841} 842 843static const struct phylink_mac_ops macb_phylink_ops = { 844 .mac_select_pcs = macb_mac_select_pcs, 845 .mac_config = macb_mac_config, 846 .mac_link_down = macb_mac_link_down, 847 .mac_link_up = macb_mac_link_up, 848}; 849 850static bool macb_phy_handle_exists(struct device_node *dn) 851{ 852 dn = of_parse_phandle(dn, "phy-handle", 0); 853 of_node_put(dn); 854 return dn != NULL; 855} 856 857static int macb_phylink_connect(struct macb *bp) 858{ 859 struct device_node *dn = bp->pdev->dev.of_node; 860 struct net_device *dev = bp->dev; 861 struct phy_device *phydev; 862 int ret; 863 864 if (dn) 865 ret = phylink_of_phy_connect(bp->phylink, dn, 0); 866 867 if (!dn || (ret && !macb_phy_handle_exists(dn))) { 868 phydev = phy_find_first(bp->mii_bus); 869 if (!phydev) { 870 netdev_err(dev, "no PHY found\n"); 871 return -ENXIO; 872 } 873 874 /* attach the mac to the phy */ 875 ret = phylink_connect_phy(bp->phylink, phydev); 876 } 877 878 if (ret) { 879 netdev_err(dev, "Could not attach PHY (%d)\n", ret); 880 return ret; 881 } 882 883 phylink_start(bp->phylink); 884 885 return 0; 886} 887 888static void macb_get_pcs_fixed_state(struct phylink_config *config, 889 struct phylink_link_state *state) 890{ 891 struct net_device *ndev = to_net_dev(config->dev); 892 struct macb *bp = netdev_priv(ndev); 893 894 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0; 895} 896 897/* based on au1000_eth. c*/ 898static int macb_mii_probe(struct net_device *dev) 899{ 900 struct macb *bp = netdev_priv(dev); 901 902 bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops; 903 bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops; 904 905 bp->phylink_config.dev = &dev->dev; 906 bp->phylink_config.type = PHYLINK_NETDEV; 907 bp->phylink_config.mac_managed_pm = true; 908 909 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { 910 bp->phylink_config.poll_fixed_state = true; 911 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; 912 } 913 914 bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | 915 MAC_10 | MAC_100; 916 917 __set_bit(PHY_INTERFACE_MODE_MII, 918 bp->phylink_config.supported_interfaces); 919 __set_bit(PHY_INTERFACE_MODE_RMII, 920 bp->phylink_config.supported_interfaces); 921 922 /* Determine what modes are supported */ 923 if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { 924 bp->phylink_config.mac_capabilities |= MAC_1000FD; 925 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) 926 bp->phylink_config.mac_capabilities |= MAC_1000HD; 927 928 __set_bit(PHY_INTERFACE_MODE_GMII, 929 bp->phylink_config.supported_interfaces); 930 phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); 931 932 if (bp->caps & MACB_CAPS_PCS) 933 __set_bit(PHY_INTERFACE_MODE_SGMII, 934 bp->phylink_config.supported_interfaces); 935 936 if (bp->caps & MACB_CAPS_HIGH_SPEED) { 937 __set_bit(PHY_INTERFACE_MODE_10GBASER, 938 bp->phylink_config.supported_interfaces); 939 bp->phylink_config.mac_capabilities |= MAC_10000FD; 940 } 941 } 942 943 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, 944 bp->phy_interface, &macb_phylink_ops); 945 if (IS_ERR(bp->phylink)) { 946 netdev_err(dev, "Could not create a phylink instance (%ld)\n", 947 PTR_ERR(bp->phylink)); 948 return PTR_ERR(bp->phylink); 949 } 950 951 return 0; 952} 953 954static int macb_mdiobus_register(struct macb *bp, struct device_node *mdio_np) 955{ 956 struct device_node *child, *np = bp->pdev->dev.of_node; 957 958 /* If we have a child named mdio, probe it instead of looking for PHYs 959 * directly under the MAC node 960 */ 961 if (mdio_np) 962 return of_mdiobus_register(bp->mii_bus, mdio_np); 963 964 /* Only create the PHY from the device tree if at least one PHY is 965 * described. Otherwise scan the entire MDIO bus. We do this to support 966 * old device tree that did not follow the best practices and did not 967 * describe their network PHYs. 968 */ 969 for_each_available_child_of_node(np, child) 970 if (of_mdiobus_child_is_phy(child)) { 971 /* The loop increments the child refcount, 972 * decrement it before returning. 973 */ 974 of_node_put(child); 975 976 return of_mdiobus_register(bp->mii_bus, np); 977 } 978 979 return mdiobus_register(bp->mii_bus); 980} 981 982static int macb_mii_init(struct macb *bp) 983{ 984 struct device_node *mdio_np, *np = bp->pdev->dev.of_node; 985 int err = -ENXIO; 986 987 /* With fixed-link, we don't need to register the MDIO bus, 988 * except if we have a child named "mdio" in the device tree. 989 * In that case, some devices may be attached to the MACB's MDIO bus. 990 */ 991 mdio_np = of_get_child_by_name(np, "mdio"); 992 if (!mdio_np && of_phy_is_fixed_link(np)) 993 return macb_mii_probe(bp->dev); 994 995 /* Enable management port */ 996 macb_writel(bp, NCR, MACB_BIT(MPE)); 997 998 bp->mii_bus = mdiobus_alloc(); 999 if (!bp->mii_bus) { 1000 err = -ENOMEM; 1001 goto err_out; 1002 } 1003 1004 bp->mii_bus->name = "MACB_mii_bus"; 1005 bp->mii_bus->read = &macb_mdio_read_c22; 1006 bp->mii_bus->write = &macb_mdio_write_c22; 1007 bp->mii_bus->read_c45 = &macb_mdio_read_c45; 1008 bp->mii_bus->write_c45 = &macb_mdio_write_c45; 1009 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", 1010 bp->pdev->name, bp->pdev->id); 1011 bp->mii_bus->priv = bp; 1012 bp->mii_bus->parent = &bp->pdev->dev; 1013 1014 dev_set_drvdata(&bp->dev->dev, bp->mii_bus); 1015 1016 err = macb_mdiobus_register(bp, mdio_np); 1017 if (err) 1018 goto err_out_free_mdiobus; 1019 1020 err = macb_mii_probe(bp->dev); 1021 if (err) 1022 goto err_out_unregister_bus; 1023 1024 return 0; 1025 1026err_out_unregister_bus: 1027 mdiobus_unregister(bp->mii_bus); 1028err_out_free_mdiobus: 1029 mdiobus_free(bp->mii_bus); 1030err_out: 1031 of_node_put(mdio_np); 1032 1033 return err; 1034} 1035 1036static void macb_update_stats(struct macb *bp) 1037{ 1038 u64 *p = &bp->hw_stats.macb.rx_pause_frames; 1039 u64 *end = &bp->hw_stats.macb.tx_pause_frames + 1; 1040 int offset = MACB_PFR; 1041 1042 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4); 1043 1044 for (; p < end; p++, offset += 4) 1045 *p += bp->macb_reg_readl(bp, offset); 1046} 1047 1048static int macb_halt_tx(struct macb *bp) 1049{ 1050 u32 status; 1051 1052 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT)); 1053 1054 /* Poll TSR until TGO is cleared or timeout. */ 1055 return read_poll_timeout_atomic(macb_readl, status, 1056 !(status & MACB_BIT(TGO)), 1057 250, MACB_HALT_TIMEOUT, false, 1058 bp, TSR); 1059} 1060 1061static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget) 1062{ 1063 if (tx_skb->mapping) { 1064 if (tx_skb->mapped_as_page) 1065 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, 1066 tx_skb->size, DMA_TO_DEVICE); 1067 else 1068 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, 1069 tx_skb->size, DMA_TO_DEVICE); 1070 tx_skb->mapping = 0; 1071 } 1072 1073 if (tx_skb->skb) { 1074 dev_consume_skb_any(tx_skb->skb); 1075 tx_skb->skb = NULL; 1076 } 1077} 1078 1079static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr) 1080{ 1081 if (macb_dma64(bp)) { 1082 struct macb_dma_desc_64 *desc_64; 1083 1084 desc_64 = macb_64b_desc(bp, desc); 1085 desc_64->addrh = upper_32_bits(addr); 1086 /* The low bits of RX address contain the RX_USED bit, clearing 1087 * of which allows packet RX. Make sure the high bits are also 1088 * visible to HW at that point. 1089 */ 1090 dma_wmb(); 1091 } 1092 1093 desc->addr = lower_32_bits(addr); 1094} 1095 1096static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc) 1097{ 1098 dma_addr_t addr = 0; 1099 1100 if (macb_dma64(bp)) { 1101 struct macb_dma_desc_64 *desc_64; 1102 1103 desc_64 = macb_64b_desc(bp, desc); 1104 addr = ((u64)(desc_64->addrh) << 32); 1105 } 1106 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr)); 1107 if (macb_dma_ptp(bp)) 1108 addr &= ~GEM_BIT(DMA_RXVALID); 1109 return addr; 1110} 1111 1112static void macb_tx_error_task(struct work_struct *work) 1113{ 1114 struct macb_queue *queue = container_of(work, struct macb_queue, 1115 tx_error_task); 1116 bool halt_timeout = false; 1117 struct macb *bp = queue->bp; 1118 u32 queue_index; 1119 u32 packets = 0; 1120 u32 bytes = 0; 1121 struct macb_tx_skb *tx_skb; 1122 struct macb_dma_desc *desc; 1123 struct sk_buff *skb; 1124 unsigned int tail; 1125 unsigned long flags; 1126 1127 queue_index = queue - bp->queues; 1128 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n", 1129 queue_index, queue->tx_tail, queue->tx_head); 1130 1131 /* Prevent the queue NAPI TX poll from running, as it calls 1132 * macb_tx_complete(), which in turn may call netif_wake_subqueue(). 1133 * As explained below, we have to halt the transmission before updating 1134 * TBQP registers so we call netif_tx_stop_all_queues() to notify the 1135 * network engine about the macb/gem being halted. 1136 */ 1137 napi_disable(&queue->napi_tx); 1138 spin_lock_irqsave(&bp->lock, flags); 1139 1140 /* Make sure nobody is trying to queue up new packets */ 1141 netif_tx_stop_all_queues(bp->dev); 1142 1143 /* Stop transmission now 1144 * (in case we have just queued new packets) 1145 * macb/gem must be halted to write TBQP register 1146 */ 1147 if (macb_halt_tx(bp)) { 1148 netdev_err(bp->dev, "BUG: halt tx timed out\n"); 1149 macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE))); 1150 halt_timeout = true; 1151 } 1152 1153 /* Treat frames in TX queue including the ones that caused the error. 1154 * Free transmit buffers in upper layer. 1155 */ 1156 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) { 1157 u32 ctrl; 1158 1159 desc = macb_tx_desc(queue, tail); 1160 ctrl = desc->ctrl; 1161 tx_skb = macb_tx_skb(queue, tail); 1162 skb = tx_skb->skb; 1163 1164 if (ctrl & MACB_BIT(TX_USED)) { 1165 /* skb is set for the last buffer of the frame */ 1166 while (!skb) { 1167 macb_tx_unmap(bp, tx_skb, 0); 1168 tail++; 1169 tx_skb = macb_tx_skb(queue, tail); 1170 skb = tx_skb->skb; 1171 } 1172 1173 /* ctrl still refers to the first buffer descriptor 1174 * since it's the only one written back by the hardware 1175 */ 1176 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) { 1177 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n", 1178 macb_tx_ring_wrap(bp, tail), 1179 skb->data); 1180 bp->dev->stats.tx_packets++; 1181 queue->stats.tx_packets++; 1182 packets++; 1183 bp->dev->stats.tx_bytes += skb->len; 1184 queue->stats.tx_bytes += skb->len; 1185 bytes += skb->len; 1186 } 1187 } else { 1188 /* "Buffers exhausted mid-frame" errors may only happen 1189 * if the driver is buggy, so complain loudly about 1190 * those. Statistics are updated by hardware. 1191 */ 1192 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED)) 1193 netdev_err(bp->dev, 1194 "BUG: TX buffers exhausted mid-frame\n"); 1195 1196 desc->ctrl = ctrl | MACB_BIT(TX_USED); 1197 } 1198 1199 macb_tx_unmap(bp, tx_skb, 0); 1200 } 1201 1202 netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), 1203 packets, bytes); 1204 1205 /* Set end of TX queue */ 1206 desc = macb_tx_desc(queue, 0); 1207 macb_set_addr(bp, desc, 0); 1208 desc->ctrl = MACB_BIT(TX_USED); 1209 1210 /* Make descriptor updates visible to hardware */ 1211 wmb(); 1212 1213 /* Reinitialize the TX desc queue */ 1214 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma)); 1215 /* Make TX ring reflect state of hardware */ 1216 queue->tx_head = 0; 1217 queue->tx_tail = 0; 1218 1219 /* Housework before enabling TX IRQ */ 1220 macb_writel(bp, TSR, macb_readl(bp, TSR)); 1221 queue_writel(queue, IER, MACB_TX_INT_FLAGS); 1222 1223 if (halt_timeout) 1224 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE)); 1225 1226 /* Now we are ready to start transmission again */ 1227 netif_tx_start_all_queues(bp->dev); 1228 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1229 1230 spin_unlock_irqrestore(&bp->lock, flags); 1231 napi_enable(&queue->napi_tx); 1232} 1233 1234static bool ptp_one_step_sync(struct sk_buff *skb) 1235{ 1236 struct ptp_header *hdr; 1237 unsigned int ptp_class; 1238 u8 msgtype; 1239 1240 /* No need to parse packet if PTP TS is not involved */ 1241 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) 1242 goto not_oss; 1243 1244 /* Identify and return whether PTP one step sync is being processed */ 1245 ptp_class = ptp_classify_raw(skb); 1246 if (ptp_class == PTP_CLASS_NONE) 1247 goto not_oss; 1248 1249 hdr = ptp_parse_header(skb, ptp_class); 1250 if (!hdr) 1251 goto not_oss; 1252 1253 if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP) 1254 goto not_oss; 1255 1256 msgtype = ptp_get_msgtype(hdr, ptp_class); 1257 if (msgtype == PTP_MSGTYPE_SYNC) 1258 return true; 1259 1260not_oss: 1261 return false; 1262} 1263 1264static int macb_tx_complete(struct macb_queue *queue, int budget) 1265{ 1266 struct macb *bp = queue->bp; 1267 u16 queue_index = queue - bp->queues; 1268 unsigned long flags; 1269 unsigned int tail; 1270 unsigned int head; 1271 int packets = 0; 1272 u32 bytes = 0; 1273 1274 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 1275 head = queue->tx_head; 1276 for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { 1277 struct macb_tx_skb *tx_skb; 1278 struct sk_buff *skb; 1279 struct macb_dma_desc *desc; 1280 u32 ctrl; 1281 1282 desc = macb_tx_desc(queue, tail); 1283 1284 /* Make hw descriptor updates visible to CPU */ 1285 rmb(); 1286 1287 ctrl = desc->ctrl; 1288 1289 /* TX_USED bit is only set by hardware on the very first buffer 1290 * descriptor of the transmitted frame. 1291 */ 1292 if (!(ctrl & MACB_BIT(TX_USED))) 1293 break; 1294 1295 /* Process all buffers of the current transmitted frame */ 1296 for (;; tail++) { 1297 tx_skb = macb_tx_skb(queue, tail); 1298 skb = tx_skb->skb; 1299 1300 /* First, update TX stats if needed */ 1301 if (skb) { 1302 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && 1303 !ptp_one_step_sync(skb)) 1304 gem_ptp_do_txstamp(bp, skb, desc); 1305 1306 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n", 1307 macb_tx_ring_wrap(bp, tail), 1308 skb->data); 1309 bp->dev->stats.tx_packets++; 1310 queue->stats.tx_packets++; 1311 bp->dev->stats.tx_bytes += skb->len; 1312 queue->stats.tx_bytes += skb->len; 1313 packets++; 1314 bytes += skb->len; 1315 } 1316 1317 /* Now we can safely release resources */ 1318 macb_tx_unmap(bp, tx_skb, budget); 1319 1320 /* skb is set only for the last buffer of the frame. 1321 * WARNING: at this point skb has been freed by 1322 * macb_tx_unmap(). 1323 */ 1324 if (skb) 1325 break; 1326 } 1327 } 1328 1329 netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), 1330 packets, bytes); 1331 1332 queue->tx_tail = tail; 1333 if (__netif_subqueue_stopped(bp->dev, queue_index) && 1334 CIRC_CNT(queue->tx_head, queue->tx_tail, 1335 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp)) 1336 netif_wake_subqueue(bp->dev, queue_index); 1337 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 1338 1339 return packets; 1340} 1341 1342static void gem_rx_refill(struct macb_queue *queue) 1343{ 1344 unsigned int entry; 1345 struct sk_buff *skb; 1346 dma_addr_t paddr; 1347 struct macb *bp = queue->bp; 1348 struct macb_dma_desc *desc; 1349 1350 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail, 1351 bp->rx_ring_size) > 0) { 1352 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head); 1353 1354 /* Make hw descriptor updates visible to CPU */ 1355 rmb(); 1356 1357 desc = macb_rx_desc(queue, entry); 1358 1359 if (!queue->rx_skbuff[entry]) { 1360 /* allocate sk_buff for this free entry in ring */ 1361 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); 1362 if (unlikely(!skb)) { 1363 netdev_err(bp->dev, 1364 "Unable to allocate sk_buff\n"); 1365 break; 1366 } 1367 1368 /* now fill corresponding descriptor entry */ 1369 paddr = dma_map_single(&bp->pdev->dev, skb->data, 1370 bp->rx_buffer_size, 1371 DMA_FROM_DEVICE); 1372 if (dma_mapping_error(&bp->pdev->dev, paddr)) { 1373 dev_kfree_skb(skb); 1374 break; 1375 } 1376 1377 queue->rx_skbuff[entry] = skb; 1378 1379 if (entry == bp->rx_ring_size - 1) 1380 paddr |= MACB_BIT(RX_WRAP); 1381 desc->ctrl = 0; 1382 /* Setting addr clears RX_USED and allows reception, 1383 * make sure ctrl is cleared first to avoid a race. 1384 */ 1385 dma_wmb(); 1386 macb_set_addr(bp, desc, paddr); 1387 1388 /* Properly align Ethernet header. 1389 * 1390 * Hardware can add dummy bytes if asked using the RBOF 1391 * field inside the NCFGR register. That feature isn't 1392 * available if hardware is RSC capable. 1393 * 1394 * We cannot fallback to doing the 2-byte shift before 1395 * DMA mapping because the address field does not allow 1396 * setting the low 2/3 bits. 1397 * It is 3 bits if HW_DMA_CAP_PTP, else 2 bits. 1398 */ 1399 if (!(bp->caps & MACB_CAPS_RSC)) 1400 skb_reserve(skb, NET_IP_ALIGN); 1401 } else { 1402 desc->ctrl = 0; 1403 dma_wmb(); 1404 desc->addr &= ~MACB_BIT(RX_USED); 1405 } 1406 queue->rx_prepared_head++; 1407 } 1408 1409 /* Make descriptor updates visible to hardware */ 1410 wmb(); 1411 1412 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n", 1413 queue, queue->rx_prepared_head, queue->rx_tail); 1414} 1415 1416/* Mark DMA descriptors from begin up to and not including end as unused */ 1417static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, 1418 unsigned int end) 1419{ 1420 unsigned int frag; 1421 1422 for (frag = begin; frag != end; frag++) { 1423 struct macb_dma_desc *desc = macb_rx_desc(queue, frag); 1424 1425 desc->addr &= ~MACB_BIT(RX_USED); 1426 } 1427 1428 /* Make descriptor updates visible to hardware */ 1429 wmb(); 1430 1431 /* When this happens, the hardware stats registers for 1432 * whatever caused this is updated, so we don't have to record 1433 * anything. 1434 */ 1435} 1436 1437static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, 1438 int budget) 1439{ 1440 struct macb *bp = queue->bp; 1441 unsigned int len; 1442 unsigned int entry; 1443 struct sk_buff *skb; 1444 struct macb_dma_desc *desc; 1445 int count = 0; 1446 1447 while (count < budget) { 1448 u32 ctrl; 1449 dma_addr_t addr; 1450 bool rxused; 1451 1452 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1453 desc = macb_rx_desc(queue, entry); 1454 1455 /* Make hw descriptor updates visible to CPU */ 1456 rmb(); 1457 1458 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; 1459 addr = macb_get_addr(bp, desc); 1460 1461 if (!rxused) 1462 break; 1463 1464 /* Ensure ctrl is at least as up-to-date as rxused */ 1465 dma_rmb(); 1466 1467 ctrl = desc->ctrl; 1468 1469 queue->rx_tail++; 1470 count++; 1471 1472 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { 1473 netdev_err(bp->dev, 1474 "not whole frame pointed by descriptor\n"); 1475 bp->dev->stats.rx_dropped++; 1476 queue->stats.rx_dropped++; 1477 break; 1478 } 1479 skb = queue->rx_skbuff[entry]; 1480 if (unlikely(!skb)) { 1481 netdev_err(bp->dev, 1482 "inconsistent Rx descriptor chain\n"); 1483 bp->dev->stats.rx_dropped++; 1484 queue->stats.rx_dropped++; 1485 break; 1486 } 1487 /* now everything is ready for receiving packet */ 1488 queue->rx_skbuff[entry] = NULL; 1489 len = ctrl & bp->rx_frm_len_mask; 1490 1491 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); 1492 1493 skb_put(skb, len); 1494 dma_unmap_single(&bp->pdev->dev, addr, 1495 bp->rx_buffer_size, DMA_FROM_DEVICE); 1496 1497 skb->protocol = eth_type_trans(skb, bp->dev); 1498 skb_checksum_none_assert(skb); 1499 if (bp->dev->features & NETIF_F_RXCSUM && 1500 !(bp->dev->flags & IFF_PROMISC) && 1501 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) 1502 skb->ip_summed = CHECKSUM_UNNECESSARY; 1503 1504 bp->dev->stats.rx_packets++; 1505 queue->stats.rx_packets++; 1506 bp->dev->stats.rx_bytes += skb->len; 1507 queue->stats.rx_bytes += skb->len; 1508 1509 gem_ptp_do_rxstamp(bp, skb, desc); 1510 1511#if defined(DEBUG) && defined(VERBOSE_DEBUG) 1512 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1513 skb->len, skb->csum); 1514 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, 1515 skb_mac_header(skb), 16, true); 1516 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, 1517 skb->data, 32, true); 1518#endif 1519 1520 napi_gro_receive(napi, skb); 1521 } 1522 1523 gem_rx_refill(queue); 1524 1525 return count; 1526} 1527 1528static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi, 1529 unsigned int first_frag, unsigned int last_frag) 1530{ 1531 unsigned int len; 1532 unsigned int frag; 1533 unsigned int offset; 1534 struct sk_buff *skb; 1535 struct macb_dma_desc *desc; 1536 struct macb *bp = queue->bp; 1537 1538 desc = macb_rx_desc(queue, last_frag); 1539 len = desc->ctrl & bp->rx_frm_len_mask; 1540 1541 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n", 1542 macb_rx_ring_wrap(bp, first_frag), 1543 macb_rx_ring_wrap(bp, last_frag), len); 1544 1545 /* The ethernet header starts NET_IP_ALIGN bytes into the 1546 * first buffer. Since the header is 14 bytes, this makes the 1547 * payload word-aligned. 1548 * 1549 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy 1550 * the two padding bytes into the skb so that we avoid hitting 1551 * the slowpath in memcpy(), and pull them off afterwards. 1552 */ 1553 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN); 1554 if (!skb) { 1555 bp->dev->stats.rx_dropped++; 1556 for (frag = first_frag; ; frag++) { 1557 desc = macb_rx_desc(queue, frag); 1558 desc->addr &= ~MACB_BIT(RX_USED); 1559 if (frag == last_frag) 1560 break; 1561 } 1562 1563 /* Make descriptor updates visible to hardware */ 1564 wmb(); 1565 1566 return 1; 1567 } 1568 1569 offset = 0; 1570 len += NET_IP_ALIGN; 1571 skb_checksum_none_assert(skb); 1572 skb_put(skb, len); 1573 1574 for (frag = first_frag; ; frag++) { 1575 unsigned int frag_len = bp->rx_buffer_size; 1576 1577 if (offset + frag_len > len) { 1578 if (unlikely(frag != last_frag)) { 1579 dev_kfree_skb_any(skb); 1580 return -1; 1581 } 1582 frag_len = len - offset; 1583 } 1584 skb_copy_to_linear_data_offset(skb, offset, 1585 macb_rx_buffer(queue, frag), 1586 frag_len); 1587 offset += bp->rx_buffer_size; 1588 desc = macb_rx_desc(queue, frag); 1589 desc->addr &= ~MACB_BIT(RX_USED); 1590 1591 if (frag == last_frag) 1592 break; 1593 } 1594 1595 /* Make descriptor updates visible to hardware */ 1596 wmb(); 1597 1598 __skb_pull(skb, NET_IP_ALIGN); 1599 skb->protocol = eth_type_trans(skb, bp->dev); 1600 1601 bp->dev->stats.rx_packets++; 1602 bp->dev->stats.rx_bytes += skb->len; 1603 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", 1604 skb->len, skb->csum); 1605 napi_gro_receive(napi, skb); 1606 1607 return 0; 1608} 1609 1610static inline void macb_init_rx_ring(struct macb_queue *queue) 1611{ 1612 struct macb *bp = queue->bp; 1613 dma_addr_t addr; 1614 struct macb_dma_desc *desc = NULL; 1615 int i; 1616 1617 addr = queue->rx_buffers_dma; 1618 for (i = 0; i < bp->rx_ring_size; i++) { 1619 desc = macb_rx_desc(queue, i); 1620 macb_set_addr(bp, desc, addr); 1621 desc->ctrl = 0; 1622 addr += bp->rx_buffer_size; 1623 } 1624 desc->addr |= MACB_BIT(RX_WRAP); 1625 queue->rx_tail = 0; 1626} 1627 1628static int macb_rx(struct macb_queue *queue, struct napi_struct *napi, 1629 int budget) 1630{ 1631 struct macb *bp = queue->bp; 1632 bool reset_rx_queue = false; 1633 int received = 0; 1634 unsigned int tail; 1635 int first_frag = -1; 1636 1637 for (tail = queue->rx_tail; budget > 0; tail++) { 1638 struct macb_dma_desc *desc = macb_rx_desc(queue, tail); 1639 u32 ctrl; 1640 1641 /* Make hw descriptor updates visible to CPU */ 1642 rmb(); 1643 1644 if (!(desc->addr & MACB_BIT(RX_USED))) 1645 break; 1646 1647 /* Ensure ctrl is at least as up-to-date as addr */ 1648 dma_rmb(); 1649 1650 ctrl = desc->ctrl; 1651 1652 if (ctrl & MACB_BIT(RX_SOF)) { 1653 if (first_frag != -1) 1654 discard_partial_frame(queue, first_frag, tail); 1655 first_frag = tail; 1656 } 1657 1658 if (ctrl & MACB_BIT(RX_EOF)) { 1659 int dropped; 1660 1661 if (unlikely(first_frag == -1)) { 1662 reset_rx_queue = true; 1663 continue; 1664 } 1665 1666 dropped = macb_rx_frame(queue, napi, first_frag, tail); 1667 first_frag = -1; 1668 if (unlikely(dropped < 0)) { 1669 reset_rx_queue = true; 1670 continue; 1671 } 1672 if (!dropped) { 1673 received++; 1674 budget--; 1675 } 1676 } 1677 } 1678 1679 if (unlikely(reset_rx_queue)) { 1680 unsigned long flags; 1681 u32 ctrl; 1682 1683 netdev_err(bp->dev, "RX queue corruption: reset it\n"); 1684 1685 spin_lock_irqsave(&bp->lock, flags); 1686 1687 ctrl = macb_readl(bp, NCR); 1688 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 1689 1690 macb_init_rx_ring(queue); 1691 queue_writel(queue, RBQP, queue->rx_ring_dma); 1692 1693 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 1694 1695 spin_unlock_irqrestore(&bp->lock, flags); 1696 return received; 1697 } 1698 1699 if (first_frag != -1) 1700 queue->rx_tail = first_frag; 1701 else 1702 queue->rx_tail = tail; 1703 1704 return received; 1705} 1706 1707static bool macb_rx_pending(struct macb_queue *queue) 1708{ 1709 struct macb *bp = queue->bp; 1710 unsigned int entry; 1711 struct macb_dma_desc *desc; 1712 1713 entry = macb_rx_ring_wrap(bp, queue->rx_tail); 1714 desc = macb_rx_desc(queue, entry); 1715 1716 /* Make hw descriptor updates visible to CPU */ 1717 rmb(); 1718 1719 return (desc->addr & MACB_BIT(RX_USED)) != 0; 1720} 1721 1722static int macb_rx_poll(struct napi_struct *napi, int budget) 1723{ 1724 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx); 1725 struct macb *bp = queue->bp; 1726 int work_done; 1727 1728 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget); 1729 1730 netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n", 1731 (unsigned int)(queue - bp->queues), work_done, budget); 1732 1733 if (work_done < budget && napi_complete_done(napi, work_done)) { 1734 queue_writel(queue, IER, bp->rx_intr_mask); 1735 1736 /* Packet completions only seem to propagate to raise 1737 * interrupts when interrupts are enabled at the time, so if 1738 * packets were received while interrupts were disabled, 1739 * they will not cause another interrupt to be generated when 1740 * interrupts are re-enabled. 1741 * Check for this case here to avoid losing a wakeup. This can 1742 * potentially race with the interrupt handler doing the same 1743 * actions if an interrupt is raised just after enabling them, 1744 * but this should be harmless. 1745 */ 1746 if (macb_rx_pending(queue)) { 1747 queue_writel(queue, IDR, bp->rx_intr_mask); 1748 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1749 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1750 netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n"); 1751 napi_schedule(napi); 1752 } 1753 } 1754 1755 /* TODO: Handle errors */ 1756 1757 return work_done; 1758} 1759 1760static void macb_tx_restart(struct macb_queue *queue) 1761{ 1762 struct macb *bp = queue->bp; 1763 unsigned int head_idx, tbqp; 1764 unsigned long flags; 1765 1766 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 1767 1768 if (queue->tx_head == queue->tx_tail) 1769 goto out_tx_ptr_unlock; 1770 1771 tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp); 1772 tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp)); 1773 head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head)); 1774 1775 if (tbqp == head_idx) 1776 goto out_tx_ptr_unlock; 1777 1778 spin_lock(&bp->lock); 1779 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 1780 spin_unlock(&bp->lock); 1781 1782out_tx_ptr_unlock: 1783 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 1784} 1785 1786static bool macb_tx_complete_pending(struct macb_queue *queue) 1787{ 1788 bool retval = false; 1789 unsigned long flags; 1790 1791 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 1792 if (queue->tx_head != queue->tx_tail) { 1793 /* Make hw descriptor updates visible to CPU */ 1794 rmb(); 1795 1796 if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED)) 1797 retval = true; 1798 } 1799 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 1800 return retval; 1801} 1802 1803static int macb_tx_poll(struct napi_struct *napi, int budget) 1804{ 1805 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx); 1806 struct macb *bp = queue->bp; 1807 int work_done; 1808 1809 work_done = macb_tx_complete(queue, budget); 1810 1811 rmb(); // ensure txubr_pending is up to date 1812 if (queue->txubr_pending) { 1813 queue->txubr_pending = false; 1814 netdev_vdbg(bp->dev, "poll: tx restart\n"); 1815 macb_tx_restart(queue); 1816 } 1817 1818 netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n", 1819 (unsigned int)(queue - bp->queues), work_done, budget); 1820 1821 if (work_done < budget && napi_complete_done(napi, work_done)) { 1822 queue_writel(queue, IER, MACB_BIT(TCOMP)); 1823 1824 /* Packet completions only seem to propagate to raise 1825 * interrupts when interrupts are enabled at the time, so if 1826 * packets were sent while interrupts were disabled, 1827 * they will not cause another interrupt to be generated when 1828 * interrupts are re-enabled. 1829 * Check for this case here to avoid losing a wakeup. This can 1830 * potentially race with the interrupt handler doing the same 1831 * actions if an interrupt is raised just after enabling them, 1832 * but this should be harmless. 1833 */ 1834 if (macb_tx_complete_pending(queue)) { 1835 queue_writel(queue, IDR, MACB_BIT(TCOMP)); 1836 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1837 queue_writel(queue, ISR, MACB_BIT(TCOMP)); 1838 netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n"); 1839 napi_schedule(napi); 1840 } 1841 } 1842 1843 return work_done; 1844} 1845 1846static void macb_hresp_error_task(struct work_struct *work) 1847{ 1848 struct macb *bp = from_work(bp, work, hresp_err_bh_work); 1849 struct net_device *dev = bp->dev; 1850 struct macb_queue *queue; 1851 unsigned int q; 1852 u32 ctrl; 1853 1854 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 1855 queue_writel(queue, IDR, bp->rx_intr_mask | 1856 MACB_TX_INT_FLAGS | 1857 MACB_BIT(HRESP)); 1858 } 1859 ctrl = macb_readl(bp, NCR); 1860 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 1861 macb_writel(bp, NCR, ctrl); 1862 1863 netif_tx_stop_all_queues(dev); 1864 netif_carrier_off(dev); 1865 1866 bp->macbgem_ops.mog_init_rings(bp); 1867 1868 /* Initialize TX and RX buffers */ 1869 macb_init_buffers(bp); 1870 1871 /* Enable interrupts */ 1872 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 1873 queue_writel(queue, IER, 1874 bp->rx_intr_mask | 1875 MACB_TX_INT_FLAGS | 1876 MACB_BIT(HRESP)); 1877 1878 ctrl |= MACB_BIT(RE) | MACB_BIT(TE); 1879 macb_writel(bp, NCR, ctrl); 1880 1881 netif_carrier_on(dev); 1882 netif_tx_start_all_queues(dev); 1883} 1884 1885static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) 1886{ 1887 struct macb_queue *queue = dev_id; 1888 struct macb *bp = queue->bp; 1889 u32 status; 1890 1891 status = queue_readl(queue, ISR); 1892 1893 if (unlikely(!status)) 1894 return IRQ_NONE; 1895 1896 spin_lock(&bp->lock); 1897 1898 if (status & MACB_BIT(WOL)) { 1899 queue_writel(queue, IDR, MACB_BIT(WOL)); 1900 macb_writel(bp, WOL, 0); 1901 netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", 1902 (unsigned int)(queue - bp->queues), 1903 (unsigned long)status); 1904 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1905 queue_writel(queue, ISR, MACB_BIT(WOL)); 1906 pm_wakeup_event(&bp->pdev->dev, 0); 1907 } 1908 1909 spin_unlock(&bp->lock); 1910 1911 return IRQ_HANDLED; 1912} 1913 1914static irqreturn_t gem_wol_interrupt(int irq, void *dev_id) 1915{ 1916 struct macb_queue *queue = dev_id; 1917 struct macb *bp = queue->bp; 1918 u32 status; 1919 1920 status = queue_readl(queue, ISR); 1921 1922 if (unlikely(!status)) 1923 return IRQ_NONE; 1924 1925 spin_lock(&bp->lock); 1926 1927 if (status & GEM_BIT(WOL)) { 1928 queue_writel(queue, IDR, GEM_BIT(WOL)); 1929 gem_writel(bp, WOL, 0); 1930 netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", 1931 (unsigned int)(queue - bp->queues), 1932 (unsigned long)status); 1933 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1934 queue_writel(queue, ISR, GEM_BIT(WOL)); 1935 pm_wakeup_event(&bp->pdev->dev, 0); 1936 } 1937 1938 spin_unlock(&bp->lock); 1939 1940 return IRQ_HANDLED; 1941} 1942 1943static irqreturn_t macb_interrupt(int irq, void *dev_id) 1944{ 1945 struct macb_queue *queue = dev_id; 1946 struct macb *bp = queue->bp; 1947 struct net_device *dev = bp->dev; 1948 u32 status, ctrl; 1949 1950 status = queue_readl(queue, ISR); 1951 1952 if (unlikely(!status)) 1953 return IRQ_NONE; 1954 1955 spin_lock(&bp->lock); 1956 1957 while (status) { 1958 /* close possible race with dev_close */ 1959 if (unlikely(!netif_running(dev))) { 1960 queue_writel(queue, IDR, -1); 1961 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1962 queue_writel(queue, ISR, -1); 1963 break; 1964 } 1965 1966 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n", 1967 (unsigned int)(queue - bp->queues), 1968 (unsigned long)status); 1969 1970 if (status & bp->rx_intr_mask) { 1971 /* There's no point taking any more interrupts 1972 * until we have processed the buffers. The 1973 * scheduling call may fail if the poll routine 1974 * is already scheduled, so disable interrupts 1975 * now. 1976 */ 1977 queue_writel(queue, IDR, bp->rx_intr_mask); 1978 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1979 queue_writel(queue, ISR, MACB_BIT(RCOMP)); 1980 1981 if (napi_schedule_prep(&queue->napi_rx)) { 1982 netdev_vdbg(bp->dev, "scheduling RX softirq\n"); 1983 __napi_schedule(&queue->napi_rx); 1984 } 1985 } 1986 1987 if (status & (MACB_BIT(TCOMP) | 1988 MACB_BIT(TXUBR))) { 1989 queue_writel(queue, IDR, MACB_BIT(TCOMP)); 1990 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 1991 queue_writel(queue, ISR, MACB_BIT(TCOMP) | 1992 MACB_BIT(TXUBR)); 1993 1994 if (status & MACB_BIT(TXUBR)) { 1995 queue->txubr_pending = true; 1996 wmb(); // ensure softirq can see update 1997 } 1998 1999 if (napi_schedule_prep(&queue->napi_tx)) { 2000 netdev_vdbg(bp->dev, "scheduling TX softirq\n"); 2001 __napi_schedule(&queue->napi_tx); 2002 } 2003 } 2004 2005 if (unlikely(status & (MACB_TX_ERR_FLAGS))) { 2006 queue_writel(queue, IDR, MACB_TX_INT_FLAGS); 2007 schedule_work(&queue->tx_error_task); 2008 2009 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2010 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); 2011 2012 break; 2013 } 2014 2015 /* Link change detection isn't possible with RMII, so we'll 2016 * add that if/when we get our hands on a full-blown MII PHY. 2017 */ 2018 2019 /* There is a hardware issue under heavy load where DMA can 2020 * stop, this causes endless "used buffer descriptor read" 2021 * interrupts but it can be cleared by re-enabling RX. See 2022 * the at91rm9200 manual, section 41.3.1 or the Zynq manual 2023 * section 16.7.4 for details. RXUBR is only enabled for 2024 * these two versions. 2025 */ 2026 if (status & MACB_BIT(RXUBR)) { 2027 ctrl = macb_readl(bp, NCR); 2028 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); 2029 wmb(); 2030 macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); 2031 2032 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2033 queue_writel(queue, ISR, MACB_BIT(RXUBR)); 2034 } 2035 2036 if (status & MACB_BIT(ISR_ROVR)) { 2037 /* We missed at least one packet */ 2038 spin_lock(&bp->stats_lock); 2039 if (macb_is_gem(bp)) 2040 bp->hw_stats.gem.rx_overruns++; 2041 else 2042 bp->hw_stats.macb.rx_overruns++; 2043 spin_unlock(&bp->stats_lock); 2044 2045 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2046 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); 2047 } 2048 2049 if (status & MACB_BIT(HRESP)) { 2050 queue_work(system_bh_wq, &bp->hresp_err_bh_work); 2051 netdev_err(dev, "DMA bus error: HRESP not OK\n"); 2052 2053 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2054 queue_writel(queue, ISR, MACB_BIT(HRESP)); 2055 } 2056 status = queue_readl(queue, ISR); 2057 } 2058 2059 spin_unlock(&bp->lock); 2060 2061 return IRQ_HANDLED; 2062} 2063 2064#ifdef CONFIG_NET_POLL_CONTROLLER 2065/* Polling receive - used by netconsole and other diagnostic tools 2066 * to allow network i/o with interrupts disabled. 2067 */ 2068static void macb_poll_controller(struct net_device *dev) 2069{ 2070 struct macb *bp = netdev_priv(dev); 2071 struct macb_queue *queue; 2072 unsigned long flags; 2073 unsigned int q; 2074 2075 local_irq_save(flags); 2076 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 2077 macb_interrupt(dev->irq, queue); 2078 local_irq_restore(flags); 2079} 2080#endif 2081 2082static unsigned int macb_tx_map(struct macb *bp, 2083 struct macb_queue *queue, 2084 struct sk_buff *skb, 2085 unsigned int hdrlen) 2086{ 2087 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; 2088 unsigned int len, i, tx_head = queue->tx_head; 2089 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; 2090 unsigned int eof = 1, mss_mfs = 0; 2091 struct macb_tx_skb *tx_skb = NULL; 2092 struct macb_dma_desc *desc; 2093 unsigned int offset, size; 2094 dma_addr_t mapping; 2095 2096 /* LSO */ 2097 if (skb_shinfo(skb)->gso_size != 0) { 2098 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2099 /* UDP - UFO */ 2100 lso_ctrl = MACB_LSO_UFO_ENABLE; 2101 else 2102 /* TCP - TSO */ 2103 lso_ctrl = MACB_LSO_TSO_ENABLE; 2104 } 2105 2106 /* First, map non-paged data */ 2107 len = skb_headlen(skb); 2108 2109 /* first buffer length */ 2110 size = hdrlen; 2111 2112 offset = 0; 2113 while (len) { 2114 tx_skb = macb_tx_skb(queue, tx_head); 2115 2116 mapping = dma_map_single(&bp->pdev->dev, 2117 skb->data + offset, 2118 size, DMA_TO_DEVICE); 2119 if (dma_mapping_error(&bp->pdev->dev, mapping)) 2120 goto dma_error; 2121 2122 /* Save info to properly release resources */ 2123 tx_skb->skb = NULL; 2124 tx_skb->mapping = mapping; 2125 tx_skb->size = size; 2126 tx_skb->mapped_as_page = false; 2127 2128 len -= size; 2129 offset += size; 2130 tx_head++; 2131 2132 size = umin(len, bp->max_tx_length); 2133 } 2134 2135 /* Then, map paged data from fragments */ 2136 for (f = 0; f < nr_frags; f++) { 2137 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2138 2139 len = skb_frag_size(frag); 2140 offset = 0; 2141 while (len) { 2142 size = umin(len, bp->max_tx_length); 2143 tx_skb = macb_tx_skb(queue, tx_head); 2144 2145 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 2146 offset, size, DMA_TO_DEVICE); 2147 if (dma_mapping_error(&bp->pdev->dev, mapping)) 2148 goto dma_error; 2149 2150 /* Save info to properly release resources */ 2151 tx_skb->skb = NULL; 2152 tx_skb->mapping = mapping; 2153 tx_skb->size = size; 2154 tx_skb->mapped_as_page = true; 2155 2156 len -= size; 2157 offset += size; 2158 tx_head++; 2159 } 2160 } 2161 2162 /* Should never happen */ 2163 if (unlikely(!tx_skb)) { 2164 netdev_err(bp->dev, "BUG! empty skb!\n"); 2165 return 0; 2166 } 2167 2168 /* This is the last buffer of the frame: save socket buffer */ 2169 tx_skb->skb = skb; 2170 2171 /* Update TX ring: update buffer descriptors in reverse order 2172 * to avoid race condition 2173 */ 2174 2175 /* Set 'TX_USED' bit in buffer descriptor at tx_head position 2176 * to set the end of TX queue 2177 */ 2178 i = tx_head; 2179 ctrl = MACB_BIT(TX_USED); 2180 desc = macb_tx_desc(queue, i); 2181 desc->ctrl = ctrl; 2182 2183 if (lso_ctrl) { 2184 if (lso_ctrl == MACB_LSO_UFO_ENABLE) 2185 /* include header and FCS in value given to h/w */ 2186 mss_mfs = skb_shinfo(skb)->gso_size + 2187 skb_transport_offset(skb) + 2188 ETH_FCS_LEN; 2189 else /* TSO */ { 2190 mss_mfs = skb_shinfo(skb)->gso_size; 2191 /* TCP Sequence Number Source Select 2192 * can be set only for TSO 2193 */ 2194 seq_ctrl = 0; 2195 } 2196 } 2197 2198 do { 2199 i--; 2200 tx_skb = macb_tx_skb(queue, i); 2201 desc = macb_tx_desc(queue, i); 2202 2203 ctrl = (u32)tx_skb->size; 2204 if (eof) { 2205 ctrl |= MACB_BIT(TX_LAST); 2206 eof = 0; 2207 } 2208 if (unlikely(macb_tx_ring_wrap(bp, i) == bp->tx_ring_size - 1)) 2209 ctrl |= MACB_BIT(TX_WRAP); 2210 2211 /* First descriptor is header descriptor */ 2212 if (i == queue->tx_head) { 2213 ctrl |= MACB_BF(TX_LSO, lso_ctrl); 2214 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl); 2215 if ((bp->dev->features & NETIF_F_HW_CSUM) && 2216 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl && 2217 !ptp_one_step_sync(skb)) 2218 ctrl |= MACB_BIT(TX_NOCRC); 2219 } else 2220 /* Only set MSS/MFS on payload descriptors 2221 * (second or later descriptor) 2222 */ 2223 ctrl |= MACB_BF(MSS_MFS, mss_mfs); 2224 2225 /* Set TX buffer descriptor */ 2226 macb_set_addr(bp, desc, tx_skb->mapping); 2227 /* desc->addr must be visible to hardware before clearing 2228 * 'TX_USED' bit in desc->ctrl. 2229 */ 2230 wmb(); 2231 desc->ctrl = ctrl; 2232 } while (i != queue->tx_head); 2233 2234 queue->tx_head = tx_head; 2235 2236 return 0; 2237 2238dma_error: 2239 netdev_err(bp->dev, "TX DMA map failed\n"); 2240 2241 for (i = queue->tx_head; i != tx_head; i++) { 2242 tx_skb = macb_tx_skb(queue, i); 2243 2244 macb_tx_unmap(bp, tx_skb, 0); 2245 } 2246 2247 return -ENOMEM; 2248} 2249 2250static netdev_features_t macb_features_check(struct sk_buff *skb, 2251 struct net_device *dev, 2252 netdev_features_t features) 2253{ 2254 unsigned int nr_frags, f; 2255 unsigned int hdrlen; 2256 2257 /* Validate LSO compatibility */ 2258 2259 /* there is only one buffer or protocol is not UDP */ 2260 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP)) 2261 return features; 2262 2263 /* length of header */ 2264 hdrlen = skb_transport_offset(skb); 2265 2266 /* For UFO only: 2267 * When software supplies two or more payload buffers all payload buffers 2268 * apart from the last must be a multiple of 8 bytes in size. 2269 */ 2270 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN)) 2271 return features & ~MACB_NETIF_LSO; 2272 2273 nr_frags = skb_shinfo(skb)->nr_frags; 2274 /* No need to check last fragment */ 2275 nr_frags--; 2276 for (f = 0; f < nr_frags; f++) { 2277 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2278 2279 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN)) 2280 return features & ~MACB_NETIF_LSO; 2281 } 2282 return features; 2283} 2284 2285static inline int macb_clear_csum(struct sk_buff *skb) 2286{ 2287 /* no change for packets without checksum offloading */ 2288 if (skb->ip_summed != CHECKSUM_PARTIAL) 2289 return 0; 2290 2291 /* make sure we can modify the header */ 2292 if (unlikely(skb_cow_head(skb, 0))) 2293 return -1; 2294 2295 /* initialize checksum field 2296 * This is required - at least for Zynq, which otherwise calculates 2297 * wrong UDP header checksums for UDP packets with UDP data len <=2 2298 */ 2299 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0; 2300 return 0; 2301} 2302 2303static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev) 2304{ 2305 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) || 2306 skb_is_nonlinear(*skb); 2307 int padlen = ETH_ZLEN - (*skb)->len; 2308 int tailroom = skb_tailroom(*skb); 2309 struct sk_buff *nskb; 2310 u32 fcs; 2311 2312 if (!(ndev->features & NETIF_F_HW_CSUM) || 2313 !((*skb)->ip_summed != CHECKSUM_PARTIAL) || 2314 skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb)) 2315 return 0; 2316 2317 if (padlen <= 0) { 2318 /* FCS could be appeded to tailroom. */ 2319 if (tailroom >= ETH_FCS_LEN) 2320 goto add_fcs; 2321 /* No room for FCS, need to reallocate skb. */ 2322 else 2323 padlen = ETH_FCS_LEN; 2324 } else { 2325 /* Add room for FCS. */ 2326 padlen += ETH_FCS_LEN; 2327 } 2328 2329 if (cloned || tailroom < padlen) { 2330 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC); 2331 if (!nskb) 2332 return -ENOMEM; 2333 2334 dev_consume_skb_any(*skb); 2335 *skb = nskb; 2336 } 2337 2338 if (padlen > ETH_FCS_LEN) 2339 skb_put_zero(*skb, padlen - ETH_FCS_LEN); 2340 2341add_fcs: 2342 /* set FCS to packet */ 2343 fcs = crc32_le(~0, (*skb)->data, (*skb)->len); 2344 fcs = ~fcs; 2345 2346 skb_put_u8(*skb, fcs & 0xff); 2347 skb_put_u8(*skb, (fcs >> 8) & 0xff); 2348 skb_put_u8(*skb, (fcs >> 16) & 0xff); 2349 skb_put_u8(*skb, (fcs >> 24) & 0xff); 2350 2351 return 0; 2352} 2353 2354static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) 2355{ 2356 u16 queue_index = skb_get_queue_mapping(skb); 2357 struct macb *bp = netdev_priv(dev); 2358 struct macb_queue *queue = &bp->queues[queue_index]; 2359 unsigned int desc_cnt, nr_frags, frag_size, f; 2360 unsigned int hdrlen; 2361 unsigned long flags; 2362 bool is_lso; 2363 netdev_tx_t ret = NETDEV_TX_OK; 2364 2365 if (macb_clear_csum(skb)) { 2366 dev_kfree_skb_any(skb); 2367 return ret; 2368 } 2369 2370 if (macb_pad_and_fcs(&skb, dev)) { 2371 dev_kfree_skb_any(skb); 2372 return ret; 2373 } 2374 2375 if (macb_dma_ptp(bp) && 2376 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) 2377 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 2378 2379 is_lso = (skb_shinfo(skb)->gso_size != 0); 2380 2381 if (is_lso) { 2382 /* length of headers */ 2383 if (ip_hdr(skb)->protocol == IPPROTO_UDP) 2384 /* only queue eth + ip headers separately for UDP */ 2385 hdrlen = skb_transport_offset(skb); 2386 else 2387 hdrlen = skb_tcp_all_headers(skb); 2388 if (skb_headlen(skb) < hdrlen) { 2389 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n"); 2390 /* if this is required, would need to copy to single buffer */ 2391 return NETDEV_TX_BUSY; 2392 } 2393 } else 2394 hdrlen = umin(skb_headlen(skb), bp->max_tx_length); 2395 2396#if defined(DEBUG) && defined(VERBOSE_DEBUG) 2397 netdev_vdbg(bp->dev, 2398 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n", 2399 queue_index, skb->len, skb->head, skb->data, 2400 skb_tail_pointer(skb), skb_end_pointer(skb)); 2401 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1, 2402 skb->data, 16, true); 2403#endif 2404 2405 /* Count how many TX buffer descriptors are needed to send this 2406 * socket buffer: skb fragments of jumbo frames may need to be 2407 * split into many buffer descriptors. 2408 */ 2409 if (is_lso && (skb_headlen(skb) > hdrlen)) 2410 /* extra header descriptor if also payload in first buffer */ 2411 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1; 2412 else 2413 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length); 2414 nr_frags = skb_shinfo(skb)->nr_frags; 2415 for (f = 0; f < nr_frags; f++) { 2416 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]); 2417 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length); 2418 } 2419 2420 spin_lock_irqsave(&queue->tx_ptr_lock, flags); 2421 2422 /* This is a hard error, log it. */ 2423 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, 2424 bp->tx_ring_size) < desc_cnt) { 2425 netif_stop_subqueue(dev, queue_index); 2426 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", 2427 queue->tx_head, queue->tx_tail); 2428 ret = NETDEV_TX_BUSY; 2429 goto unlock; 2430 } 2431 2432 /* Map socket buffer for DMA transfer */ 2433 if (macb_tx_map(bp, queue, skb, hdrlen)) { 2434 dev_kfree_skb_any(skb); 2435 goto unlock; 2436 } 2437 2438 /* Make newly initialized descriptor visible to hardware */ 2439 wmb(); 2440 skb_tx_timestamp(skb); 2441 netdev_tx_sent_queue(netdev_get_tx_queue(bp->dev, queue_index), 2442 skb->len); 2443 2444 spin_lock(&bp->lock); 2445 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); 2446 spin_unlock(&bp->lock); 2447 2448 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) 2449 netif_stop_subqueue(dev, queue_index); 2450 2451unlock: 2452 spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); 2453 2454 return ret; 2455} 2456 2457static void macb_init_rx_buffer_size(struct macb *bp, size_t size) 2458{ 2459 if (!macb_is_gem(bp)) { 2460 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; 2461 } else { 2462 bp->rx_buffer_size = size; 2463 2464 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { 2465 netdev_dbg(bp->dev, 2466 "RX buffer must be multiple of %d bytes, expanding\n", 2467 RX_BUFFER_MULTIPLE); 2468 bp->rx_buffer_size = 2469 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE); 2470 } 2471 } 2472 2473 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n", 2474 bp->dev->mtu, bp->rx_buffer_size); 2475} 2476 2477static void gem_free_rx_buffers(struct macb *bp) 2478{ 2479 struct sk_buff *skb; 2480 struct macb_dma_desc *desc; 2481 struct macb_queue *queue; 2482 dma_addr_t addr; 2483 unsigned int q; 2484 int i; 2485 2486 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2487 if (!queue->rx_skbuff) 2488 continue; 2489 2490 for (i = 0; i < bp->rx_ring_size; i++) { 2491 skb = queue->rx_skbuff[i]; 2492 2493 if (!skb) 2494 continue; 2495 2496 desc = macb_rx_desc(queue, i); 2497 addr = macb_get_addr(bp, desc); 2498 2499 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, 2500 DMA_FROM_DEVICE); 2501 dev_kfree_skb_any(skb); 2502 skb = NULL; 2503 } 2504 2505 kfree(queue->rx_skbuff); 2506 queue->rx_skbuff = NULL; 2507 } 2508} 2509 2510static void macb_free_rx_buffers(struct macb *bp) 2511{ 2512 struct macb_queue *queue = &bp->queues[0]; 2513 2514 if (queue->rx_buffers) { 2515 dma_free_coherent(&bp->pdev->dev, 2516 bp->rx_ring_size * bp->rx_buffer_size, 2517 queue->rx_buffers, queue->rx_buffers_dma); 2518 queue->rx_buffers = NULL; 2519 } 2520} 2521 2522static unsigned int macb_tx_ring_size_per_queue(struct macb *bp) 2523{ 2524 return macb_dma_desc_get_size(bp) * bp->tx_ring_size + bp->tx_bd_rd_prefetch; 2525} 2526 2527static unsigned int macb_rx_ring_size_per_queue(struct macb *bp) 2528{ 2529 return macb_dma_desc_get_size(bp) * bp->rx_ring_size + bp->rx_bd_rd_prefetch; 2530} 2531 2532static void macb_free_consistent(struct macb *bp) 2533{ 2534 struct device *dev = &bp->pdev->dev; 2535 struct macb_queue *queue; 2536 unsigned int q; 2537 size_t size; 2538 2539 if (bp->rx_ring_tieoff) { 2540 dma_free_coherent(dev, macb_dma_desc_get_size(bp), 2541 bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma); 2542 bp->rx_ring_tieoff = NULL; 2543 } 2544 2545 bp->macbgem_ops.mog_free_rx_buffers(bp); 2546 2547 size = bp->num_queues * macb_tx_ring_size_per_queue(bp); 2548 dma_free_coherent(dev, size, bp->queues[0].tx_ring, bp->queues[0].tx_ring_dma); 2549 2550 size = bp->num_queues * macb_rx_ring_size_per_queue(bp); 2551 dma_free_coherent(dev, size, bp->queues[0].rx_ring, bp->queues[0].rx_ring_dma); 2552 2553 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2554 kfree(queue->tx_skb); 2555 queue->tx_skb = NULL; 2556 queue->tx_ring = NULL; 2557 queue->rx_ring = NULL; 2558 } 2559} 2560 2561static int gem_alloc_rx_buffers(struct macb *bp) 2562{ 2563 struct macb_queue *queue; 2564 unsigned int q; 2565 int size; 2566 2567 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2568 size = bp->rx_ring_size * sizeof(struct sk_buff *); 2569 queue->rx_skbuff = kzalloc(size, GFP_KERNEL); 2570 if (!queue->rx_skbuff) 2571 return -ENOMEM; 2572 else 2573 netdev_dbg(bp->dev, 2574 "Allocated %d RX struct sk_buff entries at %p\n", 2575 bp->rx_ring_size, queue->rx_skbuff); 2576 } 2577 return 0; 2578} 2579 2580static int macb_alloc_rx_buffers(struct macb *bp) 2581{ 2582 struct macb_queue *queue = &bp->queues[0]; 2583 int size; 2584 2585 size = bp->rx_ring_size * bp->rx_buffer_size; 2586 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size, 2587 &queue->rx_buffers_dma, GFP_KERNEL); 2588 if (!queue->rx_buffers) 2589 return -ENOMEM; 2590 2591 netdev_dbg(bp->dev, 2592 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n", 2593 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers); 2594 return 0; 2595} 2596 2597static int macb_alloc_consistent(struct macb *bp) 2598{ 2599 struct device *dev = &bp->pdev->dev; 2600 dma_addr_t tx_dma, rx_dma; 2601 struct macb_queue *queue; 2602 unsigned int q; 2603 void *tx, *rx; 2604 size_t size; 2605 2606 /* 2607 * Upper 32-bits of Tx/Rx DMA descriptor for each queues much match! 2608 * We cannot enforce this guarantee, the best we can do is do a single 2609 * allocation and hope it will land into alloc_pages() that guarantees 2610 * natural alignment of physical addresses. 2611 */ 2612 2613 size = bp->num_queues * macb_tx_ring_size_per_queue(bp); 2614 tx = dma_alloc_coherent(dev, size, &tx_dma, GFP_KERNEL); 2615 if (!tx || upper_32_bits(tx_dma) != upper_32_bits(tx_dma + size - 1)) 2616 goto out_err; 2617 netdev_dbg(bp->dev, "Allocated %zu bytes for %u TX rings at %08lx (mapped %p)\n", 2618 size, bp->num_queues, (unsigned long)tx_dma, tx); 2619 2620 size = bp->num_queues * macb_rx_ring_size_per_queue(bp); 2621 rx = dma_alloc_coherent(dev, size, &rx_dma, GFP_KERNEL); 2622 if (!rx || upper_32_bits(rx_dma) != upper_32_bits(rx_dma + size - 1)) 2623 goto out_err; 2624 netdev_dbg(bp->dev, "Allocated %zu bytes for %u RX rings at %08lx (mapped %p)\n", 2625 size, bp->num_queues, (unsigned long)rx_dma, rx); 2626 2627 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2628 queue->tx_ring = tx + macb_tx_ring_size_per_queue(bp) * q; 2629 queue->tx_ring_dma = tx_dma + macb_tx_ring_size_per_queue(bp) * q; 2630 2631 queue->rx_ring = rx + macb_rx_ring_size_per_queue(bp) * q; 2632 queue->rx_ring_dma = rx_dma + macb_rx_ring_size_per_queue(bp) * q; 2633 2634 size = bp->tx_ring_size * sizeof(struct macb_tx_skb); 2635 queue->tx_skb = kmalloc(size, GFP_KERNEL); 2636 if (!queue->tx_skb) 2637 goto out_err; 2638 } 2639 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) 2640 goto out_err; 2641 2642 /* Required for tie off descriptor for PM cases */ 2643 if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) { 2644 bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev, 2645 macb_dma_desc_get_size(bp), 2646 &bp->rx_ring_tieoff_dma, 2647 GFP_KERNEL); 2648 if (!bp->rx_ring_tieoff) 2649 goto out_err; 2650 } 2651 2652 return 0; 2653 2654out_err: 2655 macb_free_consistent(bp); 2656 return -ENOMEM; 2657} 2658 2659static void macb_init_tieoff(struct macb *bp) 2660{ 2661 struct macb_dma_desc *desc = bp->rx_ring_tieoff; 2662 2663 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) 2664 return; 2665 /* Setup a wrapping descriptor with no free slots 2666 * (WRAP and USED) to tie off/disable unused RX queues. 2667 */ 2668 macb_set_addr(bp, desc, MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED)); 2669 desc->ctrl = 0; 2670} 2671 2672static void gem_init_rx_ring(struct macb_queue *queue) 2673{ 2674 queue->rx_tail = 0; 2675 queue->rx_prepared_head = 0; 2676 2677 gem_rx_refill(queue); 2678} 2679 2680static void gem_init_rings(struct macb *bp) 2681{ 2682 struct macb_queue *queue; 2683 struct macb_dma_desc *desc = NULL; 2684 unsigned int q; 2685 int i; 2686 2687 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2688 for (i = 0; i < bp->tx_ring_size; i++) { 2689 desc = macb_tx_desc(queue, i); 2690 macb_set_addr(bp, desc, 0); 2691 desc->ctrl = MACB_BIT(TX_USED); 2692 } 2693 desc->ctrl |= MACB_BIT(TX_WRAP); 2694 queue->tx_head = 0; 2695 queue->tx_tail = 0; 2696 2697 gem_init_rx_ring(queue); 2698 } 2699 2700 macb_init_tieoff(bp); 2701} 2702 2703static void macb_init_rings(struct macb *bp) 2704{ 2705 int i; 2706 struct macb_dma_desc *desc = NULL; 2707 2708 macb_init_rx_ring(&bp->queues[0]); 2709 2710 for (i = 0; i < bp->tx_ring_size; i++) { 2711 desc = macb_tx_desc(&bp->queues[0], i); 2712 macb_set_addr(bp, desc, 0); 2713 desc->ctrl = MACB_BIT(TX_USED); 2714 } 2715 bp->queues[0].tx_head = 0; 2716 bp->queues[0].tx_tail = 0; 2717 desc->ctrl |= MACB_BIT(TX_WRAP); 2718 2719 macb_init_tieoff(bp); 2720} 2721 2722static void macb_reset_hw(struct macb *bp) 2723{ 2724 struct macb_queue *queue; 2725 unsigned int q; 2726 u32 ctrl = macb_readl(bp, NCR); 2727 2728 /* Disable RX and TX (XXX: Should we halt the transmission 2729 * more gracefully?) 2730 */ 2731 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE)); 2732 2733 /* Clear the stats registers (XXX: Update stats first?) */ 2734 ctrl |= MACB_BIT(CLRSTAT); 2735 2736 macb_writel(bp, NCR, ctrl); 2737 2738 /* Clear all status flags */ 2739 macb_writel(bp, TSR, -1); 2740 macb_writel(bp, RSR, -1); 2741 2742 /* Disable RX partial store and forward and reset watermark value */ 2743 gem_writel(bp, PBUFRXCUT, 0); 2744 2745 /* Disable all interrupts */ 2746 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2747 queue_writel(queue, IDR, -1); 2748 queue_readl(queue, ISR); 2749 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 2750 queue_writel(queue, ISR, -1); 2751 } 2752} 2753 2754static u32 gem_mdc_clk_div(struct macb *bp) 2755{ 2756 u32 config; 2757 unsigned long pclk_hz = clk_get_rate(bp->pclk); 2758 2759 if (pclk_hz <= 20000000) 2760 config = GEM_BF(CLK, GEM_CLK_DIV8); 2761 else if (pclk_hz <= 40000000) 2762 config = GEM_BF(CLK, GEM_CLK_DIV16); 2763 else if (pclk_hz <= 80000000) 2764 config = GEM_BF(CLK, GEM_CLK_DIV32); 2765 else if (pclk_hz <= 120000000) 2766 config = GEM_BF(CLK, GEM_CLK_DIV48); 2767 else if (pclk_hz <= 160000000) 2768 config = GEM_BF(CLK, GEM_CLK_DIV64); 2769 else if (pclk_hz <= 240000000) 2770 config = GEM_BF(CLK, GEM_CLK_DIV96); 2771 else if (pclk_hz <= 320000000) 2772 config = GEM_BF(CLK, GEM_CLK_DIV128); 2773 else 2774 config = GEM_BF(CLK, GEM_CLK_DIV224); 2775 2776 return config; 2777} 2778 2779static u32 macb_mdc_clk_div(struct macb *bp) 2780{ 2781 u32 config; 2782 unsigned long pclk_hz; 2783 2784 if (macb_is_gem(bp)) 2785 return gem_mdc_clk_div(bp); 2786 2787 pclk_hz = clk_get_rate(bp->pclk); 2788 if (pclk_hz <= 20000000) 2789 config = MACB_BF(CLK, MACB_CLK_DIV8); 2790 else if (pclk_hz <= 40000000) 2791 config = MACB_BF(CLK, MACB_CLK_DIV16); 2792 else if (pclk_hz <= 80000000) 2793 config = MACB_BF(CLK, MACB_CLK_DIV32); 2794 else 2795 config = MACB_BF(CLK, MACB_CLK_DIV64); 2796 2797 return config; 2798} 2799 2800/* Get the DMA bus width field of the network configuration register that we 2801 * should program. We find the width from decoding the design configuration 2802 * register to find the maximum supported data bus width. 2803 */ 2804static u32 macb_dbw(struct macb *bp) 2805{ 2806 if (!macb_is_gem(bp)) 2807 return 0; 2808 2809 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) { 2810 case 4: 2811 return GEM_BF(DBW, GEM_DBW128); 2812 case 2: 2813 return GEM_BF(DBW, GEM_DBW64); 2814 case 1: 2815 default: 2816 return GEM_BF(DBW, GEM_DBW32); 2817 } 2818} 2819 2820/* Configure the receive DMA engine 2821 * - use the correct receive buffer size 2822 * - set best burst length for DMA operations 2823 * (if not supported by FIFO, it will fallback to default) 2824 * - set both rx/tx packet buffers to full memory size 2825 * These are configurable parameters for GEM. 2826 */ 2827static void macb_configure_dma(struct macb *bp) 2828{ 2829 struct macb_queue *queue; 2830 u32 buffer_size; 2831 unsigned int q; 2832 u32 dmacfg; 2833 2834 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; 2835 if (macb_is_gem(bp)) { 2836 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); 2837 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 2838 if (q) 2839 queue_writel(queue, RBQS, buffer_size); 2840 else 2841 dmacfg |= GEM_BF(RXBS, buffer_size); 2842 } 2843 if (bp->dma_burst_length) 2844 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg); 2845 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L); 2846 dmacfg &= ~GEM_BIT(ENDIA_PKT); 2847 2848 if (bp->native_io) 2849 dmacfg &= ~GEM_BIT(ENDIA_DESC); 2850 else 2851 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */ 2852 2853 if (bp->dev->features & NETIF_F_HW_CSUM) 2854 dmacfg |= GEM_BIT(TXCOEN); 2855 else 2856 dmacfg &= ~GEM_BIT(TXCOEN); 2857 2858 dmacfg &= ~GEM_BIT(ADDR64); 2859 if (macb_dma64(bp)) 2860 dmacfg |= GEM_BIT(ADDR64); 2861 if (macb_dma_ptp(bp)) 2862 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT); 2863 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", 2864 dmacfg); 2865 gem_writel(bp, DMACFG, dmacfg); 2866 } 2867} 2868 2869static void macb_init_hw(struct macb *bp) 2870{ 2871 u32 config; 2872 2873 macb_reset_hw(bp); 2874 macb_set_hwaddr(bp); 2875 2876 config = macb_mdc_clk_div(bp); 2877 /* Make eth data aligned. 2878 * If RSC capable, that offset is ignored by HW. 2879 */ 2880 if (!(bp->caps & MACB_CAPS_RSC)) 2881 config |= MACB_BF(RBOF, NET_IP_ALIGN); 2882 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */ 2883 if (bp->caps & MACB_CAPS_JUMBO) 2884 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */ 2885 else 2886 config |= MACB_BIT(BIG); /* Receive oversized frames */ 2887 if (bp->dev->flags & IFF_PROMISC) 2888 config |= MACB_BIT(CAF); /* Copy All Frames */ 2889 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM) 2890 config |= GEM_BIT(RXCOEN); 2891 if (!(bp->dev->flags & IFF_BROADCAST)) 2892 config |= MACB_BIT(NBC); /* No BroadCast */ 2893 config |= macb_dbw(bp); 2894 macb_writel(bp, NCFGR, config); 2895 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 2896 gem_writel(bp, JML, bp->jumbo_max_len); 2897 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK; 2898 if (bp->caps & MACB_CAPS_JUMBO) 2899 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK; 2900 2901 macb_configure_dma(bp); 2902 2903 /* Enable RX partial store and forward and set watermark */ 2904 if (bp->rx_watermark) 2905 gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU))); 2906} 2907 2908/* The hash address register is 64 bits long and takes up two 2909 * locations in the memory map. The least significant bits are stored 2910 * in EMAC_HSL and the most significant bits in EMAC_HSH. 2911 * 2912 * The unicast hash enable and the multicast hash enable bits in the 2913 * network configuration register enable the reception of hash matched 2914 * frames. The destination address is reduced to a 6 bit index into 2915 * the 64 bit hash register using the following hash function. The 2916 * hash function is an exclusive or of every sixth bit of the 2917 * destination address. 2918 * 2919 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47] 2920 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46] 2921 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45] 2922 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44] 2923 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43] 2924 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42] 2925 * 2926 * da[0] represents the least significant bit of the first byte 2927 * received, that is, the multicast/unicast indicator, and da[47] 2928 * represents the most significant bit of the last byte received. If 2929 * the hash index, hi[n], points to a bit that is set in the hash 2930 * register then the frame will be matched according to whether the 2931 * frame is multicast or unicast. A multicast match will be signalled 2932 * if the multicast hash enable bit is set, da[0] is 1 and the hash 2933 * index points to a bit set in the hash register. A unicast match 2934 * will be signalled if the unicast hash enable bit is set, da[0] is 0 2935 * and the hash index points to a bit set in the hash register. To 2936 * receive all multicast frames, the hash register should be set with 2937 * all ones and the multicast hash enable bit should be set in the 2938 * network configuration register. 2939 */ 2940 2941static inline int hash_bit_value(int bitnr, __u8 *addr) 2942{ 2943 if (addr[bitnr / 8] & (1 << (bitnr % 8))) 2944 return 1; 2945 return 0; 2946} 2947 2948/* Return the hash index value for the specified address. */ 2949static int hash_get_index(__u8 *addr) 2950{ 2951 int i, j, bitval; 2952 int hash_index = 0; 2953 2954 for (j = 0; j < 6; j++) { 2955 for (i = 0, bitval = 0; i < 8; i++) 2956 bitval ^= hash_bit_value(i * 6 + j, addr); 2957 2958 hash_index |= (bitval << j); 2959 } 2960 2961 return hash_index; 2962} 2963 2964/* Add multicast addresses to the internal multicast-hash table. */ 2965static void macb_sethashtable(struct net_device *dev) 2966{ 2967 struct netdev_hw_addr *ha; 2968 unsigned long mc_filter[2]; 2969 unsigned int bitnr; 2970 struct macb *bp = netdev_priv(dev); 2971 2972 mc_filter[0] = 0; 2973 mc_filter[1] = 0; 2974 2975 netdev_for_each_mc_addr(ha, dev) { 2976 bitnr = hash_get_index(ha->addr); 2977 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31); 2978 } 2979 2980 macb_or_gem_writel(bp, HRB, mc_filter[0]); 2981 macb_or_gem_writel(bp, HRT, mc_filter[1]); 2982} 2983 2984/* Enable/Disable promiscuous and multicast modes. */ 2985static void macb_set_rx_mode(struct net_device *dev) 2986{ 2987 unsigned long cfg; 2988 struct macb *bp = netdev_priv(dev); 2989 2990 cfg = macb_readl(bp, NCFGR); 2991 2992 if (dev->flags & IFF_PROMISC) { 2993 /* Enable promiscuous mode */ 2994 cfg |= MACB_BIT(CAF); 2995 2996 /* Disable RX checksum offload */ 2997 if (macb_is_gem(bp)) 2998 cfg &= ~GEM_BIT(RXCOEN); 2999 } else { 3000 /* Disable promiscuous mode */ 3001 cfg &= ~MACB_BIT(CAF); 3002 3003 /* Enable RX checksum offload only if requested */ 3004 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM) 3005 cfg |= GEM_BIT(RXCOEN); 3006 } 3007 3008 if (dev->flags & IFF_ALLMULTI) { 3009 /* Enable all multicast mode */ 3010 macb_or_gem_writel(bp, HRB, -1); 3011 macb_or_gem_writel(bp, HRT, -1); 3012 cfg |= MACB_BIT(NCFGR_MTI); 3013 } else if (!netdev_mc_empty(dev)) { 3014 /* Enable specific multicasts */ 3015 macb_sethashtable(dev); 3016 cfg |= MACB_BIT(NCFGR_MTI); 3017 } else if (dev->flags & (~IFF_ALLMULTI)) { 3018 /* Disable all multicast mode */ 3019 macb_or_gem_writel(bp, HRB, 0); 3020 macb_or_gem_writel(bp, HRT, 0); 3021 cfg &= ~MACB_BIT(NCFGR_MTI); 3022 } 3023 3024 macb_writel(bp, NCFGR, cfg); 3025} 3026 3027static int macb_open(struct net_device *dev) 3028{ 3029 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN; 3030 struct macb *bp = netdev_priv(dev); 3031 struct macb_queue *queue; 3032 unsigned int q; 3033 int err; 3034 3035 netdev_dbg(bp->dev, "open\n"); 3036 3037 err = pm_runtime_resume_and_get(&bp->pdev->dev); 3038 if (err < 0) 3039 return err; 3040 3041 /* RX buffers initialization */ 3042 macb_init_rx_buffer_size(bp, bufsz); 3043 3044 err = macb_alloc_consistent(bp); 3045 if (err) { 3046 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n", 3047 err); 3048 goto pm_exit; 3049 } 3050 3051 bp->macbgem_ops.mog_init_rings(bp); 3052 macb_init_buffers(bp); 3053 3054 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3055 napi_enable(&queue->napi_rx); 3056 napi_enable(&queue->napi_tx); 3057 } 3058 3059 macb_init_hw(bp); 3060 3061 err = phy_set_mode_ext(bp->phy, PHY_MODE_ETHERNET, bp->phy_interface); 3062 if (err) 3063 goto reset_hw; 3064 3065 err = phy_power_on(bp->phy); 3066 if (err) 3067 goto reset_hw; 3068 3069 err = macb_phylink_connect(bp); 3070 if (err) 3071 goto phy_off; 3072 3073 netif_tx_start_all_queues(dev); 3074 3075 if (bp->ptp_info) 3076 bp->ptp_info->ptp_init(dev); 3077 3078 return 0; 3079 3080phy_off: 3081 phy_power_off(bp->phy); 3082 3083reset_hw: 3084 macb_reset_hw(bp); 3085 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3086 napi_disable(&queue->napi_rx); 3087 napi_disable(&queue->napi_tx); 3088 } 3089 macb_free_consistent(bp); 3090pm_exit: 3091 pm_runtime_put_sync(&bp->pdev->dev); 3092 return err; 3093} 3094 3095static int macb_close(struct net_device *dev) 3096{ 3097 struct macb *bp = netdev_priv(dev); 3098 struct macb_queue *queue; 3099 unsigned long flags; 3100 unsigned int q; 3101 3102 netif_tx_stop_all_queues(dev); 3103 3104 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3105 napi_disable(&queue->napi_rx); 3106 napi_disable(&queue->napi_tx); 3107 netdev_tx_reset_queue(netdev_get_tx_queue(dev, q)); 3108 } 3109 3110 phylink_stop(bp->phylink); 3111 phylink_disconnect_phy(bp->phylink); 3112 3113 phy_power_off(bp->phy); 3114 3115 spin_lock_irqsave(&bp->lock, flags); 3116 macb_reset_hw(bp); 3117 netif_carrier_off(dev); 3118 spin_unlock_irqrestore(&bp->lock, flags); 3119 3120 macb_free_consistent(bp); 3121 3122 if (bp->ptp_info) 3123 bp->ptp_info->ptp_remove(dev); 3124 3125 pm_runtime_put(&bp->pdev->dev); 3126 3127 return 0; 3128} 3129 3130static int macb_change_mtu(struct net_device *dev, int new_mtu) 3131{ 3132 if (netif_running(dev)) 3133 return -EBUSY; 3134 3135 WRITE_ONCE(dev->mtu, new_mtu); 3136 3137 return 0; 3138} 3139 3140static int macb_set_mac_addr(struct net_device *dev, void *addr) 3141{ 3142 int err; 3143 3144 err = eth_mac_addr(dev, addr); 3145 if (err < 0) 3146 return err; 3147 3148 macb_set_hwaddr(netdev_priv(dev)); 3149 return 0; 3150} 3151 3152static void gem_update_stats(struct macb *bp) 3153{ 3154 struct macb_queue *queue; 3155 unsigned int i, q, idx; 3156 unsigned long *stat; 3157 3158 u64 *p = &bp->hw_stats.gem.tx_octets; 3159 3160 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) { 3161 u32 offset = gem_statistics[i].offset; 3162 u64 val = bp->macb_reg_readl(bp, offset); 3163 3164 bp->ethtool_stats[i] += val; 3165 *p += val; 3166 3167 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) { 3168 /* Add GEM_OCTTXH, GEM_OCTRXH */ 3169 val = bp->macb_reg_readl(bp, offset + 4); 3170 bp->ethtool_stats[i] += ((u64)val) << 32; 3171 *p += ((u64)val) << 32; 3172 } 3173 } 3174 3175 idx = GEM_STATS_LEN; 3176 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) 3177 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat) 3178 bp->ethtool_stats[idx++] = *stat; 3179} 3180 3181static void gem_get_stats(struct macb *bp, struct rtnl_link_stats64 *nstat) 3182{ 3183 struct gem_stats *hwstat = &bp->hw_stats.gem; 3184 3185 spin_lock_irq(&bp->stats_lock); 3186 if (netif_running(bp->dev)) 3187 gem_update_stats(bp); 3188 3189 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors + 3190 hwstat->rx_alignment_errors + 3191 hwstat->rx_resource_errors + 3192 hwstat->rx_overruns + 3193 hwstat->rx_oversize_frames + 3194 hwstat->rx_jabbers + 3195 hwstat->rx_undersized_frames + 3196 hwstat->rx_length_field_frame_errors); 3197 nstat->tx_errors = (hwstat->tx_late_collisions + 3198 hwstat->tx_excessive_collisions + 3199 hwstat->tx_underrun + 3200 hwstat->tx_carrier_sense_errors); 3201 nstat->multicast = hwstat->rx_multicast_frames; 3202 nstat->collisions = (hwstat->tx_single_collision_frames + 3203 hwstat->tx_multiple_collision_frames + 3204 hwstat->tx_excessive_collisions); 3205 nstat->rx_length_errors = (hwstat->rx_oversize_frames + 3206 hwstat->rx_jabbers + 3207 hwstat->rx_undersized_frames + 3208 hwstat->rx_length_field_frame_errors); 3209 nstat->rx_over_errors = hwstat->rx_resource_errors; 3210 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors; 3211 nstat->rx_frame_errors = hwstat->rx_alignment_errors; 3212 nstat->rx_fifo_errors = hwstat->rx_overruns; 3213 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions; 3214 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors; 3215 nstat->tx_fifo_errors = hwstat->tx_underrun; 3216 spin_unlock_irq(&bp->stats_lock); 3217} 3218 3219static void gem_get_ethtool_stats(struct net_device *dev, 3220 struct ethtool_stats *stats, u64 *data) 3221{ 3222 struct macb *bp = netdev_priv(dev); 3223 3224 spin_lock_irq(&bp->stats_lock); 3225 gem_update_stats(bp); 3226 memcpy(data, &bp->ethtool_stats, sizeof(u64) 3227 * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); 3228 spin_unlock_irq(&bp->stats_lock); 3229} 3230 3231static int gem_get_sset_count(struct net_device *dev, int sset) 3232{ 3233 struct macb *bp = netdev_priv(dev); 3234 3235 switch (sset) { 3236 case ETH_SS_STATS: 3237 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN; 3238 default: 3239 return -EOPNOTSUPP; 3240 } 3241} 3242 3243static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) 3244{ 3245 char stat_string[ETH_GSTRING_LEN]; 3246 struct macb *bp = netdev_priv(dev); 3247 struct macb_queue *queue; 3248 unsigned int i; 3249 unsigned int q; 3250 3251 switch (sset) { 3252 case ETH_SS_STATS: 3253 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN) 3254 memcpy(p, gem_statistics[i].stat_string, 3255 ETH_GSTRING_LEN); 3256 3257 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 3258 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { 3259 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", 3260 q, queue_statistics[i].stat_string); 3261 memcpy(p, stat_string, ETH_GSTRING_LEN); 3262 } 3263 } 3264 break; 3265 } 3266} 3267 3268static void macb_get_stats(struct net_device *dev, 3269 struct rtnl_link_stats64 *nstat) 3270{ 3271 struct macb *bp = netdev_priv(dev); 3272 struct macb_stats *hwstat = &bp->hw_stats.macb; 3273 3274 netdev_stats_to_stats64(nstat, &bp->dev->stats); 3275 if (macb_is_gem(bp)) { 3276 gem_get_stats(bp, nstat); 3277 return; 3278 } 3279 3280 /* read stats from hardware */ 3281 spin_lock_irq(&bp->stats_lock); 3282 macb_update_stats(bp); 3283 3284 /* Convert HW stats into netdevice stats */ 3285 nstat->rx_errors = (hwstat->rx_fcs_errors + 3286 hwstat->rx_align_errors + 3287 hwstat->rx_resource_errors + 3288 hwstat->rx_overruns + 3289 hwstat->rx_oversize_pkts + 3290 hwstat->rx_jabbers + 3291 hwstat->rx_undersize_pkts + 3292 hwstat->rx_length_mismatch); 3293 nstat->tx_errors = (hwstat->tx_late_cols + 3294 hwstat->tx_excessive_cols + 3295 hwstat->tx_underruns + 3296 hwstat->tx_carrier_errors + 3297 hwstat->sqe_test_errors); 3298 nstat->collisions = (hwstat->tx_single_cols + 3299 hwstat->tx_multiple_cols + 3300 hwstat->tx_excessive_cols); 3301 nstat->rx_length_errors = (hwstat->rx_oversize_pkts + 3302 hwstat->rx_jabbers + 3303 hwstat->rx_undersize_pkts + 3304 hwstat->rx_length_mismatch); 3305 nstat->rx_over_errors = hwstat->rx_resource_errors + 3306 hwstat->rx_overruns; 3307 nstat->rx_crc_errors = hwstat->rx_fcs_errors; 3308 nstat->rx_frame_errors = hwstat->rx_align_errors; 3309 nstat->rx_fifo_errors = hwstat->rx_overruns; 3310 /* XXX: What does "missed" mean? */ 3311 nstat->tx_aborted_errors = hwstat->tx_excessive_cols; 3312 nstat->tx_carrier_errors = hwstat->tx_carrier_errors; 3313 nstat->tx_fifo_errors = hwstat->tx_underruns; 3314 /* Don't know about heartbeat or window errors... */ 3315 spin_unlock_irq(&bp->stats_lock); 3316} 3317 3318static void macb_get_pause_stats(struct net_device *dev, 3319 struct ethtool_pause_stats *pause_stats) 3320{ 3321 struct macb *bp = netdev_priv(dev); 3322 struct macb_stats *hwstat = &bp->hw_stats.macb; 3323 3324 spin_lock_irq(&bp->stats_lock); 3325 macb_update_stats(bp); 3326 pause_stats->tx_pause_frames = hwstat->tx_pause_frames; 3327 pause_stats->rx_pause_frames = hwstat->rx_pause_frames; 3328 spin_unlock_irq(&bp->stats_lock); 3329} 3330 3331static void gem_get_pause_stats(struct net_device *dev, 3332 struct ethtool_pause_stats *pause_stats) 3333{ 3334 struct macb *bp = netdev_priv(dev); 3335 struct gem_stats *hwstat = &bp->hw_stats.gem; 3336 3337 spin_lock_irq(&bp->stats_lock); 3338 gem_update_stats(bp); 3339 pause_stats->tx_pause_frames = hwstat->tx_pause_frames; 3340 pause_stats->rx_pause_frames = hwstat->rx_pause_frames; 3341 spin_unlock_irq(&bp->stats_lock); 3342} 3343 3344static void macb_get_eth_mac_stats(struct net_device *dev, 3345 struct ethtool_eth_mac_stats *mac_stats) 3346{ 3347 struct macb *bp = netdev_priv(dev); 3348 struct macb_stats *hwstat = &bp->hw_stats.macb; 3349 3350 spin_lock_irq(&bp->stats_lock); 3351 macb_update_stats(bp); 3352 mac_stats->FramesTransmittedOK = hwstat->tx_ok; 3353 mac_stats->SingleCollisionFrames = hwstat->tx_single_cols; 3354 mac_stats->MultipleCollisionFrames = hwstat->tx_multiple_cols; 3355 mac_stats->FramesReceivedOK = hwstat->rx_ok; 3356 mac_stats->FrameCheckSequenceErrors = hwstat->rx_fcs_errors; 3357 mac_stats->AlignmentErrors = hwstat->rx_align_errors; 3358 mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred; 3359 mac_stats->LateCollisions = hwstat->tx_late_cols; 3360 mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_cols; 3361 mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underruns; 3362 mac_stats->CarrierSenseErrors = hwstat->tx_carrier_errors; 3363 mac_stats->FramesLostDueToIntMACRcvError = hwstat->rx_overruns; 3364 mac_stats->InRangeLengthErrors = hwstat->rx_length_mismatch; 3365 mac_stats->FrameTooLongErrors = hwstat->rx_oversize_pkts; 3366 spin_unlock_irq(&bp->stats_lock); 3367} 3368 3369static void gem_get_eth_mac_stats(struct net_device *dev, 3370 struct ethtool_eth_mac_stats *mac_stats) 3371{ 3372 struct macb *bp = netdev_priv(dev); 3373 struct gem_stats *hwstat = &bp->hw_stats.gem; 3374 3375 spin_lock_irq(&bp->stats_lock); 3376 gem_update_stats(bp); 3377 mac_stats->FramesTransmittedOK = hwstat->tx_frames; 3378 mac_stats->SingleCollisionFrames = hwstat->tx_single_collision_frames; 3379 mac_stats->MultipleCollisionFrames = 3380 hwstat->tx_multiple_collision_frames; 3381 mac_stats->FramesReceivedOK = hwstat->rx_frames; 3382 mac_stats->FrameCheckSequenceErrors = 3383 hwstat->rx_frame_check_sequence_errors; 3384 mac_stats->AlignmentErrors = hwstat->rx_alignment_errors; 3385 mac_stats->OctetsTransmittedOK = hwstat->tx_octets; 3386 mac_stats->FramesWithDeferredXmissions = hwstat->tx_deferred_frames; 3387 mac_stats->LateCollisions = hwstat->tx_late_collisions; 3388 mac_stats->FramesAbortedDueToXSColls = hwstat->tx_excessive_collisions; 3389 mac_stats->FramesLostDueToIntMACXmitError = hwstat->tx_underrun; 3390 mac_stats->CarrierSenseErrors = hwstat->tx_carrier_sense_errors; 3391 mac_stats->OctetsReceivedOK = hwstat->rx_octets; 3392 mac_stats->MulticastFramesXmittedOK = hwstat->tx_multicast_frames; 3393 mac_stats->BroadcastFramesXmittedOK = hwstat->tx_broadcast_frames; 3394 mac_stats->MulticastFramesReceivedOK = hwstat->rx_multicast_frames; 3395 mac_stats->BroadcastFramesReceivedOK = hwstat->rx_broadcast_frames; 3396 mac_stats->InRangeLengthErrors = hwstat->rx_length_field_frame_errors; 3397 mac_stats->FrameTooLongErrors = hwstat->rx_oversize_frames; 3398 spin_unlock_irq(&bp->stats_lock); 3399} 3400 3401/* TODO: Report SQE test errors when added to phy_stats */ 3402static void macb_get_eth_phy_stats(struct net_device *dev, 3403 struct ethtool_eth_phy_stats *phy_stats) 3404{ 3405 struct macb *bp = netdev_priv(dev); 3406 struct macb_stats *hwstat = &bp->hw_stats.macb; 3407 3408 spin_lock_irq(&bp->stats_lock); 3409 macb_update_stats(bp); 3410 phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; 3411 spin_unlock_irq(&bp->stats_lock); 3412} 3413 3414static void gem_get_eth_phy_stats(struct net_device *dev, 3415 struct ethtool_eth_phy_stats *phy_stats) 3416{ 3417 struct macb *bp = netdev_priv(dev); 3418 struct gem_stats *hwstat = &bp->hw_stats.gem; 3419 3420 spin_lock_irq(&bp->stats_lock); 3421 gem_update_stats(bp); 3422 phy_stats->SymbolErrorDuringCarrier = hwstat->rx_symbol_errors; 3423 spin_unlock_irq(&bp->stats_lock); 3424} 3425 3426static void macb_get_rmon_stats(struct net_device *dev, 3427 struct ethtool_rmon_stats *rmon_stats, 3428 const struct ethtool_rmon_hist_range **ranges) 3429{ 3430 struct macb *bp = netdev_priv(dev); 3431 struct macb_stats *hwstat = &bp->hw_stats.macb; 3432 3433 spin_lock_irq(&bp->stats_lock); 3434 macb_update_stats(bp); 3435 rmon_stats->undersize_pkts = hwstat->rx_undersize_pkts; 3436 rmon_stats->oversize_pkts = hwstat->rx_oversize_pkts; 3437 rmon_stats->jabbers = hwstat->rx_jabbers; 3438 spin_unlock_irq(&bp->stats_lock); 3439} 3440 3441static const struct ethtool_rmon_hist_range gem_rmon_ranges[] = { 3442 { 64, 64 }, 3443 { 65, 127 }, 3444 { 128, 255 }, 3445 { 256, 511 }, 3446 { 512, 1023 }, 3447 { 1024, 1518 }, 3448 { 1519, 16384 }, 3449 { }, 3450}; 3451 3452static void gem_get_rmon_stats(struct net_device *dev, 3453 struct ethtool_rmon_stats *rmon_stats, 3454 const struct ethtool_rmon_hist_range **ranges) 3455{ 3456 struct macb *bp = netdev_priv(dev); 3457 struct gem_stats *hwstat = &bp->hw_stats.gem; 3458 3459 spin_lock_irq(&bp->stats_lock); 3460 gem_update_stats(bp); 3461 rmon_stats->undersize_pkts = hwstat->rx_undersized_frames; 3462 rmon_stats->oversize_pkts = hwstat->rx_oversize_frames; 3463 rmon_stats->jabbers = hwstat->rx_jabbers; 3464 rmon_stats->hist[0] = hwstat->rx_64_byte_frames; 3465 rmon_stats->hist[1] = hwstat->rx_65_127_byte_frames; 3466 rmon_stats->hist[2] = hwstat->rx_128_255_byte_frames; 3467 rmon_stats->hist[3] = hwstat->rx_256_511_byte_frames; 3468 rmon_stats->hist[4] = hwstat->rx_512_1023_byte_frames; 3469 rmon_stats->hist[5] = hwstat->rx_1024_1518_byte_frames; 3470 rmon_stats->hist[6] = hwstat->rx_greater_than_1518_byte_frames; 3471 rmon_stats->hist_tx[0] = hwstat->tx_64_byte_frames; 3472 rmon_stats->hist_tx[1] = hwstat->tx_65_127_byte_frames; 3473 rmon_stats->hist_tx[2] = hwstat->tx_128_255_byte_frames; 3474 rmon_stats->hist_tx[3] = hwstat->tx_256_511_byte_frames; 3475 rmon_stats->hist_tx[4] = hwstat->tx_512_1023_byte_frames; 3476 rmon_stats->hist_tx[5] = hwstat->tx_1024_1518_byte_frames; 3477 rmon_stats->hist_tx[6] = hwstat->tx_greater_than_1518_byte_frames; 3478 spin_unlock_irq(&bp->stats_lock); 3479 *ranges = gem_rmon_ranges; 3480} 3481 3482static int macb_get_regs_len(struct net_device *netdev) 3483{ 3484 return MACB_GREGS_NBR * sizeof(u32); 3485} 3486 3487static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs, 3488 void *p) 3489{ 3490 struct macb *bp = netdev_priv(dev); 3491 unsigned int tail, head; 3492 u32 *regs_buff = p; 3493 3494 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1)) 3495 | MACB_GREGS_VERSION; 3496 3497 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail); 3498 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head); 3499 3500 regs_buff[0] = macb_readl(bp, NCR); 3501 regs_buff[1] = macb_or_gem_readl(bp, NCFGR); 3502 regs_buff[2] = macb_readl(bp, NSR); 3503 regs_buff[3] = macb_readl(bp, TSR); 3504 regs_buff[4] = macb_readl(bp, RBQP); 3505 regs_buff[5] = macb_readl(bp, TBQP); 3506 regs_buff[6] = macb_readl(bp, RSR); 3507 regs_buff[7] = macb_readl(bp, IMR); 3508 3509 regs_buff[8] = tail; 3510 regs_buff[9] = head; 3511 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail); 3512 regs_buff[11] = macb_tx_dma(&bp->queues[0], head); 3513 3514 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 3515 regs_buff[12] = macb_or_gem_readl(bp, USRIO); 3516 if (macb_is_gem(bp)) 3517 regs_buff[13] = gem_readl(bp, DMACFG); 3518} 3519 3520static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 3521{ 3522 struct macb *bp = netdev_priv(netdev); 3523 3524 phylink_ethtool_get_wol(bp->phylink, wol); 3525 wol->supported |= (WAKE_MAGIC | WAKE_ARP); 3526 3527 /* Add macb wolopts to phy wolopts */ 3528 wol->wolopts |= bp->wolopts; 3529} 3530 3531static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol) 3532{ 3533 struct macb *bp = netdev_priv(netdev); 3534 int ret; 3535 3536 /* Pass the order to phylink layer */ 3537 ret = phylink_ethtool_set_wol(bp->phylink, wol); 3538 /* Don't manage WoL on MAC, if PHY set_wol() fails */ 3539 if (ret && ret != -EOPNOTSUPP) 3540 return ret; 3541 3542 bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0; 3543 bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0; 3544 bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0; 3545 3546 device_set_wakeup_enable(&bp->pdev->dev, bp->wol); 3547 3548 return 0; 3549} 3550 3551static int macb_get_link_ksettings(struct net_device *netdev, 3552 struct ethtool_link_ksettings *kset) 3553{ 3554 struct macb *bp = netdev_priv(netdev); 3555 3556 return phylink_ethtool_ksettings_get(bp->phylink, kset); 3557} 3558 3559static int macb_set_link_ksettings(struct net_device *netdev, 3560 const struct ethtool_link_ksettings *kset) 3561{ 3562 struct macb *bp = netdev_priv(netdev); 3563 3564 return phylink_ethtool_ksettings_set(bp->phylink, kset); 3565} 3566 3567static void macb_get_ringparam(struct net_device *netdev, 3568 struct ethtool_ringparam *ring, 3569 struct kernel_ethtool_ringparam *kernel_ring, 3570 struct netlink_ext_ack *extack) 3571{ 3572 struct macb *bp = netdev_priv(netdev); 3573 3574 ring->rx_max_pending = MAX_RX_RING_SIZE; 3575 ring->tx_max_pending = MAX_TX_RING_SIZE; 3576 3577 ring->rx_pending = bp->rx_ring_size; 3578 ring->tx_pending = bp->tx_ring_size; 3579} 3580 3581static int macb_set_ringparam(struct net_device *netdev, 3582 struct ethtool_ringparam *ring, 3583 struct kernel_ethtool_ringparam *kernel_ring, 3584 struct netlink_ext_ack *extack) 3585{ 3586 struct macb *bp = netdev_priv(netdev); 3587 u32 new_rx_size, new_tx_size; 3588 unsigned int reset = 0; 3589 3590 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending)) 3591 return -EINVAL; 3592 3593 new_rx_size = clamp_t(u32, ring->rx_pending, 3594 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE); 3595 new_rx_size = roundup_pow_of_two(new_rx_size); 3596 3597 new_tx_size = clamp_t(u32, ring->tx_pending, 3598 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE); 3599 new_tx_size = roundup_pow_of_two(new_tx_size); 3600 3601 if ((new_tx_size == bp->tx_ring_size) && 3602 (new_rx_size == bp->rx_ring_size)) { 3603 /* nothing to do */ 3604 return 0; 3605 } 3606 3607 if (netif_running(bp->dev)) { 3608 reset = 1; 3609 macb_close(bp->dev); 3610 } 3611 3612 bp->rx_ring_size = new_rx_size; 3613 bp->tx_ring_size = new_tx_size; 3614 3615 if (reset) 3616 macb_open(bp->dev); 3617 3618 return 0; 3619} 3620 3621#ifdef CONFIG_MACB_USE_HWSTAMP 3622static unsigned int gem_get_tsu_rate(struct macb *bp) 3623{ 3624 struct clk *tsu_clk; 3625 unsigned int tsu_rate; 3626 3627 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); 3628 if (!IS_ERR(tsu_clk)) 3629 tsu_rate = clk_get_rate(tsu_clk); 3630 /* try pclk instead */ 3631 else if (!IS_ERR(bp->pclk)) { 3632 tsu_clk = bp->pclk; 3633 tsu_rate = clk_get_rate(tsu_clk); 3634 } else 3635 return -ENOTSUPP; 3636 return tsu_rate; 3637} 3638 3639static s32 gem_get_ptp_max_adj(void) 3640{ 3641 return 64000000; 3642} 3643 3644static int gem_get_ts_info(struct net_device *dev, 3645 struct kernel_ethtool_ts_info *info) 3646{ 3647 struct macb *bp = netdev_priv(dev); 3648 3649 if (!macb_dma_ptp(bp)) { 3650 ethtool_op_get_ts_info(dev, info); 3651 return 0; 3652 } 3653 3654 info->so_timestamping = 3655 SOF_TIMESTAMPING_TX_SOFTWARE | 3656 SOF_TIMESTAMPING_TX_HARDWARE | 3657 SOF_TIMESTAMPING_RX_HARDWARE | 3658 SOF_TIMESTAMPING_RAW_HARDWARE; 3659 info->tx_types = 3660 (1 << HWTSTAMP_TX_ONESTEP_SYNC) | 3661 (1 << HWTSTAMP_TX_OFF) | 3662 (1 << HWTSTAMP_TX_ON); 3663 info->rx_filters = 3664 (1 << HWTSTAMP_FILTER_NONE) | 3665 (1 << HWTSTAMP_FILTER_ALL); 3666 3667 if (bp->ptp_clock) 3668 info->phc_index = ptp_clock_index(bp->ptp_clock); 3669 3670 return 0; 3671} 3672 3673static struct macb_ptp_info gem_ptp_info = { 3674 .ptp_init = gem_ptp_init, 3675 .ptp_remove = gem_ptp_remove, 3676 .get_ptp_max_adj = gem_get_ptp_max_adj, 3677 .get_tsu_rate = gem_get_tsu_rate, 3678 .get_ts_info = gem_get_ts_info, 3679 .get_hwtst = gem_get_hwtst, 3680 .set_hwtst = gem_set_hwtst, 3681}; 3682#endif 3683 3684static int macb_get_ts_info(struct net_device *netdev, 3685 struct kernel_ethtool_ts_info *info) 3686{ 3687 struct macb *bp = netdev_priv(netdev); 3688 3689 if (bp->ptp_info) 3690 return bp->ptp_info->get_ts_info(netdev, info); 3691 3692 return ethtool_op_get_ts_info(netdev, info); 3693} 3694 3695static void gem_enable_flow_filters(struct macb *bp, bool enable) 3696{ 3697 struct net_device *netdev = bp->dev; 3698 struct ethtool_rx_fs_item *item; 3699 u32 t2_scr; 3700 int num_t2_scr; 3701 3702 if (!(netdev->features & NETIF_F_NTUPLE)) 3703 return; 3704 3705 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8)); 3706 3707 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3708 struct ethtool_rx_flow_spec *fs = &item->fs; 3709 struct ethtool_tcpip4_spec *tp4sp_m; 3710 3711 if (fs->location >= num_t2_scr) 3712 continue; 3713 3714 t2_scr = gem_readl_n(bp, SCRT2, fs->location); 3715 3716 /* enable/disable screener regs for the flow entry */ 3717 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr); 3718 3719 /* only enable fields with no masking */ 3720 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3721 3722 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF)) 3723 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr); 3724 else 3725 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr); 3726 3727 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF)) 3728 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr); 3729 else 3730 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr); 3731 3732 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF))) 3733 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr); 3734 else 3735 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr); 3736 3737 gem_writel_n(bp, SCRT2, fs->location, t2_scr); 3738 } 3739} 3740 3741static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs) 3742{ 3743 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m; 3744 uint16_t index = fs->location; 3745 u32 w0, w1, t2_scr; 3746 bool cmp_a = false; 3747 bool cmp_b = false; 3748 bool cmp_c = false; 3749 3750 if (!macb_is_gem(bp)) 3751 return; 3752 3753 tp4sp_v = &(fs->h_u.tcp_ip4_spec); 3754 tp4sp_m = &(fs->m_u.tcp_ip4_spec); 3755 3756 /* ignore field if any masking set */ 3757 if (tp4sp_m->ip4src == 0xFFFFFFFF) { 3758 /* 1st compare reg - IP source address */ 3759 w0 = 0; 3760 w1 = 0; 3761 w0 = tp4sp_v->ip4src; 3762 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3763 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3764 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1); 3765 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0); 3766 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1); 3767 cmp_a = true; 3768 } 3769 3770 /* ignore field if any masking set */ 3771 if (tp4sp_m->ip4dst == 0xFFFFFFFF) { 3772 /* 2nd compare reg - IP destination address */ 3773 w0 = 0; 3774 w1 = 0; 3775 w0 = tp4sp_v->ip4dst; 3776 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3777 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1); 3778 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1); 3779 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0); 3780 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1); 3781 cmp_b = true; 3782 } 3783 3784 /* ignore both port fields if masking set in both */ 3785 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) { 3786 /* 3rd compare reg - source port, destination port */ 3787 w0 = 0; 3788 w1 = 0; 3789 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1); 3790 if (tp4sp_m->psrc == tp4sp_m->pdst) { 3791 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0); 3792 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3793 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */ 3794 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3795 } else { 3796 /* only one port definition */ 3797 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */ 3798 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0); 3799 if (tp4sp_m->psrc == 0xFFFF) { /* src port */ 3800 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0); 3801 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1); 3802 } else { /* dst port */ 3803 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0); 3804 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1); 3805 } 3806 } 3807 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0); 3808 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1); 3809 cmp_c = true; 3810 } 3811 3812 t2_scr = 0; 3813 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr); 3814 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr); 3815 if (cmp_a) 3816 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr); 3817 if (cmp_b) 3818 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr); 3819 if (cmp_c) 3820 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr); 3821 gem_writel_n(bp, SCRT2, index, t2_scr); 3822} 3823 3824static int gem_add_flow_filter(struct net_device *netdev, 3825 struct ethtool_rxnfc *cmd) 3826{ 3827 struct macb *bp = netdev_priv(netdev); 3828 struct ethtool_rx_flow_spec *fs = &cmd->fs; 3829 struct ethtool_rx_fs_item *item, *newfs; 3830 unsigned long flags; 3831 int ret = -EINVAL; 3832 bool added = false; 3833 3834 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL); 3835 if (newfs == NULL) 3836 return -ENOMEM; 3837 memcpy(&newfs->fs, fs, sizeof(newfs->fs)); 3838 3839 netdev_dbg(netdev, 3840 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3841 fs->flow_type, (int)fs->ring_cookie, fs->location, 3842 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3843 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3844 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), 3845 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); 3846 3847 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3848 3849 /* find correct place to add in list */ 3850 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3851 if (item->fs.location > newfs->fs.location) { 3852 list_add_tail(&newfs->list, &item->list); 3853 added = true; 3854 break; 3855 } else if (item->fs.location == fs->location) { 3856 netdev_err(netdev, "Rule not added: location %d not free!\n", 3857 fs->location); 3858 ret = -EBUSY; 3859 goto err; 3860 } 3861 } 3862 if (!added) 3863 list_add_tail(&newfs->list, &bp->rx_fs_list.list); 3864 3865 gem_prog_cmp_regs(bp, fs); 3866 bp->rx_fs_list.count++; 3867 /* enable filtering if NTUPLE on */ 3868 gem_enable_flow_filters(bp, 1); 3869 3870 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3871 return 0; 3872 3873err: 3874 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3875 kfree(newfs); 3876 return ret; 3877} 3878 3879static int gem_del_flow_filter(struct net_device *netdev, 3880 struct ethtool_rxnfc *cmd) 3881{ 3882 struct macb *bp = netdev_priv(netdev); 3883 struct ethtool_rx_fs_item *item; 3884 struct ethtool_rx_flow_spec *fs; 3885 unsigned long flags; 3886 3887 spin_lock_irqsave(&bp->rx_fs_lock, flags); 3888 3889 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3890 if (item->fs.location == cmd->fs.location) { 3891 /* disable screener regs for the flow entry */ 3892 fs = &(item->fs); 3893 netdev_dbg(netdev, 3894 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n", 3895 fs->flow_type, (int)fs->ring_cookie, fs->location, 3896 htonl(fs->h_u.tcp_ip4_spec.ip4src), 3897 htonl(fs->h_u.tcp_ip4_spec.ip4dst), 3898 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc), 3899 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst)); 3900 3901 gem_writel_n(bp, SCRT2, fs->location, 0); 3902 3903 list_del(&item->list); 3904 bp->rx_fs_list.count--; 3905 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3906 kfree(item); 3907 return 0; 3908 } 3909 } 3910 3911 spin_unlock_irqrestore(&bp->rx_fs_lock, flags); 3912 return -EINVAL; 3913} 3914 3915static int gem_get_flow_entry(struct net_device *netdev, 3916 struct ethtool_rxnfc *cmd) 3917{ 3918 struct macb *bp = netdev_priv(netdev); 3919 struct ethtool_rx_fs_item *item; 3920 3921 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3922 if (item->fs.location == cmd->fs.location) { 3923 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs)); 3924 return 0; 3925 } 3926 } 3927 return -EINVAL; 3928} 3929 3930static int gem_get_all_flow_entries(struct net_device *netdev, 3931 struct ethtool_rxnfc *cmd, u32 *rule_locs) 3932{ 3933 struct macb *bp = netdev_priv(netdev); 3934 struct ethtool_rx_fs_item *item; 3935 uint32_t cnt = 0; 3936 3937 list_for_each_entry(item, &bp->rx_fs_list.list, list) { 3938 if (cnt == cmd->rule_cnt) 3939 return -EMSGSIZE; 3940 rule_locs[cnt] = item->fs.location; 3941 cnt++; 3942 } 3943 cmd->data = bp->max_tuples; 3944 cmd->rule_cnt = cnt; 3945 3946 return 0; 3947} 3948 3949static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, 3950 u32 *rule_locs) 3951{ 3952 struct macb *bp = netdev_priv(netdev); 3953 int ret = 0; 3954 3955 switch (cmd->cmd) { 3956 case ETHTOOL_GRXRINGS: 3957 cmd->data = bp->num_queues; 3958 break; 3959 case ETHTOOL_GRXCLSRLCNT: 3960 cmd->rule_cnt = bp->rx_fs_list.count; 3961 break; 3962 case ETHTOOL_GRXCLSRULE: 3963 ret = gem_get_flow_entry(netdev, cmd); 3964 break; 3965 case ETHTOOL_GRXCLSRLALL: 3966 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs); 3967 break; 3968 default: 3969 netdev_err(netdev, 3970 "Command parameter %d is not supported\n", cmd->cmd); 3971 ret = -EOPNOTSUPP; 3972 } 3973 3974 return ret; 3975} 3976 3977static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 3978{ 3979 struct macb *bp = netdev_priv(netdev); 3980 int ret; 3981 3982 if (!(netdev->hw_features & NETIF_F_NTUPLE)) 3983 return -EOPNOTSUPP; 3984 3985 switch (cmd->cmd) { 3986 case ETHTOOL_SRXCLSRLINS: 3987 if ((cmd->fs.location >= bp->max_tuples) 3988 || (cmd->fs.ring_cookie >= bp->num_queues)) { 3989 ret = -EINVAL; 3990 break; 3991 } 3992 ret = gem_add_flow_filter(netdev, cmd); 3993 break; 3994 case ETHTOOL_SRXCLSRLDEL: 3995 ret = gem_del_flow_filter(netdev, cmd); 3996 break; 3997 default: 3998 netdev_err(netdev, 3999 "Command parameter %d is not supported\n", cmd->cmd); 4000 ret = -EOPNOTSUPP; 4001 } 4002 4003 return ret; 4004} 4005 4006static const struct ethtool_ops macb_ethtool_ops = { 4007 .get_regs_len = macb_get_regs_len, 4008 .get_regs = macb_get_regs, 4009 .get_link = ethtool_op_get_link, 4010 .get_ts_info = ethtool_op_get_ts_info, 4011 .get_pause_stats = macb_get_pause_stats, 4012 .get_eth_mac_stats = macb_get_eth_mac_stats, 4013 .get_eth_phy_stats = macb_get_eth_phy_stats, 4014 .get_rmon_stats = macb_get_rmon_stats, 4015 .get_wol = macb_get_wol, 4016 .set_wol = macb_set_wol, 4017 .get_link_ksettings = macb_get_link_ksettings, 4018 .set_link_ksettings = macb_set_link_ksettings, 4019 .get_ringparam = macb_get_ringparam, 4020 .set_ringparam = macb_set_ringparam, 4021}; 4022 4023static const struct ethtool_ops gem_ethtool_ops = { 4024 .get_regs_len = macb_get_regs_len, 4025 .get_regs = macb_get_regs, 4026 .get_wol = macb_get_wol, 4027 .set_wol = macb_set_wol, 4028 .get_link = ethtool_op_get_link, 4029 .get_ts_info = macb_get_ts_info, 4030 .get_ethtool_stats = gem_get_ethtool_stats, 4031 .get_strings = gem_get_ethtool_strings, 4032 .get_sset_count = gem_get_sset_count, 4033 .get_pause_stats = gem_get_pause_stats, 4034 .get_eth_mac_stats = gem_get_eth_mac_stats, 4035 .get_eth_phy_stats = gem_get_eth_phy_stats, 4036 .get_rmon_stats = gem_get_rmon_stats, 4037 .get_link_ksettings = macb_get_link_ksettings, 4038 .set_link_ksettings = macb_set_link_ksettings, 4039 .get_ringparam = macb_get_ringparam, 4040 .set_ringparam = macb_set_ringparam, 4041 .get_rxnfc = gem_get_rxnfc, 4042 .set_rxnfc = gem_set_rxnfc, 4043}; 4044 4045static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) 4046{ 4047 struct macb *bp = netdev_priv(dev); 4048 4049 if (!netif_running(dev)) 4050 return -EINVAL; 4051 4052 return phylink_mii_ioctl(bp->phylink, rq, cmd); 4053} 4054 4055static int macb_hwtstamp_get(struct net_device *dev, 4056 struct kernel_hwtstamp_config *cfg) 4057{ 4058 struct macb *bp = netdev_priv(dev); 4059 4060 if (!netif_running(dev)) 4061 return -EINVAL; 4062 4063 if (!bp->ptp_info) 4064 return -EOPNOTSUPP; 4065 4066 return bp->ptp_info->get_hwtst(dev, cfg); 4067} 4068 4069static int macb_hwtstamp_set(struct net_device *dev, 4070 struct kernel_hwtstamp_config *cfg, 4071 struct netlink_ext_ack *extack) 4072{ 4073 struct macb *bp = netdev_priv(dev); 4074 4075 if (!netif_running(dev)) 4076 return -EINVAL; 4077 4078 if (!bp->ptp_info) 4079 return -EOPNOTSUPP; 4080 4081 return bp->ptp_info->set_hwtst(dev, cfg, extack); 4082} 4083 4084static inline void macb_set_txcsum_feature(struct macb *bp, 4085 netdev_features_t features) 4086{ 4087 u32 val; 4088 4089 if (!macb_is_gem(bp)) 4090 return; 4091 4092 val = gem_readl(bp, DMACFG); 4093 if (features & NETIF_F_HW_CSUM) 4094 val |= GEM_BIT(TXCOEN); 4095 else 4096 val &= ~GEM_BIT(TXCOEN); 4097 4098 gem_writel(bp, DMACFG, val); 4099} 4100 4101static inline void macb_set_rxcsum_feature(struct macb *bp, 4102 netdev_features_t features) 4103{ 4104 struct net_device *netdev = bp->dev; 4105 u32 val; 4106 4107 if (!macb_is_gem(bp)) 4108 return; 4109 4110 val = gem_readl(bp, NCFGR); 4111 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC)) 4112 val |= GEM_BIT(RXCOEN); 4113 else 4114 val &= ~GEM_BIT(RXCOEN); 4115 4116 gem_writel(bp, NCFGR, val); 4117} 4118 4119static inline void macb_set_rxflow_feature(struct macb *bp, 4120 netdev_features_t features) 4121{ 4122 if (!macb_is_gem(bp)) 4123 return; 4124 4125 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE)); 4126} 4127 4128static int macb_set_features(struct net_device *netdev, 4129 netdev_features_t features) 4130{ 4131 struct macb *bp = netdev_priv(netdev); 4132 netdev_features_t changed = features ^ netdev->features; 4133 4134 /* TX checksum offload */ 4135 if (changed & NETIF_F_HW_CSUM) 4136 macb_set_txcsum_feature(bp, features); 4137 4138 /* RX checksum offload */ 4139 if (changed & NETIF_F_RXCSUM) 4140 macb_set_rxcsum_feature(bp, features); 4141 4142 /* RX Flow Filters */ 4143 if (changed & NETIF_F_NTUPLE) 4144 macb_set_rxflow_feature(bp, features); 4145 4146 return 0; 4147} 4148 4149static void macb_restore_features(struct macb *bp) 4150{ 4151 struct net_device *netdev = bp->dev; 4152 netdev_features_t features = netdev->features; 4153 struct ethtool_rx_fs_item *item; 4154 4155 /* TX checksum offload */ 4156 macb_set_txcsum_feature(bp, features); 4157 4158 /* RX checksum offload */ 4159 macb_set_rxcsum_feature(bp, features); 4160 4161 /* RX Flow Filters */ 4162 list_for_each_entry(item, &bp->rx_fs_list.list, list) 4163 gem_prog_cmp_regs(bp, &item->fs); 4164 4165 macb_set_rxflow_feature(bp, features); 4166} 4167 4168static int macb_taprio_setup_replace(struct net_device *ndev, 4169 struct tc_taprio_qopt_offload *conf) 4170{ 4171 u64 total_on_time = 0, start_time_sec = 0, start_time = conf->base_time; 4172 u32 configured_queues = 0, speed = 0, start_time_nsec; 4173 struct macb_queue_enst_config *enst_queue; 4174 struct tc_taprio_sched_entry *entry; 4175 struct macb *bp = netdev_priv(ndev); 4176 struct ethtool_link_ksettings kset; 4177 struct macb_queue *queue; 4178 u32 queue_mask; 4179 u8 queue_id; 4180 size_t i; 4181 int err; 4182 4183 if (conf->num_entries > bp->num_queues) { 4184 netdev_err(ndev, "Too many TAPRIO entries: %zu > %d queues\n", 4185 conf->num_entries, bp->num_queues); 4186 return -EINVAL; 4187 } 4188 4189 if (conf->base_time < 0) { 4190 netdev_err(ndev, "Invalid base_time: must be 0 or positive, got %lld\n", 4191 conf->base_time); 4192 return -ERANGE; 4193 } 4194 4195 /* Get the current link speed */ 4196 err = phylink_ethtool_ksettings_get(bp->phylink, &kset); 4197 if (unlikely(err)) { 4198 netdev_err(ndev, "Failed to get link settings: %d\n", err); 4199 return err; 4200 } 4201 4202 speed = kset.base.speed; 4203 if (unlikely(speed <= 0)) { 4204 netdev_err(ndev, "Invalid speed: %d\n", speed); 4205 return -EINVAL; 4206 } 4207 4208 enst_queue = kcalloc(conf->num_entries, sizeof(*enst_queue), GFP_KERNEL); 4209 if (unlikely(!enst_queue)) 4210 return -ENOMEM; 4211 4212 /* Pre-validate all entries before making any hardware changes */ 4213 for (i = 0; i < conf->num_entries; i++) { 4214 entry = &conf->entries[i]; 4215 4216 if (entry->command != TC_TAPRIO_CMD_SET_GATES) { 4217 netdev_err(ndev, "Entry %zu: unsupported command %d\n", 4218 i, entry->command); 4219 err = -EOPNOTSUPP; 4220 goto cleanup; 4221 } 4222 4223 /* Validate gate_mask: must be nonzero, single queue, and within range */ 4224 if (!is_power_of_2(entry->gate_mask)) { 4225 netdev_err(ndev, "Entry %zu: gate_mask 0x%x is not a power of 2 (only one queue per entry allowed)\n", 4226 i, entry->gate_mask); 4227 err = -EINVAL; 4228 goto cleanup; 4229 } 4230 4231 /* gate_mask must not select queues outside the valid queues */ 4232 queue_id = order_base_2(entry->gate_mask); 4233 if (queue_id >= bp->num_queues) { 4234 netdev_err(ndev, "Entry %zu: gate_mask 0x%x exceeds queue range (max_queues=%d)\n", 4235 i, entry->gate_mask, bp->num_queues); 4236 err = -EINVAL; 4237 goto cleanup; 4238 } 4239 4240 /* Check for start time limits */ 4241 start_time_sec = start_time; 4242 start_time_nsec = do_div(start_time_sec, NSEC_PER_SEC); 4243 if (start_time_sec > GENMASK(GEM_START_TIME_SEC_SIZE - 1, 0)) { 4244 netdev_err(ndev, "Entry %zu: Start time %llu s exceeds hardware limit\n", 4245 i, start_time_sec); 4246 err = -ERANGE; 4247 goto cleanup; 4248 } 4249 4250 /* Check for on time limit */ 4251 if (entry->interval > enst_max_hw_interval(speed)) { 4252 netdev_err(ndev, "Entry %zu: interval %u ns exceeds hardware limit %llu ns\n", 4253 i, entry->interval, enst_max_hw_interval(speed)); 4254 err = -ERANGE; 4255 goto cleanup; 4256 } 4257 4258 /* Check for off time limit*/ 4259 if ((conf->cycle_time - entry->interval) > enst_max_hw_interval(speed)) { 4260 netdev_err(ndev, "Entry %zu: off_time %llu ns exceeds hardware limit %llu ns\n", 4261 i, conf->cycle_time - entry->interval, 4262 enst_max_hw_interval(speed)); 4263 err = -ERANGE; 4264 goto cleanup; 4265 } 4266 4267 enst_queue[i].queue_id = queue_id; 4268 enst_queue[i].start_time_mask = 4269 (start_time_sec << GEM_START_TIME_SEC_OFFSET) | 4270 start_time_nsec; 4271 enst_queue[i].on_time_bytes = 4272 enst_ns_to_hw_units(entry->interval, speed); 4273 enst_queue[i].off_time_bytes = 4274 enst_ns_to_hw_units(conf->cycle_time - entry->interval, speed); 4275 4276 configured_queues |= entry->gate_mask; 4277 total_on_time += entry->interval; 4278 start_time += entry->interval; 4279 } 4280 4281 /* Check total interval doesn't exceed cycle time */ 4282 if (total_on_time > conf->cycle_time) { 4283 netdev_err(ndev, "Total ON %llu ns exceeds cycle time %llu ns\n", 4284 total_on_time, conf->cycle_time); 4285 err = -EINVAL; 4286 goto cleanup; 4287 } 4288 4289 netdev_dbg(ndev, "TAPRIO setup: %zu entries, base_time=%lld ns, cycle_time=%llu ns\n", 4290 conf->num_entries, conf->base_time, conf->cycle_time); 4291 4292 /* All validations passed - proceed with hardware configuration */ 4293 scoped_guard(spinlock_irqsave, &bp->lock) { 4294 /* Disable ENST queues if running before configuring */ 4295 queue_mask = BIT_U32(bp->num_queues) - 1; 4296 gem_writel(bp, ENST_CONTROL, 4297 queue_mask << GEM_ENST_DISABLE_QUEUE_OFFSET); 4298 4299 for (i = 0; i < conf->num_entries; i++) { 4300 queue = &bp->queues[enst_queue[i].queue_id]; 4301 /* Configure queue timing registers */ 4302 queue_writel(queue, ENST_START_TIME, 4303 enst_queue[i].start_time_mask); 4304 queue_writel(queue, ENST_ON_TIME, 4305 enst_queue[i].on_time_bytes); 4306 queue_writel(queue, ENST_OFF_TIME, 4307 enst_queue[i].off_time_bytes); 4308 } 4309 4310 /* Enable ENST for all configured queues in one write */ 4311 gem_writel(bp, ENST_CONTROL, configured_queues); 4312 } 4313 4314 netdev_info(ndev, "TAPRIO configuration completed successfully: %zu entries, %d queues configured\n", 4315 conf->num_entries, hweight32(configured_queues)); 4316 4317cleanup: 4318 kfree(enst_queue); 4319 return err; 4320} 4321 4322static void macb_taprio_destroy(struct net_device *ndev) 4323{ 4324 struct macb *bp = netdev_priv(ndev); 4325 struct macb_queue *queue; 4326 u32 queue_mask; 4327 unsigned int q; 4328 4329 netdev_reset_tc(ndev); 4330 queue_mask = BIT_U32(bp->num_queues) - 1; 4331 4332 scoped_guard(spinlock_irqsave, &bp->lock) { 4333 /* Single disable command for all queues */ 4334 gem_writel(bp, ENST_CONTROL, 4335 queue_mask << GEM_ENST_DISABLE_QUEUE_OFFSET); 4336 4337 /* Clear all queue ENST registers in batch */ 4338 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { 4339 queue_writel(queue, ENST_START_TIME, 0); 4340 queue_writel(queue, ENST_ON_TIME, 0); 4341 queue_writel(queue, ENST_OFF_TIME, 0); 4342 } 4343 } 4344 netdev_info(ndev, "TAPRIO destroy: All gates disabled\n"); 4345} 4346 4347static int macb_setup_taprio(struct net_device *ndev, 4348 struct tc_taprio_qopt_offload *taprio) 4349{ 4350 struct macb *bp = netdev_priv(ndev); 4351 int err = 0; 4352 4353 if (unlikely(!(ndev->hw_features & NETIF_F_HW_TC))) 4354 return -EOPNOTSUPP; 4355 4356 /* Check if Device is in runtime suspend */ 4357 if (unlikely(pm_runtime_suspended(&bp->pdev->dev))) { 4358 netdev_err(ndev, "Device is in runtime suspend\n"); 4359 return -EOPNOTSUPP; 4360 } 4361 4362 switch (taprio->cmd) { 4363 case TAPRIO_CMD_REPLACE: 4364 err = macb_taprio_setup_replace(ndev, taprio); 4365 break; 4366 case TAPRIO_CMD_DESTROY: 4367 macb_taprio_destroy(ndev); 4368 break; 4369 default: 4370 err = -EOPNOTSUPP; 4371 } 4372 4373 return err; 4374} 4375 4376static int macb_setup_tc(struct net_device *dev, enum tc_setup_type type, 4377 void *type_data) 4378{ 4379 if (!dev || !type_data) 4380 return -EINVAL; 4381 4382 switch (type) { 4383 case TC_SETUP_QDISC_TAPRIO: 4384 return macb_setup_taprio(dev, type_data); 4385 default: 4386 return -EOPNOTSUPP; 4387 } 4388} 4389 4390static const struct net_device_ops macb_netdev_ops = { 4391 .ndo_open = macb_open, 4392 .ndo_stop = macb_close, 4393 .ndo_start_xmit = macb_start_xmit, 4394 .ndo_set_rx_mode = macb_set_rx_mode, 4395 .ndo_get_stats64 = macb_get_stats, 4396 .ndo_eth_ioctl = macb_ioctl, 4397 .ndo_validate_addr = eth_validate_addr, 4398 .ndo_change_mtu = macb_change_mtu, 4399 .ndo_set_mac_address = macb_set_mac_addr, 4400#ifdef CONFIG_NET_POLL_CONTROLLER 4401 .ndo_poll_controller = macb_poll_controller, 4402#endif 4403 .ndo_set_features = macb_set_features, 4404 .ndo_features_check = macb_features_check, 4405 .ndo_hwtstamp_set = macb_hwtstamp_set, 4406 .ndo_hwtstamp_get = macb_hwtstamp_get, 4407 .ndo_setup_tc = macb_setup_tc, 4408}; 4409 4410/* Configure peripheral capabilities according to device tree 4411 * and integration options used 4412 */ 4413static void macb_configure_caps(struct macb *bp, 4414 const struct macb_config *dt_conf) 4415{ 4416 struct device_node *np = bp->pdev->dev.of_node; 4417 bool refclk_ext; 4418 u32 dcfg; 4419 4420 refclk_ext = of_property_read_bool(np, "cdns,refclk-ext"); 4421 4422 if (dt_conf) 4423 bp->caps = dt_conf->caps; 4424 4425 if (hw_is_gem(bp->regs, bp->native_io)) { 4426 bp->caps |= MACB_CAPS_MACB_IS_GEM; 4427 4428 dcfg = gem_readl(bp, DCFG1); 4429 if (GEM_BFEXT(IRQCOR, dcfg) == 0) 4430 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; 4431 if (GEM_BFEXT(NO_PCS, dcfg) == 0) 4432 bp->caps |= MACB_CAPS_PCS; 4433 dcfg = gem_readl(bp, DCFG12); 4434 if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) 4435 bp->caps |= MACB_CAPS_HIGH_SPEED; 4436 dcfg = gem_readl(bp, DCFG2); 4437 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0) 4438 bp->caps |= MACB_CAPS_FIFO_MODE; 4439 if (GEM_BFEXT(PBUF_RSC, gem_readl(bp, DCFG6))) 4440 bp->caps |= MACB_CAPS_RSC; 4441 if (gem_has_ptp(bp)) { 4442 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5))) 4443 dev_err(&bp->pdev->dev, 4444 "GEM doesn't support hardware ptp.\n"); 4445 else { 4446#ifdef CONFIG_MACB_USE_HWSTAMP 4447 bp->caps |= MACB_CAPS_DMA_PTP; 4448 bp->ptp_info = &gem_ptp_info; 4449#endif 4450 } 4451 } 4452 } 4453 4454 if (refclk_ext) 4455 bp->caps |= MACB_CAPS_USRIO_HAS_CLKEN; 4456 4457 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); 4458} 4459 4460static int macb_probe_queues(struct device *dev, void __iomem *mem, bool native_io) 4461{ 4462 /* BIT(0) is never set but queue 0 always exists. */ 4463 unsigned int queue_mask = 0x1; 4464 4465 /* Use hw_is_gem() as MACB_CAPS_MACB_IS_GEM is not yet positioned. */ 4466 if (hw_is_gem(mem, native_io)) { 4467 if (native_io) 4468 queue_mask |= __raw_readl(mem + GEM_DCFG6) & 0xFF; 4469 else 4470 queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xFF; 4471 4472 if (fls(queue_mask) != ffz(queue_mask)) { 4473 dev_err(dev, "queue mask %#x has a hole\n", queue_mask); 4474 return -EINVAL; 4475 } 4476 } 4477 4478 return hweight32(queue_mask); 4479} 4480 4481static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk, 4482 struct clk *rx_clk, struct clk *tsu_clk) 4483{ 4484 struct clk_bulk_data clks[] = { 4485 { .clk = tsu_clk, }, 4486 { .clk = rx_clk, }, 4487 { .clk = pclk, }, 4488 { .clk = hclk, }, 4489 { .clk = tx_clk }, 4490 }; 4491 4492 clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); 4493} 4494 4495static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, 4496 struct clk **hclk, struct clk **tx_clk, 4497 struct clk **rx_clk, struct clk **tsu_clk) 4498{ 4499 struct macb_platform_data *pdata; 4500 int err; 4501 4502 pdata = dev_get_platdata(&pdev->dev); 4503 if (pdata) { 4504 *pclk = pdata->pclk; 4505 *hclk = pdata->hclk; 4506 } else { 4507 *pclk = devm_clk_get(&pdev->dev, "pclk"); 4508 *hclk = devm_clk_get(&pdev->dev, "hclk"); 4509 } 4510 4511 if (IS_ERR_OR_NULL(*pclk)) 4512 return dev_err_probe(&pdev->dev, 4513 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV, 4514 "failed to get pclk\n"); 4515 4516 if (IS_ERR_OR_NULL(*hclk)) 4517 return dev_err_probe(&pdev->dev, 4518 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV, 4519 "failed to get hclk\n"); 4520 4521 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk"); 4522 if (IS_ERR(*tx_clk)) 4523 return PTR_ERR(*tx_clk); 4524 4525 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk"); 4526 if (IS_ERR(*rx_clk)) 4527 return PTR_ERR(*rx_clk); 4528 4529 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk"); 4530 if (IS_ERR(*tsu_clk)) 4531 return PTR_ERR(*tsu_clk); 4532 4533 err = clk_prepare_enable(*pclk); 4534 if (err) { 4535 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 4536 return err; 4537 } 4538 4539 err = clk_prepare_enable(*hclk); 4540 if (err) { 4541 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err); 4542 goto err_disable_pclk; 4543 } 4544 4545 err = clk_prepare_enable(*tx_clk); 4546 if (err) { 4547 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); 4548 goto err_disable_hclk; 4549 } 4550 4551 err = clk_prepare_enable(*rx_clk); 4552 if (err) { 4553 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); 4554 goto err_disable_txclk; 4555 } 4556 4557 err = clk_prepare_enable(*tsu_clk); 4558 if (err) { 4559 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err); 4560 goto err_disable_rxclk; 4561 } 4562 4563 return 0; 4564 4565err_disable_rxclk: 4566 clk_disable_unprepare(*rx_clk); 4567 4568err_disable_txclk: 4569 clk_disable_unprepare(*tx_clk); 4570 4571err_disable_hclk: 4572 clk_disable_unprepare(*hclk); 4573 4574err_disable_pclk: 4575 clk_disable_unprepare(*pclk); 4576 4577 return err; 4578} 4579 4580static int macb_init(struct platform_device *pdev) 4581{ 4582 struct net_device *dev = platform_get_drvdata(pdev); 4583 unsigned int hw_q, q; 4584 struct macb *bp = netdev_priv(dev); 4585 struct macb_queue *queue; 4586 int err; 4587 u32 val, reg; 4588 4589 bp->tx_ring_size = DEFAULT_TX_RING_SIZE; 4590 bp->rx_ring_size = DEFAULT_RX_RING_SIZE; 4591 4592 /* set the queue register mapping once for all: queue0 has a special 4593 * register mapping but we don't want to test the queue index then 4594 * compute the corresponding register offset at run time. 4595 */ 4596 for (hw_q = 0, q = 0; hw_q < bp->num_queues; ++hw_q) { 4597 queue = &bp->queues[q]; 4598 queue->bp = bp; 4599 spin_lock_init(&queue->tx_ptr_lock); 4600 netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); 4601 netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); 4602 if (hw_q) { 4603 queue->ISR = GEM_ISR(hw_q - 1); 4604 queue->IER = GEM_IER(hw_q - 1); 4605 queue->IDR = GEM_IDR(hw_q - 1); 4606 queue->IMR = GEM_IMR(hw_q - 1); 4607 queue->TBQP = GEM_TBQP(hw_q - 1); 4608 queue->RBQP = GEM_RBQP(hw_q - 1); 4609 queue->RBQS = GEM_RBQS(hw_q - 1); 4610 } else { 4611 /* queue0 uses legacy registers */ 4612 queue->ISR = MACB_ISR; 4613 queue->IER = MACB_IER; 4614 queue->IDR = MACB_IDR; 4615 queue->IMR = MACB_IMR; 4616 queue->TBQP = MACB_TBQP; 4617 queue->RBQP = MACB_RBQP; 4618 } 4619 4620 queue->ENST_START_TIME = GEM_ENST_START_TIME(hw_q); 4621 queue->ENST_ON_TIME = GEM_ENST_ON_TIME(hw_q); 4622 queue->ENST_OFF_TIME = GEM_ENST_OFF_TIME(hw_q); 4623 4624 /* get irq: here we use the linux queue index, not the hardware 4625 * queue index. the queue irq definitions in the device tree 4626 * must remove the optional gaps that could exist in the 4627 * hardware queue mask. 4628 */ 4629 queue->irq = platform_get_irq(pdev, q); 4630 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt, 4631 IRQF_SHARED, dev->name, queue); 4632 if (err) { 4633 dev_err(&pdev->dev, 4634 "Unable to request IRQ %d (error %d)\n", 4635 queue->irq, err); 4636 return err; 4637 } 4638 4639 INIT_WORK(&queue->tx_error_task, macb_tx_error_task); 4640 q++; 4641 } 4642 4643 dev->netdev_ops = &macb_netdev_ops; 4644 4645 /* setup appropriated routines according to adapter type */ 4646 if (macb_is_gem(bp)) { 4647 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers; 4648 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers; 4649 bp->macbgem_ops.mog_init_rings = gem_init_rings; 4650 bp->macbgem_ops.mog_rx = gem_rx; 4651 dev->ethtool_ops = &gem_ethtool_ops; 4652 } else { 4653 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers; 4654 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers; 4655 bp->macbgem_ops.mog_init_rings = macb_init_rings; 4656 bp->macbgem_ops.mog_rx = macb_rx; 4657 dev->ethtool_ops = &macb_ethtool_ops; 4658 } 4659 4660 netdev_sw_irq_coalesce_default_on(dev); 4661 4662 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 4663 4664 /* Set features */ 4665 dev->hw_features = NETIF_F_SG; 4666 4667 /* Check LSO capability; runtime detection can be overridden by a cap 4668 * flag if the hardware is known to be buggy 4669 */ 4670 if (!(bp->caps & MACB_CAPS_NO_LSO) && 4671 GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6))) 4672 dev->hw_features |= MACB_NETIF_LSO; 4673 4674 /* Checksum offload is only available on gem with packet buffer */ 4675 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE)) 4676 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM; 4677 if (bp->caps & MACB_CAPS_SG_DISABLED) 4678 dev->hw_features &= ~NETIF_F_SG; 4679 /* Enable HW_TC if hardware supports QBV */ 4680 if (bp->caps & MACB_CAPS_QBV) 4681 dev->hw_features |= NETIF_F_HW_TC; 4682 4683 dev->features = dev->hw_features; 4684 4685 /* Check RX Flow Filters support. 4686 * Max Rx flows set by availability of screeners & compare regs: 4687 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs 4688 */ 4689 reg = gem_readl(bp, DCFG8); 4690 bp->max_tuples = umin((GEM_BFEXT(SCR2CMP, reg) / 3), 4691 GEM_BFEXT(T2SCR, reg)); 4692 INIT_LIST_HEAD(&bp->rx_fs_list.list); 4693 if (bp->max_tuples > 0) { 4694 /* also needs one ethtype match to check IPv4 */ 4695 if (GEM_BFEXT(SCR2ETH, reg) > 0) { 4696 /* program this reg now */ 4697 reg = 0; 4698 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg); 4699 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg); 4700 /* Filtering is supported in hw but don't enable it in kernel now */ 4701 dev->hw_features |= NETIF_F_NTUPLE; 4702 /* init Rx flow definitions */ 4703 bp->rx_fs_list.count = 0; 4704 spin_lock_init(&bp->rx_fs_lock); 4705 } else 4706 bp->max_tuples = 0; 4707 } 4708 4709 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { 4710 val = 0; 4711 if (phy_interface_mode_is_rgmii(bp->phy_interface)) 4712 val = bp->usrio->rgmii; 4713 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && 4714 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 4715 val = bp->usrio->rmii; 4716 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) 4717 val = bp->usrio->mii; 4718 4719 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) 4720 val |= bp->usrio->refclk; 4721 4722 macb_or_gem_writel(bp, USRIO, val); 4723 } 4724 4725 /* Set MII management clock divider */ 4726 val = macb_mdc_clk_div(bp); 4727 val |= macb_dbw(bp); 4728 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) 4729 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL); 4730 macb_writel(bp, NCFGR, val); 4731 4732 return 0; 4733} 4734 4735static const struct macb_usrio_config macb_default_usrio = { 4736 .mii = MACB_BIT(MII), 4737 .rmii = MACB_BIT(RMII), 4738 .rgmii = GEM_BIT(RGMII), 4739 .refclk = MACB_BIT(CLKEN), 4740}; 4741 4742#if defined(CONFIG_OF) 4743/* 1518 rounded up */ 4744#define AT91ETHER_MAX_RBUFF_SZ 0x600 4745/* max number of receive buffers */ 4746#define AT91ETHER_MAX_RX_DESCR 9 4747 4748static struct sifive_fu540_macb_mgmt *mgmt; 4749 4750static int at91ether_alloc_coherent(struct macb *lp) 4751{ 4752 struct macb_queue *q = &lp->queues[0]; 4753 4754 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev, 4755 (AT91ETHER_MAX_RX_DESCR * 4756 macb_dma_desc_get_size(lp)), 4757 &q->rx_ring_dma, GFP_KERNEL); 4758 if (!q->rx_ring) 4759 return -ENOMEM; 4760 4761 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev, 4762 AT91ETHER_MAX_RX_DESCR * 4763 AT91ETHER_MAX_RBUFF_SZ, 4764 &q->rx_buffers_dma, GFP_KERNEL); 4765 if (!q->rx_buffers) { 4766 dma_free_coherent(&lp->pdev->dev, 4767 AT91ETHER_MAX_RX_DESCR * 4768 macb_dma_desc_get_size(lp), 4769 q->rx_ring, q->rx_ring_dma); 4770 q->rx_ring = NULL; 4771 return -ENOMEM; 4772 } 4773 4774 return 0; 4775} 4776 4777static void at91ether_free_coherent(struct macb *lp) 4778{ 4779 struct macb_queue *q = &lp->queues[0]; 4780 4781 if (q->rx_ring) { 4782 dma_free_coherent(&lp->pdev->dev, 4783 AT91ETHER_MAX_RX_DESCR * 4784 macb_dma_desc_get_size(lp), 4785 q->rx_ring, q->rx_ring_dma); 4786 q->rx_ring = NULL; 4787 } 4788 4789 if (q->rx_buffers) { 4790 dma_free_coherent(&lp->pdev->dev, 4791 AT91ETHER_MAX_RX_DESCR * 4792 AT91ETHER_MAX_RBUFF_SZ, 4793 q->rx_buffers, q->rx_buffers_dma); 4794 q->rx_buffers = NULL; 4795 } 4796} 4797 4798/* Initialize and start the Receiver and Transmit subsystems */ 4799static int at91ether_start(struct macb *lp) 4800{ 4801 struct macb_queue *q = &lp->queues[0]; 4802 struct macb_dma_desc *desc; 4803 dma_addr_t addr; 4804 u32 ctl; 4805 int i, ret; 4806 4807 ret = at91ether_alloc_coherent(lp); 4808 if (ret) 4809 return ret; 4810 4811 addr = q->rx_buffers_dma; 4812 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) { 4813 desc = macb_rx_desc(q, i); 4814 macb_set_addr(lp, desc, addr); 4815 desc->ctrl = 0; 4816 addr += AT91ETHER_MAX_RBUFF_SZ; 4817 } 4818 4819 /* Set the Wrap bit on the last descriptor */ 4820 desc->addr |= MACB_BIT(RX_WRAP); 4821 4822 /* Reset buffer index */ 4823 q->rx_tail = 0; 4824 4825 /* Program address of descriptor list in Rx Buffer Queue register */ 4826 macb_writel(lp, RBQP, q->rx_ring_dma); 4827 4828 /* Enable Receive and Transmit */ 4829 ctl = macb_readl(lp, NCR); 4830 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE)); 4831 4832 /* Enable MAC interrupts */ 4833 macb_writel(lp, IER, MACB_BIT(RCOMP) | 4834 MACB_BIT(RXUBR) | 4835 MACB_BIT(ISR_TUND) | 4836 MACB_BIT(ISR_RLE) | 4837 MACB_BIT(TCOMP) | 4838 MACB_BIT(ISR_ROVR) | 4839 MACB_BIT(HRESP)); 4840 4841 return 0; 4842} 4843 4844static void at91ether_stop(struct macb *lp) 4845{ 4846 u32 ctl; 4847 4848 /* Disable MAC interrupts */ 4849 macb_writel(lp, IDR, MACB_BIT(RCOMP) | 4850 MACB_BIT(RXUBR) | 4851 MACB_BIT(ISR_TUND) | 4852 MACB_BIT(ISR_RLE) | 4853 MACB_BIT(TCOMP) | 4854 MACB_BIT(ISR_ROVR) | 4855 MACB_BIT(HRESP)); 4856 4857 /* Disable Receiver and Transmitter */ 4858 ctl = macb_readl(lp, NCR); 4859 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE))); 4860 4861 /* Free resources. */ 4862 at91ether_free_coherent(lp); 4863} 4864 4865/* Open the ethernet interface */ 4866static int at91ether_open(struct net_device *dev) 4867{ 4868 struct macb *lp = netdev_priv(dev); 4869 u32 ctl; 4870 int ret; 4871 4872 ret = pm_runtime_resume_and_get(&lp->pdev->dev); 4873 if (ret < 0) 4874 return ret; 4875 4876 /* Clear internal statistics */ 4877 ctl = macb_readl(lp, NCR); 4878 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT)); 4879 4880 macb_set_hwaddr(lp); 4881 4882 ret = at91ether_start(lp); 4883 if (ret) 4884 goto pm_exit; 4885 4886 ret = macb_phylink_connect(lp); 4887 if (ret) 4888 goto stop; 4889 4890 netif_start_queue(dev); 4891 4892 return 0; 4893 4894stop: 4895 at91ether_stop(lp); 4896pm_exit: 4897 pm_runtime_put_sync(&lp->pdev->dev); 4898 return ret; 4899} 4900 4901/* Close the interface */ 4902static int at91ether_close(struct net_device *dev) 4903{ 4904 struct macb *lp = netdev_priv(dev); 4905 4906 netif_stop_queue(dev); 4907 4908 phylink_stop(lp->phylink); 4909 phylink_disconnect_phy(lp->phylink); 4910 4911 at91ether_stop(lp); 4912 4913 return pm_runtime_put(&lp->pdev->dev); 4914} 4915 4916/* Transmit packet */ 4917static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, 4918 struct net_device *dev) 4919{ 4920 struct macb *lp = netdev_priv(dev); 4921 4922 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) { 4923 int desc = 0; 4924 4925 netif_stop_queue(dev); 4926 4927 /* Store packet information (to free when Tx completed) */ 4928 lp->rm9200_txq[desc].skb = skb; 4929 lp->rm9200_txq[desc].size = skb->len; 4930 lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data, 4931 skb->len, DMA_TO_DEVICE); 4932 if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) { 4933 dev_kfree_skb_any(skb); 4934 dev->stats.tx_dropped++; 4935 netdev_err(dev, "%s: DMA mapping error\n", __func__); 4936 return NETDEV_TX_OK; 4937 } 4938 4939 /* Set address of the data in the Transmit Address register */ 4940 macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping); 4941 /* Set length of the packet in the Transmit Control register */ 4942 macb_writel(lp, TCR, skb->len); 4943 4944 } else { 4945 netdev_err(dev, "%s called, but device is busy!\n", __func__); 4946 return NETDEV_TX_BUSY; 4947 } 4948 4949 return NETDEV_TX_OK; 4950} 4951 4952/* Extract received frame from buffer descriptors and sent to upper layers. 4953 * (Called from interrupt context) 4954 */ 4955static void at91ether_rx(struct net_device *dev) 4956{ 4957 struct macb *lp = netdev_priv(dev); 4958 struct macb_queue *q = &lp->queues[0]; 4959 struct macb_dma_desc *desc; 4960 unsigned char *p_recv; 4961 struct sk_buff *skb; 4962 unsigned int pktlen; 4963 4964 desc = macb_rx_desc(q, q->rx_tail); 4965 while (desc->addr & MACB_BIT(RX_USED)) { 4966 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ; 4967 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl); 4968 skb = netdev_alloc_skb(dev, pktlen + 2); 4969 if (skb) { 4970 skb_reserve(skb, 2); 4971 skb_put_data(skb, p_recv, pktlen); 4972 4973 skb->protocol = eth_type_trans(skb, dev); 4974 dev->stats.rx_packets++; 4975 dev->stats.rx_bytes += pktlen; 4976 netif_rx(skb); 4977 } else { 4978 dev->stats.rx_dropped++; 4979 } 4980 4981 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH)) 4982 dev->stats.multicast++; 4983 4984 /* reset ownership bit */ 4985 desc->addr &= ~MACB_BIT(RX_USED); 4986 4987 /* wrap after last buffer */ 4988 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1) 4989 q->rx_tail = 0; 4990 else 4991 q->rx_tail++; 4992 4993 desc = macb_rx_desc(q, q->rx_tail); 4994 } 4995} 4996 4997/* MAC interrupt handler */ 4998static irqreturn_t at91ether_interrupt(int irq, void *dev_id) 4999{ 5000 struct net_device *dev = dev_id; 5001 struct macb *lp = netdev_priv(dev); 5002 u32 intstatus, ctl; 5003 unsigned int desc; 5004 5005 /* MAC Interrupt Status register indicates what interrupts are pending. 5006 * It is automatically cleared once read. 5007 */ 5008 intstatus = macb_readl(lp, ISR); 5009 5010 /* Receive complete */ 5011 if (intstatus & MACB_BIT(RCOMP)) 5012 at91ether_rx(dev); 5013 5014 /* Transmit complete */ 5015 if (intstatus & MACB_BIT(TCOMP)) { 5016 /* The TCOM bit is set even if the transmission failed */ 5017 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE))) 5018 dev->stats.tx_errors++; 5019 5020 desc = 0; 5021 if (lp->rm9200_txq[desc].skb) { 5022 dev_consume_skb_irq(lp->rm9200_txq[desc].skb); 5023 lp->rm9200_txq[desc].skb = NULL; 5024 dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, 5025 lp->rm9200_txq[desc].size, DMA_TO_DEVICE); 5026 dev->stats.tx_packets++; 5027 dev->stats.tx_bytes += lp->rm9200_txq[desc].size; 5028 } 5029 netif_wake_queue(dev); 5030 } 5031 5032 /* Work-around for EMAC Errata section 41.3.1 */ 5033 if (intstatus & MACB_BIT(RXUBR)) { 5034 ctl = macb_readl(lp, NCR); 5035 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE)); 5036 wmb(); 5037 macb_writel(lp, NCR, ctl | MACB_BIT(RE)); 5038 } 5039 5040 if (intstatus & MACB_BIT(ISR_ROVR)) 5041 netdev_err(dev, "ROVR error\n"); 5042 5043 return IRQ_HANDLED; 5044} 5045 5046#ifdef CONFIG_NET_POLL_CONTROLLER 5047static void at91ether_poll_controller(struct net_device *dev) 5048{ 5049 unsigned long flags; 5050 5051 local_irq_save(flags); 5052 at91ether_interrupt(dev->irq, dev); 5053 local_irq_restore(flags); 5054} 5055#endif 5056 5057static const struct net_device_ops at91ether_netdev_ops = { 5058 .ndo_open = at91ether_open, 5059 .ndo_stop = at91ether_close, 5060 .ndo_start_xmit = at91ether_start_xmit, 5061 .ndo_get_stats64 = macb_get_stats, 5062 .ndo_set_rx_mode = macb_set_rx_mode, 5063 .ndo_set_mac_address = eth_mac_addr, 5064 .ndo_eth_ioctl = macb_ioctl, 5065 .ndo_validate_addr = eth_validate_addr, 5066#ifdef CONFIG_NET_POLL_CONTROLLER 5067 .ndo_poll_controller = at91ether_poll_controller, 5068#endif 5069 .ndo_hwtstamp_set = macb_hwtstamp_set, 5070 .ndo_hwtstamp_get = macb_hwtstamp_get, 5071}; 5072 5073static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk, 5074 struct clk **hclk, struct clk **tx_clk, 5075 struct clk **rx_clk, struct clk **tsu_clk) 5076{ 5077 int err; 5078 5079 *hclk = NULL; 5080 *tx_clk = NULL; 5081 *rx_clk = NULL; 5082 *tsu_clk = NULL; 5083 5084 *pclk = devm_clk_get(&pdev->dev, "ether_clk"); 5085 if (IS_ERR(*pclk)) 5086 return PTR_ERR(*pclk); 5087 5088 err = clk_prepare_enable(*pclk); 5089 if (err) { 5090 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err); 5091 return err; 5092 } 5093 5094 return 0; 5095} 5096 5097static int at91ether_init(struct platform_device *pdev) 5098{ 5099 struct net_device *dev = platform_get_drvdata(pdev); 5100 struct macb *bp = netdev_priv(dev); 5101 int err; 5102 5103 bp->queues[0].bp = bp; 5104 5105 dev->netdev_ops = &at91ether_netdev_ops; 5106 dev->ethtool_ops = &macb_ethtool_ops; 5107 5108 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt, 5109 0, dev->name, dev); 5110 if (err) 5111 return err; 5112 5113 macb_writel(bp, NCR, 0); 5114 5115 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG)); 5116 5117 return 0; 5118} 5119 5120static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw, 5121 unsigned long parent_rate) 5122{ 5123 return mgmt->rate; 5124} 5125 5126static int fu540_macb_tx_determine_rate(struct clk_hw *hw, 5127 struct clk_rate_request *req) 5128{ 5129 if (WARN_ON(req->rate < 2500000)) 5130 req->rate = 2500000; 5131 else if (req->rate == 2500000) 5132 req->rate = 2500000; 5133 else if (WARN_ON(req->rate < 13750000)) 5134 req->rate = 2500000; 5135 else if (WARN_ON(req->rate < 25000000)) 5136 req->rate = 25000000; 5137 else if (req->rate == 25000000) 5138 req->rate = 25000000; 5139 else if (WARN_ON(req->rate < 75000000)) 5140 req->rate = 25000000; 5141 else if (WARN_ON(req->rate < 125000000)) 5142 req->rate = 125000000; 5143 else if (req->rate == 125000000) 5144 req->rate = 125000000; 5145 else if (WARN_ON(req->rate > 125000000)) 5146 req->rate = 125000000; 5147 else 5148 req->rate = 125000000; 5149 5150 return 0; 5151} 5152 5153static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate, 5154 unsigned long parent_rate) 5155{ 5156 struct clk_rate_request req; 5157 int ret; 5158 5159 clk_hw_init_rate_request(hw, &req, rate); 5160 ret = fu540_macb_tx_determine_rate(hw, &req); 5161 if (ret != 0) 5162 return ret; 5163 5164 if (req.rate != 125000000) 5165 iowrite32(1, mgmt->reg); 5166 else 5167 iowrite32(0, mgmt->reg); 5168 mgmt->rate = rate; 5169 5170 return 0; 5171} 5172 5173static const struct clk_ops fu540_c000_ops = { 5174 .recalc_rate = fu540_macb_tx_recalc_rate, 5175 .determine_rate = fu540_macb_tx_determine_rate, 5176 .set_rate = fu540_macb_tx_set_rate, 5177}; 5178 5179static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, 5180 struct clk **hclk, struct clk **tx_clk, 5181 struct clk **rx_clk, struct clk **tsu_clk) 5182{ 5183 struct clk_init_data init; 5184 int err = 0; 5185 5186 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); 5187 if (err) 5188 return err; 5189 5190 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL); 5191 if (!mgmt) { 5192 err = -ENOMEM; 5193 goto err_disable_clks; 5194 } 5195 5196 init.name = "sifive-gemgxl-mgmt"; 5197 init.ops = &fu540_c000_ops; 5198 init.flags = 0; 5199 init.num_parents = 0; 5200 5201 mgmt->rate = 0; 5202 mgmt->hw.init = &init; 5203 5204 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw); 5205 if (IS_ERR(*tx_clk)) { 5206 err = PTR_ERR(*tx_clk); 5207 goto err_disable_clks; 5208 } 5209 5210 err = clk_prepare_enable(*tx_clk); 5211 if (err) { 5212 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err); 5213 *tx_clk = NULL; 5214 goto err_disable_clks; 5215 } else { 5216 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name); 5217 } 5218 5219 return 0; 5220 5221err_disable_clks: 5222 macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk); 5223 5224 return err; 5225} 5226 5227static int fu540_c000_init(struct platform_device *pdev) 5228{ 5229 mgmt->reg = devm_platform_ioremap_resource(pdev, 1); 5230 if (IS_ERR(mgmt->reg)) 5231 return PTR_ERR(mgmt->reg); 5232 5233 return macb_init(pdev); 5234} 5235 5236static int init_reset_optional(struct platform_device *pdev) 5237{ 5238 struct net_device *dev = platform_get_drvdata(pdev); 5239 struct macb *bp = netdev_priv(dev); 5240 int ret; 5241 5242 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { 5243 /* Ensure PHY device used in SGMII mode is ready */ 5244 bp->phy = devm_phy_optional_get(&pdev->dev, NULL); 5245 5246 if (IS_ERR(bp->phy)) 5247 return dev_err_probe(&pdev->dev, PTR_ERR(bp->phy), 5248 "failed to get SGMII PHY\n"); 5249 5250 ret = phy_init(bp->phy); 5251 if (ret) 5252 return dev_err_probe(&pdev->dev, ret, 5253 "failed to init SGMII PHY\n"); 5254 5255 ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG); 5256 if (!ret) { 5257 u32 pm_info[2]; 5258 5259 ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains", 5260 pm_info, ARRAY_SIZE(pm_info)); 5261 if (ret) { 5262 dev_err(&pdev->dev, "Failed to read power management information\n"); 5263 goto err_out_phy_exit; 5264 } 5265 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0); 5266 if (ret) 5267 goto err_out_phy_exit; 5268 5269 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1); 5270 if (ret) 5271 goto err_out_phy_exit; 5272 } 5273 5274 } 5275 5276 /* Fully reset controller at hardware level if mapped in device tree */ 5277 ret = device_reset_optional(&pdev->dev); 5278 if (ret) { 5279 phy_exit(bp->phy); 5280 return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); 5281 } 5282 5283 ret = macb_init(pdev); 5284 5285err_out_phy_exit: 5286 if (ret) 5287 phy_exit(bp->phy); 5288 5289 return ret; 5290} 5291 5292static int eyeq5_init(struct platform_device *pdev) 5293{ 5294 struct net_device *netdev = platform_get_drvdata(pdev); 5295 struct macb *bp = netdev_priv(netdev); 5296 struct device *dev = &pdev->dev; 5297 int ret; 5298 5299 bp->phy = devm_phy_get(dev, NULL); 5300 if (IS_ERR(bp->phy)) 5301 return dev_err_probe(dev, PTR_ERR(bp->phy), 5302 "failed to get PHY\n"); 5303 5304 ret = phy_init(bp->phy); 5305 if (ret) 5306 return dev_err_probe(dev, ret, "failed to init PHY\n"); 5307 5308 ret = macb_init(pdev); 5309 if (ret) 5310 phy_exit(bp->phy); 5311 return ret; 5312} 5313 5314static const struct macb_usrio_config sama7g5_usrio = { 5315 .mii = 0, 5316 .rmii = 1, 5317 .rgmii = 2, 5318 .refclk = BIT(2), 5319 .hdfctlen = BIT(6), 5320}; 5321 5322static const struct macb_config fu540_c000_config = { 5323 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 5324 MACB_CAPS_GEM_HAS_PTP, 5325 .dma_burst_length = 16, 5326 .clk_init = fu540_c000_clk_init, 5327 .init = fu540_c000_init, 5328 .jumbo_max_len = 10240, 5329 .usrio = &macb_default_usrio, 5330}; 5331 5332static const struct macb_config at91sam9260_config = { 5333 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 5334 .clk_init = macb_clk_init, 5335 .init = macb_init, 5336 .usrio = &macb_default_usrio, 5337}; 5338 5339static const struct macb_config sama5d3macb_config = { 5340 .caps = MACB_CAPS_SG_DISABLED | 5341 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 5342 .clk_init = macb_clk_init, 5343 .init = macb_init, 5344 .usrio = &macb_default_usrio, 5345}; 5346 5347static const struct macb_config pc302gem_config = { 5348 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, 5349 .dma_burst_length = 16, 5350 .clk_init = macb_clk_init, 5351 .init = macb_init, 5352 .usrio = &macb_default_usrio, 5353}; 5354 5355static const struct macb_config sama5d2_config = { 5356 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 5357 .dma_burst_length = 16, 5358 .clk_init = macb_clk_init, 5359 .init = macb_init, 5360 .jumbo_max_len = 10240, 5361 .usrio = &macb_default_usrio, 5362}; 5363 5364static const struct macb_config sama5d29_config = { 5365 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP, 5366 .dma_burst_length = 16, 5367 .clk_init = macb_clk_init, 5368 .init = macb_init, 5369 .usrio = &macb_default_usrio, 5370}; 5371 5372static const struct macb_config sama5d3_config = { 5373 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5374 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, 5375 .dma_burst_length = 16, 5376 .clk_init = macb_clk_init, 5377 .init = macb_init, 5378 .jumbo_max_len = 10240, 5379 .usrio = &macb_default_usrio, 5380}; 5381 5382static const struct macb_config sama5d4_config = { 5383 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, 5384 .dma_burst_length = 4, 5385 .clk_init = macb_clk_init, 5386 .init = macb_init, 5387 .usrio = &macb_default_usrio, 5388}; 5389 5390static const struct macb_config emac_config = { 5391 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC, 5392 .clk_init = at91ether_clk_init, 5393 .init = at91ether_init, 5394 .usrio = &macb_default_usrio, 5395}; 5396 5397static const struct macb_config np4_config = { 5398 .caps = MACB_CAPS_USRIO_DISABLED, 5399 .clk_init = macb_clk_init, 5400 .init = macb_init, 5401 .usrio = &macb_default_usrio, 5402}; 5403 5404static const struct macb_config zynqmp_config = { 5405 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5406 MACB_CAPS_JUMBO | 5407 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, 5408 .dma_burst_length = 16, 5409 .clk_init = macb_clk_init, 5410 .init = init_reset_optional, 5411 .jumbo_max_len = 10240, 5412 .usrio = &macb_default_usrio, 5413}; 5414 5415static const struct macb_config zynq_config = { 5416 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | 5417 MACB_CAPS_NEEDS_RSTONUBR, 5418 .dma_burst_length = 16, 5419 .clk_init = macb_clk_init, 5420 .init = macb_init, 5421 .usrio = &macb_default_usrio, 5422}; 5423 5424static const struct macb_config mpfs_config = { 5425 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5426 MACB_CAPS_JUMBO | 5427 MACB_CAPS_GEM_HAS_PTP, 5428 .dma_burst_length = 16, 5429 .clk_init = macb_clk_init, 5430 .init = init_reset_optional, 5431 .usrio = &macb_default_usrio, 5432 .max_tx_length = 4040, /* Cadence Erratum 1686 */ 5433 .jumbo_max_len = 4040, 5434}; 5435 5436static const struct macb_config sama7g5_gem_config = { 5437 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | 5438 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 5439 MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP, 5440 .dma_burst_length = 16, 5441 .clk_init = macb_clk_init, 5442 .init = macb_init, 5443 .usrio = &sama7g5_usrio, 5444}; 5445 5446static const struct macb_config sama7g5_emac_config = { 5447 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | 5448 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII | 5449 MACB_CAPS_GEM_HAS_PTP, 5450 .dma_burst_length = 16, 5451 .clk_init = macb_clk_init, 5452 .init = macb_init, 5453 .usrio = &sama7g5_usrio, 5454}; 5455 5456static const struct macb_config versal_config = { 5457 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 5458 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | 5459 MACB_CAPS_NEED_TSUCLK | MACB_CAPS_QUEUE_DISABLE | 5460 MACB_CAPS_QBV, 5461 .dma_burst_length = 16, 5462 .clk_init = macb_clk_init, 5463 .init = init_reset_optional, 5464 .jumbo_max_len = 10240, 5465 .usrio = &macb_default_usrio, 5466}; 5467 5468static const struct macb_config eyeq5_config = { 5469 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | 5470 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_QUEUE_DISABLE | 5471 MACB_CAPS_NO_LSO, 5472 .dma_burst_length = 16, 5473 .clk_init = macb_clk_init, 5474 .init = eyeq5_init, 5475 .jumbo_max_len = 10240, 5476 .usrio = &macb_default_usrio, 5477}; 5478 5479static const struct macb_config raspberrypi_rp1_config = { 5480 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | 5481 MACB_CAPS_JUMBO | 5482 MACB_CAPS_GEM_HAS_PTP, 5483 .dma_burst_length = 16, 5484 .clk_init = macb_clk_init, 5485 .init = macb_init, 5486 .usrio = &macb_default_usrio, 5487 .jumbo_max_len = 10240, 5488}; 5489 5490static const struct of_device_id macb_dt_ids[] = { 5491 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, 5492 { .compatible = "cdns,macb" }, 5493 { .compatible = "cdns,np4-macb", .data = &np4_config }, 5494 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config }, 5495 { .compatible = "cdns,gem", .data = &pc302gem_config }, 5496 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config }, 5497 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config }, 5498 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config }, 5499 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config }, 5500 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config }, 5501 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config }, 5502 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config }, 5503 { .compatible = "cdns,emac", .data = &emac_config }, 5504 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */ 5505 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */ 5506 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, 5507 { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, 5508 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, 5509 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, 5510 { .compatible = "mobileye,eyeq5-gem", .data = &eyeq5_config }, 5511 { .compatible = "raspberrypi,rp1-gem", .data = &raspberrypi_rp1_config }, 5512 { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config}, 5513 { .compatible = "xlnx,zynq-gem", .data = &zynq_config }, 5514 { .compatible = "xlnx,versal-gem", .data = &versal_config}, 5515 { /* sentinel */ } 5516}; 5517MODULE_DEVICE_TABLE(of, macb_dt_ids); 5518#endif /* CONFIG_OF */ 5519 5520static const struct macb_config default_gem_config = { 5521 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | 5522 MACB_CAPS_JUMBO | 5523 MACB_CAPS_GEM_HAS_PTP, 5524 .dma_burst_length = 16, 5525 .clk_init = macb_clk_init, 5526 .init = macb_init, 5527 .usrio = &macb_default_usrio, 5528 .jumbo_max_len = 10240, 5529}; 5530 5531static int macb_probe(struct platform_device *pdev) 5532{ 5533 const struct macb_config *macb_config = &default_gem_config; 5534 struct device_node *np = pdev->dev.of_node; 5535 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL; 5536 struct clk *tsu_clk = NULL; 5537 phy_interface_t interface; 5538 struct net_device *dev; 5539 struct resource *regs; 5540 u32 wtrmrk_rst_val; 5541 void __iomem *mem; 5542 struct macb *bp; 5543 int num_queues; 5544 bool native_io; 5545 int err, val; 5546 5547 mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); 5548 if (IS_ERR(mem)) 5549 return PTR_ERR(mem); 5550 5551 if (np) { 5552 const struct of_device_id *match; 5553 5554 match = of_match_node(macb_dt_ids, np); 5555 if (match && match->data) 5556 macb_config = match->data; 5557 } 5558 5559 err = macb_config->clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); 5560 if (err) 5561 return err; 5562 5563 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT); 5564 pm_runtime_use_autosuspend(&pdev->dev); 5565 pm_runtime_get_noresume(&pdev->dev); 5566 pm_runtime_set_active(&pdev->dev); 5567 pm_runtime_enable(&pdev->dev); 5568 native_io = hw_is_native_io(mem); 5569 5570 num_queues = macb_probe_queues(&pdev->dev, mem, native_io); 5571 if (num_queues < 0) { 5572 err = num_queues; 5573 goto err_disable_clocks; 5574 } 5575 5576 dev = alloc_etherdev_mq(sizeof(*bp), num_queues); 5577 if (!dev) { 5578 err = -ENOMEM; 5579 goto err_disable_clocks; 5580 } 5581 5582 dev->base_addr = regs->start; 5583 5584 SET_NETDEV_DEV(dev, &pdev->dev); 5585 5586 bp = netdev_priv(dev); 5587 bp->pdev = pdev; 5588 bp->dev = dev; 5589 bp->regs = mem; 5590 bp->native_io = native_io; 5591 if (native_io) { 5592 bp->macb_reg_readl = hw_readl_native; 5593 bp->macb_reg_writel = hw_writel_native; 5594 } else { 5595 bp->macb_reg_readl = hw_readl; 5596 bp->macb_reg_writel = hw_writel; 5597 } 5598 bp->num_queues = num_queues; 5599 bp->dma_burst_length = macb_config->dma_burst_length; 5600 bp->pclk = pclk; 5601 bp->hclk = hclk; 5602 bp->tx_clk = tx_clk; 5603 bp->rx_clk = rx_clk; 5604 bp->tsu_clk = tsu_clk; 5605 bp->jumbo_max_len = macb_config->jumbo_max_len; 5606 5607 if (!hw_is_gem(bp->regs, bp->native_io)) 5608 bp->max_tx_length = MACB_MAX_TX_LEN; 5609 else if (macb_config->max_tx_length) 5610 bp->max_tx_length = macb_config->max_tx_length; 5611 else 5612 bp->max_tx_length = GEM_MAX_TX_LEN; 5613 5614 bp->wol = 0; 5615 device_set_wakeup_capable(&pdev->dev, 1); 5616 5617 bp->usrio = macb_config->usrio; 5618 5619 /* By default we set to partial store and forward mode for zynqmp. 5620 * Disable if not set in devicetree. 5621 */ 5622 if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) { 5623 err = of_property_read_u32(bp->pdev->dev.of_node, 5624 "cdns,rx-watermark", 5625 &bp->rx_watermark); 5626 5627 if (!err) { 5628 /* Disable partial store and forward in case of error or 5629 * invalid watermark value 5630 */ 5631 wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1; 5632 if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) { 5633 dev_info(&bp->pdev->dev, "Invalid watermark value\n"); 5634 bp->rx_watermark = 0; 5635 } 5636 } 5637 } 5638 spin_lock_init(&bp->lock); 5639 spin_lock_init(&bp->stats_lock); 5640 5641 /* setup capabilities */ 5642 macb_configure_caps(bp, macb_config); 5643 5644#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 5645 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) { 5646 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); 5647 if (err) { 5648 dev_err(&pdev->dev, "failed to set DMA mask\n"); 5649 goto err_out_free_netdev; 5650 } 5651 bp->caps |= MACB_CAPS_DMA_64B; 5652 } 5653#endif 5654 platform_set_drvdata(pdev, dev); 5655 5656 dev->irq = platform_get_irq(pdev, 0); 5657 if (dev->irq < 0) { 5658 err = dev->irq; 5659 goto err_out_free_netdev; 5660 } 5661 5662 /* MTU range: 68 - 1518 or 10240 */ 5663 dev->min_mtu = GEM_MTU_MIN_SIZE; 5664 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) 5665 dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN; 5666 else 5667 dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN; 5668 5669 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) { 5670 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10)); 5671 if (val) 5672 bp->rx_bd_rd_prefetch = (2 << (val - 1)) * 5673 macb_dma_desc_get_size(bp); 5674 5675 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10)); 5676 if (val) 5677 bp->tx_bd_rd_prefetch = (2 << (val - 1)) * 5678 macb_dma_desc_get_size(bp); 5679 } 5680 5681 bp->rx_intr_mask = MACB_RX_INT_FLAGS; 5682 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR) 5683 bp->rx_intr_mask |= MACB_BIT(RXUBR); 5684 5685 err = of_get_ethdev_address(np, bp->dev); 5686 if (err == -EPROBE_DEFER) 5687 goto err_out_free_netdev; 5688 else if (err) 5689 macb_get_hwaddr(bp); 5690 5691 err = of_get_phy_mode(np, &interface); 5692 if (err) 5693 /* not found in DT, MII by default */ 5694 bp->phy_interface = PHY_INTERFACE_MODE_MII; 5695 else 5696 bp->phy_interface = interface; 5697 5698 /* IP specific init */ 5699 err = macb_config->init(pdev); 5700 if (err) 5701 goto err_out_free_netdev; 5702 5703 err = macb_mii_init(bp); 5704 if (err) 5705 goto err_out_phy_exit; 5706 5707 netif_carrier_off(dev); 5708 5709 err = register_netdev(dev); 5710 if (err) { 5711 dev_err(&pdev->dev, "Cannot register net device, aborting.\n"); 5712 goto err_out_unregister_mdio; 5713 } 5714 5715 INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task); 5716 5717 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", 5718 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), 5719 dev->base_addr, dev->irq, dev->dev_addr); 5720 5721 pm_runtime_put_autosuspend(&bp->pdev->dev); 5722 5723 return 0; 5724 5725err_out_unregister_mdio: 5726 mdiobus_unregister(bp->mii_bus); 5727 mdiobus_free(bp->mii_bus); 5728 5729err_out_phy_exit: 5730 phy_exit(bp->phy); 5731 5732err_out_free_netdev: 5733 free_netdev(dev); 5734 5735err_disable_clocks: 5736 macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk); 5737 pm_runtime_disable(&pdev->dev); 5738 pm_runtime_set_suspended(&pdev->dev); 5739 pm_runtime_dont_use_autosuspend(&pdev->dev); 5740 5741 return err; 5742} 5743 5744static void macb_remove(struct platform_device *pdev) 5745{ 5746 struct net_device *dev; 5747 struct macb *bp; 5748 5749 dev = platform_get_drvdata(pdev); 5750 5751 if (dev) { 5752 bp = netdev_priv(dev); 5753 unregister_netdev(dev); 5754 phy_exit(bp->phy); 5755 mdiobus_unregister(bp->mii_bus); 5756 mdiobus_free(bp->mii_bus); 5757 5758 device_set_wakeup_enable(&bp->pdev->dev, 0); 5759 cancel_work_sync(&bp->hresp_err_bh_work); 5760 pm_runtime_disable(&pdev->dev); 5761 pm_runtime_dont_use_autosuspend(&pdev->dev); 5762 pm_runtime_set_suspended(&pdev->dev); 5763 phylink_destroy(bp->phylink); 5764 free_netdev(dev); 5765 } 5766} 5767 5768static int __maybe_unused macb_suspend(struct device *dev) 5769{ 5770 struct net_device *netdev = dev_get_drvdata(dev); 5771 struct macb *bp = netdev_priv(netdev); 5772 struct in_ifaddr *ifa = NULL; 5773 struct macb_queue *queue; 5774 struct in_device *idev; 5775 unsigned long flags; 5776 u32 tmp, ifa_local; 5777 unsigned int q; 5778 int err; 5779 5780 if (!device_may_wakeup(&bp->dev->dev)) 5781 phy_exit(bp->phy); 5782 5783 if (!netif_running(netdev)) 5784 return 0; 5785 5786 if (bp->wol & MACB_WOL_ENABLED) { 5787 if (bp->wolopts & WAKE_ARP) { 5788 /* Check for IP address in WOL ARP mode */ 5789 rcu_read_lock(); 5790 idev = __in_dev_get_rcu(bp->dev); 5791 if (idev) 5792 ifa = rcu_dereference(idev->ifa_list); 5793 if (!ifa) { 5794 rcu_read_unlock(); 5795 netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n"); 5796 return -EOPNOTSUPP; 5797 } 5798 ifa_local = be32_to_cpu(ifa->ifa_local); 5799 rcu_read_unlock(); 5800 } 5801 5802 spin_lock_irqsave(&bp->lock, flags); 5803 5804 /* Disable Tx and Rx engines before disabling the queues, 5805 * this is mandatory as per the IP spec sheet 5806 */ 5807 tmp = macb_readl(bp, NCR); 5808 macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE))); 5809#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 5810 if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) 5811 macb_writel(bp, RBQPH, 5812 upper_32_bits(bp->rx_ring_tieoff_dma)); 5813#endif 5814 for (q = 0, queue = bp->queues; q < bp->num_queues; 5815 ++q, ++queue) { 5816 /* Disable RX queues */ 5817 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) { 5818 queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE)); 5819 } else { 5820 /* Tie off RX queues */ 5821 queue_writel(queue, RBQP, 5822 lower_32_bits(bp->rx_ring_tieoff_dma)); 5823 } 5824 /* Disable all interrupts */ 5825 queue_writel(queue, IDR, -1); 5826 queue_readl(queue, ISR); 5827 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 5828 queue_writel(queue, ISR, -1); 5829 } 5830 /* Enable Receive engine */ 5831 macb_writel(bp, NCR, tmp | MACB_BIT(RE)); 5832 /* Flush all status bits */ 5833 macb_writel(bp, TSR, -1); 5834 macb_writel(bp, RSR, -1); 5835 5836 tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0; 5837 if (bp->wolopts & WAKE_ARP) { 5838 tmp |= MACB_BIT(ARP); 5839 /* write IP address into register */ 5840 tmp |= MACB_BFEXT(IP, ifa_local); 5841 } 5842 spin_unlock_irqrestore(&bp->lock, flags); 5843 5844 /* Change interrupt handler and 5845 * Enable WoL IRQ on queue 0 5846 */ 5847 devm_free_irq(dev, bp->queues[0].irq, bp->queues); 5848 if (macb_is_gem(bp)) { 5849 err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, 5850 IRQF_SHARED, netdev->name, bp->queues); 5851 if (err) { 5852 dev_err(dev, 5853 "Unable to request IRQ %d (error %d)\n", 5854 bp->queues[0].irq, err); 5855 return err; 5856 } 5857 spin_lock_irqsave(&bp->lock, flags); 5858 queue_writel(bp->queues, IER, GEM_BIT(WOL)); 5859 gem_writel(bp, WOL, tmp); 5860 spin_unlock_irqrestore(&bp->lock, flags); 5861 } else { 5862 err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, 5863 IRQF_SHARED, netdev->name, bp->queues); 5864 if (err) { 5865 dev_err(dev, 5866 "Unable to request IRQ %d (error %d)\n", 5867 bp->queues[0].irq, err); 5868 return err; 5869 } 5870 spin_lock_irqsave(&bp->lock, flags); 5871 queue_writel(bp->queues, IER, MACB_BIT(WOL)); 5872 macb_writel(bp, WOL, tmp); 5873 spin_unlock_irqrestore(&bp->lock, flags); 5874 } 5875 5876 enable_irq_wake(bp->queues[0].irq); 5877 } 5878 5879 netif_device_detach(netdev); 5880 for (q = 0, queue = bp->queues; q < bp->num_queues; 5881 ++q, ++queue) { 5882 napi_disable(&queue->napi_rx); 5883 napi_disable(&queue->napi_tx); 5884 } 5885 5886 if (!(bp->wol & MACB_WOL_ENABLED)) { 5887 rtnl_lock(); 5888 phylink_stop(bp->phylink); 5889 rtnl_unlock(); 5890 spin_lock_irqsave(&bp->lock, flags); 5891 macb_reset_hw(bp); 5892 spin_unlock_irqrestore(&bp->lock, flags); 5893 } 5894 5895 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 5896 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO); 5897 5898 if (netdev->hw_features & NETIF_F_NTUPLE) 5899 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT); 5900 5901 if (bp->ptp_info) 5902 bp->ptp_info->ptp_remove(netdev); 5903 if (!device_may_wakeup(dev)) 5904 pm_runtime_force_suspend(dev); 5905 5906 return 0; 5907} 5908 5909static int __maybe_unused macb_resume(struct device *dev) 5910{ 5911 struct net_device *netdev = dev_get_drvdata(dev); 5912 struct macb *bp = netdev_priv(netdev); 5913 struct macb_queue *queue; 5914 unsigned long flags; 5915 unsigned int q; 5916 int err; 5917 5918 if (!device_may_wakeup(&bp->dev->dev)) 5919 phy_init(bp->phy); 5920 5921 if (!netif_running(netdev)) 5922 return 0; 5923 5924 if (!device_may_wakeup(dev)) 5925 pm_runtime_force_resume(dev); 5926 5927 if (bp->wol & MACB_WOL_ENABLED) { 5928 spin_lock_irqsave(&bp->lock, flags); 5929 /* Disable WoL */ 5930 if (macb_is_gem(bp)) { 5931 queue_writel(bp->queues, IDR, GEM_BIT(WOL)); 5932 gem_writel(bp, WOL, 0); 5933 } else { 5934 queue_writel(bp->queues, IDR, MACB_BIT(WOL)); 5935 macb_writel(bp, WOL, 0); 5936 } 5937 /* Clear ISR on queue 0 */ 5938 queue_readl(bp->queues, ISR); 5939 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) 5940 queue_writel(bp->queues, ISR, -1); 5941 spin_unlock_irqrestore(&bp->lock, flags); 5942 5943 /* Replace interrupt handler on queue 0 */ 5944 devm_free_irq(dev, bp->queues[0].irq, bp->queues); 5945 err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, 5946 IRQF_SHARED, netdev->name, bp->queues); 5947 if (err) { 5948 dev_err(dev, 5949 "Unable to request IRQ %d (error %d)\n", 5950 bp->queues[0].irq, err); 5951 return err; 5952 } 5953 5954 disable_irq_wake(bp->queues[0].irq); 5955 5956 /* Now make sure we disable phy before moving 5957 * to common restore path 5958 */ 5959 rtnl_lock(); 5960 phylink_stop(bp->phylink); 5961 rtnl_unlock(); 5962 } 5963 5964 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) 5965 macb_init_buffers(bp); 5966 5967 for (q = 0, queue = bp->queues; q < bp->num_queues; 5968 ++q, ++queue) { 5969 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) { 5970 if (macb_is_gem(bp)) 5971 gem_init_rx_ring(queue); 5972 else 5973 macb_init_rx_ring(queue); 5974 } 5975 5976 napi_enable(&queue->napi_rx); 5977 napi_enable(&queue->napi_tx); 5978 } 5979 5980 if (netdev->hw_features & NETIF_F_NTUPLE) 5981 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2); 5982 5983 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) 5984 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio); 5985 5986 macb_writel(bp, NCR, MACB_BIT(MPE)); 5987 macb_init_hw(bp); 5988 macb_set_rx_mode(netdev); 5989 macb_restore_features(bp); 5990 rtnl_lock(); 5991 5992 phylink_start(bp->phylink); 5993 rtnl_unlock(); 5994 5995 netif_device_attach(netdev); 5996 if (bp->ptp_info) 5997 bp->ptp_info->ptp_init(netdev); 5998 5999 return 0; 6000} 6001 6002static int __maybe_unused macb_runtime_suspend(struct device *dev) 6003{ 6004 struct net_device *netdev = dev_get_drvdata(dev); 6005 struct macb *bp = netdev_priv(netdev); 6006 6007 if (!(device_may_wakeup(dev))) 6008 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk); 6009 else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) 6010 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk); 6011 6012 return 0; 6013} 6014 6015static int __maybe_unused macb_runtime_resume(struct device *dev) 6016{ 6017 struct net_device *netdev = dev_get_drvdata(dev); 6018 struct macb *bp = netdev_priv(netdev); 6019 6020 if (!(device_may_wakeup(dev))) { 6021 clk_prepare_enable(bp->pclk); 6022 clk_prepare_enable(bp->hclk); 6023 clk_prepare_enable(bp->tx_clk); 6024 clk_prepare_enable(bp->rx_clk); 6025 clk_prepare_enable(bp->tsu_clk); 6026 } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) { 6027 clk_prepare_enable(bp->tsu_clk); 6028 } 6029 6030 return 0; 6031} 6032 6033static void macb_shutdown(struct platform_device *pdev) 6034{ 6035 struct net_device *netdev = platform_get_drvdata(pdev); 6036 6037 rtnl_lock(); 6038 6039 if (netif_running(netdev)) 6040 dev_close(netdev); 6041 6042 netif_device_detach(netdev); 6043 6044 rtnl_unlock(); 6045} 6046 6047static const struct dev_pm_ops macb_pm_ops = { 6048 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume) 6049 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL) 6050}; 6051 6052static struct platform_driver macb_driver = { 6053 .probe = macb_probe, 6054 .remove = macb_remove, 6055 .driver = { 6056 .name = "macb", 6057 .of_match_table = of_match_ptr(macb_dt_ids), 6058 .pm = &macb_pm_ops, 6059 }, 6060 .shutdown = macb_shutdown, 6061}; 6062 6063module_platform_driver(macb_driver); 6064 6065MODULE_LICENSE("GPL"); 6066MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver"); 6067MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 6068MODULE_ALIAS("platform:macb");
~/
Read-only file preview.