xgbe-drv.c

   1// SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-3-Clause)
   2/*
   3 * Copyright (c) 2014-2025, Advanced Micro Devices, Inc.
   4 * Copyright (c) 2014, Synopsys, Inc.
   5 * All rights reserved
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/spinlock.h>
  10#include <linux/tcp.h>
  11#include <linux/if_vlan.h>
  12#include <linux/interrupt.h>
  13#include <linux/clk.h>
  14#include <linux/if_ether.h>
  15#include <linux/net_tstamp.h>
  16#include <linux/phy.h>
  17#include <net/vxlan.h>
  18
  19#include "xgbe.h"
  20#include "xgbe-common.h"
  21
  22static unsigned int ecc_sec_info_threshold = 10;
  23static unsigned int ecc_sec_warn_threshold = 10000;
  24static unsigned int ecc_sec_period = 600;
  25static unsigned int ecc_ded_threshold = 2;
  26static unsigned int ecc_ded_period = 600;
  27
  28#ifdef CONFIG_AMD_XGBE_HAVE_ECC
  29/* Only expose the ECC parameters if supported */
  30module_param(ecc_sec_info_threshold, uint, 0644);
  31MODULE_PARM_DESC(ecc_sec_info_threshold,
  32		 " ECC corrected error informational threshold setting");
  33
  34module_param(ecc_sec_warn_threshold, uint, 0644);
  35MODULE_PARM_DESC(ecc_sec_warn_threshold,
  36		 " ECC corrected error warning threshold setting");
  37
  38module_param(ecc_sec_period, uint, 0644);
  39MODULE_PARM_DESC(ecc_sec_period, " ECC corrected error period (in seconds)");
  40
  41module_param(ecc_ded_threshold, uint, 0644);
  42MODULE_PARM_DESC(ecc_ded_threshold, " ECC detected error threshold setting");
  43
  44module_param(ecc_ded_period, uint, 0644);
  45MODULE_PARM_DESC(ecc_ded_period, " ECC detected error period (in seconds)");
  46#endif
  47
  48static int xgbe_one_poll(struct napi_struct *, int);
  49static int xgbe_all_poll(struct napi_struct *, int);
  50static void xgbe_stop(struct xgbe_prv_data *);
  51
  52static void *xgbe_alloc_node(size_t size, int node)
  53{
  54	void *mem;
  55
  56	mem = kzalloc_node(size, GFP_KERNEL, node);
  57	if (!mem)
  58		mem = kzalloc(size, GFP_KERNEL);
  59
  60	return mem;
  61}
  62
  63static void xgbe_free_channels(struct xgbe_prv_data *pdata)
  64{
  65	unsigned int i;
  66
  67	for (i = 0; i < ARRAY_SIZE(pdata->channel); i++) {
  68		if (!pdata->channel[i])
  69			continue;
  70
  71		kfree(pdata->channel[i]->rx_ring);
  72		kfree(pdata->channel[i]->tx_ring);
  73		kfree(pdata->channel[i]);
  74
  75		pdata->channel[i] = NULL;
  76	}
  77
  78	pdata->channel_count = 0;
  79}
  80
  81static int xgbe_alloc_channels(struct xgbe_prv_data *pdata)
  82{
  83	struct xgbe_channel *channel;
  84	struct xgbe_ring *ring;
  85	unsigned int count, i;
  86	unsigned int cpu;
  87	int node;
  88
  89	count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count);
  90	for (i = 0; i < count; i++) {
  91		/* Attempt to use a CPU on the node the device is on */
  92		cpu = cpumask_local_spread(i, dev_to_node(pdata->dev));
  93
  94		/* Set the allocation node based on the returned CPU */
  95		node = cpu_to_node(cpu);
  96
  97		channel = xgbe_alloc_node(sizeof(*channel), node);
  98		if (!channel)
  99			goto err_mem;
 100		pdata->channel[i] = channel;
 101
 102		snprintf(channel->name, sizeof(channel->name), "channel-%u", i);
 103		channel->pdata = pdata;
 104		channel->queue_index = i;
 105		channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE +
 106				    (DMA_CH_INC * i);
 107		channel->node = node;
 108		cpumask_set_cpu(cpu, &channel->affinity_mask);
 109
 110		if (pdata->per_channel_irq)
 111			channel->dma_irq = pdata->channel_irq[i];
 112
 113		if (i < pdata->tx_ring_count) {
 114			ring = xgbe_alloc_node(sizeof(*ring), node);
 115			if (!ring)
 116				goto err_mem;
 117
 118			spin_lock_init(&ring->lock);
 119			ring->node = node;
 120
 121			channel->tx_ring = ring;
 122		}
 123
 124		if (i < pdata->rx_ring_count) {
 125			ring = xgbe_alloc_node(sizeof(*ring), node);
 126			if (!ring)
 127				goto err_mem;
 128
 129			spin_lock_init(&ring->lock);
 130			ring->node = node;
 131
 132			channel->rx_ring = ring;
 133		}
 134
 135		netif_dbg(pdata, drv, pdata->netdev,
 136			  "%s: cpu=%u, node=%d\n", channel->name, cpu, node);
 137
 138		netif_dbg(pdata, drv, pdata->netdev,
 139			  "%s: dma_regs=%p, dma_irq=%d, tx=%p, rx=%p\n",
 140			  channel->name, channel->dma_regs, channel->dma_irq,
 141			  channel->tx_ring, channel->rx_ring);
 142	}
 143
 144	pdata->channel_count = count;
 145
 146	return 0;
 147
 148err_mem:
 149	xgbe_free_channels(pdata);
 150
 151	return -ENOMEM;
 152}
 153
 154static inline unsigned int xgbe_tx_avail_desc(struct xgbe_ring *ring)
 155{
 156	return (ring->rdesc_count - (ring->cur - ring->dirty));
 157}
 158
 159static inline unsigned int xgbe_rx_dirty_desc(struct xgbe_ring *ring)
 160{
 161	return (ring->cur - ring->dirty);
 162}
 163
 164static int xgbe_maybe_stop_tx_queue(struct xgbe_channel *channel,
 165				    struct xgbe_ring *ring, unsigned int count)
 166{
 167	struct xgbe_prv_data *pdata = channel->pdata;
 168
 169	if (count > xgbe_tx_avail_desc(ring)) {
 170		netif_info(pdata, drv, pdata->netdev,
 171			   "Tx queue stopped, not enough descriptors available\n");
 172		netif_stop_subqueue(pdata->netdev, channel->queue_index);
 173		ring->tx.queue_stopped = 1;
 174
 175		/* If we haven't notified the hardware because of xmit_more
 176		 * support, tell it now
 177		 */
 178		if (ring->tx.xmit_more)
 179			pdata->hw_if.tx_start_xmit(channel, ring);
 180
 181		return NETDEV_TX_BUSY;
 182	}
 183
 184	return 0;
 185}
 186
 187static int xgbe_calc_rx_buf_size(struct net_device *netdev, unsigned int mtu)
 188{
 189	unsigned int rx_buf_size;
 190
 191	rx_buf_size = mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
 192	rx_buf_size = clamp_val(rx_buf_size, XGBE_RX_MIN_BUF_SIZE, PAGE_SIZE);
 193
 194	rx_buf_size = (rx_buf_size + XGBE_RX_BUF_ALIGN - 1) &
 195		      ~(XGBE_RX_BUF_ALIGN - 1);
 196
 197	return rx_buf_size;
 198}
 199
 200static void xgbe_enable_rx_tx_int(struct xgbe_prv_data *pdata,
 201				  struct xgbe_channel *channel)
 202{
 203	struct xgbe_hw_if *hw_if = &pdata->hw_if;
 204	enum xgbe_int int_id;
 205
 206	if (channel->tx_ring && channel->rx_ring)
 207		int_id = XGMAC_INT_DMA_CH_SR_TI_RI;
 208	else if (channel->tx_ring)
 209		int_id = XGMAC_INT_DMA_CH_SR_TI;
 210	else if (channel->rx_ring)
 211		int_id = XGMAC_INT_DMA_CH_SR_RI;
 212	else
 213		return;
 214
 215	hw_if->enable_int(channel, int_id);
 216}
 217
 218static void xgbe_enable_rx_tx_ints(struct xgbe_prv_data *pdata)
 219{
 220	unsigned int i;
 221
 222	for (i = 0; i < pdata->channel_count; i++)
 223		xgbe_enable_rx_tx_int(pdata, pdata->channel[i]);
 224}
 225
 226static void xgbe_disable_rx_tx_int(struct xgbe_prv_data *pdata,
 227				   struct xgbe_channel *channel)
 228{
 229	struct xgbe_hw_if *hw_if = &pdata->hw_if;
 230	enum xgbe_int int_id;
 231
 232	if (channel->tx_ring && channel->rx_ring)
 233		int_id = XGMAC_INT_DMA_CH_SR_TI_RI;
 234	else if (channel->tx_ring)
 235		int_id = XGMAC_INT_DMA_CH_SR_TI;
 236	else if (channel->rx_ring)
 237		int_id = XGMAC_INT_DMA_CH_SR_RI;
 238	else
 239		return;
 240
 241	hw_if->disable_int(channel, int_id);
 242}
 243
 244static void xgbe_disable_rx_tx_ints(struct xgbe_prv_data *pdata)
 245{
 246	unsigned int i;
 247
 248	for (i = 0; i < pdata->channel_count; i++)
 249		xgbe_disable_rx_tx_int(pdata, pdata->channel[i]);
 250}
 251
 252static bool xgbe_ecc_sec(struct xgbe_prv_data *pdata, unsigned long *period,
 253			 unsigned int *count, const char *area)
 254{
 255	if (time_before(jiffies, *period)) {
 256		(*count)++;
 257	} else {
 258		*period = jiffies + (ecc_sec_period * HZ);
 259		*count = 1;
 260	}
 261
 262	if (*count > ecc_sec_info_threshold)
 263		dev_warn_once(pdata->dev,
 264			      "%s ECC corrected errors exceed informational threshold\n",
 265			      area);
 266
 267	if (*count > ecc_sec_warn_threshold) {
 268		dev_warn_once(pdata->dev,
 269			      "%s ECC corrected errors exceed warning threshold\n",
 270			      area);
 271		return true;
 272	}
 273
 274	return false;
 275}
 276
 277static bool xgbe_ecc_ded(struct xgbe_prv_data *pdata, unsigned long *period,
 278			 unsigned int *count, const char *area)
 279{
 280	if (time_before(jiffies, *period)) {
 281		(*count)++;
 282	} else {
 283		*period = jiffies + (ecc_ded_period * HZ);
 284		*count = 1;
 285	}
 286
 287	if (*count > ecc_ded_threshold) {
 288		netdev_alert(pdata->netdev,
 289			     "%s ECC detected errors exceed threshold\n",
 290			     area);
 291		return true;
 292	}
 293
 294	return false;
 295}
 296
 297static void xgbe_ecc_isr_bh_work(struct work_struct *work)
 298{
 299	struct xgbe_prv_data *pdata = from_work(pdata, work, ecc_bh_work);
 300	unsigned int ecc_isr;
 301	bool stop = false;
 302
 303	/* Mask status with only the interrupts we care about */
 304	ecc_isr = XP_IOREAD(pdata, XP_ECC_ISR);
 305	ecc_isr &= XP_IOREAD(pdata, XP_ECC_IER);
 306	netif_dbg(pdata, intr, pdata->netdev, "ECC_ISR=%#010x\n", ecc_isr);
 307
 308	if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, TX_DED)) {
 309		stop |= xgbe_ecc_ded(pdata, &pdata->tx_ded_period,
 310				     &pdata->tx_ded_count, "TX fifo");
 311	}
 312
 313	if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, RX_DED)) {
 314		stop |= xgbe_ecc_ded(pdata, &pdata->rx_ded_period,
 315				     &pdata->rx_ded_count, "RX fifo");
 316	}
 317
 318	if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, DESC_DED)) {
 319		stop |= xgbe_ecc_ded(pdata, &pdata->desc_ded_period,
 320				     &pdata->desc_ded_count,
 321				     "descriptor cache");
 322	}
 323
 324	if (stop) {
 325		pdata->hw_if.disable_ecc_ded(pdata);
 326		schedule_work(&pdata->stopdev_work);
 327		goto out;
 328	}
 329
 330	if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, TX_SEC)) {
 331		if (xgbe_ecc_sec(pdata, &pdata->tx_sec_period,
 332				 &pdata->tx_sec_count, "TX fifo"))
 333			pdata->hw_if.disable_ecc_sec(pdata, XGBE_ECC_SEC_TX);
 334	}
 335
 336	if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, RX_SEC))
 337		if (xgbe_ecc_sec(pdata, &pdata->rx_sec_period,
 338				 &pdata->rx_sec_count, "RX fifo"))
 339			pdata->hw_if.disable_ecc_sec(pdata, XGBE_ECC_SEC_RX);
 340
 341	if (XP_GET_BITS(ecc_isr, XP_ECC_ISR, DESC_SEC))
 342		if (xgbe_ecc_sec(pdata, &pdata->desc_sec_period,
 343				 &pdata->desc_sec_count, "descriptor cache"))
 344			pdata->hw_if.disable_ecc_sec(pdata, XGBE_ECC_SEC_DESC);
 345
 346out:
 347	/* Clear all ECC interrupts */
 348	XP_IOWRITE(pdata, XP_ECC_ISR, ecc_isr);
 349
 350	/* Reissue interrupt if status is not clear */
 351	if (pdata->vdata->irq_reissue_support)
 352		XP_IOWRITE(pdata, XP_INT_REISSUE_EN, 1 << 1);
 353}
 354
 355static irqreturn_t xgbe_ecc_isr(int irq, void *data)
 356{
 357	struct xgbe_prv_data *pdata = data;
 358
 359	if (pdata->isr_as_bh_work)
 360		queue_work(system_bh_wq, &pdata->ecc_bh_work);
 361	else
 362		xgbe_ecc_isr_bh_work(&pdata->ecc_bh_work);
 363
 364	return IRQ_HANDLED;
 365}
 366
 367static void xgbe_isr_bh_work(struct work_struct *work)
 368{
 369	struct xgbe_prv_data *pdata = from_work(pdata, work, dev_bh_work);
 370	unsigned int mac_isr, mac_tssr, mac_mdioisr;
 371	struct xgbe_hw_if *hw_if = &pdata->hw_if;
 372	bool per_ch_irq, ti, ri, rbu, fbe;
 373	unsigned int dma_isr, dma_ch_isr;
 374	struct xgbe_channel *channel;
 375	unsigned int i;
 376
 377	/* The DMA interrupt status register also reports MAC and MTL
 378	 * interrupts. So for polling mode, we just need to check for
 379	 * this register to be non-zero
 380	 */
 381	dma_isr = XGMAC_IOREAD(pdata, DMA_ISR);
 382	if (!dma_isr)
 383		goto isr_done;
 384
 385	netif_dbg(pdata, intr, pdata->netdev, "DMA_ISR=%#010x\n", dma_isr);
 386
 387	for (i = 0; i < pdata->channel_count; i++) {
 388		bool schedule_napi = false;
 389		struct napi_struct *napi;
 390
 391		if (!(dma_isr & (1 << i)))
 392			continue;
 393
 394		channel = pdata->channel[i];
 395
 396		dma_ch_isr = XGMAC_DMA_IOREAD(channel, DMA_CH_SR);
 397
 398		/* Precompute flags once */
 399		ti  = !!XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, TI);
 400		ri  = !!XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, RI);
 401		rbu = !!XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, RBU);
 402		fbe = !!XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, FBE);
 403
 404		netif_dbg(pdata, intr, pdata->netdev, "DMA_CH%u_ISR=%#010x\n",
 405			  i, dma_ch_isr);
 406
 407		per_ch_irq = pdata->per_channel_irq;
 408
 409		/*
 410		 * Decide which NAPI to use and whether to schedule:
 411		 * - When not using per-channel IRQs: schedule on global NAPI
 412		 *   if TI or RI are set.
 413		 * - RBU should also trigger NAPI (either per-channel or global)
 414		 *   to allow refill.
 415		 */
 416		if (!per_ch_irq && (ti || ri))
 417			schedule_napi = true;
 418
 419		if (rbu) {
 420			schedule_napi = true;
 421			pdata->ext_stats.rx_buffer_unavailable++;
 422		}
 423
 424		napi = per_ch_irq ? &channel->napi : &pdata->napi;
 425
 426		if (schedule_napi && napi_schedule_prep(napi)) {
 427			/* Disable interrupts appropriately before polling */
 428			if (per_ch_irq) {
 429				if (pdata->channel_irq_mode)
 430					xgbe_disable_rx_tx_int(pdata, channel);
 431				else
 432					disable_irq_nosync(channel->dma_irq);
 433			} else {
 434				xgbe_disable_rx_tx_ints(pdata);
 435			}
 436
 437			/* Turn on polling */
 438			__napi_schedule(napi);
 439		} else {
 440			/*
 441			 * Don't clear Rx/Tx status if doing per-channel DMA
 442			 * interrupts; those bits will be serviced/cleared by
 443			 * the per-channel ISR/NAPI. In non-per-channel mode
 444			 * when we're not scheduling NAPI here, ensure we don't
 445			 * accidentally clear TI/RI in HW: zero them in the
 446			 * local copy so that the eventual write-back does not
 447			 * clear TI/RI.
 448			 */
 449			XGMAC_SET_BITS(dma_ch_isr, DMA_CH_SR, TI, 0);
 450			XGMAC_SET_BITS(dma_ch_isr, DMA_CH_SR, RI, 0);
 451		}
 452
 453		/* Restart the device on a Fatal Bus Error */
 454		if (fbe)
 455			schedule_work(&pdata->restart_work);
 456
 457		/* Clear interrupt signals */
 458		XGMAC_DMA_IOWRITE(channel, DMA_CH_SR, dma_ch_isr);
 459	}
 460
 461	if (XGMAC_GET_BITS(dma_isr, DMA_ISR, MACIS)) {
 462		mac_isr = XGMAC_IOREAD(pdata, MAC_ISR);
 463
 464		netif_dbg(pdata, intr, pdata->netdev, "MAC_ISR=%#010x\n",
 465			  mac_isr);
 466
 467		if (XGMAC_GET_BITS(mac_isr, MAC_ISR, MMCTXIS))
 468			hw_if->tx_mmc_int(pdata);
 469
 470		if (XGMAC_GET_BITS(mac_isr, MAC_ISR, MMCRXIS))
 471			hw_if->rx_mmc_int(pdata);
 472
 473		if (XGMAC_GET_BITS(mac_isr, MAC_ISR, TSIS)) {
 474			mac_tssr = XGMAC_IOREAD(pdata, MAC_TSSR);
 475
 476			netif_dbg(pdata, intr, pdata->netdev,
 477				  "MAC_TSSR=%#010x\n", mac_tssr);
 478
 479			if (XGMAC_GET_BITS(mac_tssr, MAC_TSSR, TXTSC)) {
 480				/* Read Tx Timestamp to clear interrupt */
 481				pdata->tx_tstamp =
 482					xgbe_get_tx_tstamp(pdata);
 483				queue_work(pdata->dev_workqueue,
 484					   &pdata->tx_tstamp_work);
 485			}
 486		}
 487
 488		if (XGMAC_GET_BITS(mac_isr, MAC_ISR, SMI)) {
 489			mac_mdioisr = XGMAC_IOREAD(pdata, MAC_MDIOISR);
 490
 491			netif_dbg(pdata, intr, pdata->netdev,
 492				  "MAC_MDIOISR=%#010x\n", mac_mdioisr);
 493
 494			if (XGMAC_GET_BITS(mac_mdioisr, MAC_MDIOISR,
 495					   SNGLCOMPINT))
 496				complete(&pdata->mdio_complete);
 497		}
 498	}
 499
 500isr_done:
 501	/* If there is not a separate AN irq, handle it here */
 502	if (pdata->dev_irq == pdata->an_irq)
 503		pdata->phy_if.an_isr(pdata);
 504
 505	/* If there is not a separate ECC irq, handle it here */
 506	if (pdata->vdata->ecc_support && (pdata->dev_irq == pdata->ecc_irq))
 507		xgbe_ecc_isr_bh_work(&pdata->ecc_bh_work);
 508
 509	/* If there is not a separate I2C irq, handle it here */
 510	if (pdata->vdata->i2c_support && (pdata->dev_irq == pdata->i2c_irq))
 511		pdata->i2c_if.i2c_isr(pdata);
 512
 513	/* Reissue interrupt if status is not clear */
 514	if (pdata->vdata->irq_reissue_support) {
 515		unsigned int reissue_mask;
 516
 517		reissue_mask = 1 << 0;
 518		if (!pdata->per_channel_irq)
 519			reissue_mask |= 0xffff << 4;
 520
 521		XP_IOWRITE(pdata, XP_INT_REISSUE_EN, reissue_mask);
 522	}
 523}
 524
 525static irqreturn_t xgbe_isr(int irq, void *data)
 526{
 527	struct xgbe_prv_data *pdata = data;
 528
 529	if (pdata->isr_as_bh_work)
 530		queue_work(system_bh_wq, &pdata->dev_bh_work);
 531	else
 532		xgbe_isr_bh_work(&pdata->dev_bh_work);
 533
 534	return IRQ_HANDLED;
 535}
 536
 537static irqreturn_t xgbe_dma_isr(int irq, void *data)
 538{
 539	struct xgbe_channel *channel = data;
 540	struct xgbe_prv_data *pdata = channel->pdata;
 541	unsigned int dma_status;
 542
 543	/* Per channel DMA interrupts are enabled, so we use the per
 544	 * channel napi structure and not the private data napi structure
 545	 */
 546	if (napi_schedule_prep(&channel->napi)) {
 547		/* Disable Tx and Rx interrupts */
 548		if (pdata->channel_irq_mode)
 549			xgbe_disable_rx_tx_int(pdata, channel);
 550		else
 551			disable_irq_nosync(channel->dma_irq);
 552
 553		/* Turn on polling */
 554		__napi_schedule_irqoff(&channel->napi);
 555	}
 556
 557	/* Clear Tx/Rx signals */
 558	dma_status = 0;
 559	XGMAC_SET_BITS(dma_status, DMA_CH_SR, TI, 1);
 560	XGMAC_SET_BITS(dma_status, DMA_CH_SR, RI, 1);
 561	XGMAC_DMA_IOWRITE(channel, DMA_CH_SR, dma_status);
 562
 563	return IRQ_HANDLED;
 564}
 565
 566static void xgbe_tx_timer(struct timer_list *t)
 567{
 568	struct xgbe_channel *channel = timer_container_of(channel, t,
 569							  tx_timer);
 570	struct xgbe_prv_data *pdata = channel->pdata;
 571	struct napi_struct *napi;
 572
 573	DBGPR("-->xgbe_tx_timer\n");
 574
 575	napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
 576
 577	if (napi_schedule_prep(napi)) {
 578		/* Disable Tx and Rx interrupts */
 579		if (pdata->per_channel_irq)
 580			if (pdata->channel_irq_mode)
 581				xgbe_disable_rx_tx_int(pdata, channel);
 582			else
 583				disable_irq_nosync(channel->dma_irq);
 584		else
 585			xgbe_disable_rx_tx_ints(pdata);
 586
 587		/* Turn on polling */
 588		__napi_schedule(napi);
 589	}
 590
 591	channel->tx_timer_active = 0;
 592
 593	DBGPR("<--xgbe_tx_timer\n");
 594}
 595
 596static void xgbe_service(struct work_struct *work)
 597{
 598	struct xgbe_prv_data *pdata = container_of(work,
 599						   struct xgbe_prv_data,
 600						   service_work);
 601
 602	pdata->phy_if.phy_status(pdata);
 603}
 604
 605static void xgbe_service_timer(struct timer_list *t)
 606{
 607	struct xgbe_prv_data *pdata = timer_container_of(pdata, t,
 608							 service_timer);
 609	struct xgbe_channel *channel;
 610	unsigned int i;
 611
 612	queue_work(pdata->dev_workqueue, &pdata->service_work);
 613
 614	mod_timer(&pdata->service_timer, jiffies + HZ);
 615
 616	if (!pdata->tx_usecs)
 617		return;
 618
 619	for (i = 0; i < pdata->channel_count; i++) {
 620		channel = pdata->channel[i];
 621		if (!channel->tx_ring || channel->tx_timer_active)
 622			break;
 623		channel->tx_timer_active = 1;
 624		mod_timer(&channel->tx_timer,
 625			  jiffies + usecs_to_jiffies(pdata->tx_usecs));
 626	}
 627}
 628
 629static void xgbe_init_timers(struct xgbe_prv_data *pdata)
 630{
 631	struct xgbe_channel *channel;
 632	unsigned int i;
 633
 634	timer_setup(&pdata->service_timer, xgbe_service_timer, 0);
 635
 636	for (i = 0; i < pdata->channel_count; i++) {
 637		channel = pdata->channel[i];
 638		if (!channel->tx_ring)
 639			break;
 640
 641		timer_setup(&channel->tx_timer, xgbe_tx_timer, 0);
 642	}
 643}
 644
 645static void xgbe_start_timers(struct xgbe_prv_data *pdata)
 646{
 647	mod_timer(&pdata->service_timer, jiffies + HZ);
 648}
 649
 650static void xgbe_stop_timers(struct xgbe_prv_data *pdata)
 651{
 652	struct xgbe_channel *channel;
 653	unsigned int i;
 654
 655	timer_delete_sync(&pdata->service_timer);
 656
 657	for (i = 0; i < pdata->channel_count; i++) {
 658		channel = pdata->channel[i];
 659		if (!channel->tx_ring)
 660			break;
 661
 662		/* Deactivate the Tx timer */
 663		timer_delete_sync(&channel->tx_timer);
 664		channel->tx_timer_active = 0;
 665	}
 666}
 667
 668void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata)
 669{
 670	unsigned int mac_hfr0, mac_hfr1, mac_hfr2;
 671	struct xgbe_hw_features *hw_feat = &pdata->hw_feat;
 672
 673	mac_hfr0 = XGMAC_IOREAD(pdata, MAC_HWF0R);
 674	mac_hfr1 = XGMAC_IOREAD(pdata, MAC_HWF1R);
 675	mac_hfr2 = XGMAC_IOREAD(pdata, MAC_HWF2R);
 676
 677	memset(hw_feat, 0, sizeof(*hw_feat));
 678
 679	hw_feat->version = XGMAC_IOREAD(pdata, MAC_VR);
 680
 681	/* Hardware feature register 0 */
 682	hw_feat->gmii        = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, GMIISEL);
 683	hw_feat->vlhash      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, VLHASH);
 684	hw_feat->sma         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, SMASEL);
 685	hw_feat->rwk         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, RWKSEL);
 686	hw_feat->mgk         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, MGKSEL);
 687	hw_feat->mmc         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, MMCSEL);
 688	hw_feat->aoe         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, ARPOFFSEL);
 689	hw_feat->ts          = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TSSEL);
 690	hw_feat->eee         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, EEESEL);
 691	hw_feat->tx_coe      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TXCOESEL);
 692	hw_feat->rx_coe      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, RXCOESEL);
 693	hw_feat->addn_mac    = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R,
 694					      ADDMACADRSEL);
 695	hw_feat->ts_src      = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, TSSTSSEL);
 696	hw_feat->sa_vlan_ins = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, SAVLANINS);
 697	hw_feat->vxn         = XGMAC_GET_BITS(mac_hfr0, MAC_HWF0R, VXN);
 698
 699	/* Hardware feature register 1 */
 700	hw_feat->rx_fifo_size  = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
 701						RXFIFOSIZE);
 702	hw_feat->tx_fifo_size  = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
 703						TXFIFOSIZE);
 704	hw_feat->adv_ts_hi     = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, ADVTHWORD);
 705	hw_feat->dma_width     = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, ADDR64);
 706	hw_feat->dcb           = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, DCBEN);
 707	hw_feat->sph           = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, SPHEN);
 708	hw_feat->tso           = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, TSOEN);
 709	hw_feat->dma_debug     = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, DBGMEMA);
 710	hw_feat->rss           = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, RSSEN);
 711	hw_feat->tc_cnt	       = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R, NUMTC);
 712	hw_feat->hash_table_size = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
 713						  HASHTBLSZ);
 714	hw_feat->l3l4_filter_num = XGMAC_GET_BITS(mac_hfr1, MAC_HWF1R,
 715						  L3L4FNUM);
 716
 717	/* Hardware feature register 2 */
 718	hw_feat->rx_q_cnt     = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, RXQCNT);
 719	hw_feat->tx_q_cnt     = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, TXQCNT);
 720	hw_feat->rx_ch_cnt    = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, RXCHCNT);
 721	hw_feat->tx_ch_cnt    = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, TXCHCNT);
 722	hw_feat->pps_out_num  = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, PPSOUTNUM);
 723	hw_feat->aux_snap_num = XGMAC_GET_BITS(mac_hfr2, MAC_HWF2R, AUXSNAPNUM);
 724
 725	/* Sanity check and warn if hardware reports more than supported */
 726	if (hw_feat->pps_out_num > XGBE_MAX_PPS_OUT) {
 727		dev_warn(pdata->dev,
 728			 "Hardware reports %u PPS outputs, limiting to %u\n",
 729			 hw_feat->pps_out_num, XGBE_MAX_PPS_OUT);
 730		hw_feat->pps_out_num = XGBE_MAX_PPS_OUT;
 731	}
 732
 733	if (hw_feat->aux_snap_num > XGBE_MAX_AUX_SNAP) {
 734		dev_warn(pdata->dev,
 735			 "Hardware reports %u aux snapshot inputs, limiting to %u\n",
 736			 hw_feat->aux_snap_num, XGBE_MAX_AUX_SNAP);
 737		hw_feat->aux_snap_num = XGBE_MAX_AUX_SNAP;
 738	}
 739
 740	/* Translate the Hash Table size into actual number */
 741	switch (hw_feat->hash_table_size) {
 742	case 0:
 743		break;
 744	case 1:
 745		hw_feat->hash_table_size = 64;
 746		break;
 747	case 2:
 748		hw_feat->hash_table_size = 128;
 749		break;
 750	case 3:
 751		hw_feat->hash_table_size = 256;
 752		break;
 753	}
 754
 755	/* Translate the address width setting into actual number */
 756	switch (hw_feat->dma_width) {
 757	case 0:
 758		hw_feat->dma_width = 32;
 759		break;
 760	case 1:
 761		hw_feat->dma_width = 40;
 762		break;
 763	case 2:
 764		hw_feat->dma_width = 48;
 765		break;
 766	default:
 767		hw_feat->dma_width = 32;
 768	}
 769
 770	/* The Queue, Channel and TC counts are zero based so increment them
 771	 * to get the actual number
 772	 */
 773	hw_feat->rx_q_cnt++;
 774	hw_feat->tx_q_cnt++;
 775	hw_feat->rx_ch_cnt++;
 776	hw_feat->tx_ch_cnt++;
 777	hw_feat->tc_cnt++;
 778
 779	/* Translate the fifo sizes into actual numbers */
 780	hw_feat->rx_fifo_size = 1 << (hw_feat->rx_fifo_size + 7);
 781	hw_feat->tx_fifo_size = 1 << (hw_feat->tx_fifo_size + 7);
 782
 783	if (netif_msg_probe(pdata)) {
 784		dev_dbg(pdata->dev, "Hardware features:\n");
 785
 786		/* Hardware feature register 0 */
 787		dev_dbg(pdata->dev, "  1GbE support              : %s\n",
 788			hw_feat->gmii ? "yes" : "no");
 789		dev_dbg(pdata->dev, "  VLAN hash filter          : %s\n",
 790			hw_feat->vlhash ? "yes" : "no");
 791		dev_dbg(pdata->dev, "  MDIO interface            : %s\n",
 792			hw_feat->sma ? "yes" : "no");
 793		dev_dbg(pdata->dev, "  Wake-up packet support    : %s\n",
 794			hw_feat->rwk ? "yes" : "no");
 795		dev_dbg(pdata->dev, "  Magic packet support      : %s\n",
 796			hw_feat->mgk ? "yes" : "no");
 797		dev_dbg(pdata->dev, "  Management counters       : %s\n",
 798			hw_feat->mmc ? "yes" : "no");
 799		dev_dbg(pdata->dev, "  ARP offload               : %s\n",
 800			hw_feat->aoe ? "yes" : "no");
 801		dev_dbg(pdata->dev, "  IEEE 1588-2008 Timestamp  : %s\n",
 802			hw_feat->ts ? "yes" : "no");
 803		dev_dbg(pdata->dev, "  Energy Efficient Ethernet : %s\n",
 804			hw_feat->eee ? "yes" : "no");
 805		dev_dbg(pdata->dev, "  TX checksum offload       : %s\n",
 806			hw_feat->tx_coe ? "yes" : "no");
 807		dev_dbg(pdata->dev, "  RX checksum offload       : %s\n",
 808			hw_feat->rx_coe ? "yes" : "no");
 809		dev_dbg(pdata->dev, "  Additional MAC addresses  : %u\n",
 810			hw_feat->addn_mac);
 811		dev_dbg(pdata->dev, "  Timestamp source          : %s\n",
 812			(hw_feat->ts_src == 1) ? "internal" :
 813			(hw_feat->ts_src == 2) ? "external" :
 814			(hw_feat->ts_src == 3) ? "internal/external" : "n/a");
 815		dev_dbg(pdata->dev, "  SA/VLAN insertion         : %s\n",
 816			hw_feat->sa_vlan_ins ? "yes" : "no");
 817		dev_dbg(pdata->dev, "  VXLAN/NVGRE support       : %s\n",
 818			hw_feat->vxn ? "yes" : "no");
 819
 820		/* Hardware feature register 1 */
 821		dev_dbg(pdata->dev, "  RX fifo size              : %u\n",
 822			hw_feat->rx_fifo_size);
 823		dev_dbg(pdata->dev, "  TX fifo size              : %u\n",
 824			hw_feat->tx_fifo_size);
 825		dev_dbg(pdata->dev, "  IEEE 1588 high word       : %s\n",
 826			hw_feat->adv_ts_hi ? "yes" : "no");
 827		dev_dbg(pdata->dev, "  DMA width                 : %u\n",
 828			hw_feat->dma_width);
 829		dev_dbg(pdata->dev, "  Data Center Bridging      : %s\n",
 830			hw_feat->dcb ? "yes" : "no");
 831		dev_dbg(pdata->dev, "  Split header              : %s\n",
 832			hw_feat->sph ? "yes" : "no");
 833		dev_dbg(pdata->dev, "  TCP Segmentation Offload  : %s\n",
 834			hw_feat->tso ? "yes" : "no");
 835		dev_dbg(pdata->dev, "  Debug memory interface    : %s\n",
 836			hw_feat->dma_debug ? "yes" : "no");
 837		dev_dbg(pdata->dev, "  Receive Side Scaling      : %s\n",
 838			hw_feat->rss ? "yes" : "no");
 839		dev_dbg(pdata->dev, "  Traffic Class count       : %u\n",
 840			hw_feat->tc_cnt);
 841		dev_dbg(pdata->dev, "  Hash table size           : %u\n",
 842			hw_feat->hash_table_size);
 843		dev_dbg(pdata->dev, "  L3/L4 Filters             : %u\n",
 844			hw_feat->l3l4_filter_num);
 845
 846		/* Hardware feature register 2 */
 847		dev_dbg(pdata->dev, "  RX queue count            : %u\n",
 848			hw_feat->rx_q_cnt);
 849		dev_dbg(pdata->dev, "  TX queue count            : %u\n",
 850			hw_feat->tx_q_cnt);
 851		dev_dbg(pdata->dev, "  RX DMA channel count      : %u\n",
 852			hw_feat->rx_ch_cnt);
 853		dev_dbg(pdata->dev, "  TX DMA channel count      : %u\n",
 854			hw_feat->rx_ch_cnt);
 855		dev_dbg(pdata->dev, "  PPS outputs               : %u\n",
 856			hw_feat->pps_out_num);
 857		dev_dbg(pdata->dev, "  Auxiliary snapshot inputs : %u\n",
 858			hw_feat->aux_snap_num);
 859	}
 860}
 861
 862static int xgbe_vxlan_set_port(struct net_device *netdev, unsigned int table,
 863			       unsigned int entry, struct udp_tunnel_info *ti)
 864{
 865	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 866
 867	pdata->vxlan_port = be16_to_cpu(ti->port);
 868	pdata->hw_if.enable_vxlan(pdata);
 869
 870	return 0;
 871}
 872
 873static int xgbe_vxlan_unset_port(struct net_device *netdev, unsigned int table,
 874				 unsigned int entry, struct udp_tunnel_info *ti)
 875{
 876	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 877
 878	pdata->hw_if.disable_vxlan(pdata);
 879	pdata->vxlan_port = 0;
 880
 881	return 0;
 882}
 883
 884static const struct udp_tunnel_nic_info xgbe_udp_tunnels = {
 885	.set_port	= xgbe_vxlan_set_port,
 886	.unset_port	= xgbe_vxlan_unset_port,
 887	.flags		= UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
 888	.tables		= {
 889		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
 890	},
 891};
 892
 893const struct udp_tunnel_nic_info *xgbe_get_udp_tunnel_info(void)
 894{
 895	return &xgbe_udp_tunnels;
 896}
 897
 898static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add)
 899{
 900	struct xgbe_channel *channel;
 901	unsigned int i;
 902
 903	if (pdata->per_channel_irq) {
 904		for (i = 0; i < pdata->channel_count; i++) {
 905			channel = pdata->channel[i];
 906			if (add)
 907				netif_napi_add(pdata->netdev, &channel->napi,
 908					       xgbe_one_poll);
 909
 910			napi_enable(&channel->napi);
 911		}
 912	} else {
 913		if (add)
 914			netif_napi_add(pdata->netdev, &pdata->napi,
 915				       xgbe_all_poll);
 916
 917		napi_enable(&pdata->napi);
 918	}
 919}
 920
 921static void xgbe_napi_disable(struct xgbe_prv_data *pdata, unsigned int del)
 922{
 923	struct xgbe_channel *channel;
 924	unsigned int i;
 925
 926	if (pdata->per_channel_irq) {
 927		for (i = 0; i < pdata->channel_count; i++) {
 928			channel = pdata->channel[i];
 929			napi_disable(&channel->napi);
 930
 931			if (del)
 932				netif_napi_del(&channel->napi);
 933		}
 934	} else {
 935		napi_disable(&pdata->napi);
 936
 937		if (del)
 938			netif_napi_del(&pdata->napi);
 939	}
 940}
 941
 942static int xgbe_request_irqs(struct xgbe_prv_data *pdata)
 943{
 944	struct xgbe_channel *channel;
 945	struct net_device *netdev = pdata->netdev;
 946	unsigned int i;
 947	int ret;
 948
 949	INIT_WORK(&pdata->dev_bh_work, xgbe_isr_bh_work);
 950	INIT_WORK(&pdata->ecc_bh_work, xgbe_ecc_isr_bh_work);
 951
 952	ret = devm_request_irq(pdata->dev, pdata->dev_irq, xgbe_isr, 0,
 953			       netdev_name(netdev), pdata);
 954	if (ret) {
 955		netdev_alert(netdev, "error requesting irq %d\n",
 956			     pdata->dev_irq);
 957		return ret;
 958	}
 959
 960	if (pdata->vdata->ecc_support && (pdata->dev_irq != pdata->ecc_irq)) {
 961		ret = devm_request_irq(pdata->dev, pdata->ecc_irq, xgbe_ecc_isr,
 962				       0, pdata->ecc_name, pdata);
 963		if (ret) {
 964			netdev_alert(netdev, "error requesting ecc irq %d\n",
 965				     pdata->ecc_irq);
 966			goto err_dev_irq;
 967		}
 968	}
 969
 970	if (!pdata->per_channel_irq)
 971		return 0;
 972
 973	for (i = 0; i < pdata->channel_count; i++) {
 974		channel = pdata->channel[i];
 975		snprintf(channel->dma_irq_name,
 976			 sizeof(channel->dma_irq_name) - 1,
 977			 "%s-TxRx-%u", netdev_name(netdev),
 978			 channel->queue_index);
 979
 980		ret = devm_request_irq(pdata->dev, channel->dma_irq,
 981				       xgbe_dma_isr, 0,
 982				       channel->dma_irq_name, channel);
 983		if (ret) {
 984			netdev_alert(netdev, "error requesting irq %d\n",
 985				     channel->dma_irq);
 986			goto err_dma_irq;
 987		}
 988
 989		irq_set_affinity_hint(channel->dma_irq,
 990				      &channel->affinity_mask);
 991	}
 992
 993	return 0;
 994
 995err_dma_irq:
 996	/* Using an unsigned int, 'i' will go to UINT_MAX and exit */
 997	for (i--; i < pdata->channel_count; i--) {
 998		channel = pdata->channel[i];
 999
1000		irq_set_affinity_hint(channel->dma_irq, NULL);
1001		devm_free_irq(pdata->dev, channel->dma_irq, channel);
1002	}
1003
1004	if (pdata->vdata->ecc_support && (pdata->dev_irq != pdata->ecc_irq))
1005		devm_free_irq(pdata->dev, pdata->ecc_irq, pdata);
1006
1007err_dev_irq:
1008	devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
1009
1010	return ret;
1011}
1012
1013static void xgbe_free_irqs(struct xgbe_prv_data *pdata)
1014{
1015	struct xgbe_channel *channel;
1016	unsigned int i;
1017
1018	devm_free_irq(pdata->dev, pdata->dev_irq, pdata);
1019
1020	cancel_work_sync(&pdata->dev_bh_work);
1021	cancel_work_sync(&pdata->ecc_bh_work);
1022
1023	if (pdata->vdata->ecc_support && (pdata->dev_irq != pdata->ecc_irq))
1024		devm_free_irq(pdata->dev, pdata->ecc_irq, pdata);
1025
1026	if (!pdata->per_channel_irq)
1027		return;
1028
1029	for (i = 0; i < pdata->channel_count; i++) {
1030		channel = pdata->channel[i];
1031
1032		irq_set_affinity_hint(channel->dma_irq, NULL);
1033		devm_free_irq(pdata->dev, channel->dma_irq, channel);
1034	}
1035}
1036
1037void xgbe_init_tx_coalesce(struct xgbe_prv_data *pdata)
1038{
1039	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1040
1041	DBGPR("-->xgbe_init_tx_coalesce\n");
1042
1043	pdata->tx_usecs = XGMAC_INIT_DMA_TX_USECS;
1044	pdata->tx_frames = XGMAC_INIT_DMA_TX_FRAMES;
1045
1046	hw_if->config_tx_coalesce(pdata);
1047
1048	DBGPR("<--xgbe_init_tx_coalesce\n");
1049}
1050
1051void xgbe_init_rx_coalesce(struct xgbe_prv_data *pdata)
1052{
1053	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1054
1055	DBGPR("-->xgbe_init_rx_coalesce\n");
1056
1057	pdata->rx_riwt = hw_if->usec_to_riwt(pdata, XGMAC_INIT_DMA_RX_USECS);
1058	pdata->rx_usecs = XGMAC_INIT_DMA_RX_USECS;
1059	pdata->rx_frames = XGMAC_INIT_DMA_RX_FRAMES;
1060
1061	hw_if->config_rx_coalesce(pdata);
1062
1063	DBGPR("<--xgbe_init_rx_coalesce\n");
1064}
1065
1066static void xgbe_free_tx_data(struct xgbe_prv_data *pdata)
1067{
1068	struct xgbe_desc_if *desc_if = &pdata->desc_if;
1069	struct xgbe_ring *ring;
1070	struct xgbe_ring_data *rdata;
1071	unsigned int i, j;
1072
1073	DBGPR("-->xgbe_free_tx_data\n");
1074
1075	for (i = 0; i < pdata->channel_count; i++) {
1076		ring = pdata->channel[i]->tx_ring;
1077		if (!ring)
1078			break;
1079
1080		for (j = 0; j < ring->rdesc_count; j++) {
1081			rdata = XGBE_GET_DESC_DATA(ring, j);
1082			desc_if->unmap_rdata(pdata, rdata);
1083		}
1084	}
1085
1086	DBGPR("<--xgbe_free_tx_data\n");
1087}
1088
1089static void xgbe_free_rx_data(struct xgbe_prv_data *pdata)
1090{
1091	struct xgbe_desc_if *desc_if = &pdata->desc_if;
1092	struct xgbe_ring *ring;
1093	struct xgbe_ring_data *rdata;
1094	unsigned int i, j;
1095
1096	DBGPR("-->xgbe_free_rx_data\n");
1097
1098	for (i = 0; i < pdata->channel_count; i++) {
1099		ring = pdata->channel[i]->rx_ring;
1100		if (!ring)
1101			break;
1102
1103		for (j = 0; j < ring->rdesc_count; j++) {
1104			rdata = XGBE_GET_DESC_DATA(ring, j);
1105			desc_if->unmap_rdata(pdata, rdata);
1106		}
1107	}
1108
1109	DBGPR("<--xgbe_free_rx_data\n");
1110}
1111
1112static int xgbe_phy_reset(struct xgbe_prv_data *pdata)
1113{
1114	pdata->phy_speed = SPEED_UNKNOWN;
1115
1116	return pdata->phy_if.phy_reset(pdata);
1117}
1118
1119int xgbe_powerdown(struct net_device *netdev, unsigned int caller)
1120{
1121	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1122	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1123
1124	DBGPR("-->xgbe_powerdown\n");
1125
1126	if (!netif_running(netdev) ||
1127	    (caller == XGMAC_IOCTL_CONTEXT && pdata->power_down)) {
1128		netdev_alert(netdev, "Device is already powered down\n");
1129		DBGPR("<--xgbe_powerdown\n");
1130		return -EINVAL;
1131	}
1132
1133	if (caller == XGMAC_DRIVER_CONTEXT)
1134		netif_device_detach(netdev);
1135
1136	netif_tx_stop_all_queues(netdev);
1137
1138	xgbe_stop_timers(pdata);
1139	flush_workqueue(pdata->dev_workqueue);
1140
1141	hw_if->powerdown_tx(pdata);
1142	hw_if->powerdown_rx(pdata);
1143
1144	xgbe_napi_disable(pdata, 0);
1145
1146	pdata->power_down = 1;
1147
1148	DBGPR("<--xgbe_powerdown\n");
1149
1150	return 0;
1151}
1152
1153int xgbe_powerup(struct net_device *netdev, unsigned int caller)
1154{
1155	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1156	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1157
1158	DBGPR("-->xgbe_powerup\n");
1159
1160	if (!netif_running(netdev) ||
1161	    (caller == XGMAC_IOCTL_CONTEXT && !pdata->power_down)) {
1162		netdev_alert(netdev, "Device is already powered up\n");
1163		DBGPR("<--xgbe_powerup\n");
1164		return -EINVAL;
1165	}
1166
1167	pdata->power_down = 0;
1168
1169	xgbe_napi_enable(pdata, 0);
1170
1171	hw_if->powerup_tx(pdata);
1172	hw_if->powerup_rx(pdata);
1173
1174	if (caller == XGMAC_DRIVER_CONTEXT)
1175		netif_device_attach(netdev);
1176
1177	netif_tx_start_all_queues(netdev);
1178
1179	xgbe_start_timers(pdata);
1180
1181	DBGPR("<--xgbe_powerup\n");
1182
1183	return 0;
1184}
1185
1186static void xgbe_free_memory(struct xgbe_prv_data *pdata)
1187{
1188	struct xgbe_desc_if *desc_if = &pdata->desc_if;
1189
1190	/* Free the ring descriptors and buffers */
1191	desc_if->free_ring_resources(pdata);
1192
1193	/* Free the channel and ring structures */
1194	xgbe_free_channels(pdata);
1195}
1196
1197static int xgbe_alloc_memory(struct xgbe_prv_data *pdata)
1198{
1199	struct xgbe_desc_if *desc_if = &pdata->desc_if;
1200	struct net_device *netdev = pdata->netdev;
1201	int ret;
1202
1203	if (pdata->new_tx_ring_count) {
1204		pdata->tx_ring_count = pdata->new_tx_ring_count;
1205		pdata->tx_q_count = pdata->tx_ring_count;
1206		pdata->new_tx_ring_count = 0;
1207	}
1208
1209	if (pdata->new_rx_ring_count) {
1210		pdata->rx_ring_count = pdata->new_rx_ring_count;
1211		pdata->new_rx_ring_count = 0;
1212	}
1213
1214	/* Calculate the Rx buffer size before allocating rings */
1215	pdata->rx_buf_size = xgbe_calc_rx_buf_size(netdev, netdev->mtu);
1216
1217	/* Allocate the channel and ring structures */
1218	ret = xgbe_alloc_channels(pdata);
1219	if (ret)
1220		return ret;
1221
1222	/* Allocate the ring descriptors and buffers */
1223	ret = desc_if->alloc_ring_resources(pdata);
1224	if (ret)
1225		goto err_channels;
1226
1227	/* Initialize the service and Tx timers */
1228	xgbe_init_timers(pdata);
1229
1230	return 0;
1231
1232err_channels:
1233	xgbe_free_memory(pdata);
1234
1235	return ret;
1236}
1237
1238static int xgbe_start(struct xgbe_prv_data *pdata)
1239{
1240	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1241	struct xgbe_phy_if *phy_if = &pdata->phy_if;
1242	struct net_device *netdev = pdata->netdev;
1243	unsigned int i;
1244	int ret;
1245
1246	/* Set the number of queues */
1247	ret = netif_set_real_num_tx_queues(netdev, pdata->tx_ring_count);
1248	if (ret) {
1249		netdev_err(netdev, "error setting real tx queue count\n");
1250		return ret;
1251	}
1252
1253	ret = netif_set_real_num_rx_queues(netdev, pdata->rx_ring_count);
1254	if (ret) {
1255		netdev_err(netdev, "error setting real rx queue count\n");
1256		return ret;
1257	}
1258
1259	/* Set RSS lookup table data for programming */
1260	for (i = 0; i < XGBE_RSS_MAX_TABLE_SIZE; i++)
1261		XGMAC_SET_BITS(pdata->rss_table[i], MAC_RSSDR, DMCH,
1262			       i % pdata->rx_ring_count);
1263
1264	ret = hw_if->init(pdata);
1265	if (ret)
1266		return ret;
1267
1268	xgbe_napi_enable(pdata, 1);
1269
1270	ret = xgbe_request_irqs(pdata);
1271	if (ret)
1272		goto err_napi;
1273
1274	/* Reset the phy settings */
1275	ret = xgbe_phy_reset(pdata);
1276	if (ret)
1277		goto err_irqs;
1278
1279	/* Start the phy */
1280	ret = phy_if->phy_start(pdata);
1281	if (ret)
1282		goto err_irqs;
1283
1284	hw_if->enable_tx(pdata);
1285	hw_if->enable_rx(pdata);
1286	/* Synchronize flag with hardware state after enabling TX/RX.
1287	 * This prevents stale state after device restart cycles.
1288	 */
1289	pdata->data_path_stopped = false;
1290
1291	udp_tunnel_nic_reset_ntf(netdev);
1292
1293	netif_tx_start_all_queues(netdev);
1294
1295	xgbe_start_timers(pdata);
1296	queue_work(pdata->dev_workqueue, &pdata->service_work);
1297
1298	clear_bit(XGBE_STOPPED, &pdata->dev_state);
1299
1300	return 0;
1301
1302err_irqs:
1303	xgbe_free_irqs(pdata);
1304
1305err_napi:
1306	xgbe_napi_disable(pdata, 1);
1307
1308	hw_if->exit(pdata);
1309
1310	return ret;
1311}
1312
1313static void xgbe_stop(struct xgbe_prv_data *pdata)
1314{
1315	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1316	struct xgbe_phy_if *phy_if = &pdata->phy_if;
1317	struct xgbe_channel *channel;
1318	struct net_device *netdev = pdata->netdev;
1319	struct netdev_queue *txq;
1320	unsigned int i;
1321
1322	DBGPR("-->xgbe_stop\n");
1323
1324	if (test_bit(XGBE_STOPPED, &pdata->dev_state))
1325		return;
1326
1327	netif_tx_stop_all_queues(netdev);
1328	netif_carrier_off(pdata->netdev);
1329
1330	xgbe_stop_timers(pdata);
1331	flush_workqueue(pdata->dev_workqueue);
1332
1333	xgbe_vxlan_unset_port(netdev, 0, 0, NULL);
1334
1335	hw_if->disable_tx(pdata);
1336	hw_if->disable_rx(pdata);
1337
1338	phy_if->phy_stop(pdata);
1339
1340	xgbe_free_irqs(pdata);
1341
1342	xgbe_napi_disable(pdata, 1);
1343
1344	hw_if->exit(pdata);
1345
1346	for (i = 0; i < pdata->channel_count; i++) {
1347		channel = pdata->channel[i];
1348		if (!channel->tx_ring)
1349			continue;
1350
1351		txq = netdev_get_tx_queue(netdev, channel->queue_index);
1352		netdev_tx_reset_queue(txq);
1353	}
1354
1355	set_bit(XGBE_STOPPED, &pdata->dev_state);
1356
1357	DBGPR("<--xgbe_stop\n");
1358}
1359
1360static void xgbe_stopdev(struct work_struct *work)
1361{
1362	struct xgbe_prv_data *pdata = container_of(work,
1363						   struct xgbe_prv_data,
1364						   stopdev_work);
1365
1366	rtnl_lock();
1367
1368	xgbe_stop(pdata);
1369
1370	xgbe_free_tx_data(pdata);
1371	xgbe_free_rx_data(pdata);
1372
1373	rtnl_unlock();
1374
1375	netdev_alert(pdata->netdev, "device stopped\n");
1376}
1377
1378void xgbe_full_restart_dev(struct xgbe_prv_data *pdata)
1379{
1380	/* If not running, "restart" will happen on open */
1381	if (!netif_running(pdata->netdev))
1382		return;
1383
1384	xgbe_stop(pdata);
1385
1386	xgbe_free_memory(pdata);
1387	xgbe_alloc_memory(pdata);
1388
1389	xgbe_start(pdata);
1390}
1391
1392void xgbe_restart_dev(struct xgbe_prv_data *pdata)
1393{
1394	/* If not running, "restart" will happen on open */
1395	if (!netif_running(pdata->netdev))
1396		return;
1397
1398	xgbe_stop(pdata);
1399
1400	xgbe_free_tx_data(pdata);
1401	xgbe_free_rx_data(pdata);
1402
1403	xgbe_start(pdata);
1404}
1405
1406static void xgbe_restart(struct work_struct *work)
1407{
1408	struct xgbe_prv_data *pdata = container_of(work,
1409						   struct xgbe_prv_data,
1410						   restart_work);
1411
1412	rtnl_lock();
1413
1414	xgbe_restart_dev(pdata);
1415
1416	rtnl_unlock();
1417}
1418
1419static void xgbe_prep_vlan(struct sk_buff *skb, struct xgbe_packet_data *packet)
1420{
1421	if (skb_vlan_tag_present(skb))
1422		packet->vlan_ctag = skb_vlan_tag_get(skb);
1423}
1424
1425static int xgbe_prep_tso(struct sk_buff *skb, struct xgbe_packet_data *packet)
1426{
1427	int ret;
1428
1429	if (!XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1430			    TSO_ENABLE))
1431		return 0;
1432
1433	ret = skb_cow_head(skb, 0);
1434	if (ret)
1435		return ret;
1436
1437	if (XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES, VXLAN)) {
1438		packet->header_len = skb_inner_tcp_all_headers(skb);
1439		packet->tcp_header_len = inner_tcp_hdrlen(skb);
1440	} else {
1441		packet->header_len = skb_tcp_all_headers(skb);
1442		packet->tcp_header_len = tcp_hdrlen(skb);
1443	}
1444	packet->tcp_payload_len = skb->len - packet->header_len;
1445	packet->mss = skb_shinfo(skb)->gso_size;
1446
1447	DBGPR("  packet->header_len=%u\n", packet->header_len);
1448	DBGPR("  packet->tcp_header_len=%u, packet->tcp_payload_len=%u\n",
1449	      packet->tcp_header_len, packet->tcp_payload_len);
1450	DBGPR("  packet->mss=%u\n", packet->mss);
1451
1452	/* Update the number of packets that will ultimately be transmitted
1453	 * along with the extra bytes for each extra packet
1454	 */
1455	packet->tx_packets = skb_shinfo(skb)->gso_segs;
1456	packet->tx_bytes += (packet->tx_packets - 1) * packet->header_len;
1457
1458	return 0;
1459}
1460
1461static bool xgbe_is_vxlan(struct sk_buff *skb)
1462{
1463	if (!skb->encapsulation)
1464		return false;
1465
1466	if (skb->ip_summed != CHECKSUM_PARTIAL)
1467		return false;
1468
1469	switch (skb->protocol) {
1470	case htons(ETH_P_IP):
1471		if (ip_hdr(skb)->protocol != IPPROTO_UDP)
1472			return false;
1473		break;
1474
1475	case htons(ETH_P_IPV6):
1476		if (ipv6_hdr(skb)->nexthdr != IPPROTO_UDP)
1477			return false;
1478		break;
1479
1480	default:
1481		return false;
1482	}
1483
1484	if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1485	    skb->inner_protocol != htons(ETH_P_TEB) ||
1486	    (skb_inner_mac_header(skb) - skb_transport_header(skb) !=
1487	     sizeof(struct udphdr) + sizeof(struct vxlanhdr)))
1488		return false;
1489
1490	return true;
1491}
1492
1493static int xgbe_is_tso(struct sk_buff *skb)
1494{
1495	if (skb->ip_summed != CHECKSUM_PARTIAL)
1496		return 0;
1497
1498	if (!skb_is_gso(skb))
1499		return 0;
1500
1501	DBGPR("  TSO packet to be processed\n");
1502
1503	return 1;
1504}
1505
1506static void xgbe_packet_info(struct xgbe_prv_data *pdata,
1507			     struct xgbe_ring *ring, struct sk_buff *skb,
1508			     struct xgbe_packet_data *packet)
1509{
1510	skb_frag_t *frag;
1511	unsigned int context_desc;
1512	unsigned int len;
1513	unsigned int i;
1514
1515	packet->skb = skb;
1516
1517	context_desc = 0;
1518	packet->rdesc_count = 0;
1519
1520	packet->tx_packets = 1;
1521	packet->tx_bytes = skb->len;
1522
1523	if (xgbe_is_tso(skb)) {
1524		/* TSO requires an extra descriptor if mss is different */
1525		if (skb_shinfo(skb)->gso_size != ring->tx.cur_mss) {
1526			context_desc = 1;
1527			packet->rdesc_count++;
1528		}
1529
1530		/* TSO requires an extra descriptor for TSO header */
1531		packet->rdesc_count++;
1532
1533		XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1534			       TSO_ENABLE, 1);
1535		XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1536			       CSUM_ENABLE, 1);
1537	} else if (skb->ip_summed == CHECKSUM_PARTIAL)
1538		XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1539			       CSUM_ENABLE, 1);
1540
1541	if (xgbe_is_vxlan(skb))
1542		XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1543			       VXLAN, 1);
1544
1545	if (skb_vlan_tag_present(skb)) {
1546		/* VLAN requires an extra descriptor if tag is different */
1547		if (skb_vlan_tag_get(skb) != ring->tx.cur_vlan_ctag)
1548			/* We can share with the TSO context descriptor */
1549			if (!context_desc) {
1550				context_desc = 1;
1551				packet->rdesc_count++;
1552			}
1553
1554		XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1555			       VLAN_CTAG, 1);
1556	}
1557
1558	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1559	    (pdata->tstamp_config.tx_type == HWTSTAMP_TX_ON))
1560		XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES,
1561			       PTP, 1);
1562
1563	for (len = skb_headlen(skb); len;) {
1564		packet->rdesc_count++;
1565		len -= min_t(unsigned int, len, XGBE_TX_MAX_BUF_SIZE);
1566	}
1567
1568	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1569		frag = &skb_shinfo(skb)->frags[i];
1570		for (len = skb_frag_size(frag); len; ) {
1571			packet->rdesc_count++;
1572			len -= min_t(unsigned int, len, XGBE_TX_MAX_BUF_SIZE);
1573		}
1574	}
1575}
1576
1577static int xgbe_open(struct net_device *netdev)
1578{
1579	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1580	int ret;
1581
1582	/* Create the various names based on netdev name */
1583	snprintf(pdata->an_name, sizeof(pdata->an_name) - 1, "%s-pcs",
1584		 netdev_name(netdev));
1585
1586	snprintf(pdata->ecc_name, sizeof(pdata->ecc_name) - 1, "%s-ecc",
1587		 netdev_name(netdev));
1588
1589	snprintf(pdata->i2c_name, sizeof(pdata->i2c_name) - 1, "%s-i2c",
1590		 netdev_name(netdev));
1591
1592	/* Create workqueues */
1593	pdata->dev_workqueue =
1594		create_singlethread_workqueue(netdev_name(netdev));
1595	if (!pdata->dev_workqueue) {
1596		netdev_err(netdev, "device workqueue creation failed\n");
1597		return -ENOMEM;
1598	}
1599
1600	pdata->an_workqueue =
1601		create_singlethread_workqueue(pdata->an_name);
1602	if (!pdata->an_workqueue) {
1603		netdev_err(netdev, "phy workqueue creation failed\n");
1604		ret = -ENOMEM;
1605		goto err_dev_wq;
1606	}
1607
1608	/* Enable the clocks */
1609	ret = clk_prepare_enable(pdata->sysclk);
1610	if (ret) {
1611		netdev_alert(netdev, "dma clk_prepare_enable failed\n");
1612		goto err_an_wq;
1613	}
1614
1615	ret = clk_prepare_enable(pdata->ptpclk);
1616	if (ret) {
1617		netdev_alert(netdev, "ptp clk_prepare_enable failed\n");
1618		goto err_sysclk;
1619	}
1620
1621	INIT_WORK(&pdata->service_work, xgbe_service);
1622	INIT_WORK(&pdata->restart_work, xgbe_restart);
1623	INIT_WORK(&pdata->stopdev_work, xgbe_stopdev);
1624	INIT_WORK(&pdata->tx_tstamp_work, xgbe_tx_tstamp);
1625
1626	/* Initialize PTP timestamping and clock. */
1627	xgbe_init_ptp(pdata);
1628
1629	ret = xgbe_alloc_memory(pdata);
1630	if (ret)
1631		goto err_ptpclk;
1632
1633	ret = xgbe_start(pdata);
1634	if (ret)
1635		goto err_mem;
1636
1637	clear_bit(XGBE_DOWN, &pdata->dev_state);
1638
1639	return 0;
1640
1641err_mem:
1642	xgbe_free_memory(pdata);
1643
1644err_ptpclk:
1645	clk_disable_unprepare(pdata->ptpclk);
1646
1647err_sysclk:
1648	clk_disable_unprepare(pdata->sysclk);
1649
1650err_an_wq:
1651	destroy_workqueue(pdata->an_workqueue);
1652
1653err_dev_wq:
1654	destroy_workqueue(pdata->dev_workqueue);
1655
1656	return ret;
1657}
1658
1659static int xgbe_close(struct net_device *netdev)
1660{
1661	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1662
1663	/* Stop the device */
1664	xgbe_stop(pdata);
1665
1666	xgbe_free_memory(pdata);
1667
1668	/* Disable the clocks */
1669	clk_disable_unprepare(pdata->ptpclk);
1670	clk_disable_unprepare(pdata->sysclk);
1671
1672	destroy_workqueue(pdata->an_workqueue);
1673
1674	destroy_workqueue(pdata->dev_workqueue);
1675
1676	set_bit(XGBE_DOWN, &pdata->dev_state);
1677
1678	return 0;
1679}
1680
1681static netdev_tx_t xgbe_xmit(struct sk_buff *skb, struct net_device *netdev)
1682{
1683	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1684	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1685	struct xgbe_desc_if *desc_if = &pdata->desc_if;
1686	struct xgbe_channel *channel;
1687	struct xgbe_ring *ring;
1688	struct xgbe_packet_data *packet;
1689	struct netdev_queue *txq;
1690	netdev_tx_t ret;
1691
1692	DBGPR("-->xgbe_xmit: skb->len = %d\n", skb->len);
1693
1694	channel = pdata->channel[skb->queue_mapping];
1695	txq = netdev_get_tx_queue(netdev, channel->queue_index);
1696	ring = channel->tx_ring;
1697	packet = &ring->packet_data;
1698
1699	ret = NETDEV_TX_OK;
1700
1701	if (skb->len == 0) {
1702		netif_err(pdata, tx_err, netdev,
1703			  "empty skb received from stack\n");
1704		dev_kfree_skb_any(skb);
1705		goto tx_netdev_return;
1706	}
1707
1708	/* Calculate preliminary packet info */
1709	memset(packet, 0, sizeof(*packet));
1710	xgbe_packet_info(pdata, ring, skb, packet);
1711
1712	/* Check that there are enough descriptors available */
1713	ret = xgbe_maybe_stop_tx_queue(channel, ring, packet->rdesc_count);
1714	if (ret)
1715		goto tx_netdev_return;
1716
1717	ret = xgbe_prep_tso(skb, packet);
1718	if (ret) {
1719		netif_err(pdata, tx_err, netdev,
1720			  "error processing TSO packet\n");
1721		dev_kfree_skb_any(skb);
1722		goto tx_netdev_return;
1723	}
1724	xgbe_prep_vlan(skb, packet);
1725
1726	if (!desc_if->map_tx_skb(channel, skb)) {
1727		dev_kfree_skb_any(skb);
1728		goto tx_netdev_return;
1729	}
1730
1731	xgbe_prep_tx_tstamp(pdata, skb, packet);
1732
1733	/* Report on the actual number of bytes (to be) sent */
1734	netdev_tx_sent_queue(txq, packet->tx_bytes);
1735
1736	/* Configure required descriptor fields for transmission */
1737	hw_if->dev_xmit(channel);
1738
1739	if (netif_msg_pktdata(pdata))
1740		xgbe_print_pkt(netdev, skb, true);
1741
1742	/* Stop the queue in advance if there may not be enough descriptors */
1743	xgbe_maybe_stop_tx_queue(channel, ring, XGBE_TX_MAX_DESCS);
1744
1745	ret = NETDEV_TX_OK;
1746
1747tx_netdev_return:
1748	return ret;
1749}
1750
1751static void xgbe_set_rx_mode(struct net_device *netdev)
1752{
1753	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1754	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1755
1756	DBGPR("-->xgbe_set_rx_mode\n");
1757
1758	hw_if->config_rx_mode(pdata);
1759
1760	DBGPR("<--xgbe_set_rx_mode\n");
1761}
1762
1763static int xgbe_set_mac_address(struct net_device *netdev, void *addr)
1764{
1765	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1766	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1767	struct sockaddr *saddr = addr;
1768
1769	DBGPR("-->xgbe_set_mac_address\n");
1770
1771	if (!is_valid_ether_addr(saddr->sa_data))
1772		return -EADDRNOTAVAIL;
1773
1774	eth_hw_addr_set(netdev, saddr->sa_data);
1775
1776	hw_if->set_mac_address(pdata, netdev->dev_addr);
1777
1778	DBGPR("<--xgbe_set_mac_address\n");
1779
1780	return 0;
1781}
1782
1783static int xgbe_change_mtu(struct net_device *netdev, int mtu)
1784{
1785	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1786	int ret;
1787
1788	DBGPR("-->xgbe_change_mtu\n");
1789
1790	ret = xgbe_calc_rx_buf_size(netdev, mtu);
1791	if (ret < 0)
1792		return ret;
1793
1794	pdata->rx_buf_size = ret;
1795	WRITE_ONCE(netdev->mtu, mtu);
1796
1797	xgbe_restart_dev(pdata);
1798
1799	DBGPR("<--xgbe_change_mtu\n");
1800
1801	return 0;
1802}
1803
1804static void xgbe_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1805{
1806	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1807
1808	netdev_warn(netdev, "tx timeout, device restarting\n");
1809	schedule_work(&pdata->restart_work);
1810}
1811
1812static void xgbe_get_stats64(struct net_device *netdev,
1813			     struct rtnl_link_stats64 *s)
1814{
1815	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1816	struct xgbe_mmc_stats *pstats = &pdata->mmc_stats;
1817
1818	DBGPR("-->%s\n", __func__);
1819
1820	pdata->hw_if.read_mmc_stats(pdata);
1821
1822	s->rx_packets = pstats->rxframecount_gb;
1823	s->rx_bytes = pstats->rxoctetcount_gb;
1824	s->rx_errors = pstats->rxframecount_gb -
1825		       pstats->rxbroadcastframes_g -
1826		       pstats->rxmulticastframes_g -
1827		       pstats->rxunicastframes_g;
1828	s->multicast = pstats->rxmulticastframes_g;
1829	s->rx_length_errors = pstats->rxlengtherror;
1830	s->rx_crc_errors = pstats->rxcrcerror;
1831	s->rx_over_errors = pstats->rxfifooverflow;
1832
1833	s->tx_packets = pstats->txframecount_gb;
1834	s->tx_bytes = pstats->txoctetcount_gb;
1835	s->tx_errors = pstats->txframecount_gb - pstats->txframecount_g;
1836	s->tx_dropped = netdev->stats.tx_dropped;
1837
1838	DBGPR("<--%s\n", __func__);
1839}
1840
1841static int xgbe_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1842				u16 vid)
1843{
1844	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1845	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1846
1847	DBGPR("-->%s\n", __func__);
1848
1849	set_bit(vid, pdata->active_vlans);
1850	hw_if->update_vlan_hash_table(pdata);
1851
1852	DBGPR("<--%s\n", __func__);
1853
1854	return 0;
1855}
1856
1857static int xgbe_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1858				 u16 vid)
1859{
1860	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1861	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1862
1863	DBGPR("-->%s\n", __func__);
1864
1865	clear_bit(vid, pdata->active_vlans);
1866	hw_if->update_vlan_hash_table(pdata);
1867
1868	DBGPR("<--%s\n", __func__);
1869
1870	return 0;
1871}
1872
1873#ifdef CONFIG_NET_POLL_CONTROLLER
1874static void xgbe_poll_controller(struct net_device *netdev)
1875{
1876	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1877	struct xgbe_channel *channel;
1878	unsigned int i;
1879
1880	DBGPR("-->xgbe_poll_controller\n");
1881
1882	if (pdata->per_channel_irq) {
1883		for (i = 0; i < pdata->channel_count; i++) {
1884			channel = pdata->channel[i];
1885			xgbe_dma_isr(channel->dma_irq, channel);
1886		}
1887	} else {
1888		disable_irq(pdata->dev_irq);
1889		xgbe_isr(pdata->dev_irq, pdata);
1890		enable_irq(pdata->dev_irq);
1891	}
1892
1893	DBGPR("<--xgbe_poll_controller\n");
1894}
1895#endif /* End CONFIG_NET_POLL_CONTROLLER */
1896
1897static int xgbe_setup_tc(struct net_device *netdev, enum tc_setup_type type,
1898			 void *type_data)
1899{
1900	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1901	struct tc_mqprio_qopt *mqprio = type_data;
1902	u8 tc;
1903
1904	if (type != TC_SETUP_QDISC_MQPRIO)
1905		return -EOPNOTSUPP;
1906
1907	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1908	tc = mqprio->num_tc;
1909
1910	if (tc > pdata->hw_feat.tc_cnt)
1911		return -EINVAL;
1912
1913	pdata->num_tcs = tc;
1914	pdata->hw_if.config_tc(pdata);
1915
1916	return 0;
1917}
1918
1919static netdev_features_t xgbe_fix_features(struct net_device *netdev,
1920					   netdev_features_t features)
1921{
1922	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1923	netdev_features_t vxlan_base;
1924
1925	vxlan_base = NETIF_F_GSO_UDP_TUNNEL | NETIF_F_RX_UDP_TUNNEL_PORT;
1926
1927	if (!pdata->hw_feat.vxn)
1928		return features;
1929
1930	/* VXLAN CSUM requires VXLAN base */
1931	if ((features & NETIF_F_GSO_UDP_TUNNEL_CSUM) &&
1932	    !(features & NETIF_F_GSO_UDP_TUNNEL)) {
1933		netdev_notice(netdev,
1934			      "forcing tx udp tunnel support\n");
1935		features |= NETIF_F_GSO_UDP_TUNNEL;
1936	}
1937
1938	/* Can't do one without doing the other */
1939	if ((features & vxlan_base) != vxlan_base) {
1940		netdev_notice(netdev,
1941			      "forcing both tx and rx udp tunnel support\n");
1942		features |= vxlan_base;
1943	}
1944
1945	if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
1946		if (!(features & NETIF_F_GSO_UDP_TUNNEL_CSUM)) {
1947			netdev_notice(netdev,
1948				      "forcing tx udp tunnel checksumming on\n");
1949			features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1950		}
1951	} else {
1952		if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM) {
1953			netdev_notice(netdev,
1954				      "forcing tx udp tunnel checksumming off\n");
1955			features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
1956		}
1957	}
1958
1959	return features;
1960}
1961
1962static int xgbe_set_features(struct net_device *netdev,
1963			     netdev_features_t features)
1964{
1965	struct xgbe_prv_data *pdata = netdev_priv(netdev);
1966	struct xgbe_hw_if *hw_if = &pdata->hw_if;
1967	netdev_features_t rxhash, rxcsum, rxvlan, rxvlan_filter;
1968	int ret = 0;
1969
1970	rxhash = pdata->netdev_features & NETIF_F_RXHASH;
1971	rxcsum = pdata->netdev_features & NETIF_F_RXCSUM;
1972	rxvlan = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_RX;
1973	rxvlan_filter = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_FILTER;
1974
1975	if ((features & NETIF_F_RXHASH) && !rxhash)
1976		ret = hw_if->enable_rss(pdata);
1977	else if (!(features & NETIF_F_RXHASH) && rxhash)
1978		ret = hw_if->disable_rss(pdata);
1979	if (ret)
1980		return ret;
1981
1982	if ((features & NETIF_F_RXCSUM) && !rxcsum) {
1983		hw_if->enable_sph(pdata);
1984		hw_if->enable_vxlan(pdata);
1985		hw_if->enable_rx_csum(pdata);
1986		schedule_work(&pdata->restart_work);
1987	} else if (!(features & NETIF_F_RXCSUM) && rxcsum) {
1988		hw_if->disable_sph(pdata);
1989		hw_if->disable_vxlan(pdata);
1990		hw_if->disable_rx_csum(pdata);
1991		schedule_work(&pdata->restart_work);
1992	}
1993
1994	if ((features & NETIF_F_HW_VLAN_CTAG_RX) && !rxvlan)
1995		hw_if->enable_rx_vlan_stripping(pdata);
1996	else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && rxvlan)
1997		hw_if->disable_rx_vlan_stripping(pdata);
1998
1999	if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && !rxvlan_filter)
2000		hw_if->enable_rx_vlan_filtering(pdata);
2001	else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && rxvlan_filter)
2002		hw_if->disable_rx_vlan_filtering(pdata);
2003
2004	pdata->netdev_features = features;
2005
2006	DBGPR("<--xgbe_set_features\n");
2007
2008	return 0;
2009}
2010
2011static netdev_features_t xgbe_features_check(struct sk_buff *skb,
2012					     struct net_device *netdev,
2013					     netdev_features_t features)
2014{
2015	features = vlan_features_check(skb, features);
2016	features = vxlan_features_check(skb, features);
2017
2018	return features;
2019}
2020
2021static const struct net_device_ops xgbe_netdev_ops = {
2022	.ndo_open		= xgbe_open,
2023	.ndo_stop		= xgbe_close,
2024	.ndo_start_xmit		= xgbe_xmit,
2025	.ndo_set_rx_mode	= xgbe_set_rx_mode,
2026	.ndo_set_mac_address	= xgbe_set_mac_address,
2027	.ndo_validate_addr	= eth_validate_addr,
2028	.ndo_change_mtu		= xgbe_change_mtu,
2029	.ndo_tx_timeout		= xgbe_tx_timeout,
2030	.ndo_get_stats64	= xgbe_get_stats64,
2031	.ndo_vlan_rx_add_vid	= xgbe_vlan_rx_add_vid,
2032	.ndo_vlan_rx_kill_vid	= xgbe_vlan_rx_kill_vid,
2033#ifdef CONFIG_NET_POLL_CONTROLLER
2034	.ndo_poll_controller	= xgbe_poll_controller,
2035#endif
2036	.ndo_setup_tc		= xgbe_setup_tc,
2037	.ndo_fix_features	= xgbe_fix_features,
2038	.ndo_set_features	= xgbe_set_features,
2039	.ndo_features_check	= xgbe_features_check,
2040	.ndo_hwtstamp_get	= xgbe_get_hwtstamp_settings,
2041	.ndo_hwtstamp_set	= xgbe_set_hwtstamp_settings,
2042};
2043
2044const struct net_device_ops *xgbe_get_netdev_ops(void)
2045{
2046	return &xgbe_netdev_ops;
2047}
2048
2049static void xgbe_rx_refresh(struct xgbe_channel *channel)
2050{
2051	struct xgbe_prv_data *pdata = channel->pdata;
2052	struct xgbe_hw_if *hw_if = &pdata->hw_if;
2053	struct xgbe_desc_if *desc_if = &pdata->desc_if;
2054	struct xgbe_ring *ring = channel->rx_ring;
2055	struct xgbe_ring_data *rdata;
2056
2057	while (ring->dirty != ring->cur) {
2058		rdata = XGBE_GET_DESC_DATA(ring, ring->dirty);
2059
2060		/* Reset rdata values */
2061		desc_if->unmap_rdata(pdata, rdata);
2062
2063		if (desc_if->map_rx_buffer(pdata, ring, rdata))
2064			break;
2065
2066		hw_if->rx_desc_reset(pdata, rdata, ring->dirty);
2067
2068		ring->dirty++;
2069	}
2070
2071	/* Make sure everything is written before the register write */
2072	wmb();
2073
2074	/* Update the Rx Tail Pointer Register with address of
2075	 * the last cleaned entry */
2076	rdata = XGBE_GET_DESC_DATA(ring, ring->dirty - 1);
2077	XGMAC_DMA_IOWRITE(channel, DMA_CH_RDTR_LO,
2078			  lower_32_bits(rdata->rdesc_dma));
2079}
2080
2081static struct sk_buff *xgbe_create_skb(struct xgbe_prv_data *pdata,
2082				       struct napi_struct *napi,
2083				       struct xgbe_ring_data *rdata,
2084				       unsigned int len)
2085{
2086	struct sk_buff *skb;
2087	u8 *packet;
2088
2089	skb = napi_alloc_skb(napi, rdata->rx.hdr.dma_len);
2090	if (!skb)
2091		return NULL;
2092
2093	/* Pull in the header buffer which may contain just the header
2094	 * or the header plus data
2095	 */
2096	dma_sync_single_range_for_cpu(pdata->dev, rdata->rx.hdr.dma_base,
2097				      rdata->rx.hdr.dma_off,
2098				      rdata->rx.hdr.dma_len, DMA_FROM_DEVICE);
2099
2100	packet = page_address(rdata->rx.hdr.pa.pages) +
2101		 rdata->rx.hdr.pa.pages_offset;
2102	skb_copy_to_linear_data(skb, packet, len);
2103	skb_put(skb, len);
2104
2105	return skb;
2106}
2107
2108static unsigned int xgbe_rx_buf1_len(struct xgbe_ring_data *rdata,
2109				     struct xgbe_packet_data *packet)
2110{
2111	/* Always zero if not the first descriptor */
2112	if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, FIRST))
2113		return 0;
2114
2115	/* First descriptor with split header, return header length */
2116	if (rdata->rx.hdr_len)
2117		return rdata->rx.hdr_len;
2118
2119	/* First descriptor but not the last descriptor and no split header,
2120	 * so the full buffer was used
2121	 */
2122	if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, LAST))
2123		return rdata->rx.hdr.dma_len;
2124
2125	/* First descriptor and last descriptor and no split header, so
2126	 * calculate how much of the buffer was used
2127	 */
2128	return min_t(unsigned int, rdata->rx.hdr.dma_len, rdata->rx.len);
2129}
2130
2131static unsigned int xgbe_rx_buf2_len(struct xgbe_ring_data *rdata,
2132				     struct xgbe_packet_data *packet,
2133				     unsigned int len)
2134{
2135	/* Always the full buffer if not the last descriptor */
2136	if (!XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, LAST))
2137		return rdata->rx.buf.dma_len;
2138
2139	/* Last descriptor so calculate how much of the buffer was used
2140	 * for the last bit of data
2141	 */
2142	return rdata->rx.len - len;
2143}
2144
2145static int xgbe_tx_poll(struct xgbe_channel *channel)
2146{
2147	struct xgbe_prv_data *pdata = channel->pdata;
2148	struct xgbe_hw_if *hw_if = &pdata->hw_if;
2149	struct xgbe_desc_if *desc_if = &pdata->desc_if;
2150	struct xgbe_ring *ring = channel->tx_ring;
2151	struct xgbe_ring_data *rdata;
2152	struct xgbe_ring_desc *rdesc;
2153	struct net_device *netdev = pdata->netdev;
2154	struct netdev_queue *txq;
2155	int processed = 0;
2156	unsigned int tx_packets = 0, tx_bytes = 0;
2157	unsigned int cur;
2158
2159	DBGPR("-->xgbe_tx_poll\n");
2160
2161	/* Nothing to do if there isn't a Tx ring for this channel */
2162	if (!ring)
2163		return 0;
2164
2165	cur = ring->cur;
2166
2167	/* Be sure we get ring->cur before accessing descriptor data */
2168	smp_rmb();
2169
2170	txq = netdev_get_tx_queue(netdev, channel->queue_index);
2171
2172	while ((processed < XGBE_TX_DESC_MAX_PROC) &&
2173	       (ring->dirty != cur)) {
2174		rdata = XGBE_GET_DESC_DATA(ring, ring->dirty);
2175		rdesc = rdata->rdesc;
2176
2177		if (!hw_if->tx_complete(rdesc))
2178			break;
2179
2180		/* Make sure descriptor fields are read after reading the OWN
2181		 * bit */
2182		dma_rmb();
2183
2184		if (netif_msg_tx_done(pdata))
2185			xgbe_dump_tx_desc(pdata, ring, ring->dirty, 1, 0);
2186
2187		if (hw_if->is_last_desc(rdesc)) {
2188			tx_packets += rdata->tx.packets;
2189			tx_bytes += rdata->tx.bytes;
2190		}
2191
2192		/* Free the SKB and reset the descriptor for re-use */
2193		desc_if->unmap_rdata(pdata, rdata);
2194		hw_if->tx_desc_reset(rdata);
2195
2196		processed++;
2197		ring->dirty++;
2198	}
2199
2200	if (!processed)
2201		return 0;
2202
2203	netdev_tx_completed_queue(txq, tx_packets, tx_bytes);
2204
2205	if ((ring->tx.queue_stopped == 1) &&
2206	    (xgbe_tx_avail_desc(ring) > XGBE_TX_DESC_MIN_FREE)) {
2207		ring->tx.queue_stopped = 0;
2208		netif_tx_wake_queue(txq);
2209	}
2210
2211	DBGPR("<--xgbe_tx_poll: processed=%d\n", processed);
2212
2213	return processed;
2214}
2215
2216static int xgbe_rx_poll(struct xgbe_channel *channel, int budget)
2217{
2218	struct xgbe_prv_data *pdata = channel->pdata;
2219	struct xgbe_hw_if *hw_if = &pdata->hw_if;
2220	struct xgbe_ring *ring = channel->rx_ring;
2221	struct xgbe_ring_data *rdata;
2222	struct xgbe_packet_data *packet;
2223	struct net_device *netdev = pdata->netdev;
2224	struct napi_struct *napi;
2225	struct sk_buff *skb;
2226	struct skb_shared_hwtstamps *hwtstamps;
2227	unsigned int last, error, context_next, context;
2228	unsigned int len, buf1_len, buf2_len, max_len;
2229	unsigned int received = 0;
2230	int packet_count = 0;
2231
2232	DBGPR("-->xgbe_rx_poll: budget=%d\n", budget);
2233
2234	/* Nothing to do if there isn't a Rx ring for this channel */
2235	if (!ring)
2236		return 0;
2237
2238	last = 0;
2239	context_next = 0;
2240
2241	napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;
2242
2243	rdata = XGBE_GET_DESC_DATA(ring, ring->cur);
2244	packet = &ring->packet_data;
2245	while (packet_count < budget) {
2246		DBGPR("  cur = %d\n", ring->cur);
2247
2248		/* First time in loop see if we need to restore state */
2249		if (!received && rdata->state_saved) {
2250			skb = rdata->state.skb;
2251			error = rdata->state.error;
2252			len = rdata->state.len;
2253		} else {
2254			memset(packet, 0, sizeof(*packet));
2255			skb = NULL;
2256			error = 0;
2257			len = 0;
2258		}
2259
2260read_again:
2261		rdata = XGBE_GET_DESC_DATA(ring, ring->cur);
2262
2263		if (xgbe_rx_dirty_desc(ring) > (XGBE_RX_DESC_CNT >> 3))
2264			xgbe_rx_refresh(channel);
2265
2266		if (hw_if->dev_read(channel))
2267			break;
2268
2269		received++;
2270		ring->cur++;
2271
2272		last = XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES,
2273				      LAST);
2274		context_next = XGMAC_GET_BITS(packet->attributes,
2275					      RX_PACKET_ATTRIBUTES,
2276					      CONTEXT_NEXT);
2277		context = XGMAC_GET_BITS(packet->attributes,
2278					 RX_PACKET_ATTRIBUTES,
2279					 CONTEXT);
2280
2281		/* Earlier error, just drain the remaining data */
2282		if ((!last || context_next) && error)
2283			goto read_again;
2284
2285		if (error || packet->errors) {
2286			dev_kfree_skb(skb);
2287			goto next_packet;
2288		}
2289
2290		if (!context) {
2291			/* Get the data length in the descriptor buffers */
2292			buf1_len = xgbe_rx_buf1_len(rdata, packet);
2293			len += buf1_len;
2294			buf2_len = xgbe_rx_buf2_len(rdata, packet, len);
2295			len += buf2_len;
2296
2297			if (buf2_len > rdata->rx.buf.dma_len) {
2298				/* Hardware inconsistency within the descriptors
2299				 * that has resulted in a length underflow.
2300				 */
2301				error = 1;
2302				goto skip_data;
2303			}
2304
2305			if (!skb) {
2306				skb = xgbe_create_skb(pdata, napi, rdata,
2307						      buf1_len);
2308				if (!skb) {
2309					error = 1;
2310					goto skip_data;
2311				}
2312			}
2313
2314			if (buf2_len) {
2315				dma_sync_single_range_for_cpu(pdata->dev,
2316							rdata->rx.buf.dma_base,
2317							rdata->rx.buf.dma_off,
2318							rdata->rx.buf.dma_len,
2319							DMA_FROM_DEVICE);
2320
2321				skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
2322						rdata->rx.buf.pa.pages,
2323						rdata->rx.buf.pa.pages_offset,
2324						buf2_len,
2325						rdata->rx.buf.dma_len);
2326				rdata->rx.buf.pa.pages = NULL;
2327			}
2328		}
2329
2330skip_data:
2331		if (!last || context_next)
2332			goto read_again;
2333
2334		if (!skb || error) {
2335			dev_kfree_skb(skb);
2336			goto next_packet;
2337		}
2338
2339		/* Be sure we don't exceed the configured MTU */
2340		max_len = netdev->mtu + ETH_HLEN;
2341		if (!(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
2342		    (skb->protocol == htons(ETH_P_8021Q)))
2343			max_len += VLAN_HLEN;
2344
2345		if (skb->len > max_len) {
2346			netif_err(pdata, rx_err, netdev,
2347				  "packet length exceeds configured MTU\n");
2348			dev_kfree_skb(skb);
2349			goto next_packet;
2350		}
2351
2352		if (netif_msg_pktdata(pdata))
2353			xgbe_print_pkt(netdev, skb, false);
2354
2355		skb_checksum_none_assert(skb);
2356		if (XGMAC_GET_BITS(packet->attributes,
2357				   RX_PACKET_ATTRIBUTES, CSUM_DONE))
2358			skb->ip_summed = CHECKSUM_UNNECESSARY;
2359
2360		if (XGMAC_GET_BITS(packet->attributes,
2361				   RX_PACKET_ATTRIBUTES, TNP)) {
2362			skb->encapsulation = 1;
2363
2364			if (XGMAC_GET_BITS(packet->attributes,
2365					   RX_PACKET_ATTRIBUTES, TNPCSUM_DONE))
2366				skb->csum_level = 1;
2367		}
2368
2369		if (XGMAC_GET_BITS(packet->attributes,
2370				   RX_PACKET_ATTRIBUTES, VLAN_CTAG))
2371			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
2372					       packet->vlan_ctag);
2373
2374		if (XGMAC_GET_BITS(packet->attributes,
2375				   RX_PACKET_ATTRIBUTES, RX_TSTAMP)) {
2376			hwtstamps = skb_hwtstamps(skb);
2377			hwtstamps->hwtstamp = ns_to_ktime(packet->rx_tstamp);
2378		}
2379
2380		if (XGMAC_GET_BITS(packet->attributes,
2381				   RX_PACKET_ATTRIBUTES, RSS_HASH))
2382			skb_set_hash(skb, packet->rss_hash,
2383				     packet->rss_hash_type);
2384
2385		skb->dev = netdev;
2386		skb->protocol = eth_type_trans(skb, netdev);
2387		skb_record_rx_queue(skb, channel->queue_index);
2388
2389		napi_gro_receive(napi, skb);
2390
2391next_packet:
2392		packet_count++;
2393	}
2394
2395	/* Check if we need to save state before leaving */
2396	if (received && (!last || context_next)) {
2397		rdata = XGBE_GET_DESC_DATA(ring, ring->cur);
2398		rdata->state_saved = 1;
2399		rdata->state.skb = skb;
2400		rdata->state.len = len;
2401		rdata->state.error = error;
2402	}
2403
2404	DBGPR("<--xgbe_rx_poll: packet_count = %d\n", packet_count);
2405
2406	return packet_count;
2407}
2408
2409static int xgbe_one_poll(struct napi_struct *napi, int budget)
2410{
2411	struct xgbe_channel *channel = container_of(napi, struct xgbe_channel,
2412						    napi);
2413	struct xgbe_prv_data *pdata = channel->pdata;
2414	int processed = 0;
2415
2416	DBGPR("-->xgbe_one_poll: budget=%d\n", budget);
2417
2418	/* Cleanup Tx ring first */
2419	xgbe_tx_poll(channel);
2420
2421	/* Process Rx ring next */
2422	processed = xgbe_rx_poll(channel, budget);
2423
2424	/* If we processed everything, we are done */
2425	if ((processed < budget) && napi_complete_done(napi, processed)) {
2426		/* Enable Tx and Rx interrupts */
2427		if (pdata->channel_irq_mode)
2428			xgbe_enable_rx_tx_int(pdata, channel);
2429		else
2430			enable_irq(channel->dma_irq);
2431	}
2432
2433	DBGPR("<--xgbe_one_poll: received = %d\n", processed);
2434
2435	return processed;
2436}
2437
2438static int xgbe_all_poll(struct napi_struct *napi, int budget)
2439{
2440	struct xgbe_prv_data *pdata = container_of(napi, struct xgbe_prv_data,
2441						   napi);
2442	struct xgbe_channel *channel;
2443	int ring_budget;
2444	int processed, last_processed;
2445	unsigned int i;
2446
2447	DBGPR("-->xgbe_all_poll: budget=%d\n", budget);
2448
2449	processed = 0;
2450	ring_budget = budget / pdata->rx_ring_count;
2451	do {
2452		last_processed = processed;
2453
2454		for (i = 0; i < pdata->channel_count; i++) {
2455			channel = pdata->channel[i];
2456
2457			/* Cleanup Tx ring first */
2458			xgbe_tx_poll(channel);
2459
2460			/* Process Rx ring next */
2461			if (ring_budget > (budget - processed))
2462				ring_budget = budget - processed;
2463			processed += xgbe_rx_poll(channel, ring_budget);
2464		}
2465	} while ((processed < budget) && (processed != last_processed));
2466
2467	/* If we processed everything, we are done */
2468	if ((processed < budget) && napi_complete_done(napi, processed)) {
2469		/* Enable Tx and Rx interrupts */
2470		xgbe_enable_rx_tx_ints(pdata);
2471	}
2472
2473	DBGPR("<--xgbe_all_poll: received = %d\n", processed);
2474
2475	return processed;
2476}
2477
2478void xgbe_dump_tx_desc(struct xgbe_prv_data *pdata, struct xgbe_ring *ring,
2479		       unsigned int idx, unsigned int count, unsigned int flag)
2480{
2481	struct xgbe_ring_data *rdata;
2482	struct xgbe_ring_desc *rdesc;
2483
2484	while (count--) {
2485		rdata = XGBE_GET_DESC_DATA(ring, idx);
2486		rdesc = rdata->rdesc;
2487		netdev_dbg(pdata->netdev,
2488			   "TX_NORMAL_DESC[%d %s] = %08x:%08x:%08x:%08x\n", idx,
2489			   (flag == 1) ? "QUEUED FOR TX" : "TX BY DEVICE",
2490			   le32_to_cpu(rdesc->desc0),
2491			   le32_to_cpu(rdesc->desc1),
2492			   le32_to_cpu(rdesc->desc2),
2493			   le32_to_cpu(rdesc->desc3));
2494		idx++;
2495	}
2496}
2497
2498void xgbe_dump_rx_desc(struct xgbe_prv_data *pdata, struct xgbe_ring *ring,
2499		       unsigned int idx)
2500{
2501	struct xgbe_ring_data *rdata;
2502	struct xgbe_ring_desc *rdesc;
2503
2504	rdata = XGBE_GET_DESC_DATA(ring, idx);
2505	rdesc = rdata->rdesc;
2506	netdev_dbg(pdata->netdev,
2507		   "RX_NORMAL_DESC[%d RX BY DEVICE] = %08x:%08x:%08x:%08x\n",
2508		   idx, le32_to_cpu(rdesc->desc0), le32_to_cpu(rdesc->desc1),
2509		   le32_to_cpu(rdesc->desc2), le32_to_cpu(rdesc->desc3));
2510}
2511
2512void xgbe_print_pkt(struct net_device *netdev, struct sk_buff *skb, bool tx_rx)
2513{
2514	struct ethhdr *eth = (struct ethhdr *)skb->data;
2515	unsigned char buffer[128];
2516	unsigned int i;
2517
2518	netdev_dbg(netdev, "\n************** SKB dump ****************\n");
2519
2520	netdev_dbg(netdev, "%s packet of %d bytes\n",
2521		   (tx_rx ? "TX" : "RX"), skb->len);
2522
2523	netdev_dbg(netdev, "Dst MAC addr: %pM\n", eth->h_dest);
2524	netdev_dbg(netdev, "Src MAC addr: %pM\n", eth->h_source);
2525	netdev_dbg(netdev, "Protocol: %#06x\n", ntohs(eth->h_proto));
2526
2527	for (i = 0; i < skb->len; i += 32) {
2528		unsigned int len = min(skb->len - i, 32U);
2529
2530		hex_dump_to_buffer(&skb->data[i], len, 32, 1,
2531				   buffer, sizeof(buffer), false);
2532		netdev_dbg(netdev, "  %#06x: %s\n", i, buffer);
2533	}
2534
2535	netdev_dbg(netdev, "\n************** SKB dump ****************\n");
2536}