xilinx_axienet_main.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Xilinx Axi Ethernet device driver
   4 *
   5 * Copyright (c) 2008 Nissin Systems Co., Ltd.,  Yoshio Kashiwagi
   6 * Copyright (c) 2005-2008 DLA Systems,  David H. Lynch Jr. <dhlii@dlasys.net>
   7 * Copyright (c) 2008-2009 Secret Lab Technologies Ltd.
   8 * Copyright (c) 2010 - 2011 Michal Simek <monstr@monstr.eu>
   9 * Copyright (c) 2010 - 2011 PetaLogix
  10 * Copyright (c) 2019 - 2022 Calian Advanced Technologies
  11 * Copyright (c) 2010 - 2012 Xilinx, Inc. All rights reserved.
  12 *
  13 * This is a driver for the Xilinx Axi Ethernet which is used in the Virtex6
  14 * and Spartan6.
  15 *
  16 * TODO:
  17 *  - Add Axi Fifo support.
  18 *  - Factor out Axi DMA code into separate driver.
  19 *  - Test and fix basic multicast filtering.
  20 *  - Add support for extended multicast filtering.
  21 *  - Test basic VLAN support.
  22 *  - Add support for extended VLAN support.
  23 */
  24
  25#include <linux/clk.h>
  26#include <linux/delay.h>
  27#include <linux/etherdevice.h>
  28#include <linux/module.h>
  29#include <linux/netdevice.h>
  30#include <linux/of.h>
  31#include <linux/of_mdio.h>
  32#include <linux/of_net.h>
  33#include <linux/of_irq.h>
  34#include <linux/of_address.h>
  35#include <linux/platform_device.h>
  36#include <linux/skbuff.h>
  37#include <linux/math64.h>
  38#include <linux/phy.h>
  39#include <linux/mii.h>
  40#include <linux/ethtool.h>
  41#include <linux/dmaengine.h>
  42#include <linux/dma-mapping.h>
  43#include <linux/dma/xilinx_dma.h>
  44#include <linux/circ_buf.h>
  45#include <net/netdev_queues.h>
  46
  47#include "xilinx_axienet.h"
  48
  49/* Descriptors defines for Tx and Rx DMA */
  50#define TX_BD_NUM_DEFAULT		128
  51#define RX_BD_NUM_DEFAULT		1024
  52#define TX_BD_NUM_MIN			(MAX_SKB_FRAGS + 1)
  53#define TX_BD_NUM_MAX			4096
  54#define RX_BD_NUM_MAX			4096
  55#define DMA_NUM_APP_WORDS		5
  56#define LEN_APP				4
  57#define RX_BUF_NUM_DEFAULT		128
  58
  59/* Must be shorter than length of ethtool_drvinfo.driver field to fit */
  60#define DRIVER_NAME		"xaxienet"
  61#define DRIVER_DESCRIPTION	"Xilinx Axi Ethernet driver"
  62#define DRIVER_VERSION		"1.00a"
  63
  64#define AXIENET_REGS_N		40
  65
  66static void axienet_rx_submit_desc(struct net_device *ndev);
  67
  68/* Match table for of_platform binding */
  69static const struct of_device_id axienet_of_match[] = {
  70	{ .compatible = "xlnx,axi-ethernet-1.00.a", },
  71	{ .compatible = "xlnx,axi-ethernet-1.01.a", },
  72	{ .compatible = "xlnx,axi-ethernet-2.01.a", },
  73	{},
  74};
  75
  76MODULE_DEVICE_TABLE(of, axienet_of_match);
  77
  78/* Option table for setting up Axi Ethernet hardware options */
  79static struct axienet_option axienet_options[] = {
  80	/* Turn on jumbo packet support for both Rx and Tx */
  81	{
  82		.opt = XAE_OPTION_JUMBO,
  83		.reg = XAE_TC_OFFSET,
  84		.m_or = XAE_TC_JUM_MASK,
  85	}, {
  86		.opt = XAE_OPTION_JUMBO,
  87		.reg = XAE_RCW1_OFFSET,
  88		.m_or = XAE_RCW1_JUM_MASK,
  89	}, { /* Turn on VLAN packet support for both Rx and Tx */
  90		.opt = XAE_OPTION_VLAN,
  91		.reg = XAE_TC_OFFSET,
  92		.m_or = XAE_TC_VLAN_MASK,
  93	}, {
  94		.opt = XAE_OPTION_VLAN,
  95		.reg = XAE_RCW1_OFFSET,
  96		.m_or = XAE_RCW1_VLAN_MASK,
  97	}, { /* Turn on FCS stripping on receive packets */
  98		.opt = XAE_OPTION_FCS_STRIP,
  99		.reg = XAE_RCW1_OFFSET,
 100		.m_or = XAE_RCW1_FCS_MASK,
 101	}, { /* Turn on FCS insertion on transmit packets */
 102		.opt = XAE_OPTION_FCS_INSERT,
 103		.reg = XAE_TC_OFFSET,
 104		.m_or = XAE_TC_FCS_MASK,
 105	}, { /* Turn off length/type field checking on receive packets */
 106		.opt = XAE_OPTION_LENTYPE_ERR,
 107		.reg = XAE_RCW1_OFFSET,
 108		.m_or = XAE_RCW1_LT_DIS_MASK,
 109	}, { /* Turn on Rx flow control */
 110		.opt = XAE_OPTION_FLOW_CONTROL,
 111		.reg = XAE_FCC_OFFSET,
 112		.m_or = XAE_FCC_FCRX_MASK,
 113	}, { /* Turn on Tx flow control */
 114		.opt = XAE_OPTION_FLOW_CONTROL,
 115		.reg = XAE_FCC_OFFSET,
 116		.m_or = XAE_FCC_FCTX_MASK,
 117	}, { /* Turn on promiscuous frame filtering */
 118		.opt = XAE_OPTION_PROMISC,
 119		.reg = XAE_FMI_OFFSET,
 120		.m_or = XAE_FMI_PM_MASK,
 121	}, { /* Enable transmitter */
 122		.opt = XAE_OPTION_TXEN,
 123		.reg = XAE_TC_OFFSET,
 124		.m_or = XAE_TC_TX_MASK,
 125	}, { /* Enable receiver */
 126		.opt = XAE_OPTION_RXEN,
 127		.reg = XAE_RCW1_OFFSET,
 128		.m_or = XAE_RCW1_RX_MASK,
 129	},
 130	{}
 131};
 132
 133static struct skbuf_dma_descriptor *axienet_get_rx_desc(struct axienet_local *lp, int i)
 134{
 135	return lp->rx_skb_ring[i & (RX_BUF_NUM_DEFAULT - 1)];
 136}
 137
 138static struct skbuf_dma_descriptor *axienet_get_tx_desc(struct axienet_local *lp, int i)
 139{
 140	return lp->tx_skb_ring[i & (TX_BD_NUM_MAX - 1)];
 141}
 142
 143/**
 144 * axienet_dma_in32 - Memory mapped Axi DMA register read
 145 * @lp:		Pointer to axienet local structure
 146 * @reg:	Address offset from the base address of the Axi DMA core
 147 *
 148 * Return: The contents of the Axi DMA register
 149 *
 150 * This function returns the contents of the corresponding Axi DMA register.
 151 */
 152static inline u32 axienet_dma_in32(struct axienet_local *lp, off_t reg)
 153{
 154	return ioread32(lp->dma_regs + reg);
 155}
 156
 157static void desc_set_phys_addr(struct axienet_local *lp, dma_addr_t addr,
 158			       struct axidma_bd *desc)
 159{
 160	desc->phys = lower_32_bits(addr);
 161	if (lp->features & XAE_FEATURE_DMA_64BIT)
 162		desc->phys_msb = upper_32_bits(addr);
 163}
 164
 165static dma_addr_t desc_get_phys_addr(struct axienet_local *lp,
 166				     struct axidma_bd *desc)
 167{
 168	dma_addr_t ret = desc->phys;
 169
 170	if (lp->features & XAE_FEATURE_DMA_64BIT)
 171		ret |= ((dma_addr_t)desc->phys_msb << 16) << 16;
 172
 173	return ret;
 174}
 175
 176/**
 177 * axienet_dma_bd_release - Release buffer descriptor rings
 178 * @ndev:	Pointer to the net_device structure
 179 *
 180 * This function is used to release the descriptors allocated in
 181 * axienet_dma_bd_init. axienet_dma_bd_release is called when Axi Ethernet
 182 * driver stop api is called.
 183 */
 184static void axienet_dma_bd_release(struct net_device *ndev)
 185{
 186	int i;
 187	struct axienet_local *lp = netdev_priv(ndev);
 188
 189	/* If we end up here, tx_bd_v must have been DMA allocated. */
 190	dma_free_coherent(lp->dev,
 191			  sizeof(*lp->tx_bd_v) * lp->tx_bd_num,
 192			  lp->tx_bd_v,
 193			  lp->tx_bd_p);
 194
 195	if (!lp->rx_bd_v)
 196		return;
 197
 198	for (i = 0; i < lp->rx_bd_num; i++) {
 199		dma_addr_t phys;
 200
 201		/* A NULL skb means this descriptor has not been initialised
 202		 * at all.
 203		 */
 204		if (!lp->rx_bd_v[i].skb)
 205			break;
 206
 207		dev_kfree_skb(lp->rx_bd_v[i].skb);
 208
 209		/* For each descriptor, we programmed cntrl with the (non-zero)
 210		 * descriptor size, after it had been successfully allocated.
 211		 * So a non-zero value in there means we need to unmap it.
 212		 */
 213		if (lp->rx_bd_v[i].cntrl) {
 214			phys = desc_get_phys_addr(lp, &lp->rx_bd_v[i]);
 215			dma_unmap_single(lp->dev, phys,
 216					 lp->max_frm_size, DMA_FROM_DEVICE);
 217		}
 218	}
 219
 220	dma_free_coherent(lp->dev,
 221			  sizeof(*lp->rx_bd_v) * lp->rx_bd_num,
 222			  lp->rx_bd_v,
 223			  lp->rx_bd_p);
 224}
 225
 226static u64 axienet_dma_rate(struct axienet_local *lp)
 227{
 228	if (lp->axi_clk)
 229		return clk_get_rate(lp->axi_clk);
 230	return 125000000; /* arbitrary guess if no clock rate set */
 231}
 232
 233/**
 234 * axienet_calc_cr() - Calculate control register value
 235 * @lp: Device private data
 236 * @count: Number of completions before an interrupt
 237 * @usec: Microseconds after the last completion before an interrupt
 238 *
 239 * Calculate a control register value based on the coalescing settings. The
 240 * run/stop bit is not set.
 241 *
 242 * Return: Control register value with coalescing settings configured.
 243 */
 244static u32 axienet_calc_cr(struct axienet_local *lp, u32 count, u32 usec)
 245{
 246	u32 cr;
 247
 248	cr = FIELD_PREP(XAXIDMA_COALESCE_MASK, count) | XAXIDMA_IRQ_IOC_MASK |
 249	     XAXIDMA_IRQ_ERROR_MASK;
 250	/* Only set interrupt delay timer if not generating an interrupt on
 251	 * the first packet. Otherwise leave at 0 to disable delay interrupt.
 252	 */
 253	if (count > 1) {
 254		u64 clk_rate = axienet_dma_rate(lp);
 255		u32 timer;
 256
 257		/* 1 Timeout Interval = 125 * (clock period of SG clock) */
 258		timer = DIV64_U64_ROUND_CLOSEST((u64)usec * clk_rate,
 259						XAXIDMA_DELAY_SCALE);
 260
 261		timer = min(timer, FIELD_MAX(XAXIDMA_DELAY_MASK));
 262		cr |= FIELD_PREP(XAXIDMA_DELAY_MASK, timer) |
 263		      XAXIDMA_IRQ_DELAY_MASK;
 264	}
 265
 266	return cr;
 267}
 268
 269/**
 270 * axienet_coalesce_params() - Extract coalesce parameters from the CR
 271 * @lp: Device private data
 272 * @cr: The control register to parse
 273 * @count: Number of packets before an interrupt
 274 * @usec: Idle time (in usec) before an interrupt
 275 */
 276static void axienet_coalesce_params(struct axienet_local *lp, u32 cr,
 277				    u32 *count, u32 *usec)
 278{
 279	u64 clk_rate = axienet_dma_rate(lp);
 280	u64 timer = FIELD_GET(XAXIDMA_DELAY_MASK, cr);
 281
 282	*count = FIELD_GET(XAXIDMA_COALESCE_MASK, cr);
 283	*usec = DIV64_U64_ROUND_CLOSEST(timer * XAXIDMA_DELAY_SCALE, clk_rate);
 284}
 285
 286/**
 287 * axienet_dma_start - Set up DMA registers and start DMA operation
 288 * @lp:		Pointer to the axienet_local structure
 289 */
 290static void axienet_dma_start(struct axienet_local *lp)
 291{
 292	spin_lock_irq(&lp->rx_cr_lock);
 293
 294	/* Start updating the Rx channel control register */
 295	lp->rx_dma_cr &= ~XAXIDMA_CR_RUNSTOP_MASK;
 296	axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr);
 297
 298	/* Populate the tail pointer and bring the Rx Axi DMA engine out of
 299	 * halted state. This will make the Rx side ready for reception.
 300	 */
 301	axienet_dma_out_addr(lp, XAXIDMA_RX_CDESC_OFFSET, lp->rx_bd_p);
 302	lp->rx_dma_cr |= XAXIDMA_CR_RUNSTOP_MASK;
 303	axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr);
 304	axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, lp->rx_bd_p +
 305			     (sizeof(*lp->rx_bd_v) * (lp->rx_bd_num - 1)));
 306	lp->rx_dma_started = true;
 307
 308	spin_unlock_irq(&lp->rx_cr_lock);
 309	spin_lock_irq(&lp->tx_cr_lock);
 310
 311	/* Start updating the Tx channel control register */
 312	lp->tx_dma_cr &= ~XAXIDMA_CR_RUNSTOP_MASK;
 313	axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr);
 314
 315	/* Write to the RS (Run-stop) bit in the Tx channel control register.
 316	 * Tx channel is now ready to run. But only after we write to the
 317	 * tail pointer register that the Tx channel will start transmitting.
 318	 */
 319	axienet_dma_out_addr(lp, XAXIDMA_TX_CDESC_OFFSET, lp->tx_bd_p);
 320	lp->tx_dma_cr |= XAXIDMA_CR_RUNSTOP_MASK;
 321	axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr);
 322	lp->tx_dma_started = true;
 323
 324	spin_unlock_irq(&lp->tx_cr_lock);
 325}
 326
 327/**
 328 * axienet_dma_bd_init - Setup buffer descriptor rings for Axi DMA
 329 * @ndev:	Pointer to the net_device structure
 330 *
 331 * Return: 0, on success -ENOMEM, on failure
 332 *
 333 * This function is called to initialize the Rx and Tx DMA descriptor
 334 * rings. This initializes the descriptors with required default values
 335 * and is called when Axi Ethernet driver reset is called.
 336 */
 337static int axienet_dma_bd_init(struct net_device *ndev)
 338{
 339	int i;
 340	struct sk_buff *skb;
 341	struct axienet_local *lp = netdev_priv(ndev);
 342
 343	/* Reset the indexes which are used for accessing the BDs */
 344	lp->tx_bd_ci = 0;
 345	lp->tx_bd_tail = 0;
 346	lp->rx_bd_ci = 0;
 347
 348	/* Allocate the Tx and Rx buffer descriptors. */
 349	lp->tx_bd_v = dma_alloc_coherent(lp->dev,
 350					 sizeof(*lp->tx_bd_v) * lp->tx_bd_num,
 351					 &lp->tx_bd_p, GFP_KERNEL);
 352	if (!lp->tx_bd_v)
 353		return -ENOMEM;
 354
 355	lp->rx_bd_v = dma_alloc_coherent(lp->dev,
 356					 sizeof(*lp->rx_bd_v) * lp->rx_bd_num,
 357					 &lp->rx_bd_p, GFP_KERNEL);
 358	if (!lp->rx_bd_v)
 359		goto out;
 360
 361	for (i = 0; i < lp->tx_bd_num; i++) {
 362		dma_addr_t addr = lp->tx_bd_p +
 363				  sizeof(*lp->tx_bd_v) *
 364				  ((i + 1) % lp->tx_bd_num);
 365
 366		lp->tx_bd_v[i].next = lower_32_bits(addr);
 367		if (lp->features & XAE_FEATURE_DMA_64BIT)
 368			lp->tx_bd_v[i].next_msb = upper_32_bits(addr);
 369	}
 370
 371	for (i = 0; i < lp->rx_bd_num; i++) {
 372		dma_addr_t addr;
 373
 374		addr = lp->rx_bd_p + sizeof(*lp->rx_bd_v) *
 375			((i + 1) % lp->rx_bd_num);
 376		lp->rx_bd_v[i].next = lower_32_bits(addr);
 377		if (lp->features & XAE_FEATURE_DMA_64BIT)
 378			lp->rx_bd_v[i].next_msb = upper_32_bits(addr);
 379
 380		skb = netdev_alloc_skb_ip_align(ndev, lp->max_frm_size);
 381		if (!skb)
 382			goto out;
 383
 384		lp->rx_bd_v[i].skb = skb;
 385		addr = dma_map_single(lp->dev, skb->data,
 386				      lp->max_frm_size, DMA_FROM_DEVICE);
 387		if (dma_mapping_error(lp->dev, addr)) {
 388			netdev_err(ndev, "DMA mapping error\n");
 389			goto out;
 390		}
 391		desc_set_phys_addr(lp, addr, &lp->rx_bd_v[i]);
 392
 393		lp->rx_bd_v[i].cntrl = lp->max_frm_size;
 394	}
 395
 396	axienet_dma_start(lp);
 397
 398	return 0;
 399out:
 400	axienet_dma_bd_release(ndev);
 401	return -ENOMEM;
 402}
 403
 404/**
 405 * axienet_set_mac_address - Write the MAC address
 406 * @ndev:	Pointer to the net_device structure
 407 * @address:	6 byte Address to be written as MAC address
 408 *
 409 * This function is called to initialize the MAC address of the Axi Ethernet
 410 * core. It writes to the UAW0 and UAW1 registers of the core.
 411 */
 412static void axienet_set_mac_address(struct net_device *ndev,
 413				    const void *address)
 414{
 415	struct axienet_local *lp = netdev_priv(ndev);
 416
 417	if (address)
 418		eth_hw_addr_set(ndev, address);
 419	if (!is_valid_ether_addr(ndev->dev_addr))
 420		eth_hw_addr_random(ndev);
 421
 422	/* Set up unicast MAC address filter set its mac address */
 423	axienet_iow(lp, XAE_UAW0_OFFSET,
 424		    (ndev->dev_addr[0]) |
 425		    (ndev->dev_addr[1] << 8) |
 426		    (ndev->dev_addr[2] << 16) |
 427		    (ndev->dev_addr[3] << 24));
 428	axienet_iow(lp, XAE_UAW1_OFFSET,
 429		    (((axienet_ior(lp, XAE_UAW1_OFFSET)) &
 430		      ~XAE_UAW1_UNICASTADDR_MASK) |
 431		     (ndev->dev_addr[4] |
 432		     (ndev->dev_addr[5] << 8))));
 433}
 434
 435/**
 436 * netdev_set_mac_address - Write the MAC address (from outside the driver)
 437 * @ndev:	Pointer to the net_device structure
 438 * @p:		6 byte Address to be written as MAC address
 439 *
 440 * Return: 0 for all conditions. Presently, there is no failure case.
 441 *
 442 * This function is called to initialize the MAC address of the Axi Ethernet
 443 * core. It calls the core specific axienet_set_mac_address. This is the
 444 * function that goes into net_device_ops structure entry ndo_set_mac_address.
 445 */
 446static int netdev_set_mac_address(struct net_device *ndev, void *p)
 447{
 448	struct sockaddr *addr = p;
 449
 450	axienet_set_mac_address(ndev, addr->sa_data);
 451	return 0;
 452}
 453
 454/**
 455 * axienet_set_multicast_list - Prepare the multicast table
 456 * @ndev:	Pointer to the net_device structure
 457 *
 458 * This function is called to initialize the multicast table during
 459 * initialization. The Axi Ethernet basic multicast support has a four-entry
 460 * multicast table which is initialized here. Additionally this function
 461 * goes into the net_device_ops structure entry ndo_set_multicast_list. This
 462 * means whenever the multicast table entries need to be updated this
 463 * function gets called.
 464 */
 465static void axienet_set_multicast_list(struct net_device *ndev)
 466{
 467	int i = 0;
 468	u32 reg, af0reg, af1reg;
 469	struct axienet_local *lp = netdev_priv(ndev);
 470
 471	reg = axienet_ior(lp, XAE_FMI_OFFSET);
 472	reg &= ~XAE_FMI_PM_MASK;
 473	if (ndev->flags & IFF_PROMISC)
 474		reg |= XAE_FMI_PM_MASK;
 475	else
 476		reg &= ~XAE_FMI_PM_MASK;
 477	axienet_iow(lp, XAE_FMI_OFFSET, reg);
 478
 479	if (ndev->flags & IFF_ALLMULTI ||
 480	    netdev_mc_count(ndev) > XAE_MULTICAST_CAM_TABLE_NUM) {
 481		reg &= 0xFFFFFF00;
 482		axienet_iow(lp, XAE_FMI_OFFSET, reg);
 483		axienet_iow(lp, XAE_AF0_OFFSET, 1); /* Multicast bit */
 484		axienet_iow(lp, XAE_AF1_OFFSET, 0);
 485		axienet_iow(lp, XAE_AM0_OFFSET, 1); /* ditto */
 486		axienet_iow(lp, XAE_AM1_OFFSET, 0);
 487		axienet_iow(lp, XAE_FFE_OFFSET, 1);
 488		i = 1;
 489	} else if (!netdev_mc_empty(ndev)) {
 490		struct netdev_hw_addr *ha;
 491
 492		netdev_for_each_mc_addr(ha, ndev) {
 493			if (i >= XAE_MULTICAST_CAM_TABLE_NUM)
 494				break;
 495
 496			af0reg = (ha->addr[0]);
 497			af0reg |= (ha->addr[1] << 8);
 498			af0reg |= (ha->addr[2] << 16);
 499			af0reg |= (ha->addr[3] << 24);
 500
 501			af1reg = (ha->addr[4]);
 502			af1reg |= (ha->addr[5] << 8);
 503
 504			reg &= 0xFFFFFF00;
 505			reg |= i;
 506
 507			axienet_iow(lp, XAE_FMI_OFFSET, reg);
 508			axienet_iow(lp, XAE_AF0_OFFSET, af0reg);
 509			axienet_iow(lp, XAE_AF1_OFFSET, af1reg);
 510			axienet_iow(lp, XAE_AM0_OFFSET, 0xffffffff);
 511			axienet_iow(lp, XAE_AM1_OFFSET, 0x0000ffff);
 512			axienet_iow(lp, XAE_FFE_OFFSET, 1);
 513			i++;
 514		}
 515	}
 516
 517	for (; i < XAE_MULTICAST_CAM_TABLE_NUM; i++) {
 518		reg &= 0xFFFFFF00;
 519		reg |= i;
 520		axienet_iow(lp, XAE_FMI_OFFSET, reg);
 521		axienet_iow(lp, XAE_FFE_OFFSET, 0);
 522	}
 523}
 524
 525/**
 526 * axienet_setoptions - Set an Axi Ethernet option
 527 * @ndev:	Pointer to the net_device structure
 528 * @options:	Option to be enabled/disabled
 529 *
 530 * The Axi Ethernet core has multiple features which can be selectively turned
 531 * on or off. The typical options could be jumbo frame option, basic VLAN
 532 * option, promiscuous mode option etc. This function is used to set or clear
 533 * these options in the Axi Ethernet hardware. This is done through
 534 * axienet_option structure .
 535 */
 536static void axienet_setoptions(struct net_device *ndev, u32 options)
 537{
 538	int reg;
 539	struct axienet_local *lp = netdev_priv(ndev);
 540	struct axienet_option *tp = &axienet_options[0];
 541
 542	while (tp->opt) {
 543		reg = ((axienet_ior(lp, tp->reg)) & ~(tp->m_or));
 544		if (options & tp->opt)
 545			reg |= tp->m_or;
 546		axienet_iow(lp, tp->reg, reg);
 547		tp++;
 548	}
 549
 550	lp->options |= options;
 551}
 552
 553static u64 axienet_stat(struct axienet_local *lp, enum temac_stat stat)
 554{
 555	u32 counter;
 556
 557	if (lp->reset_in_progress)
 558		return lp->hw_stat_base[stat];
 559
 560	counter = axienet_ior(lp, XAE_STATS_OFFSET + stat * 8);
 561	return lp->hw_stat_base[stat] + (counter - lp->hw_last_counter[stat]);
 562}
 563
 564static void axienet_stats_update(struct axienet_local *lp, bool reset)
 565{
 566	enum temac_stat stat;
 567
 568	write_seqcount_begin(&lp->hw_stats_seqcount);
 569	lp->reset_in_progress = reset;
 570	for (stat = 0; stat < STAT_COUNT; stat++) {
 571		u32 counter = axienet_ior(lp, XAE_STATS_OFFSET + stat * 8);
 572
 573		lp->hw_stat_base[stat] += counter - lp->hw_last_counter[stat];
 574		lp->hw_last_counter[stat] = counter;
 575	}
 576	write_seqcount_end(&lp->hw_stats_seqcount);
 577}
 578
 579static void axienet_refresh_stats(struct work_struct *work)
 580{
 581	struct axienet_local *lp = container_of(work, struct axienet_local,
 582						stats_work.work);
 583
 584	mutex_lock(&lp->stats_lock);
 585	axienet_stats_update(lp, false);
 586	mutex_unlock(&lp->stats_lock);
 587
 588	/* Just less than 2^32 bytes at 2.5 GBit/s */
 589	schedule_delayed_work(&lp->stats_work, 13 * HZ);
 590}
 591
 592static int __axienet_device_reset(struct axienet_local *lp)
 593{
 594	u32 value;
 595	int ret;
 596
 597	/* Save statistics counters in case they will be reset */
 598	mutex_lock(&lp->stats_lock);
 599	if (lp->features & XAE_FEATURE_STATS)
 600		axienet_stats_update(lp, true);
 601
 602	/* Reset Axi DMA. This would reset Axi Ethernet core as well. The reset
 603	 * process of Axi DMA takes a while to complete as all pending
 604	 * commands/transfers will be flushed or completed during this
 605	 * reset process.
 606	 * Note that even though both TX and RX have their own reset register,
 607	 * they both reset the entire DMA core, so only one needs to be used.
 608	 */
 609	axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, XAXIDMA_CR_RESET_MASK);
 610	ret = read_poll_timeout(axienet_dma_in32, value,
 611				!(value & XAXIDMA_CR_RESET_MASK),
 612				DELAY_OF_ONE_MILLISEC, 50000, false, lp,
 613				XAXIDMA_TX_CR_OFFSET);
 614	if (ret) {
 615		dev_err(lp->dev, "%s: DMA reset timeout!\n", __func__);
 616		goto out;
 617	}
 618
 619	/* Wait for PhyRstCmplt bit to be set, indicating the PHY reset has finished */
 620	ret = read_poll_timeout(axienet_ior, value,
 621				value & XAE_INT_PHYRSTCMPLT_MASK,
 622				DELAY_OF_ONE_MILLISEC, 50000, false, lp,
 623				XAE_IS_OFFSET);
 624	if (ret) {
 625		dev_err(lp->dev, "%s: timeout waiting for PhyRstCmplt\n", __func__);
 626		goto out;
 627	}
 628
 629	/* Update statistics counters with new values */
 630	if (lp->features & XAE_FEATURE_STATS) {
 631		enum temac_stat stat;
 632
 633		write_seqcount_begin(&lp->hw_stats_seqcount);
 634		lp->reset_in_progress = false;
 635		for (stat = 0; stat < STAT_COUNT; stat++) {
 636			u32 counter =
 637				axienet_ior(lp, XAE_STATS_OFFSET + stat * 8);
 638
 639			lp->hw_stat_base[stat] +=
 640				lp->hw_last_counter[stat] - counter;
 641			lp->hw_last_counter[stat] = counter;
 642		}
 643		write_seqcount_end(&lp->hw_stats_seqcount);
 644	}
 645
 646out:
 647	mutex_unlock(&lp->stats_lock);
 648	return ret;
 649}
 650
 651/**
 652 * axienet_dma_stop - Stop DMA operation
 653 * @lp:		Pointer to the axienet_local structure
 654 */
 655static void axienet_dma_stop(struct axienet_local *lp)
 656{
 657	int count;
 658	u32 cr, sr;
 659
 660	spin_lock_irq(&lp->rx_cr_lock);
 661
 662	cr = lp->rx_dma_cr & ~(XAXIDMA_CR_RUNSTOP_MASK | XAXIDMA_IRQ_ALL_MASK);
 663	axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr);
 664	lp->rx_dma_started = false;
 665
 666	spin_unlock_irq(&lp->rx_cr_lock);
 667	synchronize_irq(lp->rx_irq);
 668
 669	spin_lock_irq(&lp->tx_cr_lock);
 670
 671	cr = lp->tx_dma_cr & ~(XAXIDMA_CR_RUNSTOP_MASK | XAXIDMA_IRQ_ALL_MASK);
 672	axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, cr);
 673	lp->tx_dma_started = false;
 674
 675	spin_unlock_irq(&lp->tx_cr_lock);
 676	synchronize_irq(lp->tx_irq);
 677
 678	/* Give DMAs a chance to halt gracefully */
 679	sr = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET);
 680	for (count = 0; !(sr & XAXIDMA_SR_HALT_MASK) && count < 5; ++count) {
 681		msleep(20);
 682		sr = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET);
 683	}
 684
 685	sr = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET);
 686	for (count = 0; !(sr & XAXIDMA_SR_HALT_MASK) && count < 5; ++count) {
 687		msleep(20);
 688		sr = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET);
 689	}
 690
 691	/* Do a reset to ensure DMA is really stopped */
 692	axienet_lock_mii(lp);
 693	__axienet_device_reset(lp);
 694	axienet_unlock_mii(lp);
 695}
 696
 697/**
 698 * axienet_device_reset - Reset and initialize the Axi Ethernet hardware.
 699 * @ndev:	Pointer to the net_device structure
 700 *
 701 * This function is called to reset and initialize the Axi Ethernet core. This
 702 * is typically called during initialization. It does a reset of the Axi DMA
 703 * Rx/Tx channels and initializes the Axi DMA BDs. Since Axi DMA reset lines
 704 * are connected to Axi Ethernet reset lines, this in turn resets the Axi
 705 * Ethernet core. No separate hardware reset is done for the Axi Ethernet
 706 * core.
 707 *
 708 * Return: 0 on success or a negative error number otherwise.
 709 */
 710static int axienet_device_reset(struct net_device *ndev)
 711{
 712	u32 axienet_status;
 713	struct axienet_local *lp = netdev_priv(ndev);
 714	int ret;
 715
 716	lp->max_frm_size = XAE_MAX_VLAN_FRAME_SIZE;
 717	lp->options |= XAE_OPTION_VLAN;
 718	lp->options &= (~XAE_OPTION_JUMBO);
 719
 720	if (ndev->mtu > XAE_MTU && ndev->mtu <= XAE_JUMBO_MTU) {
 721		lp->max_frm_size = ndev->mtu + VLAN_ETH_HLEN +
 722					XAE_TRL_SIZE;
 723
 724		if (lp->max_frm_size <= lp->rxmem)
 725			lp->options |= XAE_OPTION_JUMBO;
 726	}
 727
 728	if (!lp->use_dmaengine) {
 729		ret = __axienet_device_reset(lp);
 730		if (ret)
 731			return ret;
 732
 733		ret = axienet_dma_bd_init(ndev);
 734		if (ret) {
 735			netdev_err(ndev, "%s: descriptor allocation failed\n",
 736				   __func__);
 737			return ret;
 738		}
 739	}
 740
 741	axienet_status = axienet_ior(lp, XAE_RCW1_OFFSET);
 742	axienet_status &= ~XAE_RCW1_RX_MASK;
 743	axienet_iow(lp, XAE_RCW1_OFFSET, axienet_status);
 744
 745	axienet_status = axienet_ior(lp, XAE_IP_OFFSET);
 746	if (axienet_status & XAE_INT_RXRJECT_MASK)
 747		axienet_iow(lp, XAE_IS_OFFSET, XAE_INT_RXRJECT_MASK);
 748	axienet_iow(lp, XAE_IE_OFFSET, lp->eth_irq > 0 ?
 749		    XAE_INT_RECV_ERROR_MASK : 0);
 750
 751	axienet_iow(lp, XAE_FCC_OFFSET, XAE_FCC_FCRX_MASK);
 752
 753	/* Sync default options with HW but leave receiver and
 754	 * transmitter disabled.
 755	 */
 756	axienet_setoptions(ndev, lp->options &
 757			   ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN));
 758	axienet_set_mac_address(ndev, NULL);
 759	axienet_set_multicast_list(ndev);
 760	axienet_setoptions(ndev, lp->options);
 761
 762	netif_trans_update(ndev);
 763
 764	return 0;
 765}
 766
 767/**
 768 * axienet_free_tx_chain - Clean up a series of linked TX descriptors.
 769 * @lp:		Pointer to the axienet_local structure
 770 * @first_bd:	Index of first descriptor to clean up
 771 * @nr_bds:	Max number of descriptors to clean up
 772 * @force:	Whether to clean descriptors even if not complete
 773 * @sizep:	Pointer to a u32 accumulating the total byte count of
 774 *		completed packets (using skb->len). Ignored if NULL.
 775 * @budget:	NAPI budget (use 0 when not called from NAPI poll)
 776 *
 777 * Would either be called after a successful transmit operation, or after
 778 * there was an error when setting up the chain.
 779 *
 780 * Return: The number of packets handled.
 781 */
 782static int axienet_free_tx_chain(struct axienet_local *lp, u32 first_bd,
 783				 int nr_bds, bool force, u32 *sizep, int budget)
 784{
 785	struct axidma_bd *cur_p;
 786	unsigned int status;
 787	int i, packets = 0;
 788	dma_addr_t phys;
 789
 790	for (i = 0; i < nr_bds; i++) {
 791		cur_p = &lp->tx_bd_v[(first_bd + i) % lp->tx_bd_num];
 792		status = cur_p->status;
 793
 794		/* If force is not specified, clean up only descriptors
 795		 * that have been completed by the MAC.
 796		 */
 797		if (!force && !(status & XAXIDMA_BD_STS_COMPLETE_MASK))
 798			break;
 799
 800		/* Ensure we see complete descriptor update */
 801		dma_rmb();
 802		phys = desc_get_phys_addr(lp, cur_p);
 803		dma_unmap_single(lp->dev, phys,
 804				 (cur_p->cntrl & XAXIDMA_BD_CTRL_LENGTH_MASK),
 805				 DMA_TO_DEVICE);
 806
 807		if (cur_p->skb && (status & XAXIDMA_BD_STS_COMPLETE_MASK)) {
 808			if (sizep)
 809				*sizep += cur_p->skb->len;
 810			napi_consume_skb(cur_p->skb, budget);
 811			packets++;
 812		}
 813
 814		cur_p->app0 = 0;
 815		cur_p->app1 = 0;
 816		cur_p->app2 = 0;
 817		cur_p->app4 = 0;
 818		cur_p->skb = NULL;
 819		/* ensure our transmit path and device don't prematurely see status cleared */
 820		wmb();
 821		cur_p->cntrl = 0;
 822		cur_p->status = 0;
 823	}
 824
 825	if (!force) {
 826		lp->tx_bd_ci += i;
 827		if (lp->tx_bd_ci >= lp->tx_bd_num)
 828			lp->tx_bd_ci %= lp->tx_bd_num;
 829	}
 830
 831	return packets;
 832}
 833
 834/**
 835 * axienet_check_tx_bd_space - Checks if a BD/group of BDs are currently busy
 836 * @lp:		Pointer to the axienet_local structure
 837 * @num_frag:	The number of BDs to check for
 838 *
 839 * Return: 0, on success
 840 *	    NETDEV_TX_BUSY, if any of the descriptors are not free
 841 *
 842 * This function is invoked before BDs are allocated and transmission starts.
 843 * This function returns 0 if a BD or group of BDs can be allocated for
 844 * transmission. If the BD or any of the BDs are not free the function
 845 * returns a busy status.
 846 */
 847static inline int axienet_check_tx_bd_space(struct axienet_local *lp,
 848					    int num_frag)
 849{
 850	struct axidma_bd *cur_p;
 851
 852	/* Ensure we see all descriptor updates from device or TX polling */
 853	rmb();
 854	cur_p = &lp->tx_bd_v[(READ_ONCE(lp->tx_bd_tail) + num_frag) %
 855			     lp->tx_bd_num];
 856	if (cur_p->cntrl)
 857		return NETDEV_TX_BUSY;
 858	return 0;
 859}
 860
 861/**
 862 * axienet_dma_tx_cb - DMA engine callback for TX channel.
 863 * @data:       Pointer to the axienet_local structure.
 864 * @result:     error reporting through dmaengine_result.
 865 * This function is called by dmaengine driver for TX channel to notify
 866 * that the transmit is done.
 867 */
 868static void axienet_dma_tx_cb(void *data, const struct dmaengine_result *result)
 869{
 870	struct skbuf_dma_descriptor *skbuf_dma;
 871	struct axienet_local *lp = data;
 872	struct netdev_queue *txq;
 873	int len;
 874
 875	skbuf_dma = axienet_get_tx_desc(lp, lp->tx_ring_tail++);
 876	len = skbuf_dma->skb->len;
 877	txq = skb_get_tx_queue(lp->ndev, skbuf_dma->skb);
 878	u64_stats_update_begin(&lp->tx_stat_sync);
 879	u64_stats_add(&lp->tx_bytes, len);
 880	u64_stats_add(&lp->tx_packets, 1);
 881	u64_stats_update_end(&lp->tx_stat_sync);
 882	dma_unmap_sg(lp->dev, skbuf_dma->sgl, skbuf_dma->sg_len, DMA_TO_DEVICE);
 883	dev_consume_skb_any(skbuf_dma->skb);
 884	netif_txq_completed_wake(txq, 1, len,
 885				 CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX),
 886				 2);
 887}
 888
 889/**
 890 * axienet_start_xmit_dmaengine - Starts the transmission.
 891 * @skb:        sk_buff pointer that contains data to be Txed.
 892 * @ndev:       Pointer to net_device structure.
 893 *
 894 * Return: NETDEV_TX_OK on success or any non space errors.
 895 *         NETDEV_TX_BUSY when free element in TX skb ring buffer
 896 *         is not available.
 897 *
 898 * This function is invoked to initiate transmission. The
 899 * function sets the skbs, register dma callback API and submit
 900 * the dma transaction.
 901 * Additionally if checksum offloading is supported,
 902 * it populates AXI Stream Control fields with appropriate values.
 903 */
 904static netdev_tx_t
 905axienet_start_xmit_dmaengine(struct sk_buff *skb, struct net_device *ndev)
 906{
 907	struct dma_async_tx_descriptor *dma_tx_desc = NULL;
 908	struct axienet_local *lp = netdev_priv(ndev);
 909	u32 app_metadata[DMA_NUM_APP_WORDS] = {0};
 910	struct skbuf_dma_descriptor *skbuf_dma;
 911	struct dma_device *dma_dev;
 912	struct netdev_queue *txq;
 913	u32 csum_start_off;
 914	u32 csum_index_off;
 915	int sg_len;
 916	int ret;
 917
 918	dma_dev = lp->tx_chan->device;
 919	sg_len = skb_shinfo(skb)->nr_frags + 1;
 920	if (CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX) <= 1) {
 921		netif_stop_queue(ndev);
 922		if (net_ratelimit())
 923			netdev_warn(ndev, "TX ring unexpectedly full\n");
 924		return NETDEV_TX_BUSY;
 925	}
 926
 927	skbuf_dma = axienet_get_tx_desc(lp, lp->tx_ring_head);
 928	if (!skbuf_dma)
 929		goto xmit_error_drop_skb;
 930
 931	lp->tx_ring_head++;
 932	sg_init_table(skbuf_dma->sgl, sg_len);
 933	ret = skb_to_sgvec(skb, skbuf_dma->sgl, 0, skb->len);
 934	if (ret < 0)
 935		goto xmit_error_drop_skb;
 936
 937	ret = dma_map_sg(lp->dev, skbuf_dma->sgl, sg_len, DMA_TO_DEVICE);
 938	if (!ret)
 939		goto xmit_error_drop_skb;
 940
 941	/* Fill up app fields for checksum */
 942	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 943		if (lp->features & XAE_FEATURE_FULL_TX_CSUM) {
 944			/* Tx Full Checksum Offload Enabled */
 945			app_metadata[0] |= 2;
 946		} else if (lp->features & XAE_FEATURE_PARTIAL_TX_CSUM) {
 947			csum_start_off = skb_transport_offset(skb);
 948			csum_index_off = csum_start_off + skb->csum_offset;
 949			/* Tx Partial Checksum Offload Enabled */
 950			app_metadata[0] |= 1;
 951			app_metadata[1] = (csum_start_off << 16) | csum_index_off;
 952		}
 953	} else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
 954		app_metadata[0] |= 2; /* Tx Full Checksum Offload Enabled */
 955	}
 956
 957	dma_tx_desc = dma_dev->device_prep_slave_sg(lp->tx_chan, skbuf_dma->sgl,
 958			sg_len, DMA_MEM_TO_DEV,
 959			DMA_PREP_INTERRUPT, (void *)app_metadata);
 960	if (!dma_tx_desc)
 961		goto xmit_error_unmap_sg;
 962
 963	skbuf_dma->skb = skb;
 964	skbuf_dma->sg_len = sg_len;
 965	dma_tx_desc->callback_param = lp;
 966	dma_tx_desc->callback_result = axienet_dma_tx_cb;
 967	txq = skb_get_tx_queue(lp->ndev, skb);
 968	netdev_tx_sent_queue(txq, skb->len);
 969	netif_txq_maybe_stop(txq, CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX),
 970			     1, 2);
 971
 972	dmaengine_submit(dma_tx_desc);
 973	dma_async_issue_pending(lp->tx_chan);
 974	return NETDEV_TX_OK;
 975
 976xmit_error_unmap_sg:
 977	dma_unmap_sg(lp->dev, skbuf_dma->sgl, sg_len, DMA_TO_DEVICE);
 978xmit_error_drop_skb:
 979	dev_kfree_skb_any(skb);
 980	return NETDEV_TX_OK;
 981}
 982
 983/**
 984 * axienet_tx_poll - Invoked once a transmit is completed by the
 985 * Axi DMA Tx channel.
 986 * @napi:	Pointer to NAPI structure.
 987 * @budget:	Max number of TX packets to process.
 988 *
 989 * Return: Number of TX packets processed.
 990 *
 991 * This function is invoked from the NAPI processing to notify the completion
 992 * of transmit operation. It clears fields in the corresponding Tx BDs and
 993 * unmaps the corresponding buffer so that CPU can regain ownership of the
 994 * buffer. It finally invokes "netif_wake_queue" to restart transmission if
 995 * required.
 996 */
 997static int axienet_tx_poll(struct napi_struct *napi, int budget)
 998{
 999	struct axienet_local *lp = container_of(napi, struct axienet_local, napi_tx);
1000	struct net_device *ndev = lp->ndev;
1001	u32 size = 0;
1002	int packets;
1003
1004	packets = axienet_free_tx_chain(lp, lp->tx_bd_ci, lp->tx_bd_num, false,
1005					&size, budget);
1006
1007	if (packets) {
1008		netdev_completed_queue(ndev, packets, size);
1009		u64_stats_update_begin(&lp->tx_stat_sync);
1010		u64_stats_add(&lp->tx_packets, packets);
1011		u64_stats_add(&lp->tx_bytes, size);
1012		u64_stats_update_end(&lp->tx_stat_sync);
1013
1014		/* Matches barrier in axienet_start_xmit */
1015		smp_mb();
1016
1017		if (!axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1))
1018			netif_wake_queue(ndev);
1019	}
1020
1021	if (packets < budget && napi_complete_done(napi, packets)) {
1022		/* Re-enable TX completion interrupts. This should
1023		 * cause an immediate interrupt if any TX packets are
1024		 * already pending.
1025		 */
1026		spin_lock_irq(&lp->tx_cr_lock);
1027		axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, lp->tx_dma_cr);
1028		spin_unlock_irq(&lp->tx_cr_lock);
1029	}
1030	return packets;
1031}
1032
1033/**
1034 * axienet_start_xmit - Starts the transmission.
1035 * @skb:	sk_buff pointer that contains data to be Txed.
1036 * @ndev:	Pointer to net_device structure.
1037 *
1038 * Return: NETDEV_TX_OK, on success
1039 *	    NETDEV_TX_BUSY, if any of the descriptors are not free
1040 *
1041 * This function is invoked from upper layers to initiate transmission. The
1042 * function uses the next available free BDs and populates their fields to
1043 * start the transmission. Additionally if checksum offloading is supported,
1044 * it populates AXI Stream Control fields with appropriate values.
1045 */
1046static netdev_tx_t
1047axienet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
1048{
1049	u32 ii;
1050	u32 num_frag;
1051	u32 csum_start_off;
1052	u32 csum_index_off;
1053	skb_frag_t *frag;
1054	dma_addr_t tail_p, phys;
1055	u32 orig_tail_ptr, new_tail_ptr;
1056	struct axienet_local *lp = netdev_priv(ndev);
1057	struct axidma_bd *cur_p;
1058
1059	orig_tail_ptr = lp->tx_bd_tail;
1060	new_tail_ptr = orig_tail_ptr;
1061
1062	num_frag = skb_shinfo(skb)->nr_frags;
1063	cur_p = &lp->tx_bd_v[orig_tail_ptr];
1064
1065	if (axienet_check_tx_bd_space(lp, num_frag + 1)) {
1066		/* Should not happen as last start_xmit call should have
1067		 * checked for sufficient space and queue should only be
1068		 * woken when sufficient space is available.
1069		 */
1070		netif_stop_queue(ndev);
1071		if (net_ratelimit())
1072			netdev_warn(ndev, "TX ring unexpectedly full\n");
1073		return NETDEV_TX_BUSY;
1074	}
1075
1076	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1077		if (lp->features & XAE_FEATURE_FULL_TX_CSUM) {
1078			/* Tx Full Checksum Offload Enabled */
1079			cur_p->app0 |= 2;
1080		} else if (lp->features & XAE_FEATURE_PARTIAL_TX_CSUM) {
1081			csum_start_off = skb_transport_offset(skb);
1082			csum_index_off = csum_start_off + skb->csum_offset;
1083			/* Tx Partial Checksum Offload Enabled */
1084			cur_p->app0 |= 1;
1085			cur_p->app1 = (csum_start_off << 16) | csum_index_off;
1086		}
1087	} else if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
1088		cur_p->app0 |= 2; /* Tx Full Checksum Offload Enabled */
1089	}
1090
1091	phys = dma_map_single(lp->dev, skb->data,
1092			      skb_headlen(skb), DMA_TO_DEVICE);
1093	if (unlikely(dma_mapping_error(lp->dev, phys))) {
1094		if (net_ratelimit())
1095			netdev_err(ndev, "TX DMA mapping error\n");
1096		ndev->stats.tx_dropped++;
1097		dev_kfree_skb_any(skb);
1098		return NETDEV_TX_OK;
1099	}
1100	desc_set_phys_addr(lp, phys, cur_p);
1101	cur_p->cntrl = skb_headlen(skb) | XAXIDMA_BD_CTRL_TXSOF_MASK;
1102
1103	for (ii = 0; ii < num_frag; ii++) {
1104		if (++new_tail_ptr >= lp->tx_bd_num)
1105			new_tail_ptr = 0;
1106		cur_p = &lp->tx_bd_v[new_tail_ptr];
1107		frag = &skb_shinfo(skb)->frags[ii];
1108		phys = dma_map_single(lp->dev,
1109				      skb_frag_address(frag),
1110				      skb_frag_size(frag),
1111				      DMA_TO_DEVICE);
1112		if (unlikely(dma_mapping_error(lp->dev, phys))) {
1113			if (net_ratelimit())
1114				netdev_err(ndev, "TX DMA mapping error\n");
1115			ndev->stats.tx_dropped++;
1116			axienet_free_tx_chain(lp, orig_tail_ptr, ii + 1,
1117					      true, NULL, 0);
1118			dev_kfree_skb_any(skb);
1119			return NETDEV_TX_OK;
1120		}
1121		desc_set_phys_addr(lp, phys, cur_p);
1122		cur_p->cntrl = skb_frag_size(frag);
1123	}
1124
1125	cur_p->cntrl |= XAXIDMA_BD_CTRL_TXEOF_MASK;
1126	cur_p->skb = skb;
1127
1128	tail_p = lp->tx_bd_p + sizeof(*lp->tx_bd_v) * new_tail_ptr;
1129	if (++new_tail_ptr >= lp->tx_bd_num)
1130		new_tail_ptr = 0;
1131	WRITE_ONCE(lp->tx_bd_tail, new_tail_ptr);
1132	netdev_sent_queue(ndev, skb->len);
1133
1134	/* Start the transfer */
1135	axienet_dma_out_addr(lp, XAXIDMA_TX_TDESC_OFFSET, tail_p);
1136
1137	/* Stop queue if next transmit may not have space */
1138	if (axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1)) {
1139		netif_stop_queue(ndev);
1140
1141		/* Matches barrier in axienet_tx_poll */
1142		smp_mb();
1143
1144		/* Space might have just been freed - check again */
1145		if (!axienet_check_tx_bd_space(lp, MAX_SKB_FRAGS + 1))
1146			netif_wake_queue(ndev);
1147	}
1148
1149	return NETDEV_TX_OK;
1150}
1151
1152/**
1153 * axienet_dma_rx_cb - DMA engine callback for RX channel.
1154 * @data:       Pointer to the skbuf_dma_descriptor structure.
1155 * @result:     error reporting through dmaengine_result.
1156 * This function is called by dmaengine driver for RX channel to notify
1157 * that the packet is received.
1158 */
1159static void axienet_dma_rx_cb(void *data, const struct dmaengine_result *result)
1160{
1161	struct skbuf_dma_descriptor *skbuf_dma;
1162	size_t meta_len, meta_max_len, rx_len;
1163	struct axienet_local *lp = data;
1164	struct sk_buff *skb;
1165	u32 *app_metadata;
1166	int i;
1167
1168	skbuf_dma = axienet_get_rx_desc(lp, lp->rx_ring_tail++);
1169	skb = skbuf_dma->skb;
1170	app_metadata = dmaengine_desc_get_metadata_ptr(skbuf_dma->desc, &meta_len,
1171						       &meta_max_len);
1172	dma_unmap_single(lp->dev, skbuf_dma->dma_address, lp->max_frm_size,
1173			 DMA_FROM_DEVICE);
1174
1175	if (IS_ERR(app_metadata)) {
1176		if (net_ratelimit())
1177			netdev_err(lp->ndev, "Failed to get RX metadata pointer\n");
1178		dev_kfree_skb_any(skb);
1179		lp->ndev->stats.rx_dropped++;
1180		goto rx_submit;
1181	}
1182
1183	/* TODO: Derive app word index programmatically */
1184	rx_len = (app_metadata[LEN_APP] & 0xFFFF);
1185	skb_put(skb, rx_len);
1186	skb->protocol = eth_type_trans(skb, lp->ndev);
1187	skb->ip_summed = CHECKSUM_NONE;
1188
1189	__netif_rx(skb);
1190	u64_stats_update_begin(&lp->rx_stat_sync);
1191	u64_stats_add(&lp->rx_packets, 1);
1192	u64_stats_add(&lp->rx_bytes, rx_len);
1193	u64_stats_update_end(&lp->rx_stat_sync);
1194
1195rx_submit:
1196	for (i = 0; i < CIRC_SPACE(lp->rx_ring_head, lp->rx_ring_tail,
1197				   RX_BUF_NUM_DEFAULT); i++)
1198		axienet_rx_submit_desc(lp->ndev);
1199	dma_async_issue_pending(lp->rx_chan);
1200}
1201
1202/**
1203 * axienet_rx_poll - Triggered by RX ISR to complete the BD processing.
1204 * @napi:	Pointer to NAPI structure.
1205 * @budget:	Max number of RX packets to process.
1206 *
1207 * Return: Number of RX packets processed.
1208 */
1209static int axienet_rx_poll(struct napi_struct *napi, int budget)
1210{
1211	u32 length;
1212	u32 csumstatus;
1213	u32 size = 0;
1214	int packets = 0;
1215	dma_addr_t tail_p = 0;
1216	struct axidma_bd *cur_p;
1217	struct sk_buff *skb, *new_skb;
1218	struct axienet_local *lp = container_of(napi, struct axienet_local, napi_rx);
1219
1220	cur_p = &lp->rx_bd_v[lp->rx_bd_ci];
1221
1222	while (packets < budget && (cur_p->status & XAXIDMA_BD_STS_COMPLETE_MASK)) {
1223		dma_addr_t phys;
1224
1225		/* Ensure we see complete descriptor update */
1226		dma_rmb();
1227
1228		skb = cur_p->skb;
1229		cur_p->skb = NULL;
1230
1231		/* skb could be NULL if a previous pass already received the
1232		 * packet for this slot in the ring, but failed to refill it
1233		 * with a newly allocated buffer. In this case, don't try to
1234		 * receive it again.
1235		 */
1236		if (likely(skb)) {
1237			length = cur_p->app4 & 0x0000FFFF;
1238
1239			phys = desc_get_phys_addr(lp, cur_p);
1240			dma_unmap_single(lp->dev, phys, lp->max_frm_size,
1241					 DMA_FROM_DEVICE);
1242
1243			skb_put(skb, length);
1244			skb->protocol = eth_type_trans(skb, lp->ndev);
1245			/*skb_checksum_none_assert(skb);*/
1246			skb->ip_summed = CHECKSUM_NONE;
1247
1248			/* if we're doing Rx csum offload, set it up */
1249			if (lp->features & XAE_FEATURE_FULL_RX_CSUM) {
1250				csumstatus = (cur_p->app2 &
1251					      XAE_FULL_CSUM_STATUS_MASK) >> 3;
1252				if (csumstatus == XAE_IP_TCP_CSUM_VALIDATED ||
1253				    csumstatus == XAE_IP_UDP_CSUM_VALIDATED) {
1254					skb->ip_summed = CHECKSUM_UNNECESSARY;
1255				}
1256			} else if (lp->features & XAE_FEATURE_PARTIAL_RX_CSUM) {
1257				skb->csum = be32_to_cpu(cur_p->app3 & 0xFFFF);
1258				skb->ip_summed = CHECKSUM_COMPLETE;
1259			}
1260
1261			napi_gro_receive(napi, skb);
1262
1263			size += length;
1264			packets++;
1265		}
1266
1267		new_skb = napi_alloc_skb(napi, lp->max_frm_size);
1268		if (!new_skb)
1269			break;
1270
1271		phys = dma_map_single(lp->dev, new_skb->data,
1272				      lp->max_frm_size,
1273				      DMA_FROM_DEVICE);
1274		if (unlikely(dma_mapping_error(lp->dev, phys))) {
1275			if (net_ratelimit())
1276				netdev_err(lp->ndev, "RX DMA mapping error\n");
1277			dev_kfree_skb(new_skb);
1278			break;
1279		}
1280		desc_set_phys_addr(lp, phys, cur_p);
1281
1282		cur_p->cntrl = lp->max_frm_size;
1283		cur_p->status = 0;
1284		cur_p->skb = new_skb;
1285
1286		/* Only update tail_p to mark this slot as usable after it has
1287		 * been successfully refilled.
1288		 */
1289		tail_p = lp->rx_bd_p + sizeof(*lp->rx_bd_v) * lp->rx_bd_ci;
1290
1291		if (++lp->rx_bd_ci >= lp->rx_bd_num)
1292			lp->rx_bd_ci = 0;
1293		cur_p = &lp->rx_bd_v[lp->rx_bd_ci];
1294	}
1295
1296	u64_stats_update_begin(&lp->rx_stat_sync);
1297	u64_stats_add(&lp->rx_packets, packets);
1298	u64_stats_add(&lp->rx_bytes, size);
1299	u64_stats_update_end(&lp->rx_stat_sync);
1300
1301	if (tail_p)
1302		axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, tail_p);
1303
1304	if (packets < budget && napi_complete_done(napi, packets)) {
1305		if (READ_ONCE(lp->rx_dim_enabled)) {
1306			struct dim_sample sample = {
1307				.time = ktime_get(),
1308				/* Safe because we are the only writer */
1309				.pkt_ctr = u64_stats_read(&lp->rx_packets),
1310				.byte_ctr = u64_stats_read(&lp->rx_bytes),
1311				.event_ctr = READ_ONCE(lp->rx_irqs),
1312			};
1313
1314			net_dim(&lp->rx_dim, &sample);
1315		}
1316
1317		/* Re-enable RX completion interrupts. This should
1318		 * cause an immediate interrupt if any RX packets are
1319		 * already pending.
1320		 */
1321		spin_lock_irq(&lp->rx_cr_lock);
1322		axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr);
1323		spin_unlock_irq(&lp->rx_cr_lock);
1324	}
1325	return packets;
1326}
1327
1328/**
1329 * axienet_tx_irq - Tx Done Isr.
1330 * @irq:	irq number
1331 * @_ndev:	net_device pointer
1332 *
1333 * Return: IRQ_HANDLED if device generated a TX interrupt, IRQ_NONE otherwise.
1334 *
1335 * This is the Axi DMA Tx done Isr. It invokes NAPI polling to complete the
1336 * TX BD processing.
1337 */
1338static irqreturn_t axienet_tx_irq(int irq, void *_ndev)
1339{
1340	unsigned int status;
1341	struct net_device *ndev = _ndev;
1342	struct axienet_local *lp = netdev_priv(ndev);
1343
1344	status = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET);
1345
1346	if (!(status & XAXIDMA_IRQ_ALL_MASK))
1347		return IRQ_NONE;
1348
1349	axienet_dma_out32(lp, XAXIDMA_TX_SR_OFFSET, status);
1350
1351	if (unlikely(status & XAXIDMA_IRQ_ERROR_MASK)) {
1352		netdev_err(ndev, "DMA Tx error 0x%x\n", status);
1353		netdev_err(ndev, "Current BD is at: 0x%x%08x\n",
1354			   (lp->tx_bd_v[lp->tx_bd_ci]).phys_msb,
1355			   (lp->tx_bd_v[lp->tx_bd_ci]).phys);
1356		schedule_work(&lp->dma_err_task);
1357	} else {
1358		/* Disable further TX completion interrupts and schedule
1359		 * NAPI to handle the completions.
1360		 */
1361		if (napi_schedule_prep(&lp->napi_tx)) {
1362			u32 cr;
1363
1364			spin_lock(&lp->tx_cr_lock);
1365			cr = lp->tx_dma_cr;
1366			cr &= ~(XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_DELAY_MASK);
1367			axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, cr);
1368			spin_unlock(&lp->tx_cr_lock);
1369			__napi_schedule(&lp->napi_tx);
1370		}
1371	}
1372
1373	return IRQ_HANDLED;
1374}
1375
1376/**
1377 * axienet_rx_irq - Rx Isr.
1378 * @irq:	irq number
1379 * @_ndev:	net_device pointer
1380 *
1381 * Return: IRQ_HANDLED if device generated a RX interrupt, IRQ_NONE otherwise.
1382 *
1383 * This is the Axi DMA Rx Isr. It invokes NAPI polling to complete the RX BD
1384 * processing.
1385 */
1386static irqreturn_t axienet_rx_irq(int irq, void *_ndev)
1387{
1388	unsigned int status;
1389	struct net_device *ndev = _ndev;
1390	struct axienet_local *lp = netdev_priv(ndev);
1391
1392	status = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET);
1393
1394	if (!(status & XAXIDMA_IRQ_ALL_MASK))
1395		return IRQ_NONE;
1396
1397	axienet_dma_out32(lp, XAXIDMA_RX_SR_OFFSET, status);
1398
1399	if (unlikely(status & XAXIDMA_IRQ_ERROR_MASK)) {
1400		netdev_err(ndev, "DMA Rx error 0x%x\n", status);
1401		netdev_err(ndev, "Current BD is at: 0x%x%08x\n",
1402			   (lp->rx_bd_v[lp->rx_bd_ci]).phys_msb,
1403			   (lp->rx_bd_v[lp->rx_bd_ci]).phys);
1404		schedule_work(&lp->dma_err_task);
1405	} else {
1406		/* Disable further RX completion interrupts and schedule
1407		 * NAPI receive.
1408		 */
1409		WRITE_ONCE(lp->rx_irqs, READ_ONCE(lp->rx_irqs) + 1);
1410		if (napi_schedule_prep(&lp->napi_rx)) {
1411			u32 cr;
1412
1413			spin_lock(&lp->rx_cr_lock);
1414			cr = lp->rx_dma_cr;
1415			cr &= ~(XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_DELAY_MASK);
1416			axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr);
1417			spin_unlock(&lp->rx_cr_lock);
1418
1419			__napi_schedule(&lp->napi_rx);
1420		}
1421	}
1422
1423	return IRQ_HANDLED;
1424}
1425
1426/**
1427 * axienet_eth_irq - Ethernet core Isr.
1428 * @irq:	irq number
1429 * @_ndev:	net_device pointer
1430 *
1431 * Return: IRQ_HANDLED if device generated a core interrupt, IRQ_NONE otherwise.
1432 *
1433 * Handle miscellaneous conditions indicated by Ethernet core IRQ.
1434 */
1435static irqreturn_t axienet_eth_irq(int irq, void *_ndev)
1436{
1437	struct net_device *ndev = _ndev;
1438	struct axienet_local *lp = netdev_priv(ndev);
1439	unsigned int pending;
1440
1441	pending = axienet_ior(lp, XAE_IP_OFFSET);
1442	if (!pending)
1443		return IRQ_NONE;
1444
1445	if (pending & XAE_INT_RXFIFOOVR_MASK)
1446		ndev->stats.rx_missed_errors++;
1447
1448	if (pending & XAE_INT_RXRJECT_MASK)
1449		ndev->stats.rx_dropped++;
1450
1451	axienet_iow(lp, XAE_IS_OFFSET, pending);
1452	return IRQ_HANDLED;
1453}
1454
1455static void axienet_dma_err_handler(struct work_struct *work);
1456
1457/**
1458 * axienet_rx_submit_desc - Submit the rx descriptors to dmaengine.
1459 * allocate skbuff, map the scatterlist and obtain a descriptor
1460 * and then add the callback information and submit descriptor.
1461 *
1462 * @ndev:	net_device pointer
1463 *
1464 */
1465static void axienet_rx_submit_desc(struct net_device *ndev)
1466{
1467	struct dma_async_tx_descriptor *dma_rx_desc = NULL;
1468	struct axienet_local *lp = netdev_priv(ndev);
1469	struct skbuf_dma_descriptor *skbuf_dma;
1470	struct sk_buff *skb;
1471	dma_addr_t addr;
1472
1473	skbuf_dma = axienet_get_rx_desc(lp, lp->rx_ring_head);
1474	if (!skbuf_dma)
1475		return;
1476
1477	skb = netdev_alloc_skb(ndev, lp->max_frm_size);
1478	if (!skb)
1479		return;
1480
1481	sg_init_table(skbuf_dma->sgl, 1);
1482	addr = dma_map_single(lp->dev, skb->data, lp->max_frm_size, DMA_FROM_DEVICE);
1483	if (unlikely(dma_mapping_error(lp->dev, addr))) {
1484		if (net_ratelimit())
1485			netdev_err(ndev, "DMA mapping error\n");
1486		goto rx_submit_err_free_skb;
1487	}
1488	sg_dma_address(skbuf_dma->sgl) = addr;
1489	sg_dma_len(skbuf_dma->sgl) = lp->max_frm_size;
1490	dma_rx_desc = dmaengine_prep_slave_sg(lp->rx_chan, skbuf_dma->sgl,
1491					      1, DMA_DEV_TO_MEM,
1492					      DMA_PREP_INTERRUPT);
1493	if (!dma_rx_desc)
1494		goto rx_submit_err_unmap_skb;
1495
1496	skbuf_dma->skb = skb;
1497	skbuf_dma->dma_address = sg_dma_address(skbuf_dma->sgl);
1498	skbuf_dma->desc = dma_rx_desc;
1499	dma_rx_desc->callback_param = lp;
1500	dma_rx_desc->callback_result = axienet_dma_rx_cb;
1501	lp->rx_ring_head++;
1502	dmaengine_submit(dma_rx_desc);
1503
1504	return;
1505
1506rx_submit_err_unmap_skb:
1507	dma_unmap_single(lp->dev, addr, lp->max_frm_size, DMA_FROM_DEVICE);
1508rx_submit_err_free_skb:
1509	dev_kfree_skb(skb);
1510}
1511
1512/**
1513 * axienet_init_dmaengine - init the dmaengine code.
1514 * @ndev:       Pointer to net_device structure
1515 *
1516 * Return: 0, on success.
1517 *          non-zero error value on failure
1518 *
1519 * This is the dmaengine initialization code.
1520 */
1521static int axienet_init_dmaengine(struct net_device *ndev)
1522{
1523	struct axienet_local *lp = netdev_priv(ndev);
1524	struct skbuf_dma_descriptor *skbuf_dma;
1525	int i, ret;
1526
1527	lp->tx_chan = dma_request_chan(lp->dev, "tx_chan0");
1528	if (IS_ERR(lp->tx_chan)) {
1529		dev_err(lp->dev, "No Ethernet DMA (TX) channel found\n");
1530		return PTR_ERR(lp->tx_chan);
1531	}
1532
1533	lp->rx_chan = dma_request_chan(lp->dev, "rx_chan0");
1534	if (IS_ERR(lp->rx_chan)) {
1535		ret = PTR_ERR(lp->rx_chan);
1536		dev_err(lp->dev, "No Ethernet DMA (RX) channel found\n");
1537		goto err_dma_release_tx;
1538	}
1539
1540	lp->tx_ring_tail = 0;
1541	lp->tx_ring_head = 0;
1542	lp->rx_ring_tail = 0;
1543	lp->rx_ring_head = 0;
1544	lp->tx_skb_ring = kcalloc(TX_BD_NUM_MAX, sizeof(*lp->tx_skb_ring),
1545				  GFP_KERNEL);
1546	if (!lp->tx_skb_ring) {
1547		ret = -ENOMEM;
1548		goto err_dma_release_rx;
1549	}
1550	for (i = 0; i < TX_BD_NUM_MAX; i++) {
1551		skbuf_dma = kzalloc(sizeof(*skbuf_dma), GFP_KERNEL);
1552		if (!skbuf_dma) {
1553			ret = -ENOMEM;
1554			goto err_free_tx_skb_ring;
1555		}
1556		lp->tx_skb_ring[i] = skbuf_dma;
1557	}
1558
1559	lp->rx_skb_ring = kcalloc(RX_BUF_NUM_DEFAULT, sizeof(*lp->rx_skb_ring),
1560				  GFP_KERNEL);
1561	if (!lp->rx_skb_ring) {
1562		ret = -ENOMEM;
1563		goto err_free_tx_skb_ring;
1564	}
1565	for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) {
1566		skbuf_dma = kzalloc(sizeof(*skbuf_dma), GFP_KERNEL);
1567		if (!skbuf_dma) {
1568			ret = -ENOMEM;
1569			goto err_free_rx_skb_ring;
1570		}
1571		lp->rx_skb_ring[i] = skbuf_dma;
1572	}
1573	/* TODO: Instead of BD_NUM_DEFAULT use runtime support */
1574	for (i = 0; i < RX_BUF_NUM_DEFAULT; i++)
1575		axienet_rx_submit_desc(ndev);
1576	dma_async_issue_pending(lp->rx_chan);
1577
1578	return 0;
1579
1580err_free_rx_skb_ring:
1581	for (i = 0; i < RX_BUF_NUM_DEFAULT; i++)
1582		kfree(lp->rx_skb_ring[i]);
1583	kfree(lp->rx_skb_ring);
1584err_free_tx_skb_ring:
1585	for (i = 0; i < TX_BD_NUM_MAX; i++)
1586		kfree(lp->tx_skb_ring[i]);
1587	kfree(lp->tx_skb_ring);
1588err_dma_release_rx:
1589	dma_release_channel(lp->rx_chan);
1590err_dma_release_tx:
1591	dma_release_channel(lp->tx_chan);
1592	return ret;
1593}
1594
1595/**
1596 * axienet_init_legacy_dma - init the dma legacy code.
1597 * @ndev:       Pointer to net_device structure
1598 *
1599 * Return: 0, on success.
1600 *          non-zero error value on failure
1601 *
1602 * This is the dma  initialization code. It also allocates interrupt
1603 * service routines, enables the interrupt lines and ISR handling.
1604 *
1605 */
1606static int axienet_init_legacy_dma(struct net_device *ndev)
1607{
1608	int ret;
1609	struct axienet_local *lp = netdev_priv(ndev);
1610
1611	/* Enable worker thread for Axi DMA error handling */
1612	lp->stopping = false;
1613	INIT_WORK(&lp->dma_err_task, axienet_dma_err_handler);
1614
1615	napi_enable(&lp->napi_rx);
1616	napi_enable(&lp->napi_tx);
1617
1618	/* Enable interrupts for Axi DMA Tx */
1619	ret = request_irq(lp->tx_irq, axienet_tx_irq, IRQF_SHARED,
1620			  ndev->name, ndev);
1621	if (ret)
1622		goto err_tx_irq;
1623	/* Enable interrupts for Axi DMA Rx */
1624	ret = request_irq(lp->rx_irq, axienet_rx_irq, IRQF_SHARED,
1625			  ndev->name, ndev);
1626	if (ret)
1627		goto err_rx_irq;
1628	/* Enable interrupts for Axi Ethernet core (if defined) */
1629	if (lp->eth_irq > 0) {
1630		ret = request_irq(lp->eth_irq, axienet_eth_irq, IRQF_SHARED,
1631				  ndev->name, ndev);
1632		if (ret)
1633			goto err_eth_irq;
1634	}
1635
1636	return 0;
1637
1638err_eth_irq:
1639	free_irq(lp->rx_irq, ndev);
1640err_rx_irq:
1641	free_irq(lp->tx_irq, ndev);
1642err_tx_irq:
1643	napi_disable(&lp->napi_tx);
1644	napi_disable(&lp->napi_rx);
1645	cancel_work_sync(&lp->dma_err_task);
1646	dev_err(lp->dev, "request_irq() failed\n");
1647	return ret;
1648}
1649
1650/**
1651 * axienet_open - Driver open routine.
1652 * @ndev:	Pointer to net_device structure
1653 *
1654 * Return: 0, on success.
1655 *	    non-zero error value on failure
1656 *
1657 * This is the driver open routine. It calls phylink_start to start the
1658 * PHY device.
1659 * It also allocates interrupt service routines, enables the interrupt lines
1660 * and ISR handling. Axi Ethernet core is reset through Axi DMA core. Buffer
1661 * descriptors are initialized.
1662 */
1663static int axienet_open(struct net_device *ndev)
1664{
1665	int ret;
1666	struct axienet_local *lp = netdev_priv(ndev);
1667
1668	/* When we do an Axi Ethernet reset, it resets the complete core
1669	 * including the MDIO. MDIO must be disabled before resetting.
1670	 * Hold MDIO bus lock to avoid MDIO accesses during the reset.
1671	 */
1672	axienet_lock_mii(lp);
1673	ret = axienet_device_reset(ndev);
1674	axienet_unlock_mii(lp);
1675
1676	ret = phylink_of_phy_connect(lp->phylink, lp->dev->of_node, 0);
1677	if (ret) {
1678		dev_err(lp->dev, "phylink_of_phy_connect() failed: %d\n", ret);
1679		return ret;
1680	}
1681
1682	phylink_start(lp->phylink);
1683
1684	/* Start the statistics refresh work */
1685	schedule_delayed_work(&lp->stats_work, 0);
1686
1687	if (lp->use_dmaengine) {
1688		/* Enable interrupts for Axi Ethernet core (if defined) */
1689		if (lp->eth_irq > 0) {
1690			ret = request_irq(lp->eth_irq, axienet_eth_irq, IRQF_SHARED,
1691					  ndev->name, ndev);
1692			if (ret)
1693				goto err_phy;
1694		}
1695
1696		ret = axienet_init_dmaengine(ndev);
1697		if (ret < 0)
1698			goto err_free_eth_irq;
1699	} else {
1700		ret = axienet_init_legacy_dma(ndev);
1701		if (ret)
1702			goto err_phy;
1703	}
1704
1705	return 0;
1706
1707err_free_eth_irq:
1708	if (lp->eth_irq > 0)
1709		free_irq(lp->eth_irq, ndev);
1710err_phy:
1711	cancel_work_sync(&lp->rx_dim.work);
1712	cancel_delayed_work_sync(&lp->stats_work);
1713	phylink_stop(lp->phylink);
1714	phylink_disconnect_phy(lp->phylink);
1715	return ret;
1716}
1717
1718/**
1719 * axienet_stop - Driver stop routine.
1720 * @ndev:	Pointer to net_device structure
1721 *
1722 * Return: 0, on success.
1723 *
1724 * This is the driver stop routine. It calls phylink_disconnect to stop the PHY
1725 * device. It also removes the interrupt handlers and disables the interrupts.
1726 * The Axi DMA Tx/Rx BDs are released.
1727 */
1728static int axienet_stop(struct net_device *ndev)
1729{
1730	struct axienet_local *lp = netdev_priv(ndev);
1731	int i;
1732
1733	if (!lp->use_dmaengine) {
1734		WRITE_ONCE(lp->stopping, true);
1735		flush_work(&lp->dma_err_task);
1736
1737		napi_disable(&lp->napi_tx);
1738		napi_disable(&lp->napi_rx);
1739	}
1740
1741	cancel_work_sync(&lp->rx_dim.work);
1742	cancel_delayed_work_sync(&lp->stats_work);
1743
1744	phylink_stop(lp->phylink);
1745	phylink_disconnect_phy(lp->phylink);
1746
1747	axienet_setoptions(ndev, lp->options &
1748			   ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN));
1749
1750	if (!lp->use_dmaengine) {
1751		axienet_dma_stop(lp);
1752		cancel_work_sync(&lp->dma_err_task);
1753		free_irq(lp->tx_irq, ndev);
1754		free_irq(lp->rx_irq, ndev);
1755		axienet_dma_bd_release(ndev);
1756	} else {
1757		dmaengine_terminate_sync(lp->tx_chan);
1758		dmaengine_synchronize(lp->tx_chan);
1759		dmaengine_terminate_sync(lp->rx_chan);
1760		dmaengine_synchronize(lp->rx_chan);
1761
1762		for (i = 0; i < TX_BD_NUM_MAX; i++)
1763			kfree(lp->tx_skb_ring[i]);
1764		kfree(lp->tx_skb_ring);
1765		for (i = 0; i < RX_BUF_NUM_DEFAULT; i++)
1766			kfree(lp->rx_skb_ring[i]);
1767		kfree(lp->rx_skb_ring);
1768
1769		dma_release_channel(lp->rx_chan);
1770		dma_release_channel(lp->tx_chan);
1771	}
1772
1773	netdev_reset_queue(ndev);
1774	axienet_iow(lp, XAE_IE_OFFSET, 0);
1775
1776	if (lp->eth_irq > 0)
1777		free_irq(lp->eth_irq, ndev);
1778	return 0;
1779}
1780
1781/**
1782 * axienet_change_mtu - Driver change mtu routine.
1783 * @ndev:	Pointer to net_device structure
1784 * @new_mtu:	New mtu value to be applied
1785 *
1786 * Return: Always returns 0 (success).
1787 *
1788 * This is the change mtu driver routine. It checks if the Axi Ethernet
1789 * hardware supports jumbo frames before changing the mtu. This can be
1790 * called only when the device is not up.
1791 */
1792static int axienet_change_mtu(struct net_device *ndev, int new_mtu)
1793{
1794	struct axienet_local *lp = netdev_priv(ndev);
1795
1796	if (netif_running(ndev))
1797		return -EBUSY;
1798
1799	if ((new_mtu + VLAN_ETH_HLEN +
1800		XAE_TRL_SIZE) > lp->rxmem)
1801		return -EINVAL;
1802
1803	WRITE_ONCE(ndev->mtu, new_mtu);
1804
1805	return 0;
1806}
1807
1808#ifdef CONFIG_NET_POLL_CONTROLLER
1809/**
1810 * axienet_poll_controller - Axi Ethernet poll mechanism.
1811 * @ndev:	Pointer to net_device structure
1812 *
1813 * This implements Rx/Tx ISR poll mechanisms. The interrupts are disabled prior
1814 * to polling the ISRs and are enabled back after the polling is done.
1815 */
1816static void axienet_poll_controller(struct net_device *ndev)
1817{
1818	struct axienet_local *lp = netdev_priv(ndev);
1819
1820	disable_irq(lp->tx_irq);
1821	disable_irq(lp->rx_irq);
1822	axienet_rx_irq(lp->tx_irq, ndev);
1823	axienet_tx_irq(lp->rx_irq, ndev);
1824	enable_irq(lp->tx_irq);
1825	enable_irq(lp->rx_irq);
1826}
1827#endif
1828
1829static int axienet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1830{
1831	struct axienet_local *lp = netdev_priv(dev);
1832
1833	if (!netif_running(dev))
1834		return -EINVAL;
1835
1836	return phylink_mii_ioctl(lp->phylink, rq, cmd);
1837}
1838
1839static void
1840axienet_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
1841{
1842	struct axienet_local *lp = netdev_priv(dev);
1843	unsigned int start;
1844
1845	netdev_stats_to_stats64(stats, &dev->stats);
1846
1847	do {
1848		start = u64_stats_fetch_begin(&lp->rx_stat_sync);
1849		stats->rx_packets = u64_stats_read(&lp->rx_packets);
1850		stats->rx_bytes = u64_stats_read(&lp->rx_bytes);
1851	} while (u64_stats_fetch_retry(&lp->rx_stat_sync, start));
1852
1853	do {
1854		start = u64_stats_fetch_begin(&lp->tx_stat_sync);
1855		stats->tx_packets = u64_stats_read(&lp->tx_packets);
1856		stats->tx_bytes = u64_stats_read(&lp->tx_bytes);
1857	} while (u64_stats_fetch_retry(&lp->tx_stat_sync, start));
1858
1859	if (!(lp->features & XAE_FEATURE_STATS))
1860		return;
1861
1862	do {
1863		start = read_seqcount_begin(&lp->hw_stats_seqcount);
1864		stats->rx_length_errors =
1865			axienet_stat(lp, STAT_RX_LENGTH_ERRORS);
1866		stats->rx_crc_errors = axienet_stat(lp, STAT_RX_FCS_ERRORS);
1867		stats->rx_frame_errors =
1868			axienet_stat(lp, STAT_RX_ALIGNMENT_ERRORS);
1869		stats->rx_errors = axienet_stat(lp, STAT_UNDERSIZE_FRAMES) +
1870				   axienet_stat(lp, STAT_FRAGMENT_FRAMES) +
1871				   stats->rx_length_errors +
1872				   stats->rx_crc_errors +
1873				   stats->rx_frame_errors;
1874		stats->multicast = axienet_stat(lp, STAT_RX_MULTICAST_FRAMES);
1875
1876		stats->tx_aborted_errors =
1877			axienet_stat(lp, STAT_TX_EXCESS_COLLISIONS);
1878		stats->tx_fifo_errors =
1879			axienet_stat(lp, STAT_TX_UNDERRUN_ERRORS);
1880		stats->tx_window_errors =
1881			axienet_stat(lp, STAT_TX_LATE_COLLISIONS);
1882		stats->tx_errors = axienet_stat(lp, STAT_TX_EXCESS_DEFERRAL) +
1883				   stats->tx_aborted_errors +
1884				   stats->tx_fifo_errors +
1885				   stats->tx_window_errors;
1886	} while (read_seqcount_retry(&lp->hw_stats_seqcount, start));
1887}
1888
1889static const struct net_device_ops axienet_netdev_ops = {
1890	.ndo_open = axienet_open,
1891	.ndo_stop = axienet_stop,
1892	.ndo_start_xmit = axienet_start_xmit,
1893	.ndo_get_stats64 = axienet_get_stats64,
1894	.ndo_change_mtu	= axienet_change_mtu,
1895	.ndo_set_mac_address = netdev_set_mac_address,
1896	.ndo_validate_addr = eth_validate_addr,
1897	.ndo_eth_ioctl = axienet_ioctl,
1898	.ndo_set_rx_mode = axienet_set_multicast_list,
1899#ifdef CONFIG_NET_POLL_CONTROLLER
1900	.ndo_poll_controller = axienet_poll_controller,
1901#endif
1902};
1903
1904static const struct net_device_ops axienet_netdev_dmaengine_ops = {
1905	.ndo_open = axienet_open,
1906	.ndo_stop = axienet_stop,
1907	.ndo_start_xmit = axienet_start_xmit_dmaengine,
1908	.ndo_get_stats64 = axienet_get_stats64,
1909	.ndo_change_mtu	= axienet_change_mtu,
1910	.ndo_set_mac_address = netdev_set_mac_address,
1911	.ndo_validate_addr = eth_validate_addr,
1912	.ndo_eth_ioctl = axienet_ioctl,
1913	.ndo_set_rx_mode = axienet_set_multicast_list,
1914};
1915
1916/**
1917 * axienet_ethtools_get_drvinfo - Get various Axi Ethernet driver information.
1918 * @ndev:	Pointer to net_device structure
1919 * @ed:		Pointer to ethtool_drvinfo structure
1920 *
1921 * This implements ethtool command for getting the driver information.
1922 * Issue "ethtool -i ethX" under linux prompt to execute this function.
1923 */
1924static void axienet_ethtools_get_drvinfo(struct net_device *ndev,
1925					 struct ethtool_drvinfo *ed)
1926{
1927	strscpy(ed->driver, DRIVER_NAME, sizeof(ed->driver));
1928	strscpy(ed->version, DRIVER_VERSION, sizeof(ed->version));
1929}
1930
1931/**
1932 * axienet_ethtools_get_regs_len - Get the total regs length present in the
1933 *				   AxiEthernet core.
1934 * @ndev:	Pointer to net_device structure
1935 *
1936 * This implements ethtool command for getting the total register length
1937 * information.
1938 *
1939 * Return: the total regs length
1940 */
1941static int axienet_ethtools_get_regs_len(struct net_device *ndev)
1942{
1943	return sizeof(u32) * AXIENET_REGS_N;
1944}
1945
1946/**
1947 * axienet_ethtools_get_regs - Dump the contents of all registers present
1948 *			       in AxiEthernet core.
1949 * @ndev:	Pointer to net_device structure
1950 * @regs:	Pointer to ethtool_regs structure
1951 * @ret:	Void pointer used to return the contents of the registers.
1952 *
1953 * This implements ethtool command for getting the Axi Ethernet register dump.
1954 * Issue "ethtool -d ethX" to execute this function.
1955 */
1956static void axienet_ethtools_get_regs(struct net_device *ndev,
1957				      struct ethtool_regs *regs, void *ret)
1958{
1959	u32 *data = (u32 *)ret;
1960	size_t len = sizeof(u32) * AXIENET_REGS_N;
1961	struct axienet_local *lp = netdev_priv(ndev);
1962
1963	regs->version = 0;
1964	regs->len = len;
1965
1966	memset(data, 0, len);
1967	data[0] = axienet_ior(lp, XAE_RAF_OFFSET);
1968	data[1] = axienet_ior(lp, XAE_TPF_OFFSET);
1969	data[2] = axienet_ior(lp, XAE_IFGP_OFFSET);
1970	data[3] = axienet_ior(lp, XAE_IS_OFFSET);
1971	data[4] = axienet_ior(lp, XAE_IP_OFFSET);
1972	data[5] = axienet_ior(lp, XAE_IE_OFFSET);
1973	data[6] = axienet_ior(lp, XAE_TTAG_OFFSET);
1974	data[7] = axienet_ior(lp, XAE_RTAG_OFFSET);
1975	data[8] = axienet_ior(lp, XAE_UAWL_OFFSET);
1976	data[9] = axienet_ior(lp, XAE_UAWU_OFFSET);
1977	data[10] = axienet_ior(lp, XAE_TPID0_OFFSET);
1978	data[11] = axienet_ior(lp, XAE_TPID1_OFFSET);
1979	data[12] = axienet_ior(lp, XAE_PPST_OFFSET);
1980	data[13] = axienet_ior(lp, XAE_RCW0_OFFSET);
1981	data[14] = axienet_ior(lp, XAE_RCW1_OFFSET);
1982	data[15] = axienet_ior(lp, XAE_TC_OFFSET);
1983	data[16] = axienet_ior(lp, XAE_FCC_OFFSET);
1984	data[17] = axienet_ior(lp, XAE_EMMC_OFFSET);
1985	data[18] = axienet_ior(lp, XAE_PHYC_OFFSET);
1986	data[19] = axienet_ior(lp, XAE_MDIO_MC_OFFSET);
1987	data[20] = axienet_ior(lp, XAE_MDIO_MCR_OFFSET);
1988	data[21] = axienet_ior(lp, XAE_MDIO_MWD_OFFSET);
1989	data[22] = axienet_ior(lp, XAE_MDIO_MRD_OFFSET);
1990	data[27] = axienet_ior(lp, XAE_UAW0_OFFSET);
1991	data[28] = axienet_ior(lp, XAE_UAW1_OFFSET);
1992	data[29] = axienet_ior(lp, XAE_FMI_OFFSET);
1993	data[30] = axienet_ior(lp, XAE_AF0_OFFSET);
1994	data[31] = axienet_ior(lp, XAE_AF1_OFFSET);
1995	if (!lp->use_dmaengine) {
1996		data[32] = axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET);
1997		data[33] = axienet_dma_in32(lp, XAXIDMA_TX_SR_OFFSET);
1998		data[34] = axienet_dma_in32(lp, XAXIDMA_TX_CDESC_OFFSET);
1999		data[35] = axienet_dma_in32(lp, XAXIDMA_TX_TDESC_OFFSET);
2000		data[36] = axienet_dma_in32(lp, XAXIDMA_RX_CR_OFFSET);
2001		data[37] = axienet_dma_in32(lp, XAXIDMA_RX_SR_OFFSET);
2002		data[38] = axienet_dma_in32(lp, XAXIDMA_RX_CDESC_OFFSET);
2003		data[39] = axienet_dma_in32(lp, XAXIDMA_RX_TDESC_OFFSET);
2004	}
2005}
2006
2007static void
2008axienet_ethtools_get_ringparam(struct net_device *ndev,
2009			       struct ethtool_ringparam *ering,
2010			       struct kernel_ethtool_ringparam *kernel_ering,
2011			       struct netlink_ext_ack *extack)
2012{
2013	struct axienet_local *lp = netdev_priv(ndev);
2014
2015	ering->rx_max_pending = RX_BD_NUM_MAX;
2016	ering->rx_mini_max_pending = 0;
2017	ering->rx_jumbo_max_pending = 0;
2018	ering->tx_max_pending = TX_BD_NUM_MAX;
2019	ering->rx_pending = lp->rx_bd_num;
2020	ering->rx_mini_pending = 0;
2021	ering->rx_jumbo_pending = 0;
2022	ering->tx_pending = lp->tx_bd_num;
2023}
2024
2025static int
2026axienet_ethtools_set_ringparam(struct net_device *ndev,
2027			       struct ethtool_ringparam *ering,
2028			       struct kernel_ethtool_ringparam *kernel_ering,
2029			       struct netlink_ext_ack *extack)
2030{
2031	struct axienet_local *lp = netdev_priv(ndev);
2032
2033	if (ering->rx_pending > RX_BD_NUM_MAX ||
2034	    ering->rx_mini_pending ||
2035	    ering->rx_jumbo_pending ||
2036	    ering->tx_pending < TX_BD_NUM_MIN ||
2037	    ering->tx_pending > TX_BD_NUM_MAX)
2038		return -EINVAL;
2039
2040	if (netif_running(ndev))
2041		return -EBUSY;
2042
2043	lp->rx_bd_num = ering->rx_pending;
2044	lp->tx_bd_num = ering->tx_pending;
2045	return 0;
2046}
2047
2048/**
2049 * axienet_ethtools_get_pauseparam - Get the pause parameter setting for
2050 *				     Tx and Rx paths.
2051 * @ndev:	Pointer to net_device structure
2052 * @epauseparm:	Pointer to ethtool_pauseparam structure.
2053 *
2054 * This implements ethtool command for getting axi ethernet pause frame
2055 * setting. Issue "ethtool -a ethX" to execute this function.
2056 */
2057static void
2058axienet_ethtools_get_pauseparam(struct net_device *ndev,
2059				struct ethtool_pauseparam *epauseparm)
2060{
2061	struct axienet_local *lp = netdev_priv(ndev);
2062
2063	phylink_ethtool_get_pauseparam(lp->phylink, epauseparm);
2064}
2065
2066/**
2067 * axienet_ethtools_set_pauseparam - Set device pause parameter(flow control)
2068 *				     settings.
2069 * @ndev:	Pointer to net_device structure
2070 * @epauseparm:Pointer to ethtool_pauseparam structure
2071 *
2072 * This implements ethtool command for enabling flow control on Rx and Tx
2073 * paths. Issue "ethtool -A ethX tx on|off" under linux prompt to execute this
2074 * function.
2075 *
2076 * Return: 0 on success, -EFAULT if device is running
2077 */
2078static int
2079axienet_ethtools_set_pauseparam(struct net_device *ndev,
2080				struct ethtool_pauseparam *epauseparm)
2081{
2082	struct axienet_local *lp = netdev_priv(ndev);
2083
2084	return phylink_ethtool_set_pauseparam(lp->phylink, epauseparm);
2085}
2086
2087/**
2088 * axienet_update_coalesce_rx() - Set RX CR
2089 * @lp: Device private data
2090 * @cr: Value to write to the RX CR
2091 * @mask: Bits to set from @cr
2092 */
2093static void axienet_update_coalesce_rx(struct axienet_local *lp, u32 cr,
2094				       u32 mask)
2095{
2096	spin_lock_irq(&lp->rx_cr_lock);
2097	lp->rx_dma_cr &= ~mask;
2098	lp->rx_dma_cr |= cr;
2099	/* If DMA isn't started, then the settings will be applied the next
2100	 * time dma_start() is called.
2101	 */
2102	if (lp->rx_dma_started) {
2103		u32 reg = axienet_dma_in32(lp, XAXIDMA_RX_CR_OFFSET);
2104
2105		/* Don't enable IRQs if they are disabled by NAPI */
2106		if (reg & XAXIDMA_IRQ_ALL_MASK)
2107			cr = lp->rx_dma_cr;
2108		else
2109			cr = lp->rx_dma_cr & ~XAXIDMA_IRQ_ALL_MASK;
2110		axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr);
2111	}
2112	spin_unlock_irq(&lp->rx_cr_lock);
2113}
2114
2115/**
2116 * axienet_dim_coalesce_count_rx() - RX coalesce count for DIM
2117 * @lp: Device private data
2118 *
2119 * Return: RX coalescing frame count value for DIM.
2120 */
2121static u32 axienet_dim_coalesce_count_rx(struct axienet_local *lp)
2122{
2123	return min(1 << (lp->rx_dim.profile_ix << 1), 255);
2124}
2125
2126/**
2127 * axienet_rx_dim_work() - Adjust RX DIM settings
2128 * @work: The work struct
2129 */
2130static void axienet_rx_dim_work(struct work_struct *work)
2131{
2132	struct axienet_local *lp =
2133		container_of(work, struct axienet_local, rx_dim.work);
2134	u32 cr = axienet_calc_cr(lp, axienet_dim_coalesce_count_rx(lp), 0);
2135	u32 mask = XAXIDMA_COALESCE_MASK | XAXIDMA_IRQ_IOC_MASK |
2136		   XAXIDMA_IRQ_ERROR_MASK;
2137
2138	axienet_update_coalesce_rx(lp, cr, mask);
2139	lp->rx_dim.state = DIM_START_MEASURE;
2140}
2141
2142/**
2143 * axienet_update_coalesce_tx() - Set TX CR
2144 * @lp: Device private data
2145 * @cr: Value to write to the TX CR
2146 * @mask: Bits to set from @cr
2147 */
2148static void axienet_update_coalesce_tx(struct axienet_local *lp, u32 cr,
2149				       u32 mask)
2150{
2151	spin_lock_irq(&lp->tx_cr_lock);
2152	lp->tx_dma_cr &= ~mask;
2153	lp->tx_dma_cr |= cr;
2154	/* If DMA isn't started, then the settings will be applied the next
2155	 * time dma_start() is called.
2156	 */
2157	if (lp->tx_dma_started) {
2158		u32 reg = axienet_dma_in32(lp, XAXIDMA_TX_CR_OFFSET);
2159
2160		/* Don't enable IRQs if they are disabled by NAPI */
2161		if (reg & XAXIDMA_IRQ_ALL_MASK)
2162			cr = lp->tx_dma_cr;
2163		else
2164			cr = lp->tx_dma_cr & ~XAXIDMA_IRQ_ALL_MASK;
2165		axienet_dma_out32(lp, XAXIDMA_TX_CR_OFFSET, cr);
2166	}
2167	spin_unlock_irq(&lp->tx_cr_lock);
2168}
2169
2170/**
2171 * axienet_ethtools_get_coalesce - Get DMA interrupt coalescing count.
2172 * @ndev:	Pointer to net_device structure
2173 * @ecoalesce:	Pointer to ethtool_coalesce structure
2174 * @kernel_coal: ethtool CQE mode setting structure
2175 * @extack:	extack for reporting error messages
2176 *
2177 * This implements ethtool command for getting the DMA interrupt coalescing
2178 * count on Tx and Rx paths. Issue "ethtool -c ethX" under linux prompt to
2179 * execute this function.
2180 *
2181 * Return: 0 always
2182 */
2183static int
2184axienet_ethtools_get_coalesce(struct net_device *ndev,
2185			      struct ethtool_coalesce *ecoalesce,
2186			      struct kernel_ethtool_coalesce *kernel_coal,
2187			      struct netlink_ext_ack *extack)
2188{
2189	struct axienet_local *lp = netdev_priv(ndev);
2190	u32 cr;
2191
2192	ecoalesce->use_adaptive_rx_coalesce = lp->rx_dim_enabled;
2193
2194	spin_lock_irq(&lp->rx_cr_lock);
2195	cr = lp->rx_dma_cr;
2196	spin_unlock_irq(&lp->rx_cr_lock);
2197	axienet_coalesce_params(lp, cr,
2198				&ecoalesce->rx_max_coalesced_frames,
2199				&ecoalesce->rx_coalesce_usecs);
2200
2201	spin_lock_irq(&lp->tx_cr_lock);
2202	cr = lp->tx_dma_cr;
2203	spin_unlock_irq(&lp->tx_cr_lock);
2204	axienet_coalesce_params(lp, cr,
2205				&ecoalesce->tx_max_coalesced_frames,
2206				&ecoalesce->tx_coalesce_usecs);
2207	return 0;
2208}
2209
2210/**
2211 * axienet_ethtools_set_coalesce - Set DMA interrupt coalescing count.
2212 * @ndev:	Pointer to net_device structure
2213 * @ecoalesce:	Pointer to ethtool_coalesce structure
2214 * @kernel_coal: ethtool CQE mode setting structure
2215 * @extack:	extack for reporting error messages
2216 *
2217 * This implements ethtool command for setting the DMA interrupt coalescing
2218 * count on Tx and Rx paths. Issue "ethtool -C ethX rx-frames 5" under linux
2219 * prompt to execute this function.
2220 *
2221 * Return: 0, on success, Non-zero error value on failure.
2222 */
2223static int
2224axienet_ethtools_set_coalesce(struct net_device *ndev,
2225			      struct ethtool_coalesce *ecoalesce,
2226			      struct kernel_ethtool_coalesce *kernel_coal,
2227			      struct netlink_ext_ack *extack)
2228{
2229	struct axienet_local *lp = netdev_priv(ndev);
2230	bool new_dim = ecoalesce->use_adaptive_rx_coalesce;
2231	bool old_dim = lp->rx_dim_enabled;
2232	u32 cr, mask = ~XAXIDMA_CR_RUNSTOP_MASK;
2233
2234	if (ecoalesce->rx_max_coalesced_frames > 255 ||
2235	    ecoalesce->tx_max_coalesced_frames > 255) {
2236		NL_SET_ERR_MSG(extack, "frames must be less than 256");
2237		return -EINVAL;
2238	}
2239
2240	if (!ecoalesce->rx_max_coalesced_frames ||
2241	    !ecoalesce->tx_max_coalesced_frames) {
2242		NL_SET_ERR_MSG(extack, "frames must be non-zero");
2243		return -EINVAL;
2244	}
2245
2246	if (((ecoalesce->rx_max_coalesced_frames > 1 || new_dim) &&
2247	     !ecoalesce->rx_coalesce_usecs) ||
2248	    (ecoalesce->tx_max_coalesced_frames > 1 &&
2249	     !ecoalesce->tx_coalesce_usecs)) {
2250		NL_SET_ERR_MSG(extack,
2251			       "usecs must be non-zero when frames is greater than one");
2252		return -EINVAL;
2253	}
2254
2255	if (new_dim && !old_dim) {
2256		cr = axienet_calc_cr(lp, axienet_dim_coalesce_count_rx(lp),
2257				     ecoalesce->rx_coalesce_usecs);
2258	} else if (!new_dim) {
2259		if (old_dim) {
2260			WRITE_ONCE(lp->rx_dim_enabled, false);
2261			napi_synchronize(&lp->napi_rx);
2262			flush_work(&lp->rx_dim.work);
2263		}
2264
2265		cr = axienet_calc_cr(lp, ecoalesce->rx_max_coalesced_frames,
2266				     ecoalesce->rx_coalesce_usecs);
2267	} else {
2268		/* Dummy value for count just to calculate timer */
2269		cr = axienet_calc_cr(lp, 2, ecoalesce->rx_coalesce_usecs);
2270		mask = XAXIDMA_DELAY_MASK | XAXIDMA_IRQ_DELAY_MASK;
2271	}
2272
2273	axienet_update_coalesce_rx(lp, cr, mask);
2274	if (new_dim && !old_dim)
2275		WRITE_ONCE(lp->rx_dim_enabled, true);
2276
2277	cr = axienet_calc_cr(lp, ecoalesce->tx_max_coalesced_frames,
2278			     ecoalesce->tx_coalesce_usecs);
2279	axienet_update_coalesce_tx(lp, cr, ~XAXIDMA_CR_RUNSTOP_MASK);
2280	return 0;
2281}
2282
2283static int
2284axienet_ethtools_get_link_ksettings(struct net_device *ndev,
2285				    struct ethtool_link_ksettings *cmd)
2286{
2287	struct axienet_local *lp = netdev_priv(ndev);
2288
2289	return phylink_ethtool_ksettings_get(lp->phylink, cmd);
2290}
2291
2292static int
2293axienet_ethtools_set_link_ksettings(struct net_device *ndev,
2294				    const struct ethtool_link_ksettings *cmd)
2295{
2296	struct axienet_local *lp = netdev_priv(ndev);
2297
2298	return phylink_ethtool_ksettings_set(lp->phylink, cmd);
2299}
2300
2301static int axienet_ethtools_nway_reset(struct net_device *dev)
2302{
2303	struct axienet_local *lp = netdev_priv(dev);
2304
2305	return phylink_ethtool_nway_reset(lp->phylink);
2306}
2307
2308static void axienet_ethtools_get_ethtool_stats(struct net_device *dev,
2309					       struct ethtool_stats *stats,
2310					       u64 *data)
2311{
2312	struct axienet_local *lp = netdev_priv(dev);
2313	unsigned int start;
2314
2315	do {
2316		start = read_seqcount_begin(&lp->hw_stats_seqcount);
2317		data[0] = axienet_stat(lp, STAT_RX_BYTES);
2318		data[1] = axienet_stat(lp, STAT_TX_BYTES);
2319		data[2] = axienet_stat(lp, STAT_RX_VLAN_FRAMES);
2320		data[3] = axienet_stat(lp, STAT_TX_VLAN_FRAMES);
2321		data[6] = axienet_stat(lp, STAT_TX_PFC_FRAMES);
2322		data[7] = axienet_stat(lp, STAT_RX_PFC_FRAMES);
2323		data[8] = axienet_stat(lp, STAT_USER_DEFINED0);
2324		data[9] = axienet_stat(lp, STAT_USER_DEFINED1);
2325		data[10] = axienet_stat(lp, STAT_USER_DEFINED2);
2326	} while (read_seqcount_retry(&lp->hw_stats_seqcount, start));
2327}
2328
2329static const char axienet_ethtool_stats_strings[][ETH_GSTRING_LEN] = {
2330	"Received bytes",
2331	"Transmitted bytes",
2332	"RX Good VLAN Tagged Frames",
2333	"TX Good VLAN Tagged Frames",
2334	"TX Good PFC Frames",
2335	"RX Good PFC Frames",
2336	"User Defined Counter 0",
2337	"User Defined Counter 1",
2338	"User Defined Counter 2",
2339};
2340
2341static void axienet_ethtools_get_strings(struct net_device *dev, u32 stringset, u8 *data)
2342{
2343	switch (stringset) {
2344	case ETH_SS_STATS:
2345		memcpy(data, axienet_ethtool_stats_strings,
2346		       sizeof(axienet_ethtool_stats_strings));
2347		break;
2348	}
2349}
2350
2351static int axienet_ethtools_get_sset_count(struct net_device *dev, int sset)
2352{
2353	struct axienet_local *lp = netdev_priv(dev);
2354
2355	switch (sset) {
2356	case ETH_SS_STATS:
2357		if (lp->features & XAE_FEATURE_STATS)
2358			return ARRAY_SIZE(axienet_ethtool_stats_strings);
2359		fallthrough;
2360	default:
2361		return -EOPNOTSUPP;
2362	}
2363}
2364
2365static void
2366axienet_ethtools_get_pause_stats(struct net_device *dev,
2367				 struct ethtool_pause_stats *pause_stats)
2368{
2369	struct axienet_local *lp = netdev_priv(dev);
2370	unsigned int start;
2371
2372	if (!(lp->features & XAE_FEATURE_STATS))
2373		return;
2374
2375	do {
2376		start = read_seqcount_begin(&lp->hw_stats_seqcount);
2377		pause_stats->tx_pause_frames =
2378			axienet_stat(lp, STAT_TX_PAUSE_FRAMES);
2379		pause_stats->rx_pause_frames =
2380			axienet_stat(lp, STAT_RX_PAUSE_FRAMES);
2381	} while (read_seqcount_retry(&lp->hw_stats_seqcount, start));
2382}
2383
2384static void
2385axienet_ethtool_get_eth_mac_stats(struct net_device *dev,
2386				  struct ethtool_eth_mac_stats *mac_stats)
2387{
2388	struct axienet_local *lp = netdev_priv(dev);
2389	unsigned int start;
2390
2391	if (!(lp->features & XAE_FEATURE_STATS))
2392		return;
2393
2394	do {
2395		start = read_seqcount_begin(&lp->hw_stats_seqcount);
2396		mac_stats->FramesTransmittedOK =
2397			axienet_stat(lp, STAT_TX_GOOD_FRAMES);
2398		mac_stats->SingleCollisionFrames =
2399			axienet_stat(lp, STAT_TX_SINGLE_COLLISION_FRAMES);
2400		mac_stats->MultipleCollisionFrames =
2401			axienet_stat(lp, STAT_TX_MULTIPLE_COLLISION_FRAMES);
2402		mac_stats->FramesReceivedOK =
2403			axienet_stat(lp, STAT_RX_GOOD_FRAMES);
2404		mac_stats->FrameCheckSequenceErrors =
2405			axienet_stat(lp, STAT_RX_FCS_ERRORS);
2406		mac_stats->AlignmentErrors =
2407			axienet_stat(lp, STAT_RX_ALIGNMENT_ERRORS);
2408		mac_stats->FramesWithDeferredXmissions =
2409			axienet_stat(lp, STAT_TX_DEFERRED_FRAMES);
2410		mac_stats->LateCollisions =
2411			axienet_stat(lp, STAT_TX_LATE_COLLISIONS);
2412		mac_stats->FramesAbortedDueToXSColls =
2413			axienet_stat(lp, STAT_TX_EXCESS_COLLISIONS);
2414		mac_stats->MulticastFramesXmittedOK =
2415			axienet_stat(lp, STAT_TX_MULTICAST_FRAMES);
2416		mac_stats->BroadcastFramesXmittedOK =
2417			axienet_stat(lp, STAT_TX_BROADCAST_FRAMES);
2418		mac_stats->FramesWithExcessiveDeferral =
2419			axienet_stat(lp, STAT_TX_EXCESS_DEFERRAL);
2420		mac_stats->MulticastFramesReceivedOK =
2421			axienet_stat(lp, STAT_RX_MULTICAST_FRAMES);
2422		mac_stats->BroadcastFramesReceivedOK =
2423			axienet_stat(lp, STAT_RX_BROADCAST_FRAMES);
2424		mac_stats->InRangeLengthErrors =
2425			axienet_stat(lp, STAT_RX_LENGTH_ERRORS);
2426	} while (read_seqcount_retry(&lp->hw_stats_seqcount, start));
2427}
2428
2429static void
2430axienet_ethtool_get_eth_ctrl_stats(struct net_device *dev,
2431				   struct ethtool_eth_ctrl_stats *ctrl_stats)
2432{
2433	struct axienet_local *lp = netdev_priv(dev);
2434	unsigned int start;
2435
2436	if (!(lp->features & XAE_FEATURE_STATS))
2437		return;
2438
2439	do {
2440		start = read_seqcount_begin(&lp->hw_stats_seqcount);
2441		ctrl_stats->MACControlFramesTransmitted =
2442			axienet_stat(lp, STAT_TX_CONTROL_FRAMES);
2443		ctrl_stats->MACControlFramesReceived =
2444			axienet_stat(lp, STAT_RX_CONTROL_FRAMES);
2445		ctrl_stats->UnsupportedOpcodesReceived =
2446			axienet_stat(lp, STAT_RX_CONTROL_OPCODE_ERRORS);
2447	} while (read_seqcount_retry(&lp->hw_stats_seqcount, start));
2448}
2449
2450static const struct ethtool_rmon_hist_range axienet_rmon_ranges[] = {
2451	{   64,    64 },
2452	{   65,   127 },
2453	{  128,   255 },
2454	{  256,   511 },
2455	{  512,  1023 },
2456	{ 1024,  1518 },
2457	{ 1519, 16384 },
2458	{ },
2459};
2460
2461static void
2462axienet_ethtool_get_rmon_stats(struct net_device *dev,
2463			       struct ethtool_rmon_stats *rmon_stats,
2464			       const struct ethtool_rmon_hist_range **ranges)
2465{
2466	struct axienet_local *lp = netdev_priv(dev);
2467	unsigned int start;
2468
2469	if (!(lp->features & XAE_FEATURE_STATS))
2470		return;
2471
2472	do {
2473		start = read_seqcount_begin(&lp->hw_stats_seqcount);
2474		rmon_stats->undersize_pkts =
2475			axienet_stat(lp, STAT_UNDERSIZE_FRAMES);
2476		rmon_stats->oversize_pkts =
2477			axienet_stat(lp, STAT_RX_OVERSIZE_FRAMES);
2478		rmon_stats->fragments =
2479			axienet_stat(lp, STAT_FRAGMENT_FRAMES);
2480
2481		rmon_stats->hist[0] =
2482			axienet_stat(lp, STAT_RX_64_BYTE_FRAMES);
2483		rmon_stats->hist[1] =
2484			axienet_stat(lp, STAT_RX_65_127_BYTE_FRAMES);
2485		rmon_stats->hist[2] =
2486			axienet_stat(lp, STAT_RX_128_255_BYTE_FRAMES);
2487		rmon_stats->hist[3] =
2488			axienet_stat(lp, STAT_RX_256_511_BYTE_FRAMES);
2489		rmon_stats->hist[4] =
2490			axienet_stat(lp, STAT_RX_512_1023_BYTE_FRAMES);
2491		rmon_stats->hist[5] =
2492			axienet_stat(lp, STAT_RX_1024_MAX_BYTE_FRAMES);
2493		rmon_stats->hist[6] =
2494			rmon_stats->oversize_pkts;
2495
2496		rmon_stats->hist_tx[0] =
2497			axienet_stat(lp, STAT_TX_64_BYTE_FRAMES);
2498		rmon_stats->hist_tx[1] =
2499			axienet_stat(lp, STAT_TX_65_127_BYTE_FRAMES);
2500		rmon_stats->hist_tx[2] =
2501			axienet_stat(lp, STAT_TX_128_255_BYTE_FRAMES);
2502		rmon_stats->hist_tx[3] =
2503			axienet_stat(lp, STAT_TX_256_511_BYTE_FRAMES);
2504		rmon_stats->hist_tx[4] =
2505			axienet_stat(lp, STAT_TX_512_1023_BYTE_FRAMES);
2506		rmon_stats->hist_tx[5] =
2507			axienet_stat(lp, STAT_TX_1024_MAX_BYTE_FRAMES);
2508		rmon_stats->hist_tx[6] =
2509			axienet_stat(lp, STAT_TX_OVERSIZE_FRAMES);
2510	} while (read_seqcount_retry(&lp->hw_stats_seqcount, start));
2511
2512	*ranges = axienet_rmon_ranges;
2513}
2514
2515static const struct ethtool_ops axienet_ethtool_ops = {
2516	.supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES |
2517				     ETHTOOL_COALESCE_USECS |
2518				     ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
2519	.get_drvinfo    = axienet_ethtools_get_drvinfo,
2520	.get_regs_len   = axienet_ethtools_get_regs_len,
2521	.get_regs       = axienet_ethtools_get_regs,
2522	.get_link       = ethtool_op_get_link,
2523	.get_ringparam	= axienet_ethtools_get_ringparam,
2524	.set_ringparam	= axienet_ethtools_set_ringparam,
2525	.get_pauseparam = axienet_ethtools_get_pauseparam,
2526	.set_pauseparam = axienet_ethtools_set_pauseparam,
2527	.get_coalesce   = axienet_ethtools_get_coalesce,
2528	.set_coalesce   = axienet_ethtools_set_coalesce,
2529	.get_link_ksettings = axienet_ethtools_get_link_ksettings,
2530	.set_link_ksettings = axienet_ethtools_set_link_ksettings,
2531	.nway_reset	= axienet_ethtools_nway_reset,
2532	.get_ethtool_stats = axienet_ethtools_get_ethtool_stats,
2533	.get_strings    = axienet_ethtools_get_strings,
2534	.get_sset_count = axienet_ethtools_get_sset_count,
2535	.get_pause_stats = axienet_ethtools_get_pause_stats,
2536	.get_eth_mac_stats = axienet_ethtool_get_eth_mac_stats,
2537	.get_eth_ctrl_stats = axienet_ethtool_get_eth_ctrl_stats,
2538	.get_rmon_stats = axienet_ethtool_get_rmon_stats,
2539};
2540
2541static struct axienet_local *pcs_to_axienet_local(struct phylink_pcs *pcs)
2542{
2543	return container_of(pcs, struct axienet_local, pcs);
2544}
2545
2546static void axienet_pcs_get_state(struct phylink_pcs *pcs,
2547				  unsigned int neg_mode,
2548				  struct phylink_link_state *state)
2549{
2550	struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy;
2551
2552	phylink_mii_c22_pcs_get_state(pcs_phy, neg_mode, state);
2553}
2554
2555static void axienet_pcs_an_restart(struct phylink_pcs *pcs)
2556{
2557	struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy;
2558
2559	phylink_mii_c22_pcs_an_restart(pcs_phy);
2560}
2561
2562static int axienet_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode,
2563			      phy_interface_t interface,
2564			      const unsigned long *advertising,
2565			      bool permit_pause_to_mac)
2566{
2567	struct mdio_device *pcs_phy = pcs_to_axienet_local(pcs)->pcs_phy;
2568	struct net_device *ndev = pcs_to_axienet_local(pcs)->ndev;
2569	struct axienet_local *lp = netdev_priv(ndev);
2570	int ret;
2571
2572	if (lp->switch_x_sgmii) {
2573		ret = mdiodev_write(pcs_phy, XLNX_MII_STD_SELECT_REG,
2574				    interface == PHY_INTERFACE_MODE_SGMII ?
2575					XLNX_MII_STD_SELECT_SGMII : 0);
2576		if (ret < 0) {
2577			netdev_warn(ndev,
2578				    "Failed to switch PHY interface: %d\n",
2579				    ret);
2580			return ret;
2581		}
2582	}
2583
2584	ret = phylink_mii_c22_pcs_config(pcs_phy, interface, advertising,
2585					 neg_mode);
2586	if (ret < 0)
2587		netdev_warn(ndev, "Failed to configure PCS: %d\n", ret);
2588
2589	return ret;
2590}
2591
2592static const struct phylink_pcs_ops axienet_pcs_ops = {
2593	.pcs_get_state = axienet_pcs_get_state,
2594	.pcs_config = axienet_pcs_config,
2595	.pcs_an_restart = axienet_pcs_an_restart,
2596};
2597
2598static struct phylink_pcs *axienet_mac_select_pcs(struct phylink_config *config,
2599						  phy_interface_t interface)
2600{
2601	struct net_device *ndev = to_net_dev(config->dev);
2602	struct axienet_local *lp = netdev_priv(ndev);
2603
2604	if (interface == PHY_INTERFACE_MODE_1000BASEX ||
2605	    interface ==  PHY_INTERFACE_MODE_SGMII)
2606		return &lp->pcs;
2607
2608	return NULL;
2609}
2610
2611static void axienet_mac_config(struct phylink_config *config, unsigned int mode,
2612			       const struct phylink_link_state *state)
2613{
2614	/* nothing meaningful to do */
2615}
2616
2617static void axienet_mac_link_down(struct phylink_config *config,
2618				  unsigned int mode,
2619				  phy_interface_t interface)
2620{
2621	/* nothing meaningful to do */
2622}
2623
2624static void axienet_mac_link_up(struct phylink_config *config,
2625				struct phy_device *phy,
2626				unsigned int mode, phy_interface_t interface,
2627				int speed, int duplex,
2628				bool tx_pause, bool rx_pause)
2629{
2630	struct net_device *ndev = to_net_dev(config->dev);
2631	struct axienet_local *lp = netdev_priv(ndev);
2632	u32 emmc_reg, fcc_reg;
2633
2634	emmc_reg = axienet_ior(lp, XAE_EMMC_OFFSET);
2635	emmc_reg &= ~XAE_EMMC_LINKSPEED_MASK;
2636
2637	switch (speed) {
2638	case SPEED_1000:
2639		emmc_reg |= XAE_EMMC_LINKSPD_1000;
2640		break;
2641	case SPEED_100:
2642		emmc_reg |= XAE_EMMC_LINKSPD_100;
2643		break;
2644	case SPEED_10:
2645		emmc_reg |= XAE_EMMC_LINKSPD_10;
2646		break;
2647	default:
2648		dev_err(&ndev->dev,
2649			"Speed other than 10, 100 or 1Gbps is not supported\n");
2650		break;
2651	}
2652
2653	axienet_iow(lp, XAE_EMMC_OFFSET, emmc_reg);
2654
2655	fcc_reg = axienet_ior(lp, XAE_FCC_OFFSET);
2656	if (tx_pause)
2657		fcc_reg |= XAE_FCC_FCTX_MASK;
2658	else
2659		fcc_reg &= ~XAE_FCC_FCTX_MASK;
2660	if (rx_pause)
2661		fcc_reg |= XAE_FCC_FCRX_MASK;
2662	else
2663		fcc_reg &= ~XAE_FCC_FCRX_MASK;
2664	axienet_iow(lp, XAE_FCC_OFFSET, fcc_reg);
2665}
2666
2667static const struct phylink_mac_ops axienet_phylink_ops = {
2668	.mac_select_pcs = axienet_mac_select_pcs,
2669	.mac_config = axienet_mac_config,
2670	.mac_link_down = axienet_mac_link_down,
2671	.mac_link_up = axienet_mac_link_up,
2672};
2673
2674/**
2675 * axienet_dma_err_handler - Work queue task for Axi DMA Error
2676 * @work:	pointer to work_struct
2677 *
2678 * Resets the Axi DMA and Axi Ethernet devices, and reconfigures the
2679 * Tx/Rx BDs.
2680 */
2681static void axienet_dma_err_handler(struct work_struct *work)
2682{
2683	u32 i;
2684	u32 axienet_status;
2685	struct axidma_bd *cur_p;
2686	struct axienet_local *lp = container_of(work, struct axienet_local,
2687						dma_err_task);
2688	struct net_device *ndev = lp->ndev;
2689
2690	/* Don't bother if we are going to stop anyway */
2691	if (READ_ONCE(lp->stopping))
2692		return;
2693
2694	napi_disable(&lp->napi_tx);
2695	napi_disable(&lp->napi_rx);
2696
2697	axienet_setoptions(ndev, lp->options &
2698			   ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN));
2699
2700	axienet_dma_stop(lp);
2701	netdev_reset_queue(ndev);
2702
2703	for (i = 0; i < lp->tx_bd_num; i++) {
2704		cur_p = &lp->tx_bd_v[i];
2705		if (cur_p->cntrl) {
2706			dma_addr_t addr = desc_get_phys_addr(lp, cur_p);
2707
2708			dma_unmap_single(lp->dev, addr,
2709					 (cur_p->cntrl &
2710					  XAXIDMA_BD_CTRL_LENGTH_MASK),
2711					 DMA_TO_DEVICE);
2712		}
2713		if (cur_p->skb)
2714			dev_kfree_skb_irq(cur_p->skb);
2715		cur_p->phys = 0;
2716		cur_p->phys_msb = 0;
2717		cur_p->cntrl = 0;
2718		cur_p->status = 0;
2719		cur_p->app0 = 0;
2720		cur_p->app1 = 0;
2721		cur_p->app2 = 0;
2722		cur_p->app3 = 0;
2723		cur_p->app4 = 0;
2724		cur_p->skb = NULL;
2725	}
2726
2727	for (i = 0; i < lp->rx_bd_num; i++) {
2728		cur_p = &lp->rx_bd_v[i];
2729		cur_p->status = 0;
2730		cur_p->app0 = 0;
2731		cur_p->app1 = 0;
2732		cur_p->app2 = 0;
2733		cur_p->app3 = 0;
2734		cur_p->app4 = 0;
2735	}
2736
2737	lp->tx_bd_ci = 0;
2738	lp->tx_bd_tail = 0;
2739	lp->rx_bd_ci = 0;
2740
2741	axienet_dma_start(lp);
2742
2743	axienet_status = axienet_ior(lp, XAE_RCW1_OFFSET);
2744	axienet_status &= ~XAE_RCW1_RX_MASK;
2745	axienet_iow(lp, XAE_RCW1_OFFSET, axienet_status);
2746
2747	axienet_status = axienet_ior(lp, XAE_IP_OFFSET);
2748	if (axienet_status & XAE_INT_RXRJECT_MASK)
2749		axienet_iow(lp, XAE_IS_OFFSET, XAE_INT_RXRJECT_MASK);
2750	axienet_iow(lp, XAE_IE_OFFSET, lp->eth_irq > 0 ?
2751		    XAE_INT_RECV_ERROR_MASK : 0);
2752	axienet_iow(lp, XAE_FCC_OFFSET, XAE_FCC_FCRX_MASK);
2753
2754	/* Sync default options with HW but leave receiver and
2755	 * transmitter disabled.
2756	 */
2757	axienet_setoptions(ndev, lp->options &
2758			   ~(XAE_OPTION_TXEN | XAE_OPTION_RXEN));
2759	axienet_set_mac_address(ndev, NULL);
2760	axienet_set_multicast_list(ndev);
2761	napi_enable(&lp->napi_rx);
2762	napi_enable(&lp->napi_tx);
2763	axienet_setoptions(ndev, lp->options);
2764}
2765
2766/**
2767 * axienet_probe - Axi Ethernet probe function.
2768 * @pdev:	Pointer to platform device structure.
2769 *
2770 * Return: 0, on success
2771 *	    Non-zero error value on failure.
2772 *
2773 * This is the probe routine for Axi Ethernet driver. This is called before
2774 * any other driver routines are invoked. It allocates and sets up the Ethernet
2775 * device. Parses through device tree and populates fields of
2776 * axienet_local. It registers the Ethernet device.
2777 */
2778static int axienet_probe(struct platform_device *pdev)
2779{
2780	int ret;
2781	struct device_node *np;
2782	struct axienet_local *lp;
2783	struct net_device *ndev;
2784	struct resource *ethres;
2785	u8 mac_addr[ETH_ALEN];
2786	int addr_width = 32;
2787	u32 value;
2788
2789	ndev = alloc_etherdev(sizeof(*lp));
2790	if (!ndev)
2791		return -ENOMEM;
2792
2793	platform_set_drvdata(pdev, ndev);
2794
2795	SET_NETDEV_DEV(ndev, &pdev->dev);
2796	ndev->features = NETIF_F_SG;
2797	ndev->ethtool_ops = &axienet_ethtool_ops;
2798
2799	/* MTU range: 64 - 9000 */
2800	ndev->min_mtu = 64;
2801	ndev->max_mtu = XAE_JUMBO_MTU;
2802
2803	lp = netdev_priv(ndev);
2804	lp->ndev = ndev;
2805	lp->dev = &pdev->dev;
2806	lp->options = XAE_OPTION_DEFAULTS;
2807	lp->rx_bd_num = RX_BD_NUM_DEFAULT;
2808	lp->tx_bd_num = TX_BD_NUM_DEFAULT;
2809
2810	u64_stats_init(&lp->rx_stat_sync);
2811	u64_stats_init(&lp->tx_stat_sync);
2812
2813	mutex_init(&lp->stats_lock);
2814	seqcount_mutex_init(&lp->hw_stats_seqcount, &lp->stats_lock);
2815	INIT_DEFERRABLE_WORK(&lp->stats_work, axienet_refresh_stats);
2816
2817	lp->axi_clk = devm_clk_get_optional(&pdev->dev, "s_axi_lite_clk");
2818	if (!lp->axi_clk) {
2819		/* For backward compatibility, if named AXI clock is not present,
2820		 * treat the first clock specified as the AXI clock.
2821		 */
2822		lp->axi_clk = devm_clk_get_optional(&pdev->dev, NULL);
2823	}
2824	if (IS_ERR(lp->axi_clk)) {
2825		ret = PTR_ERR(lp->axi_clk);
2826		goto free_netdev;
2827	}
2828	ret = clk_prepare_enable(lp->axi_clk);
2829	if (ret) {
2830		dev_err(&pdev->dev, "Unable to enable AXI clock: %d\n", ret);
2831		goto free_netdev;
2832	}
2833
2834	lp->misc_clks[0].id = "axis_clk";
2835	lp->misc_clks[1].id = "ref_clk";
2836	lp->misc_clks[2].id = "mgt_clk";
2837
2838	ret = devm_clk_bulk_get_optional(&pdev->dev, XAE_NUM_MISC_CLOCKS, lp->misc_clks);
2839	if (ret)
2840		goto cleanup_clk;
2841
2842	ret = clk_bulk_prepare_enable(XAE_NUM_MISC_CLOCKS, lp->misc_clks);
2843	if (ret)
2844		goto cleanup_clk;
2845
2846	/* Map device registers */
2847	lp->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &ethres);
2848	if (IS_ERR(lp->regs)) {
2849		ret = PTR_ERR(lp->regs);
2850		goto cleanup_clk;
2851	}
2852	lp->regs_start = ethres->start;
2853
2854	/* Setup checksum offload, but default to off if not specified */
2855	lp->features = 0;
2856
2857	if (axienet_ior(lp, XAE_ABILITY_OFFSET) & XAE_ABILITY_STATS)
2858		lp->features |= XAE_FEATURE_STATS;
2859
2860	ret = of_property_read_u32(pdev->dev.of_node, "xlnx,txcsum", &value);
2861	if (!ret) {
2862		switch (value) {
2863		case 1:
2864			lp->features |= XAE_FEATURE_PARTIAL_TX_CSUM;
2865			/* Can checksum any contiguous range */
2866			ndev->features |= NETIF_F_HW_CSUM;
2867			break;
2868		case 2:
2869			lp->features |= XAE_FEATURE_FULL_TX_CSUM;
2870			/* Can checksum TCP/UDP over IPv4. */
2871			ndev->features |= NETIF_F_IP_CSUM;
2872			break;
2873		}
2874	}
2875	ret = of_property_read_u32(pdev->dev.of_node, "xlnx,rxcsum", &value);
2876	if (!ret) {
2877		switch (value) {
2878		case 1:
2879			lp->features |= XAE_FEATURE_PARTIAL_RX_CSUM;
2880			ndev->features |= NETIF_F_RXCSUM;
2881			break;
2882		case 2:
2883			lp->features |= XAE_FEATURE_FULL_RX_CSUM;
2884			ndev->features |= NETIF_F_RXCSUM;
2885			break;
2886		}
2887	}
2888	/* For supporting jumbo frames, the Axi Ethernet hardware must have
2889	 * a larger Rx/Tx Memory. Typically, the size must be large so that
2890	 * we can enable jumbo option and start supporting jumbo frames.
2891	 * Here we check for memory allocated for Rx/Tx in the hardware from
2892	 * the device-tree and accordingly set flags.
2893	 */
2894	of_property_read_u32(pdev->dev.of_node, "xlnx,rxmem", &lp->rxmem);
2895
2896	lp->switch_x_sgmii = of_property_read_bool(pdev->dev.of_node,
2897						   "xlnx,switch-x-sgmii");
2898
2899	/* Start with the proprietary, and broken phy_type */
2900	ret = of_property_read_u32(pdev->dev.of_node, "xlnx,phy-type", &value);
2901	if (!ret) {
2902		netdev_warn(ndev, "Please upgrade your device tree binary blob to use phy-mode");
2903		switch (value) {
2904		case XAE_PHY_TYPE_MII:
2905			lp->phy_mode = PHY_INTERFACE_MODE_MII;
2906			break;
2907		case XAE_PHY_TYPE_GMII:
2908			lp->phy_mode = PHY_INTERFACE_MODE_GMII;
2909			break;
2910		case XAE_PHY_TYPE_RGMII_2_0:
2911			lp->phy_mode = PHY_INTERFACE_MODE_RGMII_ID;
2912			break;
2913		case XAE_PHY_TYPE_SGMII:
2914			lp->phy_mode = PHY_INTERFACE_MODE_SGMII;
2915			break;
2916		case XAE_PHY_TYPE_1000BASE_X:
2917			lp->phy_mode = PHY_INTERFACE_MODE_1000BASEX;
2918			break;
2919		default:
2920			ret = -EINVAL;
2921			goto cleanup_clk;
2922		}
2923	} else {
2924		ret = of_get_phy_mode(pdev->dev.of_node, &lp->phy_mode);
2925		if (ret)
2926			goto cleanup_clk;
2927	}
2928	if (lp->switch_x_sgmii && lp->phy_mode != PHY_INTERFACE_MODE_SGMII &&
2929	    lp->phy_mode != PHY_INTERFACE_MODE_1000BASEX) {
2930		dev_err(&pdev->dev, "xlnx,switch-x-sgmii only supported with SGMII or 1000BaseX\n");
2931		ret = -EINVAL;
2932		goto cleanup_clk;
2933	}
2934
2935	if (!of_property_present(pdev->dev.of_node, "dmas")) {
2936		/* Find the DMA node, map the DMA registers, and decode the DMA IRQs */
2937		np = of_parse_phandle(pdev->dev.of_node, "axistream-connected", 0);
2938
2939		if (np) {
2940			struct resource dmares;
2941
2942			ret = of_address_to_resource(np, 0, &dmares);
2943			if (ret) {
2944				dev_err(&pdev->dev,
2945					"unable to get DMA resource\n");
2946				of_node_put(np);
2947				goto cleanup_clk;
2948			}
2949			lp->dma_regs = devm_ioremap_resource(&pdev->dev,
2950							     &dmares);
2951			lp->rx_irq = irq_of_parse_and_map(np, 1);
2952			lp->tx_irq = irq_of_parse_and_map(np, 0);
2953			of_node_put(np);
2954			lp->eth_irq = platform_get_irq_optional(pdev, 0);
2955		} else {
2956			/* Check for these resources directly on the Ethernet node. */
2957			lp->dma_regs = devm_platform_get_and_ioremap_resource(pdev, 1, NULL);
2958			lp->rx_irq = platform_get_irq(pdev, 1);
2959			lp->tx_irq = platform_get_irq(pdev, 0);
2960			lp->eth_irq = platform_get_irq_optional(pdev, 2);
2961		}
2962		if (IS_ERR(lp->dma_regs)) {
2963			dev_err(&pdev->dev, "could not map DMA regs\n");
2964			ret = PTR_ERR(lp->dma_regs);
2965			goto cleanup_clk;
2966		}
2967		if (lp->rx_irq <= 0 || lp->tx_irq <= 0) {
2968			dev_err(&pdev->dev, "could not determine irqs\n");
2969			ret = -ENOMEM;
2970			goto cleanup_clk;
2971		}
2972
2973		/* Reset core now that clocks are enabled, prior to accessing MDIO */
2974		ret = __axienet_device_reset(lp);
2975		if (ret)
2976			goto cleanup_clk;
2977
2978		/* Autodetect the need for 64-bit DMA pointers.
2979		 * When the IP is configured for a bus width bigger than 32 bits,
2980		 * writing the MSB registers is mandatory, even if they are all 0.
2981		 * We can detect this case by writing all 1's to one such register
2982		 * and see if that sticks: when the IP is configured for 32 bits
2983		 * only, those registers are RES0.
2984		 * Those MSB registers were introduced in IP v7.1, which we check first.
2985		 */
2986		if ((axienet_ior(lp, XAE_ID_OFFSET) >> 24) >= 0x9) {
2987			void __iomem *desc = lp->dma_regs + XAXIDMA_TX_CDESC_OFFSET + 4;
2988
2989			iowrite32(0x0, desc);
2990			if (ioread32(desc) == 0) {	/* sanity check */
2991				iowrite32(0xffffffff, desc);
2992				if (ioread32(desc) > 0) {
2993					lp->features |= XAE_FEATURE_DMA_64BIT;
2994					addr_width = 64;
2995					dev_info(&pdev->dev,
2996						 "autodetected 64-bit DMA range\n");
2997				}
2998				iowrite32(0x0, desc);
2999			}
3000		}
3001		if (!IS_ENABLED(CONFIG_64BIT) && lp->features & XAE_FEATURE_DMA_64BIT) {
3002			dev_err(&pdev->dev, "64-bit addressable DMA is not compatible with 32-bit architecture\n");
3003			ret = -EINVAL;
3004			goto cleanup_clk;
3005		}
3006
3007		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_width));
3008		if (ret) {
3009			dev_err(&pdev->dev, "No suitable DMA available\n");
3010			goto cleanup_clk;
3011		}
3012		netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll);
3013		netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll);
3014	} else {
3015		struct xilinx_vdma_config cfg;
3016		struct dma_chan *tx_chan;
3017
3018		lp->eth_irq = platform_get_irq_optional(pdev, 0);
3019		if (lp->eth_irq < 0 && lp->eth_irq != -ENXIO) {
3020			ret = lp->eth_irq;
3021			goto cleanup_clk;
3022		}
3023		tx_chan = dma_request_chan(lp->dev, "tx_chan0");
3024		if (IS_ERR(tx_chan)) {
3025			ret = PTR_ERR(tx_chan);
3026			dev_err_probe(lp->dev, ret, "No Ethernet DMA (TX) channel found\n");
3027			goto cleanup_clk;
3028		}
3029
3030		cfg.reset = 1;
3031		/* As name says VDMA but it has support for DMA channel reset */
3032		ret = xilinx_vdma_channel_set_config(tx_chan, &cfg);
3033		if (ret < 0) {
3034			dev_err(&pdev->dev, "Reset channel failed\n");
3035			dma_release_channel(tx_chan);
3036			goto cleanup_clk;
3037		}
3038
3039		dma_release_channel(tx_chan);
3040		lp->use_dmaengine = 1;
3041	}
3042
3043	if (lp->use_dmaengine)
3044		ndev->netdev_ops = &axienet_netdev_dmaengine_ops;
3045	else
3046		ndev->netdev_ops = &axienet_netdev_ops;
3047	/* Check for Ethernet core IRQ (optional) */
3048	if (lp->eth_irq <= 0)
3049		dev_info(&pdev->dev, "Ethernet core IRQ not defined\n");
3050
3051	/* Retrieve the MAC address */
3052	ret = of_get_mac_address(pdev->dev.of_node, mac_addr);
3053	if (!ret) {
3054		axienet_set_mac_address(ndev, mac_addr);
3055	} else {
3056		dev_warn(&pdev->dev, "could not find MAC address property: %d\n",
3057			 ret);
3058		axienet_set_mac_address(ndev, NULL);
3059	}
3060
3061	spin_lock_init(&lp->rx_cr_lock);
3062	spin_lock_init(&lp->tx_cr_lock);
3063	INIT_WORK(&lp->rx_dim.work, axienet_rx_dim_work);
3064	lp->rx_dim_enabled = true;
3065	lp->rx_dim.profile_ix = 1;
3066	lp->rx_dma_cr = axienet_calc_cr(lp, axienet_dim_coalesce_count_rx(lp),
3067					XAXIDMA_DFT_RX_USEC);
3068	lp->tx_dma_cr = axienet_calc_cr(lp, XAXIDMA_DFT_TX_THRESHOLD,
3069					XAXIDMA_DFT_TX_USEC);
3070
3071	ret = axienet_mdio_setup(lp);
3072	if (ret)
3073		dev_warn(&pdev->dev,
3074			 "error registering MDIO bus: %d\n", ret);
3075
3076	if (lp->phy_mode == PHY_INTERFACE_MODE_SGMII ||
3077	    lp->phy_mode == PHY_INTERFACE_MODE_1000BASEX) {
3078		np = of_parse_phandle(pdev->dev.of_node, "pcs-handle", 0);
3079		if (!np) {
3080			/* Deprecated: Always use "pcs-handle" for pcs_phy.
3081			 * Falling back to "phy-handle" here is only for
3082			 * backward compatibility with old device trees.
3083			 */
3084			np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0);
3085		}
3086		if (!np) {
3087			dev_err(&pdev->dev, "pcs-handle (preferred) or phy-handle required for 1000BaseX/SGMII\n");
3088			ret = -EINVAL;
3089			goto cleanup_mdio;
3090		}
3091		lp->pcs_phy = of_mdio_find_device(np);
3092		if (!lp->pcs_phy) {
3093			ret = -EPROBE_DEFER;
3094			of_node_put(np);
3095			goto cleanup_mdio;
3096		}
3097		of_node_put(np);
3098		lp->pcs.ops = &axienet_pcs_ops;
3099		lp->pcs.poll = true;
3100	}
3101
3102	lp->phylink_config.dev = &ndev->dev;
3103	lp->phylink_config.type = PHYLINK_NETDEV;
3104	lp->phylink_config.mac_managed_pm = true;
3105	lp->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE |
3106		MAC_10FD | MAC_100FD | MAC_1000FD;
3107
3108	__set_bit(lp->phy_mode, lp->phylink_config.supported_interfaces);
3109	if (lp->switch_x_sgmii) {
3110		__set_bit(PHY_INTERFACE_MODE_1000BASEX,
3111			  lp->phylink_config.supported_interfaces);
3112		__set_bit(PHY_INTERFACE_MODE_SGMII,
3113			  lp->phylink_config.supported_interfaces);
3114	}
3115
3116	lp->phylink = phylink_create(&lp->phylink_config, pdev->dev.fwnode,
3117				     lp->phy_mode,
3118				     &axienet_phylink_ops);
3119	if (IS_ERR(lp->phylink)) {
3120		ret = PTR_ERR(lp->phylink);
3121		dev_err(&pdev->dev, "phylink_create error (%i)\n", ret);
3122		goto cleanup_mdio;
3123	}
3124
3125	ret = register_netdev(lp->ndev);
3126	if (ret) {
3127		dev_err(lp->dev, "register_netdev() error (%i)\n", ret);
3128		goto cleanup_phylink;
3129	}
3130
3131	return 0;
3132
3133cleanup_phylink:
3134	phylink_destroy(lp->phylink);
3135
3136cleanup_mdio:
3137	if (lp->pcs_phy)
3138		put_device(&lp->pcs_phy->dev);
3139	if (lp->mii_bus)
3140		axienet_mdio_teardown(lp);
3141cleanup_clk:
3142	clk_bulk_disable_unprepare(XAE_NUM_MISC_CLOCKS, lp->misc_clks);
3143	clk_disable_unprepare(lp->axi_clk);
3144
3145free_netdev:
3146	free_netdev(ndev);
3147
3148	return ret;
3149}
3150
3151static void axienet_remove(struct platform_device *pdev)
3152{
3153	struct net_device *ndev = platform_get_drvdata(pdev);
3154	struct axienet_local *lp = netdev_priv(ndev);
3155
3156	unregister_netdev(ndev);
3157
3158	if (lp->phylink)
3159		phylink_destroy(lp->phylink);
3160
3161	if (lp->pcs_phy)
3162		put_device(&lp->pcs_phy->dev);
3163
3164	axienet_mdio_teardown(lp);
3165
3166	clk_bulk_disable_unprepare(XAE_NUM_MISC_CLOCKS, lp->misc_clks);
3167	clk_disable_unprepare(lp->axi_clk);
3168
3169	free_netdev(ndev);
3170}
3171
3172static void axienet_shutdown(struct platform_device *pdev)
3173{
3174	struct net_device *ndev = platform_get_drvdata(pdev);
3175
3176	rtnl_lock();
3177	netif_device_detach(ndev);
3178
3179	if (netif_running(ndev))
3180		dev_close(ndev);
3181
3182	rtnl_unlock();
3183}
3184
3185static int axienet_suspend(struct device *dev)
3186{
3187	struct net_device *ndev = dev_get_drvdata(dev);
3188
3189	if (!netif_running(ndev))
3190		return 0;
3191
3192	netif_device_detach(ndev);
3193
3194	rtnl_lock();
3195	axienet_stop(ndev);
3196	rtnl_unlock();
3197
3198	return 0;
3199}
3200
3201static int axienet_resume(struct device *dev)
3202{
3203	struct net_device *ndev = dev_get_drvdata(dev);
3204
3205	if (!netif_running(ndev))
3206		return 0;
3207
3208	rtnl_lock();
3209	axienet_open(ndev);
3210	rtnl_unlock();
3211
3212	netif_device_attach(ndev);
3213
3214	return 0;
3215}
3216
3217static DEFINE_SIMPLE_DEV_PM_OPS(axienet_pm_ops,
3218				axienet_suspend, axienet_resume);
3219
3220static struct platform_driver axienet_driver = {
3221	.probe = axienet_probe,
3222	.remove = axienet_remove,
3223	.shutdown = axienet_shutdown,
3224	.driver = {
3225		 .name = "xilinx_axienet",
3226		 .pm = &axienet_pm_ops,
3227		 .of_match_table = axienet_of_match,
3228	},
3229};
3230
3231module_platform_driver(axienet_driver);
3232
3233MODULE_DESCRIPTION("Xilinx Axi Ethernet driver");
3234MODULE_AUTHOR("Xilinx");
3235MODULE_LICENSE("GPL");