ravb_main.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Renesas Ethernet AVB device driver
   3 *
   4 * Copyright (C) 2014-2019 Renesas Electronics Corporation
   5 * Copyright (C) 2015 Renesas Solutions Corp.
   6 * Copyright (C) 2015-2016 Cogent Embedded, Inc. <source@cogentembedded.com>
   7 *
   8 * Based on the SuperH Ethernet driver
   9 */
  10
  11#include <linux/cache.h>
  12#include <linux/clk.h>
  13#include <linux/delay.h>
  14#include <linux/dma-mapping.h>
  15#include <linux/err.h>
  16#include <linux/etherdevice.h>
  17#include <linux/ethtool.h>
  18#include <linux/if_vlan.h>
  19#include <linux/kernel.h>
  20#include <linux/list.h>
  21#include <linux/module.h>
  22#include <linux/net_tstamp.h>
  23#include <linux/of.h>
  24#include <linux/of_mdio.h>
  25#include <linux/of_net.h>
  26#include <linux/platform_device.h>
  27#include <linux/pm_runtime.h>
  28#include <linux/slab.h>
  29#include <linux/spinlock.h>
  30#include <linux/reset.h>
  31#include <linux/math64.h>
  32#include <net/ip.h>
  33#include <net/page_pool/helpers.h>
  34
  35#include "ravb.h"
  36
  37#define RAVB_DEF_MSG_ENABLE \
  38		(NETIF_MSG_LINK	  | \
  39		 NETIF_MSG_TIMER  | \
  40		 NETIF_MSG_RX_ERR | \
  41		 NETIF_MSG_TX_ERR)
  42
  43void ravb_modify(struct net_device *ndev, enum ravb_reg reg, u32 clear,
  44		 u32 set)
  45{
  46	ravb_write(ndev, (ravb_read(ndev, reg) & ~clear) | set, reg);
  47}
  48
  49int ravb_wait(struct net_device *ndev, enum ravb_reg reg, u32 mask, u32 value)
  50{
  51	int i;
  52
  53	for (i = 0; i < 10000; i++) {
  54		if ((ravb_read(ndev, reg) & mask) == value)
  55			return 0;
  56		udelay(10);
  57	}
  58	return -ETIMEDOUT;
  59}
  60
  61static int ravb_set_opmode(struct net_device *ndev, u32 opmode)
  62{
  63	u32 csr_ops = 1U << (opmode & CCC_OPC);
  64	u32 ccc_mask = CCC_OPC;
  65	int error;
  66
  67	/* If gPTP active in config mode is supported it needs to be configured
  68	 * along with CSEL and operating mode in the same access. This is a
  69	 * hardware limitation.
  70	 */
  71	if (opmode & CCC_GAC)
  72		ccc_mask |= CCC_GAC | CCC_CSEL;
  73
  74	/* Set operating mode */
  75	ravb_modify(ndev, CCC, ccc_mask, opmode);
  76	/* Check if the operating mode is changed to the requested one */
  77	error = ravb_wait(ndev, CSR, CSR_OPS, csr_ops);
  78	if (error) {
  79		netdev_err(ndev, "failed to switch device to requested mode (%u)\n",
  80			   opmode & CCC_OPC);
  81	}
  82
  83	return error;
  84}
  85
  86static void ravb_set_rate_gbeth(struct net_device *ndev)
  87{
  88	struct ravb_private *priv = netdev_priv(ndev);
  89
  90	switch (priv->speed) {
  91	case 10:		/* 10BASE */
  92		ravb_write(ndev, GBETH_GECMR_SPEED_10, GECMR);
  93		break;
  94	case 100:		/* 100BASE */
  95		ravb_write(ndev, GBETH_GECMR_SPEED_100, GECMR);
  96		break;
  97	case 1000:		/* 1000BASE */
  98		ravb_write(ndev, GBETH_GECMR_SPEED_1000, GECMR);
  99		break;
 100	}
 101}
 102
 103static void ravb_set_rate_rcar(struct net_device *ndev)
 104{
 105	struct ravb_private *priv = netdev_priv(ndev);
 106
 107	switch (priv->speed) {
 108	case 100:		/* 100BASE */
 109		ravb_write(ndev, GECMR_SPEED_100, GECMR);
 110		break;
 111	case 1000:		/* 1000BASE */
 112		ravb_write(ndev, GECMR_SPEED_1000, GECMR);
 113		break;
 114	}
 115}
 116
 117/* Get MAC address from the MAC address registers
 118 *
 119 * Ethernet AVB device doesn't have ROM for MAC address.
 120 * This function gets the MAC address that was used by a bootloader.
 121 */
 122static void ravb_read_mac_address(struct device_node *np,
 123				  struct net_device *ndev)
 124{
 125	int ret;
 126
 127	ret = of_get_ethdev_address(np, ndev);
 128	if (ret) {
 129		u32 mahr = ravb_read(ndev, MAHR);
 130		u32 malr = ravb_read(ndev, MALR);
 131		u8 addr[ETH_ALEN];
 132
 133		addr[0] = (mahr >> 24) & 0xFF;
 134		addr[1] = (mahr >> 16) & 0xFF;
 135		addr[2] = (mahr >>  8) & 0xFF;
 136		addr[3] = (mahr >>  0) & 0xFF;
 137		addr[4] = (malr >>  8) & 0xFF;
 138		addr[5] = (malr >>  0) & 0xFF;
 139		eth_hw_addr_set(ndev, addr);
 140	}
 141}
 142
 143static void ravb_mdio_ctrl(struct mdiobb_ctrl *ctrl, u32 mask, int set)
 144{
 145	struct ravb_private *priv = container_of(ctrl, struct ravb_private,
 146						 mdiobb);
 147
 148	ravb_modify(priv->ndev, PIR, mask, set ? mask : 0);
 149}
 150
 151/* MDC pin control */
 152static void ravb_set_mdc(struct mdiobb_ctrl *ctrl, int level)
 153{
 154	ravb_mdio_ctrl(ctrl, PIR_MDC, level);
 155}
 156
 157/* Data I/O pin control */
 158static void ravb_set_mdio_dir(struct mdiobb_ctrl *ctrl, int output)
 159{
 160	ravb_mdio_ctrl(ctrl, PIR_MMD, output);
 161}
 162
 163/* Set data bit */
 164static void ravb_set_mdio_data(struct mdiobb_ctrl *ctrl, int value)
 165{
 166	ravb_mdio_ctrl(ctrl, PIR_MDO, value);
 167}
 168
 169/* Get data bit */
 170static int ravb_get_mdio_data(struct mdiobb_ctrl *ctrl)
 171{
 172	struct ravb_private *priv = container_of(ctrl, struct ravb_private,
 173						 mdiobb);
 174
 175	return (ravb_read(priv->ndev, PIR) & PIR_MDI) != 0;
 176}
 177
 178/* MDIO bus control struct */
 179static const struct mdiobb_ops bb_ops = {
 180	.owner = THIS_MODULE,
 181	.set_mdc = ravb_set_mdc,
 182	.set_mdio_dir = ravb_set_mdio_dir,
 183	.set_mdio_data = ravb_set_mdio_data,
 184	.get_mdio_data = ravb_get_mdio_data,
 185};
 186
 187static struct ravb_rx_desc *
 188ravb_rx_get_desc(struct ravb_private *priv, unsigned int q,
 189		 unsigned int i)
 190{
 191	return priv->rx_ring[q].raw + priv->info->rx_desc_size * i;
 192}
 193
 194/* Free TX skb function for AVB-IP */
 195static int ravb_tx_free(struct net_device *ndev, int q, bool free_txed_only)
 196{
 197	struct ravb_private *priv = netdev_priv(ndev);
 198	struct net_device_stats *stats = &priv->stats[q];
 199	unsigned int num_tx_desc = priv->num_tx_desc;
 200	struct ravb_tx_desc *desc;
 201	unsigned int entry;
 202	int free_num = 0;
 203	u32 size;
 204
 205	for (; priv->cur_tx[q] - priv->dirty_tx[q] > 0; priv->dirty_tx[q]++) {
 206		bool txed;
 207
 208		entry = priv->dirty_tx[q] % (priv->num_tx_ring[q] *
 209					     num_tx_desc);
 210		desc = &priv->tx_ring[q][entry];
 211		txed = desc->die_dt == DT_FEMPTY;
 212		if (free_txed_only && !txed)
 213			break;
 214		/* Descriptor type must be checked before all other reads */
 215		dma_rmb();
 216		size = le16_to_cpu(desc->ds_tagl) & TX_DS;
 217		/* Free the original skb. */
 218		if (priv->tx_skb[q][entry / num_tx_desc]) {
 219			dma_unmap_single(ndev->dev.parent, le32_to_cpu(desc->dptr),
 220					 size, DMA_TO_DEVICE);
 221			/* Last packet descriptor? */
 222			if (entry % num_tx_desc == num_tx_desc - 1) {
 223				entry /= num_tx_desc;
 224				dev_kfree_skb_any(priv->tx_skb[q][entry]);
 225				priv->tx_skb[q][entry] = NULL;
 226				if (txed)
 227					stats->tx_packets++;
 228			}
 229			free_num++;
 230		}
 231		if (txed)
 232			stats->tx_bytes += size;
 233		desc->die_dt = DT_EEMPTY;
 234	}
 235	return free_num;
 236}
 237
 238static void ravb_rx_ring_free(struct net_device *ndev, int q)
 239{
 240	struct ravb_private *priv = netdev_priv(ndev);
 241	unsigned int ring_size;
 242
 243	if (!priv->rx_ring[q].raw)
 244		return;
 245
 246	ring_size = priv->info->rx_desc_size * (priv->num_rx_ring[q] + 1);
 247	dma_free_coherent(ndev->dev.parent, ring_size, priv->rx_ring[q].raw,
 248			  priv->rx_desc_dma[q]);
 249	priv->rx_ring[q].raw = NULL;
 250}
 251
 252/* Free skb's and DMA buffers for Ethernet AVB */
 253static void ravb_ring_free(struct net_device *ndev, int q)
 254{
 255	struct ravb_private *priv = netdev_priv(ndev);
 256	unsigned int num_tx_desc = priv->num_tx_desc;
 257	unsigned int ring_size;
 258	unsigned int i;
 259
 260	ravb_rx_ring_free(ndev, q);
 261
 262	if (priv->tx_ring[q]) {
 263		ravb_tx_free(ndev, q, false);
 264
 265		ring_size = sizeof(struct ravb_tx_desc) *
 266			    (priv->num_tx_ring[q] * num_tx_desc + 1);
 267		dma_free_coherent(ndev->dev.parent, ring_size, priv->tx_ring[q],
 268				  priv->tx_desc_dma[q]);
 269		priv->tx_ring[q] = NULL;
 270	}
 271
 272	/* Free RX buffers */
 273	for (i = 0; i < priv->num_rx_ring[q]; i++) {
 274		if (priv->rx_buffers[q][i].page)
 275			page_pool_put_page(priv->rx_pool[q],
 276					   priv->rx_buffers[q][i].page,
 277					   0, true);
 278	}
 279	kfree(priv->rx_buffers[q]);
 280	priv->rx_buffers[q] = NULL;
 281	page_pool_destroy(priv->rx_pool[q]);
 282
 283	/* Free aligned TX buffers */
 284	kfree(priv->tx_align[q]);
 285	priv->tx_align[q] = NULL;
 286
 287	/* Free TX skb ringbuffer.
 288	 * SKBs are freed by ravb_tx_free() call above.
 289	 */
 290	kfree(priv->tx_skb[q]);
 291	priv->tx_skb[q] = NULL;
 292}
 293
 294static int
 295ravb_alloc_rx_buffer(struct net_device *ndev, int q, u32 entry, gfp_t gfp_mask,
 296		     struct ravb_rx_desc *rx_desc)
 297{
 298	struct ravb_private *priv = netdev_priv(ndev);
 299	const struct ravb_hw_info *info = priv->info;
 300	struct ravb_rx_buffer *rx_buff;
 301	dma_addr_t dma_addr;
 302	unsigned int size;
 303
 304	rx_buff = &priv->rx_buffers[q][entry];
 305	size = info->rx_buffer_size;
 306	rx_buff->page = page_pool_alloc(priv->rx_pool[q], &rx_buff->offset,
 307					&size, gfp_mask);
 308	if (unlikely(!rx_buff->page)) {
 309		/* We just set the data size to 0 for a failed mapping which
 310		 * should prevent DMA from happening...
 311		 */
 312		rx_desc->ds_cc = cpu_to_le16(0);
 313		return -ENOMEM;
 314	}
 315
 316	dma_addr = page_pool_get_dma_addr(rx_buff->page) + rx_buff->offset;
 317	dma_sync_single_for_device(ndev->dev.parent, dma_addr,
 318				   info->rx_buffer_size, DMA_FROM_DEVICE);
 319	rx_desc->dptr = cpu_to_le32(dma_addr);
 320
 321	/* The end of the RX buffer is used to store skb shared data, so we need
 322	 * to ensure that the hardware leaves enough space for this.
 323	 */
 324	rx_desc->ds_cc = cpu_to_le16(info->rx_buffer_size -
 325				     SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) -
 326				     ETH_FCS_LEN + sizeof(__sum16));
 327	return 0;
 328}
 329
 330static u32
 331ravb_rx_ring_refill(struct net_device *ndev, int q, u32 count, gfp_t gfp_mask)
 332{
 333	struct ravb_private *priv = netdev_priv(ndev);
 334	struct ravb_rx_desc *rx_desc;
 335	u32 i, entry;
 336
 337	for (i = 0; i < count; i++) {
 338		entry = (priv->dirty_rx[q] + i) % priv->num_rx_ring[q];
 339		rx_desc = ravb_rx_get_desc(priv, q, entry);
 340
 341		if (!priv->rx_buffers[q][entry].page) {
 342			if (unlikely(ravb_alloc_rx_buffer(ndev, q, entry,
 343							  gfp_mask, rx_desc)))
 344				break;
 345		}
 346		/* Descriptor type must be set after all the above writes */
 347		dma_wmb();
 348		rx_desc->die_dt = DT_FEMPTY;
 349	}
 350
 351	return i;
 352}
 353
 354/* Format skb and descriptor buffer for Ethernet AVB */
 355static void ravb_ring_format(struct net_device *ndev, int q)
 356{
 357	struct ravb_private *priv = netdev_priv(ndev);
 358	unsigned int num_tx_desc = priv->num_tx_desc;
 359	struct ravb_rx_desc *rx_desc;
 360	struct ravb_tx_desc *tx_desc;
 361	struct ravb_desc *desc;
 362	unsigned int tx_ring_size = sizeof(*tx_desc) * priv->num_tx_ring[q] *
 363				    num_tx_desc;
 364	unsigned int i;
 365
 366	priv->cur_rx[q] = 0;
 367	priv->cur_tx[q] = 0;
 368	priv->dirty_rx[q] = 0;
 369	priv->dirty_tx[q] = 0;
 370
 371	/* Regular RX descriptors have already been initialized by
 372	 * ravb_rx_ring_refill(), we just need to initialize the final link
 373	 * descriptor.
 374	 */
 375	rx_desc = ravb_rx_get_desc(priv, q, priv->num_rx_ring[q]);
 376	rx_desc->dptr = cpu_to_le32((u32)priv->rx_desc_dma[q]);
 377	rx_desc->die_dt = DT_LINKFIX; /* type */
 378
 379	memset(priv->tx_ring[q], 0, tx_ring_size);
 380	/* Build TX ring buffer */
 381	for (i = 0, tx_desc = priv->tx_ring[q]; i < priv->num_tx_ring[q];
 382	     i++, tx_desc++) {
 383		tx_desc->die_dt = DT_EEMPTY;
 384		if (num_tx_desc > 1) {
 385			tx_desc++;
 386			tx_desc->die_dt = DT_EEMPTY;
 387		}
 388	}
 389	tx_desc->dptr = cpu_to_le32((u32)priv->tx_desc_dma[q]);
 390	tx_desc->die_dt = DT_LINKFIX; /* type */
 391
 392	/* RX descriptor base address for best effort */
 393	desc = &priv->desc_bat[RX_QUEUE_OFFSET + q];
 394	desc->die_dt = DT_LINKFIX; /* type */
 395	desc->dptr = cpu_to_le32((u32)priv->rx_desc_dma[q]);
 396
 397	/* TX descriptor base address for best effort */
 398	desc = &priv->desc_bat[q];
 399	desc->die_dt = DT_LINKFIX; /* type */
 400	desc->dptr = cpu_to_le32((u32)priv->tx_desc_dma[q]);
 401}
 402
 403static void *ravb_alloc_rx_desc(struct net_device *ndev, int q)
 404{
 405	struct ravb_private *priv = netdev_priv(ndev);
 406	unsigned int ring_size;
 407
 408	ring_size = priv->info->rx_desc_size * (priv->num_rx_ring[q] + 1);
 409
 410	priv->rx_ring[q].raw = dma_alloc_coherent(ndev->dev.parent, ring_size,
 411						  &priv->rx_desc_dma[q],
 412						  GFP_KERNEL);
 413
 414	return priv->rx_ring[q].raw;
 415}
 416
 417/* Init skb and descriptor buffer for Ethernet AVB */
 418static int ravb_ring_init(struct net_device *ndev, int q)
 419{
 420	struct ravb_private *priv = netdev_priv(ndev);
 421	unsigned int num_tx_desc = priv->num_tx_desc;
 422	struct page_pool_params params = {
 423		.order = 0,
 424		.flags = PP_FLAG_DMA_MAP,
 425		.pool_size = priv->num_rx_ring[q],
 426		.nid = NUMA_NO_NODE,
 427		.dev = ndev->dev.parent,
 428		.dma_dir = DMA_FROM_DEVICE,
 429	};
 430	unsigned int ring_size;
 431	u32 num_filled;
 432
 433	/* Allocate RX page pool and buffers */
 434	priv->rx_pool[q] = page_pool_create(&params);
 435	if (IS_ERR(priv->rx_pool[q]))
 436		goto error;
 437
 438	/* Allocate RX buffers */
 439	priv->rx_buffers[q] = kcalloc(priv->num_rx_ring[q],
 440				      sizeof(*priv->rx_buffers[q]), GFP_KERNEL);
 441	if (!priv->rx_buffers[q])
 442		goto error;
 443
 444	/* Allocate TX skb rings */
 445	priv->tx_skb[q] = kcalloc(priv->num_tx_ring[q],
 446				  sizeof(*priv->tx_skb[q]), GFP_KERNEL);
 447	if (!priv->tx_skb[q])
 448		goto error;
 449
 450	/* Allocate all RX descriptors. */
 451	if (!ravb_alloc_rx_desc(ndev, q))
 452		goto error;
 453
 454	/* Populate RX ring buffer. */
 455	priv->dirty_rx[q] = 0;
 456	ring_size = priv->info->rx_desc_size * priv->num_rx_ring[q];
 457	memset(priv->rx_ring[q].raw, 0, ring_size);
 458	num_filled = ravb_rx_ring_refill(ndev, q, priv->num_rx_ring[q],
 459					 GFP_KERNEL);
 460	if (num_filled != priv->num_rx_ring[q])
 461		goto error;
 462
 463	if (num_tx_desc > 1) {
 464		/* Allocate rings for the aligned buffers */
 465		priv->tx_align[q] = kmalloc(DPTR_ALIGN * priv->num_tx_ring[q] +
 466					    DPTR_ALIGN - 1, GFP_KERNEL);
 467		if (!priv->tx_align[q])
 468			goto error;
 469	}
 470
 471	/* Allocate all TX descriptors. */
 472	ring_size = sizeof(struct ravb_tx_desc) *
 473		    (priv->num_tx_ring[q] * num_tx_desc + 1);
 474	priv->tx_ring[q] = dma_alloc_coherent(ndev->dev.parent, ring_size,
 475					      &priv->tx_desc_dma[q],
 476					      GFP_KERNEL);
 477	if (!priv->tx_ring[q])
 478		goto error;
 479
 480	return 0;
 481
 482error:
 483	ravb_ring_free(ndev, q);
 484
 485	return -ENOMEM;
 486}
 487
 488static void ravb_csum_init_gbeth(struct net_device *ndev)
 489{
 490	bool tx_enable = ndev->features & NETIF_F_HW_CSUM;
 491	bool rx_enable = ndev->features & NETIF_F_RXCSUM;
 492
 493	if (!(tx_enable || rx_enable))
 494		goto done;
 495
 496	ravb_write(ndev, 0, CSR0);
 497	if (ravb_wait(ndev, CSR0, CSR0_TPE | CSR0_RPE, 0)) {
 498		netdev_err(ndev, "Timeout enabling hardware checksum\n");
 499
 500		if (tx_enable)
 501			ndev->features &= ~NETIF_F_HW_CSUM;
 502
 503		if (rx_enable)
 504			ndev->features &= ~NETIF_F_RXCSUM;
 505	} else {
 506		if (tx_enable)
 507			ravb_write(ndev, CSR1_CSUM_ENABLE, CSR1);
 508
 509		if (rx_enable)
 510			ravb_write(ndev, CSR2_CSUM_ENABLE, CSR2);
 511	}
 512
 513done:
 514	ravb_write(ndev, CSR0_TPE | CSR0_RPE, CSR0);
 515}
 516
 517static void ravb_emac_init_gbeth(struct net_device *ndev)
 518{
 519	struct ravb_private *priv = netdev_priv(ndev);
 520
 521	if (priv->phy_interface == PHY_INTERFACE_MODE_MII) {
 522		ravb_write(ndev, (1000 << 16) | CXR35_SEL_XMII_MII, CXR35);
 523		ravb_modify(ndev, CXR31, CXR31_SEL_LINK0 | CXR31_SEL_LINK1, 0);
 524	} else {
 525		ravb_write(ndev, (1000 << 16) | CXR35_SEL_XMII_RGMII, CXR35);
 526		ravb_modify(ndev, CXR31, CXR31_SEL_LINK0 | CXR31_SEL_LINK1,
 527			    CXR31_SEL_LINK0);
 528	}
 529
 530	/* Receive frame limit set register */
 531	ravb_write(ndev, priv->info->rx_max_frame_size + ETH_FCS_LEN, RFLR);
 532
 533	/* EMAC Mode: PAUSE prohibition; Duplex; TX; RX; CRC Pass Through */
 534	ravb_write(ndev, ECMR_ZPF | ((priv->duplex > 0) ? ECMR_DM : 0) |
 535			 ECMR_TE | ECMR_RE | ECMR_RCPT |
 536			 ECMR_TXF | ECMR_RXF, ECMR);
 537
 538	ravb_set_rate_gbeth(ndev);
 539
 540	/* Set MAC address */
 541	ravb_write(ndev,
 542		   (ndev->dev_addr[0] << 24) | (ndev->dev_addr[1] << 16) |
 543		   (ndev->dev_addr[2] << 8)  | (ndev->dev_addr[3]), MAHR);
 544	ravb_write(ndev, (ndev->dev_addr[4] << 8)  | (ndev->dev_addr[5]), MALR);
 545
 546	/* E-MAC status register clear */
 547	ravb_write(ndev, ECSR_ICD | ECSR_LCHNG | ECSR_PFRI, ECSR);
 548
 549	ravb_csum_init_gbeth(ndev);
 550
 551	/* E-MAC interrupt enable register */
 552	ravb_write(ndev, ECSIPR_ICDIP, ECSIPR);
 553}
 554
 555static void ravb_emac_init_rcar(struct net_device *ndev)
 556{
 557	struct ravb_private *priv = netdev_priv(ndev);
 558
 559	/* Set receive frame length
 560	 *
 561	 * The length set here describes the frame from the destination address
 562	 * up to and including the CRC data. However only the frame data,
 563	 * excluding the CRC, are transferred to memory. To allow for the
 564	 * largest frames add the CRC length to the maximum Rx descriptor size.
 565	 */
 566	ravb_write(ndev, priv->info->rx_max_frame_size + ETH_FCS_LEN, RFLR);
 567
 568	/* EMAC Mode: PAUSE prohibition; Duplex; RX Checksum; TX; RX */
 569	ravb_write(ndev, ECMR_ZPF | ECMR_DM |
 570		   (ndev->features & NETIF_F_RXCSUM ? ECMR_RCSC : 0) |
 571		   ECMR_TE | ECMR_RE, ECMR);
 572
 573	ravb_set_rate_rcar(ndev);
 574
 575	/* Set MAC address */
 576	ravb_write(ndev,
 577		   (ndev->dev_addr[0] << 24) | (ndev->dev_addr[1] << 16) |
 578		   (ndev->dev_addr[2] << 8)  | (ndev->dev_addr[3]), MAHR);
 579	ravb_write(ndev,
 580		   (ndev->dev_addr[4] << 8)  | (ndev->dev_addr[5]), MALR);
 581
 582	/* E-MAC status register clear */
 583	ravb_write(ndev, ECSR_ICD | ECSR_MPD, ECSR);
 584
 585	/* E-MAC interrupt enable register */
 586	ravb_write(ndev, ECSIPR_ICDIP | ECSIPR_MPDIP | ECSIPR_LCHNGIP, ECSIPR);
 587}
 588
 589static void ravb_emac_init_rcar_gen4(struct net_device *ndev)
 590{
 591	struct ravb_private *priv = netdev_priv(ndev);
 592	bool mii = priv->phy_interface == PHY_INTERFACE_MODE_MII;
 593
 594	ravb_modify(ndev, APSR, APSR_MIISELECT, mii ? APSR_MIISELECT : 0);
 595
 596	ravb_emac_init_rcar(ndev);
 597}
 598
 599/* E-MAC init function */
 600static void ravb_emac_init(struct net_device *ndev)
 601{
 602	struct ravb_private *priv = netdev_priv(ndev);
 603	const struct ravb_hw_info *info = priv->info;
 604
 605	info->emac_init(ndev);
 606}
 607
 608static int ravb_dmac_init_gbeth(struct net_device *ndev)
 609{
 610	struct ravb_private *priv = netdev_priv(ndev);
 611	int error;
 612
 613	error = ravb_ring_init(ndev, RAVB_BE);
 614	if (error)
 615		return error;
 616
 617	/* Descriptor format */
 618	ravb_ring_format(ndev, RAVB_BE);
 619
 620	/* Set DMAC RX */
 621	ravb_write(ndev, 0x60000000, RCR);
 622
 623	/* Set Max Frame Length (RTC) */
 624	ravb_write(ndev, 0x7ffc0000 | priv->info->rx_max_frame_size, RTC);
 625
 626	/* Set FIFO size */
 627	ravb_write(ndev, 0x00222200, TGC);
 628
 629	ravb_write(ndev, 0, TCCR);
 630
 631	/* Frame receive */
 632	ravb_write(ndev, RIC0_FRE0, RIC0);
 633	/* Disable FIFO full warning */
 634	ravb_write(ndev, 0x0, RIC1);
 635	/* Receive FIFO full error, descriptor empty */
 636	ravb_write(ndev, RIC2_QFE0 | RIC2_RFFE, RIC2);
 637
 638	ravb_write(ndev, TIC_FTE0, TIC);
 639
 640	return 0;
 641}
 642
 643static int ravb_dmac_init_rcar(struct net_device *ndev)
 644{
 645	struct ravb_private *priv = netdev_priv(ndev);
 646	const struct ravb_hw_info *info = priv->info;
 647	int error;
 648
 649	error = ravb_ring_init(ndev, RAVB_BE);
 650	if (error)
 651		return error;
 652	error = ravb_ring_init(ndev, RAVB_NC);
 653	if (error) {
 654		ravb_ring_free(ndev, RAVB_BE);
 655		return error;
 656	}
 657
 658	/* Descriptor format */
 659	ravb_ring_format(ndev, RAVB_BE);
 660	ravb_ring_format(ndev, RAVB_NC);
 661
 662	/* Set AVB RX */
 663	ravb_write(ndev,
 664		   RCR_EFFS | RCR_ENCF | RCR_ETS0 | RCR_ESF | 0x18000000, RCR);
 665
 666	/* Set FIFO size */
 667	ravb_write(ndev, TGC_TQP_AVBMODE1 | 0x00112200, TGC);
 668
 669	/* Timestamp enable */
 670	ravb_write(ndev, TCCR_TFEN, TCCR);
 671
 672	/* Interrupt init: */
 673	if (info->multi_irqs) {
 674		/* Clear DIL.DPLx */
 675		ravb_write(ndev, 0, DIL);
 676		/* Set queue specific interrupt */
 677		ravb_write(ndev, CIE_CRIE | CIE_CTIE | CIE_CL0M, CIE);
 678	}
 679	/* Frame receive */
 680	ravb_write(ndev, RIC0_FRE0 | RIC0_FRE1, RIC0);
 681	/* Disable FIFO full warning */
 682	ravb_write(ndev, 0, RIC1);
 683	/* Receive FIFO full error, descriptor empty */
 684	ravb_write(ndev, RIC2_QFE0 | RIC2_QFE1 | RIC2_RFFE, RIC2);
 685	/* Frame transmitted, timestamp FIFO updated */
 686	ravb_write(ndev, TIC_FTE0 | TIC_FTE1 | TIC_TFUE, TIC);
 687
 688	return 0;
 689}
 690
 691/* Device init function for Ethernet AVB */
 692static int ravb_dmac_init(struct net_device *ndev)
 693{
 694	struct ravb_private *priv = netdev_priv(ndev);
 695	const struct ravb_hw_info *info = priv->info;
 696	int error;
 697
 698	/* Set CONFIG mode */
 699	error = ravb_set_opmode(ndev, CCC_OPC_CONFIG);
 700	if (error)
 701		return error;
 702
 703	error = info->dmac_init(ndev);
 704	if (error)
 705		return error;
 706
 707	/* Setting the control will start the AVB-DMAC process. */
 708	return ravb_set_opmode(ndev, CCC_OPC_OPERATION);
 709}
 710
 711static void ravb_get_tx_tstamp(struct net_device *ndev)
 712{
 713	struct ravb_private *priv = netdev_priv(ndev);
 714	struct ravb_tstamp_skb *ts_skb, *ts_skb2;
 715	struct skb_shared_hwtstamps shhwtstamps;
 716	struct sk_buff *skb;
 717	struct timespec64 ts;
 718	u16 tag, tfa_tag;
 719	int count;
 720	u32 tfa2;
 721
 722	count = (ravb_read(ndev, TSR) & TSR_TFFL) >> 8;
 723	while (count--) {
 724		tfa2 = ravb_read(ndev, TFA2);
 725		tfa_tag = (tfa2 & TFA2_TST) >> 16;
 726		ts.tv_nsec = (u64)ravb_read(ndev, TFA0);
 727		ts.tv_sec = ((u64)(tfa2 & TFA2_TSV) << 32) |
 728			    ravb_read(ndev, TFA1);
 729		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
 730		shhwtstamps.hwtstamp = timespec64_to_ktime(ts);
 731		list_for_each_entry_safe(ts_skb, ts_skb2, &priv->ts_skb_list,
 732					 list) {
 733			skb = ts_skb->skb;
 734			tag = ts_skb->tag;
 735			list_del(&ts_skb->list);
 736			kfree(ts_skb);
 737			if (tag == tfa_tag) {
 738				skb_tstamp_tx(skb, &shhwtstamps);
 739				dev_consume_skb_any(skb);
 740				break;
 741			} else {
 742				dev_kfree_skb_any(skb);
 743			}
 744		}
 745		ravb_modify(ndev, TCCR, TCCR_TFR, TCCR_TFR);
 746	}
 747}
 748
 749static void ravb_rx_csum_gbeth(struct sk_buff *skb)
 750{
 751	struct skb_shared_info *shinfo = skb_shinfo(skb);
 752	size_t csum_len;
 753	u16 *hw_csum;
 754
 755	/* The hardware checksum status is contained in 4 bytes appended to
 756	 * packet data.
 757	 *
 758	 * For ipv4, the first 2 bytes are the ip header checksum status. We can
 759	 * ignore this as it will always be re-checked in inet_gro_receive().
 760	 *
 761	 * The last 2 bytes are the protocol checksum status which will be zero
 762	 * if the checksum has been validated.
 763	 */
 764	csum_len = sizeof(*hw_csum) * 2;
 765	if (unlikely(skb->len < csum_len))
 766		return;
 767
 768	if (skb_is_nonlinear(skb)) {
 769		skb_frag_t *last_frag = &shinfo->frags[shinfo->nr_frags - 1];
 770
 771		hw_csum = (u16 *)(skb_frag_address(last_frag) +
 772				  skb_frag_size(last_frag));
 773		skb_frag_size_sub(last_frag, csum_len);
 774	} else {
 775		hw_csum = (u16 *)skb_tail_pointer(skb);
 776		skb_trim(skb, skb->len - csum_len);
 777	}
 778
 779	if (!get_unaligned(--hw_csum))
 780		skb->ip_summed = CHECKSUM_UNNECESSARY;
 781}
 782
 783static void ravb_rx_csum(struct sk_buff *skb)
 784{
 785	u8 *hw_csum;
 786
 787	/* The hardware checksum is contained in sizeof(__sum16) (2) bytes
 788	 * appended to packet data
 789	 */
 790	if (unlikely(skb->len < sizeof(__sum16)))
 791		return;
 792	hw_csum = skb_tail_pointer(skb) - sizeof(__sum16);
 793	skb->csum = csum_unfold((__force __sum16)get_unaligned_le16(hw_csum));
 794	skb->ip_summed = CHECKSUM_COMPLETE;
 795	skb_trim(skb, skb->len - sizeof(__sum16));
 796}
 797
 798/* Packet receive function for Gigabit Ethernet */
 799static int ravb_rx_gbeth(struct net_device *ndev, int budget, int q)
 800{
 801	struct ravb_private *priv = netdev_priv(ndev);
 802	const struct ravb_hw_info *info = priv->info;
 803	struct net_device_stats *stats;
 804	struct ravb_rx_desc *desc;
 805	int rx_packets = 0;
 806	u8  desc_status;
 807	u16 desc_len;
 808	u8  die_dt;
 809	int entry;
 810	int limit;
 811	int i;
 812
 813	limit = priv->dirty_rx[q] + priv->num_rx_ring[q] - priv->cur_rx[q];
 814	stats = &priv->stats[q];
 815
 816	for (i = 0; i < limit; i++, priv->cur_rx[q]++) {
 817		struct sk_buff *skb = NULL;
 818
 819		entry = priv->cur_rx[q] % priv->num_rx_ring[q];
 820		desc = &priv->rx_ring[q].desc[entry];
 821		if (rx_packets == budget || desc->die_dt == DT_FEMPTY)
 822			break;
 823
 824		/* Descriptor type must be checked before all other reads */
 825		dma_rmb();
 826		desc_status = desc->msc;
 827		desc_len = le16_to_cpu(desc->ds_cc) & RX_DS;
 828
 829		/* We use 0-byte descriptors to mark the DMA mapping errors */
 830		if (!desc_len)
 831			continue;
 832
 833		if (desc_status & MSC_MC)
 834			stats->multicast++;
 835
 836		if (desc_status & (MSC_CRC | MSC_RFE | MSC_RTSF | MSC_RTLF | MSC_CEEF)) {
 837			stats->rx_errors++;
 838			if (desc_status & MSC_CRC)
 839				stats->rx_crc_errors++;
 840			if (desc_status & MSC_RFE)
 841				stats->rx_frame_errors++;
 842			if (desc_status & (MSC_RTLF | MSC_RTSF))
 843				stats->rx_length_errors++;
 844			if (desc_status & MSC_CEEF)
 845				stats->rx_missed_errors++;
 846		} else {
 847			struct ravb_rx_buffer *rx_buff;
 848			void *rx_addr;
 849
 850			rx_buff = &priv->rx_buffers[q][entry];
 851			rx_addr = page_address(rx_buff->page) + rx_buff->offset;
 852			die_dt = desc->die_dt & 0xF0;
 853			dma_sync_single_for_cpu(ndev->dev.parent,
 854						le32_to_cpu(desc->dptr),
 855						desc_len, DMA_FROM_DEVICE);
 856
 857			switch (die_dt) {
 858			case DT_FSINGLE:
 859			case DT_FSTART:
 860				/* Start of packet: Set initial data length. */
 861				skb = napi_build_skb(rx_addr,
 862						     info->rx_buffer_size);
 863				if (unlikely(!skb)) {
 864					stats->rx_errors++;
 865					page_pool_put_page(priv->rx_pool[q],
 866							   rx_buff->page, 0,
 867							   true);
 868					goto refill;
 869				}
 870				skb_mark_for_recycle(skb);
 871				skb_put(skb, desc_len);
 872
 873				/* Save this skb if the packet spans multiple
 874				 * descriptors.
 875				 */
 876				if (die_dt == DT_FSTART)
 877					priv->rx_1st_skb = skb;
 878				break;
 879
 880			case DT_FMID:
 881			case DT_FEND:
 882				/* Continuing a packet: Add this buffer as an RX
 883				 * frag.
 884				 */
 885
 886				/* rx_1st_skb will be NULL if napi_build_skb()
 887				 * failed for the first descriptor of a
 888				 * multi-descriptor packet.
 889				 */
 890				if (unlikely(!priv->rx_1st_skb)) {
 891					stats->rx_errors++;
 892					page_pool_put_page(priv->rx_pool[q],
 893							   rx_buff->page, 0,
 894							   true);
 895
 896					/* We may find a DT_FSINGLE or DT_FSTART
 897					 * descriptor in the queue which we can
 898					 * process, so don't give up yet.
 899					 */
 900					continue;
 901				}
 902				skb_add_rx_frag(priv->rx_1st_skb,
 903						skb_shinfo(priv->rx_1st_skb)->nr_frags,
 904						rx_buff->page, rx_buff->offset,
 905						desc_len, info->rx_buffer_size);
 906
 907				/* Set skb to point at the whole packet so that
 908				 * we only need one code path for finishing a
 909				 * packet.
 910				 */
 911				skb = priv->rx_1st_skb;
 912			}
 913
 914			switch (die_dt) {
 915			case DT_FSINGLE:
 916			case DT_FEND:
 917				/* Finishing a packet: Determine protocol &
 918				 * checksum, hand off to NAPI and update our
 919				 * stats.
 920				 */
 921				skb->protocol = eth_type_trans(skb, ndev);
 922				if (ndev->features & NETIF_F_RXCSUM)
 923					ravb_rx_csum_gbeth(skb);
 924				stats->rx_bytes += skb->len;
 925				napi_gro_receive(&priv->napi[q], skb);
 926				rx_packets++;
 927
 928				/* Clear rx_1st_skb so that it will only be
 929				 * non-NULL when valid.
 930				 */
 931				priv->rx_1st_skb = NULL;
 932			}
 933
 934			/* Mark this RX buffer as consumed. */
 935			rx_buff->page = NULL;
 936		}
 937	}
 938
 939refill:
 940	/* Refill the RX ring buffers. */
 941	priv->dirty_rx[q] += ravb_rx_ring_refill(ndev, q,
 942						 priv->cur_rx[q] - priv->dirty_rx[q],
 943						 GFP_ATOMIC);
 944
 945	stats->rx_packets += rx_packets;
 946	return rx_packets;
 947}
 948
 949static void ravb_rx_rcar_hwstamp(struct ravb_private *priv, int q,
 950				 struct ravb_ex_rx_desc *desc,
 951				 struct sk_buff *skb)
 952{
 953	struct skb_shared_hwtstamps *shhwtstamps;
 954	struct timespec64 ts;
 955	bool get_ts;
 956
 957	if (q == RAVB_NC)
 958		get_ts = priv->tstamp_rx_ctrl != HWTSTAMP_FILTER_NONE;
 959	else
 960		get_ts = priv->tstamp_rx_ctrl == HWTSTAMP_FILTER_ALL;
 961
 962	if (!get_ts)
 963		return;
 964
 965	shhwtstamps = skb_hwtstamps(skb);
 966	memset(shhwtstamps, 0, sizeof(*shhwtstamps));
 967	ts.tv_sec = ((u64)le16_to_cpu(desc->ts_sh) << 32)
 968		| le32_to_cpu(desc->ts_sl);
 969	ts.tv_nsec = le32_to_cpu(desc->ts_n);
 970	shhwtstamps->hwtstamp = timespec64_to_ktime(ts);
 971}
 972
 973/* Packet receive function for Ethernet AVB */
 974static int ravb_rx_rcar(struct net_device *ndev, int budget, int q)
 975{
 976	struct ravb_private *priv = netdev_priv(ndev);
 977	const struct ravb_hw_info *info = priv->info;
 978	struct net_device_stats *stats = &priv->stats[q];
 979	struct ravb_ex_rx_desc *desc;
 980	unsigned int limit, i;
 981	struct sk_buff *skb;
 982	int rx_packets = 0;
 983	u8  desc_status;
 984	u16 pkt_len;
 985	int entry;
 986
 987	limit = priv->dirty_rx[q] + priv->num_rx_ring[q] - priv->cur_rx[q];
 988	for (i = 0; i < limit; i++, priv->cur_rx[q]++) {
 989		entry = priv->cur_rx[q] % priv->num_rx_ring[q];
 990		desc = &priv->rx_ring[q].ex_desc[entry];
 991		if (rx_packets == budget || desc->die_dt == DT_FEMPTY)
 992			break;
 993
 994		/* Descriptor type must be checked before all other reads */
 995		dma_rmb();
 996		desc_status = desc->msc;
 997		pkt_len = le16_to_cpu(desc->ds_cc) & RX_DS;
 998
 999		/* We use 0-byte descriptors to mark the DMA mapping errors */
1000		if (!pkt_len)
1001			continue;
1002
1003		if (desc_status & MSC_MC)
1004			stats->multicast++;
1005
1006		if (desc_status & (MSC_CRC | MSC_RFE | MSC_RTSF | MSC_RTLF |
1007				   MSC_CEEF)) {
1008			stats->rx_errors++;
1009			if (desc_status & MSC_CRC)
1010				stats->rx_crc_errors++;
1011			if (desc_status & MSC_RFE)
1012				stats->rx_frame_errors++;
1013			if (desc_status & (MSC_RTLF | MSC_RTSF))
1014				stats->rx_length_errors++;
1015			if (desc_status & MSC_CEEF)
1016				stats->rx_missed_errors++;
1017		} else {
1018			struct ravb_rx_buffer *rx_buff;
1019			void *rx_addr;
1020
1021			rx_buff = &priv->rx_buffers[q][entry];
1022			rx_addr = page_address(rx_buff->page) + rx_buff->offset;
1023			dma_sync_single_for_cpu(ndev->dev.parent,
1024						le32_to_cpu(desc->dptr),
1025						pkt_len, DMA_FROM_DEVICE);
1026
1027			skb = napi_build_skb(rx_addr, info->rx_buffer_size);
1028			if (unlikely(!skb)) {
1029				stats->rx_errors++;
1030				page_pool_put_page(priv->rx_pool[q],
1031						   rx_buff->page, 0, true);
1032				break;
1033			}
1034			skb_mark_for_recycle(skb);
1035
1036			ravb_rx_rcar_hwstamp(priv, q, desc, skb);
1037
1038			skb_put(skb, pkt_len);
1039			skb->protocol = eth_type_trans(skb, ndev);
1040			if (ndev->features & NETIF_F_RXCSUM)
1041				ravb_rx_csum(skb);
1042			napi_gro_receive(&priv->napi[q], skb);
1043			rx_packets++;
1044			stats->rx_bytes += pkt_len;
1045
1046			/* Mark this RX buffer as consumed. */
1047			rx_buff->page = NULL;
1048		}
1049	}
1050
1051	/* Refill the RX ring buffers. */
1052	priv->dirty_rx[q] += ravb_rx_ring_refill(ndev, q,
1053						 priv->cur_rx[q] - priv->dirty_rx[q],
1054						 GFP_ATOMIC);
1055
1056	stats->rx_packets += rx_packets;
1057	return rx_packets;
1058}
1059
1060/* Packet receive function for Ethernet AVB */
1061static int ravb_rx(struct net_device *ndev, int budget, int q)
1062{
1063	struct ravb_private *priv = netdev_priv(ndev);
1064	const struct ravb_hw_info *info = priv->info;
1065
1066	return info->receive(ndev, budget, q);
1067}
1068
1069static void ravb_rcv_snd_disable(struct net_device *ndev)
1070{
1071	/* Disable TX and RX */
1072	ravb_modify(ndev, ECMR, ECMR_RE | ECMR_TE, 0);
1073}
1074
1075static void ravb_rcv_snd_enable(struct net_device *ndev)
1076{
1077	/* Enable TX and RX */
1078	ravb_modify(ndev, ECMR, ECMR_RE | ECMR_TE, ECMR_RE | ECMR_TE);
1079}
1080
1081/* function for waiting dma process finished */
1082static int ravb_stop_dma(struct net_device *ndev)
1083{
1084	struct ravb_private *priv = netdev_priv(ndev);
1085	const struct ravb_hw_info *info = priv->info;
1086	int error;
1087
1088	/* Wait for stopping the hardware TX process */
1089	error = ravb_wait(ndev, TCCR, info->tccr_mask, 0);
1090
1091	if (error)
1092		return error;
1093
1094	error = ravb_wait(ndev, CSR, CSR_TPO0 | CSR_TPO1 | CSR_TPO2 | CSR_TPO3,
1095			  0);
1096	if (error)
1097		return error;
1098
1099	/* Stop the E-MAC's RX/TX processes. */
1100	ravb_rcv_snd_disable(ndev);
1101
1102	/* Wait for stopping the RX DMA process */
1103	error = ravb_wait(ndev, CSR, CSR_RPO, 0);
1104	if (error)
1105		return error;
1106
1107	/* Stop AVB-DMAC process */
1108	return ravb_set_opmode(ndev, CCC_OPC_CONFIG);
1109}
1110
1111/* E-MAC interrupt handler */
1112static void ravb_emac_interrupt_unlocked(struct net_device *ndev)
1113{
1114	struct ravb_private *priv = netdev_priv(ndev);
1115	u32 ecsr, psr;
1116
1117	ecsr = ravb_read(ndev, ECSR);
1118	ravb_write(ndev, ecsr, ECSR);	/* clear interrupt */
1119
1120	if (ecsr & ECSR_MPD)
1121		pm_wakeup_event(&priv->pdev->dev, 0);
1122	if (ecsr & ECSR_ICD)
1123		ndev->stats.tx_carrier_errors++;
1124	if (ecsr & ECSR_LCHNG) {
1125		/* Link changed */
1126		if (priv->no_avb_link)
1127			return;
1128		psr = ravb_read(ndev, PSR);
1129		if (priv->avb_link_active_low)
1130			psr ^= PSR_LMON;
1131		if (!(psr & PSR_LMON)) {
1132			/* DIsable RX and TX */
1133			ravb_rcv_snd_disable(ndev);
1134		} else {
1135			/* Enable RX and TX */
1136			ravb_rcv_snd_enable(ndev);
1137		}
1138	}
1139}
1140
1141static irqreturn_t ravb_emac_interrupt(int irq, void *dev_id)
1142{
1143	struct net_device *ndev = dev_id;
1144	struct ravb_private *priv = netdev_priv(ndev);
1145	struct device *dev = &priv->pdev->dev;
1146	irqreturn_t result = IRQ_HANDLED;
1147
1148	pm_runtime_get_noresume(dev);
1149
1150	if (unlikely(!pm_runtime_active(dev))) {
1151		result = IRQ_NONE;
1152		goto out_rpm_put;
1153	}
1154
1155	spin_lock(&priv->lock);
1156	ravb_emac_interrupt_unlocked(ndev);
1157	spin_unlock(&priv->lock);
1158
1159out_rpm_put:
1160	pm_runtime_put_noidle(dev);
1161	return result;
1162}
1163
1164/* Error interrupt handler */
1165static void ravb_error_interrupt(struct net_device *ndev)
1166{
1167	struct ravb_private *priv = netdev_priv(ndev);
1168	u32 eis, ris2;
1169
1170	eis = ravb_read(ndev, EIS);
1171	ravb_write(ndev, ~(EIS_QFS | EIS_RESERVED), EIS);
1172	if (eis & EIS_QFS) {
1173		ris2 = ravb_read(ndev, RIS2);
1174		ravb_write(ndev, ~(RIS2_QFF0 | RIS2_QFF1 | RIS2_RFFF | RIS2_RESERVED),
1175			   RIS2);
1176
1177		/* Receive Descriptor Empty int */
1178		if (ris2 & RIS2_QFF0)
1179			priv->stats[RAVB_BE].rx_over_errors++;
1180
1181		/* Receive Descriptor Empty int */
1182		if (ris2 & RIS2_QFF1)
1183			priv->stats[RAVB_NC].rx_over_errors++;
1184
1185		/* Receive FIFO Overflow int */
1186		if (ris2 & RIS2_RFFF)
1187			priv->rx_fifo_errors++;
1188	}
1189}
1190
1191static bool ravb_queue_interrupt(struct net_device *ndev, int q)
1192{
1193	struct ravb_private *priv = netdev_priv(ndev);
1194	const struct ravb_hw_info *info = priv->info;
1195	u32 ris0 = ravb_read(ndev, RIS0);
1196	u32 ric0 = ravb_read(ndev, RIC0);
1197	u32 tis  = ravb_read(ndev, TIS);
1198	u32 tic  = ravb_read(ndev, TIC);
1199
1200	if (((ris0 & ric0) & BIT(q)) || ((tis  & tic)  & BIT(q))) {
1201		if (napi_schedule_prep(&priv->napi[q])) {
1202			/* Mask RX and TX interrupts */
1203			if (!info->irq_en_dis) {
1204				ravb_write(ndev, ric0 & ~BIT(q), RIC0);
1205				ravb_write(ndev, tic & ~BIT(q), TIC);
1206			} else {
1207				ravb_write(ndev, BIT(q), RID0);
1208				ravb_write(ndev, BIT(q), TID);
1209			}
1210			__napi_schedule(&priv->napi[q]);
1211		} else {
1212			netdev_warn(ndev,
1213				    "ignoring interrupt, rx status 0x%08x, rx mask 0x%08x,\n",
1214				    ris0, ric0);
1215			netdev_warn(ndev,
1216				    "                    tx status 0x%08x, tx mask 0x%08x.\n",
1217				    tis, tic);
1218		}
1219		return true;
1220	}
1221	return false;
1222}
1223
1224static bool ravb_timestamp_interrupt(struct net_device *ndev)
1225{
1226	u32 tis = ravb_read(ndev, TIS);
1227
1228	if (tis & TIS_TFUF) {
1229		ravb_write(ndev, ~(TIS_TFUF | TIS_RESERVED), TIS);
1230		ravb_get_tx_tstamp(ndev);
1231		return true;
1232	}
1233	return false;
1234}
1235
1236static irqreturn_t ravb_interrupt(int irq, void *dev_id)
1237{
1238	struct net_device *ndev = dev_id;
1239	struct ravb_private *priv = netdev_priv(ndev);
1240	const struct ravb_hw_info *info = priv->info;
1241	struct device *dev = &priv->pdev->dev;
1242	irqreturn_t result = IRQ_NONE;
1243	u32 iss;
1244
1245	pm_runtime_get_noresume(dev);
1246
1247	if (unlikely(!pm_runtime_active(dev)))
1248		goto out_rpm_put;
1249
1250	spin_lock(&priv->lock);
1251	/* Get interrupt status */
1252	iss = ravb_read(ndev, ISS);
1253
1254	/* Received and transmitted interrupts */
1255	if (iss & (ISS_FRS | ISS_FTS | ISS_TFUS)) {
1256		int q;
1257
1258		/* Timestamp updated */
1259		if (ravb_timestamp_interrupt(ndev))
1260			result = IRQ_HANDLED;
1261
1262		/* Network control and best effort queue RX/TX */
1263		if (info->nc_queues) {
1264			for (q = RAVB_NC; q >= RAVB_BE; q--) {
1265				if (ravb_queue_interrupt(ndev, q))
1266					result = IRQ_HANDLED;
1267			}
1268		} else {
1269			if (ravb_queue_interrupt(ndev, RAVB_BE))
1270				result = IRQ_HANDLED;
1271		}
1272	}
1273
1274	/* E-MAC status summary */
1275	if (iss & ISS_MS) {
1276		ravb_emac_interrupt_unlocked(ndev);
1277		result = IRQ_HANDLED;
1278	}
1279
1280	/* Error status summary */
1281	if (iss & ISS_ES) {
1282		ravb_error_interrupt(ndev);
1283		result = IRQ_HANDLED;
1284	}
1285
1286	/* gPTP interrupt status summary */
1287	if (iss & ISS_CGIS) {
1288		ravb_ptp_interrupt(ndev);
1289		result = IRQ_HANDLED;
1290	}
1291
1292	spin_unlock(&priv->lock);
1293
1294out_rpm_put:
1295	pm_runtime_put_noidle(dev);
1296	return result;
1297}
1298
1299/* Timestamp/Error/gPTP interrupt handler */
1300static irqreturn_t ravb_multi_interrupt(int irq, void *dev_id)
1301{
1302	struct net_device *ndev = dev_id;
1303	struct ravb_private *priv = netdev_priv(ndev);
1304	struct device *dev = &priv->pdev->dev;
1305	irqreturn_t result = IRQ_NONE;
1306	u32 iss;
1307
1308	pm_runtime_get_noresume(dev);
1309
1310	if (unlikely(!pm_runtime_active(dev)))
1311		goto out_rpm_put;
1312
1313	spin_lock(&priv->lock);
1314	/* Get interrupt status */
1315	iss = ravb_read(ndev, ISS);
1316
1317	/* Timestamp updated */
1318	if ((iss & ISS_TFUS) && ravb_timestamp_interrupt(ndev))
1319		result = IRQ_HANDLED;
1320
1321	/* Error status summary */
1322	if (iss & ISS_ES) {
1323		ravb_error_interrupt(ndev);
1324		result = IRQ_HANDLED;
1325	}
1326
1327	/* gPTP interrupt status summary */
1328	if (iss & ISS_CGIS) {
1329		ravb_ptp_interrupt(ndev);
1330		result = IRQ_HANDLED;
1331	}
1332
1333	spin_unlock(&priv->lock);
1334
1335out_rpm_put:
1336	pm_runtime_put_noidle(dev);
1337	return result;
1338}
1339
1340static irqreturn_t ravb_dma_interrupt(int irq, void *dev_id, int q)
1341{
1342	struct net_device *ndev = dev_id;
1343	struct ravb_private *priv = netdev_priv(ndev);
1344	struct device *dev = &priv->pdev->dev;
1345	irqreturn_t result = IRQ_NONE;
1346
1347	pm_runtime_get_noresume(dev);
1348
1349	if (unlikely(!pm_runtime_active(dev)))
1350		goto out_rpm_put;
1351
1352	spin_lock(&priv->lock);
1353
1354	/* Network control/Best effort queue RX/TX */
1355	if (ravb_queue_interrupt(ndev, q))
1356		result = IRQ_HANDLED;
1357
1358	spin_unlock(&priv->lock);
1359
1360out_rpm_put:
1361	pm_runtime_put_noidle(dev);
1362	return result;
1363}
1364
1365static irqreturn_t ravb_be_interrupt(int irq, void *dev_id)
1366{
1367	return ravb_dma_interrupt(irq, dev_id, RAVB_BE);
1368}
1369
1370static irqreturn_t ravb_nc_interrupt(int irq, void *dev_id)
1371{
1372	return ravb_dma_interrupt(irq, dev_id, RAVB_NC);
1373}
1374
1375static int ravb_poll(struct napi_struct *napi, int budget)
1376{
1377	struct net_device *ndev = napi->dev;
1378	struct ravb_private *priv = netdev_priv(ndev);
1379	const struct ravb_hw_info *info = priv->info;
1380	unsigned long flags;
1381	int q = napi - priv->napi;
1382	int mask = BIT(q);
1383	int work_done;
1384
1385	/* Processing RX Descriptor Ring */
1386	/* Clear RX interrupt */
1387	ravb_write(ndev, ~(mask | RIS0_RESERVED), RIS0);
1388	work_done = ravb_rx(ndev, budget, q);
1389
1390	/* Processing TX Descriptor Ring */
1391	spin_lock_irqsave(&priv->lock, flags);
1392	/* Clear TX interrupt */
1393	ravb_write(ndev, ~(mask | TIS_RESERVED), TIS);
1394	ravb_tx_free(ndev, q, true);
1395	netif_wake_subqueue(ndev, q);
1396	spin_unlock_irqrestore(&priv->lock, flags);
1397
1398	/* Receive error message handling */
1399	priv->rx_over_errors = priv->stats[RAVB_BE].rx_over_errors;
1400	if (info->nc_queues)
1401		priv->rx_over_errors += priv->stats[RAVB_NC].rx_over_errors;
1402	if (priv->rx_over_errors != ndev->stats.rx_over_errors)
1403		ndev->stats.rx_over_errors = priv->rx_over_errors;
1404	if (priv->rx_fifo_errors != ndev->stats.rx_fifo_errors)
1405		ndev->stats.rx_fifo_errors = priv->rx_fifo_errors;
1406
1407	if (work_done < budget && napi_complete_done(napi, work_done)) {
1408		/* Re-enable RX/TX interrupts */
1409		spin_lock_irqsave(&priv->lock, flags);
1410		if (!info->irq_en_dis) {
1411			ravb_modify(ndev, RIC0, mask, mask);
1412			ravb_modify(ndev, TIC,  mask, mask);
1413		} else {
1414			ravb_write(ndev, mask, RIE0);
1415			ravb_write(ndev, mask, TIE);
1416		}
1417		spin_unlock_irqrestore(&priv->lock, flags);
1418	}
1419
1420	return work_done;
1421}
1422
1423static void ravb_set_duplex_gbeth(struct net_device *ndev)
1424{
1425	struct ravb_private *priv = netdev_priv(ndev);
1426
1427	ravb_modify(ndev, ECMR, ECMR_DM, priv->duplex > 0 ? ECMR_DM : 0);
1428}
1429
1430/* PHY state control function */
1431static void ravb_adjust_link(struct net_device *ndev)
1432{
1433	struct ravb_private *priv = netdev_priv(ndev);
1434	const struct ravb_hw_info *info = priv->info;
1435	struct phy_device *phydev = ndev->phydev;
1436	bool new_state = false;
1437	unsigned long flags;
1438
1439	spin_lock_irqsave(&priv->lock, flags);
1440
1441	/* Disable TX and RX right over here, if E-MAC change is ignored */
1442	if (priv->no_avb_link)
1443		ravb_rcv_snd_disable(ndev);
1444
1445	if (phydev->link) {
1446		if (info->half_duplex && phydev->duplex != priv->duplex) {
1447			new_state = true;
1448			priv->duplex = phydev->duplex;
1449			ravb_set_duplex_gbeth(ndev);
1450		}
1451
1452		if (phydev->speed != priv->speed) {
1453			new_state = true;
1454			priv->speed = phydev->speed;
1455			info->set_rate(ndev);
1456		}
1457		if (!priv->link) {
1458			ravb_modify(ndev, ECMR, ECMR_TXF, 0);
1459			new_state = true;
1460			priv->link = phydev->link;
1461		}
1462	} else if (priv->link) {
1463		new_state = true;
1464		priv->link = 0;
1465		priv->speed = 0;
1466		if (info->half_duplex)
1467			priv->duplex = -1;
1468	}
1469
1470	/* Enable TX and RX right over here, if E-MAC change is ignored */
1471	if (priv->no_avb_link && phydev->link)
1472		ravb_rcv_snd_enable(ndev);
1473
1474	spin_unlock_irqrestore(&priv->lock, flags);
1475
1476	if (new_state && netif_msg_link(priv))
1477		phy_print_status(phydev);
1478}
1479
1480/* PHY init function */
1481static int ravb_phy_init(struct net_device *ndev)
1482{
1483	struct device_node *np = ndev->dev.parent->of_node;
1484	struct ravb_private *priv = netdev_priv(ndev);
1485	const struct ravb_hw_info *info = priv->info;
1486	struct phy_device *phydev;
1487	struct device_node *pn;
1488	phy_interface_t iface;
1489	int err;
1490
1491	priv->link = 0;
1492	priv->speed = 0;
1493	priv->duplex = -1;
1494
1495	/* Try connecting to PHY */
1496	pn = of_parse_phandle(np, "phy-handle", 0);
1497	if (!pn) {
1498		/* In the case of a fixed PHY, the DT node associated
1499		 * to the PHY is the Ethernet MAC DT node.
1500		 */
1501		if (of_phy_is_fixed_link(np)) {
1502			err = of_phy_register_fixed_link(np);
1503			if (err)
1504				return err;
1505		}
1506		pn = of_node_get(np);
1507	}
1508
1509	iface = priv->rgmii_override ? PHY_INTERFACE_MODE_RGMII
1510				     : priv->phy_interface;
1511	phydev = of_phy_connect(ndev, pn, ravb_adjust_link, 0, iface);
1512	of_node_put(pn);
1513	if (!phydev) {
1514		netdev_err(ndev, "failed to connect PHY\n");
1515		err = -ENOENT;
1516		goto err_deregister_fixed_link;
1517	}
1518
1519	if (!info->half_duplex) {
1520		/* 10BASE, Pause and Asym Pause is not supported */
1521		phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);
1522		phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT);
1523		phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_Pause_BIT);
1524		phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_Asym_Pause_BIT);
1525
1526		/* Half Duplex is not supported */
1527		phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);
1528		phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT);
1529	}
1530
1531	phy_attached_info(phydev);
1532
1533	return 0;
1534
1535err_deregister_fixed_link:
1536	if (of_phy_is_fixed_link(np))
1537		of_phy_deregister_fixed_link(np);
1538
1539	return err;
1540}
1541
1542/* PHY control start function */
1543static int ravb_phy_start(struct net_device *ndev)
1544{
1545	int error;
1546
1547	error = ravb_phy_init(ndev);
1548	if (error)
1549		return error;
1550
1551	phy_start(ndev->phydev);
1552
1553	return 0;
1554}
1555
1556static u32 ravb_get_msglevel(struct net_device *ndev)
1557{
1558	struct ravb_private *priv = netdev_priv(ndev);
1559
1560	return priv->msg_enable;
1561}
1562
1563static void ravb_set_msglevel(struct net_device *ndev, u32 value)
1564{
1565	struct ravb_private *priv = netdev_priv(ndev);
1566
1567	priv->msg_enable = value;
1568}
1569
1570static const char ravb_gstrings_stats_gbeth[][ETH_GSTRING_LEN] = {
1571	"rx_queue_0_current",
1572	"tx_queue_0_current",
1573	"rx_queue_0_dirty",
1574	"tx_queue_0_dirty",
1575	"rx_queue_0_packets",
1576	"tx_queue_0_packets",
1577	"rx_queue_0_bytes",
1578	"tx_queue_0_bytes",
1579	"rx_queue_0_mcast_packets",
1580	"rx_queue_0_errors",
1581	"rx_queue_0_crc_errors",
1582	"rx_queue_0_frame_errors",
1583	"rx_queue_0_length_errors",
1584	"rx_queue_0_csum_offload_errors",
1585	"rx_queue_0_over_errors",
1586};
1587
1588static const char ravb_gstrings_stats[][ETH_GSTRING_LEN] = {
1589	"rx_queue_0_current",
1590	"tx_queue_0_current",
1591	"rx_queue_0_dirty",
1592	"tx_queue_0_dirty",
1593	"rx_queue_0_packets",
1594	"tx_queue_0_packets",
1595	"rx_queue_0_bytes",
1596	"tx_queue_0_bytes",
1597	"rx_queue_0_mcast_packets",
1598	"rx_queue_0_errors",
1599	"rx_queue_0_crc_errors",
1600	"rx_queue_0_frame_errors",
1601	"rx_queue_0_length_errors",
1602	"rx_queue_0_missed_errors",
1603	"rx_queue_0_over_errors",
1604
1605	"rx_queue_1_current",
1606	"tx_queue_1_current",
1607	"rx_queue_1_dirty",
1608	"tx_queue_1_dirty",
1609	"rx_queue_1_packets",
1610	"tx_queue_1_packets",
1611	"rx_queue_1_bytes",
1612	"tx_queue_1_bytes",
1613	"rx_queue_1_mcast_packets",
1614	"rx_queue_1_errors",
1615	"rx_queue_1_crc_errors",
1616	"rx_queue_1_frame_errors",
1617	"rx_queue_1_length_errors",
1618	"rx_queue_1_missed_errors",
1619	"rx_queue_1_over_errors",
1620};
1621
1622static int ravb_get_sset_count(struct net_device *netdev, int sset)
1623{
1624	struct ravb_private *priv = netdev_priv(netdev);
1625	const struct ravb_hw_info *info = priv->info;
1626
1627	switch (sset) {
1628	case ETH_SS_STATS:
1629		return info->stats_len;
1630	default:
1631		return -EOPNOTSUPP;
1632	}
1633}
1634
1635static void ravb_get_ethtool_stats(struct net_device *ndev,
1636				   struct ethtool_stats *estats, u64 *data)
1637{
1638	struct ravb_private *priv = netdev_priv(ndev);
1639	const struct ravb_hw_info *info = priv->info;
1640	int num_rx_q;
1641	int i = 0;
1642	int q;
1643
1644	num_rx_q = info->nc_queues ? NUM_RX_QUEUE : 1;
1645	/* Device-specific stats */
1646	for (q = RAVB_BE; q < num_rx_q; q++) {
1647		struct net_device_stats *stats = &priv->stats[q];
1648
1649		data[i++] = priv->cur_rx[q];
1650		data[i++] = priv->cur_tx[q];
1651		data[i++] = priv->dirty_rx[q];
1652		data[i++] = priv->dirty_tx[q];
1653		data[i++] = stats->rx_packets;
1654		data[i++] = stats->tx_packets;
1655		data[i++] = stats->rx_bytes;
1656		data[i++] = stats->tx_bytes;
1657		data[i++] = stats->multicast;
1658		data[i++] = stats->rx_errors;
1659		data[i++] = stats->rx_crc_errors;
1660		data[i++] = stats->rx_frame_errors;
1661		data[i++] = stats->rx_length_errors;
1662		data[i++] = stats->rx_missed_errors;
1663		data[i++] = stats->rx_over_errors;
1664	}
1665}
1666
1667static void ravb_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
1668{
1669	struct ravb_private *priv = netdev_priv(ndev);
1670	const struct ravb_hw_info *info = priv->info;
1671
1672	switch (stringset) {
1673	case ETH_SS_STATS:
1674		memcpy(data, info->gstrings_stats, info->gstrings_size);
1675		break;
1676	}
1677}
1678
1679static void ravb_get_ringparam(struct net_device *ndev,
1680			       struct ethtool_ringparam *ring,
1681			       struct kernel_ethtool_ringparam *kernel_ring,
1682			       struct netlink_ext_ack *extack)
1683{
1684	struct ravb_private *priv = netdev_priv(ndev);
1685
1686	ring->rx_max_pending = BE_RX_RING_MAX;
1687	ring->tx_max_pending = BE_TX_RING_MAX;
1688	ring->rx_pending = priv->num_rx_ring[RAVB_BE];
1689	ring->tx_pending = priv->num_tx_ring[RAVB_BE];
1690}
1691
1692static int ravb_set_ringparam(struct net_device *ndev,
1693			      struct ethtool_ringparam *ring,
1694			      struct kernel_ethtool_ringparam *kernel_ring,
1695			      struct netlink_ext_ack *extack)
1696{
1697	struct ravb_private *priv = netdev_priv(ndev);
1698	const struct ravb_hw_info *info = priv->info;
1699	int error;
1700
1701	if (ring->tx_pending > BE_TX_RING_MAX ||
1702	    ring->rx_pending > BE_RX_RING_MAX ||
1703	    ring->tx_pending < BE_TX_RING_MIN ||
1704	    ring->rx_pending < BE_RX_RING_MIN)
1705		return -EINVAL;
1706	if (ring->rx_mini_pending || ring->rx_jumbo_pending)
1707		return -EINVAL;
1708
1709	if (netif_running(ndev)) {
1710		netif_device_detach(ndev);
1711		/* Stop PTP Clock driver */
1712		if (info->gptp)
1713			ravb_ptp_stop(ndev);
1714		/* Wait for DMA stopping */
1715		error = ravb_stop_dma(ndev);
1716		if (error) {
1717			netdev_err(ndev,
1718				   "cannot set ringparam! Any AVB processes are still running?\n");
1719			return error;
1720		}
1721		synchronize_irq(ndev->irq);
1722
1723		/* Free all the skb's in the RX queue and the DMA buffers. */
1724		ravb_ring_free(ndev, RAVB_BE);
1725		if (info->nc_queues)
1726			ravb_ring_free(ndev, RAVB_NC);
1727	}
1728
1729	/* Set new parameters */
1730	priv->num_rx_ring[RAVB_BE] = ring->rx_pending;
1731	priv->num_tx_ring[RAVB_BE] = ring->tx_pending;
1732
1733	if (netif_running(ndev)) {
1734		error = ravb_dmac_init(ndev);
1735		if (error) {
1736			netdev_err(ndev,
1737				   "%s: ravb_dmac_init() failed, error %d\n",
1738				   __func__, error);
1739			return error;
1740		}
1741
1742		ravb_emac_init(ndev);
1743
1744		/* Initialise PTP Clock driver */
1745		if (info->gptp)
1746			ravb_ptp_init(ndev, priv->pdev);
1747
1748		netif_device_attach(ndev);
1749	}
1750
1751	return 0;
1752}
1753
1754static int ravb_get_ts_info(struct net_device *ndev,
1755			    struct kernel_ethtool_ts_info *info)
1756{
1757	struct ravb_private *priv = netdev_priv(ndev);
1758	const struct ravb_hw_info *hw_info = priv->info;
1759
1760	if (hw_info->gptp || hw_info->ccc_gac) {
1761		info->so_timestamping =
1762			SOF_TIMESTAMPING_TX_SOFTWARE |
1763			SOF_TIMESTAMPING_TX_HARDWARE |
1764			SOF_TIMESTAMPING_RX_HARDWARE |
1765			SOF_TIMESTAMPING_RAW_HARDWARE;
1766		info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
1767		info->rx_filters =
1768			(1 << HWTSTAMP_FILTER_NONE) |
1769			(1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
1770			(1 << HWTSTAMP_FILTER_ALL);
1771		info->phc_index = ptp_clock_index(priv->ptp.clock);
1772	}
1773
1774	return 0;
1775}
1776
1777static void ravb_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
1778{
1779	struct ravb_private *priv = netdev_priv(ndev);
1780
1781	wol->supported = WAKE_MAGIC;
1782	wol->wolopts = priv->wol_enabled ? WAKE_MAGIC : 0;
1783}
1784
1785static int ravb_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
1786{
1787	struct ravb_private *priv = netdev_priv(ndev);
1788	const struct ravb_hw_info *info = priv->info;
1789
1790	if (!info->magic_pkt || (wol->wolopts & ~WAKE_MAGIC))
1791		return -EOPNOTSUPP;
1792
1793	priv->wol_enabled = !!(wol->wolopts & WAKE_MAGIC);
1794
1795	device_set_wakeup_enable(&priv->pdev->dev, priv->wol_enabled);
1796
1797	return 0;
1798}
1799
1800static const struct ethtool_ops ravb_ethtool_ops = {
1801	.nway_reset		= phy_ethtool_nway_reset,
1802	.get_msglevel		= ravb_get_msglevel,
1803	.set_msglevel		= ravb_set_msglevel,
1804	.get_link		= ethtool_op_get_link,
1805	.get_strings		= ravb_get_strings,
1806	.get_ethtool_stats	= ravb_get_ethtool_stats,
1807	.get_sset_count		= ravb_get_sset_count,
1808	.get_ringparam		= ravb_get_ringparam,
1809	.set_ringparam		= ravb_set_ringparam,
1810	.get_ts_info		= ravb_get_ts_info,
1811	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
1812	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
1813	.get_wol		= ravb_get_wol,
1814	.set_wol		= ravb_set_wol,
1815};
1816
1817static int ravb_set_config_mode(struct net_device *ndev)
1818{
1819	struct ravb_private *priv = netdev_priv(ndev);
1820	const struct ravb_hw_info *info = priv->info;
1821	int error;
1822
1823	if (info->gptp) {
1824		error = ravb_set_opmode(ndev, CCC_OPC_CONFIG);
1825		if (error)
1826			return error;
1827		/* Set CSEL value */
1828		ravb_modify(ndev, CCC, CCC_CSEL, CCC_CSEL_HPB);
1829	} else if (info->ccc_gac) {
1830		error = ravb_set_opmode(ndev, CCC_OPC_CONFIG | CCC_GAC | CCC_CSEL_HPB);
1831	} else {
1832		error = ravb_set_opmode(ndev, CCC_OPC_CONFIG);
1833	}
1834
1835	return error;
1836}
1837
1838static void ravb_set_gti(struct net_device *ndev)
1839{
1840	struct ravb_private *priv = netdev_priv(ndev);
1841	const struct ravb_hw_info *info = priv->info;
1842
1843	if (!(info->gptp || info->ccc_gac))
1844		return;
1845
1846	ravb_write(ndev, priv->gti_tiv, GTI);
1847
1848	/* Request GTI loading */
1849	ravb_modify(ndev, GCCR, GCCR_LTI, GCCR_LTI);
1850}
1851
1852static int ravb_compute_gti(struct net_device *ndev)
1853{
1854	struct ravb_private *priv = netdev_priv(ndev);
1855	const struct ravb_hw_info *info = priv->info;
1856	struct device *dev = ndev->dev.parent;
1857	unsigned long rate;
1858	u64 inc;
1859
1860	if (!(info->gptp || info->ccc_gac))
1861		return 0;
1862
1863	if (info->gptp_ref_clk)
1864		rate = clk_get_rate(priv->gptp_clk);
1865	else
1866		rate = clk_get_rate(priv->clk);
1867	if (!rate)
1868		return -EINVAL;
1869
1870	inc = div64_ul(1000000000ULL << 20, rate);
1871
1872	if (inc < GTI_TIV_MIN || inc > GTI_TIV_MAX) {
1873		dev_err(dev, "gti.tiv increment 0x%llx is outside the range 0x%x - 0x%x\n",
1874			inc, GTI_TIV_MIN, GTI_TIV_MAX);
1875		return -EINVAL;
1876	}
1877	priv->gti_tiv = inc;
1878
1879	return 0;
1880}
1881
1882/* Set tx and rx clock internal delay modes */
1883static void ravb_parse_delay_mode(struct device_node *np, struct net_device *ndev)
1884{
1885	struct ravb_private *priv = netdev_priv(ndev);
1886	bool explicit_delay = false;
1887	u32 delay;
1888
1889	if (!priv->info->internal_delay)
1890		return;
1891
1892	if (!of_property_read_u32(np, "rx-internal-delay-ps", &delay)) {
1893		/* Valid values are 0 and 1800, according to DT bindings */
1894		priv->rxcidm = !!delay;
1895		explicit_delay = true;
1896	}
1897	if (!of_property_read_u32(np, "tx-internal-delay-ps", &delay)) {
1898		/* Valid values are 0 and 2000, according to DT bindings */
1899		priv->txcidm = !!delay;
1900		explicit_delay = true;
1901	}
1902
1903	if (explicit_delay)
1904		return;
1905
1906	/* Fall back to legacy rgmii-*id behavior */
1907	if (priv->phy_interface == PHY_INTERFACE_MODE_RGMII_ID ||
1908	    priv->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID) {
1909		priv->rxcidm = 1;
1910		priv->rgmii_override = 1;
1911	}
1912
1913	if (priv->phy_interface == PHY_INTERFACE_MODE_RGMII_ID ||
1914	    priv->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID) {
1915		priv->txcidm = 1;
1916		priv->rgmii_override = 1;
1917	}
1918}
1919
1920static void ravb_set_delay_mode(struct net_device *ndev)
1921{
1922	struct ravb_private *priv = netdev_priv(ndev);
1923	u32 set = 0;
1924
1925	if (!priv->info->internal_delay)
1926		return;
1927
1928	if (priv->rxcidm)
1929		set |= APSR_RDM;
1930	if (priv->txcidm)
1931		set |= APSR_TDM;
1932	ravb_modify(ndev, APSR, APSR_RDM | APSR_TDM, set);
1933}
1934
1935/* Network device open function for Ethernet AVB */
1936static int ravb_open(struct net_device *ndev)
1937{
1938	struct ravb_private *priv = netdev_priv(ndev);
1939	const struct ravb_hw_info *info = priv->info;
1940	struct device *dev = &priv->pdev->dev;
1941	int error;
1942
1943	napi_enable(&priv->napi[RAVB_BE]);
1944	if (info->nc_queues)
1945		napi_enable(&priv->napi[RAVB_NC]);
1946
1947	error = pm_runtime_resume_and_get(dev);
1948	if (error < 0)
1949		goto out_napi_off;
1950
1951	/* Set AVB config mode */
1952	error = ravb_set_config_mode(ndev);
1953	if (error)
1954		goto out_rpm_put;
1955
1956	ravb_set_delay_mode(ndev);
1957	ravb_write(ndev, priv->desc_bat_dma, DBAT);
1958
1959	/* Device init */
1960	error = ravb_dmac_init(ndev);
1961	if (error)
1962		goto out_set_reset;
1963
1964	ravb_emac_init(ndev);
1965
1966	ravb_set_gti(ndev);
1967
1968	/* Initialise PTP Clock driver */
1969	if (info->gptp || info->ccc_gac)
1970		ravb_ptp_init(ndev, priv->pdev);
1971
1972	/* PHY control start */
1973	error = ravb_phy_start(ndev);
1974	if (error)
1975		goto out_ptp_stop;
1976
1977	netif_tx_start_all_queues(ndev);
1978
1979	return 0;
1980
1981out_ptp_stop:
1982	/* Stop PTP Clock driver */
1983	if (info->gptp || info->ccc_gac)
1984		ravb_ptp_stop(ndev);
1985	ravb_stop_dma(ndev);
1986out_set_reset:
1987	ravb_set_opmode(ndev, CCC_OPC_RESET);
1988out_rpm_put:
1989	pm_runtime_put_autosuspend(dev);
1990out_napi_off:
1991	if (info->nc_queues)
1992		napi_disable(&priv->napi[RAVB_NC]);
1993	napi_disable(&priv->napi[RAVB_BE]);
1994	return error;
1995}
1996
1997/* Timeout function for Ethernet AVB */
1998static void ravb_tx_timeout(struct net_device *ndev, unsigned int txqueue)
1999{
2000	struct ravb_private *priv = netdev_priv(ndev);
2001
2002	netif_err(priv, tx_err, ndev,
2003		  "transmit timed out, status %08x, resetting...\n",
2004		  ravb_read(ndev, ISS));
2005
2006	/* tx_errors count up */
2007	ndev->stats.tx_errors++;
2008
2009	schedule_work(&priv->work);
2010}
2011
2012static void ravb_tx_timeout_work(struct work_struct *work)
2013{
2014	struct ravb_private *priv = container_of(work, struct ravb_private,
2015						 work);
2016	const struct ravb_hw_info *info = priv->info;
2017	struct net_device *ndev = priv->ndev;
2018	int error;
2019
2020	if (!rtnl_trylock()) {
2021		usleep_range(1000, 2000);
2022		schedule_work(&priv->work);
2023		return;
2024	}
2025
2026	netif_tx_stop_all_queues(ndev);
2027
2028	/* Stop PTP Clock driver */
2029	if (info->gptp)
2030		ravb_ptp_stop(ndev);
2031
2032	/* Wait for DMA stopping */
2033	if (ravb_stop_dma(ndev)) {
2034		/* If ravb_stop_dma() fails, the hardware is still operating
2035		 * for TX and/or RX. So, this should not call the following
2036		 * functions because ravb_dmac_init() is possible to fail too.
2037		 * Also, this should not retry ravb_stop_dma() again and again
2038		 * here because it's possible to wait forever. So, this just
2039		 * re-enables the TX and RX and skip the following
2040		 * re-initialization procedure.
2041		 */
2042		ravb_rcv_snd_enable(ndev);
2043		goto out;
2044	}
2045
2046	ravb_ring_free(ndev, RAVB_BE);
2047	if (info->nc_queues)
2048		ravb_ring_free(ndev, RAVB_NC);
2049
2050	/* Device init */
2051	error = ravb_dmac_init(ndev);
2052	if (error) {
2053		/* If ravb_dmac_init() fails, descriptors are freed. So, this
2054		 * should return here to avoid re-enabling the TX and RX in
2055		 * ravb_emac_init().
2056		 */
2057		netdev_err(ndev, "%s: ravb_dmac_init() failed, error %d\n",
2058			   __func__, error);
2059		goto out_unlock;
2060	}
2061	ravb_emac_init(ndev);
2062
2063out:
2064	/* Initialise PTP Clock driver */
2065	if (info->gptp)
2066		ravb_ptp_init(ndev, priv->pdev);
2067
2068	netif_tx_start_all_queues(ndev);
2069
2070out_unlock:
2071	rtnl_unlock();
2072}
2073
2074static bool ravb_can_tx_csum_gbeth(struct sk_buff *skb)
2075{
2076	u16 net_protocol = ntohs(skb->protocol);
2077	u8 inner_protocol;
2078
2079	/* GbEth IP can calculate the checksum if:
2080	 * - there are zero or one VLAN headers with TPID=0x8100
2081	 * - the network protocol is IPv4 or IPv6
2082	 * - the transport protocol is TCP, UDP or ICMP
2083	 * - the packet is not fragmented
2084	 */
2085
2086	if (net_protocol == ETH_P_8021Q) {
2087		struct vlan_hdr vhdr, *vh;
2088
2089		vh = skb_header_pointer(skb, ETH_HLEN, sizeof(vhdr), &vhdr);
2090		if (!vh)
2091			return false;
2092
2093		net_protocol = ntohs(vh->h_vlan_encapsulated_proto);
2094	}
2095
2096	switch (net_protocol) {
2097	case ETH_P_IP:
2098		inner_protocol = ip_hdr(skb)->protocol;
2099		break;
2100	case ETH_P_IPV6:
2101		inner_protocol = ipv6_hdr(skb)->nexthdr;
2102		break;
2103	default:
2104		return false;
2105	}
2106
2107	switch (inner_protocol) {
2108	case IPPROTO_TCP:
2109	case IPPROTO_UDP:
2110		return true;
2111	default:
2112		return false;
2113	}
2114}
2115
2116/* Packet transmit function for Ethernet AVB */
2117static netdev_tx_t ravb_start_xmit(struct sk_buff *skb, struct net_device *ndev)
2118{
2119	struct ravb_private *priv = netdev_priv(ndev);
2120	const struct ravb_hw_info *info = priv->info;
2121	unsigned int num_tx_desc = priv->num_tx_desc;
2122	u16 q = skb_get_queue_mapping(skb);
2123	struct ravb_tstamp_skb *ts_skb;
2124	struct ravb_tx_desc *desc;
2125	unsigned long flags;
2126	dma_addr_t dma_addr;
2127	void *buffer;
2128	u32 entry;
2129	u32 len;
2130
2131	if (skb->ip_summed == CHECKSUM_PARTIAL && !ravb_can_tx_csum_gbeth(skb))
2132		skb_checksum_help(skb);
2133
2134	spin_lock_irqsave(&priv->lock, flags);
2135	if (priv->cur_tx[q] - priv->dirty_tx[q] > (priv->num_tx_ring[q] - 1) *
2136	    num_tx_desc) {
2137		netif_err(priv, tx_queued, ndev,
2138			  "still transmitting with the full ring!\n");
2139		netif_stop_subqueue(ndev, q);
2140		spin_unlock_irqrestore(&priv->lock, flags);
2141		return NETDEV_TX_BUSY;
2142	}
2143
2144	if (skb_put_padto(skb, ETH_ZLEN))
2145		goto exit;
2146
2147	entry = priv->cur_tx[q] % (priv->num_tx_ring[q] * num_tx_desc);
2148	priv->tx_skb[q][entry / num_tx_desc] = skb;
2149
2150	if (num_tx_desc > 1) {
2151		buffer = PTR_ALIGN(priv->tx_align[q], DPTR_ALIGN) +
2152			 entry / num_tx_desc * DPTR_ALIGN;
2153		len = PTR_ALIGN(skb->data, DPTR_ALIGN) - skb->data;
2154
2155		/* Zero length DMA descriptors are problematic as they seem
2156		 * to terminate DMA transfers. Avoid them by simply using a
2157		 * length of DPTR_ALIGN (4) when skb data is aligned to
2158		 * DPTR_ALIGN.
2159		 *
2160		 * As skb is guaranteed to have at least ETH_ZLEN (60)
2161		 * bytes of data by the call to skb_put_padto() above this
2162		 * is safe with respect to both the length of the first DMA
2163		 * descriptor (len) overflowing the available data and the
2164		 * length of the second DMA descriptor (skb->len - len)
2165		 * being negative.
2166		 */
2167		if (len == 0)
2168			len = DPTR_ALIGN;
2169
2170		memcpy(buffer, skb->data, len);
2171		dma_addr = dma_map_single(ndev->dev.parent, buffer, len,
2172					  DMA_TO_DEVICE);
2173		if (dma_mapping_error(ndev->dev.parent, dma_addr))
2174			goto drop;
2175
2176		desc = &priv->tx_ring[q][entry];
2177		desc->ds_tagl = cpu_to_le16(len);
2178		desc->dptr = cpu_to_le32(dma_addr);
2179
2180		buffer = skb->data + len;
2181		len = skb->len - len;
2182		dma_addr = dma_map_single(ndev->dev.parent, buffer, len,
2183					  DMA_TO_DEVICE);
2184		if (dma_mapping_error(ndev->dev.parent, dma_addr))
2185			goto unmap;
2186
2187		desc++;
2188	} else {
2189		desc = &priv->tx_ring[q][entry];
2190		len = skb->len;
2191		dma_addr = dma_map_single(ndev->dev.parent, skb->data, skb->len,
2192					  DMA_TO_DEVICE);
2193		if (dma_mapping_error(ndev->dev.parent, dma_addr))
2194			goto drop;
2195	}
2196	desc->ds_tagl = cpu_to_le16(len);
2197	desc->dptr = cpu_to_le32(dma_addr);
2198
2199	/* TX timestamp required */
2200	if (info->gptp || info->ccc_gac) {
2201		if (q == RAVB_NC) {
2202			ts_skb = kmalloc(sizeof(*ts_skb), GFP_ATOMIC);
2203			if (!ts_skb) {
2204				if (num_tx_desc > 1) {
2205					desc--;
2206					dma_unmap_single(ndev->dev.parent, dma_addr,
2207							 len, DMA_TO_DEVICE);
2208				}
2209				goto unmap;
2210			}
2211			ts_skb->skb = skb_get(skb);
2212			ts_skb->tag = priv->ts_skb_tag++;
2213			priv->ts_skb_tag &= 0x3ff;
2214			list_add_tail(&ts_skb->list, &priv->ts_skb_list);
2215
2216			/* TAG and timestamp required flag */
2217			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2218			desc->tagh_tsr = (ts_skb->tag >> 4) | TX_TSR;
2219			desc->ds_tagl |= cpu_to_le16(ts_skb->tag << 12);
2220		}
2221
2222		skb_tx_timestamp(skb);
2223	}
2224
2225	if (num_tx_desc > 1) {
2226		desc->die_dt = DT_FEND;
2227		desc--;
2228		/* When using multi-descriptors, DT_FEND needs to get written
2229		 * before DT_FSTART, but the compiler may reorder the memory
2230		 * writes in an attempt to optimize the code.
2231		 * Use a dma_wmb() barrier to make sure DT_FEND and DT_FSTART
2232		 * are written exactly in the order shown in the code.
2233		 * This is particularly important for cases where the DMA engine
2234		 * is already running when we are running this code. If the DMA
2235		 * sees DT_FSTART without the corresponding DT_FEND it will enter
2236		 * an error condition.
2237		 */
2238		dma_wmb();
2239		desc->die_dt = DT_FSTART;
2240	} else {
2241		/* Descriptor type must be set after all the above writes */
2242		dma_wmb();
2243		desc->die_dt = DT_FSINGLE;
2244	}
2245
2246	/* Before ringing the doorbell we need to make sure that the latest
2247	 * writes have been committed to memory, otherwise it could delay
2248	 * things until the doorbell is rang again.
2249	 * This is in replacement of the read operation mentioned in the HW
2250	 * manuals.
2251	 */
2252	dma_wmb();
2253	ravb_modify(ndev, TCCR, TCCR_TSRQ0 << q, TCCR_TSRQ0 << q);
2254
2255	priv->cur_tx[q] += num_tx_desc;
2256	if (priv->cur_tx[q] - priv->dirty_tx[q] >
2257	    (priv->num_tx_ring[q] - 1) * num_tx_desc &&
2258	    !ravb_tx_free(ndev, q, true))
2259		netif_stop_subqueue(ndev, q);
2260
2261exit:
2262	spin_unlock_irqrestore(&priv->lock, flags);
2263	return NETDEV_TX_OK;
2264
2265unmap:
2266	dma_unmap_single(ndev->dev.parent, le32_to_cpu(desc->dptr),
2267			 le16_to_cpu(desc->ds_tagl), DMA_TO_DEVICE);
2268drop:
2269	dev_kfree_skb_any(skb);
2270	priv->tx_skb[q][entry / num_tx_desc] = NULL;
2271	goto exit;
2272}
2273
2274static u16 ravb_select_queue(struct net_device *ndev, struct sk_buff *skb,
2275			     struct net_device *sb_dev)
2276{
2277	/* If skb needs TX timestamp, it is handled in network control queue */
2278	return (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) ? RAVB_NC :
2279							       RAVB_BE;
2280
2281}
2282
2283static struct net_device_stats *ravb_get_stats(struct net_device *ndev)
2284{
2285	struct ravb_private *priv = netdev_priv(ndev);
2286	const struct ravb_hw_info *info = priv->info;
2287	struct net_device_stats *nstats, *stats0, *stats1;
2288	struct device *dev = &priv->pdev->dev;
2289
2290	nstats = &ndev->stats;
2291
2292	pm_runtime_get_noresume(dev);
2293
2294	if (!pm_runtime_active(dev))
2295		goto out_rpm_put;
2296
2297	stats0 = &priv->stats[RAVB_BE];
2298
2299	if (info->tx_counters) {
2300		nstats->tx_dropped += ravb_read(ndev, TROCR);
2301		ravb_write(ndev, 0, TROCR);	/* (write clear) */
2302	}
2303
2304	if (info->carrier_counters) {
2305		nstats->collisions += ravb_read(ndev, CXR41);
2306		ravb_write(ndev, 0, CXR41);	/* (write clear) */
2307		nstats->tx_carrier_errors += ravb_read(ndev, CXR42);
2308		ravb_write(ndev, 0, CXR42);	/* (write clear) */
2309	}
2310
2311	nstats->rx_packets = stats0->rx_packets;
2312	nstats->tx_packets = stats0->tx_packets;
2313	nstats->rx_bytes = stats0->rx_bytes;
2314	nstats->tx_bytes = stats0->tx_bytes;
2315	nstats->multicast = stats0->multicast;
2316	nstats->rx_errors = stats0->rx_errors;
2317	nstats->rx_crc_errors = stats0->rx_crc_errors;
2318	nstats->rx_frame_errors = stats0->rx_frame_errors;
2319	nstats->rx_length_errors = stats0->rx_length_errors;
2320	nstats->rx_missed_errors = stats0->rx_missed_errors;
2321	nstats->rx_over_errors = stats0->rx_over_errors;
2322	if (info->nc_queues) {
2323		stats1 = &priv->stats[RAVB_NC];
2324
2325		nstats->rx_packets += stats1->rx_packets;
2326		nstats->tx_packets += stats1->tx_packets;
2327		nstats->rx_bytes += stats1->rx_bytes;
2328		nstats->tx_bytes += stats1->tx_bytes;
2329		nstats->multicast += stats1->multicast;
2330		nstats->rx_errors += stats1->rx_errors;
2331		nstats->rx_crc_errors += stats1->rx_crc_errors;
2332		nstats->rx_frame_errors += stats1->rx_frame_errors;
2333		nstats->rx_length_errors += stats1->rx_length_errors;
2334		nstats->rx_missed_errors += stats1->rx_missed_errors;
2335		nstats->rx_over_errors += stats1->rx_over_errors;
2336	}
2337
2338out_rpm_put:
2339	pm_runtime_put_noidle(dev);
2340	return nstats;
2341}
2342
2343/* Update promiscuous bit */
2344static void ravb_set_rx_mode(struct net_device *ndev)
2345{
2346	struct ravb_private *priv = netdev_priv(ndev);
2347	unsigned long flags;
2348
2349	spin_lock_irqsave(&priv->lock, flags);
2350	ravb_modify(ndev, ECMR, ECMR_PRM,
2351		    ndev->flags & IFF_PROMISC ? ECMR_PRM : 0);
2352	spin_unlock_irqrestore(&priv->lock, flags);
2353}
2354
2355/* Device close function for Ethernet AVB */
2356static int ravb_close(struct net_device *ndev)
2357{
2358	struct device_node *np = ndev->dev.parent->of_node;
2359	struct ravb_private *priv = netdev_priv(ndev);
2360	const struct ravb_hw_info *info = priv->info;
2361	struct ravb_tstamp_skb *ts_skb, *ts_skb2;
2362	struct device *dev = &priv->pdev->dev;
2363	int error;
2364
2365	netif_tx_stop_all_queues(ndev);
2366
2367	/* Disable interrupts by clearing the interrupt masks. */
2368	ravb_write(ndev, 0, RIC0);
2369	ravb_write(ndev, 0, RIC2);
2370	ravb_write(ndev, 0, TIC);
2371
2372	/* PHY disconnect */
2373	if (ndev->phydev) {
2374		phy_stop(ndev->phydev);
2375		phy_disconnect(ndev->phydev);
2376		if (of_phy_is_fixed_link(np))
2377			of_phy_deregister_fixed_link(np);
2378	}
2379
2380	/* Stop PTP Clock driver */
2381	if (info->gptp || info->ccc_gac)
2382		ravb_ptp_stop(ndev);
2383
2384	/* Set the config mode to stop the AVB-DMAC's processes */
2385	if (ravb_stop_dma(ndev) < 0)
2386		netdev_err(ndev,
2387			   "device will be stopped after h/w processes are done.\n");
2388
2389	/* Clear the timestamp list */
2390	if (info->gptp || info->ccc_gac) {
2391		list_for_each_entry_safe(ts_skb, ts_skb2, &priv->ts_skb_list, list) {
2392			list_del(&ts_skb->list);
2393			kfree_skb(ts_skb->skb);
2394			kfree(ts_skb);
2395		}
2396	}
2397
2398	cancel_work_sync(&priv->work);
2399
2400	if (info->nc_queues)
2401		napi_disable(&priv->napi[RAVB_NC]);
2402	napi_disable(&priv->napi[RAVB_BE]);
2403
2404	/* Free all the skb's in the RX queue and the DMA buffers. */
2405	ravb_ring_free(ndev, RAVB_BE);
2406	if (info->nc_queues)
2407		ravb_ring_free(ndev, RAVB_NC);
2408
2409	/* Update statistics. */
2410	ravb_get_stats(ndev);
2411
2412	/* Set reset mode. */
2413	error = ravb_set_opmode(ndev, CCC_OPC_RESET);
2414	if (error)
2415		return error;
2416
2417	pm_runtime_put_autosuspend(dev);
2418
2419	return 0;
2420}
2421
2422static int ravb_hwtstamp_get(struct net_device *ndev,
2423			     struct kernel_hwtstamp_config *config)
2424{
2425	struct ravb_private *priv = netdev_priv(ndev);
2426
2427	config->flags = 0;
2428	config->tx_type = priv->tstamp_tx_ctrl;
2429	config->rx_filter = priv->tstamp_rx_ctrl;
2430
2431	return 0;
2432}
2433
2434/* Control hardware time stamping */
2435static int ravb_hwtstamp_set(struct net_device *ndev,
2436			     struct kernel_hwtstamp_config *config,
2437			     struct netlink_ext_ack *extack)
2438{
2439	struct ravb_private *priv = netdev_priv(ndev);
2440	enum hwtstamp_rx_filters tstamp_rx_ctrl;
2441	enum hwtstamp_tx_types tstamp_tx_ctrl;
2442
2443	switch (config->tx_type) {
2444	case HWTSTAMP_TX_OFF:
2445	case HWTSTAMP_TX_ON:
2446		tstamp_tx_ctrl = config->tx_type;
2447		break;
2448	default:
2449		return -ERANGE;
2450	}
2451
2452	switch (config->rx_filter) {
2453	case HWTSTAMP_FILTER_NONE:
2454	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2455		tstamp_rx_ctrl = config->rx_filter;
2456		break;
2457	default:
2458		config->rx_filter = HWTSTAMP_FILTER_ALL;
2459		tstamp_rx_ctrl = HWTSTAMP_FILTER_ALL;
2460	}
2461
2462	priv->tstamp_tx_ctrl = tstamp_tx_ctrl;
2463	priv->tstamp_rx_ctrl = tstamp_rx_ctrl;
2464
2465	return 0;
2466}
2467
2468static int ravb_change_mtu(struct net_device *ndev, int new_mtu)
2469{
2470	struct ravb_private *priv = netdev_priv(ndev);
2471
2472	WRITE_ONCE(ndev->mtu, new_mtu);
2473
2474	if (netif_running(ndev)) {
2475		synchronize_irq(priv->emac_irq);
2476		ravb_emac_init(ndev);
2477	}
2478
2479	netdev_update_features(ndev);
2480
2481	return 0;
2482}
2483
2484static void ravb_set_rx_csum(struct net_device *ndev, bool enable)
2485{
2486	struct ravb_private *priv = netdev_priv(ndev);
2487	unsigned long flags;
2488
2489	spin_lock_irqsave(&priv->lock, flags);
2490
2491	/* Disable TX and RX */
2492	ravb_rcv_snd_disable(ndev);
2493
2494	/* Modify RX Checksum setting */
2495	ravb_modify(ndev, ECMR, ECMR_RCSC, enable ? ECMR_RCSC : 0);
2496
2497	/* Enable TX and RX */
2498	ravb_rcv_snd_enable(ndev);
2499
2500	spin_unlock_irqrestore(&priv->lock, flags);
2501}
2502
2503static int ravb_endisable_csum_gbeth(struct net_device *ndev, enum ravb_reg reg,
2504				     u32 val, u32 mask)
2505{
2506	u32 csr0 = CSR0_TPE | CSR0_RPE;
2507	int ret;
2508
2509	ravb_write(ndev, csr0 & ~mask, CSR0);
2510	ret = ravb_wait(ndev, CSR0, mask, 0);
2511	if (!ret)
2512		ravb_write(ndev, val, reg);
2513
2514	ravb_write(ndev, csr0, CSR0);
2515
2516	return ret;
2517}
2518
2519static int ravb_set_features_gbeth(struct net_device *ndev,
2520				   netdev_features_t features)
2521{
2522	netdev_features_t changed = ndev->features ^ features;
2523	struct ravb_private *priv = netdev_priv(ndev);
2524	unsigned long flags;
2525	int ret = 0;
2526	u32 val;
2527
2528	spin_lock_irqsave(&priv->lock, flags);
2529	if (changed & NETIF_F_RXCSUM) {
2530		if (features & NETIF_F_RXCSUM)
2531			val = CSR2_CSUM_ENABLE;
2532		else
2533			val = 0;
2534
2535		ret = ravb_endisable_csum_gbeth(ndev, CSR2, val, CSR0_RPE);
2536		if (ret)
2537			goto done;
2538	}
2539
2540	if (changed & NETIF_F_HW_CSUM) {
2541		if (features & NETIF_F_HW_CSUM)
2542			val = CSR1_CSUM_ENABLE;
2543		else
2544			val = 0;
2545
2546		ret = ravb_endisable_csum_gbeth(ndev, CSR1, val, CSR0_TPE);
2547		if (ret)
2548			goto done;
2549	}
2550
2551done:
2552	spin_unlock_irqrestore(&priv->lock, flags);
2553
2554	return ret;
2555}
2556
2557static int ravb_set_features_rcar(struct net_device *ndev,
2558				  netdev_features_t features)
2559{
2560	netdev_features_t changed = ndev->features ^ features;
2561
2562	if (changed & NETIF_F_RXCSUM)
2563		ravb_set_rx_csum(ndev, features & NETIF_F_RXCSUM);
2564
2565	return 0;
2566}
2567
2568static int ravb_set_features(struct net_device *ndev,
2569			     netdev_features_t features)
2570{
2571	struct ravb_private *priv = netdev_priv(ndev);
2572	const struct ravb_hw_info *info = priv->info;
2573	struct device *dev = &priv->pdev->dev;
2574	int ret;
2575
2576	pm_runtime_get_noresume(dev);
2577
2578	if (pm_runtime_active(dev))
2579		ret = info->set_feature(ndev, features);
2580	else
2581		ret = 0;
2582
2583	pm_runtime_put_noidle(dev);
2584
2585	if (ret)
2586		return ret;
2587
2588	ndev->features = features;
2589
2590	return 0;
2591}
2592
2593static const struct net_device_ops ravb_netdev_ops = {
2594	.ndo_open		= ravb_open,
2595	.ndo_stop		= ravb_close,
2596	.ndo_start_xmit		= ravb_start_xmit,
2597	.ndo_select_queue	= ravb_select_queue,
2598	.ndo_get_stats		= ravb_get_stats,
2599	.ndo_set_rx_mode	= ravb_set_rx_mode,
2600	.ndo_tx_timeout		= ravb_tx_timeout,
2601	.ndo_eth_ioctl		= phy_do_ioctl_running,
2602	.ndo_change_mtu		= ravb_change_mtu,
2603	.ndo_validate_addr	= eth_validate_addr,
2604	.ndo_set_mac_address	= eth_mac_addr,
2605	.ndo_set_features	= ravb_set_features,
2606	.ndo_hwtstamp_get	= ravb_hwtstamp_get,
2607	.ndo_hwtstamp_set	= ravb_hwtstamp_set,
2608};
2609
2610/* MDIO bus init function */
2611static int ravb_mdio_init(struct ravb_private *priv)
2612{
2613	struct platform_device *pdev = priv->pdev;
2614	struct device *dev = &pdev->dev;
2615	struct device_node *mdio_node;
2616	struct phy_device *phydev;
2617	struct device_node *pn;
2618	int error;
2619
2620	/* Bitbang init */
2621	priv->mdiobb.ops = &bb_ops;
2622
2623	/* MII controller setting */
2624	priv->mii_bus = alloc_mdio_bitbang(&priv->mdiobb);
2625	if (!priv->mii_bus)
2626		return -ENOMEM;
2627
2628	/* Hook up MII support for ethtool */
2629	priv->mii_bus->name = "ravb_mii";
2630	priv->mii_bus->parent = dev;
2631	snprintf(priv->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",
2632		 pdev->name, pdev->id);
2633
2634	/* Register MDIO bus */
2635	mdio_node = of_get_child_by_name(dev->of_node, "mdio");
2636	if (!mdio_node) {
2637		/* backwards compatibility for DT lacking mdio subnode */
2638		mdio_node = of_node_get(dev->of_node);
2639	}
2640	error = of_mdiobus_register(priv->mii_bus, mdio_node);
2641	of_node_put(mdio_node);
2642	if (error)
2643		goto out_free_bus;
2644
2645	pn = of_parse_phandle(dev->of_node, "phy-handle", 0);
2646	phydev = of_phy_find_device(pn);
2647	if (phydev) {
2648		phydev->mac_managed_pm = true;
2649		put_device(&phydev->mdio.dev);
2650	}
2651	of_node_put(pn);
2652
2653	return 0;
2654
2655out_free_bus:
2656	free_mdio_bitbang(priv->mii_bus);
2657	return error;
2658}
2659
2660/* MDIO bus release function */
2661static int ravb_mdio_release(struct ravb_private *priv)
2662{
2663	/* Unregister mdio bus */
2664	mdiobus_unregister(priv->mii_bus);
2665
2666	/* Free bitbang info */
2667	free_mdio_bitbang(priv->mii_bus);
2668
2669	return 0;
2670}
2671
2672static const struct ravb_hw_info ravb_gen2_hw_info = {
2673	.receive = ravb_rx_rcar,
2674	.set_rate = ravb_set_rate_rcar,
2675	.set_feature = ravb_set_features_rcar,
2676	.dmac_init = ravb_dmac_init_rcar,
2677	.emac_init = ravb_emac_init_rcar,
2678	.gstrings_stats = ravb_gstrings_stats,
2679	.gstrings_size = sizeof(ravb_gstrings_stats),
2680	.net_hw_features = NETIF_F_RXCSUM,
2681	.net_features = NETIF_F_RXCSUM,
2682	.stats_len = ARRAY_SIZE(ravb_gstrings_stats),
2683	.tccr_mask = TCCR_TSRQ0 | TCCR_TSRQ1 | TCCR_TSRQ2 | TCCR_TSRQ3,
2684	.tx_max_frame_size = SZ_2K,
2685	.rx_max_frame_size = SZ_2K,
2686	.rx_buffer_size = SZ_2K +
2687			  SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
2688	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
2689	.dbat_entry_num = 22,
2690	.aligned_tx = 1,
2691	.gptp = 1,
2692	.nc_queues = 1,
2693	.magic_pkt = 1,
2694};
2695
2696static const struct ravb_hw_info ravb_gen3_hw_info = {
2697	.receive = ravb_rx_rcar,
2698	.set_rate = ravb_set_rate_rcar,
2699	.set_feature = ravb_set_features_rcar,
2700	.dmac_init = ravb_dmac_init_rcar,
2701	.emac_init = ravb_emac_init_rcar,
2702	.gstrings_stats = ravb_gstrings_stats,
2703	.gstrings_size = sizeof(ravb_gstrings_stats),
2704	.net_hw_features = NETIF_F_RXCSUM,
2705	.net_features = NETIF_F_RXCSUM,
2706	.stats_len = ARRAY_SIZE(ravb_gstrings_stats),
2707	.tccr_mask = TCCR_TSRQ0 | TCCR_TSRQ1 | TCCR_TSRQ2 | TCCR_TSRQ3,
2708	.tx_max_frame_size = SZ_2K,
2709	.rx_max_frame_size = SZ_2K,
2710	.rx_buffer_size = SZ_2K +
2711			  SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
2712	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
2713	.dbat_entry_num = 22,
2714	.internal_delay = 1,
2715	.tx_counters = 1,
2716	.multi_irqs = 1,
2717	.irq_en_dis = 1,
2718	.ccc_gac = 1,
2719	.nc_queues = 1,
2720	.magic_pkt = 1,
2721};
2722
2723static const struct ravb_hw_info ravb_gen4_hw_info = {
2724	.receive = ravb_rx_rcar,
2725	.set_rate = ravb_set_rate_rcar,
2726	.set_feature = ravb_set_features_rcar,
2727	.dmac_init = ravb_dmac_init_rcar,
2728	.emac_init = ravb_emac_init_rcar_gen4,
2729	.gstrings_stats = ravb_gstrings_stats,
2730	.gstrings_size = sizeof(ravb_gstrings_stats),
2731	.net_hw_features = NETIF_F_RXCSUM,
2732	.net_features = NETIF_F_RXCSUM,
2733	.stats_len = ARRAY_SIZE(ravb_gstrings_stats),
2734	.tccr_mask = TCCR_TSRQ0 | TCCR_TSRQ1 | TCCR_TSRQ2 | TCCR_TSRQ3,
2735	.tx_max_frame_size = SZ_2K,
2736	.rx_max_frame_size = SZ_2K,
2737	.rx_buffer_size = SZ_2K +
2738			  SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
2739	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
2740	.dbat_entry_num = 22,
2741	.internal_delay = 1,
2742	.tx_counters = 1,
2743	.multi_irqs = 1,
2744	.irq_en_dis = 1,
2745	.ccc_gac = 1,
2746	.nc_queues = 1,
2747	.magic_pkt = 1,
2748};
2749
2750static const struct ravb_hw_info ravb_rzv2m_hw_info = {
2751	.receive = ravb_rx_rcar,
2752	.set_rate = ravb_set_rate_rcar,
2753	.set_feature = ravb_set_features_rcar,
2754	.dmac_init = ravb_dmac_init_rcar,
2755	.emac_init = ravb_emac_init_rcar,
2756	.gstrings_stats = ravb_gstrings_stats,
2757	.gstrings_size = sizeof(ravb_gstrings_stats),
2758	.net_hw_features = NETIF_F_RXCSUM,
2759	.net_features = NETIF_F_RXCSUM,
2760	.stats_len = ARRAY_SIZE(ravb_gstrings_stats),
2761	.tccr_mask = TCCR_TSRQ0 | TCCR_TSRQ1 | TCCR_TSRQ2 | TCCR_TSRQ3,
2762	.tx_max_frame_size = SZ_2K,
2763	.rx_max_frame_size = SZ_2K,
2764	.rx_buffer_size = SZ_2K +
2765			  SKB_DATA_ALIGN(sizeof(struct skb_shared_info)),
2766	.rx_desc_size = sizeof(struct ravb_ex_rx_desc),
2767	.dbat_entry_num = 22,
2768	.multi_irqs = 1,
2769	.err_mgmt_irqs = 1,
2770	.gptp = 1,
2771	.gptp_ref_clk = 1,
2772	.nc_queues = 1,
2773	.magic_pkt = 1,
2774};
2775
2776static const struct ravb_hw_info gbeth_hw_info = {
2777	.receive = ravb_rx_gbeth,
2778	.set_rate = ravb_set_rate_gbeth,
2779	.set_feature = ravb_set_features_gbeth,
2780	.dmac_init = ravb_dmac_init_gbeth,
2781	.emac_init = ravb_emac_init_gbeth,
2782	.gstrings_stats = ravb_gstrings_stats_gbeth,
2783	.gstrings_size = sizeof(ravb_gstrings_stats_gbeth),
2784	.net_hw_features = NETIF_F_RXCSUM | NETIF_F_HW_CSUM,
2785	.net_features = NETIF_F_RXCSUM | NETIF_F_HW_CSUM,
2786	.vlan_features = NETIF_F_RXCSUM | NETIF_F_HW_CSUM,
2787	.stats_len = ARRAY_SIZE(ravb_gstrings_stats_gbeth),
2788	.tccr_mask = TCCR_TSRQ0,
2789	.tx_max_frame_size = 1522,
2790	.rx_max_frame_size = SZ_8K,
2791	.rx_buffer_size = SZ_2K,
2792	.rx_desc_size = sizeof(struct ravb_rx_desc),
2793	.dbat_entry_num = 2,
2794	.aligned_tx = 1,
2795	.coalesce_irqs = 1,
2796	.tx_counters = 1,
2797	.carrier_counters = 1,
2798	.half_duplex = 1,
2799};
2800
2801static const struct of_device_id ravb_match_table[] = {
2802	{ .compatible = "renesas,etheravb-r8a7790", .data = &ravb_gen2_hw_info },
2803	{ .compatible = "renesas,etheravb-r8a7794", .data = &ravb_gen2_hw_info },
2804	{ .compatible = "renesas,etheravb-rcar-gen2", .data = &ravb_gen2_hw_info },
2805	{ .compatible = "renesas,etheravb-r8a7795", .data = &ravb_gen3_hw_info },
2806	{ .compatible = "renesas,etheravb-rcar-gen3", .data = &ravb_gen3_hw_info },
2807	{ .compatible = "renesas,etheravb-rcar-gen4", .data = &ravb_gen4_hw_info },
2808	{ .compatible = "renesas,etheravb-rzv2m", .data = &ravb_rzv2m_hw_info },
2809	{ .compatible = "renesas,rzg2l-gbeth", .data = &gbeth_hw_info },
2810	{ }
2811};
2812MODULE_DEVICE_TABLE(of, ravb_match_table);
2813
2814static int ravb_setup_irq(struct ravb_private *priv, const char *irq_name,
2815			  const char *ch, int *irq, irq_handler_t handler)
2816{
2817	struct platform_device *pdev = priv->pdev;
2818	struct net_device *ndev = priv->ndev;
2819	struct device *dev = &pdev->dev;
2820	const char *devname = dev_name(dev);
2821	unsigned long flags;
2822	int error, irq_num;
2823
2824	if (irq_name) {
2825		devname = devm_kasprintf(dev, GFP_KERNEL, "%s:%s", devname, ch);
2826		if (!devname)
2827			return -ENOMEM;
2828
2829		irq_num = platform_get_irq_byname(pdev, irq_name);
2830		flags = 0;
2831	} else {
2832		irq_num = platform_get_irq(pdev, 0);
2833		flags = IRQF_SHARED;
2834	}
2835	if (irq_num < 0)
2836		return irq_num;
2837
2838	if (irq)
2839		*irq = irq_num;
2840
2841	error = devm_request_irq(dev, irq_num, handler, flags, devname, ndev);
2842	if (error)
2843		netdev_err(ndev, "cannot request IRQ %s\n", devname);
2844
2845	return error;
2846}
2847
2848static int ravb_setup_irqs(struct ravb_private *priv)
2849{
2850	const struct ravb_hw_info *info = priv->info;
2851	struct net_device *ndev = priv->ndev;
2852	const char *irq_name, *emac_irq_name;
2853	int error;
2854
2855	if (!info->multi_irqs)
2856		return ravb_setup_irq(priv, NULL, NULL, &ndev->irq, ravb_interrupt);
2857
2858	if (info->err_mgmt_irqs) {
2859		irq_name = "dia";
2860		emac_irq_name = "line3";
2861	} else {
2862		irq_name = "ch22";
2863		emac_irq_name = "ch24";
2864	}
2865
2866	error = ravb_setup_irq(priv, irq_name, "ch22:multi", &ndev->irq, ravb_multi_interrupt);
2867	if (error)
2868		return error;
2869
2870	error = ravb_setup_irq(priv, emac_irq_name, "ch24:emac", &priv->emac_irq,
2871			       ravb_emac_interrupt);
2872	if (error)
2873		return error;
2874
2875	if (info->err_mgmt_irqs) {
2876		error = ravb_setup_irq(priv, "err_a", "err_a", NULL, ravb_multi_interrupt);
2877		if (error)
2878			return error;
2879
2880		error = ravb_setup_irq(priv, "mgmt_a", "mgmt_a", NULL, ravb_multi_interrupt);
2881		if (error)
2882			return error;
2883	}
2884
2885	error = ravb_setup_irq(priv, "ch0", "ch0:rx_be", NULL, ravb_be_interrupt);
2886	if (error)
2887		return error;
2888
2889	error = ravb_setup_irq(priv, "ch1", "ch1:rx_nc", NULL, ravb_nc_interrupt);
2890	if (error)
2891		return error;
2892
2893	error = ravb_setup_irq(priv, "ch18", "ch18:tx_be", NULL, ravb_be_interrupt);
2894	if (error)
2895		return error;
2896
2897	return ravb_setup_irq(priv, "ch19", "ch19:tx_nc", NULL, ravb_nc_interrupt);
2898}
2899
2900static int ravb_probe(struct platform_device *pdev)
2901{
2902	struct device_node *np = pdev->dev.of_node;
2903	const struct ravb_hw_info *info;
2904	struct reset_control *rstc;
2905	struct ravb_private *priv;
2906	struct net_device *ndev;
2907	struct resource *res;
2908	int error, q;
2909
2910	if (!np) {
2911		dev_err(&pdev->dev,
2912			"this driver is required to be instantiated from device tree\n");
2913		return -EINVAL;
2914	}
2915
2916	rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
2917	if (IS_ERR(rstc))
2918		return dev_err_probe(&pdev->dev, PTR_ERR(rstc),
2919				     "failed to get cpg reset\n");
2920
2921	info = of_device_get_match_data(&pdev->dev);
2922
2923	ndev = alloc_etherdev_mqs(sizeof(struct ravb_private),
2924				  info->nc_queues ? NUM_TX_QUEUE : 1,
2925				  info->nc_queues ? NUM_RX_QUEUE : 1);
2926	if (!ndev)
2927		return -ENOMEM;
2928
2929	ndev->features = info->net_features;
2930	ndev->hw_features = info->net_hw_features;
2931	ndev->vlan_features = info->vlan_features;
2932
2933	error = reset_control_deassert(rstc);
2934	if (error)
2935		goto out_free_netdev;
2936
2937	SET_NETDEV_DEV(ndev, &pdev->dev);
2938
2939	priv = netdev_priv(ndev);
2940	priv->info = info;
2941	priv->rstc = rstc;
2942	priv->ndev = ndev;
2943	priv->pdev = pdev;
2944	priv->num_tx_ring[RAVB_BE] = BE_TX_RING_SIZE;
2945	priv->num_rx_ring[RAVB_BE] = BE_RX_RING_SIZE;
2946	if (info->nc_queues) {
2947		priv->num_tx_ring[RAVB_NC] = NC_TX_RING_SIZE;
2948		priv->num_rx_ring[RAVB_NC] = NC_RX_RING_SIZE;
2949	}
2950
2951	error = ravb_setup_irqs(priv);
2952	if (error)
2953		goto out_reset_assert;
2954
2955	priv->clk = devm_clk_get(&pdev->dev, NULL);
2956	if (IS_ERR(priv->clk)) {
2957		error = PTR_ERR(priv->clk);
2958		goto out_reset_assert;
2959	}
2960
2961	if (info->gptp_ref_clk) {
2962		priv->gptp_clk = devm_clk_get(&pdev->dev, "gptp");
2963		if (IS_ERR(priv->gptp_clk)) {
2964			error = PTR_ERR(priv->gptp_clk);
2965			goto out_reset_assert;
2966		}
2967	}
2968
2969	priv->refclk = devm_clk_get_optional(&pdev->dev, "refclk");
2970	if (IS_ERR(priv->refclk)) {
2971		error = PTR_ERR(priv->refclk);
2972		goto out_reset_assert;
2973	}
2974	clk_prepare(priv->refclk);
2975
2976	platform_set_drvdata(pdev, ndev);
2977	pm_runtime_set_autosuspend_delay(&pdev->dev, 100);
2978	pm_runtime_use_autosuspend(&pdev->dev);
2979	pm_runtime_enable(&pdev->dev);
2980	error = pm_runtime_resume_and_get(&pdev->dev);
2981	if (error < 0)
2982		goto out_rpm_disable;
2983
2984	priv->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2985	if (IS_ERR(priv->addr)) {
2986		error = PTR_ERR(priv->addr);
2987		goto out_rpm_put;
2988	}
2989
2990	/* The Ether-specific entries in the device structure. */
2991	ndev->base_addr = res->start;
2992
2993	spin_lock_init(&priv->lock);
2994	INIT_WORK(&priv->work, ravb_tx_timeout_work);
2995
2996	error = of_get_phy_mode(np, &priv->phy_interface);
2997	if (error && error != -ENODEV)
2998		goto out_rpm_put;
2999
3000	priv->no_avb_link = of_property_read_bool(np, "renesas,no-ether-link");
3001	priv->avb_link_active_low =
3002		of_property_read_bool(np, "renesas,ether-link-active-low");
3003
3004	ndev->max_mtu = info->tx_max_frame_size -
3005		(ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN);
3006	ndev->min_mtu = ETH_MIN_MTU;
3007
3008	/* FIXME: R-Car Gen2 has 4byte alignment restriction for tx buffer
3009	 * Use two descriptor to handle such situation. First descriptor to
3010	 * handle aligned data buffer and second descriptor to handle the
3011	 * overflow data because of alignment.
3012	 */
3013	priv->num_tx_desc = info->aligned_tx ? 2 : 1;
3014
3015	/* Set function */
3016	ndev->netdev_ops = &ravb_netdev_ops;
3017	ndev->ethtool_ops = &ravb_ethtool_ops;
3018
3019	error = ravb_compute_gti(ndev);
3020	if (error)
3021		goto out_rpm_put;
3022
3023	ravb_parse_delay_mode(np, ndev);
3024
3025	/* Allocate descriptor base address table */
3026	priv->desc_bat_size = sizeof(struct ravb_desc) * info->dbat_entry_num;
3027	priv->desc_bat = dma_alloc_coherent(ndev->dev.parent, priv->desc_bat_size,
3028					    &priv->desc_bat_dma, GFP_KERNEL);
3029	if (!priv->desc_bat) {
3030		dev_err(&pdev->dev,
3031			"Cannot allocate desc base address table (size %d bytes)\n",
3032			priv->desc_bat_size);
3033		error = -ENOMEM;
3034		goto out_rpm_put;
3035	}
3036	for (q = RAVB_BE; q < info->dbat_entry_num; q++)
3037		priv->desc_bat[q].die_dt = DT_EOS;
3038
3039	/* Initialise HW timestamp list */
3040	INIT_LIST_HEAD(&priv->ts_skb_list);
3041
3042	/* Debug message level */
3043	priv->msg_enable = RAVB_DEF_MSG_ENABLE;
3044
3045	/* Set config mode as this is needed for PHY initialization. */
3046	error = ravb_set_opmode(ndev, CCC_OPC_CONFIG);
3047	if (error)
3048		goto out_rpm_put;
3049
3050	/* Read and set MAC address */
3051	ravb_read_mac_address(np, ndev);
3052	if (!is_valid_ether_addr(ndev->dev_addr)) {
3053		dev_warn(&pdev->dev,
3054			 "no valid MAC address supplied, using a random one\n");
3055		eth_hw_addr_random(ndev);
3056	}
3057
3058	/* MDIO bus init */
3059	error = ravb_mdio_init(priv);
3060	if (error) {
3061		dev_err(&pdev->dev, "failed to initialize MDIO\n");
3062		goto out_reset_mode;
3063	}
3064
3065	/* Undo previous switch to config opmode. */
3066	error = ravb_set_opmode(ndev, CCC_OPC_RESET);
3067	if (error)
3068		goto out_mdio_release;
3069
3070	netif_napi_add(ndev, &priv->napi[RAVB_BE], ravb_poll);
3071	if (info->nc_queues)
3072		netif_napi_add(ndev, &priv->napi[RAVB_NC], ravb_poll);
3073
3074	if (info->coalesce_irqs) {
3075		netdev_sw_irq_coalesce_default_on(ndev);
3076		if (num_present_cpus() == 1)
3077			netif_threaded_enable(ndev);
3078	}
3079
3080	/* Network device register */
3081	error = register_netdev(ndev);
3082	if (error)
3083		goto out_napi_del;
3084
3085	device_set_wakeup_capable(&pdev->dev, 1);
3086
3087	/* Print device information */
3088	netdev_info(ndev, "Base address at %#x, %pM, IRQ %d.\n",
3089		    (u32)ndev->base_addr, ndev->dev_addr, ndev->irq);
3090
3091	pm_runtime_put_autosuspend(&pdev->dev);
3092
3093	return 0;
3094
3095out_napi_del:
3096	if (info->nc_queues)
3097		netif_napi_del(&priv->napi[RAVB_NC]);
3098
3099	netif_napi_del(&priv->napi[RAVB_BE]);
3100out_mdio_release:
3101	ravb_mdio_release(priv);
3102out_reset_mode:
3103	ravb_set_opmode(ndev, CCC_OPC_RESET);
3104	dma_free_coherent(ndev->dev.parent, priv->desc_bat_size, priv->desc_bat,
3105			  priv->desc_bat_dma);
3106out_rpm_put:
3107	pm_runtime_put(&pdev->dev);
3108out_rpm_disable:
3109	pm_runtime_disable(&pdev->dev);
3110	pm_runtime_dont_use_autosuspend(&pdev->dev);
3111	clk_unprepare(priv->refclk);
3112out_reset_assert:
3113	reset_control_assert(rstc);
3114out_free_netdev:
3115	free_netdev(ndev);
3116	return error;
3117}
3118
3119static void ravb_remove(struct platform_device *pdev)
3120{
3121	struct net_device *ndev = platform_get_drvdata(pdev);
3122	struct ravb_private *priv = netdev_priv(ndev);
3123	const struct ravb_hw_info *info = priv->info;
3124	struct device *dev = &priv->pdev->dev;
3125	int error;
3126
3127	error = pm_runtime_resume_and_get(dev);
3128	if (error < 0)
3129		return;
3130
3131	unregister_netdev(ndev);
3132	if (info->nc_queues)
3133		netif_napi_del(&priv->napi[RAVB_NC]);
3134	netif_napi_del(&priv->napi[RAVB_BE]);
3135
3136	ravb_mdio_release(priv);
3137
3138	dma_free_coherent(ndev->dev.parent, priv->desc_bat_size, priv->desc_bat,
3139			  priv->desc_bat_dma);
3140
3141	pm_runtime_put_sync_suspend(&pdev->dev);
3142	pm_runtime_disable(&pdev->dev);
3143	pm_runtime_dont_use_autosuspend(dev);
3144	clk_unprepare(priv->refclk);
3145	reset_control_assert(priv->rstc);
3146	free_netdev(ndev);
3147	platform_set_drvdata(pdev, NULL);
3148}
3149
3150static int ravb_wol_setup(struct net_device *ndev)
3151{
3152	struct ravb_private *priv = netdev_priv(ndev);
3153	const struct ravb_hw_info *info = priv->info;
3154
3155	/* Disable interrupts by clearing the interrupt masks. */
3156	ravb_write(ndev, 0, RIC0);
3157	ravb_write(ndev, 0, RIC2);
3158	ravb_write(ndev, 0, TIC);
3159
3160	/* Only allow ECI interrupts */
3161	synchronize_irq(priv->emac_irq);
3162	if (info->nc_queues)
3163		napi_disable(&priv->napi[RAVB_NC]);
3164	napi_disable(&priv->napi[RAVB_BE]);
3165	ravb_write(ndev, ECSIPR_MPDIP, ECSIPR);
3166
3167	/* Enable MagicPacket */
3168	ravb_modify(ndev, ECMR, ECMR_MPDE, ECMR_MPDE);
3169
3170	if (priv->info->ccc_gac)
3171		ravb_ptp_stop(ndev);
3172
3173	return enable_irq_wake(priv->emac_irq);
3174}
3175
3176static int ravb_wol_restore(struct net_device *ndev)
3177{
3178	struct ravb_private *priv = netdev_priv(ndev);
3179	const struct ravb_hw_info *info = priv->info;
3180	int error;
3181
3182	/* Set reset mode to rearm the WoL logic. */
3183	error = ravb_set_opmode(ndev, CCC_OPC_RESET);
3184	if (error)
3185		return error;
3186
3187	/* Set AVB config mode. */
3188	error = ravb_set_config_mode(ndev);
3189	if (error)
3190		return error;
3191
3192	if (priv->info->ccc_gac)
3193		ravb_ptp_init(ndev, priv->pdev);
3194
3195	if (info->nc_queues)
3196		napi_enable(&priv->napi[RAVB_NC]);
3197	napi_enable(&priv->napi[RAVB_BE]);
3198
3199	/* Disable MagicPacket */
3200	ravb_modify(ndev, ECMR, ECMR_MPDE, 0);
3201
3202	ravb_close(ndev);
3203
3204	return disable_irq_wake(priv->emac_irq);
3205}
3206
3207static int ravb_suspend(struct device *dev)
3208{
3209	struct net_device *ndev = dev_get_drvdata(dev);
3210	struct ravb_private *priv = netdev_priv(ndev);
3211	int ret;
3212
3213	if (!netif_running(ndev))
3214		goto reset_assert;
3215
3216	netif_device_detach(ndev);
3217
3218	rtnl_lock();
3219	if (priv->wol_enabled) {
3220		ret = ravb_wol_setup(ndev);
3221		rtnl_unlock();
3222		return ret;
3223	}
3224
3225	ret = ravb_close(ndev);
3226	rtnl_unlock();
3227	if (ret)
3228		return ret;
3229
3230	ret = pm_runtime_force_suspend(&priv->pdev->dev);
3231	if (ret)
3232		return ret;
3233
3234reset_assert:
3235	return reset_control_assert(priv->rstc);
3236}
3237
3238static int ravb_resume(struct device *dev)
3239{
3240	struct net_device *ndev = dev_get_drvdata(dev);
3241	struct ravb_private *priv = netdev_priv(ndev);
3242	int ret;
3243
3244	ret = reset_control_deassert(priv->rstc);
3245	if (ret)
3246		return ret;
3247
3248	if (!netif_running(ndev))
3249		return 0;
3250
3251	rtnl_lock();
3252	/* If WoL is enabled restore the interface. */
3253	if (priv->wol_enabled)
3254		ret = ravb_wol_restore(ndev);
3255	else
3256		ret = pm_runtime_force_resume(dev);
3257	if (ret) {
3258		rtnl_unlock();
3259		return ret;
3260	}
3261
3262	/* Reopening the interface will restore the device to the working state. */
3263	ret = ravb_open(ndev);
3264	rtnl_unlock();
3265	if (ret < 0)
3266		goto out_rpm_put;
3267
3268	ravb_set_rx_mode(ndev);
3269	netif_device_attach(ndev);
3270
3271	return 0;
3272
3273out_rpm_put:
3274	if (!priv->wol_enabled)
3275		pm_runtime_put_autosuspend(dev);
3276
3277	return ret;
3278}
3279
3280static int ravb_runtime_suspend(struct device *dev)
3281{
3282	struct net_device *ndev = dev_get_drvdata(dev);
3283	struct ravb_private *priv = netdev_priv(ndev);
3284
3285	clk_disable(priv->refclk);
3286
3287	return 0;
3288}
3289
3290static int ravb_runtime_resume(struct device *dev)
3291{
3292	struct net_device *ndev = dev_get_drvdata(dev);
3293	struct ravb_private *priv = netdev_priv(ndev);
3294
3295	return clk_enable(priv->refclk);
3296}
3297
3298static const struct dev_pm_ops ravb_dev_pm_ops = {
3299	SYSTEM_SLEEP_PM_OPS(ravb_suspend, ravb_resume)
3300	RUNTIME_PM_OPS(ravb_runtime_suspend, ravb_runtime_resume, NULL)
3301};
3302
3303static struct platform_driver ravb_driver = {
3304	.probe		= ravb_probe,
3305	.remove		= ravb_remove,
3306	.driver = {
3307		.name	= "ravb",
3308		.pm	= pm_ptr(&ravb_dev_pm_ops),
3309		.of_match_table = ravb_match_table,
3310	},
3311};
3312
3313module_platform_driver(ravb_driver);
3314
3315MODULE_AUTHOR("Mitsuhiro Kimura, Masaru Nagai");
3316MODULE_DESCRIPTION("Renesas Ethernet AVB driver");
3317MODULE_LICENSE("GPL v2");