gemini.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Ethernet device driver for Cortina Systems Gemini SoC
   3 * Also known as the StorLink SL3512 and SL3516 (SL351x) or Lepus
   4 * Net Engine and Gigabit Ethernet MAC (GMAC)
   5 * This hardware contains a TCP Offload Engine (TOE) but currently the
   6 * driver does not make use of it.
   7 *
   8 * Authors:
   9 * Linus Walleij <linus.walleij@linaro.org>
  10 * Tobias Waldvogel <tobias.waldvogel@gmail.com> (OpenWRT)
  11 * Michał Mirosław <mirq-linux@rere.qmqm.pl>
  12 * Paulius Zaleckas <paulius.zaleckas@gmail.com>
  13 * Giuseppe De Robertis <Giuseppe.DeRobertis@ba.infn.it>
  14 * Gary Chen & Ch Hsu Storlink Semiconductor
  15 */
  16#include <linux/kernel.h>
  17#include <linux/init.h>
  18#include <linux/module.h>
  19#include <linux/platform_device.h>
  20#include <linux/spinlock.h>
  21#include <linux/slab.h>
  22#include <linux/dma-mapping.h>
  23#include <linux/cache.h>
  24#include <linux/interrupt.h>
  25#include <linux/reset.h>
  26#include <linux/clk.h>
  27#include <linux/of.h>
  28#include <linux/of_mdio.h>
  29#include <linux/of_net.h>
  30#include <linux/of_platform.h>
  31#include <linux/etherdevice.h>
  32#include <linux/if_vlan.h>
  33#include <linux/skbuff.h>
  34#include <linux/phy.h>
  35#include <linux/crc32.h>
  36#include <linux/ethtool.h>
  37#include <linux/tcp.h>
  38#include <linux/u64_stats_sync.h>
  39
  40#include <linux/in.h>
  41#include <linux/ip.h>
  42#include <linux/ipv6.h>
  43#include <net/gro.h>
  44
  45#include "gemini.h"
  46
  47#define DRV_NAME		"gmac-gemini"
  48
  49#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  50static int debug = -1;
  51module_param(debug, int, 0);
  52MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  53
  54#define HSIZE_8			0x00
  55#define HSIZE_16		0x01
  56#define HSIZE_32		0x02
  57
  58#define HBURST_SINGLE		0x00
  59#define HBURST_INCR		0x01
  60#define HBURST_INCR4		0x02
  61#define HBURST_INCR8		0x03
  62
  63#define HPROT_DATA_CACHE	BIT(0)
  64#define HPROT_PRIVILIGED	BIT(1)
  65#define HPROT_BUFFERABLE	BIT(2)
  66#define HPROT_CACHABLE		BIT(3)
  67
  68#define DEFAULT_RX_COALESCE_NSECS	0
  69#define DEFAULT_GMAC_RXQ_ORDER		9
  70#define DEFAULT_GMAC_TXQ_ORDER		8
  71#define DEFAULT_RX_BUF_ORDER		11
  72#define TX_MAX_FRAGS			16
  73#define TX_QUEUE_NUM			1	/* max: 6 */
  74#define RX_MAX_ALLOC_ORDER		2
  75
  76#define GMAC0_IRQ0_2 (GMAC0_TXDERR_INT_BIT | GMAC0_TXPERR_INT_BIT | \
  77		      GMAC0_RXDERR_INT_BIT | GMAC0_RXPERR_INT_BIT)
  78#define GMAC0_IRQ0_TXQ0_INTS (GMAC0_SWTQ00_EOF_INT_BIT | \
  79			      GMAC0_SWTQ00_FIN_INT_BIT)
  80#define GMAC0_IRQ4_8 (GMAC0_MIB_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT)
  81
  82#define GMAC_OFFLOAD_FEATURES (NETIF_F_SG | NETIF_F_IP_CSUM | \
  83			       NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | \
  84			       NETIF_F_TSO | NETIF_F_TSO_ECN | NETIF_F_TSO6)
  85
  86/**
  87 * struct gmac_queue_page - page buffer per-page info
  88 * @page: the page struct
  89 * @mapping: the dma address handle
  90 */
  91struct gmac_queue_page {
  92	struct page *page;
  93	dma_addr_t mapping;
  94};
  95
  96struct gmac_txq {
  97	struct gmac_txdesc *ring;
  98	struct sk_buff	**skb;
  99	unsigned int	cptr;
 100	unsigned int	noirq_packets;
 101};
 102
 103struct gemini_ethernet;
 104
 105struct gemini_ethernet_port {
 106	u8 id; /* 0 or 1 */
 107
 108	struct gemini_ethernet *geth;
 109	struct net_device *netdev;
 110	struct device *dev;
 111	void __iomem *dma_base;
 112	void __iomem *gmac_base;
 113	struct clk *pclk;
 114	struct reset_control *reset;
 115	int irq;
 116	__le32 mac_addr[3];
 117
 118	void __iomem		*rxq_rwptr;
 119	struct gmac_rxdesc	*rxq_ring;
 120	unsigned int		rxq_order;
 121
 122	struct napi_struct	napi;
 123	struct hrtimer		rx_coalesce_timer;
 124	unsigned int		rx_coalesce_nsecs;
 125	unsigned int		freeq_refill;
 126	struct gmac_txq		txq[TX_QUEUE_NUM];
 127	unsigned int		txq_order;
 128	unsigned int		irq_every_tx_packets;
 129
 130	dma_addr_t		rxq_dma_base;
 131	dma_addr_t		txq_dma_base;
 132
 133	unsigned int		msg_enable;
 134	spinlock_t		config_lock; /* Locks config register */
 135
 136	struct u64_stats_sync	tx_stats_syncp;
 137	struct u64_stats_sync	rx_stats_syncp;
 138	struct u64_stats_sync	ir_stats_syncp;
 139
 140	struct rtnl_link_stats64 stats;
 141	u64			hw_stats[RX_STATS_NUM];
 142	u64			rx_stats[RX_STATUS_NUM];
 143	u64			rx_csum_stats[RX_CHKSUM_NUM];
 144	u64			rx_napi_exits;
 145	u64			tx_frag_stats[TX_MAX_FRAGS];
 146	u64			tx_frags_linearized;
 147	u64			tx_hw_csummed;
 148};
 149
 150struct gemini_ethernet {
 151	struct device *dev;
 152	void __iomem *base;
 153	struct gemini_ethernet_port *port0;
 154	struct gemini_ethernet_port *port1;
 155	bool initialized;
 156
 157	spinlock_t	irq_lock; /* Locks IRQ-related registers */
 158	unsigned int	freeq_order;
 159	unsigned int	freeq_frag_order;
 160	struct gmac_rxdesc *freeq_ring;
 161	dma_addr_t	freeq_dma_base;
 162	struct gmac_queue_page	*freeq_pages;
 163	unsigned int	num_freeq_pages;
 164	spinlock_t	freeq_lock; /* Locks queue from reentrance */
 165};
 166
 167#define GMAC_STATS_NUM	( \
 168	RX_STATS_NUM + RX_STATUS_NUM + RX_CHKSUM_NUM + 1 + \
 169	TX_MAX_FRAGS + 2)
 170
 171static const char gmac_stats_strings[GMAC_STATS_NUM][ETH_GSTRING_LEN] = {
 172	"GMAC_IN_DISCARDS",
 173	"GMAC_IN_ERRORS",
 174	"GMAC_IN_MCAST",
 175	"GMAC_IN_BCAST",
 176	"GMAC_IN_MAC1",
 177	"GMAC_IN_MAC2",
 178	"RX_STATUS_GOOD_FRAME",
 179	"RX_STATUS_TOO_LONG_GOOD_CRC",
 180	"RX_STATUS_RUNT_FRAME",
 181	"RX_STATUS_SFD_NOT_FOUND",
 182	"RX_STATUS_CRC_ERROR",
 183	"RX_STATUS_TOO_LONG_BAD_CRC",
 184	"RX_STATUS_ALIGNMENT_ERROR",
 185	"RX_STATUS_TOO_LONG_BAD_ALIGN",
 186	"RX_STATUS_RX_ERR",
 187	"RX_STATUS_DA_FILTERED",
 188	"RX_STATUS_BUFFER_FULL",
 189	"RX_STATUS_11",
 190	"RX_STATUS_12",
 191	"RX_STATUS_13",
 192	"RX_STATUS_14",
 193	"RX_STATUS_15",
 194	"RX_CHKSUM_IP_UDP_TCP_OK",
 195	"RX_CHKSUM_IP_OK_ONLY",
 196	"RX_CHKSUM_NONE",
 197	"RX_CHKSUM_3",
 198	"RX_CHKSUM_IP_ERR_UNKNOWN",
 199	"RX_CHKSUM_IP_ERR",
 200	"RX_CHKSUM_TCP_UDP_ERR",
 201	"RX_CHKSUM_7",
 202	"RX_NAPI_EXITS",
 203	"TX_FRAGS[1]",
 204	"TX_FRAGS[2]",
 205	"TX_FRAGS[3]",
 206	"TX_FRAGS[4]",
 207	"TX_FRAGS[5]",
 208	"TX_FRAGS[6]",
 209	"TX_FRAGS[7]",
 210	"TX_FRAGS[8]",
 211	"TX_FRAGS[9]",
 212	"TX_FRAGS[10]",
 213	"TX_FRAGS[11]",
 214	"TX_FRAGS[12]",
 215	"TX_FRAGS[13]",
 216	"TX_FRAGS[14]",
 217	"TX_FRAGS[15]",
 218	"TX_FRAGS[16+]",
 219	"TX_FRAGS_LINEARIZED",
 220	"TX_HW_CSUMMED",
 221};
 222
 223static void gmac_dump_dma_state(struct net_device *netdev);
 224
 225static void gmac_update_config0_reg(struct net_device *netdev,
 226				    u32 val, u32 vmask)
 227{
 228	struct gemini_ethernet_port *port = netdev_priv(netdev);
 229	unsigned long flags;
 230	u32 reg;
 231
 232	spin_lock_irqsave(&port->config_lock, flags);
 233
 234	reg = readl(port->gmac_base + GMAC_CONFIG0);
 235	reg = (reg & ~vmask) | val;
 236	writel(reg, port->gmac_base + GMAC_CONFIG0);
 237
 238	spin_unlock_irqrestore(&port->config_lock, flags);
 239}
 240
 241static void gmac_enable_tx_rx(struct net_device *netdev)
 242{
 243	struct gemini_ethernet_port *port = netdev_priv(netdev);
 244	unsigned long flags;
 245	u32 reg;
 246
 247	spin_lock_irqsave(&port->config_lock, flags);
 248
 249	reg = readl(port->gmac_base + GMAC_CONFIG0);
 250	reg &= ~CONFIG0_TX_RX_DISABLE;
 251	writel(reg, port->gmac_base + GMAC_CONFIG0);
 252
 253	spin_unlock_irqrestore(&port->config_lock, flags);
 254}
 255
 256static void gmac_disable_tx_rx(struct net_device *netdev)
 257{
 258	struct gemini_ethernet_port *port = netdev_priv(netdev);
 259	unsigned long flags;
 260	u32 val;
 261
 262	spin_lock_irqsave(&port->config_lock, flags);
 263
 264	val = readl(port->gmac_base + GMAC_CONFIG0);
 265	val |= CONFIG0_TX_RX_DISABLE;
 266	writel(val, port->gmac_base + GMAC_CONFIG0);
 267
 268	spin_unlock_irqrestore(&port->config_lock, flags);
 269
 270	mdelay(10);	/* let GMAC consume packet */
 271}
 272
 273static void gmac_set_flow_control(struct net_device *netdev, bool tx, bool rx)
 274{
 275	struct gemini_ethernet_port *port = netdev_priv(netdev);
 276	unsigned long flags;
 277	u32 val;
 278
 279	spin_lock_irqsave(&port->config_lock, flags);
 280
 281	val = readl(port->gmac_base + GMAC_CONFIG0);
 282	val &= ~CONFIG0_FLOW_CTL;
 283	if (tx)
 284		val |= CONFIG0_FLOW_TX;
 285	if (rx)
 286		val |= CONFIG0_FLOW_RX;
 287	writel(val, port->gmac_base + GMAC_CONFIG0);
 288
 289	spin_unlock_irqrestore(&port->config_lock, flags);
 290}
 291
 292static void gmac_adjust_link(struct net_device *netdev)
 293{
 294	struct gemini_ethernet_port *port = netdev_priv(netdev);
 295	struct phy_device *phydev = netdev->phydev;
 296	union gmac_status status, old_status;
 297	bool pause_tx = false;
 298	bool pause_rx = false;
 299
 300	status.bits32 = readl(port->gmac_base + GMAC_STATUS);
 301	old_status.bits32 = status.bits32;
 302	status.bits.link = phydev->link;
 303	status.bits.duplex = phydev->duplex;
 304
 305	switch (phydev->speed) {
 306	case 1000:
 307		status.bits.speed = GMAC_SPEED_1000;
 308		if (phy_interface_mode_is_rgmii(phydev->interface))
 309			status.bits.mii_rmii = GMAC_PHY_RGMII_1000;
 310		netdev_dbg(netdev, "connect %s to RGMII @ 1Gbit\n",
 311			   phydev_name(phydev));
 312		break;
 313	case 100:
 314		status.bits.speed = GMAC_SPEED_100;
 315		if (phy_interface_mode_is_rgmii(phydev->interface))
 316			status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 317		netdev_dbg(netdev, "connect %s to RGMII @ 100 Mbit\n",
 318			   phydev_name(phydev));
 319		break;
 320	case 10:
 321		status.bits.speed = GMAC_SPEED_10;
 322		if (phy_interface_mode_is_rgmii(phydev->interface))
 323			status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 324		netdev_dbg(netdev, "connect %s to RGMII @ 10 Mbit\n",
 325			   phydev_name(phydev));
 326		break;
 327	default:
 328		netdev_warn(netdev, "Unsupported PHY speed (%d) on %s\n",
 329			    phydev->speed, phydev_name(phydev));
 330	}
 331
 332	if (phydev->duplex == DUPLEX_FULL) {
 333		phy_get_pause(phydev, &pause_tx, &pause_rx);
 334		netdev_dbg(netdev, "set negotiated pause params pause TX = %s, pause RX = %s\n",
 335			   pause_tx ? "ON" : "OFF", pause_rx ? "ON" : "OFF");
 336	}
 337
 338	gmac_set_flow_control(netdev, pause_tx, pause_rx);
 339
 340	if (old_status.bits32 == status.bits32)
 341		return;
 342
 343	if (netif_msg_link(port)) {
 344		phy_print_status(phydev);
 345		netdev_info(netdev, "link flow control: %s\n",
 346			    phydev->pause
 347			    ? (phydev->asym_pause ? "tx" : "both")
 348			    : (phydev->asym_pause ? "rx" : "none")
 349		);
 350	}
 351
 352	gmac_disable_tx_rx(netdev);
 353	writel(status.bits32, port->gmac_base + GMAC_STATUS);
 354	gmac_enable_tx_rx(netdev);
 355}
 356
 357static int gmac_setup_phy(struct net_device *netdev)
 358{
 359	struct gemini_ethernet_port *port = netdev_priv(netdev);
 360	union gmac_status status = { .bits32 = 0 };
 361	struct device *dev = port->dev;
 362	struct phy_device *phy;
 363
 364	phy = of_phy_get_and_connect(netdev,
 365				     dev->of_node,
 366				     gmac_adjust_link);
 367	if (!phy)
 368		return -ENODEV;
 369	netdev->phydev = phy;
 370
 371	phy_set_max_speed(phy, SPEED_1000);
 372	phy_support_asym_pause(phy);
 373
 374	/* set PHY interface type */
 375	switch (phy->interface) {
 376	case PHY_INTERFACE_MODE_MII:
 377		netdev_dbg(netdev,
 378			   "MII: set GMAC0 to GMII mode, GMAC1 disabled\n");
 379		status.bits.mii_rmii = GMAC_PHY_MII;
 380		break;
 381	case PHY_INTERFACE_MODE_GMII:
 382		netdev_dbg(netdev,
 383			   "GMII: set GMAC0 to GMII mode, GMAC1 disabled\n");
 384		status.bits.mii_rmii = GMAC_PHY_GMII;
 385		break;
 386	case PHY_INTERFACE_MODE_RGMII:
 387	case PHY_INTERFACE_MODE_RGMII_ID:
 388	case PHY_INTERFACE_MODE_RGMII_TXID:
 389	case PHY_INTERFACE_MODE_RGMII_RXID:
 390		netdev_dbg(netdev,
 391			   "RGMII: set GMAC0 and GMAC1 to MII/RGMII mode\n");
 392		status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
 393		break;
 394	default:
 395		netdev_err(netdev, "Unsupported MII interface\n");
 396		phy_disconnect(phy);
 397		netdev->phydev = NULL;
 398		return -EINVAL;
 399	}
 400	writel(status.bits32, port->gmac_base + GMAC_STATUS);
 401
 402	if (netif_msg_link(port))
 403		phy_attached_info(phy);
 404
 405	return 0;
 406}
 407
 408/* The maximum frame length is not logically enumerated in the
 409 * hardware, so we do a table lookup to find the applicable max
 410 * frame length.
 411 */
 412struct gmac_max_framelen {
 413	unsigned int max_l3_len;
 414	u8 val;
 415};
 416
 417static const struct gmac_max_framelen gmac_maxlens[] = {
 418	{
 419		.max_l3_len = 1518,
 420		.val = CONFIG0_MAXLEN_1518,
 421	},
 422	{
 423		.max_l3_len = 1522,
 424		.val = CONFIG0_MAXLEN_1522,
 425	},
 426	{
 427		.max_l3_len = 1536,
 428		.val = CONFIG0_MAXLEN_1536,
 429	},
 430	{
 431		.max_l3_len = 1548,
 432		.val = CONFIG0_MAXLEN_1548,
 433	},
 434	{
 435		.max_l3_len = 9212,
 436		.val = CONFIG0_MAXLEN_9k,
 437	},
 438	{
 439		.max_l3_len = 10236,
 440		.val = CONFIG0_MAXLEN_10k,
 441	},
 442};
 443
 444static int gmac_pick_rx_max_len(unsigned int max_l3_len)
 445{
 446	const struct gmac_max_framelen *maxlen;
 447	int maxtot;
 448	int i;
 449
 450	maxtot = max_l3_len + ETH_HLEN + VLAN_HLEN;
 451
 452	for (i = 0; i < ARRAY_SIZE(gmac_maxlens); i++) {
 453		maxlen = &gmac_maxlens[i];
 454		if (maxtot <= maxlen->max_l3_len)
 455			return maxlen->val;
 456	}
 457
 458	return -1;
 459}
 460
 461static int gmac_init(struct net_device *netdev)
 462{
 463	struct gemini_ethernet_port *port = netdev_priv(netdev);
 464	union gmac_config0 config0 = { .bits = {
 465		.dis_tx = 1,
 466		.dis_rx = 1,
 467		.ipv4_rx_chksum = 1,
 468		.ipv6_rx_chksum = 1,
 469		.rx_err_detect = 1,
 470		.rgmm_edge = 1,
 471		.port0_chk_hwq = 1,
 472		.port1_chk_hwq = 1,
 473		.port0_chk_toeq = 1,
 474		.port1_chk_toeq = 1,
 475		.port0_chk_classq = 1,
 476		.port1_chk_classq = 1,
 477	} };
 478	union gmac_ahb_weight ahb_weight = { .bits = {
 479		.rx_weight = 1,
 480		.tx_weight = 1,
 481		.hash_weight = 1,
 482		.pre_req = 0x1f,
 483		.tq_dv_threshold = 0,
 484	} };
 485	union gmac_tx_wcr0 hw_weigh = { .bits = {
 486		.hw_tq3 = 1,
 487		.hw_tq2 = 1,
 488		.hw_tq1 = 1,
 489		.hw_tq0 = 1,
 490	} };
 491	union gmac_tx_wcr1 sw_weigh = { .bits = {
 492		.sw_tq5 = 1,
 493		.sw_tq4 = 1,
 494		.sw_tq3 = 1,
 495		.sw_tq2 = 1,
 496		.sw_tq1 = 1,
 497		.sw_tq0 = 1,
 498	} };
 499	union gmac_config1 config1 = { .bits = {
 500		.set_threshold = 16,
 501		.rel_threshold = 24,
 502	} };
 503	union gmac_config2 config2 = { .bits = {
 504		.set_threshold = 16,
 505		.rel_threshold = 32,
 506	} };
 507	union gmac_config3 config3 = { .bits = {
 508		.set_threshold = 0,
 509		.rel_threshold = 0,
 510	} };
 511	union gmac_config0 tmp;
 512
 513	config0.bits.max_len = gmac_pick_rx_max_len(netdev->mtu);
 514	tmp.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
 515	config0.bits.reserved = tmp.bits.reserved;
 516	writel(config0.bits32, port->gmac_base + GMAC_CONFIG0);
 517	writel(config1.bits32, port->gmac_base + GMAC_CONFIG1);
 518	writel(config2.bits32, port->gmac_base + GMAC_CONFIG2);
 519	writel(config3.bits32, port->gmac_base + GMAC_CONFIG3);
 520
 521	readl(port->dma_base + GMAC_AHB_WEIGHT_REG);
 522	writel(ahb_weight.bits32, port->dma_base + GMAC_AHB_WEIGHT_REG);
 523
 524	writel(hw_weigh.bits32,
 525	       port->dma_base + GMAC_TX_WEIGHTING_CTRL_0_REG);
 526	writel(sw_weigh.bits32,
 527	       port->dma_base + GMAC_TX_WEIGHTING_CTRL_1_REG);
 528
 529	port->rxq_order = DEFAULT_GMAC_RXQ_ORDER;
 530	port->txq_order = DEFAULT_GMAC_TXQ_ORDER;
 531	port->rx_coalesce_nsecs = DEFAULT_RX_COALESCE_NSECS;
 532
 533	/* Mark every quarter of the queue a packet for interrupt
 534	 * in order to be able to wake up the queue if it was stopped
 535	 */
 536	port->irq_every_tx_packets = 1 << (port->txq_order - 2);
 537
 538	return 0;
 539}
 540
 541static int gmac_setup_txqs(struct net_device *netdev)
 542{
 543	struct gemini_ethernet_port *port = netdev_priv(netdev);
 544	unsigned int n_txq = netdev->num_tx_queues;
 545	struct gemini_ethernet *geth = port->geth;
 546	size_t entries = 1 << port->txq_order;
 547	struct gmac_txq *txq = port->txq;
 548	struct gmac_txdesc *desc_ring;
 549	size_t len = n_txq * entries;
 550	struct sk_buff **skb_tab;
 551	void __iomem *rwptr_reg;
 552	unsigned int r;
 553	int i;
 554
 555	rwptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
 556
 557	skb_tab = kcalloc(len, sizeof(*skb_tab), GFP_KERNEL);
 558	if (!skb_tab)
 559		return -ENOMEM;
 560
 561	desc_ring = dma_alloc_coherent(geth->dev, len * sizeof(*desc_ring),
 562				       &port->txq_dma_base, GFP_KERNEL);
 563
 564	if (!desc_ring) {
 565		kfree(skb_tab);
 566		return -ENOMEM;
 567	}
 568
 569	if (port->txq_dma_base & ~DMA_Q_BASE_MASK) {
 570		dev_warn(geth->dev, "TX queue base is not aligned\n");
 571		dma_free_coherent(geth->dev, len * sizeof(*desc_ring),
 572				  desc_ring, port->txq_dma_base);
 573		kfree(skb_tab);
 574		return -ENOMEM;
 575	}
 576
 577	writel(port->txq_dma_base | port->txq_order,
 578	       port->dma_base + GMAC_SW_TX_QUEUE_BASE_REG);
 579
 580	for (i = 0; i < n_txq; i++) {
 581		txq->ring = desc_ring;
 582		txq->skb = skb_tab;
 583		txq->noirq_packets = 0;
 584
 585		r = readw(rwptr_reg);
 586		rwptr_reg += 2;
 587		writew(r, rwptr_reg);
 588		rwptr_reg += 2;
 589		txq->cptr = r;
 590
 591		txq++;
 592		desc_ring += entries;
 593		skb_tab += entries;
 594	}
 595
 596	return 0;
 597}
 598
 599static void gmac_clean_txq(struct net_device *netdev, struct gmac_txq *txq,
 600			   unsigned int r)
 601{
 602	struct gemini_ethernet_port *port = netdev_priv(netdev);
 603	unsigned int m = (1 << port->txq_order) - 1;
 604	struct gemini_ethernet *geth = port->geth;
 605	unsigned int c = txq->cptr;
 606	union gmac_txdesc_0 word0;
 607	union gmac_txdesc_1 word1;
 608	unsigned int hwchksum = 0;
 609	unsigned long bytes = 0;
 610	struct gmac_txdesc *txd;
 611	unsigned short nfrags;
 612	unsigned int errs = 0;
 613	unsigned int pkts = 0;
 614	unsigned int word3;
 615	dma_addr_t mapping;
 616
 617	if (c == r)
 618		return;
 619
 620	while (c != r) {
 621		txd = txq->ring + c;
 622		word0 = txd->word0;
 623		word1 = txd->word1;
 624		mapping = txd->word2.buf_adr;
 625		word3 = txd->word3.bits32;
 626
 627		dma_unmap_single(geth->dev, mapping,
 628				 word0.bits.buffer_size, DMA_TO_DEVICE);
 629
 630		if (word3 & EOF_BIT)
 631			dev_kfree_skb(txq->skb[c]);
 632
 633		c++;
 634		c &= m;
 635
 636		if (!(word3 & SOF_BIT))
 637			continue;
 638
 639		if (!word0.bits.status_tx_ok) {
 640			errs++;
 641			continue;
 642		}
 643
 644		pkts++;
 645		bytes += txd->word1.bits.byte_count;
 646
 647		if (word1.bits32 & TSS_CHECKUM_ENABLE)
 648			hwchksum++;
 649
 650		nfrags = word0.bits.desc_count - 1;
 651		if (nfrags) {
 652			if (nfrags >= TX_MAX_FRAGS)
 653				nfrags = TX_MAX_FRAGS - 1;
 654
 655			u64_stats_update_begin(&port->tx_stats_syncp);
 656			port->tx_frag_stats[nfrags]++;
 657			u64_stats_update_end(&port->tx_stats_syncp);
 658		}
 659	}
 660
 661	u64_stats_update_begin(&port->ir_stats_syncp);
 662	port->stats.tx_errors += errs;
 663	port->stats.tx_packets += pkts;
 664	port->stats.tx_bytes += bytes;
 665	port->tx_hw_csummed += hwchksum;
 666	u64_stats_update_end(&port->ir_stats_syncp);
 667
 668	txq->cptr = c;
 669}
 670
 671static void gmac_cleanup_txqs(struct net_device *netdev)
 672{
 673	struct gemini_ethernet_port *port = netdev_priv(netdev);
 674	unsigned int n_txq = netdev->num_tx_queues;
 675	struct gemini_ethernet *geth = port->geth;
 676	void __iomem *rwptr_reg;
 677	unsigned int r, i;
 678
 679	rwptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
 680
 681	for (i = 0; i < n_txq; i++) {
 682		r = readw(rwptr_reg);
 683		rwptr_reg += 2;
 684		writew(r, rwptr_reg);
 685		rwptr_reg += 2;
 686
 687		gmac_clean_txq(netdev, port->txq + i, r);
 688	}
 689	writel(0, port->dma_base + GMAC_SW_TX_QUEUE_BASE_REG);
 690
 691	kfree(port->txq->skb);
 692	dma_free_coherent(geth->dev,
 693			  n_txq * sizeof(*port->txq->ring) << port->txq_order,
 694			  port->txq->ring, port->txq_dma_base);
 695}
 696
 697static int gmac_setup_rxq(struct net_device *netdev)
 698{
 699	struct gemini_ethernet_port *port = netdev_priv(netdev);
 700	struct gemini_ethernet *geth = port->geth;
 701	struct nontoe_qhdr __iomem *qhdr;
 702
 703	qhdr = geth->base + TOE_DEFAULT_Q_HDR_BASE(netdev->dev_id);
 704	port->rxq_rwptr = &qhdr->word1;
 705
 706	/* Remap a slew of memory to use for the RX queue */
 707	port->rxq_ring = dma_alloc_coherent(geth->dev,
 708				sizeof(*port->rxq_ring) << port->rxq_order,
 709				&port->rxq_dma_base, GFP_KERNEL);
 710	if (!port->rxq_ring)
 711		return -ENOMEM;
 712	if (port->rxq_dma_base & ~NONTOE_QHDR0_BASE_MASK) {
 713		dev_warn(geth->dev, "RX queue base is not aligned\n");
 714		return -ENOMEM;
 715	}
 716
 717	writel(port->rxq_dma_base | port->rxq_order, &qhdr->word0);
 718	writel(0, port->rxq_rwptr);
 719	return 0;
 720}
 721
 722static struct gmac_queue_page *
 723gmac_get_queue_page(struct gemini_ethernet *geth,
 724		    struct gemini_ethernet_port *port,
 725		    dma_addr_t addr)
 726{
 727	struct gmac_queue_page *gpage;
 728	dma_addr_t mapping;
 729	int i;
 730
 731	/* Only look for even pages */
 732	mapping = addr & PAGE_MASK;
 733
 734	if (!geth->freeq_pages) {
 735		dev_err(geth->dev, "try to get page with no page list\n");
 736		return NULL;
 737	}
 738
 739	/* Look up a ring buffer page from virtual mapping */
 740	for (i = 0; i < geth->num_freeq_pages; i++) {
 741		gpage = &geth->freeq_pages[i];
 742		if (gpage->mapping == mapping)
 743			return gpage;
 744	}
 745
 746	return NULL;
 747}
 748
 749static void gmac_cleanup_rxq(struct net_device *netdev)
 750{
 751	struct gemini_ethernet_port *port = netdev_priv(netdev);
 752	struct gemini_ethernet *geth = port->geth;
 753	struct gmac_rxdesc *rxd = port->rxq_ring;
 754	static struct gmac_queue_page *gpage;
 755	struct nontoe_qhdr __iomem *qhdr;
 756	void __iomem *dma_reg;
 757	void __iomem *ptr_reg;
 758	dma_addr_t mapping;
 759	union dma_rwptr rw;
 760	unsigned int r, w;
 761
 762	qhdr = geth->base +
 763		TOE_DEFAULT_Q_HDR_BASE(netdev->dev_id);
 764	dma_reg = &qhdr->word0;
 765	ptr_reg = &qhdr->word1;
 766
 767	rw.bits32 = readl(ptr_reg);
 768	r = rw.bits.rptr;
 769	w = rw.bits.wptr;
 770	writew(r, ptr_reg + 2);
 771
 772	writel(0, dma_reg);
 773
 774	/* Loop from read pointer to write pointer of the RX queue
 775	 * and free up all pages by the queue.
 776	 */
 777	while (r != w) {
 778		mapping = rxd[r].word2.buf_adr;
 779		r++;
 780		r &= ((1 << port->rxq_order) - 1);
 781
 782		if (!mapping)
 783			continue;
 784
 785		/* Freeq pointers are one page off */
 786		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
 787		if (!gpage) {
 788			dev_err(geth->dev, "could not find page\n");
 789			continue;
 790		}
 791		/* Release the RX queue reference to the page */
 792		put_page(gpage->page);
 793	}
 794
 795	dma_free_coherent(geth->dev, sizeof(*port->rxq_ring) << port->rxq_order,
 796			  port->rxq_ring, port->rxq_dma_base);
 797}
 798
 799static struct page *geth_freeq_alloc_map_page(struct gemini_ethernet *geth,
 800					      int pn)
 801{
 802	struct gmac_rxdesc *freeq_entry;
 803	struct gmac_queue_page *gpage;
 804	unsigned int fpp_order;
 805	unsigned int frag_len;
 806	dma_addr_t mapping;
 807	struct page *page;
 808	int i;
 809
 810	/* First allocate and DMA map a single page */
 811	page = alloc_page(GFP_ATOMIC);
 812	if (!page)
 813		return NULL;
 814
 815	mapping = dma_map_single(geth->dev, page_address(page),
 816				 PAGE_SIZE, DMA_FROM_DEVICE);
 817	if (dma_mapping_error(geth->dev, mapping)) {
 818		put_page(page);
 819		return NULL;
 820	}
 821
 822	/* The assign the page mapping (physical address) to the buffer address
 823	 * in the hardware queue. PAGE_SHIFT on ARM is 12 (1 page is 4096 bytes,
 824	 * 4k), and the default RX frag order is 11 (fragments are up 20 2048
 825	 * bytes, 2k) so fpp_order (fragments per page order) is default 1. Thus
 826	 * each page normally needs two entries in the queue.
 827	 */
 828	frag_len = 1 << geth->freeq_frag_order; /* Usually 2048 */
 829	fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 830	freeq_entry = geth->freeq_ring + (pn << fpp_order);
 831	dev_dbg(geth->dev, "allocate page %d fragment length %d fragments per page %d, freeq entry %p\n",
 832		 pn, frag_len, (1 << fpp_order), freeq_entry);
 833	for (i = (1 << fpp_order); i > 0; i--) {
 834		freeq_entry->word2.buf_adr = mapping;
 835		freeq_entry++;
 836		mapping += frag_len;
 837	}
 838
 839	/* If the freeq entry already has a page mapped, then unmap it. */
 840	gpage = &geth->freeq_pages[pn];
 841	if (gpage->page) {
 842		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
 843		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
 844		/* This should be the last reference to the page so it gets
 845		 * released
 846		 */
 847		put_page(gpage->page);
 848	}
 849
 850	/* Then put our new mapping into the page table */
 851	dev_dbg(geth->dev, "page %d, DMA addr: %08x, page %p\n",
 852		pn, (unsigned int)mapping, page);
 853	gpage->mapping = mapping;
 854	gpage->page = page;
 855
 856	return page;
 857}
 858
 859/**
 860 * geth_fill_freeq() - Fill the freeq with empty fragments to use
 861 * @geth: the ethernet adapter
 862 * @refill: whether to reset the queue by filling in all freeq entries or
 863 * just refill it, usually the interrupt to refill the queue happens when
 864 * the queue is half empty.
 865 */
 866static unsigned int geth_fill_freeq(struct gemini_ethernet *geth, bool refill)
 867{
 868	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 869	unsigned int count = 0;
 870	unsigned int pn, epn;
 871	unsigned long flags;
 872	union dma_rwptr rw;
 873	unsigned int m_pn;
 874
 875	/* Mask for page */
 876	m_pn = (1 << (geth->freeq_order - fpp_order)) - 1;
 877
 878	spin_lock_irqsave(&geth->freeq_lock, flags);
 879
 880	rw.bits32 = readl(geth->base + GLOBAL_SWFQ_RWPTR_REG);
 881	pn = (refill ? rw.bits.wptr : rw.bits.rptr) >> fpp_order;
 882	epn = (rw.bits.rptr >> fpp_order) - 1;
 883	epn &= m_pn;
 884
 885	/* Loop over the freeq ring buffer entries */
 886	while (pn != epn) {
 887		struct gmac_queue_page *gpage;
 888		struct page *page;
 889
 890		gpage = &geth->freeq_pages[pn];
 891		page = gpage->page;
 892
 893		dev_dbg(geth->dev, "fill entry %d page ref count %d add %d refs\n",
 894			pn, page_ref_count(page), 1 << fpp_order);
 895
 896		if (page_ref_count(page) > 1) {
 897			unsigned int fl = (pn - epn) & m_pn;
 898
 899			if (fl > 64 >> fpp_order)
 900				break;
 901
 902			page = geth_freeq_alloc_map_page(geth, pn);
 903			if (!page)
 904				break;
 905		}
 906
 907		/* Add one reference per fragment in the page */
 908		page_ref_add(page, 1 << fpp_order);
 909		count += 1 << fpp_order;
 910		pn++;
 911		pn &= m_pn;
 912	}
 913
 914	writew(pn << fpp_order, geth->base + GLOBAL_SWFQ_RWPTR_REG + 2);
 915
 916	spin_unlock_irqrestore(&geth->freeq_lock, flags);
 917
 918	return count;
 919}
 920
 921static int geth_setup_freeq(struct gemini_ethernet *geth)
 922{
 923	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 924	unsigned int frag_len = 1 << geth->freeq_frag_order;
 925	unsigned int len = 1 << geth->freeq_order;
 926	unsigned int pages = len >> fpp_order;
 927	union queue_threshold qt;
 928	union dma_skb_size skbsz;
 929	unsigned int filled;
 930	unsigned int pn;
 931
 932	geth->freeq_ring = dma_alloc_coherent(geth->dev,
 933		sizeof(*geth->freeq_ring) << geth->freeq_order,
 934		&geth->freeq_dma_base, GFP_KERNEL);
 935	if (!geth->freeq_ring)
 936		return -ENOMEM;
 937	if (geth->freeq_dma_base & ~DMA_Q_BASE_MASK) {
 938		dev_warn(geth->dev, "queue ring base is not aligned\n");
 939		goto err_freeq;
 940	}
 941
 942	/* Allocate a mapping to page look-up index */
 943	geth->freeq_pages = kcalloc(pages, sizeof(*geth->freeq_pages),
 944				    GFP_KERNEL);
 945	if (!geth->freeq_pages)
 946		goto err_freeq;
 947	geth->num_freeq_pages = pages;
 948
 949	dev_info(geth->dev, "allocate %d pages for queue\n", pages);
 950	for (pn = 0; pn < pages; pn++)
 951		if (!geth_freeq_alloc_map_page(geth, pn))
 952			goto err_freeq_alloc;
 953
 954	filled = geth_fill_freeq(geth, false);
 955	if (!filled)
 956		goto err_freeq_alloc;
 957
 958	qt.bits32 = readl(geth->base + GLOBAL_QUEUE_THRESHOLD_REG);
 959	qt.bits.swfq_empty = 32;
 960	writel(qt.bits32, geth->base + GLOBAL_QUEUE_THRESHOLD_REG);
 961
 962	skbsz.bits.sw_skb_size = 1 << geth->freeq_frag_order;
 963	writel(skbsz.bits32, geth->base + GLOBAL_DMA_SKB_SIZE_REG);
 964	writel(geth->freeq_dma_base | geth->freeq_order,
 965	       geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
 966
 967	return 0;
 968
 969err_freeq_alloc:
 970	while (pn > 0) {
 971		struct gmac_queue_page *gpage;
 972		dma_addr_t mapping;
 973
 974		--pn;
 975		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
 976		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
 977		gpage = &geth->freeq_pages[pn];
 978		put_page(gpage->page);
 979	}
 980
 981	kfree(geth->freeq_pages);
 982err_freeq:
 983	dma_free_coherent(geth->dev,
 984			  sizeof(*geth->freeq_ring) << geth->freeq_order,
 985			  geth->freeq_ring, geth->freeq_dma_base);
 986	geth->freeq_ring = NULL;
 987	return -ENOMEM;
 988}
 989
 990/**
 991 * geth_cleanup_freeq() - cleanup the DMA mappings and free the queue
 992 * @geth: the Gemini global ethernet state
 993 */
 994static void geth_cleanup_freeq(struct gemini_ethernet *geth)
 995{
 996	unsigned int fpp_order = PAGE_SHIFT - geth->freeq_frag_order;
 997	unsigned int frag_len = 1 << geth->freeq_frag_order;
 998	unsigned int len = 1 << geth->freeq_order;
 999	unsigned int pages = len >> fpp_order;
1000	unsigned int pn;
1001
1002	writew(readw(geth->base + GLOBAL_SWFQ_RWPTR_REG),
1003	       geth->base + GLOBAL_SWFQ_RWPTR_REG + 2);
1004	writel(0, geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
1005
1006	for (pn = 0; pn < pages; pn++) {
1007		struct gmac_queue_page *gpage;
1008		dma_addr_t mapping;
1009
1010		mapping = geth->freeq_ring[pn << fpp_order].word2.buf_adr;
1011		dma_unmap_single(geth->dev, mapping, frag_len, DMA_FROM_DEVICE);
1012
1013		gpage = &geth->freeq_pages[pn];
1014		while (page_ref_count(gpage->page) > 0)
1015			put_page(gpage->page);
1016	}
1017
1018	kfree(geth->freeq_pages);
1019
1020	dma_free_coherent(geth->dev,
1021			  sizeof(*geth->freeq_ring) << geth->freeq_order,
1022			  geth->freeq_ring, geth->freeq_dma_base);
1023}
1024
1025/**
1026 * geth_resize_freeq() - resize the software queue depth
1027 * @port: the port requesting the change
1028 *
1029 * This gets called at least once during probe() so the device queue gets
1030 * "resized" from the hardware defaults. Since both ports/net devices share
1031 * the same hardware queue, some synchronization between the ports is
1032 * needed.
1033 */
1034static int geth_resize_freeq(struct gemini_ethernet_port *port)
1035{
1036	struct gemini_ethernet *geth = port->geth;
1037	struct net_device *netdev = port->netdev;
1038	struct gemini_ethernet_port *other_port;
1039	struct net_device *other_netdev;
1040	unsigned int new_size = 0;
1041	unsigned int new_order;
1042	unsigned long flags;
1043	u32 en;
1044	int ret;
1045
1046	if (netdev->dev_id == 0)
1047		other_netdev = geth->port1->netdev;
1048	else
1049		other_netdev = geth->port0->netdev;
1050
1051	if (other_netdev && netif_running(other_netdev))
1052		return -EBUSY;
1053
1054	new_size = 1 << (port->rxq_order + 1);
1055	netdev_dbg(netdev, "port %d size: %d order %d\n",
1056		   netdev->dev_id,
1057		   new_size,
1058		   port->rxq_order);
1059	if (other_netdev) {
1060		other_port = netdev_priv(other_netdev);
1061		new_size += 1 << (other_port->rxq_order + 1);
1062		netdev_dbg(other_netdev, "port %d size: %d order %d\n",
1063			   other_netdev->dev_id,
1064			   (1 << (other_port->rxq_order + 1)),
1065			   other_port->rxq_order);
1066	}
1067
1068	new_order = min(15, ilog2(new_size - 1) + 1);
1069	dev_dbg(geth->dev, "set shared queue to size %d order %d\n",
1070		new_size, new_order);
1071	if (geth->freeq_order == new_order)
1072		return 0;
1073
1074	spin_lock_irqsave(&geth->irq_lock, flags);
1075
1076	/* Disable the software queue IRQs */
1077	en = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1078	en &= ~SWFQ_EMPTY_INT_BIT;
1079	writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1080	spin_unlock_irqrestore(&geth->irq_lock, flags);
1081
1082	/* Drop the old queue */
1083	if (geth->freeq_ring)
1084		geth_cleanup_freeq(geth);
1085
1086	/* Allocate a new queue with the desired order */
1087	geth->freeq_order = new_order;
1088	ret = geth_setup_freeq(geth);
1089
1090	/* Restart the interrupts - NOTE if this is the first resize
1091	 * after probe(), this is where the interrupts get turned on
1092	 * in the first place.
1093	 */
1094	spin_lock_irqsave(&geth->irq_lock, flags);
1095	en |= SWFQ_EMPTY_INT_BIT;
1096	writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1097	spin_unlock_irqrestore(&geth->irq_lock, flags);
1098
1099	return ret;
1100}
1101
1102static void gmac_tx_irq_enable(struct net_device *netdev,
1103			       unsigned int txq, int en)
1104{
1105	struct gemini_ethernet_port *port = netdev_priv(netdev);
1106	struct gemini_ethernet *geth = port->geth;
1107	unsigned long flags;
1108	u32 val, mask;
1109
1110	netdev_dbg(netdev, "%s device %d\n", __func__, netdev->dev_id);
1111
1112	spin_lock_irqsave(&geth->irq_lock, flags);
1113
1114	mask = GMAC0_IRQ0_TXQ0_INTS << (6 * netdev->dev_id + txq);
1115
1116	if (en)
1117		writel(mask, geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
1118
1119	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1120	val = en ? val | mask : val & ~mask;
1121	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1122
1123	spin_unlock_irqrestore(&geth->irq_lock, flags);
1124}
1125
1126static void gmac_tx_irq(struct net_device *netdev, unsigned int txq_num)
1127{
1128	struct netdev_queue *ntxq = netdev_get_tx_queue(netdev, txq_num);
1129
1130	gmac_tx_irq_enable(netdev, txq_num, 0);
1131	netif_tx_wake_queue(ntxq);
1132}
1133
1134static int gmac_map_tx_bufs(struct net_device *netdev, struct sk_buff *skb,
1135			    struct gmac_txq *txq, unsigned short *desc)
1136{
1137	struct gemini_ethernet_port *port = netdev_priv(netdev);
1138	struct skb_shared_info *skb_si =  skb_shinfo(skb);
1139	unsigned short m = (1 << port->txq_order) - 1;
1140	short frag, last_frag = skb_si->nr_frags - 1;
1141	struct gemini_ethernet *geth = port->geth;
1142	unsigned int word1, word3, buflen;
1143	unsigned short w = *desc;
1144	struct gmac_txdesc *txd;
1145	skb_frag_t *skb_frag;
1146	dma_addr_t mapping;
1147	bool tcp = false;
1148	void *buffer;
1149	u16 mss;
1150	int ret;
1151
1152	word1 = skb->len;
1153	word3 = SOF_BIT;
1154
1155	/* Determine if we are doing TCP */
1156	if (skb->protocol == htons(ETH_P_IP))
1157		tcp = (ip_hdr(skb)->protocol == IPPROTO_TCP);
1158	else
1159		/* IPv6 */
1160		tcp = (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP);
1161
1162	mss = skb_shinfo(skb)->gso_size;
1163	if (mss) {
1164		/* This means we are dealing with TCP and skb->len is the
1165		 * sum total of all the segments. The TSO will deal with
1166		 * chopping this up for us.
1167		 */
1168		/* The accelerator needs the full frame size here */
1169		mss += skb_tcp_all_headers(skb);
1170		netdev_dbg(netdev, "segment offloading mss = %04x len=%04x\n",
1171			   mss, skb->len);
1172		word1 |= TSS_MTU_ENABLE_BIT;
1173		word3 |= mss;
1174	} else if (tcp) {
1175		/* Even if we are not using TSO, use the hardware offloader
1176		 * for transferring the TCP frame: this hardware has partial
1177		 * TCP awareness (called TOE - TCP Offload Engine) and will
1178		 * according to the datasheet put packets belonging to the
1179		 * same TCP connection in the same queue for the TOE/TSO
1180		 * engine to process. The engine will deal with chopping
1181		 * up frames that exceed ETH_DATA_LEN which the
1182		 * checksumming engine cannot handle (see below) into
1183		 * manageable chunks. It flawlessly deals with quite big
1184		 * frames and frames containing custom DSA EtherTypes.
1185		 */
1186		mss = netdev->mtu + skb_tcp_all_headers(skb);
1187		mss = min(mss, skb->len);
1188		netdev_dbg(netdev, "TOE/TSO len %04x mtu %04x mss %04x\n",
1189			   skb->len, netdev->mtu, mss);
1190		word1 |= TSS_MTU_ENABLE_BIT;
1191		word3 |= mss;
1192	} else if (skb->len >= ETH_FRAME_LEN) {
1193		/* Hardware offloaded checksumming isn't working on non-TCP frames
1194		 * bigger than 1514 bytes. A hypothesis about this is that the
1195		 * checksum buffer is only 1518 bytes, so when the frames get
1196		 * bigger they get truncated, or the last few bytes get
1197		 * overwritten by the FCS.
1198		 *
1199		 * Just use software checksumming and bypass on bigger frames.
1200		 */
1201		if (skb->ip_summed == CHECKSUM_PARTIAL) {
1202			ret = skb_checksum_help(skb);
1203			if (ret)
1204				return ret;
1205		}
1206		word1 |= TSS_BYPASS_BIT;
1207	}
1208
1209	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1210		/* We do not switch off the checksumming on non TCP/UDP
1211		 * frames: as is shown from tests, the checksumming engine
1212		 * is smart enough to see that a frame is not actually TCP
1213		 * or UDP and then just pass it through without any changes
1214		 * to the frame.
1215		 */
1216		if (skb->protocol == htons(ETH_P_IP))
1217			word1 |= TSS_IP_CHKSUM_BIT;
1218		else
1219			word1 |= TSS_IPV6_ENABLE_BIT;
1220
1221		word1 |= tcp ? TSS_TCP_CHKSUM_BIT : TSS_UDP_CHKSUM_BIT;
1222	}
1223
1224	frag = -1;
1225	while (frag <= last_frag) {
1226		if (frag == -1) {
1227			buffer = skb->data;
1228			buflen = skb_headlen(skb);
1229		} else {
1230			skb_frag = skb_si->frags + frag;
1231			buffer = skb_frag_address(skb_frag);
1232			buflen = skb_frag_size(skb_frag);
1233		}
1234
1235		if (frag == last_frag) {
1236			word3 |= EOF_BIT;
1237			txq->skb[w] = skb;
1238		}
1239
1240		mapping = dma_map_single(geth->dev, buffer, buflen,
1241					 DMA_TO_DEVICE);
1242		if (dma_mapping_error(geth->dev, mapping))
1243			goto map_error;
1244
1245		txd = txq->ring + w;
1246		txd->word0.bits32 = buflen;
1247		txd->word1.bits32 = word1;
1248		txd->word2.buf_adr = mapping;
1249		txd->word3.bits32 = word3;
1250
1251		word3 &= MTU_SIZE_BIT_MASK;
1252		w++;
1253		w &= m;
1254		frag++;
1255	}
1256
1257	*desc = w;
1258	return 0;
1259
1260map_error:
1261	while (w != *desc) {
1262		w--;
1263		w &= m;
1264
1265		dma_unmap_page(geth->dev, txq->ring[w].word2.buf_adr,
1266			       txq->ring[w].word0.bits.buffer_size,
1267			       DMA_TO_DEVICE);
1268	}
1269	return -ENOMEM;
1270}
1271
1272static netdev_tx_t gmac_start_xmit(struct sk_buff *skb,
1273				   struct net_device *netdev)
1274{
1275	struct gemini_ethernet_port *port = netdev_priv(netdev);
1276	unsigned short m = (1 << port->txq_order) - 1;
1277	struct netdev_queue *ntxq;
1278	unsigned short r, w, d;
1279	void __iomem *ptr_reg;
1280	struct gmac_txq *txq;
1281	int txq_num, nfrags;
1282	union dma_rwptr rw;
1283
1284	if (skb->len >= 0x10000)
1285		goto out_drop_free;
1286
1287	txq_num = skb_get_queue_mapping(skb);
1288	ptr_reg = port->dma_base + GMAC_SW_TX_QUEUE_PTR_REG(txq_num);
1289	txq = &port->txq[txq_num];
1290	ntxq = netdev_get_tx_queue(netdev, txq_num);
1291	nfrags = skb_shinfo(skb)->nr_frags;
1292
1293	rw.bits32 = readl(ptr_reg);
1294	r = rw.bits.rptr;
1295	w = rw.bits.wptr;
1296
1297	d = txq->cptr - w - 1;
1298	d &= m;
1299
1300	if (d < nfrags + 2) {
1301		gmac_clean_txq(netdev, txq, r);
1302		d = txq->cptr - w - 1;
1303		d &= m;
1304
1305		if (d < nfrags + 2) {
1306			netif_tx_stop_queue(ntxq);
1307
1308			d = txq->cptr + nfrags + 16;
1309			d &= m;
1310			txq->ring[d].word3.bits.eofie = 1;
1311			gmac_tx_irq_enable(netdev, txq_num, 1);
1312
1313			u64_stats_update_begin(&port->tx_stats_syncp);
1314			netdev->stats.tx_fifo_errors++;
1315			u64_stats_update_end(&port->tx_stats_syncp);
1316			return NETDEV_TX_BUSY;
1317		}
1318	}
1319
1320	if (gmac_map_tx_bufs(netdev, skb, txq, &w)) {
1321		if (skb_linearize(skb))
1322			goto out_drop;
1323
1324		u64_stats_update_begin(&port->tx_stats_syncp);
1325		port->tx_frags_linearized++;
1326		u64_stats_update_end(&port->tx_stats_syncp);
1327
1328		if (gmac_map_tx_bufs(netdev, skb, txq, &w))
1329			goto out_drop_free;
1330	}
1331
1332	writew(w, ptr_reg + 2);
1333
1334	gmac_clean_txq(netdev, txq, r);
1335	return NETDEV_TX_OK;
1336
1337out_drop_free:
1338	dev_kfree_skb(skb);
1339out_drop:
1340	u64_stats_update_begin(&port->tx_stats_syncp);
1341	port->stats.tx_dropped++;
1342	u64_stats_update_end(&port->tx_stats_syncp);
1343	return NETDEV_TX_OK;
1344}
1345
1346static void gmac_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1347{
1348	netdev_err(netdev, "Tx timeout\n");
1349	gmac_dump_dma_state(netdev);
1350}
1351
1352static void gmac_enable_irq(struct net_device *netdev, int enable)
1353{
1354	struct gemini_ethernet_port *port = netdev_priv(netdev);
1355	struct gemini_ethernet *geth = port->geth;
1356	unsigned long flags;
1357	u32 val, mask;
1358
1359	netdev_dbg(netdev, "%s device %d %s\n", __func__,
1360		   netdev->dev_id, enable ? "enable" : "disable");
1361	spin_lock_irqsave(&geth->irq_lock, flags);
1362
1363	mask = GMAC0_IRQ0_2 << (netdev->dev_id * 2);
1364	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1365	val = enable ? (val | mask) : (val & ~mask);
1366	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1367
1368	mask = DEFAULT_Q0_INT_BIT << netdev->dev_id;
1369	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1370	val = enable ? (val | mask) : (val & ~mask);
1371	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1372
1373	mask = GMAC0_IRQ4_8 << (netdev->dev_id * 8);
1374	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1375	val = enable ? (val | mask) : (val & ~mask);
1376	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1377
1378	spin_unlock_irqrestore(&geth->irq_lock, flags);
1379}
1380
1381static void gmac_enable_rx_irq(struct net_device *netdev, int enable)
1382{
1383	struct gemini_ethernet_port *port = netdev_priv(netdev);
1384	struct gemini_ethernet *geth = port->geth;
1385	unsigned long flags;
1386	u32 val, mask;
1387
1388	netdev_dbg(netdev, "%s device %d %s\n", __func__, netdev->dev_id,
1389		   enable ? "enable" : "disable");
1390	spin_lock_irqsave(&geth->irq_lock, flags);
1391	mask = DEFAULT_Q0_INT_BIT << netdev->dev_id;
1392
1393	val = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1394	val = enable ? (val | mask) : (val & ~mask);
1395	writel(val, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1396
1397	spin_unlock_irqrestore(&geth->irq_lock, flags);
1398}
1399
1400static struct sk_buff *gmac_skb_if_good_frame(struct gemini_ethernet_port *port,
1401					      union gmac_rxdesc_0 word0,
1402					      unsigned int frame_len)
1403{
1404	unsigned int rx_csum = word0.bits.chksum_status;
1405	unsigned int rx_status = word0.bits.status;
1406	struct sk_buff *skb = NULL;
1407
1408	port->rx_stats[rx_status]++;
1409	port->rx_csum_stats[rx_csum]++;
1410
1411	if (word0.bits.derr || word0.bits.perr ||
1412	    rx_status || frame_len < ETH_ZLEN ||
1413	    rx_csum >= RX_CHKSUM_IP_ERR_UNKNOWN) {
1414		port->stats.rx_errors++;
1415
1416		if (frame_len < ETH_ZLEN || RX_ERROR_LENGTH(rx_status))
1417			port->stats.rx_length_errors++;
1418		if (RX_ERROR_OVER(rx_status))
1419			port->stats.rx_over_errors++;
1420		if (RX_ERROR_CRC(rx_status))
1421			port->stats.rx_crc_errors++;
1422		if (RX_ERROR_FRAME(rx_status))
1423			port->stats.rx_frame_errors++;
1424		return NULL;
1425	}
1426
1427	skb = napi_get_frags(&port->napi);
1428	if (!skb)
1429		goto update_exit;
1430
1431	if (rx_csum == RX_CHKSUM_IP_UDP_TCP_OK)
1432		skb->ip_summed = CHECKSUM_UNNECESSARY;
1433
1434update_exit:
1435	port->stats.rx_bytes += frame_len;
1436	port->stats.rx_packets++;
1437	return skb;
1438}
1439
1440static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget)
1441{
1442	struct gemini_ethernet_port *port = netdev_priv(netdev);
1443	unsigned short m = (1 << port->rxq_order) - 1;
1444	struct gemini_ethernet *geth = port->geth;
1445	void __iomem *ptr_reg = port->rxq_rwptr;
1446	unsigned int frame_len, frag_len;
1447	struct gmac_rxdesc *rx = NULL;
1448	struct gmac_queue_page *gpage;
1449	static struct sk_buff *skb;
1450	union gmac_rxdesc_0 word0;
1451	union gmac_rxdesc_1 word1;
1452	union gmac_rxdesc_3 word3;
1453	struct page *page = NULL;
1454	unsigned int page_offs;
1455	unsigned long flags;
1456	unsigned short r, w;
1457	union dma_rwptr rw;
1458	dma_addr_t mapping;
1459	int frag_nr = 0;
1460
1461	spin_lock_irqsave(&geth->irq_lock, flags);
1462	rw.bits32 = readl(ptr_reg);
1463	/* Reset interrupt as all packages until here are taken into account */
1464	writel(DEFAULT_Q0_INT_BIT << netdev->dev_id,
1465	       geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
1466	spin_unlock_irqrestore(&geth->irq_lock, flags);
1467
1468	r = rw.bits.rptr;
1469	w = rw.bits.wptr;
1470
1471	while (budget && w != r) {
1472		rx = port->rxq_ring + r;
1473		word0 = rx->word0;
1474		word1 = rx->word1;
1475		mapping = rx->word2.buf_adr;
1476		word3 = rx->word3;
1477
1478		r++;
1479		r &= m;
1480
1481		frag_len = word0.bits.buffer_size;
1482		frame_len = word1.bits.byte_count;
1483		page_offs = mapping & ~PAGE_MASK;
1484
1485		if (!mapping) {
1486			netdev_err(netdev,
1487				   "rxq[%u]: HW BUG: zero DMA desc\n", r);
1488			goto err_drop;
1489		}
1490
1491		/* Freeq pointers are one page off */
1492		gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE);
1493		if (!gpage) {
1494			dev_err(geth->dev, "could not find mapping\n");
1495			continue;
1496		}
1497		page = gpage->page;
1498
1499		if (word3.bits32 & SOF_BIT) {
1500			if (skb) {
1501				napi_free_frags(&port->napi);
1502				port->stats.rx_dropped++;
1503			}
1504
1505			skb = gmac_skb_if_good_frame(port, word0, frame_len);
1506			if (!skb)
1507				goto err_drop;
1508
1509			page_offs += NET_IP_ALIGN;
1510			frag_len -= NET_IP_ALIGN;
1511			frag_nr = 0;
1512
1513		} else if (!skb) {
1514			put_page(page);
1515			continue;
1516		}
1517
1518		if (word3.bits32 & EOF_BIT)
1519			frag_len = frame_len - skb->len;
1520
1521		/* append page frag to skb */
1522		if (frag_nr == MAX_SKB_FRAGS)
1523			goto err_drop;
1524
1525		if (frag_len == 0)
1526			netdev_err(netdev, "Received fragment with len = 0\n");
1527
1528		skb_fill_page_desc(skb, frag_nr, page, page_offs, frag_len);
1529		skb->len += frag_len;
1530		skb->data_len += frag_len;
1531		skb->truesize += frag_len;
1532		frag_nr++;
1533
1534		if (word3.bits32 & EOF_BIT) {
1535			napi_gro_frags(&port->napi);
1536			skb = NULL;
1537			--budget;
1538		}
1539		continue;
1540
1541err_drop:
1542		if (skb) {
1543			napi_free_frags(&port->napi);
1544			skb = NULL;
1545		}
1546
1547		if (mapping)
1548			put_page(page);
1549
1550		port->stats.rx_dropped++;
1551	}
1552
1553	writew(r, ptr_reg);
1554	return budget;
1555}
1556
1557static int gmac_napi_poll(struct napi_struct *napi, int budget)
1558{
1559	struct gemini_ethernet_port *port = netdev_priv(napi->dev);
1560	struct gemini_ethernet *geth = port->geth;
1561	unsigned int freeq_threshold;
1562	unsigned int received;
1563
1564	freeq_threshold = 1 << (geth->freeq_order - 1);
1565	u64_stats_update_begin(&port->rx_stats_syncp);
1566
1567	received = gmac_rx(napi->dev, budget);
1568	if (received < budget) {
1569		napi_gro_flush(napi, false);
1570		napi_complete_done(napi, received);
1571		gmac_enable_rx_irq(napi->dev, 1);
1572		++port->rx_napi_exits;
1573	}
1574
1575	port->freeq_refill += (budget - received);
1576	if (port->freeq_refill > freeq_threshold) {
1577		port->freeq_refill -= freeq_threshold;
1578		geth_fill_freeq(geth, true);
1579	}
1580
1581	u64_stats_update_end(&port->rx_stats_syncp);
1582	return received;
1583}
1584
1585static void gmac_dump_dma_state(struct net_device *netdev)
1586{
1587	struct gemini_ethernet_port *port = netdev_priv(netdev);
1588	struct gemini_ethernet *geth = port->geth;
1589	void __iomem *ptr_reg;
1590	u32 reg[5];
1591
1592	/* Interrupt status */
1593	reg[0] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
1594	reg[1] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
1595	reg[2] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_2_REG);
1596	reg[3] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_3_REG);
1597	reg[4] = readl(geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
1598	netdev_err(netdev, "IRQ status: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1599		   reg[0], reg[1], reg[2], reg[3], reg[4]);
1600
1601	/* Interrupt enable */
1602	reg[0] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
1603	reg[1] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
1604	reg[2] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_2_REG);
1605	reg[3] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_3_REG);
1606	reg[4] = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
1607	netdev_err(netdev, "IRQ enable: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1608		   reg[0], reg[1], reg[2], reg[3], reg[4]);
1609
1610	/* RX DMA status */
1611	reg[0] = readl(port->dma_base + GMAC_DMA_RX_FIRST_DESC_REG);
1612	reg[1] = readl(port->dma_base + GMAC_DMA_RX_CURR_DESC_REG);
1613	reg[2] = GET_RPTR(port->rxq_rwptr);
1614	reg[3] = GET_WPTR(port->rxq_rwptr);
1615	netdev_err(netdev, "RX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
1616		   reg[0], reg[1], reg[2], reg[3]);
1617
1618	reg[0] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD0_REG);
1619	reg[1] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD1_REG);
1620	reg[2] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD2_REG);
1621	reg[3] = readl(port->dma_base + GMAC_DMA_RX_DESC_WORD3_REG);
1622	netdev_err(netdev, "RX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1623		   reg[0], reg[1], reg[2], reg[3]);
1624
1625	/* TX DMA status */
1626	ptr_reg = port->dma_base + GMAC_SW_TX_QUEUE0_PTR_REG;
1627
1628	reg[0] = readl(port->dma_base + GMAC_DMA_TX_FIRST_DESC_REG);
1629	reg[1] = readl(port->dma_base + GMAC_DMA_TX_CURR_DESC_REG);
1630	reg[2] = GET_RPTR(ptr_reg);
1631	reg[3] = GET_WPTR(ptr_reg);
1632	netdev_err(netdev, "TX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
1633		   reg[0], reg[1], reg[2], reg[3]);
1634
1635	reg[0] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD0_REG);
1636	reg[1] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD1_REG);
1637	reg[2] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD2_REG);
1638	reg[3] = readl(port->dma_base + GMAC_DMA_TX_DESC_WORD3_REG);
1639	netdev_err(netdev, "TX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1640		   reg[0], reg[1], reg[2], reg[3]);
1641
1642	/* FREE queues status */
1643	ptr_reg = geth->base + GLOBAL_SWFQ_RWPTR_REG;
1644
1645	reg[0] = GET_RPTR(ptr_reg);
1646	reg[1] = GET_WPTR(ptr_reg);
1647
1648	ptr_reg = geth->base + GLOBAL_HWFQ_RWPTR_REG;
1649
1650	reg[2] = GET_RPTR(ptr_reg);
1651	reg[3] = GET_WPTR(ptr_reg);
1652	netdev_err(netdev, "FQ SW ptr: %u %u, HW ptr: %u %u\n",
1653		   reg[0], reg[1], reg[2], reg[3]);
1654}
1655
1656static void gmac_update_hw_stats(struct net_device *netdev)
1657{
1658	struct gemini_ethernet_port *port = netdev_priv(netdev);
1659	unsigned int rx_discards, rx_mcast, rx_bcast;
1660	struct gemini_ethernet *geth = port->geth;
1661	unsigned long flags;
1662
1663	spin_lock_irqsave(&geth->irq_lock, flags);
1664	u64_stats_update_begin(&port->ir_stats_syncp);
1665
1666	rx_discards = readl(port->gmac_base + GMAC_IN_DISCARDS);
1667	port->hw_stats[0] += rx_discards;
1668	port->hw_stats[1] += readl(port->gmac_base + GMAC_IN_ERRORS);
1669	rx_mcast = readl(port->gmac_base + GMAC_IN_MCAST);
1670	port->hw_stats[2] += rx_mcast;
1671	rx_bcast = readl(port->gmac_base + GMAC_IN_BCAST);
1672	port->hw_stats[3] += rx_bcast;
1673	port->hw_stats[4] += readl(port->gmac_base + GMAC_IN_MAC1);
1674	port->hw_stats[5] += readl(port->gmac_base + GMAC_IN_MAC2);
1675
1676	port->stats.rx_missed_errors += rx_discards;
1677	port->stats.multicast += rx_mcast;
1678	port->stats.multicast += rx_bcast;
1679
1680	writel(GMAC0_MIB_INT_BIT << (netdev->dev_id * 8),
1681	       geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
1682
1683	u64_stats_update_end(&port->ir_stats_syncp);
1684	spin_unlock_irqrestore(&geth->irq_lock, flags);
1685}
1686
1687/**
1688 * gmac_get_intr_flags() - get interrupt status flags for a port from
1689 * @netdev: the net device for the port to get flags from
1690 * @i: the interrupt status register 0..4
1691 */
1692static u32 gmac_get_intr_flags(struct net_device *netdev, int i)
1693{
1694	struct gemini_ethernet_port *port = netdev_priv(netdev);
1695	struct gemini_ethernet *geth = port->geth;
1696	void __iomem *irqif_reg, *irqen_reg;
1697	unsigned int offs, val;
1698
1699	/* Calculate the offset using the stride of the status registers */
1700	offs = i * (GLOBAL_INTERRUPT_STATUS_1_REG -
1701		    GLOBAL_INTERRUPT_STATUS_0_REG);
1702
1703	irqif_reg = geth->base + GLOBAL_INTERRUPT_STATUS_0_REG + offs;
1704	irqen_reg = geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG + offs;
1705
1706	val = readl(irqif_reg) & readl(irqen_reg);
1707	return val;
1708}
1709
1710static enum hrtimer_restart gmac_coalesce_delay_expired(struct hrtimer *timer)
1711{
1712	struct gemini_ethernet_port *port =
1713		container_of(timer, struct gemini_ethernet_port,
1714			     rx_coalesce_timer);
1715
1716	napi_schedule(&port->napi);
1717	return HRTIMER_NORESTART;
1718}
1719
1720static irqreturn_t gmac_irq(int irq, void *data)
1721{
1722	struct gemini_ethernet_port *port;
1723	struct net_device *netdev = data;
1724	struct gemini_ethernet *geth;
1725	u32 val, orr = 0;
1726
1727	port = netdev_priv(netdev);
1728	geth = port->geth;
1729
1730	val = gmac_get_intr_flags(netdev, 0);
1731	orr |= val;
1732
1733	if (val & (GMAC0_IRQ0_2 << (netdev->dev_id * 2))) {
1734		/* Oh, crap */
1735		netdev_err(netdev, "hw failure/sw bug\n");
1736		gmac_dump_dma_state(netdev);
1737
1738		/* don't know how to recover, just reduce losses */
1739		gmac_enable_irq(netdev, 0);
1740		return IRQ_HANDLED;
1741	}
1742
1743	if (val & (GMAC0_IRQ0_TXQ0_INTS << (netdev->dev_id * 6)))
1744		gmac_tx_irq(netdev, 0);
1745
1746	val = gmac_get_intr_flags(netdev, 1);
1747	orr |= val;
1748
1749	if (val & (DEFAULT_Q0_INT_BIT << netdev->dev_id)) {
1750		gmac_enable_rx_irq(netdev, 0);
1751
1752		if (!port->rx_coalesce_nsecs) {
1753			napi_schedule(&port->napi);
1754		} else {
1755			ktime_t ktime;
1756
1757			ktime = ktime_set(0, port->rx_coalesce_nsecs);
1758			hrtimer_start(&port->rx_coalesce_timer, ktime,
1759				      HRTIMER_MODE_REL);
1760		}
1761	}
1762
1763	val = gmac_get_intr_flags(netdev, 4);
1764	orr |= val;
1765
1766	if (val & (GMAC0_MIB_INT_BIT << (netdev->dev_id * 8)))
1767		gmac_update_hw_stats(netdev);
1768
1769	if (val & (GMAC0_RX_OVERRUN_INT_BIT << (netdev->dev_id * 8))) {
1770		spin_lock(&geth->irq_lock);
1771		writel(GMAC0_RXDERR_INT_BIT << (netdev->dev_id * 8),
1772		       geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
1773		u64_stats_update_begin(&port->ir_stats_syncp);
1774		++port->stats.rx_fifo_errors;
1775		u64_stats_update_end(&port->ir_stats_syncp);
1776		spin_unlock(&geth->irq_lock);
1777	}
1778
1779	return orr ? IRQ_HANDLED : IRQ_NONE;
1780}
1781
1782static void gmac_start_dma(struct gemini_ethernet_port *port)
1783{
1784	void __iomem *dma_ctrl_reg = port->dma_base + GMAC_DMA_CTRL_REG;
1785	union gmac_dma_ctrl dma_ctrl;
1786
1787	dma_ctrl.bits32 = readl(dma_ctrl_reg);
1788	dma_ctrl.bits.rd_enable = 1;
1789	dma_ctrl.bits.td_enable = 1;
1790	dma_ctrl.bits.loopback = 0;
1791	dma_ctrl.bits.drop_small_ack = 0;
1792	dma_ctrl.bits.rd_insert_bytes = NET_IP_ALIGN;
1793	dma_ctrl.bits.rd_prot = HPROT_DATA_CACHE | HPROT_PRIVILIGED;
1794	dma_ctrl.bits.rd_burst_size = HBURST_INCR8;
1795	dma_ctrl.bits.rd_bus = HSIZE_8;
1796	dma_ctrl.bits.td_prot = HPROT_DATA_CACHE;
1797	dma_ctrl.bits.td_burst_size = HBURST_INCR8;
1798	dma_ctrl.bits.td_bus = HSIZE_8;
1799
1800	writel(dma_ctrl.bits32, dma_ctrl_reg);
1801}
1802
1803static void gmac_stop_dma(struct gemini_ethernet_port *port)
1804{
1805	void __iomem *dma_ctrl_reg = port->dma_base + GMAC_DMA_CTRL_REG;
1806	union gmac_dma_ctrl dma_ctrl;
1807
1808	dma_ctrl.bits32 = readl(dma_ctrl_reg);
1809	dma_ctrl.bits.rd_enable = 0;
1810	dma_ctrl.bits.td_enable = 0;
1811	writel(dma_ctrl.bits32, dma_ctrl_reg);
1812}
1813
1814static int gmac_open(struct net_device *netdev)
1815{
1816	struct gemini_ethernet_port *port = netdev_priv(netdev);
1817	int err;
1818
1819	err = request_irq(netdev->irq, gmac_irq,
1820			  IRQF_SHARED, netdev->name, netdev);
1821	if (err) {
1822		netdev_err(netdev, "no IRQ\n");
1823		return err;
1824	}
1825
1826	netif_carrier_off(netdev);
1827	phy_start(netdev->phydev);
1828
1829	err = geth_resize_freeq(port);
1830	/* It's fine if it's just busy, the other port has set up
1831	 * the freeq in that case.
1832	 */
1833	if (err && (err != -EBUSY)) {
1834		netdev_err(netdev, "could not resize freeq\n");
1835		goto err_stop_phy;
1836	}
1837
1838	err = gmac_setup_rxq(netdev);
1839	if (err) {
1840		netdev_err(netdev, "could not setup RXQ\n");
1841		goto err_stop_phy;
1842	}
1843
1844	err = gmac_setup_txqs(netdev);
1845	if (err) {
1846		netdev_err(netdev, "could not setup TXQs\n");
1847		gmac_cleanup_rxq(netdev);
1848		goto err_stop_phy;
1849	}
1850
1851	napi_enable(&port->napi);
1852
1853	gmac_start_dma(port);
1854	gmac_enable_irq(netdev, 1);
1855	gmac_enable_tx_rx(netdev);
1856	netif_tx_start_all_queues(netdev);
1857
1858	hrtimer_setup(&port->rx_coalesce_timer, &gmac_coalesce_delay_expired, CLOCK_MONOTONIC,
1859		      HRTIMER_MODE_REL);
1860
1861	netdev_dbg(netdev, "opened\n");
1862
1863	return 0;
1864
1865err_stop_phy:
1866	phy_stop(netdev->phydev);
1867	free_irq(netdev->irq, netdev);
1868	return err;
1869}
1870
1871static int gmac_stop(struct net_device *netdev)
1872{
1873	struct gemini_ethernet_port *port = netdev_priv(netdev);
1874
1875	hrtimer_cancel(&port->rx_coalesce_timer);
1876	netif_tx_stop_all_queues(netdev);
1877	gmac_disable_tx_rx(netdev);
1878	gmac_stop_dma(port);
1879	napi_disable(&port->napi);
1880
1881	gmac_enable_irq(netdev, 0);
1882	gmac_cleanup_rxq(netdev);
1883	gmac_cleanup_txqs(netdev);
1884
1885	phy_stop(netdev->phydev);
1886	free_irq(netdev->irq, netdev);
1887
1888	gmac_update_hw_stats(netdev);
1889	return 0;
1890}
1891
1892static void gmac_set_rx_mode(struct net_device *netdev)
1893{
1894	struct gemini_ethernet_port *port = netdev_priv(netdev);
1895	union gmac_rx_fltr filter = { .bits = {
1896		.broadcast = 1,
1897		.multicast = 1,
1898		.unicast = 1,
1899	} };
1900	struct netdev_hw_addr *ha;
1901	unsigned int bit_nr;
1902	u32 mc_filter[2];
1903
1904	mc_filter[1] = 0;
1905	mc_filter[0] = 0;
1906
1907	if (netdev->flags & IFF_PROMISC) {
1908		filter.bits.error = 1;
1909		filter.bits.promiscuous = 1;
1910		mc_filter[1] = ~0;
1911		mc_filter[0] = ~0;
1912	} else if (netdev->flags & IFF_ALLMULTI) {
1913		mc_filter[1] = ~0;
1914		mc_filter[0] = ~0;
1915	} else {
1916		netdev_for_each_mc_addr(ha, netdev) {
1917			bit_nr = ~crc32_le(~0, ha->addr, ETH_ALEN) & 0x3f;
1918			mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 0x1f);
1919		}
1920	}
1921
1922	writel(mc_filter[0], port->gmac_base + GMAC_MCAST_FIL0);
1923	writel(mc_filter[1], port->gmac_base + GMAC_MCAST_FIL1);
1924	writel(filter.bits32, port->gmac_base + GMAC_RX_FLTR);
1925}
1926
1927static void gmac_write_mac_address(struct net_device *netdev)
1928{
1929	struct gemini_ethernet_port *port = netdev_priv(netdev);
1930	__le32 addr[3];
1931
1932	memset(addr, 0, sizeof(addr));
1933	memcpy(addr, netdev->dev_addr, ETH_ALEN);
1934
1935	writel(le32_to_cpu(addr[0]), port->gmac_base + GMAC_STA_ADD0);
1936	writel(le32_to_cpu(addr[1]), port->gmac_base + GMAC_STA_ADD1);
1937	writel(le32_to_cpu(addr[2]), port->gmac_base + GMAC_STA_ADD2);
1938}
1939
1940static int gmac_set_mac_address(struct net_device *netdev, void *addr)
1941{
1942	struct sockaddr *sa = addr;
1943
1944	eth_hw_addr_set(netdev, sa->sa_data);
1945	gmac_write_mac_address(netdev);
1946
1947	return 0;
1948}
1949
1950static void gmac_clear_hw_stats(struct net_device *netdev)
1951{
1952	struct gemini_ethernet_port *port = netdev_priv(netdev);
1953
1954	readl(port->gmac_base + GMAC_IN_DISCARDS);
1955	readl(port->gmac_base + GMAC_IN_ERRORS);
1956	readl(port->gmac_base + GMAC_IN_MCAST);
1957	readl(port->gmac_base + GMAC_IN_BCAST);
1958	readl(port->gmac_base + GMAC_IN_MAC1);
1959	readl(port->gmac_base + GMAC_IN_MAC2);
1960}
1961
1962static void gmac_get_stats64(struct net_device *netdev,
1963			     struct rtnl_link_stats64 *stats)
1964{
1965	struct gemini_ethernet_port *port = netdev_priv(netdev);
1966	unsigned int start;
1967
1968	gmac_update_hw_stats(netdev);
1969
1970	/* Racing with RX NAPI */
1971	do {
1972		start = u64_stats_fetch_begin(&port->rx_stats_syncp);
1973
1974		stats->rx_packets = port->stats.rx_packets;
1975		stats->rx_bytes = port->stats.rx_bytes;
1976		stats->rx_errors = port->stats.rx_errors;
1977		stats->rx_dropped = port->stats.rx_dropped;
1978
1979		stats->rx_length_errors = port->stats.rx_length_errors;
1980		stats->rx_over_errors = port->stats.rx_over_errors;
1981		stats->rx_crc_errors = port->stats.rx_crc_errors;
1982		stats->rx_frame_errors = port->stats.rx_frame_errors;
1983
1984	} while (u64_stats_fetch_retry(&port->rx_stats_syncp, start));
1985
1986	/* Racing with MIB and TX completion interrupts */
1987	do {
1988		start = u64_stats_fetch_begin(&port->ir_stats_syncp);
1989
1990		stats->tx_errors = port->stats.tx_errors;
1991		stats->tx_packets = port->stats.tx_packets;
1992		stats->tx_bytes = port->stats.tx_bytes;
1993
1994		stats->multicast = port->stats.multicast;
1995		stats->rx_missed_errors = port->stats.rx_missed_errors;
1996		stats->rx_fifo_errors = port->stats.rx_fifo_errors;
1997
1998	} while (u64_stats_fetch_retry(&port->ir_stats_syncp, start));
1999
2000	/* Racing with hard_start_xmit */
2001	do {
2002		start = u64_stats_fetch_begin(&port->tx_stats_syncp);
2003
2004		stats->tx_dropped = port->stats.tx_dropped;
2005
2006	} while (u64_stats_fetch_retry(&port->tx_stats_syncp, start));
2007
2008	stats->rx_dropped += stats->rx_missed_errors;
2009}
2010
2011static int gmac_change_mtu(struct net_device *netdev, int new_mtu)
2012{
2013	int max_len = gmac_pick_rx_max_len(new_mtu);
2014
2015	if (max_len < 0)
2016		return -EINVAL;
2017
2018	gmac_disable_tx_rx(netdev);
2019
2020	WRITE_ONCE(netdev->mtu, new_mtu);
2021	gmac_update_config0_reg(netdev, max_len << CONFIG0_MAXLEN_SHIFT,
2022				CONFIG0_MAXLEN_MASK);
2023
2024	netdev_update_features(netdev);
2025
2026	gmac_enable_tx_rx(netdev);
2027
2028	return 0;
2029}
2030
2031static int gmac_set_features(struct net_device *netdev,
2032			     netdev_features_t features)
2033{
2034	struct gemini_ethernet_port *port = netdev_priv(netdev);
2035	int enable = features & NETIF_F_RXCSUM;
2036	unsigned long flags;
2037	u32 reg;
2038
2039	spin_lock_irqsave(&port->config_lock, flags);
2040
2041	reg = readl(port->gmac_base + GMAC_CONFIG0);
2042	reg = enable ? reg | CONFIG0_RX_CHKSUM : reg & ~CONFIG0_RX_CHKSUM;
2043	writel(reg, port->gmac_base + GMAC_CONFIG0);
2044
2045	spin_unlock_irqrestore(&port->config_lock, flags);
2046	return 0;
2047}
2048
2049static int gmac_get_sset_count(struct net_device *netdev, int sset)
2050{
2051	return sset == ETH_SS_STATS ? GMAC_STATS_NUM : 0;
2052}
2053
2054static void gmac_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
2055{
2056	if (stringset != ETH_SS_STATS)
2057		return;
2058
2059	memcpy(data, gmac_stats_strings, sizeof(gmac_stats_strings));
2060}
2061
2062static void gmac_get_ethtool_stats(struct net_device *netdev,
2063				   struct ethtool_stats *estats, u64 *values)
2064{
2065	struct gemini_ethernet_port *port = netdev_priv(netdev);
2066	unsigned int start;
2067	u64 *p;
2068	int i;
2069
2070	gmac_update_hw_stats(netdev);
2071
2072	/* Racing with MIB interrupt */
2073	do {
2074		p = values;
2075		start = u64_stats_fetch_begin(&port->ir_stats_syncp);
2076
2077		for (i = 0; i < RX_STATS_NUM; i++)
2078			*p++ = port->hw_stats[i];
2079
2080	} while (u64_stats_fetch_retry(&port->ir_stats_syncp, start));
2081	values = p;
2082
2083	/* Racing with RX NAPI */
2084	do {
2085		p = values;
2086		start = u64_stats_fetch_begin(&port->rx_stats_syncp);
2087
2088		for (i = 0; i < RX_STATUS_NUM; i++)
2089			*p++ = port->rx_stats[i];
2090		for (i = 0; i < RX_CHKSUM_NUM; i++)
2091			*p++ = port->rx_csum_stats[i];
2092		*p++ = port->rx_napi_exits;
2093
2094	} while (u64_stats_fetch_retry(&port->rx_stats_syncp, start));
2095	values = p;
2096
2097	/* Racing with TX start_xmit */
2098	do {
2099		p = values;
2100		start = u64_stats_fetch_begin(&port->tx_stats_syncp);
2101
2102		for (i = 0; i < TX_MAX_FRAGS; i++) {
2103			*values++ = port->tx_frag_stats[i];
2104			port->tx_frag_stats[i] = 0;
2105		}
2106		*values++ = port->tx_frags_linearized;
2107		*values++ = port->tx_hw_csummed;
2108
2109	} while (u64_stats_fetch_retry(&port->tx_stats_syncp, start));
2110}
2111
2112static int gmac_get_ksettings(struct net_device *netdev,
2113			      struct ethtool_link_ksettings *cmd)
2114{
2115	if (!netdev->phydev)
2116		return -ENXIO;
2117	phy_ethtool_ksettings_get(netdev->phydev, cmd);
2118
2119	return 0;
2120}
2121
2122static int gmac_set_ksettings(struct net_device *netdev,
2123			      const struct ethtool_link_ksettings *cmd)
2124{
2125	if (!netdev->phydev)
2126		return -ENXIO;
2127	return phy_ethtool_ksettings_set(netdev->phydev, cmd);
2128}
2129
2130static int gmac_nway_reset(struct net_device *netdev)
2131{
2132	if (!netdev->phydev)
2133		return -ENXIO;
2134	return phy_start_aneg(netdev->phydev);
2135}
2136
2137static void gmac_get_pauseparam(struct net_device *netdev,
2138				struct ethtool_pauseparam *pparam)
2139{
2140	struct gemini_ethernet_port *port = netdev_priv(netdev);
2141	union gmac_config0 config0;
2142
2143	config0.bits32 = readl(port->gmac_base + GMAC_CONFIG0);
2144
2145	pparam->rx_pause = config0.bits.rx_fc_en;
2146	pparam->tx_pause = config0.bits.tx_fc_en;
2147	pparam->autoneg = true;
2148}
2149
2150static int gmac_set_pauseparam(struct net_device *netdev,
2151			       struct ethtool_pauseparam *pparam)
2152{
2153	struct phy_device *phydev = netdev->phydev;
2154
2155	if (!pparam->autoneg)
2156		return -EOPNOTSUPP;
2157
2158	phy_set_asym_pause(phydev, pparam->rx_pause, pparam->tx_pause);
2159
2160	return 0;
2161}
2162
2163static void gmac_get_ringparam(struct net_device *netdev,
2164			       struct ethtool_ringparam *rp,
2165			       struct kernel_ethtool_ringparam *kernel_rp,
2166			       struct netlink_ext_ack *extack)
2167{
2168	struct gemini_ethernet_port *port = netdev_priv(netdev);
2169
2170	readl(port->gmac_base + GMAC_CONFIG0);
2171
2172	rp->rx_max_pending = 1 << 15;
2173	rp->rx_mini_max_pending = 0;
2174	rp->rx_jumbo_max_pending = 0;
2175	rp->tx_max_pending = 1 << 15;
2176
2177	rp->rx_pending = 1 << port->rxq_order;
2178	rp->rx_mini_pending = 0;
2179	rp->rx_jumbo_pending = 0;
2180	rp->tx_pending = 1 << port->txq_order;
2181}
2182
2183static int gmac_set_ringparam(struct net_device *netdev,
2184			      struct ethtool_ringparam *rp,
2185			      struct kernel_ethtool_ringparam *kernel_rp,
2186			      struct netlink_ext_ack *extack)
2187{
2188	struct gemini_ethernet_port *port = netdev_priv(netdev);
2189	int err = 0;
2190
2191	if (netif_running(netdev))
2192		return -EBUSY;
2193
2194	if (rp->rx_pending) {
2195		port->rxq_order = min(15, ilog2(rp->rx_pending - 1) + 1);
2196		err = geth_resize_freeq(port);
2197	}
2198	if (rp->tx_pending) {
2199		port->txq_order = min(15, ilog2(rp->tx_pending - 1) + 1);
2200		port->irq_every_tx_packets = 1 << (port->txq_order - 2);
2201	}
2202
2203	return err;
2204}
2205
2206static int gmac_get_coalesce(struct net_device *netdev,
2207			     struct ethtool_coalesce *ecmd,
2208			     struct kernel_ethtool_coalesce *kernel_coal,
2209			     struct netlink_ext_ack *extack)
2210{
2211	struct gemini_ethernet_port *port = netdev_priv(netdev);
2212
2213	ecmd->rx_max_coalesced_frames = 1;
2214	ecmd->tx_max_coalesced_frames = port->irq_every_tx_packets;
2215	ecmd->rx_coalesce_usecs = port->rx_coalesce_nsecs / 1000;
2216
2217	return 0;
2218}
2219
2220static int gmac_set_coalesce(struct net_device *netdev,
2221			     struct ethtool_coalesce *ecmd,
2222			     struct kernel_ethtool_coalesce *kernel_coal,
2223			     struct netlink_ext_ack *extack)
2224{
2225	struct gemini_ethernet_port *port = netdev_priv(netdev);
2226
2227	if (ecmd->tx_max_coalesced_frames < 1)
2228		return -EINVAL;
2229	if (ecmd->tx_max_coalesced_frames >= 1 << port->txq_order)
2230		return -EINVAL;
2231
2232	port->irq_every_tx_packets = ecmd->tx_max_coalesced_frames;
2233	port->rx_coalesce_nsecs = ecmd->rx_coalesce_usecs * 1000;
2234
2235	return 0;
2236}
2237
2238static u32 gmac_get_msglevel(struct net_device *netdev)
2239{
2240	struct gemini_ethernet_port *port = netdev_priv(netdev);
2241
2242	return port->msg_enable;
2243}
2244
2245static void gmac_set_msglevel(struct net_device *netdev, u32 level)
2246{
2247	struct gemini_ethernet_port *port = netdev_priv(netdev);
2248
2249	port->msg_enable = level;
2250}
2251
2252static void gmac_get_drvinfo(struct net_device *netdev,
2253			     struct ethtool_drvinfo *info)
2254{
2255	strcpy(info->driver,  DRV_NAME);
2256	strcpy(info->bus_info, netdev->dev_id ? "1" : "0");
2257}
2258
2259static const struct net_device_ops gmac_351x_ops = {
2260	.ndo_init		= gmac_init,
2261	.ndo_open		= gmac_open,
2262	.ndo_stop		= gmac_stop,
2263	.ndo_start_xmit		= gmac_start_xmit,
2264	.ndo_tx_timeout		= gmac_tx_timeout,
2265	.ndo_set_rx_mode	= gmac_set_rx_mode,
2266	.ndo_set_mac_address	= gmac_set_mac_address,
2267	.ndo_get_stats64	= gmac_get_stats64,
2268	.ndo_change_mtu		= gmac_change_mtu,
2269	.ndo_set_features	= gmac_set_features,
2270};
2271
2272static const struct ethtool_ops gmac_351x_ethtool_ops = {
2273	.supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
2274				     ETHTOOL_COALESCE_MAX_FRAMES,
2275	.get_sset_count	= gmac_get_sset_count,
2276	.get_strings	= gmac_get_strings,
2277	.get_ethtool_stats = gmac_get_ethtool_stats,
2278	.get_link	= ethtool_op_get_link,
2279	.get_link_ksettings = gmac_get_ksettings,
2280	.set_link_ksettings = gmac_set_ksettings,
2281	.nway_reset	= gmac_nway_reset,
2282	.get_pauseparam	= gmac_get_pauseparam,
2283	.set_pauseparam = gmac_set_pauseparam,
2284	.get_ringparam	= gmac_get_ringparam,
2285	.set_ringparam	= gmac_set_ringparam,
2286	.get_coalesce	= gmac_get_coalesce,
2287	.set_coalesce	= gmac_set_coalesce,
2288	.get_msglevel	= gmac_get_msglevel,
2289	.set_msglevel	= gmac_set_msglevel,
2290	.get_drvinfo	= gmac_get_drvinfo,
2291};
2292
2293static irqreturn_t gemini_port_irq_thread(int irq, void *data)
2294{
2295	unsigned long irqmask = SWFQ_EMPTY_INT_BIT;
2296	struct gemini_ethernet_port *port = data;
2297	struct gemini_ethernet *geth;
2298	unsigned long flags;
2299
2300	geth = port->geth;
2301	/* The queue is half empty so refill it */
2302	geth_fill_freeq(geth, true);
2303
2304	spin_lock_irqsave(&geth->irq_lock, flags);
2305	/* ACK queue interrupt */
2306	writel(irqmask, geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2307	/* Enable queue interrupt again */
2308	irqmask |= readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2309	writel(irqmask, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2310	spin_unlock_irqrestore(&geth->irq_lock, flags);
2311
2312	return IRQ_HANDLED;
2313}
2314
2315static irqreturn_t gemini_port_irq(int irq, void *data)
2316{
2317	struct gemini_ethernet_port *port = data;
2318	struct gemini_ethernet *geth;
2319	irqreturn_t ret = IRQ_NONE;
2320	u32 val, en;
2321
2322	geth = port->geth;
2323	spin_lock(&geth->irq_lock);
2324
2325	val = readl(geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2326	en = readl(geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2327
2328	if (val & en & SWFQ_EMPTY_INT_BIT) {
2329		/* Disable the queue empty interrupt while we work on
2330		 * processing the queue. Also disable overrun interrupts
2331		 * as there is not much we can do about it here.
2332		 */
2333		en &= ~(SWFQ_EMPTY_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT
2334					   | GMAC1_RX_OVERRUN_INT_BIT);
2335		writel(en, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2336		ret = IRQ_WAKE_THREAD;
2337	}
2338
2339	spin_unlock(&geth->irq_lock);
2340
2341	return ret;
2342}
2343
2344static void gemini_port_remove(struct gemini_ethernet_port *port)
2345{
2346	if (port->netdev) {
2347		phy_disconnect(port->netdev->phydev);
2348		unregister_netdev(port->netdev);
2349	}
2350	clk_disable_unprepare(port->pclk);
2351	geth_cleanup_freeq(port->geth);
2352}
2353
2354static void gemini_ethernet_init(struct gemini_ethernet *geth)
2355{
2356	/* Only do this once both ports are online */
2357	if (geth->initialized)
2358		return;
2359	if (geth->port0 && geth->port1)
2360		geth->initialized = true;
2361	else
2362		return;
2363
2364	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_0_REG);
2365	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_1_REG);
2366	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_2_REG);
2367	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_3_REG);
2368	writel(0, geth->base + GLOBAL_INTERRUPT_ENABLE_4_REG);
2369
2370	/* Interrupt config:
2371	 *
2372	 *	GMAC0 intr bits ------> int0 ----> eth0
2373	 *	GMAC1 intr bits ------> int1 ----> eth1
2374	 *	TOE intr -------------> int1 ----> eth1
2375	 *	Classification Intr --> int0 ----> eth0
2376	 *	Default Q0 -----------> int0 ----> eth0
2377	 *	Default Q1 -----------> int1 ----> eth1
2378	 *	FreeQ intr -----------> int1 ----> eth1
2379	 */
2380	writel(0xCCFC0FC0, geth->base + GLOBAL_INTERRUPT_SELECT_0_REG);
2381	writel(0x00F00002, geth->base + GLOBAL_INTERRUPT_SELECT_1_REG);
2382	writel(0xFFFFFFFF, geth->base + GLOBAL_INTERRUPT_SELECT_2_REG);
2383	writel(0xFFFFFFFF, geth->base + GLOBAL_INTERRUPT_SELECT_3_REG);
2384	writel(0xFF000003, geth->base + GLOBAL_INTERRUPT_SELECT_4_REG);
2385
2386	/* edge-triggered interrupts packed to level-triggered one... */
2387	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_0_REG);
2388	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_1_REG);
2389	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_2_REG);
2390	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_3_REG);
2391	writel(~0, geth->base + GLOBAL_INTERRUPT_STATUS_4_REG);
2392
2393	/* Set up queue */
2394	writel(0, geth->base + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
2395	writel(0, geth->base + GLOBAL_HW_FREEQ_BASE_SIZE_REG);
2396	writel(0, geth->base + GLOBAL_SWFQ_RWPTR_REG);
2397	writel(0, geth->base + GLOBAL_HWFQ_RWPTR_REG);
2398
2399	geth->freeq_frag_order = DEFAULT_RX_BUF_ORDER;
2400	/* This makes the queue resize on probe() so that we
2401	 * set up and enable the queue IRQ. FIXME: fragile.
2402	 */
2403	geth->freeq_order = 1;
2404}
2405
2406static void gemini_port_save_mac_addr(struct gemini_ethernet_port *port)
2407{
2408	port->mac_addr[0] =
2409		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD0));
2410	port->mac_addr[1] =
2411		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD1));
2412	port->mac_addr[2] =
2413		cpu_to_le32(readl(port->gmac_base + GMAC_STA_ADD2));
2414}
2415
2416static int gemini_ethernet_port_probe(struct platform_device *pdev)
2417{
2418	char *port_names[2] = { "ethernet0", "ethernet1" };
2419	struct device_node *np = pdev->dev.of_node;
2420	struct gemini_ethernet_port *port;
2421	struct device *dev = &pdev->dev;
2422	struct gemini_ethernet *geth;
2423	struct net_device *netdev;
2424	struct device *parent;
2425	u8 mac[ETH_ALEN];
2426	unsigned int id;
2427	int irq;
2428	int ret;
2429
2430	parent = dev->parent;
2431	geth = dev_get_drvdata(parent);
2432
2433	if (!strcmp(dev_name(dev), "60008000.ethernet-port"))
2434		id = 0;
2435	else if (!strcmp(dev_name(dev), "6000c000.ethernet-port"))
2436		id = 1;
2437	else
2438		return -ENODEV;
2439
2440	dev_info(dev, "probe %s ID %d\n", dev_name(dev), id);
2441
2442	netdev = devm_alloc_etherdev_mqs(dev, sizeof(*port), TX_QUEUE_NUM, TX_QUEUE_NUM);
2443	if (!netdev) {
2444		dev_err(dev, "Can't allocate ethernet device #%d\n", id);
2445		return -ENOMEM;
2446	}
2447
2448	port = netdev_priv(netdev);
2449	SET_NETDEV_DEV(netdev, dev);
2450	port->netdev = netdev;
2451	port->id = id;
2452	port->geth = geth;
2453	port->dev = dev;
2454	port->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2455
2456	/* DMA memory */
2457	port->dma_base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
2458	if (IS_ERR(port->dma_base)) {
2459		dev_err(dev, "get DMA address failed\n");
2460		return PTR_ERR(port->dma_base);
2461	}
2462
2463	/* GMAC config memory */
2464	port->gmac_base = devm_platform_get_and_ioremap_resource(pdev, 1, NULL);
2465	if (IS_ERR(port->gmac_base)) {
2466		dev_err(dev, "get GMAC address failed\n");
2467		return PTR_ERR(port->gmac_base);
2468	}
2469
2470	/* Interrupt */
2471	irq = platform_get_irq(pdev, 0);
2472	if (irq < 0)
2473		return irq;
2474	port->irq = irq;
2475
2476	/* Clock the port */
2477	port->pclk = devm_clk_get(dev, "PCLK");
2478	if (IS_ERR(port->pclk)) {
2479		dev_err(dev, "no PCLK\n");
2480		return PTR_ERR(port->pclk);
2481	}
2482	ret = clk_prepare_enable(port->pclk);
2483	if (ret)
2484		return ret;
2485
2486	/* Maybe there is a nice ethernet address we should use */
2487	gemini_port_save_mac_addr(port);
2488
2489	/* Reset the port */
2490	port->reset = devm_reset_control_get_exclusive(dev, NULL);
2491	if (IS_ERR(port->reset)) {
2492		dev_err(dev, "no reset\n");
2493		ret = PTR_ERR(port->reset);
2494		goto unprepare;
2495	}
2496	reset_control_reset(port->reset);
2497	usleep_range(100, 500);
2498
2499	/* Assign pointer in the main state container */
2500	if (!id)
2501		geth->port0 = port;
2502	else
2503		geth->port1 = port;
2504
2505	/* This will just be done once both ports are up and reset */
2506	gemini_ethernet_init(geth);
2507
2508	platform_set_drvdata(pdev, port);
2509
2510	/* Set up and register the netdev */
2511	netdev->dev_id = port->id;
2512	netdev->irq = irq;
2513	netdev->netdev_ops = &gmac_351x_ops;
2514	netdev->ethtool_ops = &gmac_351x_ethtool_ops;
2515
2516	spin_lock_init(&port->config_lock);
2517	gmac_clear_hw_stats(netdev);
2518
2519	netdev->hw_features = GMAC_OFFLOAD_FEATURES;
2520	netdev->features |= GMAC_OFFLOAD_FEATURES | NETIF_F_GRO;
2521	/* We can receive jumbo frames up to 10236 bytes but only
2522	 * transmit 2047 bytes so, let's accept payloads of 2047
2523	 * bytes minus VLAN and ethernet header
2524	 */
2525	netdev->min_mtu = ETH_MIN_MTU;
2526	netdev->max_mtu = MTU_SIZE_BIT_MASK - VLAN_ETH_HLEN;
2527
2528	port->freeq_refill = 0;
2529	netif_napi_add(netdev, &port->napi, gmac_napi_poll);
2530
2531	ret = of_get_mac_address(np, mac);
2532	if (!ret) {
2533		dev_info(dev, "Setting macaddr from DT %pM\n", mac);
2534		memcpy(port->mac_addr, mac, ETH_ALEN);
2535	}
2536
2537	if (is_valid_ether_addr((void *)port->mac_addr)) {
2538		eth_hw_addr_set(netdev, (u8 *)port->mac_addr);
2539	} else {
2540		dev_dbg(dev, "ethernet address 0x%08x%08x%08x invalid\n",
2541			port->mac_addr[0], port->mac_addr[1],
2542			port->mac_addr[2]);
2543		dev_info(dev, "using a random ethernet address\n");
2544		eth_hw_addr_random(netdev);
2545	}
2546	gmac_write_mac_address(netdev);
2547
2548	ret = devm_request_threaded_irq(port->dev,
2549					port->irq,
2550					gemini_port_irq,
2551					gemini_port_irq_thread,
2552					IRQF_SHARED,
2553					port_names[port->id],
2554					port);
2555	if (ret)
2556		goto unprepare;
2557
2558	ret = gmac_setup_phy(netdev);
2559	if (ret) {
2560		netdev_err(netdev,
2561			   "PHY init failed\n");
2562		goto unprepare;
2563	}
2564
2565	ret = register_netdev(netdev);
2566	if (ret)
2567		goto unprepare;
2568
2569	return 0;
2570
2571unprepare:
2572	clk_disable_unprepare(port->pclk);
2573	return ret;
2574}
2575
2576static void gemini_ethernet_port_remove(struct platform_device *pdev)
2577{
2578	struct gemini_ethernet_port *port = platform_get_drvdata(pdev);
2579
2580	gemini_port_remove(port);
2581}
2582
2583static const struct of_device_id gemini_ethernet_port_of_match[] = {
2584	{
2585		.compatible = "cortina,gemini-ethernet-port",
2586	},
2587	{},
2588};
2589MODULE_DEVICE_TABLE(of, gemini_ethernet_port_of_match);
2590
2591static struct platform_driver gemini_ethernet_port_driver = {
2592	.driver = {
2593		.name = "gemini-ethernet-port",
2594		.of_match_table = gemini_ethernet_port_of_match,
2595	},
2596	.probe = gemini_ethernet_port_probe,
2597	.remove = gemini_ethernet_port_remove,
2598};
2599
2600static int gemini_ethernet_probe(struct platform_device *pdev)
2601{
2602	struct device *dev = &pdev->dev;
2603	struct gemini_ethernet *geth;
2604	unsigned int retry = 5;
2605	u32 val;
2606
2607	/* Global registers */
2608	geth = devm_kzalloc(dev, sizeof(*geth), GFP_KERNEL);
2609	if (!geth)
2610		return -ENOMEM;
2611	geth->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
2612	if (IS_ERR(geth->base))
2613		return PTR_ERR(geth->base);
2614	geth->dev = dev;
2615
2616	/* Wait for ports to stabilize */
2617	do {
2618		udelay(2);
2619		val = readl(geth->base + GLOBAL_TOE_VERSION_REG);
2620		barrier();
2621	} while (!val && --retry);
2622	if (!retry) {
2623		dev_err(dev, "failed to reset ethernet\n");
2624		return -EIO;
2625	}
2626	dev_info(dev, "Ethernet device ID: 0x%03x, revision 0x%01x\n",
2627		 (val >> 4) & 0xFFFU, val & 0xFU);
2628
2629	spin_lock_init(&geth->irq_lock);
2630	spin_lock_init(&geth->freeq_lock);
2631
2632	/* The children will use this */
2633	platform_set_drvdata(pdev, geth);
2634
2635	/* Spawn child devices for the two ports */
2636	return devm_of_platform_populate(dev);
2637}
2638
2639static void gemini_ethernet_remove(struct platform_device *pdev)
2640{
2641	struct gemini_ethernet *geth = platform_get_drvdata(pdev);
2642
2643	geth_cleanup_freeq(geth);
2644	geth->initialized = false;
2645}
2646
2647static const struct of_device_id gemini_ethernet_of_match[] = {
2648	{
2649		.compatible = "cortina,gemini-ethernet",
2650	},
2651	{},
2652};
2653MODULE_DEVICE_TABLE(of, gemini_ethernet_of_match);
2654
2655static struct platform_driver gemini_ethernet_driver = {
2656	.driver = {
2657		.name = DRV_NAME,
2658		.of_match_table = gemini_ethernet_of_match,
2659	},
2660	.probe = gemini_ethernet_probe,
2661	.remove = gemini_ethernet_remove,
2662};
2663
2664static int __init gemini_ethernet_module_init(void)
2665{
2666	int ret;
2667
2668	ret = platform_driver_register(&gemini_ethernet_port_driver);
2669	if (ret)
2670		return ret;
2671
2672	ret = platform_driver_register(&gemini_ethernet_driver);
2673	if (ret) {
2674		platform_driver_unregister(&gemini_ethernet_port_driver);
2675		return ret;
2676	}
2677
2678	return 0;
2679}
2680module_init(gemini_ethernet_module_init);
2681
2682static void __exit gemini_ethernet_module_exit(void)
2683{
2684	platform_driver_unregister(&gemini_ethernet_driver);
2685	platform_driver_unregister(&gemini_ethernet_port_driver);
2686}
2687module_exit(gemini_ethernet_module_exit);
2688
2689MODULE_AUTHOR("Linus Walleij <linus.walleij@linaro.org>");
2690MODULE_DESCRIPTION("StorLink SL351x (Gemini) ethernet driver");
2691MODULE_LICENSE("GPL");
2692MODULE_ALIAS("platform:" DRV_NAME);