dwxgmac2_core.c

   1// SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2/*
   3 * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
   4 * stmmac XGMAC support.
   5 */
   6
   7#include <linux/bitrev.h>
   8#include <linux/crc32.h>
   9#include <linux/iopoll.h>
  10#include "stmmac.h"
  11#include "stmmac_fpe.h"
  12#include "stmmac_ptp.h"
  13#include "stmmac_vlan.h"
  14#include "dwxlgmac2.h"
  15#include "dwxgmac2.h"
  16
  17static void dwxgmac2_core_init(struct mac_device_info *hw,
  18			       struct net_device *dev)
  19{
  20	void __iomem *ioaddr = hw->pcsr;
  21	u32 tx, rx;
  22
  23	tx = readl(ioaddr + XGMAC_TX_CONFIG);
  24	rx = readl(ioaddr + XGMAC_RX_CONFIG);
  25
  26	writel(tx | XGMAC_CORE_INIT_TX, ioaddr + XGMAC_TX_CONFIG);
  27	writel(rx | XGMAC_CORE_INIT_RX, ioaddr + XGMAC_RX_CONFIG);
  28	writel(XGMAC_INT_DEFAULT_EN, ioaddr + XGMAC_INT_EN);
  29}
  30
  31static void dwxgmac2_irq_modify(struct mac_device_info *hw, u32 disable,
  32				u32 enable)
  33{
  34	void __iomem *int_mask = hw->pcsr + XGMAC_INT_EN;
  35	unsigned long flags;
  36	u32 value;
  37
  38	spin_lock_irqsave(&hw->irq_ctrl_lock, flags);
  39	value = readl(int_mask) & ~disable;
  40	value |= enable;
  41	writel(value, int_mask);
  42	spin_unlock_irqrestore(&hw->irq_ctrl_lock, flags);
  43}
  44
  45static void dwxgmac2_update_caps(struct stmmac_priv *priv)
  46{
  47	if (!priv->dma_cap.mbps_10_100)
  48		priv->hw->link.caps &= ~(MAC_10 | MAC_100);
  49	else if (!priv->dma_cap.half_duplex)
  50		priv->hw->link.caps &= ~(MAC_10HD | MAC_100HD);
  51}
  52
  53static void dwxgmac2_set_mac(void __iomem *ioaddr, bool enable)
  54{
  55	u32 tx = readl(ioaddr + XGMAC_TX_CONFIG);
  56	u32 rx = readl(ioaddr + XGMAC_RX_CONFIG);
  57
  58	if (enable) {
  59		tx |= XGMAC_CONFIG_TE;
  60		rx |= XGMAC_CONFIG_RE;
  61	} else {
  62		tx &= ~XGMAC_CONFIG_TE;
  63		rx &= ~XGMAC_CONFIG_RE;
  64	}
  65
  66	writel(tx, ioaddr + XGMAC_TX_CONFIG);
  67	writel(rx, ioaddr + XGMAC_RX_CONFIG);
  68}
  69
  70static int dwxgmac2_rx_ipc(struct mac_device_info *hw)
  71{
  72	void __iomem *ioaddr = hw->pcsr;
  73	u32 value;
  74
  75	value = readl(ioaddr + XGMAC_RX_CONFIG);
  76	if (hw->rx_csum)
  77		value |= XGMAC_CONFIG_IPC;
  78	else
  79		value &= ~XGMAC_CONFIG_IPC;
  80	writel(value, ioaddr + XGMAC_RX_CONFIG);
  81
  82	return !!(readl(ioaddr + XGMAC_RX_CONFIG) & XGMAC_CONFIG_IPC);
  83}
  84
  85static void dwxgmac2_rx_queue_enable(struct mac_device_info *hw, u8 mode,
  86				     u32 queue)
  87{
  88	void __iomem *ioaddr = hw->pcsr;
  89	u32 value;
  90
  91	value = readl(ioaddr + XGMAC_RXQ_CTRL0) & ~XGMAC_RXQEN(queue);
  92	if (mode == MTL_QUEUE_AVB)
  93		value |= 0x1 << XGMAC_RXQEN_SHIFT(queue);
  94	else if (mode == MTL_QUEUE_DCB)
  95		value |= 0x2 << XGMAC_RXQEN_SHIFT(queue);
  96	writel(value, ioaddr + XGMAC_RXQ_CTRL0);
  97}
  98
  99static void dwxgmac2_rx_queue_prio(struct mac_device_info *hw, u32 prio,
 100				   u32 queue)
 101{
 102	void __iomem *ioaddr = hw->pcsr;
 103	u32 clear_mask = 0;
 104	u32 ctrl2, ctrl3;
 105	int i;
 106
 107	ctrl2 = readl(ioaddr + XGMAC_RXQ_CTRL2);
 108	ctrl3 = readl(ioaddr + XGMAC_RXQ_CTRL3);
 109
 110	/* The software must ensure that the same priority
 111	 * is not mapped to multiple Rx queues
 112	 */
 113	for (i = 0; i < 4; i++)
 114		clear_mask |= ((prio << XGMAC_PSRQ_SHIFT(i)) &
 115						XGMAC_PSRQ(i));
 116
 117	ctrl2 &= ~clear_mask;
 118	ctrl3 &= ~clear_mask;
 119
 120	/* First assign new priorities to a queue, then
 121	 * clear them from others queues
 122	 */
 123	if (queue < 4) {
 124		ctrl2 |= (prio << XGMAC_PSRQ_SHIFT(queue)) &
 125						XGMAC_PSRQ(queue);
 126
 127		writel(ctrl2, ioaddr + XGMAC_RXQ_CTRL2);
 128		writel(ctrl3, ioaddr + XGMAC_RXQ_CTRL3);
 129	} else {
 130		queue -= 4;
 131
 132		ctrl3 |= (prio << XGMAC_PSRQ_SHIFT(queue)) &
 133						XGMAC_PSRQ(queue);
 134
 135		writel(ctrl3, ioaddr + XGMAC_RXQ_CTRL3);
 136		writel(ctrl2, ioaddr + XGMAC_RXQ_CTRL2);
 137	}
 138}
 139
 140static void dwxgmac2_tx_queue_prio(struct mac_device_info *hw, u32 prio,
 141				   u32 queue)
 142{
 143	void __iomem *ioaddr = hw->pcsr;
 144	u32 value, reg;
 145
 146	reg = (queue < 4) ? XGMAC_TC_PRTY_MAP0 : XGMAC_TC_PRTY_MAP1;
 147	if (queue >= 4)
 148		queue -= 4;
 149
 150	value = readl(ioaddr + reg);
 151	value &= ~XGMAC_PSTC(queue);
 152	value |= (prio << XGMAC_PSTC_SHIFT(queue)) & XGMAC_PSTC(queue);
 153
 154	writel(value, ioaddr + reg);
 155}
 156
 157static void dwxgmac2_rx_queue_routing(struct mac_device_info *hw,
 158				      u8 packet, u32 queue)
 159{
 160	void __iomem *ioaddr = hw->pcsr;
 161	u32 value;
 162
 163	static const struct stmmac_rx_routing dwxgmac2_route_possibilities[] = {
 164		{ XGMAC_AVCPQ, XGMAC_AVCPQ_SHIFT },
 165		{ XGMAC_PTPQ, XGMAC_PTPQ_SHIFT },
 166		{ XGMAC_DCBCPQ, XGMAC_DCBCPQ_SHIFT },
 167		{ XGMAC_UPQ, XGMAC_UPQ_SHIFT },
 168		{ XGMAC_MCBCQ, XGMAC_MCBCQ_SHIFT },
 169	};
 170
 171	value = readl(ioaddr + XGMAC_RXQ_CTRL1);
 172
 173	/* routing configuration */
 174	value &= ~dwxgmac2_route_possibilities[packet - 1].reg_mask;
 175	value |= (queue << dwxgmac2_route_possibilities[packet - 1].reg_shift) &
 176		 dwxgmac2_route_possibilities[packet - 1].reg_mask;
 177
 178	/* some packets require extra ops */
 179	if (packet == PACKET_AVCPQ)
 180		value |= FIELD_PREP(XGMAC_TACPQE, 1);
 181	else if (packet == PACKET_MCBCQ)
 182		value |= FIELD_PREP(XGMAC_MCBCQEN, 1);
 183
 184	writel(value, ioaddr + XGMAC_RXQ_CTRL1);
 185}
 186
 187static void dwxgmac2_prog_mtl_rx_algorithms(struct mac_device_info *hw,
 188					    u32 rx_alg)
 189{
 190	void __iomem *ioaddr = hw->pcsr;
 191	u32 value;
 192
 193	value = readl(ioaddr + XGMAC_MTL_OPMODE);
 194	value &= ~XGMAC_RAA;
 195
 196	switch (rx_alg) {
 197	case MTL_RX_ALGORITHM_SP:
 198		break;
 199	case MTL_RX_ALGORITHM_WSP:
 200		value |= XGMAC_RAA;
 201		break;
 202	default:
 203		break;
 204	}
 205
 206	writel(value, ioaddr + XGMAC_MTL_OPMODE);
 207}
 208
 209static void dwxgmac2_prog_mtl_tx_algorithms(struct mac_device_info *hw,
 210					    u32 tx_alg)
 211{
 212	void __iomem *ioaddr = hw->pcsr;
 213	bool ets = true;
 214	u32 value;
 215	int i;
 216
 217	value = readl(ioaddr + XGMAC_MTL_OPMODE);
 218	value &= ~XGMAC_ETSALG;
 219
 220	switch (tx_alg) {
 221	case MTL_TX_ALGORITHM_WRR:
 222		value |= XGMAC_WRR;
 223		break;
 224	case MTL_TX_ALGORITHM_WFQ:
 225		value |= XGMAC_WFQ;
 226		break;
 227	case MTL_TX_ALGORITHM_DWRR:
 228		value |= XGMAC_DWRR;
 229		break;
 230	default:
 231		ets = false;
 232		break;
 233	}
 234
 235	writel(value, ioaddr + XGMAC_MTL_OPMODE);
 236
 237	/* Set ETS if desired */
 238	for (i = 0; i < MTL_MAX_TX_QUEUES; i++) {
 239		value = readl(ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(i));
 240		value &= ~XGMAC_TSA;
 241		if (ets)
 242			value |= XGMAC_ETS;
 243		writel(value, ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(i));
 244	}
 245}
 246
 247static void dwxgmac2_set_mtl_tx_queue_weight(struct stmmac_priv *priv,
 248					     struct mac_device_info *hw,
 249					     u32 weight, u32 queue)
 250{
 251	void __iomem *ioaddr = hw->pcsr;
 252
 253	writel(weight, ioaddr + XGMAC_MTL_TCx_QUANTUM_WEIGHT(queue));
 254}
 255
 256static void dwxgmac2_map_mtl_to_dma(struct mac_device_info *hw, u32 queue,
 257				    u32 chan)
 258{
 259	void __iomem *ioaddr = hw->pcsr;
 260	u32 value, reg;
 261
 262	reg = (queue < 4) ? XGMAC_MTL_RXQ_DMA_MAP0 : XGMAC_MTL_RXQ_DMA_MAP1;
 263	if (queue >= 4)
 264		queue -= 4;
 265
 266	value = readl(ioaddr + reg);
 267	value &= ~XGMAC_QxMDMACH(queue);
 268	value |= (chan << XGMAC_QxMDMACH_SHIFT(queue)) & XGMAC_QxMDMACH(queue);
 269
 270	writel(value, ioaddr + reg);
 271}
 272
 273static void dwxgmac2_config_cbs(struct stmmac_priv *priv,
 274				struct mac_device_info *hw,
 275				u32 send_slope, u32 idle_slope,
 276				u32 high_credit, u32 low_credit, u32 queue)
 277{
 278	void __iomem *ioaddr = hw->pcsr;
 279	u32 value;
 280
 281	writel(send_slope, ioaddr + XGMAC_MTL_TCx_SENDSLOPE(queue));
 282	writel(idle_slope, ioaddr + XGMAC_MTL_TCx_QUANTUM_WEIGHT(queue));
 283	writel(high_credit, ioaddr + XGMAC_MTL_TCx_HICREDIT(queue));
 284	writel(low_credit, ioaddr + XGMAC_MTL_TCx_LOCREDIT(queue));
 285
 286	value = readl(ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(queue));
 287	value &= ~XGMAC_TSA;
 288	value |= XGMAC_CC | XGMAC_CBS;
 289	writel(value, ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(queue));
 290}
 291
 292static void dwxgmac2_dump_regs(struct mac_device_info *hw, u32 *reg_space)
 293{
 294	void __iomem *ioaddr = hw->pcsr;
 295	int i;
 296
 297	for (i = 0; i < XGMAC_MAC_REGSIZE; i++)
 298		reg_space[i] = readl(ioaddr + i * 4);
 299}
 300
 301static int dwxgmac2_host_irq_status(struct mac_device_info *hw,
 302				    struct stmmac_extra_stats *x)
 303{
 304	void __iomem *ioaddr = hw->pcsr;
 305	u32 stat, en;
 306	int ret = 0;
 307
 308	en = readl(ioaddr + XGMAC_INT_EN);
 309	stat = readl(ioaddr + XGMAC_INT_STATUS);
 310
 311	stat &= en;
 312
 313	if (stat & XGMAC_PMTIS) {
 314		x->irq_receive_pmt_irq_n++;
 315		readl(ioaddr + XGMAC_PMT);
 316	}
 317
 318	if (stat & XGMAC_LPIIS) {
 319		u32 lpi = readl(ioaddr + XGMAC_LPI_CTRL);
 320
 321		if (lpi & LPI_CTRL_STATUS_TLPIEN) {
 322			ret |= CORE_IRQ_TX_PATH_IN_LPI_MODE;
 323			x->irq_tx_path_in_lpi_mode_n++;
 324		}
 325		if (lpi & LPI_CTRL_STATUS_TLPIEX) {
 326			ret |= CORE_IRQ_TX_PATH_EXIT_LPI_MODE;
 327			x->irq_tx_path_exit_lpi_mode_n++;
 328		}
 329		if (lpi & LPI_CTRL_STATUS_RLPIEN)
 330			x->irq_rx_path_in_lpi_mode_n++;
 331		if (lpi & LPI_CTRL_STATUS_RLPIEX)
 332			x->irq_rx_path_exit_lpi_mode_n++;
 333	}
 334
 335	return ret;
 336}
 337
 338static int dwxgmac2_host_mtl_irq_status(struct stmmac_priv *priv,
 339					struct mac_device_info *hw, u32 chan)
 340{
 341	void __iomem *ioaddr = hw->pcsr;
 342	int ret = 0;
 343	u32 status;
 344
 345	status = readl(ioaddr + XGMAC_MTL_INT_STATUS);
 346	if (status & BIT(chan)) {
 347		u32 chan_status = readl(ioaddr + XGMAC_MTL_QINT_STATUS(chan));
 348
 349		if (chan_status & XGMAC_RXOVFIS)
 350			ret |= CORE_IRQ_MTL_RX_OVERFLOW;
 351
 352		writel(~0x0, ioaddr + XGMAC_MTL_QINT_STATUS(chan));
 353	}
 354
 355	return ret;
 356}
 357
 358static void dwxgmac2_flow_ctrl(struct mac_device_info *hw, unsigned int duplex,
 359			       unsigned int fc, unsigned int pause_time,
 360			       u32 tx_cnt)
 361{
 362	void __iomem *ioaddr = hw->pcsr;
 363	u32 i;
 364
 365	if (fc & FLOW_RX)
 366		writel(XGMAC_RFE, ioaddr + XGMAC_RX_FLOW_CTRL);
 367	if (fc & FLOW_TX) {
 368		for (i = 0; i < tx_cnt; i++) {
 369			u32 value = XGMAC_TFE;
 370
 371			if (duplex)
 372				value |= pause_time << XGMAC_PT_SHIFT;
 373
 374			writel(value, ioaddr + XGMAC_Qx_TX_FLOW_CTRL(i));
 375		}
 376	}
 377}
 378
 379static void dwxgmac2_pmt(struct mac_device_info *hw, unsigned long mode)
 380{
 381	void __iomem *ioaddr = hw->pcsr;
 382	u32 val = 0x0;
 383
 384	if (mode & WAKE_MAGIC)
 385		val |= XGMAC_PWRDWN | XGMAC_MGKPKTEN;
 386	if (mode & WAKE_UCAST)
 387		val |= XGMAC_PWRDWN | XGMAC_GLBLUCAST | XGMAC_RWKPKTEN;
 388	if (val) {
 389		u32 cfg = readl(ioaddr + XGMAC_RX_CONFIG);
 390		cfg |= XGMAC_CONFIG_RE;
 391		writel(cfg, ioaddr + XGMAC_RX_CONFIG);
 392	}
 393
 394	writel(val, ioaddr + XGMAC_PMT);
 395}
 396
 397static void dwxgmac2_set_umac_addr(struct mac_device_info *hw,
 398				   const unsigned char *addr,
 399				   unsigned int reg_n)
 400{
 401	void __iomem *ioaddr = hw->pcsr;
 402	u32 value;
 403
 404	value = (addr[5] << 8) | addr[4];
 405	writel(value | XGMAC_AE, ioaddr + XGMAC_ADDRx_HIGH(reg_n));
 406
 407	value = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0];
 408	writel(value, ioaddr + XGMAC_ADDRx_LOW(reg_n));
 409}
 410
 411static void dwxgmac2_get_umac_addr(struct mac_device_info *hw,
 412				   unsigned char *addr, unsigned int reg_n)
 413{
 414	void __iomem *ioaddr = hw->pcsr;
 415	u32 hi_addr, lo_addr;
 416
 417	/* Read the MAC address from the hardware */
 418	hi_addr = readl(ioaddr + XGMAC_ADDRx_HIGH(reg_n));
 419	lo_addr = readl(ioaddr + XGMAC_ADDRx_LOW(reg_n));
 420
 421	/* Extract the MAC address from the high and low words */
 422	addr[0] = lo_addr & 0xff;
 423	addr[1] = (lo_addr >> 8) & 0xff;
 424	addr[2] = (lo_addr >> 16) & 0xff;
 425	addr[3] = (lo_addr >> 24) & 0xff;
 426	addr[4] = hi_addr & 0xff;
 427	addr[5] = (hi_addr >> 8) & 0xff;
 428}
 429
 430static int dwxgmac2_set_lpi_mode(struct mac_device_info *hw,
 431				 enum stmmac_lpi_mode mode,
 432				 bool en_tx_lpi_clockgating, u32 et)
 433{
 434	void __iomem *ioaddr = hw->pcsr;
 435	u32 value;
 436
 437	if (mode == STMMAC_LPI_TIMER)
 438		return -EOPNOTSUPP;
 439
 440	value = readl(ioaddr + XGMAC_LPI_CTRL);
 441	if (mode == STMMAC_LPI_FORCED) {
 442		value |= LPI_CTRL_STATUS_LPIEN | LPI_CTRL_STATUS_LPITXA;
 443		if (en_tx_lpi_clockgating)
 444			value |= LPI_CTRL_STATUS_LPITCSE;
 445	} else {
 446		value &= ~(LPI_CTRL_STATUS_LPIEN | LPI_CTRL_STATUS_LPITXA |
 447			   LPI_CTRL_STATUS_LPITCSE);
 448	}
 449	writel(value, ioaddr + XGMAC_LPI_CTRL);
 450
 451	return 0;
 452}
 453
 454static void dwxgmac2_set_eee_pls(struct mac_device_info *hw, int link)
 455{
 456	void __iomem *ioaddr = hw->pcsr;
 457	u32 value;
 458
 459	value = readl(ioaddr + XGMAC_LPI_CTRL);
 460	if (link)
 461		value |= LPI_CTRL_STATUS_PLS;
 462	else
 463		value &= ~LPI_CTRL_STATUS_PLS;
 464	writel(value, ioaddr + XGMAC_LPI_CTRL);
 465}
 466
 467static void dwxgmac2_set_eee_timer(struct mac_device_info *hw, int ls, int tw)
 468{
 469	void __iomem *ioaddr = hw->pcsr;
 470	u32 value;
 471
 472	value = (tw & 0xffff) | ((ls & 0x3ff) << 16);
 473	writel(value, ioaddr + XGMAC_LPI_TIMER_CTRL);
 474}
 475
 476static void dwxgmac2_set_mchash(void __iomem *ioaddr, u32 *mcfilterbits,
 477				int mcbitslog2)
 478{
 479	int numhashregs, regs;
 480
 481	switch (mcbitslog2) {
 482	case 6:
 483		numhashregs = 2;
 484		break;
 485	case 7:
 486		numhashregs = 4;
 487		break;
 488	case 8:
 489		numhashregs = 8;
 490		break;
 491	default:
 492		return;
 493	}
 494
 495	for (regs = 0; regs < numhashregs; regs++)
 496		writel(mcfilterbits[regs], ioaddr + XGMAC_HASH_TABLE(regs));
 497}
 498
 499static void dwxgmac2_set_filter(struct mac_device_info *hw,
 500				struct net_device *dev)
 501{
 502	void __iomem *ioaddr = (void __iomem *)dev->base_addr;
 503	u32 value = readl(ioaddr + XGMAC_PACKET_FILTER);
 504	int mcbitslog2 = hw->mcast_bits_log2;
 505	u32 mc_filter[8];
 506	int i;
 507
 508	value &= ~(XGMAC_FILTER_PR | XGMAC_FILTER_HMC | XGMAC_FILTER_PM);
 509	value |= XGMAC_FILTER_HPF;
 510
 511	memset(mc_filter, 0, sizeof(mc_filter));
 512
 513	if (dev->flags & IFF_PROMISC) {
 514		value |= XGMAC_FILTER_PR;
 515		value |= XGMAC_FILTER_PCF;
 516	} else if ((dev->flags & IFF_ALLMULTI) ||
 517		   (netdev_mc_count(dev) > hw->multicast_filter_bins)) {
 518		value |= XGMAC_FILTER_PM;
 519
 520		for (i = 0; i < XGMAC_MAX_HASH_TABLE; i++)
 521			writel(~0x0, ioaddr + XGMAC_HASH_TABLE(i));
 522	} else if (!netdev_mc_empty(dev) && (dev->flags & IFF_MULTICAST)) {
 523		struct netdev_hw_addr *ha;
 524
 525		value |= XGMAC_FILTER_HMC;
 526
 527		netdev_for_each_mc_addr(ha, dev) {
 528			u32 nr = (bitrev32(~crc32_le(~0, ha->addr, 6)) >>
 529					(32 - mcbitslog2));
 530			mc_filter[nr >> 5] |= (1 << (nr & 0x1F));
 531		}
 532	}
 533
 534	dwxgmac2_set_mchash(ioaddr, mc_filter, mcbitslog2);
 535
 536	/* Handle multiple unicast addresses */
 537	if (netdev_uc_count(dev) > hw->unicast_filter_entries) {
 538		value |= XGMAC_FILTER_PR;
 539	} else {
 540		struct netdev_hw_addr *ha;
 541		int reg = 1;
 542
 543		netdev_for_each_uc_addr(ha, dev) {
 544			dwxgmac2_set_umac_addr(hw, ha->addr, reg);
 545			reg++;
 546		}
 547
 548		for ( ; reg < XGMAC_ADDR_MAX; reg++) {
 549			writel(0, ioaddr + XGMAC_ADDRx_HIGH(reg));
 550			writel(0, ioaddr + XGMAC_ADDRx_LOW(reg));
 551		}
 552	}
 553
 554	writel(value, ioaddr + XGMAC_PACKET_FILTER);
 555}
 556
 557static void dwxgmac2_set_mac_loopback(void __iomem *ioaddr, bool enable)
 558{
 559	u32 value = readl(ioaddr + XGMAC_RX_CONFIG);
 560
 561	if (enable)
 562		value |= XGMAC_CONFIG_LM;
 563	else
 564		value &= ~XGMAC_CONFIG_LM;
 565
 566	writel(value, ioaddr + XGMAC_RX_CONFIG);
 567}
 568
 569static int dwxgmac2_rss_write_reg(void __iomem *ioaddr, bool is_key, int idx,
 570				  u32 val)
 571{
 572	u32 ctrl = 0;
 573
 574	writel(val, ioaddr + XGMAC_RSS_DATA);
 575	ctrl |= idx << XGMAC_RSSIA_SHIFT;
 576	ctrl |= is_key ? XGMAC_ADDRT : 0x0;
 577	ctrl |= XGMAC_OB;
 578	writel(ctrl, ioaddr + XGMAC_RSS_ADDR);
 579
 580	return readl_poll_timeout(ioaddr + XGMAC_RSS_ADDR, ctrl,
 581				  !(ctrl & XGMAC_OB), 100, 10000);
 582}
 583
 584static int dwxgmac2_rss_configure(struct mac_device_info *hw,
 585				  struct stmmac_rss *cfg, u32 num_rxq)
 586{
 587	void __iomem *ioaddr = hw->pcsr;
 588	u32 value, *key;
 589	int i, ret;
 590
 591	value = readl(ioaddr + XGMAC_RSS_CTRL);
 592	if (!cfg || !cfg->enable) {
 593		value &= ~XGMAC_RSSE;
 594		writel(value, ioaddr + XGMAC_RSS_CTRL);
 595		return 0;
 596	}
 597
 598	key = (u32 *)cfg->key;
 599	for (i = 0; i < (ARRAY_SIZE(cfg->key) / sizeof(u32)); i++) {
 600		ret = dwxgmac2_rss_write_reg(ioaddr, true, i, key[i]);
 601		if (ret)
 602			return ret;
 603	}
 604
 605	for (i = 0; i < ARRAY_SIZE(cfg->table); i++) {
 606		ret = dwxgmac2_rss_write_reg(ioaddr, false, i, cfg->table[i]);
 607		if (ret)
 608			return ret;
 609	}
 610
 611	for (i = 0; i < num_rxq; i++)
 612		dwxgmac2_map_mtl_to_dma(hw, i, XGMAC_QDDMACH);
 613
 614	value |= XGMAC_UDP4TE | XGMAC_TCP4TE | XGMAC_IP2TE | XGMAC_RSSE;
 615	writel(value, ioaddr + XGMAC_RSS_CTRL);
 616	return 0;
 617}
 618
 619struct dwxgmac3_error_desc {
 620	bool valid;
 621	const char *desc;
 622	const char *detailed_desc;
 623};
 624
 625#define STAT_OFF(field)		offsetof(struct stmmac_safety_stats, field)
 626
 627static void dwxgmac3_log_error(struct net_device *ndev, u32 value, bool corr,
 628			       const char *module_name,
 629			       const struct dwxgmac3_error_desc *desc,
 630			       unsigned long field_offset,
 631			       struct stmmac_safety_stats *stats)
 632{
 633	unsigned long loc, mask;
 634	u8 *bptr = (u8 *)stats;
 635	unsigned long *ptr;
 636
 637	ptr = (unsigned long *)(bptr + field_offset);
 638
 639	mask = value;
 640	for_each_set_bit(loc, &mask, 32) {
 641		netdev_err(ndev, "Found %s error in %s: '%s: %s'\n", corr ?
 642				"correctable" : "uncorrectable", module_name,
 643				desc[loc].desc, desc[loc].detailed_desc);
 644
 645		/* Update counters */
 646		ptr[loc]++;
 647	}
 648}
 649
 650static const struct dwxgmac3_error_desc dwxgmac3_mac_errors[32]= {
 651	{ true, "ATPES", "Application Transmit Interface Parity Check Error" },
 652	{ true, "DPES", "Descriptor Cache Data Path Parity Check Error" },
 653	{ true, "TPES", "TSO Data Path Parity Check Error" },
 654	{ true, "TSOPES", "TSO Header Data Path Parity Check Error" },
 655	{ true, "MTPES", "MTL Data Path Parity Check Error" },
 656	{ true, "MTSPES", "MTL TX Status Data Path Parity Check Error" },
 657	{ true, "MTBUPES", "MAC TBU Data Path Parity Check Error" },
 658	{ true, "MTFCPES", "MAC TFC Data Path Parity Check Error" },
 659	{ true, "ARPES", "Application Receive Interface Data Path Parity Check Error" },
 660	{ true, "MRWCPES", "MTL RWC Data Path Parity Check Error" },
 661	{ true, "MRRCPES", "MTL RCC Data Path Parity Check Error" },
 662	{ true, "CWPES", "CSR Write Data Path Parity Check Error" },
 663	{ true, "ASRPES", "AXI Slave Read Data Path Parity Check Error" },
 664	{ true, "TTES", "TX FSM Timeout Error" },
 665	{ true, "RTES", "RX FSM Timeout Error" },
 666	{ true, "CTES", "CSR FSM Timeout Error" },
 667	{ true, "ATES", "APP FSM Timeout Error" },
 668	{ true, "PTES", "PTP FSM Timeout Error" },
 669	{ false, "UNKNOWN", "Unknown Error" }, /* 18 */
 670	{ false, "UNKNOWN", "Unknown Error" }, /* 19 */
 671	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
 672	{ true, "MSTTES", "Master Read/Write Timeout Error" },
 673	{ true, "SLVTES", "Slave Read/Write Timeout Error" },
 674	{ true, "ATITES", "Application Timeout on ATI Interface Error" },
 675	{ true, "ARITES", "Application Timeout on ARI Interface Error" },
 676	{ true, "FSMPES", "FSM State Parity Error" },
 677	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
 678	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
 679	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
 680	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
 681	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
 682	{ true, "CPI", "Control Register Parity Check Error" },
 683};
 684
 685static void dwxgmac3_handle_mac_err(struct net_device *ndev,
 686				    void __iomem *ioaddr, bool correctable,
 687				    struct stmmac_safety_stats *stats)
 688{
 689	u32 value;
 690
 691	value = readl(ioaddr + XGMAC_MAC_DPP_FSM_INT_STATUS);
 692	writel(value, ioaddr + XGMAC_MAC_DPP_FSM_INT_STATUS);
 693
 694	dwxgmac3_log_error(ndev, value, correctable, "MAC",
 695			   dwxgmac3_mac_errors, STAT_OFF(mac_errors), stats);
 696}
 697
 698static const struct dwxgmac3_error_desc dwxgmac3_mtl_errors[32]= {
 699	{ true, "TXCES", "MTL TX Memory Error" },
 700	{ true, "TXAMS", "MTL TX Memory Address Mismatch Error" },
 701	{ true, "TXUES", "MTL TX Memory Error" },
 702	{ false, "UNKNOWN", "Unknown Error" }, /* 3 */
 703	{ true, "RXCES", "MTL RX Memory Error" },
 704	{ true, "RXAMS", "MTL RX Memory Address Mismatch Error" },
 705	{ true, "RXUES", "MTL RX Memory Error" },
 706	{ false, "UNKNOWN", "Unknown Error" }, /* 7 */
 707	{ true, "ECES", "MTL EST Memory Error" },
 708	{ true, "EAMS", "MTL EST Memory Address Mismatch Error" },
 709	{ true, "EUES", "MTL EST Memory Error" },
 710	{ false, "UNKNOWN", "Unknown Error" }, /* 11 */
 711	{ true, "RPCES", "MTL RX Parser Memory Error" },
 712	{ true, "RPAMS", "MTL RX Parser Memory Address Mismatch Error" },
 713	{ true, "RPUES", "MTL RX Parser Memory Error" },
 714	{ false, "UNKNOWN", "Unknown Error" }, /* 15 */
 715	{ false, "UNKNOWN", "Unknown Error" }, /* 16 */
 716	{ false, "UNKNOWN", "Unknown Error" }, /* 17 */
 717	{ false, "UNKNOWN", "Unknown Error" }, /* 18 */
 718	{ false, "UNKNOWN", "Unknown Error" }, /* 19 */
 719	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
 720	{ false, "UNKNOWN", "Unknown Error" }, /* 21 */
 721	{ false, "UNKNOWN", "Unknown Error" }, /* 22 */
 722	{ false, "UNKNOWN", "Unknown Error" }, /* 23 */
 723	{ false, "UNKNOWN", "Unknown Error" }, /* 24 */
 724	{ false, "UNKNOWN", "Unknown Error" }, /* 25 */
 725	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
 726	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
 727	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
 728	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
 729	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
 730	{ false, "UNKNOWN", "Unknown Error" }, /* 31 */
 731};
 732
 733static void dwxgmac3_handle_mtl_err(struct net_device *ndev,
 734				    void __iomem *ioaddr, bool correctable,
 735				    struct stmmac_safety_stats *stats)
 736{
 737	u32 value;
 738
 739	value = readl(ioaddr + XGMAC_MTL_ECC_INT_STATUS);
 740	writel(value, ioaddr + XGMAC_MTL_ECC_INT_STATUS);
 741
 742	dwxgmac3_log_error(ndev, value, correctable, "MTL",
 743			   dwxgmac3_mtl_errors, STAT_OFF(mtl_errors), stats);
 744}
 745
 746static const struct dwxgmac3_error_desc dwxgmac3_dma_errors[32]= {
 747	{ true, "TCES", "DMA TSO Memory Error" },
 748	{ true, "TAMS", "DMA TSO Memory Address Mismatch Error" },
 749	{ true, "TUES", "DMA TSO Memory Error" },
 750	{ false, "UNKNOWN", "Unknown Error" }, /* 3 */
 751	{ true, "DCES", "DMA DCACHE Memory Error" },
 752	{ true, "DAMS", "DMA DCACHE Address Mismatch Error" },
 753	{ true, "DUES", "DMA DCACHE Memory Error" },
 754	{ false, "UNKNOWN", "Unknown Error" }, /* 7 */
 755	{ false, "UNKNOWN", "Unknown Error" }, /* 8 */
 756	{ false, "UNKNOWN", "Unknown Error" }, /* 9 */
 757	{ false, "UNKNOWN", "Unknown Error" }, /* 10 */
 758	{ false, "UNKNOWN", "Unknown Error" }, /* 11 */
 759	{ false, "UNKNOWN", "Unknown Error" }, /* 12 */
 760	{ false, "UNKNOWN", "Unknown Error" }, /* 13 */
 761	{ false, "UNKNOWN", "Unknown Error" }, /* 14 */
 762	{ false, "UNKNOWN", "Unknown Error" }, /* 15 */
 763	{ false, "UNKNOWN", "Unknown Error" }, /* 16 */
 764	{ false, "UNKNOWN", "Unknown Error" }, /* 17 */
 765	{ false, "UNKNOWN", "Unknown Error" }, /* 18 */
 766	{ false, "UNKNOWN", "Unknown Error" }, /* 19 */
 767	{ false, "UNKNOWN", "Unknown Error" }, /* 20 */
 768	{ false, "UNKNOWN", "Unknown Error" }, /* 21 */
 769	{ false, "UNKNOWN", "Unknown Error" }, /* 22 */
 770	{ false, "UNKNOWN", "Unknown Error" }, /* 23 */
 771	{ false, "UNKNOWN", "Unknown Error" }, /* 24 */
 772	{ false, "UNKNOWN", "Unknown Error" }, /* 25 */
 773	{ false, "UNKNOWN", "Unknown Error" }, /* 26 */
 774	{ false, "UNKNOWN", "Unknown Error" }, /* 27 */
 775	{ false, "UNKNOWN", "Unknown Error" }, /* 28 */
 776	{ false, "UNKNOWN", "Unknown Error" }, /* 29 */
 777	{ false, "UNKNOWN", "Unknown Error" }, /* 30 */
 778	{ false, "UNKNOWN", "Unknown Error" }, /* 31 */
 779};
 780
 781static const char dpp_rx_err[] = "Read Rx Descriptor Parity checker Error";
 782static const char dpp_tx_err[] = "Read Tx Descriptor Parity checker Error";
 783static const struct dwxgmac3_error_desc dwxgmac3_dma_dpp_errors[32] = {
 784	{ true, "TDPES0", dpp_tx_err },
 785	{ true, "TDPES1", dpp_tx_err },
 786	{ true, "TDPES2", dpp_tx_err },
 787	{ true, "TDPES3", dpp_tx_err },
 788	{ true, "TDPES4", dpp_tx_err },
 789	{ true, "TDPES5", dpp_tx_err },
 790	{ true, "TDPES6", dpp_tx_err },
 791	{ true, "TDPES7", dpp_tx_err },
 792	{ true, "TDPES8", dpp_tx_err },
 793	{ true, "TDPES9", dpp_tx_err },
 794	{ true, "TDPES10", dpp_tx_err },
 795	{ true, "TDPES11", dpp_tx_err },
 796	{ true, "TDPES12", dpp_tx_err },
 797	{ true, "TDPES13", dpp_tx_err },
 798	{ true, "TDPES14", dpp_tx_err },
 799	{ true, "TDPES15", dpp_tx_err },
 800	{ true, "RDPES0", dpp_rx_err },
 801	{ true, "RDPES1", dpp_rx_err },
 802	{ true, "RDPES2", dpp_rx_err },
 803	{ true, "RDPES3", dpp_rx_err },
 804	{ true, "RDPES4", dpp_rx_err },
 805	{ true, "RDPES5", dpp_rx_err },
 806	{ true, "RDPES6", dpp_rx_err },
 807	{ true, "RDPES7", dpp_rx_err },
 808	{ true, "RDPES8", dpp_rx_err },
 809	{ true, "RDPES9", dpp_rx_err },
 810	{ true, "RDPES10", dpp_rx_err },
 811	{ true, "RDPES11", dpp_rx_err },
 812	{ true, "RDPES12", dpp_rx_err },
 813	{ true, "RDPES13", dpp_rx_err },
 814	{ true, "RDPES14", dpp_rx_err },
 815	{ true, "RDPES15", dpp_rx_err },
 816};
 817
 818static void dwxgmac3_handle_dma_err(struct net_device *ndev,
 819				    void __iomem *ioaddr, bool correctable,
 820				    struct stmmac_safety_stats *stats)
 821{
 822	u32 value;
 823
 824	value = readl(ioaddr + XGMAC_DMA_ECC_INT_STATUS);
 825	writel(value, ioaddr + XGMAC_DMA_ECC_INT_STATUS);
 826
 827	dwxgmac3_log_error(ndev, value, correctable, "DMA",
 828			   dwxgmac3_dma_errors, STAT_OFF(dma_errors), stats);
 829
 830	value = readl(ioaddr + XGMAC_DMA_DPP_INT_STATUS);
 831	writel(value, ioaddr + XGMAC_DMA_DPP_INT_STATUS);
 832
 833	dwxgmac3_log_error(ndev, value, false, "DMA_DPP",
 834			   dwxgmac3_dma_dpp_errors,
 835			   STAT_OFF(dma_dpp_errors), stats);
 836}
 837
 838static int
 839dwxgmac3_safety_feat_config(void __iomem *ioaddr, unsigned int asp,
 840			    struct stmmac_safety_feature_cfg *safety_cfg)
 841{
 842	u32 value;
 843
 844	if (!asp)
 845		return -EINVAL;
 846
 847	/* 1. Enable Safety Features */
 848	writel(0x0, ioaddr + XGMAC_MTL_ECC_CONTROL);
 849
 850	/* 2. Enable MTL Safety Interrupts */
 851	value = readl(ioaddr + XGMAC_MTL_ECC_INT_ENABLE);
 852	value |= XGMAC_RPCEIE; /* RX Parser Memory Correctable Error */
 853	value |= XGMAC_ECEIE; /* EST Memory Correctable Error */
 854	value |= XGMAC_RXCEIE; /* RX Memory Correctable Error */
 855	value |= XGMAC_TXCEIE; /* TX Memory Correctable Error */
 856	writel(value, ioaddr + XGMAC_MTL_ECC_INT_ENABLE);
 857
 858	/* 3. Enable DMA Safety Interrupts */
 859	value = readl(ioaddr + XGMAC_DMA_ECC_INT_ENABLE);
 860	value |= XGMAC_DCEIE; /* Descriptor Cache Memory Correctable Error */
 861	value |= XGMAC_TCEIE; /* TSO Memory Correctable Error */
 862	writel(value, ioaddr + XGMAC_DMA_ECC_INT_ENABLE);
 863
 864	/* 0x2: Without ECC or Parity Ports on External Application Interface
 865	 * 0x4: Only ECC Protection for External Memory feature is selected
 866	 */
 867	if (asp == 0x2 || asp == 0x4)
 868		return 0;
 869
 870	/* 4. Enable Parity and Timeout for FSM */
 871	value = readl(ioaddr + XGMAC_MAC_FSM_CONTROL);
 872	value |= XGMAC_PRTYEN; /* FSM Parity Feature */
 873	value |= XGMAC_TMOUTEN; /* FSM Timeout Feature */
 874	writel(value, ioaddr + XGMAC_MAC_FSM_CONTROL);
 875
 876	/* 5. Enable Data Path Parity Protection */
 877	value = readl(ioaddr + XGMAC_MTL_DPP_CONTROL);
 878	/* already enabled by default, explicit enable it again */
 879	value &= ~XGMAC_DPP_DISABLE;
 880	writel(value, ioaddr + XGMAC_MTL_DPP_CONTROL);
 881
 882	return 0;
 883}
 884
 885static int dwxgmac3_safety_feat_irq_status(struct net_device *ndev,
 886					   void __iomem *ioaddr,
 887					   unsigned int asp,
 888					   struct stmmac_safety_stats *stats)
 889{
 890	bool err, corr;
 891	u32 mtl, dma;
 892	int ret = 0;
 893
 894	if (!asp)
 895		return -EINVAL;
 896
 897	mtl = readl(ioaddr + XGMAC_MTL_SAFETY_INT_STATUS);
 898	dma = readl(ioaddr + XGMAC_DMA_SAFETY_INT_STATUS);
 899
 900	err = (mtl & XGMAC_MCSIS) || (dma & XGMAC_MCSIS);
 901	corr = false;
 902	if (err) {
 903		dwxgmac3_handle_mac_err(ndev, ioaddr, corr, stats);
 904		ret |= !corr;
 905	}
 906
 907	err = (mtl & (XGMAC_MEUIS | XGMAC_MECIS)) ||
 908	      (dma & (XGMAC_MSUIS | XGMAC_MSCIS));
 909	corr = (mtl & XGMAC_MECIS) || (dma & XGMAC_MSCIS);
 910	if (err) {
 911		dwxgmac3_handle_mtl_err(ndev, ioaddr, corr, stats);
 912		ret |= !corr;
 913	}
 914
 915	/* DMA_DPP_Interrupt_Status is indicated by MCSIS bit in
 916	 * DMA_Safety_Interrupt_Status, so we handle DMA Data Path
 917	 * Parity Errors here
 918	 */
 919	err = dma & (XGMAC_DEUIS | XGMAC_DECIS | XGMAC_MCSIS);
 920	corr = dma & XGMAC_DECIS;
 921	if (err) {
 922		dwxgmac3_handle_dma_err(ndev, ioaddr, corr, stats);
 923		ret |= !corr;
 924	}
 925
 926	return ret;
 927}
 928
 929static const struct dwxgmac3_error {
 930	const struct dwxgmac3_error_desc *desc;
 931} dwxgmac3_all_errors[] = {
 932	{ dwxgmac3_mac_errors },
 933	{ dwxgmac3_mtl_errors },
 934	{ dwxgmac3_dma_errors },
 935	{ dwxgmac3_dma_dpp_errors },
 936};
 937
 938static int dwxgmac3_safety_feat_dump(struct stmmac_safety_stats *stats,
 939				     int index, unsigned long *count,
 940				     const char **desc)
 941{
 942	int module = index / 32, offset = index % 32;
 943	unsigned long *ptr = (unsigned long *)stats;
 944
 945	if (module >= ARRAY_SIZE(dwxgmac3_all_errors))
 946		return -EINVAL;
 947	if (!dwxgmac3_all_errors[module].desc[offset].valid)
 948		return -EINVAL;
 949	if (count)
 950		*count = *(ptr + index);
 951	if (desc)
 952		*desc = dwxgmac3_all_errors[module].desc[offset].desc;
 953	return 0;
 954}
 955
 956static int dwxgmac3_rxp_disable(void __iomem *ioaddr)
 957{
 958	u32 val = readl(ioaddr + XGMAC_MTL_OPMODE);
 959
 960	val &= ~XGMAC_FRPE;
 961	writel(val, ioaddr + XGMAC_MTL_OPMODE);
 962
 963	return 0;
 964}
 965
 966static void dwxgmac3_rxp_enable(void __iomem *ioaddr)
 967{
 968	u32 val;
 969
 970	val = readl(ioaddr + XGMAC_MTL_OPMODE);
 971	val |= XGMAC_FRPE;
 972	writel(val, ioaddr + XGMAC_MTL_OPMODE);
 973}
 974
 975static int dwxgmac3_rxp_update_single_entry(void __iomem *ioaddr,
 976					    struct stmmac_tc_entry *entry,
 977					    int pos)
 978{
 979	int ret, i;
 980
 981	for (i = 0; i < (sizeof(entry->val) / sizeof(u32)); i++) {
 982		int real_pos = pos * (sizeof(entry->val) / sizeof(u32)) + i;
 983		u32 val;
 984
 985		/* Wait for ready */
 986		ret = readl_poll_timeout(ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST,
 987					 val, !(val & XGMAC_STARTBUSY), 1, 10000);
 988		if (ret)
 989			return ret;
 990
 991		/* Write data */
 992		val = *((u32 *)&entry->val + i);
 993		writel(val, ioaddr + XGMAC_MTL_RXP_IACC_DATA);
 994
 995		/* Write pos */
 996		val = real_pos & XGMAC_ADDR;
 997		writel(val, ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST);
 998
 999		/* Write OP */
1000		val |= XGMAC_WRRDN;
1001		writel(val, ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST);
1002
1003		/* Start Write */
1004		val |= XGMAC_STARTBUSY;
1005		writel(val, ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST);
1006
1007		/* Wait for done */
1008		ret = readl_poll_timeout(ioaddr + XGMAC_MTL_RXP_IACC_CTRL_ST,
1009					 val, !(val & XGMAC_STARTBUSY), 1, 10000);
1010		if (ret)
1011			return ret;
1012	}
1013
1014	return 0;
1015}
1016
1017static struct stmmac_tc_entry *
1018dwxgmac3_rxp_get_next_entry(struct stmmac_tc_entry *entries,
1019			    unsigned int count, u32 curr_prio)
1020{
1021	struct stmmac_tc_entry *entry;
1022	u32 min_prio = ~0x0;
1023	int i, min_prio_idx;
1024	bool found = false;
1025
1026	for (i = count - 1; i >= 0; i--) {
1027		entry = &entries[i];
1028
1029		/* Do not update unused entries */
1030		if (!entry->in_use)
1031			continue;
1032		/* Do not update already updated entries (i.e. fragments) */
1033		if (entry->in_hw)
1034			continue;
1035		/* Let last entry be updated last */
1036		if (entry->is_last)
1037			continue;
1038		/* Do not return fragments */
1039		if (entry->is_frag)
1040			continue;
1041		/* Check if we already checked this prio */
1042		if (entry->prio < curr_prio)
1043			continue;
1044		/* Check if this is the minimum prio */
1045		if (entry->prio < min_prio) {
1046			min_prio = entry->prio;
1047			min_prio_idx = i;
1048			found = true;
1049		}
1050	}
1051
1052	if (found)
1053		return &entries[min_prio_idx];
1054	return NULL;
1055}
1056
1057static int dwxgmac3_rxp_config(void __iomem *ioaddr,
1058			       struct stmmac_tc_entry *entries,
1059			       unsigned int count)
1060{
1061	struct stmmac_tc_entry *entry, *frag;
1062	int i, ret, nve = 0;
1063	u32 curr_prio = 0;
1064	u32 old_val, val;
1065
1066	/* Force disable RX */
1067	old_val = readl(ioaddr + XGMAC_RX_CONFIG);
1068	val = old_val & ~XGMAC_CONFIG_RE;
1069	writel(val, ioaddr + XGMAC_RX_CONFIG);
1070
1071	/* Disable RX Parser */
1072	ret = dwxgmac3_rxp_disable(ioaddr);
1073	if (ret)
1074		goto re_enable;
1075
1076	/* Set all entries as NOT in HW */
1077	for (i = 0; i < count; i++) {
1078		entry = &entries[i];
1079		entry->in_hw = false;
1080	}
1081
1082	/* Update entries by reverse order */
1083	while (1) {
1084		entry = dwxgmac3_rxp_get_next_entry(entries, count, curr_prio);
1085		if (!entry)
1086			break;
1087
1088		curr_prio = entry->prio;
1089		frag = entry->frag_ptr;
1090
1091		/* Set special fragment requirements */
1092		if (frag) {
1093			entry->val.af = 0;
1094			entry->val.rf = 0;
1095			entry->val.nc = 1;
1096			entry->val.ok_index = nve + 2;
1097		}
1098
1099		ret = dwxgmac3_rxp_update_single_entry(ioaddr, entry, nve);
1100		if (ret)
1101			goto re_enable;
1102
1103		entry->table_pos = nve++;
1104		entry->in_hw = true;
1105
1106		if (frag && !frag->in_hw) {
1107			ret = dwxgmac3_rxp_update_single_entry(ioaddr, frag, nve);
1108			if (ret)
1109				goto re_enable;
1110			frag->table_pos = nve++;
1111			frag->in_hw = true;
1112		}
1113	}
1114
1115	if (!nve)
1116		goto re_enable;
1117
1118	/* Update all pass entry */
1119	for (i = 0; i < count; i++) {
1120		entry = &entries[i];
1121		if (!entry->is_last)
1122			continue;
1123
1124		ret = dwxgmac3_rxp_update_single_entry(ioaddr, entry, nve);
1125		if (ret)
1126			goto re_enable;
1127
1128		entry->table_pos = nve++;
1129	}
1130
1131	/* Assume n. of parsable entries == n. of valid entries */
1132	val = (nve << 16) & XGMAC_NPE;
1133	val |= nve & XGMAC_NVE;
1134	writel(val, ioaddr + XGMAC_MTL_RXP_CONTROL_STATUS);
1135
1136	/* Enable RX Parser */
1137	dwxgmac3_rxp_enable(ioaddr);
1138
1139re_enable:
1140	/* Re-enable RX */
1141	writel(old_val, ioaddr + XGMAC_RX_CONFIG);
1142	return ret;
1143}
1144
1145static int dwxgmac2_get_mac_tx_timestamp(struct mac_device_info *hw, u64 *ts)
1146{
1147	void __iomem *ioaddr = hw->pcsr;
1148	u32 value;
1149
1150	if (readl_poll_timeout_atomic(ioaddr + XGMAC_TIMESTAMP_STATUS,
1151				      value, value & XGMAC_TXTSC, 100, 10000))
1152		return -EBUSY;
1153
1154	*ts = readl(ioaddr + XGMAC_TXTIMESTAMP_NSEC) & XGMAC_TXTSSTSLO;
1155	*ts += readl(ioaddr + XGMAC_TXTIMESTAMP_SEC) * 1000000000ULL;
1156	return 0;
1157}
1158
1159static int dwxgmac2_flex_pps_config(void __iomem *ioaddr, int index,
1160				    struct stmmac_pps_cfg *cfg, bool enable,
1161				    u32 sub_second_inc, u32 systime_flags)
1162{
1163	u32 tnsec = readl(ioaddr + XGMAC_PPSx_TARGET_TIME_NSEC(index));
1164	u32 val = readl(ioaddr + XGMAC_PPS_CONTROL);
1165	u64 period;
1166
1167	if (!cfg->available)
1168		return -EINVAL;
1169	if (tnsec & XGMAC_TRGTBUSY0)
1170		return -EBUSY;
1171	if (!sub_second_inc || !systime_flags)
1172		return -EINVAL;
1173
1174	val &= ~XGMAC_PPSx_MASK(index);
1175
1176	if (!enable) {
1177		val |= XGMAC_PPSCMDx(index, XGMAC_PPSCMD_STOP);
1178		writel(val, ioaddr + XGMAC_PPS_CONTROL);
1179		return 0;
1180	}
1181
1182	val |= XGMAC_PPSCMDx(index, XGMAC_PPSCMD_START);
1183	val |= XGMAC_TRGTMODSELx(index, XGMAC_PPSCMD_START);
1184
1185	/* XGMAC Core has 4 PPS outputs at most.
1186	 *
1187	 * Prior XGMAC Core 3.20, Fixed mode or Flexible mode are selectable for
1188	 * PPS0 only via PPSEN0. PPS{1,2,3} are in Flexible mode by default,
1189	 * and can not be switched to Fixed mode, since PPSEN{1,2,3} are
1190	 * read-only reserved to 0.
1191	 * But we always set PPSEN{1,2,3} do not make things worse ;-)
1192	 *
1193	 * From XGMAC Core 3.20 and later, PPSEN{0,1,2,3} are writable and must
1194	 * be set, or the PPS outputs stay in Fixed PPS mode by default.
1195	 */
1196	val |= XGMAC_PPSENx(index);
1197
1198	writel(cfg->start.tv_sec, ioaddr + XGMAC_PPSx_TARGET_TIME_SEC(index));
1199
1200	if (!(systime_flags & PTP_TCR_TSCTRLSSR))
1201		cfg->start.tv_nsec = (cfg->start.tv_nsec * 1000) / 465;
1202	writel(cfg->start.tv_nsec, ioaddr + XGMAC_PPSx_TARGET_TIME_NSEC(index));
1203
1204	period = cfg->period.tv_sec * 1000000000;
1205	period += cfg->period.tv_nsec;
1206
1207	do_div(period, sub_second_inc);
1208
1209	if (period <= 1)
1210		return -EINVAL;
1211
1212	writel(period - 1, ioaddr + XGMAC_PPSx_INTERVAL(index));
1213
1214	period >>= 1;
1215	if (period <= 1)
1216		return -EINVAL;
1217
1218	writel(period - 1, ioaddr + XGMAC_PPSx_WIDTH(index));
1219
1220	/* Finally, activate it */
1221	writel(val, ioaddr + XGMAC_PPS_CONTROL);
1222	return 0;
1223}
1224
1225static void dwxgmac2_sarc_configure(void __iomem *ioaddr, int val)
1226{
1227	u32 value = readl(ioaddr + XGMAC_TX_CONFIG);
1228
1229	value &= ~XGMAC_CONFIG_SARC;
1230	value |= val << XGMAC_CONFIG_SARC_SHIFT;
1231
1232	writel(value, ioaddr + XGMAC_TX_CONFIG);
1233}
1234
1235static int dwxgmac2_filter_wait(struct mac_device_info *hw)
1236{
1237	void __iomem *ioaddr = hw->pcsr;
1238	u32 value;
1239
1240	if (readl_poll_timeout(ioaddr + XGMAC_L3L4_ADDR_CTRL, value,
1241			       !(value & XGMAC_XB), 100, 10000))
1242		return -EBUSY;
1243	return 0;
1244}
1245
1246static int dwxgmac2_filter_read(struct mac_device_info *hw, u32 filter_no,
1247				u8 reg, u32 *data)
1248{
1249	void __iomem *ioaddr = hw->pcsr;
1250	u32 value;
1251	int ret;
1252
1253	ret = dwxgmac2_filter_wait(hw);
1254	if (ret)
1255		return ret;
1256
1257	value = ((filter_no << XGMAC_IDDR_FNUM) | reg) << XGMAC_IDDR_SHIFT;
1258	value |= XGMAC_TT | XGMAC_XB;
1259	writel(value, ioaddr + XGMAC_L3L4_ADDR_CTRL);
1260
1261	ret = dwxgmac2_filter_wait(hw);
1262	if (ret)
1263		return ret;
1264
1265	*data = readl(ioaddr + XGMAC_L3L4_DATA);
1266	return 0;
1267}
1268
1269static int dwxgmac2_filter_write(struct mac_device_info *hw, u32 filter_no,
1270				 u8 reg, u32 data)
1271{
1272	void __iomem *ioaddr = hw->pcsr;
1273	u32 value;
1274	int ret;
1275
1276	ret = dwxgmac2_filter_wait(hw);
1277	if (ret)
1278		return ret;
1279
1280	writel(data, ioaddr + XGMAC_L3L4_DATA);
1281
1282	value = ((filter_no << XGMAC_IDDR_FNUM) | reg) << XGMAC_IDDR_SHIFT;
1283	value |= XGMAC_XB;
1284	writel(value, ioaddr + XGMAC_L3L4_ADDR_CTRL);
1285
1286	return dwxgmac2_filter_wait(hw);
1287}
1288
1289static int dwxgmac2_config_l3_filter(struct mac_device_info *hw, u32 filter_no,
1290				     bool en, bool ipv6, bool sa, bool inv,
1291				     u32 match)
1292{
1293	void __iomem *ioaddr = hw->pcsr;
1294	u32 value;
1295	int ret;
1296
1297	value = readl(ioaddr + XGMAC_PACKET_FILTER);
1298	value |= XGMAC_FILTER_IPFE;
1299	writel(value, ioaddr + XGMAC_PACKET_FILTER);
1300
1301	ret = dwxgmac2_filter_read(hw, filter_no, XGMAC_L3L4_CTRL, &value);
1302	if (ret)
1303		return ret;
1304
1305	/* For IPv6 not both SA/DA filters can be active */
1306	if (ipv6) {
1307		value |= XGMAC_L3PEN0;
1308		value &= ~(XGMAC_L3SAM0 | XGMAC_L3SAIM0);
1309		value &= ~(XGMAC_L3DAM0 | XGMAC_L3DAIM0);
1310		if (sa) {
1311			value |= XGMAC_L3SAM0;
1312			if (inv)
1313				value |= XGMAC_L3SAIM0;
1314		} else {
1315			value |= XGMAC_L3DAM0;
1316			if (inv)
1317				value |= XGMAC_L3DAIM0;
1318		}
1319	} else {
1320		value &= ~XGMAC_L3PEN0;
1321		if (sa) {
1322			value |= XGMAC_L3SAM0;
1323			if (inv)
1324				value |= XGMAC_L3SAIM0;
1325		} else {
1326			value |= XGMAC_L3DAM0;
1327			if (inv)
1328				value |= XGMAC_L3DAIM0;
1329		}
1330	}
1331
1332	ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, value);
1333	if (ret)
1334		return ret;
1335
1336	if (sa) {
1337		ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3_ADDR0, match);
1338		if (ret)
1339			return ret;
1340	} else {
1341		ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3_ADDR1, match);
1342		if (ret)
1343			return ret;
1344	}
1345
1346	if (!en)
1347		return dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, 0);
1348
1349	return 0;
1350}
1351
1352static int dwxgmac2_config_l4_filter(struct mac_device_info *hw, u32 filter_no,
1353				     bool en, bool udp, bool sa, bool inv,
1354				     u32 match)
1355{
1356	void __iomem *ioaddr = hw->pcsr;
1357	u32 value;
1358	int ret;
1359
1360	value = readl(ioaddr + XGMAC_PACKET_FILTER);
1361	value |= XGMAC_FILTER_IPFE;
1362	writel(value, ioaddr + XGMAC_PACKET_FILTER);
1363
1364	ret = dwxgmac2_filter_read(hw, filter_no, XGMAC_L3L4_CTRL, &value);
1365	if (ret)
1366		return ret;
1367
1368	if (udp) {
1369		value |= XGMAC_L4PEN0;
1370	} else {
1371		value &= ~XGMAC_L4PEN0;
1372	}
1373
1374	value &= ~(XGMAC_L4SPM0 | XGMAC_L4SPIM0);
1375	value &= ~(XGMAC_L4DPM0 | XGMAC_L4DPIM0);
1376	if (sa) {
1377		value |= XGMAC_L4SPM0;
1378		if (inv)
1379			value |= XGMAC_L4SPIM0;
1380	} else {
1381		value |= XGMAC_L4DPM0;
1382		if (inv)
1383			value |= XGMAC_L4DPIM0;
1384	}
1385
1386	ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, value);
1387	if (ret)
1388		return ret;
1389
1390	if (sa) {
1391		value = match & XGMAC_L4SP0;
1392
1393		ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L4_ADDR, value);
1394		if (ret)
1395			return ret;
1396	} else {
1397		value = (match << XGMAC_L4DP0_SHIFT) & XGMAC_L4DP0;
1398
1399		ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L4_ADDR, value);
1400		if (ret)
1401			return ret;
1402	}
1403
1404	if (!en)
1405		return dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, 0);
1406
1407	return 0;
1408}
1409
1410static void dwxgmac2_set_arp_offload(struct mac_device_info *hw, bool en,
1411				     u32 addr)
1412{
1413	void __iomem *ioaddr = hw->pcsr;
1414	u32 value;
1415
1416	writel(addr, ioaddr + XGMAC_ARP_ADDR);
1417
1418	value = readl(ioaddr + XGMAC_RX_CONFIG);
1419	if (en)
1420		value |= XGMAC_CONFIG_ARPEN;
1421	else
1422		value &= ~XGMAC_CONFIG_ARPEN;
1423	writel(value, ioaddr + XGMAC_RX_CONFIG);
1424}
1425
1426const struct stmmac_ops dwxgmac210_ops = {
1427	.core_init = dwxgmac2_core_init,
1428	.irq_modify = dwxgmac2_irq_modify,
1429	.update_caps = dwxgmac2_update_caps,
1430	.set_mac = dwxgmac2_set_mac,
1431	.rx_ipc = dwxgmac2_rx_ipc,
1432	.rx_queue_enable = dwxgmac2_rx_queue_enable,
1433	.rx_queue_prio = dwxgmac2_rx_queue_prio,
1434	.tx_queue_prio = dwxgmac2_tx_queue_prio,
1435	.rx_queue_routing = dwxgmac2_rx_queue_routing,
1436	.prog_mtl_rx_algorithms = dwxgmac2_prog_mtl_rx_algorithms,
1437	.prog_mtl_tx_algorithms = dwxgmac2_prog_mtl_tx_algorithms,
1438	.set_mtl_tx_queue_weight = dwxgmac2_set_mtl_tx_queue_weight,
1439	.map_mtl_to_dma = dwxgmac2_map_mtl_to_dma,
1440	.config_cbs = dwxgmac2_config_cbs,
1441	.dump_regs = dwxgmac2_dump_regs,
1442	.host_irq_status = dwxgmac2_host_irq_status,
1443	.host_mtl_irq_status = dwxgmac2_host_mtl_irq_status,
1444	.flow_ctrl = dwxgmac2_flow_ctrl,
1445	.pmt = dwxgmac2_pmt,
1446	.set_umac_addr = dwxgmac2_set_umac_addr,
1447	.get_umac_addr = dwxgmac2_get_umac_addr,
1448	.set_lpi_mode = dwxgmac2_set_lpi_mode,
1449	.set_eee_timer = dwxgmac2_set_eee_timer,
1450	.set_eee_pls = dwxgmac2_set_eee_pls,
1451	.debug = NULL,
1452	.set_filter = dwxgmac2_set_filter,
1453	.safety_feat_config = dwxgmac3_safety_feat_config,
1454	.safety_feat_irq_status = dwxgmac3_safety_feat_irq_status,
1455	.safety_feat_dump = dwxgmac3_safety_feat_dump,
1456	.set_mac_loopback = dwxgmac2_set_mac_loopback,
1457	.rss_configure = dwxgmac2_rss_configure,
1458	.rxp_config = dwxgmac3_rxp_config,
1459	.get_mac_tx_timestamp = dwxgmac2_get_mac_tx_timestamp,
1460	.flex_pps_config = dwxgmac2_flex_pps_config,
1461	.sarc_configure = dwxgmac2_sarc_configure,
1462	.config_l3_filter = dwxgmac2_config_l3_filter,
1463	.config_l4_filter = dwxgmac2_config_l4_filter,
1464	.set_arp_offload = dwxgmac2_set_arp_offload,
1465	.fpe_map_preemption_class = dwxgmac3_fpe_map_preemption_class,
1466};
1467
1468static void dwxlgmac2_rx_queue_enable(struct mac_device_info *hw, u8 mode,
1469				      u32 queue)
1470{
1471	void __iomem *ioaddr = hw->pcsr;
1472	u32 value;
1473
1474	value = readl(ioaddr + XLGMAC_RXQ_ENABLE_CTRL0) & ~XGMAC_RXQEN(queue);
1475	if (mode == MTL_QUEUE_AVB)
1476		value |= 0x1 << XGMAC_RXQEN_SHIFT(queue);
1477	else if (mode == MTL_QUEUE_DCB)
1478		value |= 0x2 << XGMAC_RXQEN_SHIFT(queue);
1479	writel(value, ioaddr + XLGMAC_RXQ_ENABLE_CTRL0);
1480}
1481
1482const struct stmmac_ops dwxlgmac2_ops = {
1483	.core_init = dwxgmac2_core_init,
1484	.irq_modify = dwxgmac2_irq_modify,
1485	.set_mac = dwxgmac2_set_mac,
1486	.rx_ipc = dwxgmac2_rx_ipc,
1487	.rx_queue_enable = dwxlgmac2_rx_queue_enable,
1488	.rx_queue_prio = dwxgmac2_rx_queue_prio,
1489	.tx_queue_prio = dwxgmac2_tx_queue_prio,
1490	.rx_queue_routing = dwxgmac2_rx_queue_routing,
1491	.prog_mtl_rx_algorithms = dwxgmac2_prog_mtl_rx_algorithms,
1492	.prog_mtl_tx_algorithms = dwxgmac2_prog_mtl_tx_algorithms,
1493	.set_mtl_tx_queue_weight = dwxgmac2_set_mtl_tx_queue_weight,
1494	.map_mtl_to_dma = dwxgmac2_map_mtl_to_dma,
1495	.config_cbs = dwxgmac2_config_cbs,
1496	.dump_regs = dwxgmac2_dump_regs,
1497	.host_irq_status = dwxgmac2_host_irq_status,
1498	.host_mtl_irq_status = dwxgmac2_host_mtl_irq_status,
1499	.flow_ctrl = dwxgmac2_flow_ctrl,
1500	.pmt = dwxgmac2_pmt,
1501	.set_umac_addr = dwxgmac2_set_umac_addr,
1502	.get_umac_addr = dwxgmac2_get_umac_addr,
1503	.set_lpi_mode = dwxgmac2_set_lpi_mode,
1504	.set_eee_timer = dwxgmac2_set_eee_timer,
1505	.set_eee_pls = dwxgmac2_set_eee_pls,
1506	.debug = NULL,
1507	.set_filter = dwxgmac2_set_filter,
1508	.safety_feat_config = dwxgmac3_safety_feat_config,
1509	.safety_feat_irq_status = dwxgmac3_safety_feat_irq_status,
1510	.safety_feat_dump = dwxgmac3_safety_feat_dump,
1511	.set_mac_loopback = dwxgmac2_set_mac_loopback,
1512	.rss_configure = dwxgmac2_rss_configure,
1513	.rxp_config = dwxgmac3_rxp_config,
1514	.get_mac_tx_timestamp = dwxgmac2_get_mac_tx_timestamp,
1515	.flex_pps_config = dwxgmac2_flex_pps_config,
1516	.sarc_configure = dwxgmac2_sarc_configure,
1517	.config_l3_filter = dwxgmac2_config_l3_filter,
1518	.config_l4_filter = dwxgmac2_config_l4_filter,
1519	.set_arp_offload = dwxgmac2_set_arp_offload,
1520	.fpe_map_preemption_class = dwxgmac3_fpe_map_preemption_class,
1521};
1522
1523int dwxgmac2_setup(struct stmmac_priv *priv)
1524{
1525	struct mac_device_info *mac = priv->hw;
1526
1527	dev_info(priv->device, "\tXGMAC2\n");
1528
1529	priv->dev->priv_flags |= IFF_UNICAST_FLT;
1530	mac->pcsr = priv->ioaddr;
1531	mac->multicast_filter_bins = priv->plat->multicast_filter_bins;
1532	mac->unicast_filter_entries = priv->plat->unicast_filter_entries;
1533	mac->mcast_bits_log2 = 0;
1534
1535	if (mac->multicast_filter_bins)
1536		mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins);
1537
1538	mac->link.caps = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
1539			 MAC_10 | MAC_100 | MAC_1000FD |
1540			 MAC_2500FD | MAC_5000FD | MAC_10000FD;
1541	mac->link.duplex = 0;
1542	mac->link.speed10 = XGMAC_CONFIG_SS_10_MII;
1543	mac->link.speed100 = XGMAC_CONFIG_SS_100_MII;
1544	mac->link.speed1000 = XGMAC_CONFIG_SS_1000_GMII;
1545	mac->link.speed2500 = XGMAC_CONFIG_SS_2500_GMII;
1546	mac->link.xgmii.speed2500 = XGMAC_CONFIG_SS_2500;
1547	mac->link.xgmii.speed5000 = XGMAC_CONFIG_SS_5000;
1548	mac->link.xgmii.speed10000 = XGMAC_CONFIG_SS_10000;
1549	mac->link.speed_mask = XGMAC_CONFIG_SS_MASK;
1550
1551	mac->mii.addr = XGMAC_MDIO_ADDR;
1552	mac->mii.data = XGMAC_MDIO_DATA;
1553	mac->mii.addr_shift = 16;
1554	mac->mii.addr_mask = GENMASK(20, 16);
1555	mac->mii.reg_shift = 0;
1556	mac->mii.reg_mask = GENMASK(15, 0);
1557	mac->mii.clk_csr_shift = 19;
1558	mac->mii.clk_csr_mask = GENMASK(21, 19);
1559	mac->num_vlan = stmmac_get_num_vlan(priv->ioaddr);
1560
1561	return 0;
1562}
1563
1564int dwxlgmac2_setup(struct stmmac_priv *priv)
1565{
1566	struct mac_device_info *mac = priv->hw;
1567
1568	dev_info(priv->device, "\tXLGMAC\n");
1569
1570	priv->dev->priv_flags |= IFF_UNICAST_FLT;
1571	mac->pcsr = priv->ioaddr;
1572	mac->multicast_filter_bins = priv->plat->multicast_filter_bins;
1573	mac->unicast_filter_entries = priv->plat->unicast_filter_entries;
1574	mac->mcast_bits_log2 = 0;
1575
1576	if (mac->multicast_filter_bins)
1577		mac->mcast_bits_log2 = ilog2(mac->multicast_filter_bins);
1578
1579	mac->link.caps = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
1580			 MAC_1000FD | MAC_2500FD | MAC_5000FD |
1581			 MAC_10000FD | MAC_25000FD |
1582			 MAC_40000FD | MAC_50000FD |
1583			 MAC_100000FD;
1584	mac->link.duplex = 0;
1585	mac->link.speed1000 = XLGMAC_CONFIG_SS_1000;
1586	mac->link.speed2500 = XLGMAC_CONFIG_SS_2500;
1587	mac->link.xgmii.speed10000 = XLGMAC_CONFIG_SS_10G;
1588	mac->link.xlgmii.speed25000 = XLGMAC_CONFIG_SS_25G;
1589	mac->link.xlgmii.speed40000 = XLGMAC_CONFIG_SS_40G;
1590	mac->link.xlgmii.speed50000 = XLGMAC_CONFIG_SS_50G;
1591	mac->link.xlgmii.speed100000 = XLGMAC_CONFIG_SS_100G;
1592	mac->link.speed_mask = XLGMAC_CONFIG_SS;
1593
1594	mac->mii.addr = XGMAC_MDIO_ADDR;
1595	mac->mii.data = XGMAC_MDIO_DATA;
1596	mac->mii.addr_shift = 16;
1597	mac->mii.addr_mask = GENMASK(20, 16);
1598	mac->mii.reg_shift = 0;
1599	mac->mii.reg_mask = GENMASK(15, 0);
1600	mac->mii.clk_csr_shift = 19;
1601	mac->mii.clk_csr_mask = GENMASK(21, 19);
1602
1603	return 0;
1604}