dpaa_eth.c

   1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
   2/*
   3 * Copyright 2008 - 2016 Freescale Semiconductor Inc.
   4 * Copyright 2020 NXP
   5 */
   6
   7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9#include <linux/init.h>
  10#include <linux/mod_devicetable.h>
  11#include <linux/module.h>
  12#include <linux/of_mdio.h>
  13#include <linux/of_net.h>
  14#include <linux/io.h>
  15#include <linux/if_arp.h>
  16#include <linux/if_vlan.h>
  17#include <linux/icmp.h>
  18#include <linux/ip.h>
  19#include <linux/ipv6.h>
  20#include <linux/platform_device.h>
  21#include <linux/udp.h>
  22#include <linux/tcp.h>
  23#include <linux/net.h>
  24#include <linux/skbuff.h>
  25#include <linux/etherdevice.h>
  26#include <linux/if_ether.h>
  27#include <linux/highmem.h>
  28#include <linux/percpu.h>
  29#include <linux/dma-mapping.h>
  30#include <linux/sort.h>
  31#include <linux/bpf.h>
  32#include <linux/bpf_trace.h>
  33#include <soc/fsl/bman.h>
  34#include <soc/fsl/qman.h>
  35#include "fman.h"
  36#include "fman_port.h"
  37#include "mac.h"
  38#include "dpaa_eth.h"
  39
  40/* CREATE_TRACE_POINTS only needs to be defined once. Other dpaa files
  41 * using trace events only need to #include <trace/events/sched.h>
  42 */
  43#define CREATE_TRACE_POINTS
  44#include "dpaa_eth_trace.h"
  45
  46static int debug = -1;
  47module_param(debug, int, 0444);
  48MODULE_PARM_DESC(debug, "Module/Driver verbosity level (0=none,...,16=all)");
  49
  50static u16 tx_timeout = 1000;
  51module_param(tx_timeout, ushort, 0444);
  52MODULE_PARM_DESC(tx_timeout, "The Tx timeout in ms");
  53
  54#define FM_FD_STAT_RX_ERRORS						\
  55	(FM_FD_ERR_DMA | FM_FD_ERR_PHYSICAL	| \
  56	 FM_FD_ERR_SIZE | FM_FD_ERR_CLS_DISCARD | \
  57	 FM_FD_ERR_EXTRACTION | FM_FD_ERR_NO_SCHEME	| \
  58	 FM_FD_ERR_PRS_TIMEOUT | FM_FD_ERR_PRS_ILL_INSTRUCT | \
  59	 FM_FD_ERR_PRS_HDR_ERR)
  60
  61#define FM_FD_STAT_TX_ERRORS \
  62	(FM_FD_ERR_UNSUPPORTED_FORMAT | \
  63	 FM_FD_ERR_LENGTH | FM_FD_ERR_DMA)
  64
  65#define DPAA_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | \
  66			  NETIF_MSG_LINK | NETIF_MSG_IFUP | \
  67			  NETIF_MSG_IFDOWN | NETIF_MSG_HW)
  68
  69#define DPAA_INGRESS_CS_THRESHOLD 0x10000000
  70/* Ingress congestion threshold on FMan ports
  71 * The size in bytes of the ingress tail-drop threshold on FMan ports.
  72 * Traffic piling up above this value will be rejected by QMan and discarded
  73 * by FMan.
  74 */
  75
  76/* Size in bytes of the FQ taildrop threshold */
  77#define DPAA_FQ_TD 0x200000
  78
  79#define DPAA_CS_THRESHOLD_1G 0x06000000
  80/* Egress congestion threshold on 1G ports, range 0x1000 .. 0x10000000
  81 * The size in bytes of the egress Congestion State notification threshold on
  82 * 1G ports. The 1G dTSECs can quite easily be flooded by cores doing Tx in a
  83 * tight loop (e.g. by sending UDP datagrams at "while(1) speed"),
  84 * and the larger the frame size, the more acute the problem.
  85 * So we have to find a balance between these factors:
  86 * - avoiding the device staying congested for a prolonged time (risking
  87 *   the netdev watchdog to fire - see also the tx_timeout module param);
  88 * - affecting performance of protocols such as TCP, which otherwise
  89 *   behave well under the congestion notification mechanism;
  90 * - preventing the Tx cores from tightly-looping (as if the congestion
  91 *   threshold was too low to be effective);
  92 * - running out of memory if the CS threshold is set too high.
  93 */
  94
  95#define DPAA_CS_THRESHOLD_10G 0x10000000
  96/* The size in bytes of the egress Congestion State notification threshold on
  97 * 10G ports, range 0x1000 .. 0x10000000
  98 */
  99
 100/* Largest value that the FQD's OAL field can hold */
 101#define FSL_QMAN_MAX_OAL	127
 102
 103/* Default alignment for start of data in an Rx FD */
 104#ifdef CONFIG_DPAA_ERRATUM_A050385
 105/* aligning data start to 64 avoids DMA transaction splits, unless the buffer
 106 * is crossing a 4k page boundary
 107 */
 108#define DPAA_FD_DATA_ALIGNMENT  (fman_has_errata_a050385() ? 64 : 16)
 109/* aligning to 256 avoids DMA transaction splits caused by 4k page boundary
 110 * crossings; also, all SG fragments except the last must have a size multiple
 111 * of 256 to avoid DMA transaction splits
 112 */
 113#define DPAA_A050385_ALIGN 256
 114#define DPAA_FD_RX_DATA_ALIGNMENT (fman_has_errata_a050385() ? \
 115				   DPAA_A050385_ALIGN : 16)
 116#else
 117#define DPAA_FD_DATA_ALIGNMENT  16
 118#define DPAA_FD_RX_DATA_ALIGNMENT DPAA_FD_DATA_ALIGNMENT
 119#endif
 120
 121/* The DPAA requires 256 bytes reserved and mapped for the SGT */
 122#define DPAA_SGT_SIZE 256
 123
 124/* Values for the L3R field of the FM Parse Results
 125 */
 126/* L3 Type field: First IP Present IPv4 */
 127#define FM_L3_PARSE_RESULT_IPV4	0x8000
 128/* L3 Type field: First IP Present IPv6 */
 129#define FM_L3_PARSE_RESULT_IPV6	0x4000
 130/* Values for the L4R field of the FM Parse Results */
 131/* L4 Type field: UDP */
 132#define FM_L4_PARSE_RESULT_UDP	0x40
 133/* L4 Type field: TCP */
 134#define FM_L4_PARSE_RESULT_TCP	0x20
 135
 136/* FD status field indicating whether the FM Parser has attempted to validate
 137 * the L4 csum of the frame.
 138 * Note that having this bit set doesn't necessarily imply that the checksum
 139 * is valid. One would have to check the parse results to find that out.
 140 */
 141#define FM_FD_STAT_L4CV         0x00000004
 142
 143#define DPAA_SGT_MAX_ENTRIES 16 /* maximum number of entries in SG Table */
 144#define DPAA_BUFF_RELEASE_MAX 8 /* maximum number of buffers released at once */
 145
 146#define FSL_DPAA_BPID_INV		0xff
 147#define FSL_DPAA_ETH_MAX_BUF_COUNT	128
 148#define FSL_DPAA_ETH_REFILL_THRESHOLD	80
 149
 150#define DPAA_TX_PRIV_DATA_SIZE	16
 151#define DPAA_PARSE_RESULTS_SIZE sizeof(struct fman_prs_result)
 152#define DPAA_TIME_STAMP_SIZE 8
 153#define DPAA_HASH_RESULTS_SIZE 8
 154#define DPAA_HWA_SIZE (DPAA_PARSE_RESULTS_SIZE + DPAA_TIME_STAMP_SIZE \
 155		       + DPAA_HASH_RESULTS_SIZE)
 156#define DPAA_RX_PRIV_DATA_DEFAULT_SIZE (DPAA_TX_PRIV_DATA_SIZE + \
 157					XDP_PACKET_HEADROOM - DPAA_HWA_SIZE)
 158#ifdef CONFIG_DPAA_ERRATUM_A050385
 159#define DPAA_RX_PRIV_DATA_A050385_SIZE (DPAA_A050385_ALIGN - DPAA_HWA_SIZE)
 160#define DPAA_RX_PRIV_DATA_SIZE (fman_has_errata_a050385() ? \
 161				DPAA_RX_PRIV_DATA_A050385_SIZE : \
 162				DPAA_RX_PRIV_DATA_DEFAULT_SIZE)
 163#else
 164#define DPAA_RX_PRIV_DATA_SIZE DPAA_RX_PRIV_DATA_DEFAULT_SIZE
 165#endif
 166
 167#define DPAA_ETH_PCD_RXQ_NUM	128
 168
 169#define DPAA_ENQUEUE_RETRIES	100000
 170
 171enum port_type {RX, TX};
 172
 173struct fm_port_fqs {
 174	struct dpaa_fq *tx_defq;
 175	struct dpaa_fq *tx_errq;
 176	struct dpaa_fq *rx_defq;
 177	struct dpaa_fq *rx_errq;
 178	struct dpaa_fq *rx_pcdq;
 179};
 180
 181/* All the dpa bps in use at any moment */
 182static struct dpaa_bp *dpaa_bp_array[BM_MAX_NUM_OF_POOLS];
 183
 184#define DPAA_BP_RAW_SIZE 4096
 185
 186#ifdef CONFIG_DPAA_ERRATUM_A050385
 187#define dpaa_bp_size(raw_size) (SKB_WITH_OVERHEAD(raw_size) & \
 188				~(DPAA_A050385_ALIGN - 1))
 189#else
 190#define dpaa_bp_size(raw_size) SKB_WITH_OVERHEAD(raw_size)
 191#endif
 192
 193static int dpaa_max_frm;
 194
 195static int dpaa_rx_extra_headroom;
 196
 197#define dpaa_get_max_mtu()	\
 198	(dpaa_max_frm - (VLAN_ETH_HLEN + ETH_FCS_LEN))
 199
 200static void dpaa_eth_cgr_set_speed(struct mac_device *mac_dev, int speed);
 201
 202static int dpaa_netdev_init(struct net_device *net_dev,
 203			    const struct net_device_ops *dpaa_ops,
 204			    u16 tx_timeout)
 205{
 206	struct dpaa_priv *priv = netdev_priv(net_dev);
 207	struct device *dev = net_dev->dev.parent;
 208	struct mac_device *mac_dev = priv->mac_dev;
 209	struct dpaa_percpu_priv *percpu_priv;
 210	const u8 *mac_addr;
 211	int i, err;
 212
 213	/* Although we access another CPU's private data here
 214	 * we do it at initialization so it is safe
 215	 */
 216	for_each_possible_cpu(i) {
 217		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
 218		percpu_priv->net_dev = net_dev;
 219	}
 220
 221	net_dev->netdev_ops = dpaa_ops;
 222	mac_addr = mac_dev->addr;
 223
 224	net_dev->mem_start = (unsigned long)priv->mac_dev->res->start;
 225	net_dev->mem_end = (unsigned long)priv->mac_dev->res->end;
 226
 227	net_dev->min_mtu = ETH_MIN_MTU;
 228	net_dev->max_mtu = dpaa_get_max_mtu();
 229
 230	net_dev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 231				 NETIF_F_RXHASH);
 232
 233	net_dev->hw_features |= NETIF_F_SG | NETIF_F_HIGHDMA;
 234	/* The kernels enables GSO automatically, if we declare NETIF_F_SG.
 235	 * For conformity, we'll still declare GSO explicitly.
 236	 */
 237	net_dev->features |= NETIF_F_GSO;
 238	net_dev->features |= NETIF_F_RXCSUM;
 239
 240	net_dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
 241	net_dev->lltx = true;
 242	/* we do not want shared skbs on TX */
 243	net_dev->priv_flags &= ~IFF_TX_SKB_SHARING;
 244
 245	net_dev->features |= net_dev->hw_features;
 246	net_dev->vlan_features = net_dev->features;
 247
 248	net_dev->xdp_features = NETDEV_XDP_ACT_BASIC |
 249				NETDEV_XDP_ACT_REDIRECT |
 250				NETDEV_XDP_ACT_NDO_XMIT;
 251
 252	if (is_valid_ether_addr(mac_addr)) {
 253		memcpy(net_dev->perm_addr, mac_addr, net_dev->addr_len);
 254		eth_hw_addr_set(net_dev, mac_addr);
 255	} else {
 256		eth_hw_addr_random(net_dev);
 257		err = mac_dev->change_addr(mac_dev->fman_mac,
 258			(const enet_addr_t *)net_dev->dev_addr);
 259		if (err) {
 260			dev_err(dev, "Failed to set random MAC address\n");
 261			return -EINVAL;
 262		}
 263		dev_info(dev, "Using random MAC address: %pM\n",
 264			 net_dev->dev_addr);
 265	}
 266
 267	net_dev->ethtool_ops = &dpaa_ethtool_ops;
 268
 269	net_dev->needed_headroom = priv->tx_headroom;
 270	net_dev->watchdog_timeo = msecs_to_jiffies(tx_timeout);
 271
 272	/* The rest of the config is filled in by the mac device already */
 273	mac_dev->phylink_config.dev = &net_dev->dev;
 274	mac_dev->phylink_config.type = PHYLINK_NETDEV;
 275	mac_dev->update_speed = dpaa_eth_cgr_set_speed;
 276	mac_dev->phylink = phylink_create(&mac_dev->phylink_config,
 277					  dev_fwnode(mac_dev->dev),
 278					  mac_dev->phy_if,
 279					  mac_dev->phylink_ops);
 280	if (IS_ERR(mac_dev->phylink)) {
 281		err = PTR_ERR(mac_dev->phylink);
 282		dev_err_probe(dev, err, "Could not create phylink\n");
 283		return err;
 284	}
 285
 286	/* start without the RUNNING flag, phylib controls it later */
 287	netif_carrier_off(net_dev);
 288
 289	err = register_netdev(net_dev);
 290	if (err < 0) {
 291		dev_err(dev, "register_netdev() = %d\n", err);
 292		phylink_destroy(mac_dev->phylink);
 293		return err;
 294	}
 295
 296	return 0;
 297}
 298
 299static int dpaa_stop(struct net_device *net_dev)
 300{
 301	struct mac_device *mac_dev;
 302	struct dpaa_priv *priv;
 303	int i, error;
 304	int err = 0;
 305
 306	priv = netdev_priv(net_dev);
 307	mac_dev = priv->mac_dev;
 308
 309	netif_tx_stop_all_queues(net_dev);
 310	/* Allow the Fman (Tx) port to process in-flight frames before we
 311	 * try switching it off.
 312	 */
 313	msleep(200);
 314
 315	phylink_stop(mac_dev->phylink);
 316	mac_dev->disable(mac_dev->fman_mac);
 317
 318	for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
 319		error = fman_port_disable(mac_dev->port[i]);
 320		if (error)
 321			err = error;
 322	}
 323
 324	phylink_disconnect_phy(mac_dev->phylink);
 325	net_dev->phydev = NULL;
 326
 327	msleep(200);
 328
 329	return err;
 330}
 331
 332static void dpaa_tx_timeout(struct net_device *net_dev, unsigned int txqueue)
 333{
 334	struct dpaa_percpu_priv *percpu_priv;
 335	const struct dpaa_priv	*priv;
 336
 337	priv = netdev_priv(net_dev);
 338	percpu_priv = this_cpu_ptr(priv->percpu_priv);
 339
 340	netif_crit(priv, timer, net_dev, "Transmit timeout latency: %u ms\n",
 341		   jiffies_to_msecs(jiffies - dev_trans_start(net_dev)));
 342
 343	percpu_priv->stats.tx_errors++;
 344}
 345
 346/* Calculates the statistics for the given device by adding the statistics
 347 * collected by each CPU.
 348 */
 349static void dpaa_get_stats64(struct net_device *net_dev,
 350			     struct rtnl_link_stats64 *s)
 351{
 352	int numstats = sizeof(struct rtnl_link_stats64) / sizeof(u64);
 353	struct dpaa_priv *priv = netdev_priv(net_dev);
 354	struct dpaa_percpu_priv *percpu_priv;
 355	u64 *netstats = (u64 *)s;
 356	u64 *cpustats;
 357	int i, j;
 358
 359	for_each_possible_cpu(i) {
 360		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
 361
 362		cpustats = (u64 *)&percpu_priv->stats;
 363
 364		/* add stats from all CPUs */
 365		for (j = 0; j < numstats; j++)
 366			netstats[j] += cpustats[j];
 367	}
 368}
 369
 370static int dpaa_setup_tc(struct net_device *net_dev, enum tc_setup_type type,
 371			 void *type_data)
 372{
 373	struct dpaa_priv *priv = netdev_priv(net_dev);
 374	int num_txqs_per_tc = dpaa_num_txqs_per_tc();
 375	struct tc_mqprio_qopt *mqprio = type_data;
 376	u8 num_tc;
 377	int i;
 378
 379	if (type != TC_SETUP_QDISC_MQPRIO)
 380		return -EOPNOTSUPP;
 381
 382	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
 383	num_tc = mqprio->num_tc;
 384
 385	if (num_tc == priv->num_tc)
 386		return 0;
 387
 388	if (!num_tc) {
 389		netdev_reset_tc(net_dev);
 390		goto out;
 391	}
 392
 393	if (num_tc > DPAA_TC_NUM) {
 394		netdev_err(net_dev, "Too many traffic classes: max %d supported.\n",
 395			   DPAA_TC_NUM);
 396		return -EINVAL;
 397	}
 398
 399	netdev_set_num_tc(net_dev, num_tc);
 400
 401	for (i = 0; i < num_tc; i++)
 402		netdev_set_tc_queue(net_dev, i, num_txqs_per_tc,
 403				    i * num_txqs_per_tc);
 404
 405out:
 406	priv->num_tc = num_tc ? : 1;
 407	netif_set_real_num_tx_queues(net_dev, priv->num_tc * num_txqs_per_tc);
 408	return 0;
 409}
 410
 411static struct mac_device *dpaa_mac_dev_get(struct platform_device *pdev)
 412{
 413	struct dpaa_eth_data *eth_data;
 414	struct device *dpaa_dev;
 415	struct mac_device *mac_dev;
 416
 417	dpaa_dev = &pdev->dev;
 418	eth_data = dpaa_dev->platform_data;
 419	if (!eth_data) {
 420		dev_err(dpaa_dev, "eth_data missing\n");
 421		return ERR_PTR(-ENODEV);
 422	}
 423	mac_dev = eth_data->mac_dev;
 424	if (!mac_dev) {
 425		dev_err(dpaa_dev, "mac_dev missing\n");
 426		return ERR_PTR(-EINVAL);
 427	}
 428
 429	return mac_dev;
 430}
 431
 432static int dpaa_set_mac_address(struct net_device *net_dev, void *addr)
 433{
 434	const struct dpaa_priv *priv;
 435	struct mac_device *mac_dev;
 436	struct sockaddr old_addr;
 437	int err;
 438
 439	priv = netdev_priv(net_dev);
 440
 441	memcpy(old_addr.sa_data, net_dev->dev_addr,  ETH_ALEN);
 442
 443	err = eth_mac_addr(net_dev, addr);
 444	if (err < 0) {
 445		netif_err(priv, drv, net_dev, "eth_mac_addr() = %d\n", err);
 446		return err;
 447	}
 448
 449	mac_dev = priv->mac_dev;
 450
 451	err = mac_dev->change_addr(mac_dev->fman_mac,
 452				   (const enet_addr_t *)net_dev->dev_addr);
 453	if (err < 0) {
 454		netif_err(priv, drv, net_dev, "mac_dev->change_addr() = %d\n",
 455			  err);
 456		/* reverting to previous address */
 457		eth_mac_addr(net_dev, &old_addr);
 458
 459		return err;
 460	}
 461
 462	return 0;
 463}
 464
 465static int dpaa_addr_sync(struct net_device *net_dev, const u8 *addr)
 466{
 467	const struct dpaa_priv *priv = netdev_priv(net_dev);
 468
 469	return priv->mac_dev->add_hash_mac_addr(priv->mac_dev->fman_mac,
 470						(enet_addr_t *)addr);
 471}
 472
 473static int dpaa_addr_unsync(struct net_device *net_dev, const u8 *addr)
 474{
 475	const struct dpaa_priv *priv = netdev_priv(net_dev);
 476
 477	return priv->mac_dev->remove_hash_mac_addr(priv->mac_dev->fman_mac,
 478						   (enet_addr_t *)addr);
 479}
 480
 481static void dpaa_set_rx_mode(struct net_device *net_dev)
 482{
 483	const struct dpaa_priv	*priv;
 484	int err;
 485
 486	priv = netdev_priv(net_dev);
 487
 488	if (!!(net_dev->flags & IFF_PROMISC) != priv->mac_dev->promisc) {
 489		priv->mac_dev->promisc = !priv->mac_dev->promisc;
 490		err = priv->mac_dev->set_promisc(priv->mac_dev->fman_mac,
 491						 priv->mac_dev->promisc);
 492		if (err < 0)
 493			netif_err(priv, drv, net_dev,
 494				  "mac_dev->set_promisc() = %d\n",
 495				  err);
 496	}
 497
 498	if (!!(net_dev->flags & IFF_ALLMULTI) != priv->mac_dev->allmulti) {
 499		priv->mac_dev->allmulti = !priv->mac_dev->allmulti;
 500		err = priv->mac_dev->set_allmulti(priv->mac_dev->fman_mac,
 501						  priv->mac_dev->allmulti);
 502		if (err < 0)
 503			netif_err(priv, drv, net_dev,
 504				  "mac_dev->set_allmulti() = %d\n",
 505				  err);
 506	}
 507
 508	err = __dev_mc_sync(net_dev, dpaa_addr_sync, dpaa_addr_unsync);
 509	if (err < 0)
 510		netif_err(priv, drv, net_dev, "dpaa_addr_sync() = %d\n",
 511			  err);
 512}
 513
 514static struct dpaa_bp *dpaa_bpid2pool(int bpid)
 515{
 516	if (WARN_ON(bpid < 0 || bpid >= BM_MAX_NUM_OF_POOLS))
 517		return NULL;
 518
 519	return dpaa_bp_array[bpid];
 520}
 521
 522/* checks if this bpool is already allocated */
 523static bool dpaa_bpid2pool_use(int bpid)
 524{
 525	if (dpaa_bpid2pool(bpid)) {
 526		refcount_inc(&dpaa_bp_array[bpid]->refs);
 527		return true;
 528	}
 529
 530	return false;
 531}
 532
 533/* called only once per bpid by dpaa_bp_alloc_pool() */
 534static void dpaa_bpid2pool_map(int bpid, struct dpaa_bp *dpaa_bp)
 535{
 536	dpaa_bp_array[bpid] = dpaa_bp;
 537	refcount_set(&dpaa_bp->refs, 1);
 538}
 539
 540static int dpaa_bp_alloc_pool(struct dpaa_bp *dpaa_bp)
 541{
 542	int err;
 543
 544	if (dpaa_bp->size == 0 || dpaa_bp->config_count == 0) {
 545		pr_err("%s: Buffer pool is not properly initialized! Missing size or initial number of buffers\n",
 546		       __func__);
 547		return -EINVAL;
 548	}
 549
 550	/* If the pool is already specified, we only create one per bpid */
 551	if (dpaa_bp->bpid != FSL_DPAA_BPID_INV &&
 552	    dpaa_bpid2pool_use(dpaa_bp->bpid))
 553		return 0;
 554
 555	if (dpaa_bp->bpid == FSL_DPAA_BPID_INV) {
 556		dpaa_bp->pool = bman_new_pool();
 557		if (!dpaa_bp->pool) {
 558			pr_err("%s: bman_new_pool() failed\n",
 559			       __func__);
 560			return -ENODEV;
 561		}
 562
 563		dpaa_bp->bpid = (u8)bman_get_bpid(dpaa_bp->pool);
 564	}
 565
 566	if (dpaa_bp->seed_cb) {
 567		err = dpaa_bp->seed_cb(dpaa_bp);
 568		if (err)
 569			goto pool_seed_failed;
 570	}
 571
 572	dpaa_bpid2pool_map(dpaa_bp->bpid, dpaa_bp);
 573
 574	return 0;
 575
 576pool_seed_failed:
 577	pr_err("%s: pool seeding failed\n", __func__);
 578	bman_free_pool(dpaa_bp->pool);
 579
 580	return err;
 581}
 582
 583/* remove and free all the buffers from the given buffer pool */
 584static void dpaa_bp_drain(struct dpaa_bp *bp)
 585{
 586	u8 num = 8;
 587	int ret;
 588
 589	do {
 590		struct bm_buffer bmb[8];
 591		int i;
 592
 593		ret = bman_acquire(bp->pool, bmb, num);
 594		if (ret < 0) {
 595			if (num == 8) {
 596				/* we have less than 8 buffers left;
 597				 * drain them one by one
 598				 */
 599				num = 1;
 600				ret = 1;
 601				continue;
 602			} else {
 603				/* Pool is fully drained */
 604				break;
 605			}
 606		}
 607
 608		if (bp->free_buf_cb)
 609			for (i = 0; i < num; i++)
 610				bp->free_buf_cb(bp, &bmb[i]);
 611	} while (ret > 0);
 612}
 613
 614static void dpaa_bp_free(struct dpaa_bp *dpaa_bp)
 615{
 616	struct dpaa_bp *bp = dpaa_bpid2pool(dpaa_bp->bpid);
 617
 618	/* the mapping between bpid and dpaa_bp is done very late in the
 619	 * allocation procedure; if something failed before the mapping, the bp
 620	 * was not configured, therefore we don't need the below instructions
 621	 */
 622	if (!bp)
 623		return;
 624
 625	if (!refcount_dec_and_test(&bp->refs))
 626		return;
 627
 628	if (bp->free_buf_cb)
 629		dpaa_bp_drain(bp);
 630
 631	dpaa_bp_array[bp->bpid] = NULL;
 632	bman_free_pool(bp->pool);
 633}
 634
 635static void dpaa_bps_free(struct dpaa_priv *priv)
 636{
 637	dpaa_bp_free(priv->dpaa_bp);
 638}
 639
 640/* Use multiple WQs for FQ assignment:
 641 *	- Tx Confirmation queues go to WQ1.
 642 *	- Rx Error and Tx Error queues go to WQ5 (giving them a better chance
 643 *	  to be scheduled, in case there are many more FQs in WQ6).
 644 *	- Rx Default goes to WQ6.
 645 *	- Tx queues go to different WQs depending on their priority. Equal
 646 *	  chunks of NR_CPUS queues go to WQ6 (lowest priority), WQ2, WQ1 and
 647 *	  WQ0 (highest priority).
 648 * This ensures that Tx-confirmed buffers are timely released. In particular,
 649 * it avoids congestion on the Tx Confirm FQs, which can pile up PFDRs if they
 650 * are greatly outnumbered by other FQs in the system, while
 651 * dequeue scheduling is round-robin.
 652 */
 653static inline void dpaa_assign_wq(struct dpaa_fq *fq, int idx)
 654{
 655	switch (fq->fq_type) {
 656	case FQ_TYPE_TX_CONFIRM:
 657	case FQ_TYPE_TX_CONF_MQ:
 658		fq->wq = 1;
 659		break;
 660	case FQ_TYPE_RX_ERROR:
 661	case FQ_TYPE_TX_ERROR:
 662		fq->wq = 5;
 663		break;
 664	case FQ_TYPE_RX_DEFAULT:
 665	case FQ_TYPE_RX_PCD:
 666		fq->wq = 6;
 667		break;
 668	case FQ_TYPE_TX:
 669		switch (idx / dpaa_num_txqs_per_tc()) {
 670		case 0:
 671			/* Low priority (best effort) */
 672			fq->wq = 6;
 673			break;
 674		case 1:
 675			/* Medium priority */
 676			fq->wq = 2;
 677			break;
 678		case 2:
 679			/* High priority */
 680			fq->wq = 1;
 681			break;
 682		case 3:
 683			/* Very high priority */
 684			fq->wq = 0;
 685			break;
 686		default:
 687			WARN(1, "Too many TX FQs: more than %zu!\n",
 688			     dpaa_max_num_txqs());
 689		}
 690		break;
 691	default:
 692		WARN(1, "Invalid FQ type %d for FQID %d!\n",
 693		     fq->fq_type, fq->fqid);
 694	}
 695}
 696
 697static struct dpaa_fq *dpaa_fq_alloc(struct device *dev,
 698				     u32 start, u32 count,
 699				     struct list_head *list,
 700				     enum dpaa_fq_type fq_type)
 701{
 702	struct dpaa_fq *dpaa_fq;
 703	int i;
 704
 705	dpaa_fq = devm_kcalloc(dev, count, sizeof(*dpaa_fq),
 706			       GFP_KERNEL);
 707	if (!dpaa_fq)
 708		return NULL;
 709
 710	for (i = 0; i < count; i++) {
 711		dpaa_fq[i].fq_type = fq_type;
 712		dpaa_fq[i].fqid = start ? start + i : 0;
 713		list_add_tail(&dpaa_fq[i].list, list);
 714	}
 715
 716	for (i = 0; i < count; i++)
 717		dpaa_assign_wq(dpaa_fq + i, i);
 718
 719	return dpaa_fq;
 720}
 721
 722static int dpaa_alloc_all_fqs(struct device *dev, struct list_head *list,
 723			      struct fm_port_fqs *port_fqs)
 724{
 725	struct dpaa_fq *dpaa_fq;
 726	u32 fq_base, fq_base_aligned, i;
 727
 728	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_ERROR);
 729	if (!dpaa_fq)
 730		goto fq_alloc_failed;
 731
 732	port_fqs->rx_errq = &dpaa_fq[0];
 733
 734	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_RX_DEFAULT);
 735	if (!dpaa_fq)
 736		goto fq_alloc_failed;
 737
 738	port_fqs->rx_defq = &dpaa_fq[0];
 739
 740	/* the PCD FQIDs range needs to be aligned for correct operation */
 741	if (qman_alloc_fqid_range(&fq_base, 2 * DPAA_ETH_PCD_RXQ_NUM))
 742		goto fq_alloc_failed;
 743
 744	fq_base_aligned = ALIGN(fq_base, DPAA_ETH_PCD_RXQ_NUM);
 745
 746	for (i = fq_base; i < fq_base_aligned; i++)
 747		qman_release_fqid(i);
 748
 749	for (i = fq_base_aligned + DPAA_ETH_PCD_RXQ_NUM;
 750	     i < (fq_base + 2 * DPAA_ETH_PCD_RXQ_NUM); i++)
 751		qman_release_fqid(i);
 752
 753	dpaa_fq = dpaa_fq_alloc(dev, fq_base_aligned, DPAA_ETH_PCD_RXQ_NUM,
 754				list, FQ_TYPE_RX_PCD);
 755	if (!dpaa_fq)
 756		goto fq_alloc_failed;
 757
 758	port_fqs->rx_pcdq = &dpaa_fq[0];
 759
 760	if (!dpaa_fq_alloc(dev, 0, dpaa_max_num_txqs(), list,
 761			   FQ_TYPE_TX_CONF_MQ))
 762		goto fq_alloc_failed;
 763
 764	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_TX_ERROR);
 765	if (!dpaa_fq)
 766		goto fq_alloc_failed;
 767
 768	port_fqs->tx_errq = &dpaa_fq[0];
 769
 770	dpaa_fq = dpaa_fq_alloc(dev, 0, 1, list, FQ_TYPE_TX_CONFIRM);
 771	if (!dpaa_fq)
 772		goto fq_alloc_failed;
 773
 774	port_fqs->tx_defq = &dpaa_fq[0];
 775
 776	if (!dpaa_fq_alloc(dev, 0, dpaa_max_num_txqs(), list, FQ_TYPE_TX))
 777		goto fq_alloc_failed;
 778
 779	return 0;
 780
 781fq_alloc_failed:
 782	dev_err(dev, "dpaa_fq_alloc() failed\n");
 783	return -ENOMEM;
 784}
 785
 786static u32 rx_pool_channel;
 787static DEFINE_SPINLOCK(rx_pool_channel_init);
 788
 789static int dpaa_get_channel(void)
 790{
 791	spin_lock(&rx_pool_channel_init);
 792	if (!rx_pool_channel) {
 793		u32 pool;
 794		int ret;
 795
 796		ret = qman_alloc_pool(&pool);
 797
 798		if (!ret)
 799			rx_pool_channel = pool;
 800	}
 801	spin_unlock(&rx_pool_channel_init);
 802	if (!rx_pool_channel)
 803		return -ENOMEM;
 804	return rx_pool_channel;
 805}
 806
 807static void dpaa_release_channel(void)
 808{
 809	qman_release_pool(rx_pool_channel);
 810}
 811
 812static void dpaa_eth_add_channel(u16 channel, struct device *dev)
 813{
 814	u32 pool = QM_SDQCR_CHANNELS_POOL_CONV(channel);
 815	const cpumask_t *cpus = qman_affine_cpus();
 816	struct qman_portal *portal;
 817	int cpu;
 818
 819	for_each_cpu_and(cpu, cpus, cpu_online_mask) {
 820		portal = qman_get_affine_portal(cpu);
 821		qman_p_static_dequeue_add(portal, pool);
 822		qman_start_using_portal(portal, dev);
 823	}
 824}
 825
 826/* Congestion group state change notification callback.
 827 * Stops the device's egress queues while they are congested and
 828 * wakes them upon exiting congested state.
 829 * Also updates some CGR-related stats.
 830 */
 831static void dpaa_eth_cgscn(struct qman_portal *qm, struct qman_cgr *cgr,
 832			   int congested)
 833{
 834	struct dpaa_priv *priv = (struct dpaa_priv *)container_of(cgr,
 835		struct dpaa_priv, cgr_data.cgr);
 836
 837	if (congested) {
 838		priv->cgr_data.congestion_start_jiffies = jiffies;
 839		netif_tx_stop_all_queues(priv->net_dev);
 840		priv->cgr_data.cgr_congested_count++;
 841	} else {
 842		priv->cgr_data.congested_jiffies +=
 843			(jiffies - priv->cgr_data.congestion_start_jiffies);
 844		netif_tx_wake_all_queues(priv->net_dev);
 845	}
 846}
 847
 848static int dpaa_eth_cgr_init(struct dpaa_priv *priv)
 849{
 850	struct qm_mcc_initcgr initcgr;
 851	u32 cs_th;
 852	int err;
 853
 854	err = qman_alloc_cgrid(&priv->cgr_data.cgr.cgrid);
 855	if (err < 0) {
 856		if (netif_msg_drv(priv))
 857			pr_err("%s: Error %d allocating CGR ID\n",
 858			       __func__, err);
 859		goto out_error;
 860	}
 861	priv->cgr_data.cgr.cb = dpaa_eth_cgscn;
 862
 863	/* Enable Congestion State Change Notifications and CS taildrop */
 864	memset(&initcgr, 0, sizeof(initcgr));
 865	initcgr.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES);
 866	initcgr.cgr.cscn_en = QM_CGR_EN;
 867
 868	/* Set different thresholds based on the configured MAC speed.
 869	 * This may turn suboptimal if the MAC is reconfigured at another
 870	 * speed, so MACs must call dpaa_eth_cgr_set_speed in their link_up
 871	 * callback.
 872	 */
 873	if (priv->mac_dev->phylink_config.mac_capabilities & MAC_10000FD)
 874		cs_th = DPAA_CS_THRESHOLD_10G;
 875	else
 876		cs_th = DPAA_CS_THRESHOLD_1G;
 877	qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1);
 878
 879	initcgr.we_mask |= cpu_to_be16(QM_CGR_WE_CSTD_EN);
 880	initcgr.cgr.cstd_en = QM_CGR_EN;
 881
 882	err = qman_create_cgr(&priv->cgr_data.cgr, QMAN_CGR_FLAG_USE_INIT,
 883			      &initcgr);
 884	if (err < 0) {
 885		if (netif_msg_drv(priv))
 886			pr_err("%s: Error %d creating CGR with ID %d\n",
 887			       __func__, err, priv->cgr_data.cgr.cgrid);
 888		qman_release_cgrid(priv->cgr_data.cgr.cgrid);
 889		goto out_error;
 890	}
 891	if (netif_msg_drv(priv))
 892		pr_debug("Created CGR %d for netdev with hwaddr %pM on QMan channel %d\n",
 893			 priv->cgr_data.cgr.cgrid, priv->mac_dev->addr,
 894			 priv->cgr_data.cgr.chan);
 895
 896out_error:
 897	return err;
 898}
 899
 900static void dpaa_eth_cgr_set_speed(struct mac_device *mac_dev, int speed)
 901{
 902	struct net_device *net_dev = to_net_dev(mac_dev->phylink_config.dev);
 903	struct dpaa_priv *priv = netdev_priv(net_dev);
 904	struct qm_mcc_initcgr opts = { };
 905	u32 cs_th;
 906	int err;
 907
 908	opts.we_mask = cpu_to_be16(QM_CGR_WE_CS_THRES);
 909	switch (speed) {
 910	case SPEED_10000:
 911		cs_th = DPAA_CS_THRESHOLD_10G;
 912		break;
 913	case SPEED_1000:
 914	default:
 915		cs_th = DPAA_CS_THRESHOLD_1G;
 916		break;
 917	}
 918	qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, cs_th, 1);
 919
 920	err = qman_update_cgr_safe(&priv->cgr_data.cgr, &opts);
 921	if (err)
 922		netdev_err(net_dev, "could not update speed: %d\n", err);
 923}
 924
 925static inline void dpaa_setup_ingress(const struct dpaa_priv *priv,
 926				      struct dpaa_fq *fq,
 927				      const struct qman_fq *template)
 928{
 929	fq->fq_base = *template;
 930	fq->net_dev = priv->net_dev;
 931
 932	fq->flags = QMAN_FQ_FLAG_NO_ENQUEUE;
 933	fq->channel = priv->channel;
 934}
 935
 936static inline void dpaa_setup_egress(const struct dpaa_priv *priv,
 937				     struct dpaa_fq *fq,
 938				     struct fman_port *port,
 939				     const struct qman_fq *template)
 940{
 941	fq->fq_base = *template;
 942	fq->net_dev = priv->net_dev;
 943
 944	if (port) {
 945		fq->flags = QMAN_FQ_FLAG_TO_DCPORTAL;
 946		fq->channel = (u16)fman_port_get_qman_channel_id(port);
 947	} else {
 948		fq->flags = QMAN_FQ_FLAG_NO_MODIFY;
 949	}
 950}
 951
 952static int dpaa_fq_setup(struct dpaa_priv *priv,
 953			 const struct dpaa_fq_cbs *fq_cbs,
 954			 struct fman_port *tx_port)
 955{
 956	int egress_cnt = 0, conf_cnt = 0, num_portals = 0, portal_cnt = 0, cpu;
 957	const cpumask_t *affine_cpus = qman_affine_cpus();
 958	struct dpaa_fq *fq;
 959	u16 *channels;
 960
 961	channels = kcalloc(num_possible_cpus(), sizeof(u16), GFP_KERNEL);
 962	if (!channels)
 963		return -ENOMEM;
 964
 965	for_each_cpu_and(cpu, affine_cpus, cpu_online_mask)
 966		channels[num_portals++] = qman_affine_channel(cpu);
 967
 968	if (num_portals == 0)
 969		dev_err(priv->net_dev->dev.parent,
 970			"No Qman software (affine) channels found\n");
 971
 972	/* Initialize each FQ in the list */
 973	list_for_each_entry(fq, &priv->dpaa_fq_list, list) {
 974		switch (fq->fq_type) {
 975		case FQ_TYPE_RX_DEFAULT:
 976			dpaa_setup_ingress(priv, fq, &fq_cbs->rx_defq);
 977			break;
 978		case FQ_TYPE_RX_ERROR:
 979			dpaa_setup_ingress(priv, fq, &fq_cbs->rx_errq);
 980			break;
 981		case FQ_TYPE_RX_PCD:
 982			if (!num_portals)
 983				continue;
 984			dpaa_setup_ingress(priv, fq, &fq_cbs->rx_defq);
 985			fq->channel = channels[portal_cnt++ % num_portals];
 986			break;
 987		case FQ_TYPE_TX:
 988			dpaa_setup_egress(priv, fq, tx_port,
 989					  &fq_cbs->egress_ern);
 990			priv->egress_fqs[egress_cnt++] = &fq->fq_base;
 991			break;
 992		case FQ_TYPE_TX_CONF_MQ:
 993			priv->conf_fqs[conf_cnt++] = &fq->fq_base;
 994			fallthrough;
 995		case FQ_TYPE_TX_CONFIRM:
 996			dpaa_setup_ingress(priv, fq, &fq_cbs->tx_defq);
 997			break;
 998		case FQ_TYPE_TX_ERROR:
 999			dpaa_setup_ingress(priv, fq, &fq_cbs->tx_errq);
1000			break;
1001		default:
1002			dev_warn(priv->net_dev->dev.parent,
1003				 "Unknown FQ type detected!\n");
1004			break;
1005		}
1006	}
1007
1008	kfree(channels);
1009
1010	return 0;
1011}
1012
1013static inline int dpaa_tx_fq_to_id(const struct dpaa_priv *priv,
1014				   struct qman_fq *tx_fq)
1015{
1016	int i;
1017
1018	for (i = 0; i < dpaa_max_num_txqs(); i++)
1019		if (priv->egress_fqs[i] == tx_fq)
1020			return i;
1021
1022	return -EINVAL;
1023}
1024
1025static int dpaa_fq_init(struct dpaa_fq *dpaa_fq, bool td_enable)
1026{
1027	const struct dpaa_priv	*priv;
1028	struct qman_fq *confq = NULL;
1029	struct qm_mcc_initfq initfq;
1030	struct device *dev;
1031	struct qman_fq *fq;
1032	int queue_id;
1033	int err;
1034
1035	priv = netdev_priv(dpaa_fq->net_dev);
1036	dev = dpaa_fq->net_dev->dev.parent;
1037
1038	if (dpaa_fq->fqid == 0)
1039		dpaa_fq->flags |= QMAN_FQ_FLAG_DYNAMIC_FQID;
1040
1041	dpaa_fq->init = !(dpaa_fq->flags & QMAN_FQ_FLAG_NO_MODIFY);
1042
1043	err = qman_create_fq(dpaa_fq->fqid, dpaa_fq->flags, &dpaa_fq->fq_base);
1044	if (err) {
1045		dev_err(dev, "qman_create_fq() failed\n");
1046		return err;
1047	}
1048	fq = &dpaa_fq->fq_base;
1049
1050	if (dpaa_fq->init) {
1051		memset(&initfq, 0, sizeof(initfq));
1052
1053		initfq.we_mask = cpu_to_be16(QM_INITFQ_WE_FQCTRL);
1054		/* Note: we may get to keep an empty FQ in cache */
1055		initfq.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_PREFERINCACHE);
1056
1057		/* Try to reduce the number of portal interrupts for
1058		 * Tx Confirmation FQs.
1059		 */
1060		if (dpaa_fq->fq_type == FQ_TYPE_TX_CONFIRM)
1061			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_AVOIDBLOCK);
1062
1063		/* FQ placement */
1064		initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_DESTWQ);
1065
1066		qm_fqd_set_destwq(&initfq.fqd, dpaa_fq->channel, dpaa_fq->wq);
1067
1068		/* Put all egress queues in a congestion group of their own.
1069		 * Sensu stricto, the Tx confirmation queues are Rx FQs,
1070		 * rather than Tx - but they nonetheless account for the
1071		 * memory footprint on behalf of egress traffic. We therefore
1072		 * place them in the netdev's CGR, along with the Tx FQs.
1073		 */
1074		if (dpaa_fq->fq_type == FQ_TYPE_TX ||
1075		    dpaa_fq->fq_type == FQ_TYPE_TX_CONFIRM ||
1076		    dpaa_fq->fq_type == FQ_TYPE_TX_CONF_MQ) {
1077			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CGID);
1078			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_CGE);
1079			initfq.fqd.cgid = (u8)priv->cgr_data.cgr.cgrid;
1080			/* Set a fixed overhead accounting, in an attempt to
1081			 * reduce the impact of fixed-size skb shells and the
1082			 * driver's needed headroom on system memory. This is
1083			 * especially the case when the egress traffic is
1084			 * composed of small datagrams.
1085			 * Unfortunately, QMan's OAL value is capped to an
1086			 * insufficient value, but even that is better than
1087			 * no overhead accounting at all.
1088			 */
1089			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_OAC);
1090			qm_fqd_set_oac(&initfq.fqd, QM_OAC_CG);
1091			qm_fqd_set_oal(&initfq.fqd,
1092				       min(sizeof(struct sk_buff) +
1093				       priv->tx_headroom,
1094				       (size_t)FSL_QMAN_MAX_OAL));
1095		}
1096
1097		if (td_enable) {
1098			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_TDTHRESH);
1099			qm_fqd_set_taildrop(&initfq.fqd, DPAA_FQ_TD, 1);
1100			initfq.fqd.fq_ctrl = cpu_to_be16(QM_FQCTRL_TDE);
1101		}
1102
1103		if (dpaa_fq->fq_type == FQ_TYPE_TX) {
1104			queue_id = dpaa_tx_fq_to_id(priv, &dpaa_fq->fq_base);
1105			if (queue_id >= 0)
1106				confq = priv->conf_fqs[queue_id];
1107			if (confq) {
1108				initfq.we_mask |=
1109					cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
1110			/* ContextA: OVOM=1(use contextA2 bits instead of ICAD)
1111			 *	     A2V=1 (contextA A2 field is valid)
1112			 *	     A0V=1 (contextA A0 field is valid)
1113			 *	     B0V=1 (contextB field is valid)
1114			 * ContextA A2: EBD=1 (deallocate buffers inside FMan)
1115			 * ContextB B0(ASPID): 0 (absolute Virtual Storage ID)
1116			 */
1117				qm_fqd_context_a_set64(&initfq.fqd,
1118						       0x1e00000080000000ULL);
1119			}
1120		}
1121
1122		/* Put all the ingress queues in our "ingress CGR". */
1123		if (priv->use_ingress_cgr &&
1124		    (dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
1125		     dpaa_fq->fq_type == FQ_TYPE_RX_ERROR ||
1126		     dpaa_fq->fq_type == FQ_TYPE_RX_PCD)) {
1127			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CGID);
1128			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_CGE);
1129			initfq.fqd.cgid = (u8)priv->ingress_cgr.cgrid;
1130			/* Set a fixed overhead accounting, just like for the
1131			 * egress CGR.
1132			 */
1133			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_OAC);
1134			qm_fqd_set_oac(&initfq.fqd, QM_OAC_CG);
1135			qm_fqd_set_oal(&initfq.fqd,
1136				       min(sizeof(struct sk_buff) +
1137				       priv->tx_headroom,
1138				       (size_t)FSL_QMAN_MAX_OAL));
1139		}
1140
1141		/* Initialization common to all ingress queues */
1142		if (dpaa_fq->flags & QMAN_FQ_FLAG_NO_ENQUEUE) {
1143			initfq.we_mask |= cpu_to_be16(QM_INITFQ_WE_CONTEXTA);
1144			initfq.fqd.fq_ctrl |= cpu_to_be16(QM_FQCTRL_HOLDACTIVE |
1145						QM_FQCTRL_CTXASTASHING);
1146			initfq.fqd.context_a.stashing.exclusive =
1147				QM_STASHING_EXCL_DATA | QM_STASHING_EXCL_CTX |
1148				QM_STASHING_EXCL_ANNOTATION;
1149			qm_fqd_set_stashing(&initfq.fqd, 1, 2,
1150					    DIV_ROUND_UP(sizeof(struct qman_fq),
1151							 64));
1152		}
1153
1154		err = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &initfq);
1155		if (err < 0) {
1156			dev_err(dev, "qman_init_fq(%u) = %d\n",
1157				qman_fq_fqid(fq), err);
1158			qman_destroy_fq(fq);
1159			return err;
1160		}
1161	}
1162
1163	dpaa_fq->fqid = qman_fq_fqid(fq);
1164
1165	if (dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
1166	    dpaa_fq->fq_type == FQ_TYPE_RX_PCD) {
1167		err = xdp_rxq_info_reg(&dpaa_fq->xdp_rxq, dpaa_fq->net_dev,
1168				       dpaa_fq->fqid, 0);
1169		if (err) {
1170			dev_err(dev, "xdp_rxq_info_reg() = %d\n", err);
1171			return err;
1172		}
1173
1174		err = xdp_rxq_info_reg_mem_model(&dpaa_fq->xdp_rxq,
1175						 MEM_TYPE_PAGE_ORDER0, NULL);
1176		if (err) {
1177			dev_err(dev, "xdp_rxq_info_reg_mem_model() = %d\n",
1178				err);
1179			xdp_rxq_info_unreg(&dpaa_fq->xdp_rxq);
1180			return err;
1181		}
1182	}
1183
1184	return 0;
1185}
1186
1187static int dpaa_fq_free_entry(struct device *dev, struct qman_fq *fq)
1188{
1189	const struct dpaa_priv  *priv;
1190	struct dpaa_fq *dpaa_fq;
1191	int err, error;
1192
1193	err = 0;
1194
1195	dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
1196	priv = netdev_priv(dpaa_fq->net_dev);
1197
1198	if (dpaa_fq->init) {
1199		err = qman_retire_fq(fq, NULL);
1200		if (err < 0 && netif_msg_drv(priv))
1201			dev_err(dev, "qman_retire_fq(%u) = %d\n",
1202				qman_fq_fqid(fq), err);
1203
1204		error = qman_oos_fq(fq);
1205		if (error < 0 && netif_msg_drv(priv)) {
1206			dev_err(dev, "qman_oos_fq(%u) = %d\n",
1207				qman_fq_fqid(fq), error);
1208			if (err >= 0)
1209				err = error;
1210		}
1211	}
1212
1213	if ((dpaa_fq->fq_type == FQ_TYPE_RX_DEFAULT ||
1214	     dpaa_fq->fq_type == FQ_TYPE_RX_PCD) &&
1215	    xdp_rxq_info_is_reg(&dpaa_fq->xdp_rxq))
1216		xdp_rxq_info_unreg(&dpaa_fq->xdp_rxq);
1217
1218	qman_destroy_fq(fq);
1219	list_del(&dpaa_fq->list);
1220
1221	return err;
1222}
1223
1224static int dpaa_fq_free(struct device *dev, struct list_head *list)
1225{
1226	struct dpaa_fq *dpaa_fq, *tmp;
1227	int err, error;
1228
1229	err = 0;
1230	list_for_each_entry_safe(dpaa_fq, tmp, list, list) {
1231		error = dpaa_fq_free_entry(dev, (struct qman_fq *)dpaa_fq);
1232		if (error < 0 && err >= 0)
1233			err = error;
1234	}
1235
1236	return err;
1237}
1238
1239static int dpaa_eth_init_tx_port(struct fman_port *port, struct dpaa_fq *errq,
1240				 struct dpaa_fq *defq,
1241				 struct dpaa_buffer_layout *buf_layout)
1242{
1243	struct fman_buffer_prefix_content buf_prefix_content;
1244	struct fman_port_params params;
1245	int err;
1246
1247	memset(&params, 0, sizeof(params));
1248	memset(&buf_prefix_content, 0, sizeof(buf_prefix_content));
1249
1250	buf_prefix_content.priv_data_size = buf_layout->priv_data_size;
1251	buf_prefix_content.pass_prs_result = true;
1252	buf_prefix_content.pass_hash_result = true;
1253	buf_prefix_content.pass_time_stamp = true;
1254	buf_prefix_content.data_align = DPAA_FD_DATA_ALIGNMENT;
1255
1256	params.specific_params.non_rx_params.err_fqid = errq->fqid;
1257	params.specific_params.non_rx_params.dflt_fqid = defq->fqid;
1258
1259	err = fman_port_config(port, &params);
1260	if (err) {
1261		pr_err("%s: fman_port_config failed\n", __func__);
1262		return err;
1263	}
1264
1265	err = fman_port_cfg_buf_prefix_content(port, &buf_prefix_content);
1266	if (err) {
1267		pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1268		       __func__);
1269		return err;
1270	}
1271
1272	err = fman_port_init(port);
1273	if (err)
1274		pr_err("%s: fm_port_init failed\n", __func__);
1275
1276	return err;
1277}
1278
1279static int dpaa_eth_init_rx_port(struct fman_port *port, struct dpaa_bp *bp,
1280				 struct dpaa_fq *errq,
1281				 struct dpaa_fq *defq, struct dpaa_fq *pcdq,
1282				 struct dpaa_buffer_layout *buf_layout)
1283{
1284	struct fman_buffer_prefix_content buf_prefix_content;
1285	struct fman_port_rx_params *rx_p;
1286	struct fman_port_params params;
1287	int err;
1288
1289	memset(&params, 0, sizeof(params));
1290	memset(&buf_prefix_content, 0, sizeof(buf_prefix_content));
1291
1292	buf_prefix_content.priv_data_size = buf_layout->priv_data_size;
1293	buf_prefix_content.pass_prs_result = true;
1294	buf_prefix_content.pass_hash_result = true;
1295	buf_prefix_content.pass_time_stamp = true;
1296	buf_prefix_content.data_align = DPAA_FD_RX_DATA_ALIGNMENT;
1297
1298	rx_p = &params.specific_params.rx_params;
1299	rx_p->err_fqid = errq->fqid;
1300	rx_p->dflt_fqid = defq->fqid;
1301	if (pcdq) {
1302		rx_p->pcd_base_fqid = pcdq->fqid;
1303		rx_p->pcd_fqs_count = DPAA_ETH_PCD_RXQ_NUM;
1304	}
1305
1306	rx_p->ext_buf_pools.num_of_pools_used = 1;
1307	rx_p->ext_buf_pools.ext_buf_pool[0].id =  bp->bpid;
1308	rx_p->ext_buf_pools.ext_buf_pool[0].size = (u16)bp->size;
1309
1310	err = fman_port_config(port, &params);
1311	if (err) {
1312		pr_err("%s: fman_port_config failed\n", __func__);
1313		return err;
1314	}
1315
1316	err = fman_port_cfg_buf_prefix_content(port, &buf_prefix_content);
1317	if (err) {
1318		pr_err("%s: fman_port_cfg_buf_prefix_content failed\n",
1319		       __func__);
1320		return err;
1321	}
1322
1323	err = fman_port_init(port);
1324	if (err)
1325		pr_err("%s: fm_port_init failed\n", __func__);
1326
1327	return err;
1328}
1329
1330static int dpaa_eth_init_ports(struct mac_device *mac_dev,
1331			       struct dpaa_bp *bp,
1332			       struct fm_port_fqs *port_fqs,
1333			       struct dpaa_buffer_layout *buf_layout,
1334			       struct device *dev)
1335{
1336	struct fman_port *rxport = mac_dev->port[RX];
1337	struct fman_port *txport = mac_dev->port[TX];
1338	int err;
1339
1340	err = dpaa_eth_init_tx_port(txport, port_fqs->tx_errq,
1341				    port_fqs->tx_defq, &buf_layout[TX]);
1342	if (err)
1343		return err;
1344
1345	err = dpaa_eth_init_rx_port(rxport, bp, port_fqs->rx_errq,
1346				    port_fqs->rx_defq, port_fqs->rx_pcdq,
1347				    &buf_layout[RX]);
1348
1349	return err;
1350}
1351
1352static int dpaa_bman_release(const struct dpaa_bp *dpaa_bp,
1353			     struct bm_buffer *bmb, int cnt)
1354{
1355	int err;
1356
1357	err = bman_release(dpaa_bp->pool, bmb, cnt);
1358	/* Should never occur, address anyway to avoid leaking the buffers */
1359	if (WARN_ON(err) && dpaa_bp->free_buf_cb)
1360		while (cnt-- > 0)
1361			dpaa_bp->free_buf_cb(dpaa_bp, &bmb[cnt]);
1362
1363	return cnt;
1364}
1365
1366static void dpaa_release_sgt_members(struct qm_sg_entry *sgt)
1367{
1368	struct bm_buffer bmb[DPAA_BUFF_RELEASE_MAX];
1369	struct dpaa_bp *dpaa_bp;
1370	int i = 0, j;
1371
1372	memset(bmb, 0, sizeof(bmb));
1373
1374	do {
1375		dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
1376		if (!dpaa_bp)
1377			return;
1378
1379		j = 0;
1380		do {
1381			WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1382
1383			bm_buffer_set64(&bmb[j], qm_sg_entry_get64(&sgt[i]));
1384
1385			j++; i++;
1386		} while (j < ARRAY_SIZE(bmb) &&
1387				!qm_sg_entry_is_final(&sgt[i - 1]) &&
1388				sgt[i - 1].bpid == sgt[i].bpid);
1389
1390		dpaa_bman_release(dpaa_bp, bmb, j);
1391	} while (!qm_sg_entry_is_final(&sgt[i - 1]));
1392}
1393
1394static void dpaa_fd_release(const struct net_device *net_dev,
1395			    const struct qm_fd *fd)
1396{
1397	struct qm_sg_entry *sgt;
1398	struct dpaa_bp *dpaa_bp;
1399	struct bm_buffer bmb;
1400	dma_addr_t addr;
1401	void *vaddr;
1402
1403	bmb.data = 0;
1404	bm_buffer_set64(&bmb, qm_fd_addr(fd));
1405
1406	dpaa_bp = dpaa_bpid2pool(fd->bpid);
1407	if (!dpaa_bp)
1408		return;
1409
1410	if (qm_fd_get_format(fd) == qm_fd_sg) {
1411		vaddr = phys_to_virt(qm_fd_addr(fd));
1412		sgt = vaddr + qm_fd_get_offset(fd);
1413
1414		dma_unmap_page(dpaa_bp->priv->rx_dma_dev, qm_fd_addr(fd),
1415			       DPAA_BP_RAW_SIZE, DMA_FROM_DEVICE);
1416
1417		dpaa_release_sgt_members(sgt);
1418
1419		addr = dma_map_page(dpaa_bp->priv->rx_dma_dev,
1420				    virt_to_page(vaddr), 0, DPAA_BP_RAW_SIZE,
1421				    DMA_FROM_DEVICE);
1422		if (dma_mapping_error(dpaa_bp->priv->rx_dma_dev, addr)) {
1423			netdev_err(net_dev, "DMA mapping failed\n");
1424			return;
1425		}
1426		bm_buffer_set64(&bmb, addr);
1427	}
1428
1429	dpaa_bman_release(dpaa_bp, &bmb, 1);
1430}
1431
1432static void count_ern(struct dpaa_percpu_priv *percpu_priv,
1433		      const union qm_mr_entry *msg)
1434{
1435	switch (msg->ern.rc & QM_MR_RC_MASK) {
1436	case QM_MR_RC_CGR_TAILDROP:
1437		percpu_priv->ern_cnt.cg_tdrop++;
1438		break;
1439	case QM_MR_RC_WRED:
1440		percpu_priv->ern_cnt.wred++;
1441		break;
1442	case QM_MR_RC_ERROR:
1443		percpu_priv->ern_cnt.err_cond++;
1444		break;
1445	case QM_MR_RC_ORPWINDOW_EARLY:
1446		percpu_priv->ern_cnt.early_window++;
1447		break;
1448	case QM_MR_RC_ORPWINDOW_LATE:
1449		percpu_priv->ern_cnt.late_window++;
1450		break;
1451	case QM_MR_RC_FQ_TAILDROP:
1452		percpu_priv->ern_cnt.fq_tdrop++;
1453		break;
1454	case QM_MR_RC_ORPWINDOW_RETIRED:
1455		percpu_priv->ern_cnt.fq_retired++;
1456		break;
1457	case QM_MR_RC_ORP_ZERO:
1458		percpu_priv->ern_cnt.orp_zero++;
1459		break;
1460	}
1461}
1462
1463/* Turn on HW checksum computation for this outgoing frame.
1464 * If the current protocol is not something we support in this regard
1465 * (or if the stack has already computed the SW checksum), we do nothing.
1466 *
1467 * Returns 0 if all goes well (or HW csum doesn't apply), and a negative value
1468 * otherwise.
1469 *
1470 * Note that this function may modify the fd->cmd field and the skb data buffer
1471 * (the Parse Results area).
1472 */
1473static int dpaa_enable_tx_csum(struct dpaa_priv *priv,
1474			       struct sk_buff *skb,
1475			       struct qm_fd *fd,
1476			       void *parse_results)
1477{
1478	struct fman_prs_result *parse_result;
1479	u16 ethertype = ntohs(skb->protocol);
1480	struct ipv6hdr *ipv6h = NULL;
1481	struct iphdr *iph;
1482	int retval = 0;
1483	u8 l4_proto;
1484
1485	if (skb->ip_summed != CHECKSUM_PARTIAL)
1486		return 0;
1487
1488	/* Note: L3 csum seems to be already computed in sw, but we can't choose
1489	 * L4 alone from the FM configuration anyway.
1490	 */
1491
1492	/* Fill in some fields of the Parse Results array, so the FMan
1493	 * can find them as if they came from the FMan Parser.
1494	 */
1495	parse_result = (struct fman_prs_result *)parse_results;
1496
1497	/* If we're dealing with VLAN, get the real Ethernet type */
1498	if (ethertype == ETH_P_8021Q)
1499		ethertype = ntohs(skb_vlan_eth_hdr(skb)->h_vlan_encapsulated_proto);
1500
1501	/* Fill in the relevant L3 parse result fields
1502	 * and read the L4 protocol type
1503	 */
1504	switch (ethertype) {
1505	case ETH_P_IP:
1506		parse_result->l3r = cpu_to_be16(FM_L3_PARSE_RESULT_IPV4);
1507		iph = ip_hdr(skb);
1508		WARN_ON(!iph);
1509		l4_proto = iph->protocol;
1510		break;
1511	case ETH_P_IPV6:
1512		parse_result->l3r = cpu_to_be16(FM_L3_PARSE_RESULT_IPV6);
1513		ipv6h = ipv6_hdr(skb);
1514		WARN_ON(!ipv6h);
1515		l4_proto = ipv6h->nexthdr;
1516		break;
1517	default:
1518		/* We shouldn't even be here */
1519		if (net_ratelimit())
1520			netif_alert(priv, tx_err, priv->net_dev,
1521				    "Can't compute HW csum for L3 proto 0x%x\n",
1522				    ntohs(skb->protocol));
1523		retval = -EIO;
1524		goto return_error;
1525	}
1526
1527	/* Fill in the relevant L4 parse result fields */
1528	switch (l4_proto) {
1529	case IPPROTO_UDP:
1530		parse_result->l4r = FM_L4_PARSE_RESULT_UDP;
1531		break;
1532	case IPPROTO_TCP:
1533		parse_result->l4r = FM_L4_PARSE_RESULT_TCP;
1534		break;
1535	default:
1536		if (net_ratelimit())
1537			netif_alert(priv, tx_err, priv->net_dev,
1538				    "Can't compute HW csum for L4 proto 0x%x\n",
1539				    l4_proto);
1540		retval = -EIO;
1541		goto return_error;
1542	}
1543
1544	/* At index 0 is IPOffset_1 as defined in the Parse Results */
1545	parse_result->ip_off[0] = (u8)skb_network_offset(skb);
1546	parse_result->l4_off = (u8)skb_transport_offset(skb);
1547
1548	/* Enable L3 (and L4, if TCP or UDP) HW checksum. */
1549	fd->cmd |= cpu_to_be32(FM_FD_CMD_RPD | FM_FD_CMD_DTC);
1550
1551	/* On P1023 and similar platforms fd->cmd interpretation could
1552	 * be disabled by setting CONTEXT_A bit ICMD; currently this bit
1553	 * is not set so we do not need to check; in the future, if/when
1554	 * using context_a we need to check this bit
1555	 */
1556
1557return_error:
1558	return retval;
1559}
1560
1561static int dpaa_bp_add_8_bufs(const struct dpaa_bp *dpaa_bp)
1562{
1563	struct net_device *net_dev = dpaa_bp->priv->net_dev;
1564	struct bm_buffer bmb[8];
1565	dma_addr_t addr;
1566	struct page *p;
1567	u8 i;
1568
1569	for (i = 0; i < 8; i++) {
1570		p = dev_alloc_pages(0);
1571		if (unlikely(!p)) {
1572			netdev_err(net_dev, "dev_alloc_pages() failed\n");
1573			goto release_previous_buffs;
1574		}
1575
1576		addr = dma_map_page(dpaa_bp->priv->rx_dma_dev, p, 0,
1577				    DPAA_BP_RAW_SIZE, DMA_FROM_DEVICE);
1578		if (unlikely(dma_mapping_error(dpaa_bp->priv->rx_dma_dev,
1579					       addr))) {
1580			netdev_err(net_dev, "DMA map failed\n");
1581			goto release_previous_buffs;
1582		}
1583
1584		bmb[i].data = 0;
1585		bm_buffer_set64(&bmb[i], addr);
1586	}
1587
1588release_bufs:
1589	return dpaa_bman_release(dpaa_bp, bmb, i);
1590
1591release_previous_buffs:
1592	WARN_ONCE(1, "dpaa_eth: failed to add buffers on Rx\n");
1593
1594	bm_buffer_set64(&bmb[i], 0);
1595	/* Avoid releasing a completely null buffer; bman_release() requires
1596	 * at least one buffer.
1597	 */
1598	if (likely(i))
1599		goto release_bufs;
1600
1601	return 0;
1602}
1603
1604static int dpaa_bp_seed(struct dpaa_bp *dpaa_bp)
1605{
1606	int i;
1607
1608	/* Give each CPU an allotment of "config_count" buffers */
1609	for_each_possible_cpu(i) {
1610		int *count_ptr = per_cpu_ptr(dpaa_bp->percpu_count, i);
1611		int j;
1612
1613		/* Although we access another CPU's counters here
1614		 * we do it at boot time so it is safe
1615		 */
1616		for (j = 0; j < dpaa_bp->config_count; j += 8)
1617			*count_ptr += dpaa_bp_add_8_bufs(dpaa_bp);
1618	}
1619	return 0;
1620}
1621
1622/* Add buffers/(pages) for Rx processing whenever bpool count falls below
1623 * REFILL_THRESHOLD.
1624 */
1625static int dpaa_eth_refill_bpool(struct dpaa_bp *dpaa_bp, int *countptr)
1626{
1627	int count = *countptr;
1628	int new_bufs;
1629
1630	if (unlikely(count < FSL_DPAA_ETH_REFILL_THRESHOLD)) {
1631		do {
1632			new_bufs = dpaa_bp_add_8_bufs(dpaa_bp);
1633			if (unlikely(!new_bufs)) {
1634				/* Avoid looping forever if we've temporarily
1635				 * run out of memory. We'll try again at the
1636				 * next NAPI cycle.
1637				 */
1638				break;
1639			}
1640			count += new_bufs;
1641		} while (count < FSL_DPAA_ETH_MAX_BUF_COUNT);
1642
1643		*countptr = count;
1644		if (unlikely(count < FSL_DPAA_ETH_MAX_BUF_COUNT))
1645			return -ENOMEM;
1646	}
1647
1648	return 0;
1649}
1650
1651static int dpaa_eth_refill_bpools(struct dpaa_priv *priv)
1652{
1653	struct dpaa_bp *dpaa_bp;
1654	int *countptr;
1655
1656	dpaa_bp = priv->dpaa_bp;
1657	if (!dpaa_bp)
1658		return -EINVAL;
1659	countptr = this_cpu_ptr(dpaa_bp->percpu_count);
1660
1661	return dpaa_eth_refill_bpool(dpaa_bp, countptr);
1662}
1663
1664/* Cleanup function for outgoing frame descriptors that were built on Tx path,
1665 * either contiguous frames or scatter/gather ones.
1666 * Skb freeing is not handled here.
1667 *
1668 * This function may be called on error paths in the Tx function, so guard
1669 * against cases when not all fd relevant fields were filled in. To avoid
1670 * reading the invalid transmission timestamp for the error paths set ts to
1671 * false.
1672 *
1673 * Return the skb backpointer, since for S/G frames the buffer containing it
1674 * gets freed here.
1675 *
1676 * No skb backpointer is set when transmitting XDP frames. Cleanup the buffer
1677 * and return NULL in this case.
1678 */
1679static struct sk_buff *dpaa_cleanup_tx_fd(const struct dpaa_priv *priv,
1680					  const struct qm_fd *fd, bool ts)
1681{
1682	const enum dma_data_direction dma_dir = DMA_TO_DEVICE;
1683	struct device *dev = priv->net_dev->dev.parent;
1684	struct skb_shared_hwtstamps shhwtstamps;
1685	dma_addr_t addr = qm_fd_addr(fd);
1686	void *vaddr = phys_to_virt(addr);
1687	const struct qm_sg_entry *sgt;
1688	struct dpaa_eth_swbp *swbp;
1689	struct sk_buff *skb;
1690	u64 ns;
1691	int i;
1692
1693	if (unlikely(qm_fd_get_format(fd) == qm_fd_sg)) {
1694		dma_unmap_page(priv->tx_dma_dev, addr,
1695			       qm_fd_get_offset(fd) + DPAA_SGT_SIZE,
1696			       dma_dir);
1697
1698		/* The sgt buffer has been allocated with netdev_alloc_frag(),
1699		 * it's from lowmem.
1700		 */
1701		sgt = vaddr + qm_fd_get_offset(fd);
1702
1703		/* sgt[0] is from lowmem, was dma_map_single()-ed */
1704		dma_unmap_single(priv->tx_dma_dev, qm_sg_addr(&sgt[0]),
1705				 qm_sg_entry_get_len(&sgt[0]), dma_dir);
1706
1707		/* remaining pages were mapped with skb_frag_dma_map() */
1708		for (i = 1; (i < DPAA_SGT_MAX_ENTRIES) &&
1709		     !qm_sg_entry_is_final(&sgt[i - 1]); i++) {
1710			WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1711
1712			dma_unmap_page(priv->tx_dma_dev, qm_sg_addr(&sgt[i]),
1713				       qm_sg_entry_get_len(&sgt[i]), dma_dir);
1714		}
1715	} else {
1716		dma_unmap_single(priv->tx_dma_dev, addr,
1717				 qm_fd_get_offset(fd) + qm_fd_get_length(fd),
1718				 dma_dir);
1719	}
1720
1721	swbp = (struct dpaa_eth_swbp *)vaddr;
1722	skb = swbp->skb;
1723
1724	/* No skb backpointer is set when running XDP. An xdp_frame
1725	 * backpointer is saved instead.
1726	 */
1727	if (!skb) {
1728		xdp_return_frame(swbp->xdpf);
1729		return NULL;
1730	}
1731
1732	/* DMA unmapping is required before accessing the HW provided info */
1733	if (ts && priv->tx_tstamp &&
1734	    skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
1735		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
1736
1737		if (!fman_port_get_tstamp(priv->mac_dev->port[TX], vaddr,
1738					  &ns)) {
1739			shhwtstamps.hwtstamp = ns_to_ktime(ns);
1740			skb_tstamp_tx(skb, &shhwtstamps);
1741		} else {
1742			dev_warn(dev, "fman_port_get_tstamp failed!\n");
1743		}
1744	}
1745
1746	if (qm_fd_get_format(fd) == qm_fd_sg)
1747		/* Free the page that we allocated on Tx for the SGT */
1748		free_pages((unsigned long)vaddr, 0);
1749
1750	return skb;
1751}
1752
1753static u8 rx_csum_offload(const struct dpaa_priv *priv, const struct qm_fd *fd)
1754{
1755	/* The parser has run and performed L4 checksum validation.
1756	 * We know there were no parser errors (and implicitly no
1757	 * L4 csum error), otherwise we wouldn't be here.
1758	 */
1759	if ((priv->net_dev->features & NETIF_F_RXCSUM) &&
1760	    (be32_to_cpu(fd->status) & FM_FD_STAT_L4CV))
1761		return CHECKSUM_UNNECESSARY;
1762
1763	/* We're here because either the parser didn't run or the L4 checksum
1764	 * was not verified. This may include the case of a UDP frame with
1765	 * checksum zero or an L4 proto other than TCP/UDP
1766	 */
1767	return CHECKSUM_NONE;
1768}
1769
1770#define PTR_IS_ALIGNED(x, a) (IS_ALIGNED((unsigned long)(x), (a)))
1771
1772/* Build a linear skb around the received buffer.
1773 * We are guaranteed there is enough room at the end of the data buffer to
1774 * accommodate the shared info area of the skb.
1775 */
1776static struct sk_buff *contig_fd_to_skb(const struct dpaa_priv *priv,
1777					const struct qm_fd *fd)
1778{
1779	ssize_t fd_off = qm_fd_get_offset(fd);
1780	dma_addr_t addr = qm_fd_addr(fd);
1781	struct dpaa_bp *dpaa_bp;
1782	struct sk_buff *skb;
1783	void *vaddr;
1784
1785	vaddr = phys_to_virt(addr);
1786	WARN_ON(!IS_ALIGNED((unsigned long)vaddr, SMP_CACHE_BYTES));
1787
1788	dpaa_bp = dpaa_bpid2pool(fd->bpid);
1789	if (!dpaa_bp)
1790		goto free_buffer;
1791
1792	skb = build_skb(vaddr, dpaa_bp->size +
1793			SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
1794	if (WARN_ONCE(!skb, "Build skb failure on Rx\n"))
1795		goto free_buffer;
1796	skb_reserve(skb, fd_off);
1797	skb_put(skb, qm_fd_get_length(fd));
1798
1799	skb->ip_summed = rx_csum_offload(priv, fd);
1800
1801	return skb;
1802
1803free_buffer:
1804	free_pages((unsigned long)vaddr, 0);
1805	return NULL;
1806}
1807
1808/* Build an skb with the data of the first S/G entry in the linear portion and
1809 * the rest of the frame as skb fragments.
1810 *
1811 * The page fragment holding the S/G Table is recycled here.
1812 */
1813static struct sk_buff *sg_fd_to_skb(const struct dpaa_priv *priv,
1814				    const struct qm_fd *fd)
1815{
1816	ssize_t fd_off = qm_fd_get_offset(fd);
1817	dma_addr_t addr = qm_fd_addr(fd);
1818	const struct qm_sg_entry *sgt;
1819	struct page *page, *head_page;
1820	struct dpaa_bp *dpaa_bp;
1821	void *vaddr, *sg_vaddr;
1822	struct sk_buff *skb;
1823	dma_addr_t sg_addr;
1824	int page_offset;
1825	unsigned int sz;
1826	int *count_ptr;
1827	int i, j;
1828
1829	vaddr = phys_to_virt(addr);
1830	WARN_ON(!IS_ALIGNED((unsigned long)vaddr, SMP_CACHE_BYTES));
1831
1832	/* Iterate through the SGT entries and add data buffers to the skb */
1833	sgt = vaddr + fd_off;
1834	skb = NULL;
1835	for (i = 0; i < DPAA_SGT_MAX_ENTRIES; i++) {
1836		/* Extension bit is not supported */
1837		WARN_ON(qm_sg_entry_is_ext(&sgt[i]));
1838
1839		sg_addr = qm_sg_addr(&sgt[i]);
1840		sg_vaddr = phys_to_virt(sg_addr);
1841		WARN_ON(!PTR_IS_ALIGNED(sg_vaddr, SMP_CACHE_BYTES));
1842
1843		dma_unmap_page(priv->rx_dma_dev, sg_addr,
1844			       DPAA_BP_RAW_SIZE, DMA_FROM_DEVICE);
1845
1846		/* We may use multiple Rx pools */
1847		dpaa_bp = dpaa_bpid2pool(sgt[i].bpid);
1848		if (!dpaa_bp)
1849			goto free_buffers;
1850
1851		if (!skb) {
1852			sz = dpaa_bp->size +
1853				SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1854			skb = build_skb(sg_vaddr, sz);
1855			if (WARN_ON(!skb))
1856				goto free_buffers;
1857
1858			skb->ip_summed = rx_csum_offload(priv, fd);
1859
1860			/* Make sure forwarded skbs will have enough space
1861			 * on Tx, if extra headers are added.
1862			 */
1863			WARN_ON(fd_off != priv->rx_headroom);
1864			/* The offset to data start within the buffer holding
1865			 * the SGT should always be equal to the offset to data
1866			 * start within the first buffer holding the frame.
1867			 */
1868			WARN_ON_ONCE(fd_off != qm_sg_entry_get_off(&sgt[i]));
1869			skb_reserve(skb, fd_off);
1870			skb_put(skb, qm_sg_entry_get_len(&sgt[i]));
1871		} else {
1872			/* Not the first S/G entry; all data from buffer will
1873			 * be added in an skb fragment; fragment index is offset
1874			 * by one since first S/G entry was incorporated in the
1875			 * linear part of the skb.
1876			 *
1877			 * Caution: 'page' may be a tail page.
1878			 */
1879			page = virt_to_page(sg_vaddr);
1880			head_page = virt_to_head_page(sg_vaddr);
1881
1882			/* Compute offset of sg_vaddr in (possibly tail) page */
1883			page_offset = ((unsigned long)sg_vaddr &
1884					(PAGE_SIZE - 1)) +
1885				(page_address(page) - page_address(head_page));
1886
1887			/* Non-initial SGT entries should not have a buffer
1888			 * offset.
1889			 */
1890			WARN_ON_ONCE(qm_sg_entry_get_off(&sgt[i]));
1891
1892			/* skb_add_rx_frag() does no checking on the page; if
1893			 * we pass it a tail page, we'll end up with
1894			 * bad page accounting and eventually with segfaults.
1895			 */
1896			skb_add_rx_frag(skb, i - 1, head_page, page_offset,
1897					qm_sg_entry_get_len(&sgt[i]),
1898					dpaa_bp->size);
1899		}
1900
1901		/* Update the pool count for the current {cpu x bpool} */
1902		count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
1903		(*count_ptr)--;
1904
1905		if (qm_sg_entry_is_final(&sgt[i]))
1906			break;
1907	}
1908	WARN_ONCE(i == DPAA_SGT_MAX_ENTRIES, "No final bit on SGT\n");
1909
1910	/* free the SG table buffer */
1911	free_pages((unsigned long)vaddr, 0);
1912
1913	return skb;
1914
1915free_buffers:
1916	/* free all the SG entries */
1917	for (j = 0; j < DPAA_SGT_MAX_ENTRIES ; j++) {
1918		sg_addr = qm_sg_addr(&sgt[j]);
1919		sg_vaddr = phys_to_virt(sg_addr);
1920		/* all pages 0..i were unmaped */
1921		if (j > i)
1922			dma_unmap_page(priv->rx_dma_dev, qm_sg_addr(&sgt[j]),
1923				       DPAA_BP_RAW_SIZE, DMA_FROM_DEVICE);
1924		free_pages((unsigned long)sg_vaddr, 0);
1925		/* counters 0..i-1 were decremented */
1926		if (j >= i) {
1927			dpaa_bp = dpaa_bpid2pool(sgt[j].bpid);
1928			if (dpaa_bp) {
1929				count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
1930				(*count_ptr)--;
1931			}
1932		}
1933
1934		if (qm_sg_entry_is_final(&sgt[j]))
1935			break;
1936	}
1937	/* free the SGT fragment */
1938	free_pages((unsigned long)vaddr, 0);
1939
1940	return NULL;
1941}
1942
1943static int skb_to_contig_fd(struct dpaa_priv *priv,
1944			    struct sk_buff *skb, struct qm_fd *fd,
1945			    int *offset)
1946{
1947	struct net_device *net_dev = priv->net_dev;
1948	enum dma_data_direction dma_dir;
1949	struct dpaa_eth_swbp *swbp;
1950	unsigned char *buff_start;
1951	dma_addr_t addr;
1952	int err;
1953
1954	/* We are guaranteed to have at least tx_headroom bytes
1955	 * available, so just use that for offset.
1956	 */
1957	fd->bpid = FSL_DPAA_BPID_INV;
1958	buff_start = skb->data - priv->tx_headroom;
1959	dma_dir = DMA_TO_DEVICE;
1960
1961	swbp = (struct dpaa_eth_swbp *)buff_start;
1962	swbp->skb = skb;
1963
1964	/* Enable L3/L4 hardware checksum computation.
1965	 *
1966	 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
1967	 * need to write into the skb.
1968	 */
1969	err = dpaa_enable_tx_csum(priv, skb, fd,
1970				  buff_start + DPAA_TX_PRIV_DATA_SIZE);
1971	if (unlikely(err < 0)) {
1972		if (net_ratelimit())
1973			netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",
1974				  err);
1975		return err;
1976	}
1977
1978	/* Fill in the rest of the FD fields */
1979	qm_fd_set_contig(fd, priv->tx_headroom, skb->len);
1980	fd->cmd |= cpu_to_be32(FM_FD_CMD_FCO);
1981
1982	/* Map the entire buffer size that may be seen by FMan, but no more */
1983	addr = dma_map_single(priv->tx_dma_dev, buff_start,
1984			      priv->tx_headroom + skb->len, dma_dir);
1985	if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
1986		if (net_ratelimit())
1987			netif_err(priv, tx_err, net_dev, "dma_map_single() failed\n");
1988		return -EINVAL;
1989	}
1990	qm_fd_addr_set64(fd, addr);
1991
1992	return 0;
1993}
1994
1995static int skb_to_sg_fd(struct dpaa_priv *priv,
1996			struct sk_buff *skb, struct qm_fd *fd)
1997{
1998	const enum dma_data_direction dma_dir = DMA_TO_DEVICE;
1999	const int nr_frags = skb_shinfo(skb)->nr_frags;
2000	struct net_device *net_dev = priv->net_dev;
2001	struct dpaa_eth_swbp *swbp;
2002	struct qm_sg_entry *sgt;
2003	void *buff_start;
2004	skb_frag_t *frag;
2005	dma_addr_t addr;
2006	size_t frag_len;
2007	struct page *p;
2008	int i, j, err;
2009
2010	/* get a page to store the SGTable */
2011	p = dev_alloc_pages(0);
2012	if (unlikely(!p)) {
2013		netdev_err(net_dev, "dev_alloc_pages() failed\n");
2014		return -ENOMEM;
2015	}
2016	buff_start = page_address(p);
2017
2018	/* Enable L3/L4 hardware checksum computation.
2019	 *
2020	 * We must do this before dma_map_single(DMA_TO_DEVICE), because we may
2021	 * need to write into the skb.
2022	 */
2023	err = dpaa_enable_tx_csum(priv, skb, fd,
2024				  buff_start + DPAA_TX_PRIV_DATA_SIZE);
2025	if (unlikely(err < 0)) {
2026		if (net_ratelimit())
2027			netif_err(priv, tx_err, net_dev, "HW csum error: %d\n",
2028				  err);
2029		goto csum_failed;
2030	}
2031
2032	/* SGT[0] is used by the linear part */
2033	sgt = (struct qm_sg_entry *)(buff_start + priv->tx_headroom);
2034	frag_len = skb_headlen(skb);
2035	qm_sg_entry_set_len(&sgt[0], frag_len);
2036	sgt[0].bpid = FSL_DPAA_BPID_INV;
2037	sgt[0].offset = 0;
2038	addr = dma_map_single(priv->tx_dma_dev, skb->data,
2039			      skb_headlen(skb), dma_dir);
2040	if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
2041		netdev_err(priv->net_dev, "DMA mapping failed\n");
2042		err = -EINVAL;
2043		goto sg0_map_failed;
2044	}
2045	qm_sg_entry_set64(&sgt[0], addr);
2046
2047	/* populate the rest of SGT entries */
2048	for (i = 0; i < nr_frags; i++) {
2049		frag = &skb_shinfo(skb)->frags[i];
2050		frag_len = skb_frag_size(frag);
2051		WARN_ON(!skb_frag_page(frag));
2052		addr = skb_frag_dma_map(priv->tx_dma_dev, frag, 0,
2053					frag_len, dma_dir);
2054		if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
2055			netdev_err(priv->net_dev, "DMA mapping failed\n");
2056			err = -EINVAL;
2057			goto sg_map_failed;
2058		}
2059
2060		qm_sg_entry_set_len(&sgt[i + 1], frag_len);
2061		sgt[i + 1].bpid = FSL_DPAA_BPID_INV;
2062		sgt[i + 1].offset = 0;
2063
2064		/* keep the offset in the address */
2065		qm_sg_entry_set64(&sgt[i + 1], addr);
2066	}
2067
2068	/* Set the final bit in the last used entry of the SGT */
2069	qm_sg_entry_set_f(&sgt[nr_frags], frag_len);
2070
2071	/* set fd offset to priv->tx_headroom */
2072	qm_fd_set_sg(fd, priv->tx_headroom, skb->len);
2073
2074	/* DMA map the SGT page */
2075	swbp = (struct dpaa_eth_swbp *)buff_start;
2076	swbp->skb = skb;
2077
2078	addr = dma_map_page(priv->tx_dma_dev, p, 0,
2079			    priv->tx_headroom + DPAA_SGT_SIZE, dma_dir);
2080	if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
2081		netdev_err(priv->net_dev, "DMA mapping failed\n");
2082		err = -EINVAL;
2083		goto sgt_map_failed;
2084	}
2085
2086	fd->bpid = FSL_DPAA_BPID_INV;
2087	fd->cmd |= cpu_to_be32(FM_FD_CMD_FCO);
2088	qm_fd_addr_set64(fd, addr);
2089
2090	return 0;
2091
2092sgt_map_failed:
2093sg_map_failed:
2094	for (j = 0; j < i; j++)
2095		dma_unmap_page(priv->tx_dma_dev, qm_sg_addr(&sgt[j]),
2096			       qm_sg_entry_get_len(&sgt[j]), dma_dir);
2097sg0_map_failed:
2098csum_failed:
2099	free_pages((unsigned long)buff_start, 0);
2100
2101	return err;
2102}
2103
2104static inline int dpaa_xmit(struct dpaa_priv *priv,
2105			    struct rtnl_link_stats64 *percpu_stats,
2106			    int queue,
2107			    struct qm_fd *fd)
2108{
2109	struct qman_fq *egress_fq;
2110	int err, i;
2111
2112	egress_fq = priv->egress_fqs[queue];
2113	if (fd->bpid == FSL_DPAA_BPID_INV)
2114		fd->cmd |= cpu_to_be32(qman_fq_fqid(priv->conf_fqs[queue]));
2115
2116	/* Trace this Tx fd */
2117	trace_dpaa_tx_fd(priv->net_dev, egress_fq, fd);
2118
2119	for (i = 0; i < DPAA_ENQUEUE_RETRIES; i++) {
2120		err = qman_enqueue(egress_fq, fd);
2121		if (err != -EBUSY)
2122			break;
2123	}
2124
2125	if (unlikely(err < 0)) {
2126		percpu_stats->tx_fifo_errors++;
2127		return err;
2128	}
2129
2130	percpu_stats->tx_packets++;
2131	percpu_stats->tx_bytes += qm_fd_get_length(fd);
2132
2133	return 0;
2134}
2135
2136#ifdef CONFIG_DPAA_ERRATUM_A050385
2137static int dpaa_a050385_wa_skb(struct net_device *net_dev, struct sk_buff **s)
2138{
2139	struct dpaa_priv *priv = netdev_priv(net_dev);
2140	struct sk_buff *new_skb, *skb = *s;
2141	unsigned char *start, i;
2142
2143	/* check linear buffer alignment */
2144	if (!PTR_IS_ALIGNED(skb->data, DPAA_A050385_ALIGN))
2145		goto workaround;
2146
2147	/* linear buffers just need to have an aligned start */
2148	if (!skb_is_nonlinear(skb))
2149		return 0;
2150
2151	/* linear data size for nonlinear skbs needs to be aligned */
2152	if (!IS_ALIGNED(skb_headlen(skb), DPAA_A050385_ALIGN))
2153		goto workaround;
2154
2155	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2156		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2157
2158		/* all fragments need to have aligned start addresses */
2159		if (!IS_ALIGNED(skb_frag_off(frag), DPAA_A050385_ALIGN))
2160			goto workaround;
2161
2162		/* all but last fragment need to have aligned sizes */
2163		if (!IS_ALIGNED(skb_frag_size(frag), DPAA_A050385_ALIGN) &&
2164		    (i < skb_shinfo(skb)->nr_frags - 1))
2165			goto workaround;
2166	}
2167
2168	return 0;
2169
2170workaround:
2171	/* copy all the skb content into a new linear buffer */
2172	new_skb = netdev_alloc_skb(net_dev, skb->len + DPAA_A050385_ALIGN - 1 +
2173						priv->tx_headroom);
2174	if (!new_skb)
2175		return -ENOMEM;
2176
2177	/* NET_SKB_PAD bytes already reserved, adding up to tx_headroom */
2178	skb_reserve(new_skb, priv->tx_headroom - NET_SKB_PAD);
2179
2180	/* Workaround for DPAA_A050385 requires data start to be aligned */
2181	start = PTR_ALIGN(new_skb->data, DPAA_A050385_ALIGN);
2182	if (start - new_skb->data)
2183		skb_reserve(new_skb, start - new_skb->data);
2184
2185	skb_put(new_skb, skb->len);
2186	skb_copy_bits(skb, 0, new_skb->data, skb->len);
2187	skb_copy_header(new_skb, skb);
2188	new_skb->dev = skb->dev;
2189
2190	/* Copy relevant timestamp info from the old skb to the new */
2191	if (priv->tx_tstamp) {
2192		skb_shinfo(new_skb)->tx_flags = skb_shinfo(skb)->tx_flags;
2193		skb_shinfo(new_skb)->hwtstamps = skb_shinfo(skb)->hwtstamps;
2194		skb_shinfo(new_skb)->tskey = skb_shinfo(skb)->tskey;
2195		if (skb->sk)
2196			skb_set_owner_w(new_skb, skb->sk);
2197	}
2198
2199	/* We move the headroom when we align it so we have to reset the
2200	 * network and transport header offsets relative to the new data
2201	 * pointer. The checksum offload relies on these offsets.
2202	 */
2203	skb_set_network_header(new_skb, skb_network_offset(skb));
2204	skb_set_transport_header(new_skb, skb_transport_offset(skb));
2205
2206	dev_kfree_skb(skb);
2207	*s = new_skb;
2208
2209	return 0;
2210}
2211
2212static int dpaa_a050385_wa_xdpf(struct dpaa_priv *priv,
2213				struct xdp_frame **init_xdpf)
2214{
2215	struct xdp_frame *new_xdpf, *xdpf = *init_xdpf;
2216	void *new_buff, *aligned_data;
2217	struct page *p;
2218	u32 data_shift;
2219	int headroom;
2220
2221	/* Check the data alignment and make sure the headroom is large
2222	 * enough to store the xdpf backpointer. Use an aligned headroom
2223	 * value.
2224	 *
2225	 * Due to alignment constraints, we give XDP access to the full 256
2226	 * byte frame headroom. If the XDP program uses all of it, copy the
2227	 * data to a new buffer and make room for storing the backpointer.
2228	 */
2229	if (PTR_IS_ALIGNED(xdpf->data, DPAA_FD_DATA_ALIGNMENT) &&
2230	    xdpf->headroom >= priv->tx_headroom) {
2231		xdpf->headroom = priv->tx_headroom;
2232		return 0;
2233	}
2234
2235	/* Try to move the data inside the buffer just enough to align it and
2236	 * store the xdpf backpointer. If the available headroom isn't large
2237	 * enough, resort to allocating a new buffer and copying the data.
2238	 */
2239	aligned_data = PTR_ALIGN_DOWN(xdpf->data, DPAA_FD_DATA_ALIGNMENT);
2240	data_shift = xdpf->data - aligned_data;
2241
2242	/* The XDP frame's headroom needs to be large enough to accommodate
2243	 * shifting the data as well as storing the xdpf backpointer.
2244	 */
2245	if (xdpf->headroom  >= data_shift + priv->tx_headroom) {
2246		memmove(aligned_data, xdpf->data, xdpf->len);
2247		xdpf->data = aligned_data;
2248		xdpf->headroom = priv->tx_headroom;
2249		return 0;
2250	}
2251
2252	/* The new xdp_frame is stored in the new buffer. Reserve enough space
2253	 * in the headroom for storing it along with the driver's private
2254	 * info. The headroom needs to be aligned to DPAA_FD_DATA_ALIGNMENT to
2255	 * guarantee the data's alignment in the buffer.
2256	 */
2257	headroom = ALIGN(sizeof(*new_xdpf) + priv->tx_headroom,
2258			 DPAA_FD_DATA_ALIGNMENT);
2259
2260	/* Assure the extended headroom and data don't overflow the buffer,
2261	 * while maintaining the mandatory tailroom.
2262	 */
2263	if (headroom + xdpf->len > DPAA_BP_RAW_SIZE -
2264			SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
2265		return -ENOMEM;
2266
2267	p = dev_alloc_pages(0);
2268	if (unlikely(!p))
2269		return -ENOMEM;
2270
2271	/* Copy the data to the new buffer at a properly aligned offset */
2272	new_buff = page_address(p);
2273	memcpy(new_buff + headroom, xdpf->data, xdpf->len);
2274
2275	/* Create an XDP frame around the new buffer in a similar fashion
2276	 * to xdp_convert_buff_to_frame.
2277	 */
2278	new_xdpf = new_buff;
2279	new_xdpf->data = new_buff + headroom;
2280	new_xdpf->len = xdpf->len;
2281	new_xdpf->headroom = priv->tx_headroom;
2282	new_xdpf->frame_sz = DPAA_BP_RAW_SIZE;
2283	new_xdpf->mem_type = MEM_TYPE_PAGE_ORDER0;
2284
2285	/* Release the initial buffer */
2286	xdp_return_frame_rx_napi(xdpf);
2287
2288	*init_xdpf = new_xdpf;
2289	return 0;
2290}
2291#endif
2292
2293static netdev_tx_t
2294dpaa_start_xmit(struct sk_buff *skb, struct net_device *net_dev)
2295{
2296	const int queue_mapping = skb_get_queue_mapping(skb);
2297	struct rtnl_link_stats64 *percpu_stats;
2298	struct dpaa_percpu_priv *percpu_priv;
2299	struct netdev_queue *txq;
2300	struct dpaa_priv *priv;
2301	struct qm_fd fd;
2302	bool nonlinear;
2303	int offset = 0;
2304	int err = 0;
2305
2306	priv = netdev_priv(net_dev);
2307	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2308	percpu_stats = &percpu_priv->stats;
2309
2310	qm_fd_clear_fd(&fd);
2311
2312	/* Packet data is always read as 32-bit words, so zero out any part of
2313	 * the skb which might be sent if we have to pad the packet
2314	 */
2315	if (__skb_put_padto(skb, ETH_ZLEN, false))
2316		goto enomem;
2317
2318	nonlinear = skb_is_nonlinear(skb);
2319	if (!nonlinear) {
2320		/* We're going to store the skb backpointer at the beginning
2321		 * of the data buffer, so we need a privately owned skb
2322		 *
2323		 * We've made sure skb is not shared in dev->priv_flags,
2324		 * we need to verify the skb head is not cloned
2325		 */
2326		if (skb_cow_head(skb, priv->tx_headroom))
2327			goto enomem;
2328
2329		WARN_ON(skb_is_nonlinear(skb));
2330	}
2331
2332	/* MAX_SKB_FRAGS is equal or larger than our dpaa_SGT_MAX_ENTRIES;
2333	 * make sure we don't feed FMan with more fragments than it supports.
2334	 */
2335	if (unlikely(nonlinear &&
2336		     (skb_shinfo(skb)->nr_frags >= DPAA_SGT_MAX_ENTRIES))) {
2337		/* If the egress skb contains more fragments than we support
2338		 * we have no choice but to linearize it ourselves.
2339		 */
2340		if (__skb_linearize(skb))
2341			goto enomem;
2342
2343		nonlinear = skb_is_nonlinear(skb);
2344	}
2345
2346#ifdef CONFIG_DPAA_ERRATUM_A050385
2347	if (unlikely(fman_has_errata_a050385())) {
2348		if (dpaa_a050385_wa_skb(net_dev, &skb))
2349			goto enomem;
2350		nonlinear = skb_is_nonlinear(skb);
2351	}
2352#endif
2353
2354	if (nonlinear) {
2355		/* Just create a S/G fd based on the skb */
2356		err = skb_to_sg_fd(priv, skb, &fd);
2357		percpu_priv->tx_frag_skbuffs++;
2358	} else {
2359		/* Create a contig FD from this skb */
2360		err = skb_to_contig_fd(priv, skb, &fd, &offset);
2361	}
2362	if (unlikely(err < 0))
2363		goto skb_to_fd_failed;
2364
2365	txq = netdev_get_tx_queue(net_dev, queue_mapping);
2366
2367	/* LLTX requires to do our own update of trans_start */
2368	txq_trans_cond_update(txq);
2369
2370	if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
2371		fd.cmd |= cpu_to_be32(FM_FD_CMD_UPD);
2372		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2373	}
2374
2375	if (likely(dpaa_xmit(priv, percpu_stats, queue_mapping, &fd) == 0))
2376		return NETDEV_TX_OK;
2377
2378	dpaa_cleanup_tx_fd(priv, &fd, false);
2379skb_to_fd_failed:
2380enomem:
2381	percpu_stats->tx_errors++;
2382	dev_kfree_skb(skb);
2383	return NETDEV_TX_OK;
2384}
2385
2386static void dpaa_rx_error(struct net_device *net_dev,
2387			  const struct dpaa_priv *priv,
2388			  struct dpaa_percpu_priv *percpu_priv,
2389			  const struct qm_fd *fd,
2390			  u32 fqid)
2391{
2392	if (net_ratelimit())
2393		netif_err(priv, hw, net_dev, "Err FD status = 0x%08x\n",
2394			  be32_to_cpu(fd->status) & FM_FD_STAT_RX_ERRORS);
2395
2396	percpu_priv->stats.rx_errors++;
2397
2398	if (be32_to_cpu(fd->status) & FM_FD_ERR_DMA)
2399		percpu_priv->rx_errors.dme++;
2400	if (be32_to_cpu(fd->status) & FM_FD_ERR_PHYSICAL)
2401		percpu_priv->rx_errors.fpe++;
2402	if (be32_to_cpu(fd->status) & FM_FD_ERR_SIZE)
2403		percpu_priv->rx_errors.fse++;
2404	if (be32_to_cpu(fd->status) & FM_FD_ERR_PRS_HDR_ERR)
2405		percpu_priv->rx_errors.phe++;
2406
2407	dpaa_fd_release(net_dev, fd);
2408}
2409
2410static void dpaa_tx_error(struct net_device *net_dev,
2411			  const struct dpaa_priv *priv,
2412			  struct dpaa_percpu_priv *percpu_priv,
2413			  const struct qm_fd *fd,
2414			  u32 fqid)
2415{
2416	struct sk_buff *skb;
2417
2418	if (net_ratelimit())
2419		netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
2420			   be32_to_cpu(fd->status) & FM_FD_STAT_TX_ERRORS);
2421
2422	percpu_priv->stats.tx_errors++;
2423
2424	skb = dpaa_cleanup_tx_fd(priv, fd, false);
2425	dev_kfree_skb(skb);
2426}
2427
2428static int dpaa_eth_poll(struct napi_struct *napi, int budget)
2429{
2430	struct dpaa_napi_portal *np =
2431			container_of(napi, struct dpaa_napi_portal, napi);
2432	int cleaned;
2433
2434	np->xdp_act = 0;
2435
2436	cleaned = qman_p_poll_dqrr(np->p, budget);
2437
2438	if (np->xdp_act & XDP_REDIRECT)
2439		xdp_do_flush();
2440
2441	if (cleaned < budget) {
2442		napi_complete_done(napi, cleaned);
2443		qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
2444	} else if (np->down) {
2445		qman_p_irqsource_add(np->p, QM_PIRQ_DQRI);
2446	}
2447
2448	return cleaned;
2449}
2450
2451static void dpaa_tx_conf(struct net_device *net_dev,
2452			 const struct dpaa_priv *priv,
2453			 struct dpaa_percpu_priv *percpu_priv,
2454			 const struct qm_fd *fd,
2455			 u32 fqid)
2456{
2457	struct sk_buff	*skb;
2458
2459	if (unlikely(be32_to_cpu(fd->status) & FM_FD_STAT_TX_ERRORS)) {
2460		if (net_ratelimit())
2461			netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
2462				   be32_to_cpu(fd->status) &
2463				   FM_FD_STAT_TX_ERRORS);
2464
2465		percpu_priv->stats.tx_errors++;
2466	}
2467
2468	percpu_priv->tx_confirm++;
2469
2470	skb = dpaa_cleanup_tx_fd(priv, fd, true);
2471
2472	consume_skb(skb);
2473}
2474
2475static inline int dpaa_eth_napi_schedule(struct dpaa_percpu_priv *percpu_priv,
2476					 struct qman_portal *portal, bool sched_napi)
2477{
2478	if (sched_napi) {
2479		/* Disable QMan IRQ and invoke NAPI */
2480		qman_p_irqsource_remove(portal, QM_PIRQ_DQRI);
2481
2482		percpu_priv->np.p = portal;
2483		napi_schedule(&percpu_priv->np.napi);
2484		percpu_priv->in_interrupt++;
2485		return 1;
2486	}
2487	return 0;
2488}
2489
2490static enum qman_cb_dqrr_result rx_error_dqrr(struct qman_portal *portal,
2491					      struct qman_fq *fq,
2492					      const struct qm_dqrr_entry *dq,
2493					      bool sched_napi)
2494{
2495	struct dpaa_fq *dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
2496	struct dpaa_percpu_priv *percpu_priv;
2497	struct net_device *net_dev;
2498	struct dpaa_bp *dpaa_bp;
2499	struct dpaa_priv *priv;
2500
2501	net_dev = dpaa_fq->net_dev;
2502	priv = netdev_priv(net_dev);
2503	dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);
2504	if (!dpaa_bp)
2505		return qman_cb_dqrr_consume;
2506
2507	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2508
2509	if (dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi))
2510		return qman_cb_dqrr_stop;
2511
2512	dpaa_eth_refill_bpools(priv);
2513	dpaa_rx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2514
2515	return qman_cb_dqrr_consume;
2516}
2517
2518static int dpaa_xdp_xmit_frame(struct net_device *net_dev,
2519			       struct xdp_frame *xdpf)
2520{
2521	struct dpaa_priv *priv = netdev_priv(net_dev);
2522	struct rtnl_link_stats64 *percpu_stats;
2523	struct dpaa_percpu_priv *percpu_priv;
2524	struct dpaa_eth_swbp *swbp;
2525	struct netdev_queue *txq;
2526	void *buff_start;
2527	struct qm_fd fd;
2528	dma_addr_t addr;
2529	int err;
2530
2531	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2532	percpu_stats = &percpu_priv->stats;
2533
2534#ifdef CONFIG_DPAA_ERRATUM_A050385
2535	if (unlikely(fman_has_errata_a050385())) {
2536		if (dpaa_a050385_wa_xdpf(priv, &xdpf)) {
2537			err = -ENOMEM;
2538			goto out_error;
2539		}
2540	}
2541#endif
2542
2543	if (xdpf->headroom < DPAA_TX_PRIV_DATA_SIZE) {
2544		err = -EINVAL;
2545		goto out_error;
2546	}
2547
2548	buff_start = xdpf->data - xdpf->headroom;
2549
2550	/* Leave empty the skb backpointer at the start of the buffer.
2551	 * Save the XDP frame for easy cleanup on confirmation.
2552	 */
2553	swbp = (struct dpaa_eth_swbp *)buff_start;
2554	swbp->skb = NULL;
2555	swbp->xdpf = xdpf;
2556
2557	qm_fd_clear_fd(&fd);
2558	fd.bpid = FSL_DPAA_BPID_INV;
2559	fd.cmd |= cpu_to_be32(FM_FD_CMD_FCO);
2560	qm_fd_set_contig(&fd, xdpf->headroom, xdpf->len);
2561
2562	addr = dma_map_single(priv->tx_dma_dev, buff_start,
2563			      xdpf->headroom + xdpf->len,
2564			      DMA_TO_DEVICE);
2565	if (unlikely(dma_mapping_error(priv->tx_dma_dev, addr))) {
2566		err = -EINVAL;
2567		goto out_error;
2568	}
2569
2570	qm_fd_addr_set64(&fd, addr);
2571
2572	/* Bump the trans_start */
2573	txq = netdev_get_tx_queue(net_dev, smp_processor_id());
2574	txq_trans_cond_update(txq);
2575
2576	err = dpaa_xmit(priv, percpu_stats, smp_processor_id(), &fd);
2577	if (err) {
2578		dma_unmap_single(priv->tx_dma_dev, addr,
2579				 qm_fd_get_offset(&fd) + qm_fd_get_length(&fd),
2580				 DMA_TO_DEVICE);
2581		goto out_error;
2582	}
2583
2584	return 0;
2585
2586out_error:
2587	percpu_stats->tx_errors++;
2588	return err;
2589}
2590
2591static u32 dpaa_run_xdp(struct dpaa_priv *priv, struct qm_fd *fd, void *vaddr,
2592			struct dpaa_fq *dpaa_fq, unsigned int *xdp_meta_len)
2593{
2594	ssize_t fd_off = qm_fd_get_offset(fd);
2595	struct bpf_prog *xdp_prog;
2596	struct xdp_frame *xdpf;
2597	struct xdp_buff xdp;
2598	u32 xdp_act;
2599	int err;
2600
2601	xdp_prog = READ_ONCE(priv->xdp_prog);
2602	if (!xdp_prog)
2603		return XDP_PASS;
2604
2605	xdp_init_buff(&xdp, DPAA_BP_RAW_SIZE - DPAA_TX_PRIV_DATA_SIZE,
2606		      &dpaa_fq->xdp_rxq);
2607	xdp_prepare_buff(&xdp, vaddr + fd_off - XDP_PACKET_HEADROOM,
2608			 XDP_PACKET_HEADROOM, qm_fd_get_length(fd), true);
2609
2610	/* We reserve a fixed headroom of 256 bytes under the erratum and we
2611	 * offer it all to XDP programs to use. If no room is left for the
2612	 * xdpf backpointer on TX, we will need to copy the data.
2613	 * Disable metadata support since data realignments might be required
2614	 * and the information can be lost.
2615	 */
2616#ifdef CONFIG_DPAA_ERRATUM_A050385
2617	if (unlikely(fman_has_errata_a050385())) {
2618		xdp_set_data_meta_invalid(&xdp);
2619		xdp.data_hard_start = vaddr;
2620		xdp.frame_sz = DPAA_BP_RAW_SIZE;
2621	}
2622#endif
2623
2624	xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp);
2625
2626	/* Update the length and the offset of the FD */
2627	qm_fd_set_contig(fd, xdp.data - vaddr, xdp.data_end - xdp.data);
2628
2629	switch (xdp_act) {
2630	case XDP_PASS:
2631#ifdef CONFIG_DPAA_ERRATUM_A050385
2632		*xdp_meta_len = xdp_data_meta_unsupported(&xdp) ? 0 :
2633				xdp.data - xdp.data_meta;
2634#else
2635		*xdp_meta_len = xdp.data - xdp.data_meta;
2636#endif
2637		break;
2638	case XDP_TX:
2639		/* We can access the full headroom when sending the frame
2640		 * back out
2641		 */
2642		xdp.data_hard_start = vaddr;
2643		xdp.frame_sz = DPAA_BP_RAW_SIZE;
2644		xdpf = xdp_convert_buff_to_frame(&xdp);
2645		if (unlikely(!xdpf)) {
2646			free_pages((unsigned long)vaddr, 0);
2647			break;
2648		}
2649
2650		if (dpaa_xdp_xmit_frame(priv->net_dev, xdpf))
2651			xdp_return_frame_rx_napi(xdpf);
2652
2653		break;
2654	case XDP_REDIRECT:
2655		/* Allow redirect to use the full headroom */
2656		xdp.data_hard_start = vaddr;
2657		xdp.frame_sz = DPAA_BP_RAW_SIZE;
2658
2659		err = xdp_do_redirect(priv->net_dev, &xdp, xdp_prog);
2660		if (err) {
2661			trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
2662			free_pages((unsigned long)vaddr, 0);
2663		}
2664		break;
2665	default:
2666		bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);
2667		fallthrough;
2668	case XDP_ABORTED:
2669		trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
2670		fallthrough;
2671	case XDP_DROP:
2672		/* Free the buffer */
2673		free_pages((unsigned long)vaddr, 0);
2674		break;
2675	}
2676
2677	return xdp_act;
2678}
2679
2680static enum qman_cb_dqrr_result rx_default_dqrr(struct qman_portal *portal,
2681						struct qman_fq *fq,
2682						const struct qm_dqrr_entry *dq,
2683						bool sched_napi)
2684{
2685	bool ts_valid = false, hash_valid = false;
2686	struct skb_shared_hwtstamps *shhwtstamps;
2687	unsigned int skb_len, xdp_meta_len = 0;
2688	struct rtnl_link_stats64 *percpu_stats;
2689	struct dpaa_percpu_priv *percpu_priv;
2690	const struct qm_fd *fd = &dq->fd;
2691	dma_addr_t addr = qm_fd_addr(fd);
2692	struct dpaa_napi_portal *np;
2693	enum qm_fd_format fd_format;
2694	struct net_device *net_dev;
2695	u32 fd_status, hash_offset;
2696	struct qm_sg_entry *sgt;
2697	struct dpaa_bp *dpaa_bp;
2698	struct dpaa_fq *dpaa_fq;
2699	struct dpaa_priv *priv;
2700	struct sk_buff *skb;
2701	int *count_ptr;
2702	u32 xdp_act;
2703	void *vaddr;
2704	u32 hash;
2705	u64 ns;
2706
2707	dpaa_fq = container_of(fq, struct dpaa_fq, fq_base);
2708	fd_status = be32_to_cpu(fd->status);
2709	fd_format = qm_fd_get_format(fd);
2710	net_dev = dpaa_fq->net_dev;
2711	priv = netdev_priv(net_dev);
2712	dpaa_bp = dpaa_bpid2pool(dq->fd.bpid);
2713	if (!dpaa_bp)
2714		return qman_cb_dqrr_consume;
2715
2716	/* Trace the Rx fd */
2717	trace_dpaa_rx_fd(net_dev, fq, &dq->fd);
2718
2719	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2720	percpu_stats = &percpu_priv->stats;
2721	np = &percpu_priv->np;
2722
2723	if (unlikely(dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi)))
2724		return qman_cb_dqrr_stop;
2725
2726	/* Make sure we didn't run out of buffers */
2727	if (unlikely(dpaa_eth_refill_bpools(priv))) {
2728		/* Unable to refill the buffer pool due to insufficient
2729		 * system memory. Just release the frame back into the pool,
2730		 * otherwise we'll soon end up with an empty buffer pool.
2731		 */
2732		dpaa_fd_release(net_dev, &dq->fd);
2733		return qman_cb_dqrr_consume;
2734	}
2735
2736	if (unlikely(fd_status & FM_FD_STAT_RX_ERRORS) != 0) {
2737		if (net_ratelimit())
2738			netif_warn(priv, hw, net_dev, "FD status = 0x%08x\n",
2739				   fd_status & FM_FD_STAT_RX_ERRORS);
2740
2741		percpu_stats->rx_errors++;
2742		dpaa_fd_release(net_dev, fd);
2743		return qman_cb_dqrr_consume;
2744	}
2745
2746	dma_unmap_page(dpaa_bp->priv->rx_dma_dev, addr, DPAA_BP_RAW_SIZE,
2747		       DMA_FROM_DEVICE);
2748
2749	/* prefetch the first 64 bytes of the frame or the SGT start */
2750	vaddr = phys_to_virt(addr);
2751	prefetch(vaddr + qm_fd_get_offset(fd));
2752
2753	/* The only FD types that we may receive are contig and S/G */
2754	WARN_ON((fd_format != qm_fd_contig) && (fd_format != qm_fd_sg));
2755
2756	/* Account for either the contig buffer or the SGT buffer (depending on
2757	 * which case we were in) having been removed from the pool.
2758	 */
2759	count_ptr = this_cpu_ptr(dpaa_bp->percpu_count);
2760	(*count_ptr)--;
2761
2762	/* Extract the timestamp stored in the headroom before running XDP */
2763	if (priv->rx_tstamp) {
2764		if (!fman_port_get_tstamp(priv->mac_dev->port[RX], vaddr, &ns))
2765			ts_valid = true;
2766		else
2767			WARN_ONCE(1, "fman_port_get_tstamp failed!\n");
2768	}
2769
2770	/* Extract the hash stored in the headroom before running XDP */
2771	if (net_dev->features & NETIF_F_RXHASH && priv->keygen_in_use &&
2772	    !fman_port_get_hash_result_offset(priv->mac_dev->port[RX],
2773					      &hash_offset)) {
2774		hash = be32_to_cpu(*(__be32 *)(vaddr + hash_offset));
2775		hash_valid = true;
2776	}
2777
2778	if (likely(fd_format == qm_fd_contig)) {
2779		xdp_act = dpaa_run_xdp(priv, (struct qm_fd *)fd, vaddr,
2780				       dpaa_fq, &xdp_meta_len);
2781		np->xdp_act |= xdp_act;
2782		if (xdp_act != XDP_PASS) {
2783			percpu_stats->rx_packets++;
2784			percpu_stats->rx_bytes += qm_fd_get_length(fd);
2785			return qman_cb_dqrr_consume;
2786		}
2787		skb = contig_fd_to_skb(priv, fd);
2788	} else {
2789		/* XDP doesn't support S/G frames. Return the fragments to the
2790		 * buffer pool and release the SGT.
2791		 */
2792		if (READ_ONCE(priv->xdp_prog)) {
2793			WARN_ONCE(1, "S/G frames not supported under XDP\n");
2794			sgt = vaddr + qm_fd_get_offset(fd);
2795			dpaa_release_sgt_members(sgt);
2796			free_pages((unsigned long)vaddr, 0);
2797			return qman_cb_dqrr_consume;
2798		}
2799		skb = sg_fd_to_skb(priv, fd);
2800	}
2801	if (!skb)
2802		return qman_cb_dqrr_consume;
2803
2804	if (xdp_meta_len)
2805		skb_metadata_set(skb, xdp_meta_len);
2806
2807	/* Set the previously extracted timestamp */
2808	if (ts_valid) {
2809		shhwtstamps = skb_hwtstamps(skb);
2810		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
2811		shhwtstamps->hwtstamp = ns_to_ktime(ns);
2812	}
2813
2814	skb->protocol = eth_type_trans(skb, net_dev);
2815
2816	/* Set the previously extracted hash */
2817	if (hash_valid) {
2818		enum pkt_hash_types type;
2819
2820		/* if L4 exists, it was used in the hash generation */
2821		type = be32_to_cpu(fd->status) & FM_FD_STAT_L4CV ?
2822			PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
2823		skb_set_hash(skb, hash, type);
2824	}
2825
2826	skb_len = skb->len;
2827
2828	if (unlikely(netif_receive_skb(skb) == NET_RX_DROP)) {
2829		percpu_stats->rx_dropped++;
2830		return qman_cb_dqrr_consume;
2831	}
2832
2833	percpu_stats->rx_packets++;
2834	percpu_stats->rx_bytes += skb_len;
2835
2836	return qman_cb_dqrr_consume;
2837}
2838
2839static enum qman_cb_dqrr_result conf_error_dqrr(struct qman_portal *portal,
2840						struct qman_fq *fq,
2841						const struct qm_dqrr_entry *dq,
2842						bool sched_napi)
2843{
2844	struct dpaa_percpu_priv *percpu_priv;
2845	struct net_device *net_dev;
2846	struct dpaa_priv *priv;
2847
2848	net_dev = ((struct dpaa_fq *)fq)->net_dev;
2849	priv = netdev_priv(net_dev);
2850
2851	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2852
2853	if (dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi))
2854		return qman_cb_dqrr_stop;
2855
2856	dpaa_tx_error(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2857
2858	return qman_cb_dqrr_consume;
2859}
2860
2861static enum qman_cb_dqrr_result conf_dflt_dqrr(struct qman_portal *portal,
2862					       struct qman_fq *fq,
2863					       const struct qm_dqrr_entry *dq,
2864					       bool sched_napi)
2865{
2866	struct dpaa_percpu_priv *percpu_priv;
2867	struct net_device *net_dev;
2868	struct dpaa_priv *priv;
2869
2870	net_dev = ((struct dpaa_fq *)fq)->net_dev;
2871	priv = netdev_priv(net_dev);
2872
2873	/* Trace the fd */
2874	trace_dpaa_tx_conf_fd(net_dev, fq, &dq->fd);
2875
2876	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2877
2878	if (dpaa_eth_napi_schedule(percpu_priv, portal, sched_napi))
2879		return qman_cb_dqrr_stop;
2880
2881	dpaa_tx_conf(net_dev, priv, percpu_priv, &dq->fd, fq->fqid);
2882
2883	return qman_cb_dqrr_consume;
2884}
2885
2886static void egress_ern(struct qman_portal *portal,
2887		       struct qman_fq *fq,
2888		       const union qm_mr_entry *msg)
2889{
2890	const struct qm_fd *fd = &msg->ern.fd;
2891	struct dpaa_percpu_priv *percpu_priv;
2892	const struct dpaa_priv *priv;
2893	struct net_device *net_dev;
2894	struct sk_buff *skb;
2895
2896	net_dev = ((struct dpaa_fq *)fq)->net_dev;
2897	priv = netdev_priv(net_dev);
2898	percpu_priv = this_cpu_ptr(priv->percpu_priv);
2899
2900	percpu_priv->stats.tx_dropped++;
2901	percpu_priv->stats.tx_fifo_errors++;
2902	count_ern(percpu_priv, msg);
2903
2904	skb = dpaa_cleanup_tx_fd(priv, fd, false);
2905	dev_kfree_skb_any(skb);
2906}
2907
2908static const struct dpaa_fq_cbs dpaa_fq_cbs = {
2909	.rx_defq = { .cb = { .dqrr = rx_default_dqrr } },
2910	.tx_defq = { .cb = { .dqrr = conf_dflt_dqrr } },
2911	.rx_errq = { .cb = { .dqrr = rx_error_dqrr } },
2912	.tx_errq = { .cb = { .dqrr = conf_error_dqrr } },
2913	.egress_ern = { .cb = { .ern = egress_ern } }
2914};
2915
2916static void dpaa_eth_napi_enable(struct dpaa_priv *priv)
2917{
2918	struct dpaa_percpu_priv *percpu_priv;
2919	int i;
2920
2921	for_each_online_cpu(i) {
2922		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
2923
2924		percpu_priv->np.down = false;
2925		napi_enable(&percpu_priv->np.napi);
2926	}
2927}
2928
2929static void dpaa_eth_napi_disable(struct dpaa_priv *priv)
2930{
2931	struct dpaa_percpu_priv *percpu_priv;
2932	int i;
2933
2934	for_each_online_cpu(i) {
2935		percpu_priv = per_cpu_ptr(priv->percpu_priv, i);
2936
2937		percpu_priv->np.down = true;
2938		napi_disable(&percpu_priv->np.napi);
2939	}
2940}
2941
2942static int dpaa_open(struct net_device *net_dev)
2943{
2944	struct mac_device *mac_dev;
2945	struct dpaa_priv *priv;
2946	int err, i;
2947
2948	priv = netdev_priv(net_dev);
2949	mac_dev = priv->mac_dev;
2950	dpaa_eth_napi_enable(priv);
2951
2952	err = phylink_of_phy_connect(mac_dev->phylink,
2953				     mac_dev->dev->of_node, 0);
2954	if (err)
2955		goto phy_init_failed;
2956
2957	for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
2958		err = fman_port_enable(mac_dev->port[i]);
2959		if (err)
2960			goto mac_start_failed;
2961	}
2962
2963	err = priv->mac_dev->enable(mac_dev->fman_mac);
2964	if (err < 0) {
2965		netif_err(priv, ifup, net_dev, "mac_dev->enable() = %d\n", err);
2966		goto mac_start_failed;
2967	}
2968	phylink_start(mac_dev->phylink);
2969
2970	netif_tx_start_all_queues(net_dev);
2971
2972	return 0;
2973
2974mac_start_failed:
2975	for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++)
2976		fman_port_disable(mac_dev->port[i]);
2977	phylink_disconnect_phy(mac_dev->phylink);
2978
2979phy_init_failed:
2980	dpaa_eth_napi_disable(priv);
2981
2982	return err;
2983}
2984
2985static int dpaa_eth_stop(struct net_device *net_dev)
2986{
2987	struct dpaa_priv *priv;
2988	int err;
2989
2990	err = dpaa_stop(net_dev);
2991
2992	priv = netdev_priv(net_dev);
2993	dpaa_eth_napi_disable(priv);
2994
2995	return err;
2996}
2997
2998static bool xdp_validate_mtu(struct dpaa_priv *priv, int mtu)
2999{
3000	int max_contig_data = priv->dpaa_bp->size - priv->rx_headroom;
3001
3002	/* We do not support S/G fragments when XDP is enabled.
3003	 * Limit the MTU in relation to the buffer size.
3004	 */
3005	if (mtu + VLAN_ETH_HLEN + ETH_FCS_LEN > max_contig_data) {
3006		dev_warn(priv->net_dev->dev.parent,
3007			 "The maximum MTU for XDP is %d\n",
3008			 max_contig_data - VLAN_ETH_HLEN - ETH_FCS_LEN);
3009		return false;
3010	}
3011
3012	return true;
3013}
3014
3015static int dpaa_change_mtu(struct net_device *net_dev, int new_mtu)
3016{
3017	struct dpaa_priv *priv = netdev_priv(net_dev);
3018
3019	if (priv->xdp_prog && !xdp_validate_mtu(priv, new_mtu))
3020		return -EINVAL;
3021
3022	WRITE_ONCE(net_dev->mtu, new_mtu);
3023	return 0;
3024}
3025
3026static int dpaa_setup_xdp(struct net_device *net_dev, struct netdev_bpf *bpf)
3027{
3028	struct dpaa_priv *priv = netdev_priv(net_dev);
3029	struct bpf_prog *old_prog;
3030	int err;
3031	bool up;
3032
3033	/* S/G fragments are not supported in XDP-mode */
3034	if (bpf->prog && !xdp_validate_mtu(priv, net_dev->mtu)) {
3035		NL_SET_ERR_MSG_MOD(bpf->extack, "MTU too large for XDP");
3036		return -EINVAL;
3037	}
3038
3039	up = netif_running(net_dev);
3040
3041	if (up)
3042		dpaa_eth_stop(net_dev);
3043
3044	old_prog = xchg(&priv->xdp_prog, bpf->prog);
3045	if (old_prog)
3046		bpf_prog_put(old_prog);
3047
3048	if (up) {
3049		err = dpaa_open(net_dev);
3050		if (err) {
3051			NL_SET_ERR_MSG_MOD(bpf->extack, "dpaa_open() failed");
3052			return err;
3053		}
3054	}
3055
3056	return 0;
3057}
3058
3059static int dpaa_xdp(struct net_device *net_dev, struct netdev_bpf *xdp)
3060{
3061	switch (xdp->command) {
3062	case XDP_SETUP_PROG:
3063		return dpaa_setup_xdp(net_dev, xdp);
3064	default:
3065		return -EINVAL;
3066	}
3067}
3068
3069static int dpaa_xdp_xmit(struct net_device *net_dev, int n,
3070			 struct xdp_frame **frames, u32 flags)
3071{
3072	struct xdp_frame *xdpf;
3073	int i, nxmit = 0;
3074
3075	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
3076		return -EINVAL;
3077
3078	if (!netif_running(net_dev))
3079		return -ENETDOWN;
3080
3081	for (i = 0; i < n; i++) {
3082		xdpf = frames[i];
3083		if (dpaa_xdp_xmit_frame(net_dev, xdpf))
3084			break;
3085		nxmit++;
3086	}
3087
3088	return nxmit;
3089}
3090
3091static int dpaa_hwtstamp_get(struct net_device *dev,
3092			     struct kernel_hwtstamp_config *config)
3093{
3094	struct dpaa_priv *priv = netdev_priv(dev);
3095
3096	config->tx_type = priv->tx_tstamp ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
3097	config->rx_filter = priv->rx_tstamp ? HWTSTAMP_FILTER_ALL :
3098			    HWTSTAMP_FILTER_NONE;
3099
3100	return 0;
3101}
3102
3103static int dpaa_hwtstamp_set(struct net_device *dev,
3104			     struct kernel_hwtstamp_config *config,
3105			     struct netlink_ext_ack *extack)
3106{
3107	struct dpaa_priv *priv = netdev_priv(dev);
3108
3109	switch (config->tx_type) {
3110	case HWTSTAMP_TX_OFF:
3111		/* Couldn't disable rx/tx timestamping separately.
3112		 * Do nothing here.
3113		 */
3114		priv->tx_tstamp = false;
3115		break;
3116	case HWTSTAMP_TX_ON:
3117		priv->mac_dev->set_tstamp(priv->mac_dev->fman_mac, true);
3118		priv->tx_tstamp = true;
3119		break;
3120	default:
3121		return -ERANGE;
3122	}
3123
3124	if (config->rx_filter == HWTSTAMP_FILTER_NONE) {
3125		/* Couldn't disable rx/tx timestamping separately.
3126		 * Do nothing here.
3127		 */
3128		priv->rx_tstamp = false;
3129	} else {
3130		priv->mac_dev->set_tstamp(priv->mac_dev->fman_mac, true);
3131		priv->rx_tstamp = true;
3132		/* TS is set for all frame types, not only those requested */
3133		config->rx_filter = HWTSTAMP_FILTER_ALL;
3134	}
3135
3136	return 0;
3137}
3138
3139static int dpaa_ioctl(struct net_device *net_dev, struct ifreq *rq, int cmd)
3140{
3141	struct dpaa_priv *priv = netdev_priv(net_dev);
3142
3143	return phylink_mii_ioctl(priv->mac_dev->phylink, rq, cmd);
3144}
3145
3146static const struct net_device_ops dpaa_ops = {
3147	.ndo_open = dpaa_open,
3148	.ndo_start_xmit = dpaa_start_xmit,
3149	.ndo_stop = dpaa_eth_stop,
3150	.ndo_tx_timeout = dpaa_tx_timeout,
3151	.ndo_get_stats64 = dpaa_get_stats64,
3152	.ndo_set_mac_address = dpaa_set_mac_address,
3153	.ndo_validate_addr = eth_validate_addr,
3154	.ndo_set_rx_mode = dpaa_set_rx_mode,
3155	.ndo_eth_ioctl = dpaa_ioctl,
3156	.ndo_setup_tc = dpaa_setup_tc,
3157	.ndo_change_mtu = dpaa_change_mtu,
3158	.ndo_bpf = dpaa_xdp,
3159	.ndo_xdp_xmit = dpaa_xdp_xmit,
3160	.ndo_hwtstamp_get = dpaa_hwtstamp_get,
3161	.ndo_hwtstamp_set = dpaa_hwtstamp_set,
3162};
3163
3164static int dpaa_napi_add(struct net_device *net_dev)
3165{
3166	struct dpaa_priv *priv = netdev_priv(net_dev);
3167	struct dpaa_percpu_priv *percpu_priv;
3168	int cpu;
3169
3170	for_each_possible_cpu(cpu) {
3171		percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
3172
3173		netif_napi_add(net_dev, &percpu_priv->np.napi, dpaa_eth_poll);
3174	}
3175
3176	return 0;
3177}
3178
3179static void dpaa_napi_del(struct net_device *net_dev)
3180{
3181	struct dpaa_priv *priv = netdev_priv(net_dev);
3182	struct dpaa_percpu_priv *percpu_priv;
3183	int cpu;
3184
3185	for_each_possible_cpu(cpu) {
3186		percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
3187
3188		__netif_napi_del(&percpu_priv->np.napi);
3189	}
3190	synchronize_net();
3191}
3192
3193static inline void dpaa_bp_free_pf(const struct dpaa_bp *bp,
3194				   struct bm_buffer *bmb)
3195{
3196	dma_addr_t addr = bm_buf_addr(bmb);
3197
3198	dma_unmap_page(bp->priv->rx_dma_dev, addr, DPAA_BP_RAW_SIZE,
3199		       DMA_FROM_DEVICE);
3200
3201	skb_free_frag(phys_to_virt(addr));
3202}
3203
3204/* Alloc the dpaa_bp struct and configure default values */
3205static struct dpaa_bp *dpaa_bp_alloc(struct device *dev)
3206{
3207	struct dpaa_bp *dpaa_bp;
3208
3209	dpaa_bp = devm_kzalloc(dev, sizeof(*dpaa_bp), GFP_KERNEL);
3210	if (!dpaa_bp)
3211		return ERR_PTR(-ENOMEM);
3212
3213	dpaa_bp->bpid = FSL_DPAA_BPID_INV;
3214	dpaa_bp->percpu_count = devm_alloc_percpu(dev, *dpaa_bp->percpu_count);
3215	if (!dpaa_bp->percpu_count)
3216		return ERR_PTR(-ENOMEM);
3217
3218	dpaa_bp->config_count = FSL_DPAA_ETH_MAX_BUF_COUNT;
3219
3220	dpaa_bp->seed_cb = dpaa_bp_seed;
3221	dpaa_bp->free_buf_cb = dpaa_bp_free_pf;
3222
3223	return dpaa_bp;
3224}
3225
3226/* Place all ingress FQs (Rx Default, Rx Error) in a dedicated CGR.
3227 * We won't be sending congestion notifications to FMan; for now, we just use
3228 * this CGR to generate enqueue rejections to FMan in order to drop the frames
3229 * before they reach our ingress queues and eat up memory.
3230 */
3231static int dpaa_ingress_cgr_init(struct dpaa_priv *priv)
3232{
3233	struct qm_mcc_initcgr initcgr;
3234	u32 cs_th;
3235	int err;
3236
3237	err = qman_alloc_cgrid(&priv->ingress_cgr.cgrid);
3238	if (err < 0) {
3239		if (netif_msg_drv(priv))
3240			pr_err("Error %d allocating CGR ID\n", err);
3241		goto out_error;
3242	}
3243
3244	/* Enable CS TD, but disable Congestion State Change Notifications. */
3245	memset(&initcgr, 0, sizeof(initcgr));
3246	initcgr.we_mask = cpu_to_be16(QM_CGR_WE_CS_THRES);
3247	initcgr.cgr.cscn_en = QM_CGR_EN;
3248	cs_th = DPAA_INGRESS_CS_THRESHOLD;
3249	qm_cgr_cs_thres_set64(&initcgr.cgr.cs_thres, cs_th, 1);
3250
3251	initcgr.we_mask |= cpu_to_be16(QM_CGR_WE_CSTD_EN);
3252	initcgr.cgr.cstd_en = QM_CGR_EN;
3253
3254	/* This CGR will be associated with the SWP affined to the current CPU.
3255	 * However, we'll place all our ingress FQs in it.
3256	 */
3257	err = qman_create_cgr(&priv->ingress_cgr, QMAN_CGR_FLAG_USE_INIT,
3258			      &initcgr);
3259	if (err < 0) {
3260		if (netif_msg_drv(priv))
3261			pr_err("Error %d creating ingress CGR with ID %d\n",
3262			       err, priv->ingress_cgr.cgrid);
3263		qman_release_cgrid(priv->ingress_cgr.cgrid);
3264		goto out_error;
3265	}
3266	if (netif_msg_drv(priv))
3267		pr_debug("Created ingress CGR %d for netdev with hwaddr %pM\n",
3268			 priv->ingress_cgr.cgrid, priv->mac_dev->addr);
3269
3270	priv->use_ingress_cgr = true;
3271
3272out_error:
3273	return err;
3274}
3275
3276static u16 dpaa_get_headroom(struct dpaa_buffer_layout *bl,
3277			     enum port_type port)
3278{
3279	u16 headroom;
3280
3281	/* The frame headroom must accommodate:
3282	 * - the driver private data area
3283	 * - parse results, hash results, timestamp if selected
3284	 * If either hash results or time stamp are selected, both will
3285	 * be copied to/from the frame headroom, as TS is located between PR and
3286	 * HR in the IC and IC copy size has a granularity of 16bytes
3287	 * (see description of FMBM_RICP and FMBM_TICP registers in DPAARM)
3288	 *
3289	 * Also make sure the headroom is a multiple of data_align bytes
3290	 */
3291	headroom = (u16)(bl[port].priv_data_size + DPAA_HWA_SIZE);
3292
3293	if (port == RX) {
3294#ifdef CONFIG_DPAA_ERRATUM_A050385
3295		if (unlikely(fman_has_errata_a050385()))
3296			headroom = XDP_PACKET_HEADROOM;
3297#endif
3298
3299		return ALIGN(headroom, DPAA_FD_RX_DATA_ALIGNMENT);
3300	} else {
3301		return ALIGN(headroom, DPAA_FD_DATA_ALIGNMENT);
3302	}
3303}
3304
3305static int dpaa_eth_probe(struct platform_device *pdev)
3306{
3307	struct net_device *net_dev = NULL;
3308	struct dpaa_bp *dpaa_bp = NULL;
3309	struct dpaa_fq *dpaa_fq, *tmp;
3310	struct dpaa_priv *priv = NULL;
3311	struct fm_port_fqs port_fqs;
3312	struct mac_device *mac_dev;
3313	int err = 0, channel;
3314	struct device *dev;
3315
3316	dev = &pdev->dev;
3317
3318	err = bman_is_probed();
3319	if (!err)
3320		return -EPROBE_DEFER;
3321	if (err < 0) {
3322		dev_err(dev, "failing probe due to bman probe error\n");
3323		return -ENODEV;
3324	}
3325	err = qman_is_probed();
3326	if (!err)
3327		return -EPROBE_DEFER;
3328	if (err < 0) {
3329		dev_err(dev, "failing probe due to qman probe error\n");
3330		return -ENODEV;
3331	}
3332	err = bman_portals_probed();
3333	if (!err)
3334		return -EPROBE_DEFER;
3335	if (err < 0) {
3336		dev_err(dev,
3337			"failing probe due to bman portals probe error\n");
3338		return -ENODEV;
3339	}
3340	err = qman_portals_probed();
3341	if (!err)
3342		return -EPROBE_DEFER;
3343	if (err < 0) {
3344		dev_err(dev,
3345			"failing probe due to qman portals probe error\n");
3346		return -ENODEV;
3347	}
3348
3349	/* Allocate this early, so we can store relevant information in
3350	 * the private area
3351	 */
3352	net_dev = alloc_etherdev_mq(sizeof(*priv), dpaa_max_num_txqs());
3353	if (!net_dev) {
3354		dev_err(dev, "alloc_etherdev_mq() failed\n");
3355		return -ENOMEM;
3356	}
3357
3358	/* Do this here, so we can be verbose early */
3359	SET_NETDEV_DEV(net_dev, dev->parent);
3360	dev_set_drvdata(dev, net_dev);
3361
3362	priv = netdev_priv(net_dev);
3363	priv->net_dev = net_dev;
3364
3365	priv->msg_enable = netif_msg_init(debug, DPAA_MSG_DEFAULT);
3366
3367	priv->egress_fqs = devm_kcalloc(dev, dpaa_max_num_txqs(),
3368					sizeof(*priv->egress_fqs),
3369					GFP_KERNEL);
3370	if (!priv->egress_fqs) {
3371		err = -ENOMEM;
3372		goto free_netdev;
3373	}
3374
3375	priv->conf_fqs = devm_kcalloc(dev, dpaa_max_num_txqs(),
3376				      sizeof(*priv->conf_fqs),
3377				      GFP_KERNEL);
3378	if (!priv->conf_fqs) {
3379		err = -ENOMEM;
3380		goto free_netdev;
3381	}
3382
3383	mac_dev = dpaa_mac_dev_get(pdev);
3384	if (IS_ERR(mac_dev)) {
3385		netdev_err(net_dev, "dpaa_mac_dev_get() failed\n");
3386		err = PTR_ERR(mac_dev);
3387		goto free_netdev;
3388	}
3389
3390	/* Devices used for DMA mapping */
3391	priv->rx_dma_dev = fman_port_get_device(mac_dev->port[RX]);
3392	priv->tx_dma_dev = fman_port_get_device(mac_dev->port[TX]);
3393	err = dma_coerce_mask_and_coherent(priv->rx_dma_dev, DMA_BIT_MASK(40));
3394	if (!err)
3395		err = dma_coerce_mask_and_coherent(priv->tx_dma_dev,
3396						   DMA_BIT_MASK(40));
3397	if (err) {
3398		netdev_err(net_dev, "dma_coerce_mask_and_coherent() failed\n");
3399		goto free_netdev;
3400	}
3401
3402	/* If fsl_fm_max_frm is set to a higher value than the all-common 1500,
3403	 * we choose conservatively and let the user explicitly set a higher
3404	 * MTU via ifconfig. Otherwise, the user may end up with different MTUs
3405	 * in the same LAN.
3406	 * If on the other hand fsl_fm_max_frm has been chosen below 1500,
3407	 * start with the maximum allowed.
3408	 */
3409	net_dev->mtu = min(dpaa_get_max_mtu(), ETH_DATA_LEN);
3410
3411	netdev_dbg(net_dev, "Setting initial MTU on net device: %d\n",
3412		   net_dev->mtu);
3413
3414	priv->buf_layout[RX].priv_data_size = DPAA_RX_PRIV_DATA_SIZE; /* Rx */
3415	priv->buf_layout[TX].priv_data_size = DPAA_TX_PRIV_DATA_SIZE; /* Tx */
3416
3417	/* bp init */
3418	dpaa_bp = dpaa_bp_alloc(dev);
3419	if (IS_ERR(dpaa_bp)) {
3420		err = PTR_ERR(dpaa_bp);
3421		goto free_dpaa_bps;
3422	}
3423	/* the raw size of the buffers used for reception */
3424	dpaa_bp->raw_size = DPAA_BP_RAW_SIZE;
3425	/* avoid runtime computations by keeping the usable size here */
3426	dpaa_bp->size = dpaa_bp_size(dpaa_bp->raw_size);
3427	dpaa_bp->priv = priv;
3428
3429	err = dpaa_bp_alloc_pool(dpaa_bp);
3430	if (err < 0)
3431		goto free_dpaa_bps;
3432	priv->dpaa_bp = dpaa_bp;
3433
3434	INIT_LIST_HEAD(&priv->dpaa_fq_list);
3435
3436	memset(&port_fqs, 0, sizeof(port_fqs));
3437
3438	err = dpaa_alloc_all_fqs(dev, &priv->dpaa_fq_list, &port_fqs);
3439	if (err < 0) {
3440		dev_err(dev, "dpaa_alloc_all_fqs() failed\n");
3441		goto free_dpaa_bps;
3442	}
3443
3444	priv->mac_dev = mac_dev;
3445
3446	channel = dpaa_get_channel();
3447	if (channel < 0) {
3448		dev_err(dev, "dpaa_get_channel() failed\n");
3449		err = channel;
3450		goto free_dpaa_bps;
3451	}
3452
3453	priv->channel = (u16)channel;
3454
3455	/* Walk the CPUs with affine portals
3456	 * and add this pool channel to each's dequeue mask.
3457	 */
3458	dpaa_eth_add_channel(priv->channel, &pdev->dev);
3459
3460	err = dpaa_fq_setup(priv, &dpaa_fq_cbs, priv->mac_dev->port[TX]);
3461	if (err)
3462		goto free_dpaa_bps;
3463
3464	/* Create a congestion group for this netdev, with
3465	 * dynamically-allocated CGR ID.
3466	 * Must be executed after probing the MAC, but before
3467	 * assigning the egress FQs to the CGRs.
3468	 */
3469	err = dpaa_eth_cgr_init(priv);
3470	if (err < 0) {
3471		dev_err(dev, "Error initializing CGR\n");
3472		goto free_dpaa_bps;
3473	}
3474
3475	err = dpaa_ingress_cgr_init(priv);
3476	if (err < 0) {
3477		dev_err(dev, "Error initializing ingress CGR\n");
3478		goto delete_egress_cgr;
3479	}
3480
3481	/* Add the FQs to the interface, and make them active */
3482	list_for_each_entry_safe(dpaa_fq, tmp, &priv->dpaa_fq_list, list) {
3483		err = dpaa_fq_init(dpaa_fq, false);
3484		if (err < 0)
3485			goto free_dpaa_fqs;
3486	}
3487
3488	priv->tx_headroom = dpaa_get_headroom(priv->buf_layout, TX);
3489	priv->rx_headroom = dpaa_get_headroom(priv->buf_layout, RX);
3490
3491	/* All real interfaces need their ports initialized */
3492	err = dpaa_eth_init_ports(mac_dev, dpaa_bp, &port_fqs,
3493				  &priv->buf_layout[0], dev);
3494	if (err)
3495		goto free_dpaa_fqs;
3496
3497	/* Rx traffic distribution based on keygen hashing defaults to on */
3498	priv->keygen_in_use = true;
3499
3500	priv->percpu_priv = devm_alloc_percpu(dev, *priv->percpu_priv);
3501	if (!priv->percpu_priv) {
3502		dev_err(dev, "devm_alloc_percpu() failed\n");
3503		err = -ENOMEM;
3504		goto free_dpaa_fqs;
3505	}
3506
3507	priv->num_tc = 1;
3508	netif_set_real_num_tx_queues(net_dev,
3509				     priv->num_tc * dpaa_num_txqs_per_tc());
3510
3511	/* Initialize NAPI */
3512	err = dpaa_napi_add(net_dev);
3513	if (err < 0)
3514		goto delete_dpaa_napi;
3515
3516	err = dpaa_netdev_init(net_dev, &dpaa_ops, tx_timeout);
3517	if (err < 0)
3518		goto delete_dpaa_napi;
3519
3520	dpaa_eth_sysfs_init(&net_dev->dev);
3521
3522	netif_info(priv, probe, net_dev, "Probed interface %s\n",
3523		   net_dev->name);
3524
3525	return 0;
3526
3527delete_dpaa_napi:
3528	dpaa_napi_del(net_dev);
3529free_dpaa_fqs:
3530	dpaa_fq_free(dev, &priv->dpaa_fq_list);
3531	qman_delete_cgr_safe(&priv->ingress_cgr);
3532	qman_release_cgrid(priv->ingress_cgr.cgrid);
3533delete_egress_cgr:
3534	qman_delete_cgr_safe(&priv->cgr_data.cgr);
3535	qman_release_cgrid(priv->cgr_data.cgr.cgrid);
3536free_dpaa_bps:
3537	dpaa_bps_free(priv);
3538free_netdev:
3539	dev_set_drvdata(dev, NULL);
3540	free_netdev(net_dev);
3541
3542	return err;
3543}
3544
3545static void dpaa_remove(struct platform_device *pdev)
3546{
3547	struct net_device *net_dev;
3548	struct dpaa_priv *priv;
3549	struct device *dev;
3550	int err;
3551
3552	dev = &pdev->dev;
3553	net_dev = dev_get_drvdata(dev);
3554
3555	priv = netdev_priv(net_dev);
3556
3557	dpaa_eth_sysfs_remove(dev);
3558
3559	dev_set_drvdata(dev, NULL);
3560	unregister_netdev(net_dev);
3561	phylink_destroy(priv->mac_dev->phylink);
3562
3563	err = dpaa_fq_free(dev, &priv->dpaa_fq_list);
3564	if (err)
3565		dev_err(dev, "Failed to free FQs on remove (%pE)\n",
3566			ERR_PTR(err));
3567
3568	qman_delete_cgr_safe(&priv->ingress_cgr);
3569	qman_release_cgrid(priv->ingress_cgr.cgrid);
3570	qman_delete_cgr_safe(&priv->cgr_data.cgr);
3571	qman_release_cgrid(priv->cgr_data.cgr.cgrid);
3572
3573	dpaa_napi_del(net_dev);
3574
3575	dpaa_bps_free(priv);
3576
3577	free_netdev(net_dev);
3578}
3579
3580static const struct platform_device_id dpaa_devtype[] = {
3581	{
3582		.name = "dpaa-ethernet",
3583		.driver_data = 0,
3584	}, {
3585	}
3586};
3587MODULE_DEVICE_TABLE(platform, dpaa_devtype);
3588
3589static struct platform_driver dpaa_driver = {
3590	.driver = {
3591		.name = KBUILD_MODNAME,
3592	},
3593	.id_table = dpaa_devtype,
3594	.probe = dpaa_eth_probe,
3595	.remove = dpaa_remove
3596};
3597
3598static int __init dpaa_load(void)
3599{
3600	int err;
3601
3602	pr_debug("FSL DPAA Ethernet driver\n");
3603
3604	/* initialize dpaa_eth mirror values */
3605	dpaa_rx_extra_headroom = fman_get_rx_extra_headroom();
3606	dpaa_max_frm = fman_get_max_frm();
3607
3608	err = platform_driver_register(&dpaa_driver);
3609	if (err < 0)
3610		pr_err("Error, platform_driver_register() = %d\n", err);
3611
3612	return err;
3613}
3614module_init(dpaa_load);
3615
3616static void __exit dpaa_unload(void)
3617{
3618	platform_driver_unregister(&dpaa_driver);
3619
3620	/* Only one channel is used and needs to be released after all
3621	 * interfaces are removed
3622	 */
3623	dpaa_release_channel();
3624}
3625module_exit(dpaa_unload);
3626
3627MODULE_LICENSE("Dual BSD/GPL");
3628MODULE_DESCRIPTION("FSL DPAA Ethernet driver");