mana_en.c

   1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2/* Copyright (c) 2021, Microsoft Corporation. */
   3
   4#include <uapi/linux/bpf.h>
   5
   6#include <linux/debugfs.h>
   7#include <linux/inetdevice.h>
   8#include <linux/etherdevice.h>
   9#include <linux/ethtool.h>
  10#include <linux/filter.h>
  11#include <linux/mm.h>
  12#include <linux/pci.h>
  13#include <linux/export.h>
  14#include <linux/skbuff.h>
  15
  16#include <net/checksum.h>
  17#include <net/ip6_checksum.h>
  18#include <net/netdev_lock.h>
  19#include <net/page_pool/helpers.h>
  20#include <net/xdp.h>
  21
  22#include <net/mana/mana.h>
  23#include <net/mana/mana_auxiliary.h>
  24#include <net/mana/hw_channel.h>
  25
  26static DEFINE_IDA(mana_adev_ida);
  27
  28static int mana_adev_idx_alloc(void)
  29{
  30	return ida_alloc(&mana_adev_ida, GFP_KERNEL);
  31}
  32
  33static void mana_adev_idx_free(int idx)
  34{
  35	ida_free(&mana_adev_ida, idx);
  36}
  37
  38static ssize_t mana_dbg_q_read(struct file *filp, char __user *buf, size_t count,
  39			       loff_t *pos)
  40{
  41	struct gdma_queue *gdma_q = filp->private_data;
  42
  43	return simple_read_from_buffer(buf, count, pos, gdma_q->queue_mem_ptr,
  44				       gdma_q->queue_size);
  45}
  46
  47static const struct file_operations mana_dbg_q_fops = {
  48	.owner  = THIS_MODULE,
  49	.open   = simple_open,
  50	.read   = mana_dbg_q_read,
  51};
  52
  53static bool mana_en_need_log(struct mana_port_context *apc, int err)
  54{
  55	if (apc && apc->ac && apc->ac->gdma_dev &&
  56	    apc->ac->gdma_dev->gdma_context)
  57		return mana_need_log(apc->ac->gdma_dev->gdma_context, err);
  58	else
  59		return true;
  60}
  61
  62static void mana_put_rx_page(struct mana_rxq *rxq, struct page *page,
  63			     bool from_pool)
  64{
  65	if (from_pool)
  66		page_pool_put_full_page(rxq->page_pool, page, false);
  67	else
  68		put_page(page);
  69}
  70
  71/* Microsoft Azure Network Adapter (MANA) functions */
  72
  73static int mana_open(struct net_device *ndev)
  74{
  75	struct mana_port_context *apc = netdev_priv(ndev);
  76	int err;
  77	err = mana_alloc_queues(ndev);
  78
  79	if (err) {
  80		netdev_err(ndev, "%s failed to allocate queues: %d\n", __func__, err);
  81		return err;
  82	}
  83
  84	apc->port_is_up = true;
  85
  86	/* Ensure port state updated before txq state */
  87	smp_wmb();
  88
  89	netif_tx_wake_all_queues(ndev);
  90	netdev_dbg(ndev, "%s successful\n", __func__);
  91	return 0;
  92}
  93
  94static int mana_close(struct net_device *ndev)
  95{
  96	struct mana_port_context *apc = netdev_priv(ndev);
  97
  98	if (!apc->port_is_up)
  99		return 0;
 100
 101	return mana_detach(ndev, true);
 102}
 103
 104static void mana_link_state_handle(struct work_struct *w)
 105{
 106	struct mana_context *ac;
 107	struct net_device *ndev;
 108	u32 link_event;
 109	bool link_up;
 110	int i;
 111
 112	ac = container_of(w, struct mana_context, link_change_work);
 113
 114	rtnl_lock();
 115
 116	link_event = READ_ONCE(ac->link_event);
 117
 118	if (link_event == HWC_DATA_HW_LINK_CONNECT)
 119		link_up = true;
 120	else if (link_event == HWC_DATA_HW_LINK_DISCONNECT)
 121		link_up = false;
 122	else
 123		goto out;
 124
 125	/* Process all ports */
 126	for (i = 0; i < ac->num_ports; i++) {
 127		ndev = ac->ports[i];
 128		if (!ndev)
 129			continue;
 130
 131		if (link_up) {
 132			netif_carrier_on(ndev);
 133
 134			__netdev_notify_peers(ndev);
 135		} else {
 136			netif_carrier_off(ndev);
 137		}
 138	}
 139
 140out:
 141	rtnl_unlock();
 142}
 143
 144static bool mana_can_tx(struct gdma_queue *wq)
 145{
 146	return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE;
 147}
 148
 149static unsigned int mana_checksum_info(struct sk_buff *skb)
 150{
 151	if (skb->protocol == htons(ETH_P_IP)) {
 152		struct iphdr *ip = ip_hdr(skb);
 153
 154		if (ip->protocol == IPPROTO_TCP)
 155			return IPPROTO_TCP;
 156
 157		if (ip->protocol == IPPROTO_UDP)
 158			return IPPROTO_UDP;
 159	} else if (skb->protocol == htons(ETH_P_IPV6)) {
 160		struct ipv6hdr *ip6 = ipv6_hdr(skb);
 161
 162		if (ip6->nexthdr == IPPROTO_TCP)
 163			return IPPROTO_TCP;
 164
 165		if (ip6->nexthdr == IPPROTO_UDP)
 166			return IPPROTO_UDP;
 167	}
 168
 169	/* No csum offloading */
 170	return 0;
 171}
 172
 173static void mana_add_sge(struct mana_tx_package *tp, struct mana_skb_head *ash,
 174			 int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey)
 175{
 176	ash->dma_handle[sg_i] = da;
 177	ash->size[sg_i] = sge_len;
 178
 179	tp->wqe_req.sgl[sg_i].address = da;
 180	tp->wqe_req.sgl[sg_i].mem_key = gpa_mkey;
 181	tp->wqe_req.sgl[sg_i].size = sge_len;
 182}
 183
 184static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc,
 185			struct mana_tx_package *tp, int gso_hs)
 186{
 187	struct mana_skb_head *ash = (struct mana_skb_head *)skb->head;
 188	int hsg = 1; /* num of SGEs of linear part */
 189	struct gdma_dev *gd = apc->ac->gdma_dev;
 190	int skb_hlen = skb_headlen(skb);
 191	int sge0_len, sge1_len = 0;
 192	struct gdma_context *gc;
 193	struct device *dev;
 194	skb_frag_t *frag;
 195	dma_addr_t da;
 196	int sg_i;
 197	int i;
 198
 199	gc = gd->gdma_context;
 200	dev = gc->dev;
 201
 202	if (gso_hs && gso_hs < skb_hlen) {
 203		sge0_len = gso_hs;
 204		sge1_len = skb_hlen - gso_hs;
 205	} else {
 206		sge0_len = skb_hlen;
 207	}
 208
 209	da = dma_map_single(dev, skb->data, sge0_len, DMA_TO_DEVICE);
 210	if (dma_mapping_error(dev, da))
 211		return -ENOMEM;
 212
 213	mana_add_sge(tp, ash, 0, da, sge0_len, gd->gpa_mkey);
 214
 215	if (sge1_len) {
 216		sg_i = 1;
 217		da = dma_map_single(dev, skb->data + sge0_len, sge1_len,
 218				    DMA_TO_DEVICE);
 219		if (dma_mapping_error(dev, da))
 220			goto frag_err;
 221
 222		mana_add_sge(tp, ash, sg_i, da, sge1_len, gd->gpa_mkey);
 223		hsg = 2;
 224	}
 225
 226	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 227		sg_i = hsg + i;
 228
 229		frag = &skb_shinfo(skb)->frags[i];
 230		da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag),
 231				      DMA_TO_DEVICE);
 232		if (dma_mapping_error(dev, da))
 233			goto frag_err;
 234
 235		mana_add_sge(tp, ash, sg_i, da, skb_frag_size(frag),
 236			     gd->gpa_mkey);
 237	}
 238
 239	return 0;
 240
 241frag_err:
 242	if (net_ratelimit())
 243		netdev_err(apc->ndev, "Failed to map skb of size %u to DMA\n",
 244			   skb->len);
 245	for (i = sg_i - 1; i >= hsg; i--)
 246		dma_unmap_page(dev, ash->dma_handle[i], ash->size[i],
 247			       DMA_TO_DEVICE);
 248
 249	for (i = hsg - 1; i >= 0; i--)
 250		dma_unmap_single(dev, ash->dma_handle[i], ash->size[i],
 251				 DMA_TO_DEVICE);
 252
 253	return -ENOMEM;
 254}
 255
 256/* Handle the case when GSO SKB linear length is too large.
 257 * MANA NIC requires GSO packets to put only the packet header to SGE0.
 258 * So, we need 2 SGEs for the skb linear part which contains more than the
 259 * header.
 260 * Return a positive value for the number of SGEs, or a negative value
 261 * for an error.
 262 */
 263static int mana_fix_skb_head(struct net_device *ndev, struct sk_buff *skb,
 264			     int gso_hs)
 265{
 266	int num_sge = 1 + skb_shinfo(skb)->nr_frags;
 267	int skb_hlen = skb_headlen(skb);
 268
 269	if (gso_hs < skb_hlen) {
 270		num_sge++;
 271	} else if (gso_hs > skb_hlen) {
 272		if (net_ratelimit())
 273			netdev_err(ndev,
 274				   "TX nonlinear head: hs:%d, skb_hlen:%d\n",
 275				   gso_hs, skb_hlen);
 276
 277		return -EINVAL;
 278	}
 279
 280	return num_sge;
 281}
 282
 283/* Get the GSO packet's header size */
 284static int mana_get_gso_hs(struct sk_buff *skb)
 285{
 286	int gso_hs;
 287
 288	if (skb->encapsulation) {
 289		gso_hs = skb_inner_tcp_all_headers(skb);
 290	} else {
 291		if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
 292			gso_hs = skb_transport_offset(skb) +
 293				 sizeof(struct udphdr);
 294		} else {
 295			gso_hs = skb_tcp_all_headers(skb);
 296		}
 297	}
 298
 299	return gso_hs;
 300}
 301
 302netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 303{
 304	enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT;
 305	struct mana_port_context *apc = netdev_priv(ndev);
 306	int gso_hs = 0; /* zero for non-GSO pkts */
 307	u16 txq_idx = skb_get_queue_mapping(skb);
 308	struct gdma_dev *gd = apc->ac->gdma_dev;
 309	bool ipv4 = false, ipv6 = false;
 310	struct mana_tx_package pkg = {};
 311	struct netdev_queue *net_txq;
 312	struct mana_stats_tx *tx_stats;
 313	struct gdma_queue *gdma_sq;
 314	int err, len, num_gso_seg;
 315	unsigned int csum_type;
 316	struct mana_txq *txq;
 317	struct mana_cq *cq;
 318
 319	if (unlikely(!apc->port_is_up))
 320		goto tx_drop;
 321
 322	if (skb_cow_head(skb, MANA_HEADROOM))
 323		goto tx_drop_count;
 324
 325	if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
 326		goto tx_drop_count;
 327
 328	txq = &apc->tx_qp[txq_idx].txq;
 329	gdma_sq = txq->gdma_sq;
 330	cq = &apc->tx_qp[txq_idx].tx_cq;
 331	tx_stats = &txq->stats;
 332
 333	BUILD_BUG_ON(MAX_TX_WQE_SGL_ENTRIES != MANA_MAX_TX_WQE_SGL_ENTRIES);
 334	if (MAX_SKB_FRAGS + 2 > MAX_TX_WQE_SGL_ENTRIES &&
 335	    skb_shinfo(skb)->nr_frags + 2 > MAX_TX_WQE_SGL_ENTRIES) {
 336		/* GSO skb with Hardware SGE limit exceeded is not expected here
 337		 * as they are handled in mana_features_check() callback
 338		 */
 339		if (skb_linearize(skb)) {
 340			netdev_warn_once(ndev, "Failed to linearize skb with nr_frags=%d and is_gso=%d\n",
 341					 skb_shinfo(skb)->nr_frags,
 342					 skb_is_gso(skb));
 343			goto tx_drop_count;
 344		}
 345		apc->eth_stats.tx_linear_pkt_cnt++;
 346	}
 347
 348	pkg.tx_oob.s_oob.vcq_num = cq->gdma_id;
 349	pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame;
 350
 351	if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) {
 352		pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset;
 353		pkt_fmt = MANA_LONG_PKT_FMT;
 354	} else {
 355		pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset;
 356	}
 357
 358	if (skb_vlan_tag_present(skb)) {
 359		pkt_fmt = MANA_LONG_PKT_FMT;
 360		pkg.tx_oob.l_oob.inject_vlan_pri_tag = 1;
 361		pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb);
 362		pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb);
 363		pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb);
 364	}
 365
 366	pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt;
 367
 368	if (pkt_fmt == MANA_SHORT_PKT_FMT) {
 369		pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob);
 370		u64_stats_update_begin(&tx_stats->syncp);
 371		tx_stats->short_pkt_fmt++;
 372		u64_stats_update_end(&tx_stats->syncp);
 373	} else {
 374		pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob);
 375		u64_stats_update_begin(&tx_stats->syncp);
 376		tx_stats->long_pkt_fmt++;
 377		u64_stats_update_end(&tx_stats->syncp);
 378	}
 379
 380	pkg.wqe_req.inline_oob_data = &pkg.tx_oob;
 381	pkg.wqe_req.flags = 0;
 382	pkg.wqe_req.client_data_unit = 0;
 383
 384	pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags;
 385
 386	if (skb->protocol == htons(ETH_P_IP))
 387		ipv4 = true;
 388	else if (skb->protocol == htons(ETH_P_IPV6))
 389		ipv6 = true;
 390
 391	if (skb_is_gso(skb)) {
 392		int num_sge;
 393
 394		gso_hs = mana_get_gso_hs(skb);
 395
 396		num_sge = mana_fix_skb_head(ndev, skb, gso_hs);
 397		if (num_sge > 0)
 398			pkg.wqe_req.num_sge = num_sge;
 399		else
 400			goto tx_drop_count;
 401
 402		u64_stats_update_begin(&tx_stats->syncp);
 403		if (skb->encapsulation) {
 404			tx_stats->tso_inner_packets++;
 405			tx_stats->tso_inner_bytes += skb->len - gso_hs;
 406		} else {
 407			tx_stats->tso_packets++;
 408			tx_stats->tso_bytes += skb->len - gso_hs;
 409		}
 410		u64_stats_update_end(&tx_stats->syncp);
 411
 412		pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
 413		pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
 414
 415		pkg.tx_oob.s_oob.comp_iphdr_csum = 1;
 416		pkg.tx_oob.s_oob.comp_tcp_csum = 1;
 417		pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);
 418
 419		pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size;
 420		pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0;
 421		if (ipv4) {
 422			ip_hdr(skb)->tot_len = 0;
 423			ip_hdr(skb)->check = 0;
 424			tcp_hdr(skb)->check =
 425				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
 426						   ip_hdr(skb)->daddr, 0,
 427						   IPPROTO_TCP, 0);
 428		} else {
 429			ipv6_hdr(skb)->payload_len = 0;
 430			tcp_hdr(skb)->check =
 431				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
 432						 &ipv6_hdr(skb)->daddr, 0,
 433						 IPPROTO_TCP, 0);
 434		}
 435	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 436		csum_type = mana_checksum_info(skb);
 437
 438		u64_stats_update_begin(&tx_stats->syncp);
 439		tx_stats->csum_partial++;
 440		u64_stats_update_end(&tx_stats->syncp);
 441
 442		if (csum_type == IPPROTO_TCP) {
 443			pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
 444			pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
 445
 446			pkg.tx_oob.s_oob.comp_tcp_csum = 1;
 447			pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);
 448
 449		} else if (csum_type == IPPROTO_UDP) {
 450			pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
 451			pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
 452
 453			pkg.tx_oob.s_oob.comp_udp_csum = 1;
 454		} else {
 455			/* Can't do offload of this type of checksum */
 456			if (skb_checksum_help(skb))
 457				goto tx_drop_count;
 458		}
 459	}
 460
 461	if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) {
 462		pkg.wqe_req.sgl = pkg.sgl_array;
 463	} else {
 464		pkg.sgl_ptr = kmalloc_array(pkg.wqe_req.num_sge,
 465					    sizeof(struct gdma_sge),
 466					    GFP_ATOMIC);
 467		if (!pkg.sgl_ptr)
 468			goto tx_drop_count;
 469
 470		pkg.wqe_req.sgl = pkg.sgl_ptr;
 471	}
 472
 473	if (mana_map_skb(skb, apc, &pkg, gso_hs)) {
 474		u64_stats_update_begin(&tx_stats->syncp);
 475		tx_stats->mana_map_err++;
 476		u64_stats_update_end(&tx_stats->syncp);
 477		goto free_sgl_ptr;
 478	}
 479
 480	skb_queue_tail(&txq->pending_skbs, skb);
 481
 482	len = skb->len;
 483	num_gso_seg = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
 484	net_txq = netdev_get_tx_queue(ndev, txq_idx);
 485
 486	err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req,
 487					(struct gdma_posted_wqe_info *)skb->cb);
 488	if (!mana_can_tx(gdma_sq)) {
 489		netif_tx_stop_queue(net_txq);
 490		apc->eth_stats.stop_queue++;
 491	}
 492
 493	if (err) {
 494		(void)skb_dequeue_tail(&txq->pending_skbs);
 495		mana_unmap_skb(skb, apc);
 496		netdev_warn(ndev, "Failed to post TX OOB: %d\n", err);
 497		goto free_sgl_ptr;
 498	}
 499
 500	err = NETDEV_TX_OK;
 501	atomic_inc(&txq->pending_sends);
 502
 503	mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq);
 504
 505	/* skb may be freed after mana_gd_post_work_request. Do not use it. */
 506	skb = NULL;
 507
 508	/* Populated the packet and bytes counters based on post GSO packet
 509	 * calculations
 510	 */
 511	tx_stats = &txq->stats;
 512	u64_stats_update_begin(&tx_stats->syncp);
 513	tx_stats->packets += num_gso_seg;
 514	tx_stats->bytes += len + ((num_gso_seg - 1) * gso_hs);
 515	u64_stats_update_end(&tx_stats->syncp);
 516
 517	if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) {
 518		netif_tx_wake_queue(net_txq);
 519		apc->eth_stats.wake_queue++;
 520	}
 521
 522	kfree(pkg.sgl_ptr);
 523	return err;
 524
 525free_sgl_ptr:
 526	kfree(pkg.sgl_ptr);
 527tx_drop_count:
 528	ndev->stats.tx_dropped++;
 529tx_drop:
 530	dev_kfree_skb_any(skb);
 531	return NETDEV_TX_OK;
 532}
 533
 534#if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES)
 535static netdev_features_t mana_features_check(struct sk_buff *skb,
 536					     struct net_device *ndev,
 537					     netdev_features_t features)
 538{
 539	if (skb_shinfo(skb)->nr_frags + 2 > MAX_TX_WQE_SGL_ENTRIES) {
 540		/* Exceeds HW SGE limit.
 541		 * GSO case:
 542		 *   Disable GSO so the stack will software-segment the skb
 543		 *   into smaller skbs that fit the SGE budget.
 544		 * Non-GSO case:
 545		 *   The xmit path will attempt skb_linearize() as a fallback.
 546		 */
 547		features &= ~NETIF_F_GSO_MASK;
 548	}
 549	return features;
 550}
 551#endif
 552
 553static void mana_get_stats64(struct net_device *ndev,
 554			     struct rtnl_link_stats64 *st)
 555{
 556	struct mana_port_context *apc = netdev_priv(ndev);
 557	unsigned int num_queues = apc->num_queues;
 558	struct mana_stats_rx *rx_stats;
 559	struct mana_stats_tx *tx_stats;
 560	unsigned int start;
 561	u64 packets, bytes;
 562	int q;
 563
 564	if (!apc->port_is_up)
 565		return;
 566
 567	netdev_stats_to_stats64(st, &ndev->stats);
 568
 569	if (apc->ac->hwc_timeout_occurred)
 570		netdev_warn_once(ndev, "HWC timeout occurred\n");
 571
 572	st->rx_missed_errors = apc->ac->hc_stats.hc_rx_discards_no_wqe;
 573
 574	for (q = 0; q < num_queues; q++) {
 575		rx_stats = &apc->rxqs[q]->stats;
 576
 577		do {
 578			start = u64_stats_fetch_begin(&rx_stats->syncp);
 579			packets = rx_stats->packets;
 580			bytes = rx_stats->bytes;
 581		} while (u64_stats_fetch_retry(&rx_stats->syncp, start));
 582
 583		st->rx_packets += packets;
 584		st->rx_bytes += bytes;
 585	}
 586
 587	for (q = 0; q < num_queues; q++) {
 588		tx_stats = &apc->tx_qp[q].txq.stats;
 589
 590		do {
 591			start = u64_stats_fetch_begin(&tx_stats->syncp);
 592			packets = tx_stats->packets;
 593			bytes = tx_stats->bytes;
 594		} while (u64_stats_fetch_retry(&tx_stats->syncp, start));
 595
 596		st->tx_packets += packets;
 597		st->tx_bytes += bytes;
 598	}
 599}
 600
 601static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb,
 602			     int old_q)
 603{
 604	struct mana_port_context *apc = netdev_priv(ndev);
 605	u32 hash = skb_get_hash(skb);
 606	struct sock *sk = skb->sk;
 607	int txq;
 608
 609	txq = apc->indir_table[hash & (apc->indir_table_sz - 1)];
 610
 611	if (txq != old_q && sk && sk_fullsock(sk) &&
 612	    rcu_access_pointer(sk->sk_dst_cache))
 613		sk_tx_queue_set(sk, txq);
 614
 615	return txq;
 616}
 617
 618static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb,
 619			     struct net_device *sb_dev)
 620{
 621	int txq;
 622
 623	if (ndev->real_num_tx_queues == 1)
 624		return 0;
 625
 626	txq = sk_tx_queue_get(skb->sk);
 627
 628	if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) {
 629		if (skb_rx_queue_recorded(skb))
 630			txq = skb_get_rx_queue(skb);
 631		else
 632			txq = mana_get_tx_queue(ndev, skb, txq);
 633	}
 634
 635	return txq;
 636}
 637
 638/* Release pre-allocated RX buffers */
 639void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc)
 640{
 641	struct device *dev;
 642	int i;
 643
 644	dev = mpc->ac->gdma_dev->gdma_context->dev;
 645
 646	if (!mpc->rxbufs_pre)
 647		goto out1;
 648
 649	if (!mpc->das_pre)
 650		goto out2;
 651
 652	while (mpc->rxbpre_total) {
 653		i = --mpc->rxbpre_total;
 654		dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize,
 655				 DMA_FROM_DEVICE);
 656		put_page(virt_to_head_page(mpc->rxbufs_pre[i]));
 657	}
 658
 659	kfree(mpc->das_pre);
 660	mpc->das_pre = NULL;
 661
 662out2:
 663	kfree(mpc->rxbufs_pre);
 664	mpc->rxbufs_pre = NULL;
 665
 666out1:
 667	mpc->rxbpre_datasize = 0;
 668	mpc->rxbpre_alloc_size = 0;
 669	mpc->rxbpre_headroom = 0;
 670}
 671
 672/* Get a buffer from the pre-allocated RX buffers */
 673static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da)
 674{
 675	struct net_device *ndev = rxq->ndev;
 676	struct mana_port_context *mpc;
 677	void *va;
 678
 679	mpc = netdev_priv(ndev);
 680
 681	if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) {
 682		netdev_err(ndev, "No RX pre-allocated bufs\n");
 683		return NULL;
 684	}
 685
 686	/* Check sizes to catch unexpected coding error */
 687	if (mpc->rxbpre_datasize != rxq->datasize) {
 688		netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n",
 689			   mpc->rxbpre_datasize, rxq->datasize);
 690		return NULL;
 691	}
 692
 693	if (mpc->rxbpre_alloc_size != rxq->alloc_size) {
 694		netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n",
 695			   mpc->rxbpre_alloc_size, rxq->alloc_size);
 696		return NULL;
 697	}
 698
 699	if (mpc->rxbpre_headroom != rxq->headroom) {
 700		netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n",
 701			   mpc->rxbpre_headroom, rxq->headroom);
 702		return NULL;
 703	}
 704
 705	mpc->rxbpre_total--;
 706
 707	*da = mpc->das_pre[mpc->rxbpre_total];
 708	va = mpc->rxbufs_pre[mpc->rxbpre_total];
 709	mpc->rxbufs_pre[mpc->rxbpre_total] = NULL;
 710
 711	/* Deallocate the array after all buffers are gone */
 712	if (!mpc->rxbpre_total)
 713		mana_pre_dealloc_rxbufs(mpc);
 714
 715	return va;
 716}
 717
 718/* Get RX buffer's data size, alloc size, XDP headroom based on MTU */
 719static void mana_get_rxbuf_cfg(struct mana_port_context *apc,
 720			       int mtu, u32 *datasize, u32 *alloc_size,
 721			       u32 *headroom, u32 *frag_count)
 722{
 723	u32 len, buf_size;
 724
 725	/* Calculate datasize first (consistent across all cases) */
 726	*datasize = mtu + ETH_HLEN;
 727
 728	/* For xdp and jumbo frames make sure only one packet fits per page */
 729	if (mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || mana_xdp_get(apc)) {
 730		if (mana_xdp_get(apc)) {
 731			*headroom = XDP_PACKET_HEADROOM;
 732			*alloc_size = PAGE_SIZE;
 733		} else {
 734			*headroom = 0; /* no support for XDP */
 735			*alloc_size = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD +
 736						     *headroom);
 737		}
 738
 739		*frag_count = 1;
 740
 741		/* In the single-buffer path, napi_build_skb() must see the
 742		 * actual backing allocation size so skb->truesize reflects
 743		 * the full page (or higher-order page), not just the usable
 744		 * packet area.
 745		 */
 746		*alloc_size = PAGE_SIZE << get_order(*alloc_size);
 747		return;
 748	}
 749
 750	/* Standard MTU case - optimize for multiple packets per page */
 751	*headroom = 0;
 752
 753	/* Calculate base buffer size needed */
 754	len = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + *headroom);
 755	buf_size = ALIGN(len, MANA_RX_FRAG_ALIGNMENT);
 756
 757	/* Calculate how many packets can fit in a page */
 758	*frag_count = PAGE_SIZE / buf_size;
 759	*alloc_size = buf_size;
 760}
 761
 762int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu, int num_queues)
 763{
 764	struct device *dev;
 765	struct page *page;
 766	dma_addr_t da;
 767	int num_rxb;
 768	void *va;
 769	int i;
 770
 771	mana_get_rxbuf_cfg(mpc, new_mtu, &mpc->rxbpre_datasize,
 772			   &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom,
 773			   &mpc->rxbpre_frag_count);
 774
 775	dev = mpc->ac->gdma_dev->gdma_context->dev;
 776
 777	num_rxb = num_queues * mpc->rx_queue_size;
 778
 779	WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n");
 780	mpc->rxbufs_pre = kmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL);
 781	if (!mpc->rxbufs_pre)
 782		goto error;
 783
 784	mpc->das_pre = kmalloc_array(num_rxb, sizeof(dma_addr_t), GFP_KERNEL);
 785	if (!mpc->das_pre)
 786		goto error;
 787
 788	mpc->rxbpre_total = 0;
 789
 790	for (i = 0; i < num_rxb; i++) {
 791		page = dev_alloc_pages(get_order(mpc->rxbpre_alloc_size));
 792		if (!page)
 793			goto error;
 794
 795		va = page_to_virt(page);
 796
 797		da = dma_map_single(dev, va + mpc->rxbpre_headroom,
 798				    mpc->rxbpre_datasize, DMA_FROM_DEVICE);
 799		if (dma_mapping_error(dev, da)) {
 800			put_page(page);
 801			goto error;
 802		}
 803
 804		mpc->rxbufs_pre[i] = va;
 805		mpc->das_pre[i] = da;
 806		mpc->rxbpre_total = i + 1;
 807	}
 808
 809	return 0;
 810
 811error:
 812	netdev_err(mpc->ndev, "Failed to pre-allocate RX buffers for %d queues\n", num_queues);
 813	mana_pre_dealloc_rxbufs(mpc);
 814	return -ENOMEM;
 815}
 816
 817static int mana_change_mtu(struct net_device *ndev, int new_mtu)
 818{
 819	struct mana_port_context *mpc = netdev_priv(ndev);
 820	unsigned int old_mtu = ndev->mtu;
 821	int err;
 822
 823	/* Pre-allocate buffers to prevent failure in mana_attach later */
 824	err = mana_pre_alloc_rxbufs(mpc, new_mtu, mpc->num_queues);
 825	if (err) {
 826		netdev_err(ndev, "Insufficient memory for new MTU\n");
 827		return err;
 828	}
 829
 830	err = mana_detach(ndev, false);
 831	if (err) {
 832		netdev_err(ndev, "mana_detach failed: %d\n", err);
 833		goto out;
 834	}
 835
 836	WRITE_ONCE(ndev->mtu, new_mtu);
 837
 838	err = mana_attach(ndev);
 839	if (err) {
 840		netdev_err(ndev, "mana_attach failed: %d\n", err);
 841		WRITE_ONCE(ndev->mtu, old_mtu);
 842	}
 843
 844out:
 845	mana_pre_dealloc_rxbufs(mpc);
 846	return err;
 847}
 848
 849static int mana_shaper_set(struct net_shaper_binding *binding,
 850			   const struct net_shaper *shaper,
 851			   struct netlink_ext_ack *extack)
 852{
 853	struct mana_port_context *apc = netdev_priv(binding->netdev);
 854	u32 old_speed, rate;
 855	int err;
 856
 857	if (shaper->handle.scope != NET_SHAPER_SCOPE_NETDEV) {
 858		NL_SET_ERR_MSG_MOD(extack, "net shaper scope should be netdev");
 859		return -EINVAL;
 860	}
 861
 862	if (apc->handle.id && shaper->handle.id != apc->handle.id) {
 863		NL_SET_ERR_MSG_MOD(extack, "Cannot create multiple shapers");
 864		return -EOPNOTSUPP;
 865	}
 866
 867	if (!shaper->bw_max || (shaper->bw_max % 100000000)) {
 868		NL_SET_ERR_MSG_MOD(extack, "Please use multiples of 100Mbps for bandwidth");
 869		return -EINVAL;
 870	}
 871
 872	rate = div_u64(shaper->bw_max, 1000); /* Convert bps to Kbps */
 873	rate = div_u64(rate, 1000);	      /* Convert Kbps to Mbps */
 874
 875	/* Get current speed */
 876	err = mana_query_link_cfg(apc);
 877	old_speed = (err) ? SPEED_UNKNOWN : apc->speed;
 878
 879	if (!err) {
 880		err = mana_set_bw_clamp(apc, rate, TRI_STATE_TRUE);
 881		apc->speed = (err) ? old_speed : rate;
 882		apc->handle = (err) ? apc->handle : shaper->handle;
 883	}
 884
 885	return err;
 886}
 887
 888static int mana_shaper_del(struct net_shaper_binding *binding,
 889			   const struct net_shaper_handle *handle,
 890			   struct netlink_ext_ack *extack)
 891{
 892	struct mana_port_context *apc = netdev_priv(binding->netdev);
 893	int err;
 894
 895	err = mana_set_bw_clamp(apc, 0, TRI_STATE_FALSE);
 896
 897	if (!err) {
 898		/* Reset mana port context parameters */
 899		apc->handle.id = 0;
 900		apc->handle.scope = NET_SHAPER_SCOPE_UNSPEC;
 901		apc->speed = apc->max_speed;
 902	}
 903
 904	return err;
 905}
 906
 907static void mana_shaper_cap(struct net_shaper_binding *binding,
 908			    enum net_shaper_scope scope,
 909			    unsigned long *flags)
 910{
 911	*flags = BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MAX) |
 912		 BIT(NET_SHAPER_A_CAPS_SUPPORT_METRIC_BPS);
 913}
 914
 915static const struct net_shaper_ops mana_shaper_ops = {
 916	.set = mana_shaper_set,
 917	.delete = mana_shaper_del,
 918	.capabilities = mana_shaper_cap,
 919};
 920
 921static const struct net_device_ops mana_devops = {
 922	.ndo_open		= mana_open,
 923	.ndo_stop		= mana_close,
 924	.ndo_select_queue	= mana_select_queue,
 925#if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES)
 926	.ndo_features_check	= mana_features_check,
 927#endif
 928	.ndo_start_xmit		= mana_start_xmit,
 929	.ndo_validate_addr	= eth_validate_addr,
 930	.ndo_get_stats64	= mana_get_stats64,
 931	.ndo_bpf		= mana_bpf,
 932	.ndo_xdp_xmit		= mana_xdp_xmit,
 933	.ndo_change_mtu		= mana_change_mtu,
 934	.net_shaper_ops         = &mana_shaper_ops,
 935};
 936
 937static void mana_cleanup_port_context(struct mana_port_context *apc)
 938{
 939	/*
 940	 * make sure subsequent cleanup attempts don't end up removing already
 941	 * cleaned dentry pointer
 942	 */
 943	debugfs_remove(apc->mana_port_debugfs);
 944	apc->mana_port_debugfs = NULL;
 945	kfree(apc->rxqs);
 946	apc->rxqs = NULL;
 947}
 948
 949static void mana_cleanup_indir_table(struct mana_port_context *apc)
 950{
 951	apc->indir_table_sz = 0;
 952	kfree(apc->indir_table);
 953	kfree(apc->rxobj_table);
 954}
 955
 956static int mana_init_port_context(struct mana_port_context *apc)
 957{
 958	apc->rxqs = kcalloc(apc->num_queues, sizeof(struct mana_rxq *),
 959			    GFP_KERNEL);
 960
 961	return !apc->rxqs ? -ENOMEM : 0;
 962}
 963
 964static int mana_send_request(struct mana_context *ac, void *in_buf,
 965			     u32 in_len, void *out_buf, u32 out_len)
 966{
 967	struct gdma_context *gc = ac->gdma_dev->gdma_context;
 968	struct gdma_resp_hdr *resp = out_buf;
 969	struct gdma_req_hdr *req = in_buf;
 970	struct device *dev = gc->dev;
 971	static atomic_t activity_id;
 972	int err;
 973
 974	req->dev_id = gc->mana.dev_id;
 975	req->activity_id = atomic_inc_return(&activity_id);
 976
 977	err = mana_gd_send_request(gc, in_len, in_buf, out_len,
 978				   out_buf);
 979	if (err || resp->status) {
 980		if (err == -EOPNOTSUPP)
 981			return err;
 982
 983		if (req->req.msg_type != MANA_QUERY_PHY_STAT &&
 984		    mana_need_log(gc, err))
 985			dev_err(dev, "Failed to send mana message: %d, 0x%x\n",
 986				err, resp->status);
 987		return err ? err : -EPROTO;
 988	}
 989
 990	if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 ||
 991	    req->activity_id != resp->activity_id) {
 992		dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n",
 993			req->dev_id.as_uint32, resp->dev_id.as_uint32,
 994			req->activity_id, resp->activity_id);
 995		return -EPROTO;
 996	}
 997
 998	return 0;
 999}
1000
1001static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr,
1002				const enum mana_command_code expected_code,
1003				const u32 min_size)
1004{
1005	if (resp_hdr->response.msg_type != expected_code)
1006		return -EPROTO;
1007
1008	if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1)
1009		return -EPROTO;
1010
1011	if (resp_hdr->response.msg_size < min_size)
1012		return -EPROTO;
1013
1014	return 0;
1015}
1016
1017static int mana_pf_register_hw_vport(struct mana_port_context *apc)
1018{
1019	struct mana_register_hw_vport_resp resp = {};
1020	struct mana_register_hw_vport_req req = {};
1021	int err;
1022
1023	mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT,
1024			     sizeof(req), sizeof(resp));
1025	req.attached_gfid = 1;
1026	req.is_pf_default_vport = 1;
1027	req.allow_all_ether_types = 1;
1028
1029	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1030				sizeof(resp));
1031	if (err) {
1032		netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err);
1033		return err;
1034	}
1035
1036	err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT,
1037				   sizeof(resp));
1038	if (err || resp.hdr.status) {
1039		netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n",
1040			   err, resp.hdr.status);
1041		return err ? err : -EPROTO;
1042	}
1043
1044	apc->port_handle = resp.hw_vport_handle;
1045	return 0;
1046}
1047
1048static void mana_pf_deregister_hw_vport(struct mana_port_context *apc)
1049{
1050	struct mana_deregister_hw_vport_resp resp = {};
1051	struct mana_deregister_hw_vport_req req = {};
1052	int err;
1053
1054	mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT,
1055			     sizeof(req), sizeof(resp));
1056	req.hw_vport_handle = apc->port_handle;
1057
1058	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1059				sizeof(resp));
1060	if (err) {
1061		if (mana_en_need_log(apc, err))
1062			netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n",
1063				   err);
1064
1065		return;
1066	}
1067
1068	err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT,
1069				   sizeof(resp));
1070	if (err || resp.hdr.status)
1071		netdev_err(apc->ndev,
1072			   "Failed to deregister hw vPort: %d, 0x%x\n",
1073			   err, resp.hdr.status);
1074}
1075
1076static int mana_pf_register_filter(struct mana_port_context *apc)
1077{
1078	struct mana_register_filter_resp resp = {};
1079	struct mana_register_filter_req req = {};
1080	int err;
1081
1082	mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER,
1083			     sizeof(req), sizeof(resp));
1084	req.vport = apc->port_handle;
1085	memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN);
1086
1087	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1088				sizeof(resp));
1089	if (err) {
1090		netdev_err(apc->ndev, "Failed to register filter: %d\n", err);
1091		return err;
1092	}
1093
1094	err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER,
1095				   sizeof(resp));
1096	if (err || resp.hdr.status) {
1097		netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n",
1098			   err, resp.hdr.status);
1099		return err ? err : -EPROTO;
1100	}
1101
1102	apc->pf_filter_handle = resp.filter_handle;
1103	return 0;
1104}
1105
1106static void mana_pf_deregister_filter(struct mana_port_context *apc)
1107{
1108	struct mana_deregister_filter_resp resp = {};
1109	struct mana_deregister_filter_req req = {};
1110	int err;
1111
1112	mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER,
1113			     sizeof(req), sizeof(resp));
1114	req.filter_handle = apc->pf_filter_handle;
1115
1116	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1117				sizeof(resp));
1118	if (err) {
1119		if (mana_en_need_log(apc, err))
1120			netdev_err(apc->ndev, "Failed to unregister filter: %d\n",
1121				   err);
1122
1123		return;
1124	}
1125
1126	err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER,
1127				   sizeof(resp));
1128	if (err || resp.hdr.status)
1129		netdev_err(apc->ndev,
1130			   "Failed to deregister filter: %d, 0x%x\n",
1131			   err, resp.hdr.status);
1132}
1133
1134static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver,
1135				 u32 proto_minor_ver, u32 proto_micro_ver,
1136				 u16 *max_num_vports, u8 *bm_hostmode)
1137{
1138	struct gdma_context *gc = ac->gdma_dev->gdma_context;
1139	struct mana_query_device_cfg_resp resp = {};
1140	struct mana_query_device_cfg_req req = {};
1141	struct device *dev = gc->dev;
1142	int err = 0;
1143
1144	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG,
1145			     sizeof(req), sizeof(resp));
1146
1147	req.hdr.resp.msg_version = GDMA_MESSAGE_V3;
1148
1149	req.proto_major_ver = proto_major_ver;
1150	req.proto_minor_ver = proto_minor_ver;
1151	req.proto_micro_ver = proto_micro_ver;
1152
1153	err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp));
1154	if (err) {
1155		dev_err(dev, "Failed to query config: %d", err);
1156		return err;
1157	}
1158
1159	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG,
1160				   sizeof(resp));
1161	if (err || resp.hdr.status) {
1162		dev_err(dev, "Invalid query result: %d, 0x%x\n", err,
1163			resp.hdr.status);
1164		if (!err)
1165			err = -EPROTO;
1166		return err;
1167	}
1168
1169	*max_num_vports = resp.max_num_vports;
1170
1171	if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2)
1172		gc->adapter_mtu = resp.adapter_mtu;
1173	else
1174		gc->adapter_mtu = ETH_FRAME_LEN;
1175
1176	if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V3)
1177		*bm_hostmode = resp.bm_hostmode;
1178	else
1179		*bm_hostmode = 0;
1180
1181	debugfs_create_u16("adapter-MTU", 0400, gc->mana_pci_debugfs, &gc->adapter_mtu);
1182
1183	return 0;
1184}
1185
1186static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index,
1187				u32 *max_sq, u32 *max_rq, u32 *num_indir_entry)
1188{
1189	struct mana_query_vport_cfg_resp resp = {};
1190	struct mana_query_vport_cfg_req req = {};
1191	int err;
1192
1193	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG,
1194			     sizeof(req), sizeof(resp));
1195
1196	req.vport_index = vport_index;
1197
1198	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1199				sizeof(resp));
1200	if (err)
1201		return err;
1202
1203	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG,
1204				   sizeof(resp));
1205	if (err)
1206		return err;
1207
1208	if (resp.hdr.status)
1209		return -EPROTO;
1210
1211	*max_sq = resp.max_num_sq;
1212	*max_rq = resp.max_num_rq;
1213	if (resp.num_indirection_ent > 0 &&
1214	    resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE &&
1215	    is_power_of_2(resp.num_indirection_ent)) {
1216		*num_indir_entry = resp.num_indirection_ent;
1217	} else {
1218		netdev_warn(apc->ndev,
1219			    "Setting indirection table size to default %d for vPort %d\n",
1220			    MANA_INDIRECT_TABLE_DEF_SIZE, apc->port_idx);
1221		*num_indir_entry = MANA_INDIRECT_TABLE_DEF_SIZE;
1222	}
1223
1224	apc->port_handle = resp.vport;
1225	ether_addr_copy(apc->mac_addr, resp.mac_addr);
1226
1227	return 0;
1228}
1229
1230void mana_uncfg_vport(struct mana_port_context *apc)
1231{
1232	mutex_lock(&apc->vport_mutex);
1233	apc->vport_use_count--;
1234	WARN_ON(apc->vport_use_count < 0);
1235	mutex_unlock(&apc->vport_mutex);
1236}
1237EXPORT_SYMBOL_NS(mana_uncfg_vport, "NET_MANA");
1238
1239int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id,
1240		   u32 doorbell_pg_id)
1241{
1242	struct mana_config_vport_resp resp = {};
1243	struct mana_config_vport_req req = {};
1244	int err;
1245
1246	/* This function is used to program the Ethernet port in the hardware
1247	 * table. It can be called from the Ethernet driver or the RDMA driver.
1248	 *
1249	 * For Ethernet usage, the hardware supports only one active user on a
1250	 * physical port. The driver checks on the port usage before programming
1251	 * the hardware when creating the RAW QP (RDMA driver) or exposing the
1252	 * device to kernel NET layer (Ethernet driver).
1253	 *
1254	 * Because the RDMA driver doesn't know in advance which QP type the
1255	 * user will create, it exposes the device with all its ports. The user
1256	 * may not be able to create RAW QP on a port if this port is already
1257	 * in used by the Ethernet driver from the kernel.
1258	 *
1259	 * This physical port limitation only applies to the RAW QP. For RC QP,
1260	 * the hardware doesn't have this limitation. The user can create RC
1261	 * QPs on a physical port up to the hardware limits independent of the
1262	 * Ethernet usage on the same port.
1263	 */
1264	mutex_lock(&apc->vport_mutex);
1265	if (apc->vport_use_count > 0) {
1266		mutex_unlock(&apc->vport_mutex);
1267		return -EBUSY;
1268	}
1269	apc->vport_use_count++;
1270	mutex_unlock(&apc->vport_mutex);
1271
1272	mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX,
1273			     sizeof(req), sizeof(resp));
1274	req.vport = apc->port_handle;
1275	req.pdid = protection_dom_id;
1276	req.doorbell_pageid = doorbell_pg_id;
1277
1278	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1279				sizeof(resp));
1280	if (err) {
1281		netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err);
1282		goto out;
1283	}
1284
1285	err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX,
1286				   sizeof(resp));
1287	if (err || resp.hdr.status) {
1288		netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n",
1289			   err, resp.hdr.status);
1290		if (!err)
1291			err = -EPROTO;
1292
1293		goto out;
1294	}
1295
1296	apc->tx_shortform_allowed = resp.short_form_allowed;
1297	apc->tx_vp_offset = resp.tx_vport_offset;
1298
1299	netdev_info(apc->ndev, "Configured vPort %llu PD %u DB %u\n",
1300		    apc->port_handle, protection_dom_id, doorbell_pg_id);
1301out:
1302	if (err)
1303		mana_uncfg_vport(apc);
1304
1305	return err;
1306}
1307EXPORT_SYMBOL_NS(mana_cfg_vport, "NET_MANA");
1308
1309static int mana_cfg_vport_steering(struct mana_port_context *apc,
1310				   enum TRI_STATE rx,
1311				   bool update_default_rxobj, bool update_key,
1312				   bool update_tab)
1313{
1314	struct mana_cfg_rx_steer_req_v2 *req;
1315	struct mana_cfg_rx_steer_resp resp = {};
1316	struct net_device *ndev = apc->ndev;
1317	u32 req_buf_size;
1318	int err;
1319
1320	req_buf_size = struct_size(req, indir_tab, apc->indir_table_sz);
1321	req = kzalloc(req_buf_size, GFP_KERNEL);
1322	if (!req)
1323		return -ENOMEM;
1324
1325	mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
1326			     sizeof(resp));
1327
1328	req->hdr.req.msg_version = GDMA_MESSAGE_V2;
1329
1330	req->vport = apc->port_handle;
1331	req->num_indir_entries = apc->indir_table_sz;
1332	req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2,
1333					 indir_tab);
1334	req->rx_enable = rx;
1335	req->rss_enable = apc->rss_state;
1336	req->update_default_rxobj = update_default_rxobj;
1337	req->update_hashkey = update_key;
1338	req->update_indir_tab = update_tab;
1339	req->default_rxobj = apc->default_rxobj;
1340	req->cqe_coalescing_enable = 0;
1341
1342	if (update_key)
1343		memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
1344
1345	if (update_tab)
1346		memcpy(req->indir_tab, apc->rxobj_table,
1347		       flex_array_size(req, indir_tab, req->num_indir_entries));
1348
1349	err = mana_send_request(apc->ac, req, req_buf_size, &resp,
1350				sizeof(resp));
1351	if (err) {
1352		if (mana_en_need_log(apc, err))
1353			netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);
1354
1355		goto out;
1356	}
1357
1358	err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX,
1359				   sizeof(resp));
1360	if (err) {
1361		netdev_err(ndev, "vPort RX configuration failed: %d\n", err);
1362		goto out;
1363	}
1364
1365	if (resp.hdr.status) {
1366		netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
1367			   resp.hdr.status);
1368		err = -EPROTO;
1369	}
1370
1371	netdev_info(ndev, "Configured steering vPort %llu entries %u\n",
1372		    apc->port_handle, apc->indir_table_sz);
1373out:
1374	kfree(req);
1375	return err;
1376}
1377
1378int mana_query_link_cfg(struct mana_port_context *apc)
1379{
1380	struct net_device *ndev = apc->ndev;
1381	struct mana_query_link_config_resp resp = {};
1382	struct mana_query_link_config_req req = {};
1383	int err;
1384
1385	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_LINK_CONFIG,
1386			     sizeof(req), sizeof(resp));
1387
1388	req.vport = apc->port_handle;
1389	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;
1390
1391	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1392				sizeof(resp));
1393
1394	if (err) {
1395		if (err == -EOPNOTSUPP) {
1396			netdev_info_once(ndev, "MANA_QUERY_LINK_CONFIG not supported\n");
1397			return err;
1398		}
1399		netdev_err(ndev, "Failed to query link config: %d\n", err);
1400		return err;
1401	}
1402
1403	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_LINK_CONFIG,
1404				   sizeof(resp));
1405
1406	if (err || resp.hdr.status) {
1407		netdev_err(ndev, "Failed to query link config: %d, 0x%x\n", err,
1408			   resp.hdr.status);
1409		if (!err)
1410			err = -EOPNOTSUPP;
1411		return err;
1412	}
1413
1414	if (resp.qos_unconfigured) {
1415		err = -EINVAL;
1416		return err;
1417	}
1418	apc->speed = resp.link_speed_mbps;
1419	apc->max_speed = resp.qos_speed_mbps;
1420	return 0;
1421}
1422
1423int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed,
1424		      int enable_clamping)
1425{
1426	struct mana_set_bw_clamp_resp resp = {};
1427	struct mana_set_bw_clamp_req req = {};
1428	struct net_device *ndev = apc->ndev;
1429	int err;
1430
1431	mana_gd_init_req_hdr(&req.hdr, MANA_SET_BW_CLAMP,
1432			     sizeof(req), sizeof(resp));
1433	req.vport = apc->port_handle;
1434	req.link_speed_mbps = speed;
1435	req.enable_clamping = enable_clamping;
1436
1437	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1438				sizeof(resp));
1439
1440	if (err) {
1441		if (err == -EOPNOTSUPP) {
1442			netdev_info_once(ndev, "MANA_SET_BW_CLAMP not supported\n");
1443			return err;
1444		}
1445		netdev_err(ndev, "Failed to set bandwidth clamp for speed %u, err = %d",
1446			   speed, err);
1447		return err;
1448	}
1449
1450	err = mana_verify_resp_hdr(&resp.hdr, MANA_SET_BW_CLAMP,
1451				   sizeof(resp));
1452
1453	if (err || resp.hdr.status) {
1454		netdev_err(ndev, "Failed to set bandwidth clamp: %d, 0x%x\n", err,
1455			   resp.hdr.status);
1456		if (!err)
1457			err = -EOPNOTSUPP;
1458		return err;
1459	}
1460
1461	if (resp.qos_unconfigured)
1462		netdev_info(ndev, "QoS is unconfigured\n");
1463
1464	return 0;
1465}
1466
1467int mana_create_wq_obj(struct mana_port_context *apc,
1468		       mana_handle_t vport,
1469		       u32 wq_type, struct mana_obj_spec *wq_spec,
1470		       struct mana_obj_spec *cq_spec,
1471		       mana_handle_t *wq_obj)
1472{
1473	struct mana_create_wqobj_resp resp = {};
1474	struct mana_create_wqobj_req req = {};
1475	struct net_device *ndev = apc->ndev;
1476	int err;
1477
1478	mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ,
1479			     sizeof(req), sizeof(resp));
1480	req.vport = vport;
1481	req.wq_type = wq_type;
1482	req.wq_gdma_region = wq_spec->gdma_region;
1483	req.cq_gdma_region = cq_spec->gdma_region;
1484	req.wq_size = wq_spec->queue_size;
1485	req.cq_size = cq_spec->queue_size;
1486	req.cq_moderation_ctx_id = cq_spec->modr_ctx_id;
1487	req.cq_parent_qid = cq_spec->attached_eq;
1488
1489	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1490				sizeof(resp));
1491	if (err) {
1492		netdev_err(ndev, "Failed to create WQ object: %d\n", err);
1493		goto out;
1494	}
1495
1496	err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ,
1497				   sizeof(resp));
1498	if (err || resp.hdr.status) {
1499		netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err,
1500			   resp.hdr.status);
1501		if (!err)
1502			err = -EPROTO;
1503		goto out;
1504	}
1505
1506	if (resp.wq_obj == INVALID_MANA_HANDLE) {
1507		netdev_err(ndev, "Got an invalid WQ object handle\n");
1508		err = -EPROTO;
1509		goto out;
1510	}
1511
1512	*wq_obj = resp.wq_obj;
1513	wq_spec->queue_index = resp.wq_id;
1514	cq_spec->queue_index = resp.cq_id;
1515
1516	return 0;
1517out:
1518	return err;
1519}
1520EXPORT_SYMBOL_NS(mana_create_wq_obj, "NET_MANA");
1521
1522void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type,
1523			 mana_handle_t wq_obj)
1524{
1525	struct mana_destroy_wqobj_resp resp = {};
1526	struct mana_destroy_wqobj_req req = {};
1527	struct net_device *ndev = apc->ndev;
1528	int err;
1529
1530	mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ,
1531			     sizeof(req), sizeof(resp));
1532	req.wq_type = wq_type;
1533	req.wq_obj_handle = wq_obj;
1534
1535	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1536				sizeof(resp));
1537	if (err) {
1538		if (mana_en_need_log(apc, err))
1539			netdev_err(ndev, "Failed to destroy WQ object: %d\n", err);
1540
1541		return;
1542	}
1543
1544	err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ,
1545				   sizeof(resp));
1546	if (err || resp.hdr.status)
1547		netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err,
1548			   resp.hdr.status);
1549}
1550EXPORT_SYMBOL_NS(mana_destroy_wq_obj, "NET_MANA");
1551
1552static void mana_destroy_eq(struct mana_context *ac)
1553{
1554	struct gdma_context *gc = ac->gdma_dev->gdma_context;
1555	struct gdma_queue *eq;
1556	int i;
1557
1558	if (!ac->eqs)
1559		return;
1560
1561	debugfs_remove_recursive(ac->mana_eqs_debugfs);
1562	ac->mana_eqs_debugfs = NULL;
1563
1564	for (i = 0; i < gc->max_num_queues; i++) {
1565		eq = ac->eqs[i].eq;
1566		if (!eq)
1567			continue;
1568
1569		mana_gd_destroy_queue(gc, eq);
1570	}
1571
1572	kfree(ac->eqs);
1573	ac->eqs = NULL;
1574}
1575
1576static void mana_create_eq_debugfs(struct mana_context *ac, int i)
1577{
1578	struct mana_eq eq = ac->eqs[i];
1579	char eqnum[32];
1580
1581	sprintf(eqnum, "eq%d", i);
1582	eq.mana_eq_debugfs = debugfs_create_dir(eqnum, ac->mana_eqs_debugfs);
1583	debugfs_create_u32("head", 0400, eq.mana_eq_debugfs, &eq.eq->head);
1584	debugfs_create_u32("tail", 0400, eq.mana_eq_debugfs, &eq.eq->tail);
1585	debugfs_create_file("eq_dump", 0400, eq.mana_eq_debugfs, eq.eq, &mana_dbg_q_fops);
1586}
1587
1588static int mana_create_eq(struct mana_context *ac)
1589{
1590	struct gdma_dev *gd = ac->gdma_dev;
1591	struct gdma_context *gc = gd->gdma_context;
1592	struct gdma_queue_spec spec = {};
1593	int err;
1594	int i;
1595
1596	ac->eqs = kcalloc(gc->max_num_queues, sizeof(struct mana_eq),
1597			  GFP_KERNEL);
1598	if (!ac->eqs)
1599		return -ENOMEM;
1600
1601	spec.type = GDMA_EQ;
1602	spec.monitor_avl_buf = false;
1603	spec.queue_size = EQ_SIZE;
1604	spec.eq.callback = NULL;
1605	spec.eq.context = ac->eqs;
1606	spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;
1607
1608	ac->mana_eqs_debugfs = debugfs_create_dir("EQs", gc->mana_pci_debugfs);
1609
1610	for (i = 0; i < gc->max_num_queues; i++) {
1611		spec.eq.msix_index = (i + 1) % gc->num_msix_usable;
1612		err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq);
1613		if (err) {
1614			dev_err(gc->dev, "Failed to create EQ %d : %d\n", i, err);
1615			goto out;
1616		}
1617		mana_create_eq_debugfs(ac, i);
1618	}
1619
1620	return 0;
1621out:
1622	mana_destroy_eq(ac);
1623	return err;
1624}
1625
1626static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq)
1627{
1628	struct mana_fence_rq_resp resp = {};
1629	struct mana_fence_rq_req req = {};
1630	int err;
1631
1632	init_completion(&rxq->fence_event);
1633
1634	mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ,
1635			     sizeof(req), sizeof(resp));
1636	req.wq_obj_handle =  rxq->rxobj;
1637
1638	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
1639				sizeof(resp));
1640	if (err) {
1641		netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n",
1642			   rxq->rxq_idx, err);
1643		return err;
1644	}
1645
1646	err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp));
1647	if (err || resp.hdr.status) {
1648		netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n",
1649			   rxq->rxq_idx, err, resp.hdr.status);
1650		if (!err)
1651			err = -EPROTO;
1652
1653		return err;
1654	}
1655
1656	if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) {
1657		netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n",
1658			   rxq->rxq_idx);
1659		return -ETIMEDOUT;
1660	}
1661
1662	return 0;
1663}
1664
1665static void mana_fence_rqs(struct mana_port_context *apc)
1666{
1667	unsigned int rxq_idx;
1668	struct mana_rxq *rxq;
1669	int err;
1670
1671	for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
1672		rxq = apc->rxqs[rxq_idx];
1673		err = mana_fence_rq(apc, rxq);
1674
1675		/* In case of any error, use sleep instead. */
1676		if (err)
1677			msleep(100);
1678	}
1679}
1680
1681static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units)
1682{
1683	u32 used_space_old;
1684	u32 used_space_new;
1685
1686	used_space_old = wq->head - wq->tail;
1687	used_space_new = wq->head - (wq->tail + num_units);
1688
1689	if (WARN_ON_ONCE(used_space_new > used_space_old))
1690		return -ERANGE;
1691
1692	wq->tail += num_units;
1693	return 0;
1694}
1695
1696void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc)
1697{
1698	struct mana_skb_head *ash = (struct mana_skb_head *)skb->head;
1699	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
1700	struct device *dev = gc->dev;
1701	int hsg, i;
1702
1703	/* Number of SGEs of linear part */
1704	hsg = (skb_is_gso(skb) && skb_headlen(skb) > ash->size[0]) ? 2 : 1;
1705
1706	for (i = 0; i < hsg; i++)
1707		dma_unmap_single(dev, ash->dma_handle[i], ash->size[i],
1708				 DMA_TO_DEVICE);
1709
1710	for (i = hsg; i < skb_shinfo(skb)->nr_frags + hsg; i++)
1711		dma_unmap_page(dev, ash->dma_handle[i], ash->size[i],
1712			       DMA_TO_DEVICE);
1713}
1714
1715static void mana_poll_tx_cq(struct mana_cq *cq)
1716{
1717	struct gdma_comp *completions = cq->gdma_comp_buf;
1718	struct gdma_posted_wqe_info *wqe_info;
1719	unsigned int pkt_transmitted = 0;
1720	unsigned int wqe_unit_cnt = 0;
1721	struct mana_txq *txq = cq->txq;
1722	struct mana_port_context *apc;
1723	struct netdev_queue *net_txq;
1724	struct gdma_queue *gdma_wq;
1725	unsigned int avail_space;
1726	struct net_device *ndev;
1727	struct sk_buff *skb;
1728	bool txq_stopped;
1729	int comp_read;
1730	int i;
1731
1732	ndev = txq->ndev;
1733	apc = netdev_priv(ndev);
1734
1735	/* Limit CQEs polled to 4 wraparounds of the CQ to ensure the
1736	 * doorbell can be rung in time for the hardware's requirement
1737	 * of at least one doorbell ring every 8 wraparounds.
1738	 */
1739	comp_read = mana_gd_poll_cq(cq->gdma_cq, completions,
1740				    min((cq->gdma_cq->queue_size /
1741					  COMP_ENTRY_SIZE) * 4,
1742					 CQE_POLLING_BUFFER));
1743
1744	if (comp_read < 1)
1745		return;
1746
1747	for (i = 0; i < comp_read; i++) {
1748		struct mana_tx_comp_oob *cqe_oob;
1749
1750		if (WARN_ON_ONCE(!completions[i].is_sq))
1751			return;
1752
1753		cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data;
1754		if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type !=
1755				 MANA_CQE_COMPLETION))
1756			return;
1757
1758		switch (cqe_oob->cqe_hdr.cqe_type) {
1759		case CQE_TX_OKAY:
1760			break;
1761
1762		case CQE_TX_SA_DROP:
1763		case CQE_TX_MTU_DROP:
1764		case CQE_TX_INVALID_OOB:
1765		case CQE_TX_INVALID_ETH_TYPE:
1766		case CQE_TX_HDR_PROCESSING_ERROR:
1767		case CQE_TX_VF_DISABLED:
1768		case CQE_TX_VPORT_IDX_OUT_OF_RANGE:
1769		case CQE_TX_VPORT_DISABLED:
1770		case CQE_TX_VLAN_TAGGING_VIOLATION:
1771			if (net_ratelimit())
1772				netdev_err(ndev, "TX: CQE error %d\n",
1773					   cqe_oob->cqe_hdr.cqe_type);
1774
1775			apc->eth_stats.tx_cqe_err++;
1776			break;
1777
1778		default:
1779			/* If the CQE type is unknown, log an error,
1780			 * and still free the SKB, update tail, etc.
1781			 */
1782			if (net_ratelimit())
1783				netdev_err(ndev, "TX: unknown CQE type %d\n",
1784					   cqe_oob->cqe_hdr.cqe_type);
1785
1786			apc->eth_stats.tx_cqe_unknown_type++;
1787			break;
1788		}
1789
1790		if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num))
1791			return;
1792
1793		skb = skb_dequeue(&txq->pending_skbs);
1794		if (WARN_ON_ONCE(!skb))
1795			return;
1796
1797		wqe_info = (struct gdma_posted_wqe_info *)skb->cb;
1798		wqe_unit_cnt += wqe_info->wqe_size_in_bu;
1799
1800		mana_unmap_skb(skb, apc);
1801
1802		napi_consume_skb(skb, cq->budget);
1803
1804		pkt_transmitted++;
1805	}
1806
1807	if (WARN_ON_ONCE(wqe_unit_cnt == 0))
1808		return;
1809
1810	mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt);
1811
1812	gdma_wq = txq->gdma_sq;
1813	avail_space = mana_gd_wq_avail_space(gdma_wq);
1814
1815	/* Ensure tail updated before checking q stop */
1816	smp_mb();
1817
1818	net_txq = txq->net_txq;
1819	txq_stopped = netif_tx_queue_stopped(net_txq);
1820
1821	/* Ensure checking txq_stopped before apc->port_is_up. */
1822	smp_rmb();
1823
1824	if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) {
1825		netif_tx_wake_queue(net_txq);
1826		apc->eth_stats.wake_queue++;
1827	}
1828
1829	if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0)
1830		WARN_ON_ONCE(1);
1831
1832	cq->work_done = pkt_transmitted;
1833}
1834
1835static void mana_post_pkt_rxq(struct mana_rxq *rxq)
1836{
1837	struct mana_recv_buf_oob *recv_buf_oob;
1838	u32 curr_index;
1839	int err;
1840
1841	curr_index = rxq->buf_index++;
1842	if (rxq->buf_index == rxq->num_rx_buf)
1843		rxq->buf_index = 0;
1844
1845	recv_buf_oob = &rxq->rx_oobs[curr_index];
1846
1847	err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req,
1848					&recv_buf_oob->wqe_inf);
1849	if (WARN_ON_ONCE(err))
1850		return;
1851
1852	WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1);
1853}
1854
1855static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va,
1856				      uint pkt_len, struct xdp_buff *xdp)
1857{
1858	struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size);
1859
1860	if (!skb)
1861		return NULL;
1862
1863	if (xdp->data_hard_start) {
1864		u32 metasize = xdp->data - xdp->data_meta;
1865
1866		skb_reserve(skb, xdp->data - xdp->data_hard_start);
1867		skb_put(skb, xdp->data_end - xdp->data);
1868		if (metasize)
1869			skb_metadata_set(skb, metasize);
1870		return skb;
1871	}
1872
1873	skb_reserve(skb, rxq->headroom);
1874	skb_put(skb, pkt_len);
1875
1876	return skb;
1877}
1878
1879static void mana_rx_skb(void *buf_va, bool from_pool,
1880			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq)
1881{
1882	struct mana_stats_rx *rx_stats = &rxq->stats;
1883	struct net_device *ndev = rxq->ndev;
1884	uint pkt_len = cqe->ppi[0].pkt_len;
1885	u16 rxq_idx = rxq->rxq_idx;
1886	struct napi_struct *napi;
1887	struct xdp_buff xdp = {};
1888	struct sk_buff *skb;
1889	u32 hash_value;
1890	u32 act;
1891
1892	rxq->rx_cq.work_done++;
1893	napi = &rxq->rx_cq.napi;
1894
1895	if (!buf_va) {
1896		++ndev->stats.rx_dropped;
1897		return;
1898	}
1899
1900	act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len);
1901
1902	if (act == XDP_REDIRECT && !rxq->xdp_rc)
1903		return;
1904
1905	if (act != XDP_PASS && act != XDP_TX)
1906		goto drop_xdp;
1907
1908	skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp);
1909
1910	if (!skb)
1911		goto drop;
1912
1913	if (from_pool)
1914		skb_mark_for_recycle(skb);
1915
1916	skb->dev = napi->dev;
1917
1918	skb->protocol = eth_type_trans(skb, ndev);
1919	skb_checksum_none_assert(skb);
1920	skb_record_rx_queue(skb, rxq_idx);
1921
1922	if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) {
1923		if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed)
1924			skb->ip_summed = CHECKSUM_UNNECESSARY;
1925	}
1926
1927	if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {
1928		hash_value = cqe->ppi[0].pkt_hash;
1929
1930		if (cqe->rx_hashtype & MANA_HASH_L4)
1931			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);
1932		else
1933			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3);
1934	}
1935
1936	if (cqe->rx_vlantag_present) {
1937		u16 vlan_tci = cqe->rx_vlan_id;
1938
1939		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci);
1940	}
1941
1942	u64_stats_update_begin(&rx_stats->syncp);
1943	rx_stats->packets++;
1944	rx_stats->bytes += pkt_len;
1945
1946	if (act == XDP_TX)
1947		rx_stats->xdp_tx++;
1948	u64_stats_update_end(&rx_stats->syncp);
1949
1950	if (act == XDP_TX) {
1951		skb_set_queue_mapping(skb, rxq_idx);
1952		mana_xdp_tx(skb, ndev);
1953		return;
1954	}
1955
1956	napi_gro_receive(napi, skb);
1957
1958	return;
1959
1960drop_xdp:
1961	u64_stats_update_begin(&rx_stats->syncp);
1962	rx_stats->xdp_drop++;
1963	u64_stats_update_end(&rx_stats->syncp);
1964
1965drop:
1966	if (from_pool) {
1967		if (rxq->frag_count == 1)
1968			page_pool_recycle_direct(rxq->page_pool,
1969						 virt_to_head_page(buf_va));
1970		else
1971			page_pool_free_va(rxq->page_pool, buf_va, true);
1972	} else {
1973		WARN_ON_ONCE(rxq->xdp_save_va);
1974		/* Save for reuse */
1975		rxq->xdp_save_va = buf_va;
1976	}
1977
1978	++ndev->stats.rx_dropped;
1979
1980	return;
1981}
1982
1983static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev,
1984			     dma_addr_t *da, bool *from_pool)
1985{
1986	struct page *page;
1987	u32 offset;
1988	void *va;
1989	*from_pool = false;
1990
1991	/* Don't use fragments for jumbo frames or XDP where it's 1 fragment
1992	 * per page.
1993	 */
1994	if (rxq->frag_count == 1) {
1995		/* Reuse XDP dropped page if available */
1996		if (rxq->xdp_save_va) {
1997			va = rxq->xdp_save_va;
1998			page = virt_to_head_page(va);
1999			rxq->xdp_save_va = NULL;
2000		} else {
2001			page = page_pool_dev_alloc_pages(rxq->page_pool);
2002			if (!page)
2003				return NULL;
2004
2005			*from_pool = true;
2006			va = page_to_virt(page);
2007		}
2008
2009		*da = dma_map_single(dev, va + rxq->headroom, rxq->datasize,
2010				     DMA_FROM_DEVICE);
2011		if (dma_mapping_error(dev, *da)) {
2012			mana_put_rx_page(rxq, page, *from_pool);
2013			return NULL;
2014		}
2015
2016		return va;
2017	}
2018
2019	page =  page_pool_dev_alloc_frag(rxq->page_pool, &offset,
2020					 rxq->alloc_size);
2021	if (!page)
2022		return NULL;
2023
2024	va  = page_to_virt(page) + offset;
2025	*da = page_pool_get_dma_addr(page) + offset + rxq->headroom;
2026	*from_pool = true;
2027
2028	return va;
2029}
2030
2031/* Allocate frag for rx buffer, and save the old buf */
2032static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq,
2033			       struct mana_recv_buf_oob *rxoob, void **old_buf,
2034			       bool *old_fp)
2035{
2036	bool from_pool;
2037	dma_addr_t da;
2038	void *va;
2039
2040	va = mana_get_rxfrag(rxq, dev, &da, &from_pool);
2041	if (!va)
2042		return;
2043	if (!rxoob->from_pool || rxq->frag_count == 1)
2044		dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize,
2045				 DMA_FROM_DEVICE);
2046	*old_buf = rxoob->buf_va;
2047	*old_fp = rxoob->from_pool;
2048
2049	rxoob->buf_va = va;
2050	rxoob->sgl[0].address = da;
2051	rxoob->from_pool = from_pool;
2052}
2053
2054static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
2055				struct gdma_comp *cqe)
2056{
2057	struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data;
2058	struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;
2059	struct net_device *ndev = rxq->ndev;
2060	struct mana_recv_buf_oob *rxbuf_oob;
2061	struct mana_port_context *apc;
2062	struct device *dev = gc->dev;
2063	void *old_buf = NULL;
2064	u32 curr, pktlen;
2065	bool old_fp;
2066
2067	apc = netdev_priv(ndev);
2068
2069	switch (oob->cqe_hdr.cqe_type) {
2070	case CQE_RX_OKAY:
2071		break;
2072
2073	case CQE_RX_TRUNCATED:
2074		++ndev->stats.rx_dropped;
2075		rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];
2076		netdev_warn_once(ndev, "Dropped a truncated packet\n");
2077		goto drop;
2078
2079	case CQE_RX_COALESCED_4:
2080		netdev_err(ndev, "RX coalescing is unsupported\n");
2081		apc->eth_stats.rx_coalesced_err++;
2082		return;
2083
2084	case CQE_RX_OBJECT_FENCE:
2085		complete(&rxq->fence_event);
2086		return;
2087
2088	default:
2089		netdev_err(ndev, "Unknown RX CQE type = %d\n",
2090			   oob->cqe_hdr.cqe_type);
2091		apc->eth_stats.rx_cqe_unknown_type++;
2092		return;
2093	}
2094
2095	pktlen = oob->ppi[0].pkt_len;
2096
2097	if (pktlen == 0) {
2098		/* data packets should never have packetlength of zero */
2099		netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
2100			   rxq->gdma_id, cq->gdma_id, rxq->rxobj);
2101		return;
2102	}
2103
2104	curr = rxq->buf_index;
2105	rxbuf_oob = &rxq->rx_oobs[curr];
2106	WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
2107
2108	mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp);
2109
2110	/* Unsuccessful refill will have old_buf == NULL.
2111	 * In this case, mana_rx_skb() will drop the packet.
2112	 */
2113	mana_rx_skb(old_buf, old_fp, oob, rxq);
2114
2115drop:
2116	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
2117
2118	mana_post_pkt_rxq(rxq);
2119}
2120
2121static void mana_poll_rx_cq(struct mana_cq *cq)
2122{
2123	struct gdma_comp *comp = cq->gdma_comp_buf;
2124	struct mana_rxq *rxq = cq->rxq;
2125	int comp_read, i;
2126
2127	/* Limit CQEs polled to 4 wraparounds of the CQ to ensure the
2128	 * doorbell can be rung in time for the hardware's requirement
2129	 * of at least one doorbell ring every 8 wraparounds.
2130	 */
2131	comp_read = mana_gd_poll_cq(cq->gdma_cq, comp,
2132				    min((cq->gdma_cq->queue_size /
2133					  COMP_ENTRY_SIZE) * 4,
2134					 CQE_POLLING_BUFFER));
2135	WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER);
2136
2137	rxq->xdp_flush = false;
2138
2139	for (i = 0; i < comp_read; i++) {
2140		if (WARN_ON_ONCE(comp[i].is_sq))
2141			return;
2142
2143		/* verify recv cqe references the right rxq */
2144		if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id))
2145			return;
2146
2147		mana_process_rx_cqe(rxq, cq, &comp[i]);
2148	}
2149
2150	if (comp_read > 0) {
2151		struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;
2152
2153		mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq);
2154	}
2155
2156	if (rxq->xdp_flush)
2157		xdp_do_flush();
2158}
2159
2160static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue)
2161{
2162	struct mana_cq *cq = context;
2163	int w;
2164
2165	WARN_ON_ONCE(cq->gdma_cq != gdma_queue);
2166
2167	if (cq->type == MANA_CQ_TYPE_RX)
2168		mana_poll_rx_cq(cq);
2169	else
2170		mana_poll_tx_cq(cq);
2171
2172	w = cq->work_done;
2173	cq->work_done_since_doorbell += w;
2174
2175	if (w < cq->budget) {
2176		mana_gd_ring_cq(gdma_queue, SET_ARM_BIT);
2177		cq->work_done_since_doorbell = 0;
2178		napi_complete_done(&cq->napi, w);
2179	} else if (cq->work_done_since_doorbell >=
2180		   (cq->gdma_cq->queue_size / COMP_ENTRY_SIZE) * 4) {
2181		/* MANA hardware requires at least one doorbell ring every 8
2182		 * wraparounds of CQ even if there is no need to arm the CQ.
2183		 * This driver rings the doorbell as soon as it has processed
2184		 * 4 wraparounds.
2185		 */
2186		mana_gd_ring_cq(gdma_queue, 0);
2187		cq->work_done_since_doorbell = 0;
2188	}
2189
2190	return w;
2191}
2192
2193static int mana_poll(struct napi_struct *napi, int budget)
2194{
2195	struct mana_cq *cq = container_of(napi, struct mana_cq, napi);
2196	int w;
2197
2198	cq->work_done = 0;
2199	cq->budget = budget;
2200
2201	w = mana_cq_handler(cq, cq->gdma_cq);
2202
2203	return min(w, budget);
2204}
2205
2206static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue)
2207{
2208	struct mana_cq *cq = context;
2209
2210	napi_schedule_irqoff(&cq->napi);
2211}
2212
2213static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq)
2214{
2215	struct gdma_dev *gd = apc->ac->gdma_dev;
2216
2217	if (!cq->gdma_cq)
2218		return;
2219
2220	mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq);
2221}
2222
2223static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq)
2224{
2225	struct gdma_dev *gd = apc->ac->gdma_dev;
2226
2227	if (!txq->gdma_sq)
2228		return;
2229
2230	mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq);
2231}
2232
2233static void mana_destroy_txq(struct mana_port_context *apc)
2234{
2235	struct napi_struct *napi;
2236	int i;
2237
2238	if (!apc->tx_qp)
2239		return;
2240
2241	for (i = 0; i < apc->num_queues; i++) {
2242		debugfs_remove_recursive(apc->tx_qp[i].mana_tx_debugfs);
2243		apc->tx_qp[i].mana_tx_debugfs = NULL;
2244
2245		napi = &apc->tx_qp[i].tx_cq.napi;
2246		if (apc->tx_qp[i].txq.napi_initialized) {
2247			napi_synchronize(napi);
2248			napi_disable_locked(napi);
2249			netif_napi_del_locked(napi);
2250			apc->tx_qp[i].txq.napi_initialized = false;
2251		}
2252		mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object);
2253
2254		mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq);
2255
2256		mana_deinit_txq(apc, &apc->tx_qp[i].txq);
2257	}
2258
2259	kfree(apc->tx_qp);
2260	apc->tx_qp = NULL;
2261}
2262
2263static void mana_create_txq_debugfs(struct mana_port_context *apc, int idx)
2264{
2265	struct mana_tx_qp *tx_qp = &apc->tx_qp[idx];
2266	char qnum[32];
2267
2268	sprintf(qnum, "TX-%d", idx);
2269	tx_qp->mana_tx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs);
2270	debugfs_create_u32("sq_head", 0400, tx_qp->mana_tx_debugfs,
2271			   &tx_qp->txq.gdma_sq->head);
2272	debugfs_create_u32("sq_tail", 0400, tx_qp->mana_tx_debugfs,
2273			   &tx_qp->txq.gdma_sq->tail);
2274	debugfs_create_u32("sq_pend_skb_qlen", 0400, tx_qp->mana_tx_debugfs,
2275			   &tx_qp->txq.pending_skbs.qlen);
2276	debugfs_create_u32("cq_head", 0400, tx_qp->mana_tx_debugfs,
2277			   &tx_qp->tx_cq.gdma_cq->head);
2278	debugfs_create_u32("cq_tail", 0400, tx_qp->mana_tx_debugfs,
2279			   &tx_qp->tx_cq.gdma_cq->tail);
2280	debugfs_create_u32("cq_budget", 0400, tx_qp->mana_tx_debugfs,
2281			   &tx_qp->tx_cq.budget);
2282	debugfs_create_file("txq_dump", 0400, tx_qp->mana_tx_debugfs,
2283			    tx_qp->txq.gdma_sq, &mana_dbg_q_fops);
2284	debugfs_create_file("cq_dump", 0400, tx_qp->mana_tx_debugfs,
2285			    tx_qp->tx_cq.gdma_cq, &mana_dbg_q_fops);
2286}
2287
2288static int mana_create_txq(struct mana_port_context *apc,
2289			   struct net_device *net)
2290{
2291	struct mana_context *ac = apc->ac;
2292	struct gdma_dev *gd = ac->gdma_dev;
2293	struct mana_obj_spec wq_spec;
2294	struct mana_obj_spec cq_spec;
2295	struct gdma_queue_spec spec;
2296	struct gdma_context *gc;
2297	struct mana_txq *txq;
2298	struct mana_cq *cq;
2299	u32 txq_size;
2300	u32 cq_size;
2301	int err;
2302	int i;
2303
2304	apc->tx_qp = kcalloc(apc->num_queues, sizeof(struct mana_tx_qp),
2305			     GFP_KERNEL);
2306	if (!apc->tx_qp)
2307		return -ENOMEM;
2308
2309	/*  The minimum size of the WQE is 32 bytes, hence
2310	 *  apc->tx_queue_size represents the maximum number of WQEs
2311	 *  the SQ can store. This value is then used to size other queues
2312	 *  to prevent overflow.
2313	 *  Also note that the txq_size is always going to be MANA_PAGE_ALIGNED,
2314	 *  as min val of apc->tx_queue_size is 128 and that would make
2315	 *  txq_size 128*32 = 4096 and the other higher values of apc->tx_queue_size
2316	 *  are always power of two
2317	 */
2318	txq_size = apc->tx_queue_size * 32;
2319
2320	cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE;
2321
2322	gc = gd->gdma_context;
2323
2324	for (i = 0; i < apc->num_queues; i++) {
2325		apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE;
2326
2327		/* Create SQ */
2328		txq = &apc->tx_qp[i].txq;
2329
2330		u64_stats_init(&txq->stats.syncp);
2331		txq->ndev = net;
2332		txq->net_txq = netdev_get_tx_queue(net, i);
2333		txq->vp_offset = apc->tx_vp_offset;
2334		txq->napi_initialized = false;
2335		skb_queue_head_init(&txq->pending_skbs);
2336
2337		memset(&spec, 0, sizeof(spec));
2338		spec.type = GDMA_SQ;
2339		spec.monitor_avl_buf = true;
2340		spec.queue_size = txq_size;
2341		err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq);
2342		if (err)
2343			goto out;
2344
2345		/* Create SQ's CQ */
2346		cq = &apc->tx_qp[i].tx_cq;
2347		cq->type = MANA_CQ_TYPE_TX;
2348
2349		cq->txq = txq;
2350
2351		memset(&spec, 0, sizeof(spec));
2352		spec.type = GDMA_CQ;
2353		spec.monitor_avl_buf = false;
2354		spec.queue_size = cq_size;
2355		spec.cq.callback = mana_schedule_napi;
2356		spec.cq.parent_eq = ac->eqs[i].eq;
2357		spec.cq.context = cq;
2358		err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);
2359		if (err)
2360			goto out;
2361
2362		memset(&wq_spec, 0, sizeof(wq_spec));
2363		memset(&cq_spec, 0, sizeof(cq_spec));
2364
2365		wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle;
2366		wq_spec.queue_size = txq->gdma_sq->queue_size;
2367
2368		cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle;
2369		cq_spec.queue_size = cq->gdma_cq->queue_size;
2370		cq_spec.modr_ctx_id = 0;
2371		cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
2372
2373		err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ,
2374					 &wq_spec, &cq_spec,
2375					 &apc->tx_qp[i].tx_object);
2376
2377		if (err)
2378			goto out;
2379
2380		txq->gdma_sq->id = wq_spec.queue_index;
2381		cq->gdma_cq->id = cq_spec.queue_index;
2382
2383		txq->gdma_sq->mem_info.dma_region_handle =
2384			GDMA_INVALID_DMA_REGION;
2385		cq->gdma_cq->mem_info.dma_region_handle =
2386			GDMA_INVALID_DMA_REGION;
2387
2388		txq->gdma_txq_id = txq->gdma_sq->id;
2389
2390		cq->gdma_id = cq->gdma_cq->id;
2391
2392		if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
2393			err = -EINVAL;
2394			goto out;
2395		}
2396
2397		gc->cq_table[cq->gdma_id] = cq->gdma_cq;
2398
2399		mana_create_txq_debugfs(apc, i);
2400
2401		set_bit(NAPI_STATE_NO_BUSY_POLL, &cq->napi.state);
2402		netif_napi_add_locked(net, &cq->napi, mana_poll);
2403		napi_enable_locked(&cq->napi);
2404		txq->napi_initialized = true;
2405
2406		mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
2407	}
2408
2409	return 0;
2410out:
2411	netdev_err(net, "Failed to create %d TX queues, %d\n",
2412		   apc->num_queues, err);
2413	mana_destroy_txq(apc);
2414	return err;
2415}
2416
2417static void mana_destroy_rxq(struct mana_port_context *apc,
2418			     struct mana_rxq *rxq, bool napi_initialized)
2419
2420{
2421	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
2422	struct mana_recv_buf_oob *rx_oob;
2423	struct device *dev = gc->dev;
2424	struct napi_struct *napi;
2425	struct page *page;
2426	int i;
2427
2428	if (!rxq)
2429		return;
2430
2431	debugfs_remove_recursive(rxq->mana_rx_debugfs);
2432	rxq->mana_rx_debugfs = NULL;
2433
2434	napi = &rxq->rx_cq.napi;
2435
2436	if (napi_initialized) {
2437		napi_synchronize(napi);
2438
2439		napi_disable_locked(napi);
2440		netif_napi_del_locked(napi);
2441	}
2442	xdp_rxq_info_unreg(&rxq->xdp_rxq);
2443
2444	mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj);
2445
2446	mana_deinit_cq(apc, &rxq->rx_cq);
2447
2448	if (rxq->xdp_save_va)
2449		put_page(virt_to_head_page(rxq->xdp_save_va));
2450
2451	for (i = 0; i < rxq->num_rx_buf; i++) {
2452		rx_oob = &rxq->rx_oobs[i];
2453
2454		if (!rx_oob->buf_va)
2455			continue;
2456
2457		page = virt_to_head_page(rx_oob->buf_va);
2458
2459		if (rxq->frag_count == 1 || !rx_oob->from_pool) {
2460			dma_unmap_single(dev, rx_oob->sgl[0].address,
2461					 rx_oob->sgl[0].size, DMA_FROM_DEVICE);
2462			mana_put_rx_page(rxq, page, rx_oob->from_pool);
2463		} else {
2464			page_pool_free_va(rxq->page_pool, rx_oob->buf_va, true);
2465		}
2466
2467		rx_oob->buf_va = NULL;
2468	}
2469
2470	page_pool_destroy(rxq->page_pool);
2471
2472	if (rxq->gdma_rq)
2473		mana_gd_destroy_queue(gc, rxq->gdma_rq);
2474
2475	kfree(rxq);
2476}
2477
2478static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key,
2479			    struct mana_rxq *rxq, struct device *dev)
2480{
2481	struct mana_port_context *mpc = netdev_priv(rxq->ndev);
2482	bool from_pool = false;
2483	dma_addr_t da;
2484	void *va;
2485
2486	if (mpc->rxbufs_pre)
2487		va = mana_get_rxbuf_pre(rxq, &da);
2488	else
2489		va = mana_get_rxfrag(rxq, dev, &da, &from_pool);
2490
2491	if (!va)
2492		return -ENOMEM;
2493
2494	rx_oob->buf_va = va;
2495	rx_oob->from_pool = from_pool;
2496
2497	rx_oob->sgl[0].address = da;
2498	rx_oob->sgl[0].size = rxq->datasize;
2499	rx_oob->sgl[0].mem_key = mem_key;
2500
2501	return 0;
2502}
2503
2504#define MANA_WQE_HEADER_SIZE 16
2505#define MANA_WQE_SGE_SIZE 16
2506
2507static int mana_alloc_rx_wqe(struct mana_port_context *apc,
2508			     struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size)
2509{
2510	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
2511	struct mana_recv_buf_oob *rx_oob;
2512	struct device *dev = gc->dev;
2513	u32 buf_idx;
2514	int ret;
2515
2516	WARN_ON(rxq->datasize == 0);
2517
2518	*rxq_size = 0;
2519	*cq_size = 0;
2520
2521	for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {
2522		rx_oob = &rxq->rx_oobs[buf_idx];
2523		memset(rx_oob, 0, sizeof(*rx_oob));
2524
2525		rx_oob->num_sge = 1;
2526
2527		ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq,
2528				       dev);
2529		if (ret)
2530			return ret;
2531
2532		rx_oob->wqe_req.sgl = rx_oob->sgl;
2533		rx_oob->wqe_req.num_sge = rx_oob->num_sge;
2534		rx_oob->wqe_req.inline_oob_size = 0;
2535		rx_oob->wqe_req.inline_oob_data = NULL;
2536		rx_oob->wqe_req.flags = 0;
2537		rx_oob->wqe_req.client_data_unit = 0;
2538
2539		*rxq_size += ALIGN(MANA_WQE_HEADER_SIZE +
2540				   MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32);
2541		*cq_size += COMP_ENTRY_SIZE;
2542	}
2543
2544	return 0;
2545}
2546
2547static int mana_push_wqe(struct mana_rxq *rxq)
2548{
2549	struct mana_recv_buf_oob *rx_oob;
2550	u32 buf_idx;
2551	int err;
2552
2553	for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {
2554		rx_oob = &rxq->rx_oobs[buf_idx];
2555
2556		err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req,
2557					    &rx_oob->wqe_inf);
2558		if (err)
2559			return -ENOSPC;
2560	}
2561
2562	return 0;
2563}
2564
2565static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc)
2566{
2567	struct mana_port_context *mpc = netdev_priv(rxq->ndev);
2568	struct page_pool_params pprm = {};
2569	int ret;
2570
2571	pprm.pool_size = mpc->rx_queue_size / rxq->frag_count + 1;
2572	pprm.nid = gc->numa_node;
2573	pprm.napi = &rxq->rx_cq.napi;
2574	pprm.netdev = rxq->ndev;
2575	pprm.order = get_order(rxq->alloc_size);
2576	pprm.queue_idx = rxq->rxq_idx;
2577	pprm.dev = gc->dev;
2578
2579	/* Let the page pool do the dma map when page sharing with multiple
2580	 * fragments enabled for rx buffers.
2581	 */
2582	if (rxq->frag_count > 1) {
2583		pprm.flags =  PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
2584		pprm.max_len = PAGE_SIZE;
2585		pprm.dma_dir = DMA_FROM_DEVICE;
2586	}
2587
2588	rxq->page_pool = page_pool_create(&pprm);
2589
2590	if (IS_ERR(rxq->page_pool)) {
2591		ret = PTR_ERR(rxq->page_pool);
2592		rxq->page_pool = NULL;
2593		return ret;
2594	}
2595
2596	return 0;
2597}
2598
2599static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
2600					u32 rxq_idx, struct mana_eq *eq,
2601					struct net_device *ndev)
2602{
2603	struct gdma_dev *gd = apc->ac->gdma_dev;
2604	struct mana_obj_spec wq_spec;
2605	struct mana_obj_spec cq_spec;
2606	struct gdma_queue_spec spec;
2607	struct mana_cq *cq = NULL;
2608	struct gdma_context *gc;
2609	u32 cq_size, rq_size;
2610	struct mana_rxq *rxq;
2611	int err;
2612
2613	gc = gd->gdma_context;
2614
2615	rxq = kzalloc(struct_size(rxq, rx_oobs, apc->rx_queue_size),
2616		      GFP_KERNEL);
2617	if (!rxq)
2618		return NULL;
2619
2620	rxq->ndev = ndev;
2621	rxq->num_rx_buf = apc->rx_queue_size;
2622	rxq->rxq_idx = rxq_idx;
2623	rxq->rxobj = INVALID_MANA_HANDLE;
2624
2625	mana_get_rxbuf_cfg(apc, ndev->mtu, &rxq->datasize, &rxq->alloc_size,
2626			   &rxq->headroom, &rxq->frag_count);
2627	/* Create page pool for RX queue */
2628	err = mana_create_page_pool(rxq, gc);
2629	if (err) {
2630		netdev_err(ndev, "Create page pool err:%d\n", err);
2631		goto out;
2632	}
2633
2634	err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size);
2635	if (err)
2636		goto out;
2637
2638	rq_size = MANA_PAGE_ALIGN(rq_size);
2639	cq_size = MANA_PAGE_ALIGN(cq_size);
2640
2641	/* Create RQ */
2642	memset(&spec, 0, sizeof(spec));
2643	spec.type = GDMA_RQ;
2644	spec.monitor_avl_buf = true;
2645	spec.queue_size = rq_size;
2646	err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq);
2647	if (err)
2648		goto out;
2649
2650	/* Create RQ's CQ */
2651	cq = &rxq->rx_cq;
2652	cq->type = MANA_CQ_TYPE_RX;
2653	cq->rxq = rxq;
2654
2655	memset(&spec, 0, sizeof(spec));
2656	spec.type = GDMA_CQ;
2657	spec.monitor_avl_buf = false;
2658	spec.queue_size = cq_size;
2659	spec.cq.callback = mana_schedule_napi;
2660	spec.cq.parent_eq = eq->eq;
2661	spec.cq.context = cq;
2662	err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);
2663	if (err)
2664		goto out;
2665
2666	memset(&wq_spec, 0, sizeof(wq_spec));
2667	memset(&cq_spec, 0, sizeof(cq_spec));
2668	wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle;
2669	wq_spec.queue_size = rxq->gdma_rq->queue_size;
2670
2671	cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle;
2672	cq_spec.queue_size = cq->gdma_cq->queue_size;
2673	cq_spec.modr_ctx_id = 0;
2674	cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
2675
2676	err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ,
2677				 &wq_spec, &cq_spec, &rxq->rxobj);
2678	if (err)
2679		goto out;
2680
2681	rxq->gdma_rq->id = wq_spec.queue_index;
2682	cq->gdma_cq->id = cq_spec.queue_index;
2683
2684	rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;
2685	cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;
2686
2687	rxq->gdma_id = rxq->gdma_rq->id;
2688	cq->gdma_id = cq->gdma_cq->id;
2689
2690	err = mana_push_wqe(rxq);
2691	if (err)
2692		goto out;
2693
2694	if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
2695		err = -EINVAL;
2696		goto out;
2697	}
2698
2699	gc->cq_table[cq->gdma_id] = cq->gdma_cq;
2700
2701	netif_napi_add_weight_locked(ndev, &cq->napi, mana_poll, 1);
2702
2703	WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx,
2704				 cq->napi.napi_id));
2705	WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL,
2706					   rxq->page_pool));
2707
2708	napi_enable_locked(&cq->napi);
2709
2710	mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
2711out:
2712	if (!err)
2713		return rxq;
2714
2715	netdev_err(ndev, "Failed to create RXQ: err = %d\n", err);
2716
2717	mana_destroy_rxq(apc, rxq, false);
2718
2719	if (cq)
2720		mana_deinit_cq(apc, cq);
2721
2722	return NULL;
2723}
2724
2725static void mana_create_rxq_debugfs(struct mana_port_context *apc, int idx)
2726{
2727	struct mana_rxq *rxq;
2728	char qnum[32];
2729
2730	rxq = apc->rxqs[idx];
2731
2732	sprintf(qnum, "RX-%d", idx);
2733	rxq->mana_rx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs);
2734	debugfs_create_u32("rq_head", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->head);
2735	debugfs_create_u32("rq_tail", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->tail);
2736	debugfs_create_u32("rq_nbuf", 0400, rxq->mana_rx_debugfs, &rxq->num_rx_buf);
2737	debugfs_create_u32("cq_head", 0400, rxq->mana_rx_debugfs,
2738			   &rxq->rx_cq.gdma_cq->head);
2739	debugfs_create_u32("cq_tail", 0400, rxq->mana_rx_debugfs,
2740			   &rxq->rx_cq.gdma_cq->tail);
2741	debugfs_create_u32("cq_budget", 0400, rxq->mana_rx_debugfs, &rxq->rx_cq.budget);
2742	debugfs_create_file("rxq_dump", 0400, rxq->mana_rx_debugfs, rxq->gdma_rq, &mana_dbg_q_fops);
2743	debugfs_create_file("cq_dump", 0400, rxq->mana_rx_debugfs, rxq->rx_cq.gdma_cq,
2744			    &mana_dbg_q_fops);
2745}
2746
2747static int mana_add_rx_queues(struct mana_port_context *apc,
2748			      struct net_device *ndev)
2749{
2750	struct mana_context *ac = apc->ac;
2751	struct mana_rxq *rxq;
2752	int err = 0;
2753	int i;
2754
2755	for (i = 0; i < apc->num_queues; i++) {
2756		rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev);
2757		if (!rxq) {
2758			err = -ENOMEM;
2759			netdev_err(ndev, "Failed to create rxq %d : %d\n", i, err);
2760			goto out;
2761		}
2762
2763		u64_stats_init(&rxq->stats.syncp);
2764
2765		apc->rxqs[i] = rxq;
2766
2767		mana_create_rxq_debugfs(apc, i);
2768	}
2769
2770	apc->default_rxobj = apc->rxqs[0]->rxobj;
2771out:
2772	return err;
2773}
2774
2775static void mana_destroy_vport(struct mana_port_context *apc)
2776{
2777	struct gdma_dev *gd = apc->ac->gdma_dev;
2778	struct mana_rxq *rxq;
2779	u32 rxq_idx;
2780
2781	for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
2782		rxq = apc->rxqs[rxq_idx];
2783		if (!rxq)
2784			continue;
2785
2786		mana_destroy_rxq(apc, rxq, true);
2787		apc->rxqs[rxq_idx] = NULL;
2788	}
2789
2790	mana_destroy_txq(apc);
2791	mana_uncfg_vport(apc);
2792
2793	if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode)
2794		mana_pf_deregister_hw_vport(apc);
2795}
2796
2797static int mana_create_vport(struct mana_port_context *apc,
2798			     struct net_device *net)
2799{
2800	struct gdma_dev *gd = apc->ac->gdma_dev;
2801	int err;
2802
2803	apc->default_rxobj = INVALID_MANA_HANDLE;
2804
2805	if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) {
2806		err = mana_pf_register_hw_vport(apc);
2807		if (err)
2808			return err;
2809	}
2810
2811	err = mana_cfg_vport(apc, gd->pdid, gd->doorbell);
2812	if (err)
2813		return err;
2814
2815	return mana_create_txq(apc, net);
2816}
2817
2818static int mana_rss_table_alloc(struct mana_port_context *apc)
2819{
2820	if (!apc->indir_table_sz) {
2821		netdev_err(apc->ndev,
2822			   "Indirection table size not set for vPort %d\n",
2823			   apc->port_idx);
2824		return -EINVAL;
2825	}
2826
2827	apc->indir_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL);
2828	if (!apc->indir_table)
2829		return -ENOMEM;
2830
2831	apc->rxobj_table = kcalloc(apc->indir_table_sz, sizeof(mana_handle_t), GFP_KERNEL);
2832	if (!apc->rxobj_table) {
2833		kfree(apc->indir_table);
2834		return -ENOMEM;
2835	}
2836
2837	return 0;
2838}
2839
2840static void mana_rss_table_init(struct mana_port_context *apc)
2841{
2842	int i;
2843
2844	for (i = 0; i < apc->indir_table_sz; i++)
2845		apc->indir_table[i] =
2846			ethtool_rxfh_indir_default(i, apc->num_queues);
2847}
2848
2849int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx,
2850		    bool update_hash, bool update_tab)
2851{
2852	u32 queue_idx;
2853	int err;
2854	int i;
2855
2856	if (update_tab) {
2857		for (i = 0; i < apc->indir_table_sz; i++) {
2858			queue_idx = apc->indir_table[i];
2859			apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj;
2860		}
2861	}
2862
2863	err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab);
2864	if (err)
2865		return err;
2866
2867	mana_fence_rqs(apc);
2868
2869	return 0;
2870}
2871
2872int mana_query_gf_stats(struct mana_context *ac)
2873{
2874	struct gdma_context *gc = ac->gdma_dev->gdma_context;
2875	struct mana_query_gf_stat_resp resp = {};
2876	struct mana_query_gf_stat_req req = {};
2877	struct device *dev = gc->dev;
2878	int err;
2879
2880	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_GF_STAT,
2881			     sizeof(req), sizeof(resp));
2882	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;
2883	req.req_stats = STATISTICS_FLAGS_RX_DISCARDS_NO_WQE |
2884			STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED |
2885			STATISTICS_FLAGS_HC_RX_BYTES |
2886			STATISTICS_FLAGS_HC_RX_UCAST_PACKETS |
2887			STATISTICS_FLAGS_HC_RX_UCAST_BYTES |
2888			STATISTICS_FLAGS_HC_RX_MCAST_PACKETS |
2889			STATISTICS_FLAGS_HC_RX_MCAST_BYTES |
2890			STATISTICS_FLAGS_HC_RX_BCAST_PACKETS |
2891			STATISTICS_FLAGS_HC_RX_BCAST_BYTES |
2892			STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED |
2893			STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED |
2894			STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS |
2895			STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT |
2896			STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT |
2897			STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT |
2898			STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT |
2899			STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT |
2900			STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION |
2901			STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB |
2902			STATISTICS_FLAGS_HC_TX_BYTES |
2903			STATISTICS_FLAGS_HC_TX_UCAST_PACKETS |
2904			STATISTICS_FLAGS_HC_TX_UCAST_BYTES |
2905			STATISTICS_FLAGS_HC_TX_MCAST_PACKETS |
2906			STATISTICS_FLAGS_HC_TX_MCAST_BYTES |
2907			STATISTICS_FLAGS_HC_TX_BCAST_PACKETS |
2908			STATISTICS_FLAGS_HC_TX_BCAST_BYTES |
2909			STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR;
2910
2911	err = mana_send_request(ac, &req, sizeof(req), &resp,
2912				sizeof(resp));
2913	if (err) {
2914		dev_err(dev, "Failed to query GF stats: %d\n", err);
2915		return err;
2916	}
2917	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_GF_STAT,
2918				   sizeof(resp));
2919	if (err || resp.hdr.status) {
2920		dev_err(dev, "Failed to query GF stats: %d, 0x%x\n", err,
2921			resp.hdr.status);
2922		return err;
2923	}
2924
2925	ac->hc_stats.hc_rx_discards_no_wqe = resp.rx_discards_nowqe;
2926	ac->hc_stats.hc_rx_err_vport_disabled = resp.rx_err_vport_disabled;
2927	ac->hc_stats.hc_rx_bytes = resp.hc_rx_bytes;
2928	ac->hc_stats.hc_rx_ucast_pkts = resp.hc_rx_ucast_pkts;
2929	ac->hc_stats.hc_rx_ucast_bytes = resp.hc_rx_ucast_bytes;
2930	ac->hc_stats.hc_rx_bcast_pkts = resp.hc_rx_bcast_pkts;
2931	ac->hc_stats.hc_rx_bcast_bytes = resp.hc_rx_bcast_bytes;
2932	ac->hc_stats.hc_rx_mcast_pkts = resp.hc_rx_mcast_pkts;
2933	ac->hc_stats.hc_rx_mcast_bytes = resp.hc_rx_mcast_bytes;
2934	ac->hc_stats.hc_tx_err_gf_disabled = resp.tx_err_gf_disabled;
2935	ac->hc_stats.hc_tx_err_vport_disabled = resp.tx_err_vport_disabled;
2936	ac->hc_stats.hc_tx_err_inval_vportoffset_pkt =
2937					     resp.tx_err_inval_vport_offset_pkt;
2938	ac->hc_stats.hc_tx_err_vlan_enforcement =
2939					     resp.tx_err_vlan_enforcement;
2940	ac->hc_stats.hc_tx_err_eth_type_enforcement =
2941					     resp.tx_err_ethtype_enforcement;
2942	ac->hc_stats.hc_tx_err_sa_enforcement = resp.tx_err_SA_enforcement;
2943	ac->hc_stats.hc_tx_err_sqpdid_enforcement =
2944					     resp.tx_err_SQPDID_enforcement;
2945	ac->hc_stats.hc_tx_err_cqpdid_enforcement =
2946					     resp.tx_err_CQPDID_enforcement;
2947	ac->hc_stats.hc_tx_err_mtu_violation = resp.tx_err_mtu_violation;
2948	ac->hc_stats.hc_tx_err_inval_oob = resp.tx_err_inval_oob;
2949	ac->hc_stats.hc_tx_bytes = resp.hc_tx_bytes;
2950	ac->hc_stats.hc_tx_ucast_pkts = resp.hc_tx_ucast_pkts;
2951	ac->hc_stats.hc_tx_ucast_bytes = resp.hc_tx_ucast_bytes;
2952	ac->hc_stats.hc_tx_bcast_pkts = resp.hc_tx_bcast_pkts;
2953	ac->hc_stats.hc_tx_bcast_bytes = resp.hc_tx_bcast_bytes;
2954	ac->hc_stats.hc_tx_mcast_pkts = resp.hc_tx_mcast_pkts;
2955	ac->hc_stats.hc_tx_mcast_bytes = resp.hc_tx_mcast_bytes;
2956	ac->hc_stats.hc_tx_err_gdma = resp.tx_err_gdma;
2957
2958	return 0;
2959}
2960
2961void mana_query_phy_stats(struct mana_port_context *apc)
2962{
2963	struct mana_query_phy_stat_resp resp = {};
2964	struct mana_query_phy_stat_req req = {};
2965	struct net_device *ndev = apc->ndev;
2966	int err;
2967
2968	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_PHY_STAT,
2969			     sizeof(req), sizeof(resp));
2970	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
2971				sizeof(resp));
2972	if (err)
2973		return;
2974
2975	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_PHY_STAT,
2976				   sizeof(resp));
2977	if (err || resp.hdr.status) {
2978		netdev_err(ndev,
2979			   "Failed to query PHY stats: %d, resp:0x%x\n",
2980				err, resp.hdr.status);
2981		return;
2982	}
2983
2984	/* Aggregate drop counters */
2985	apc->phy_stats.rx_pkt_drop_phy = resp.rx_pkt_drop_phy;
2986	apc->phy_stats.tx_pkt_drop_phy = resp.tx_pkt_drop_phy;
2987
2988	/* Per TC traffic Counters */
2989	apc->phy_stats.rx_pkt_tc0_phy = resp.rx_pkt_tc0_phy;
2990	apc->phy_stats.tx_pkt_tc0_phy = resp.tx_pkt_tc0_phy;
2991	apc->phy_stats.rx_pkt_tc1_phy = resp.rx_pkt_tc1_phy;
2992	apc->phy_stats.tx_pkt_tc1_phy = resp.tx_pkt_tc1_phy;
2993	apc->phy_stats.rx_pkt_tc2_phy = resp.rx_pkt_tc2_phy;
2994	apc->phy_stats.tx_pkt_tc2_phy = resp.tx_pkt_tc2_phy;
2995	apc->phy_stats.rx_pkt_tc3_phy = resp.rx_pkt_tc3_phy;
2996	apc->phy_stats.tx_pkt_tc3_phy = resp.tx_pkt_tc3_phy;
2997	apc->phy_stats.rx_pkt_tc4_phy = resp.rx_pkt_tc4_phy;
2998	apc->phy_stats.tx_pkt_tc4_phy = resp.tx_pkt_tc4_phy;
2999	apc->phy_stats.rx_pkt_tc5_phy = resp.rx_pkt_tc5_phy;
3000	apc->phy_stats.tx_pkt_tc5_phy = resp.tx_pkt_tc5_phy;
3001	apc->phy_stats.rx_pkt_tc6_phy = resp.rx_pkt_tc6_phy;
3002	apc->phy_stats.tx_pkt_tc6_phy = resp.tx_pkt_tc6_phy;
3003	apc->phy_stats.rx_pkt_tc7_phy = resp.rx_pkt_tc7_phy;
3004	apc->phy_stats.tx_pkt_tc7_phy = resp.tx_pkt_tc7_phy;
3005
3006	/* Per TC byte Counters */
3007	apc->phy_stats.rx_byte_tc0_phy = resp.rx_byte_tc0_phy;
3008	apc->phy_stats.tx_byte_tc0_phy = resp.tx_byte_tc0_phy;
3009	apc->phy_stats.rx_byte_tc1_phy = resp.rx_byte_tc1_phy;
3010	apc->phy_stats.tx_byte_tc1_phy = resp.tx_byte_tc1_phy;
3011	apc->phy_stats.rx_byte_tc2_phy = resp.rx_byte_tc2_phy;
3012	apc->phy_stats.tx_byte_tc2_phy = resp.tx_byte_tc2_phy;
3013	apc->phy_stats.rx_byte_tc3_phy = resp.rx_byte_tc3_phy;
3014	apc->phy_stats.tx_byte_tc3_phy = resp.tx_byte_tc3_phy;
3015	apc->phy_stats.rx_byte_tc4_phy = resp.rx_byte_tc4_phy;
3016	apc->phy_stats.tx_byte_tc4_phy = resp.tx_byte_tc4_phy;
3017	apc->phy_stats.rx_byte_tc5_phy = resp.rx_byte_tc5_phy;
3018	apc->phy_stats.tx_byte_tc5_phy = resp.tx_byte_tc5_phy;
3019	apc->phy_stats.rx_byte_tc6_phy = resp.rx_byte_tc6_phy;
3020	apc->phy_stats.tx_byte_tc6_phy = resp.tx_byte_tc6_phy;
3021	apc->phy_stats.rx_byte_tc7_phy = resp.rx_byte_tc7_phy;
3022	apc->phy_stats.tx_byte_tc7_phy = resp.tx_byte_tc7_phy;
3023
3024	/* Per TC pause Counters */
3025	apc->phy_stats.rx_pause_tc0_phy = resp.rx_pause_tc0_phy;
3026	apc->phy_stats.tx_pause_tc0_phy = resp.tx_pause_tc0_phy;
3027	apc->phy_stats.rx_pause_tc1_phy = resp.rx_pause_tc1_phy;
3028	apc->phy_stats.tx_pause_tc1_phy = resp.tx_pause_tc1_phy;
3029	apc->phy_stats.rx_pause_tc2_phy = resp.rx_pause_tc2_phy;
3030	apc->phy_stats.tx_pause_tc2_phy = resp.tx_pause_tc2_phy;
3031	apc->phy_stats.rx_pause_tc3_phy = resp.rx_pause_tc3_phy;
3032	apc->phy_stats.tx_pause_tc3_phy = resp.tx_pause_tc3_phy;
3033	apc->phy_stats.rx_pause_tc4_phy = resp.rx_pause_tc4_phy;
3034	apc->phy_stats.tx_pause_tc4_phy = resp.tx_pause_tc4_phy;
3035	apc->phy_stats.rx_pause_tc5_phy = resp.rx_pause_tc5_phy;
3036	apc->phy_stats.tx_pause_tc5_phy = resp.tx_pause_tc5_phy;
3037	apc->phy_stats.rx_pause_tc6_phy = resp.rx_pause_tc6_phy;
3038	apc->phy_stats.tx_pause_tc6_phy = resp.tx_pause_tc6_phy;
3039	apc->phy_stats.rx_pause_tc7_phy = resp.rx_pause_tc7_phy;
3040	apc->phy_stats.tx_pause_tc7_phy = resp.tx_pause_tc7_phy;
3041}
3042
3043static int mana_init_port(struct net_device *ndev)
3044{
3045	struct mana_port_context *apc = netdev_priv(ndev);
3046	struct gdma_dev *gd = apc->ac->gdma_dev;
3047	u32 max_txq, max_rxq, max_queues;
3048	int port_idx = apc->port_idx;
3049	struct gdma_context *gc;
3050	char vport[32];
3051	int err;
3052
3053	err = mana_init_port_context(apc);
3054	if (err)
3055		return err;
3056
3057	gc = gd->gdma_context;
3058
3059	err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq,
3060				   &apc->indir_table_sz);
3061	if (err) {
3062		netdev_err(ndev, "Failed to query info for vPort %d\n",
3063			   port_idx);
3064		goto reset_apc;
3065	}
3066
3067	max_queues = min_t(u32, max_txq, max_rxq);
3068	if (apc->max_queues > max_queues)
3069		apc->max_queues = max_queues;
3070
3071	if (apc->num_queues > apc->max_queues)
3072		apc->num_queues = apc->max_queues;
3073
3074	eth_hw_addr_set(ndev, apc->mac_addr);
3075	sprintf(vport, "vport%d", port_idx);
3076	apc->mana_port_debugfs = debugfs_create_dir(vport, gc->mana_pci_debugfs);
3077	return 0;
3078
3079reset_apc:
3080	mana_cleanup_port_context(apc);
3081	return err;
3082}
3083
3084int mana_alloc_queues(struct net_device *ndev)
3085{
3086	struct mana_port_context *apc = netdev_priv(ndev);
3087	struct gdma_dev *gd = apc->ac->gdma_dev;
3088	int err;
3089
3090	err = mana_create_vport(apc, ndev);
3091	if (err) {
3092		netdev_err(ndev, "Failed to create vPort %u : %d\n", apc->port_idx, err);
3093		return err;
3094	}
3095
3096	err = netif_set_real_num_tx_queues(ndev, apc->num_queues);
3097	if (err) {
3098		netdev_err(ndev,
3099			   "netif_set_real_num_tx_queues () failed for ndev with num_queues %u : %d\n",
3100			   apc->num_queues, err);
3101		goto destroy_vport;
3102	}
3103
3104	err = mana_add_rx_queues(apc, ndev);
3105	if (err)
3106		goto destroy_vport;
3107
3108	apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE;
3109
3110	err = netif_set_real_num_rx_queues(ndev, apc->num_queues);
3111	if (err) {
3112		netdev_err(ndev,
3113			   "netif_set_real_num_rx_queues () failed for ndev with num_queues %u : %d\n",
3114			   apc->num_queues, err);
3115		goto destroy_vport;
3116	}
3117
3118	mana_rss_table_init(apc);
3119
3120	err = mana_config_rss(apc, TRI_STATE_TRUE, true, true);
3121	if (err) {
3122		netdev_err(ndev, "Failed to configure RSS table: %d\n", err);
3123		goto destroy_vport;
3124	}
3125
3126	if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) {
3127		err = mana_pf_register_filter(apc);
3128		if (err)
3129			goto destroy_vport;
3130	}
3131
3132	mana_chn_setxdp(apc, mana_xdp_get(apc));
3133
3134	return 0;
3135
3136destroy_vport:
3137	mana_destroy_vport(apc);
3138	return err;
3139}
3140
3141int mana_attach(struct net_device *ndev)
3142{
3143	struct mana_port_context *apc = netdev_priv(ndev);
3144	int err;
3145
3146	ASSERT_RTNL();
3147
3148	err = mana_init_port(ndev);
3149	if (err)
3150		return err;
3151
3152	if (apc->port_st_save) {
3153		err = mana_alloc_queues(ndev);
3154		if (err) {
3155			mana_cleanup_port_context(apc);
3156			return err;
3157		}
3158	}
3159
3160	apc->port_is_up = apc->port_st_save;
3161
3162	/* Ensure port state updated before txq state */
3163	smp_wmb();
3164
3165	netif_device_attach(ndev);
3166
3167	return 0;
3168}
3169
3170static int mana_dealloc_queues(struct net_device *ndev)
3171{
3172	struct mana_port_context *apc = netdev_priv(ndev);
3173	unsigned long timeout = jiffies + 120 * HZ;
3174	struct gdma_dev *gd = apc->ac->gdma_dev;
3175	struct mana_txq *txq;
3176	struct sk_buff *skb;
3177	int i, err;
3178	u32 tsleep;
3179
3180	if (apc->port_is_up)
3181		return -EINVAL;
3182
3183	mana_chn_setxdp(apc, NULL);
3184
3185	if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode)
3186		mana_pf_deregister_filter(apc);
3187
3188	/* No packet can be transmitted now since apc->port_is_up is false.
3189	 * There is still a tiny chance that mana_poll_tx_cq() can re-enable
3190	 * a txq because it may not timely see apc->port_is_up being cleared
3191	 * to false, but it doesn't matter since mana_start_xmit() drops any
3192	 * new packets due to apc->port_is_up being false.
3193	 *
3194	 * Drain all the in-flight TX packets.
3195	 * A timeout of 120 seconds for all the queues is used.
3196	 * This will break the while loop when h/w is not responding.
3197	 * This value of 120 has been decided here considering max
3198	 * number of queues.
3199	 */
3200
3201	for (i = 0; i < apc->num_queues; i++) {
3202		txq = &apc->tx_qp[i].txq;
3203		tsleep = 1000;
3204		while (atomic_read(&txq->pending_sends) > 0 &&
3205		       time_before(jiffies, timeout)) {
3206			usleep_range(tsleep, tsleep + 1000);
3207			tsleep <<= 1;
3208		}
3209		if (atomic_read(&txq->pending_sends)) {
3210			err = pcie_flr(to_pci_dev(gd->gdma_context->dev));
3211			if (err) {
3212				netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n",
3213					   err, atomic_read(&txq->pending_sends),
3214					   txq->gdma_txq_id);
3215			}
3216			break;
3217		}
3218	}
3219
3220	for (i = 0; i < apc->num_queues; i++) {
3221		txq = &apc->tx_qp[i].txq;
3222		while ((skb = skb_dequeue(&txq->pending_skbs))) {
3223			mana_unmap_skb(skb, apc);
3224			dev_kfree_skb_any(skb);
3225		}
3226		atomic_set(&txq->pending_sends, 0);
3227	}
3228	/* We're 100% sure the queues can no longer be woken up, because
3229	 * we're sure now mana_poll_tx_cq() can't be running.
3230	 */
3231
3232	apc->rss_state = TRI_STATE_FALSE;
3233	err = mana_config_rss(apc, TRI_STATE_FALSE, false, false);
3234	if (err && mana_en_need_log(apc, err))
3235		netdev_err(ndev, "Failed to disable vPort: %d\n", err);
3236
3237	/* Even in err case, still need to cleanup the vPort */
3238	mana_destroy_vport(apc);
3239
3240	return 0;
3241}
3242
3243int mana_detach(struct net_device *ndev, bool from_close)
3244{
3245	struct mana_port_context *apc = netdev_priv(ndev);
3246	int err;
3247
3248	ASSERT_RTNL();
3249
3250	apc->port_st_save = apc->port_is_up;
3251	apc->port_is_up = false;
3252
3253	/* Ensure port state updated before txq state */
3254	smp_wmb();
3255
3256	netif_tx_disable(ndev);
3257
3258	if (apc->port_st_save) {
3259		err = mana_dealloc_queues(ndev);
3260		if (err) {
3261			netdev_err(ndev, "%s failed to deallocate queues: %d\n", __func__, err);
3262			return err;
3263		}
3264	}
3265
3266	if (!from_close) {
3267		netif_device_detach(ndev);
3268		mana_cleanup_port_context(apc);
3269	}
3270
3271	return 0;
3272}
3273
3274static int mana_probe_port(struct mana_context *ac, int port_idx,
3275			   struct net_device **ndev_storage)
3276{
3277	struct gdma_context *gc = ac->gdma_dev->gdma_context;
3278	struct mana_port_context *apc;
3279	struct net_device *ndev;
3280	int err;
3281
3282	ndev = alloc_etherdev_mq(sizeof(struct mana_port_context),
3283				 gc->max_num_queues);
3284	if (!ndev)
3285		return -ENOMEM;
3286
3287	*ndev_storage = ndev;
3288
3289	apc = netdev_priv(ndev);
3290	apc->ac = ac;
3291	apc->ndev = ndev;
3292	apc->max_queues = gc->max_num_queues;
3293	apc->num_queues = gc->max_num_queues;
3294	apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE;
3295	apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE;
3296	apc->port_handle = INVALID_MANA_HANDLE;
3297	apc->pf_filter_handle = INVALID_MANA_HANDLE;
3298	apc->port_idx = port_idx;
3299
3300	mutex_init(&apc->vport_mutex);
3301	apc->vport_use_count = 0;
3302
3303	ndev->netdev_ops = &mana_devops;
3304	ndev->ethtool_ops = &mana_ethtool_ops;
3305	ndev->mtu = ETH_DATA_LEN;
3306	ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;
3307	ndev->min_mtu = ETH_MIN_MTU;
3308	ndev->needed_headroom = MANA_HEADROOM;
3309	ndev->dev_port = port_idx;
3310	SET_NETDEV_DEV(ndev, gc->dev);
3311
3312	netif_set_tso_max_size(ndev, GSO_MAX_SIZE);
3313
3314	netif_carrier_off(ndev);
3315
3316	netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE);
3317
3318	err = mana_init_port(ndev);
3319	if (err)
3320		goto free_net;
3321
3322	err = mana_rss_table_alloc(apc);
3323	if (err)
3324		goto reset_apc;
3325
3326	netdev_lockdep_set_classes(ndev);
3327
3328	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3329	ndev->hw_features |= NETIF_F_RXCSUM;
3330	ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
3331	ndev->hw_features |= NETIF_F_RXHASH;
3332	ndev->features = ndev->hw_features | NETIF_F_HW_VLAN_CTAG_TX |
3333			 NETIF_F_HW_VLAN_CTAG_RX;
3334	ndev->vlan_features = ndev->features;
3335	xdp_set_features_flag(ndev, NETDEV_XDP_ACT_BASIC |
3336			      NETDEV_XDP_ACT_REDIRECT |
3337			      NETDEV_XDP_ACT_NDO_XMIT);
3338
3339	err = register_netdev(ndev);
3340	if (err) {
3341		netdev_err(ndev, "Unable to register netdev.\n");
3342		goto free_indir;
3343	}
3344
3345	netif_carrier_on(ndev);
3346
3347	debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, &apc->speed);
3348
3349	return 0;
3350
3351free_indir:
3352	mana_cleanup_indir_table(apc);
3353reset_apc:
3354	mana_cleanup_port_context(apc);
3355free_net:
3356	*ndev_storage = NULL;
3357	netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err);
3358	free_netdev(ndev);
3359	return err;
3360}
3361
3362static void adev_release(struct device *dev)
3363{
3364	struct mana_adev *madev = container_of(dev, struct mana_adev, adev.dev);
3365
3366	kfree(madev);
3367}
3368
3369static void remove_adev(struct gdma_dev *gd)
3370{
3371	struct auxiliary_device *adev = gd->adev;
3372	int id = adev->id;
3373
3374	auxiliary_device_delete(adev);
3375	auxiliary_device_uninit(adev);
3376
3377	mana_adev_idx_free(id);
3378	gd->adev = NULL;
3379}
3380
3381static int add_adev(struct gdma_dev *gd, const char *name)
3382{
3383	struct auxiliary_device *adev;
3384	struct mana_adev *madev;
3385	int ret;
3386	int id;
3387
3388	madev = kzalloc(sizeof(*madev), GFP_KERNEL);
3389	if (!madev)
3390		return -ENOMEM;
3391
3392	adev = &madev->adev;
3393	ret = mana_adev_idx_alloc();
3394	if (ret < 0)
3395		goto idx_fail;
3396	id = ret;
3397	adev->id = id;
3398
3399	adev->name = name;
3400	adev->dev.parent = gd->gdma_context->dev;
3401	adev->dev.release = adev_release;
3402	madev->mdev = gd;
3403
3404	ret = auxiliary_device_init(adev);
3405	if (ret)
3406		goto init_fail;
3407
3408	/* madev is owned by the auxiliary device */
3409	madev = NULL;
3410	ret = auxiliary_device_add(adev);
3411	if (ret)
3412		goto add_fail;
3413
3414	gd->adev = adev;
3415	dev_dbg(gd->gdma_context->dev,
3416		"Auxiliary device added successfully\n");
3417	return 0;
3418
3419add_fail:
3420	auxiliary_device_uninit(adev);
3421
3422init_fail:
3423	mana_adev_idx_free(id);
3424
3425idx_fail:
3426	kfree(madev);
3427
3428	return ret;
3429}
3430
3431static void mana_rdma_service_handle(struct work_struct *work)
3432{
3433	struct mana_service_work *serv_work =
3434		container_of(work, struct mana_service_work, work);
3435	struct gdma_dev *gd = serv_work->gdma_dev;
3436	struct device *dev = gd->gdma_context->dev;
3437	int ret;
3438
3439	if (READ_ONCE(gd->rdma_teardown))
3440		goto out;
3441
3442	switch (serv_work->event) {
3443	case GDMA_SERVICE_TYPE_RDMA_SUSPEND:
3444		if (!gd->adev || gd->is_suspended)
3445			break;
3446
3447		remove_adev(gd);
3448		gd->is_suspended = true;
3449		break;
3450
3451	case GDMA_SERVICE_TYPE_RDMA_RESUME:
3452		if (!gd->is_suspended)
3453			break;
3454
3455		ret = add_adev(gd, "rdma");
3456		if (ret)
3457			dev_err(dev, "Failed to add adev on resume: %d\n", ret);
3458		else
3459			gd->is_suspended = false;
3460		break;
3461
3462	default:
3463		dev_warn(dev, "unknown adev service event %u\n",
3464			 serv_work->event);
3465		break;
3466	}
3467
3468out:
3469	kfree(serv_work);
3470}
3471
3472int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event)
3473{
3474	struct gdma_dev *gd = &gc->mana_ib;
3475	struct mana_service_work *serv_work;
3476
3477	if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) {
3478		/* RDMA device is not detected on pci */
3479		return 0;
3480	}
3481
3482	serv_work = kzalloc(sizeof(*serv_work), GFP_ATOMIC);
3483	if (!serv_work)
3484		return -ENOMEM;
3485
3486	serv_work->event = event;
3487	serv_work->gdma_dev = gd;
3488
3489	INIT_WORK(&serv_work->work, mana_rdma_service_handle);
3490	queue_work(gc->service_wq, &serv_work->work);
3491
3492	return 0;
3493}
3494
3495#define MANA_GF_STATS_PERIOD (2 * HZ)
3496
3497static void mana_gf_stats_work_handler(struct work_struct *work)
3498{
3499	struct mana_context *ac =
3500		container_of(to_delayed_work(work), struct mana_context, gf_stats_work);
3501	int err;
3502
3503	err = mana_query_gf_stats(ac);
3504	if (err == -ETIMEDOUT) {
3505		/* HWC timeout detected - reset stats and stop rescheduling */
3506		ac->hwc_timeout_occurred = true;
3507		memset(&ac->hc_stats, 0, sizeof(ac->hc_stats));
3508		return;
3509	}
3510	schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD);
3511}
3512
3513int mana_probe(struct gdma_dev *gd, bool resuming)
3514{
3515	struct gdma_context *gc = gd->gdma_context;
3516	struct mana_context *ac = gd->driver_data;
3517	struct device *dev = gc->dev;
3518	u8 bm_hostmode = 0;
3519	u16 num_ports = 0;
3520	int err;
3521	int i;
3522
3523	dev_info(dev,
3524		 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n",
3525		 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION);
3526
3527	err = mana_gd_register_device(gd);
3528	if (err)
3529		return err;
3530
3531	if (!resuming) {
3532		ac = kzalloc(sizeof(*ac), GFP_KERNEL);
3533		if (!ac)
3534			return -ENOMEM;
3535
3536		ac->gdma_dev = gd;
3537		gd->driver_data = ac;
3538	}
3539
3540	err = mana_create_eq(ac);
3541	if (err) {
3542		dev_err(dev, "Failed to create EQs: %d\n", err);
3543		goto out;
3544	}
3545
3546	err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION,
3547				    MANA_MICRO_VERSION, &num_ports, &bm_hostmode);
3548	if (err)
3549		goto out;
3550
3551	ac->bm_hostmode = bm_hostmode;
3552
3553	if (!resuming) {
3554		ac->num_ports = num_ports;
3555
3556		INIT_WORK(&ac->link_change_work, mana_link_state_handle);
3557	} else {
3558		if (ac->num_ports != num_ports) {
3559			dev_err(dev, "The number of vPorts changed: %d->%d\n",
3560				ac->num_ports, num_ports);
3561			err = -EPROTO;
3562			goto out;
3563		}
3564
3565		enable_work(&ac->link_change_work);
3566	}
3567
3568	if (ac->num_ports == 0)
3569		dev_err(dev, "Failed to detect any vPort\n");
3570
3571	if (ac->num_ports > MAX_PORTS_IN_MANA_DEV)
3572		ac->num_ports = MAX_PORTS_IN_MANA_DEV;
3573
3574	if (!resuming) {
3575		for (i = 0; i < ac->num_ports; i++) {
3576			err = mana_probe_port(ac, i, &ac->ports[i]);
3577			/* we log the port for which the probe failed and stop
3578			 * probes for subsequent ports.
3579			 * Note that we keep running ports, for which the probes
3580			 * were successful, unless add_adev fails too
3581			 */
3582			if (err) {
3583				dev_err(dev, "Probe Failed for port %d\n", i);
3584				break;
3585			}
3586		}
3587	} else {
3588		for (i = 0; i < ac->num_ports; i++) {
3589			rtnl_lock();
3590			err = mana_attach(ac->ports[i]);
3591			rtnl_unlock();
3592			/* we log the port for which the attach failed and stop
3593			 * attach for subsequent ports
3594			 * Note that we keep running ports, for which the attach
3595			 * were successful, unless add_adev fails too
3596			 */
3597			if (err) {
3598				dev_err(dev, "Attach Failed for port %d\n", i);
3599				break;
3600			}
3601		}
3602	}
3603
3604	err = add_adev(gd, "eth");
3605
3606	INIT_DELAYED_WORK(&ac->gf_stats_work, mana_gf_stats_work_handler);
3607	schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD);
3608
3609out:
3610	if (err) {
3611		mana_remove(gd, false);
3612	} else {
3613		dev_dbg(dev, "gd=%p, id=%u, num_ports=%d, type=%u, instance=%u\n",
3614			gd, gd->dev_id.as_uint32, ac->num_ports,
3615			gd->dev_id.type, gd->dev_id.instance);
3616		dev_dbg(dev, "%s succeeded\n", __func__);
3617	}
3618
3619	return err;
3620}
3621
3622void mana_remove(struct gdma_dev *gd, bool suspending)
3623{
3624	struct gdma_context *gc = gd->gdma_context;
3625	struct mana_context *ac = gd->driver_data;
3626	struct mana_port_context *apc;
3627	struct device *dev = gc->dev;
3628	struct net_device *ndev;
3629	int err;
3630	int i;
3631
3632	disable_work_sync(&ac->link_change_work);
3633	cancel_delayed_work_sync(&ac->gf_stats_work);
3634
3635	/* adev currently doesn't support suspending, always remove it */
3636	if (gd->adev)
3637		remove_adev(gd);
3638
3639	for (i = 0; i < ac->num_ports; i++) {
3640		ndev = ac->ports[i];
3641		apc = netdev_priv(ndev);
3642		if (!ndev) {
3643			if (i == 0)
3644				dev_err(dev, "No net device to remove\n");
3645			goto out;
3646		}
3647
3648		/* All cleanup actions should stay after rtnl_lock(), otherwise
3649		 * other functions may access partially cleaned up data.
3650		 */
3651		rtnl_lock();
3652
3653		err = mana_detach(ndev, false);
3654		if (err)
3655			netdev_err(ndev, "Failed to detach vPort %d: %d\n",
3656				   i, err);
3657
3658		if (suspending) {
3659			/* No need to unregister the ndev. */
3660			rtnl_unlock();
3661			continue;
3662		}
3663
3664		unregister_netdevice(ndev);
3665		mana_cleanup_indir_table(apc);
3666
3667		rtnl_unlock();
3668
3669		free_netdev(ndev);
3670	}
3671
3672	mana_destroy_eq(ac);
3673out:
3674	mana_gd_deregister_device(gd);
3675
3676	if (suspending)
3677		return;
3678
3679	gd->driver_data = NULL;
3680	gd->gdma_context = NULL;
3681	kfree(ac);
3682	dev_dbg(dev, "%s succeeded\n", __func__);
3683}
3684
3685int mana_rdma_probe(struct gdma_dev *gd)
3686{
3687	int err = 0;
3688
3689	if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) {
3690		/* RDMA device is not detected on pci */
3691		return err;
3692	}
3693
3694	err = mana_gd_register_device(gd);
3695	if (err)
3696		return err;
3697
3698	err = add_adev(gd, "rdma");
3699	if (err)
3700		mana_gd_deregister_device(gd);
3701
3702	return err;
3703}
3704
3705void mana_rdma_remove(struct gdma_dev *gd)
3706{
3707	struct gdma_context *gc = gd->gdma_context;
3708
3709	if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) {
3710		/* RDMA device is not detected on pci */
3711		return;
3712	}
3713
3714	WRITE_ONCE(gd->rdma_teardown, true);
3715
3716	if (gc->service_wq)
3717		flush_workqueue(gc->service_wq);
3718
3719	if (gd->adev)
3720		remove_adev(gd);
3721
3722	mana_gd_deregister_device(gd);
3723}
3724
3725struct net_device *mana_get_primary_netdev(struct mana_context *ac,
3726					   u32 port_index,
3727					   netdevice_tracker *tracker)
3728{
3729	struct net_device *ndev;
3730
3731	if (port_index >= ac->num_ports)
3732		return NULL;
3733
3734	rcu_read_lock();
3735
3736	/* If mana is used in netvsc, the upper netdevice should be returned. */
3737	ndev = netdev_master_upper_dev_get_rcu(ac->ports[port_index]);
3738
3739	/* If there is no upper device, use the parent Ethernet device */
3740	if (!ndev)
3741		ndev = ac->ports[port_index];
3742
3743	netdev_hold(ndev, tracker, GFP_ATOMIC);
3744	rcu_read_unlock();
3745
3746	return ndev;
3747}
3748EXPORT_SYMBOL_NS(mana_get_primary_netdev, "NET_MANA");