en_rx.c

   1/*
   2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 *
  32 */
  33
  34#include <linux/bpf.h>
  35#include <linux/bpf_trace.h>
  36#include <linux/mlx4/cq.h>
  37#include <linux/slab.h>
  38#include <linux/mlx4/qp.h>
  39#include <linux/skbuff.h>
  40#include <linux/rculist.h>
  41#include <linux/if_ether.h>
  42#include <linux/if_vlan.h>
  43#include <linux/vmalloc.h>
  44#include <linux/irq.h>
  45#include <linux/skbuff_ref.h>
  46
  47#include <net/ip.h>
  48#if IS_ENABLED(CONFIG_IPV6)
  49#include <net/ip6_checksum.h>
  50#endif
  51#include <net/page_pool/helpers.h>
  52
  53#include "mlx4_en.h"
  54
  55static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv,
  56			       struct mlx4_en_rx_ring *ring,
  57			       struct mlx4_en_rx_desc *rx_desc,
  58			       struct mlx4_en_rx_alloc *frags,
  59			       gfp_t gfp)
  60{
  61	dma_addr_t dma;
  62	int i;
  63
  64	for (i = 0; i < priv->num_frags; i++, frags++) {
  65		if (!frags->page) {
  66			frags->page = page_pool_alloc_pages(ring->pp, gfp);
  67			if (!frags->page) {
  68				ring->alloc_fail++;
  69				return -ENOMEM;
  70			}
  71			page_pool_fragment_page(frags->page, 1);
  72			frags->page_offset = priv->rx_headroom;
  73
  74			ring->rx_alloc_pages++;
  75		}
  76		dma = page_pool_get_dma_addr(frags->page);
  77		rx_desc->data[i].addr = cpu_to_be64(dma + frags->page_offset);
  78	}
  79	return 0;
  80}
  81
  82static void mlx4_en_free_frag(const struct mlx4_en_priv *priv,
  83			      struct mlx4_en_rx_ring *ring,
  84			      struct mlx4_en_rx_alloc *frag)
  85{
  86	if (frag->page)
  87		page_pool_put_full_page(ring->pp, frag->page, false);
  88	/* We need to clear all fields, otherwise a change of priv->log_rx_info
  89	 * could lead to see garbage later in frag->page.
  90	 */
  91	memset(frag, 0, sizeof(*frag));
  92}
  93
  94static void mlx4_en_init_rx_desc(const struct mlx4_en_priv *priv,
  95				 struct mlx4_en_rx_ring *ring, int index)
  96{
  97	struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
  98	int possible_frags;
  99	int i;
 100
 101	/* Set size and memtype fields */
 102	for (i = 0; i < priv->num_frags; i++) {
 103		rx_desc->data[i].byte_count =
 104			cpu_to_be32(priv->frag_info[i].frag_size);
 105		rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);
 106	}
 107
 108	/* If the number of used fragments does not fill up the ring stride,
 109	 * remaining (unused) fragments must be padded with null address/size
 110	 * and a special memory key */
 111	possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
 112	for (i = priv->num_frags; i < possible_frags; i++) {
 113		rx_desc->data[i].byte_count = 0;
 114		rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
 115		rx_desc->data[i].addr = 0;
 116	}
 117}
 118
 119static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
 120				   struct mlx4_en_rx_ring *ring, int index,
 121				   gfp_t gfp)
 122{
 123	struct mlx4_en_rx_desc *rx_desc = ring->buf +
 124		(index << ring->log_stride);
 125	struct mlx4_en_rx_alloc *frags = ring->rx_info +
 126					(index << priv->log_rx_info);
 127
 128	return mlx4_en_alloc_frags(priv, ring, rx_desc, frags, gfp);
 129}
 130
 131static bool mlx4_en_is_ring_empty(const struct mlx4_en_rx_ring *ring)
 132{
 133	return ring->prod == ring->cons;
 134}
 135
 136static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
 137{
 138	*ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
 139}
 140
 141/* slow path */
 142static void mlx4_en_free_rx_desc(const struct mlx4_en_priv *priv,
 143				 struct mlx4_en_rx_ring *ring,
 144				 int index)
 145{
 146	struct mlx4_en_rx_alloc *frags;
 147	int nr;
 148
 149	frags = ring->rx_info + (index << priv->log_rx_info);
 150	for (nr = 0; nr < priv->num_frags; nr++) {
 151		en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
 152		mlx4_en_free_frag(priv, ring, frags + nr);
 153	}
 154}
 155
 156/* Function not in fast-path */
 157static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
 158{
 159	struct mlx4_en_rx_ring *ring;
 160	int ring_ind;
 161	int buf_ind;
 162	int new_size;
 163
 164	for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
 165		for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
 166			ring = priv->rx_ring[ring_ind];
 167
 168			if (mlx4_en_prepare_rx_desc(priv, ring,
 169						    ring->actual_size,
 170						    GFP_KERNEL)) {
 171				if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
 172					en_err(priv, "Failed to allocate enough rx buffers\n");
 173					return -ENOMEM;
 174				} else {
 175					new_size = rounddown_pow_of_two(ring->actual_size);
 176					en_warn(priv, "Only %d buffers allocated reducing ring size to %d\n",
 177						ring->actual_size, new_size);
 178					goto reduce_rings;
 179				}
 180			}
 181			ring->actual_size++;
 182			ring->prod++;
 183		}
 184	}
 185	return 0;
 186
 187reduce_rings:
 188	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
 189		ring = priv->rx_ring[ring_ind];
 190		while (ring->actual_size > new_size) {
 191			ring->actual_size--;
 192			ring->prod--;
 193			mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
 194		}
 195	}
 196
 197	return 0;
 198}
 199
 200static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
 201				struct mlx4_en_rx_ring *ring)
 202{
 203	int index;
 204
 205	en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
 206	       ring->cons, ring->prod);
 207
 208	/* Unmap and free Rx buffers */
 209	for (index = 0; index < ring->size; index++) {
 210		en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
 211		mlx4_en_free_rx_desc(priv, ring, index);
 212	}
 213	ring->cons = 0;
 214	ring->prod = 0;
 215}
 216
 217void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev)
 218{
 219	int i;
 220	int num_of_eqs;
 221	int num_rx_rings;
 222	struct mlx4_dev *dev = mdev->dev;
 223
 224	mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) {
 225		num_of_eqs = max_t(int, MIN_RX_RINGS,
 226				   min_t(int,
 227					 mlx4_get_eqs_per_port(mdev->dev, i),
 228					 DEF_RX_RINGS));
 229
 230		num_rx_rings = mlx4_low_memory_profile() ? MIN_RX_RINGS :
 231			min_t(int, num_of_eqs, num_online_cpus());
 232		mdev->profile.prof[i].rx_ring_num =
 233			rounddown_pow_of_two(num_rx_rings);
 234	}
 235}
 236
 237int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
 238			   struct mlx4_en_rx_ring **pring,
 239			   u32 size, u16 stride, int node, int queue_index)
 240{
 241	struct mlx4_en_dev *mdev = priv->mdev;
 242	struct page_pool_params pp = {};
 243	struct mlx4_en_rx_ring *ring;
 244	int err = -ENOMEM;
 245	int tmp;
 246
 247	ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, node);
 248	if (!ring) {
 249		en_err(priv, "Failed to allocate RX ring structure\n");
 250		return -ENOMEM;
 251	}
 252
 253	ring->prod = 0;
 254	ring->cons = 0;
 255	ring->size = size;
 256	ring->size_mask = size - 1;
 257	ring->stride = stride;
 258	ring->log_stride = ffs(ring->stride) - 1;
 259	ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
 260
 261	pp.flags = PP_FLAG_DMA_MAP;
 262	pp.pool_size = size * DIV_ROUND_UP(priv->rx_skb_size, PAGE_SIZE);
 263	pp.nid = node;
 264	pp.napi = &priv->rx_cq[queue_index]->napi;
 265	pp.netdev = priv->dev;
 266	pp.dev = &mdev->dev->persist->pdev->dev;
 267	pp.dma_dir = priv->dma_dir;
 268
 269	ring->pp = page_pool_create(&pp);
 270	if (IS_ERR(ring->pp)) {
 271		err = PTR_ERR(ring->pp);
 272		goto err_ring;
 273	}
 274
 275	if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0)
 276		goto err_pp;
 277
 278	err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_POOL,
 279					 ring->pp);
 280	if (err)
 281		goto err_xdp_info;
 282
 283	tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
 284					sizeof(struct mlx4_en_rx_alloc));
 285	ring->rx_info = kvzalloc_node(tmp, GFP_KERNEL, node);
 286	if (!ring->rx_info) {
 287		err = -ENOMEM;
 288		goto err_xdp_info;
 289	}
 290
 291	en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
 292		 ring->rx_info, tmp);
 293
 294	/* Allocate HW buffers on provided NUMA node */
 295	set_dev_node(&mdev->dev->persist->pdev->dev, node);
 296	err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
 297	set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
 298	if (err)
 299		goto err_info;
 300
 301	ring->buf = ring->wqres.buf.direct.buf;
 302
 303	ring->hwtstamp_rx_filter = priv->hwtstamp_config.rx_filter;
 304
 305	*pring = ring;
 306	return 0;
 307
 308err_info:
 309	kvfree(ring->rx_info);
 310	ring->rx_info = NULL;
 311err_xdp_info:
 312	xdp_rxq_info_unreg(&ring->xdp_rxq);
 313err_pp:
 314	page_pool_destroy(ring->pp);
 315err_ring:
 316	kfree(ring);
 317	*pring = NULL;
 318
 319	return err;
 320}
 321
 322int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
 323{
 324	struct mlx4_en_rx_ring *ring;
 325	int i;
 326	int ring_ind;
 327	int err;
 328	int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
 329					DS_SIZE * priv->num_frags);
 330
 331	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
 332		ring = priv->rx_ring[ring_ind];
 333
 334		ring->prod = 0;
 335		ring->cons = 0;
 336		ring->actual_size = 0;
 337		ring->cqn = priv->rx_cq[ring_ind]->mcq.cqn;
 338
 339		ring->stride = stride;
 340		if (ring->stride <= TXBB_SIZE) {
 341			/* Stamp first unused send wqe */
 342			__be32 *ptr = (__be32 *)ring->buf;
 343			__be32 stamp = cpu_to_be32(1 << STAMP_SHIFT);
 344			*ptr = stamp;
 345			/* Move pointer to start of rx section */
 346			ring->buf += TXBB_SIZE;
 347		}
 348
 349		ring->log_stride = ffs(ring->stride) - 1;
 350		ring->buf_size = ring->size * ring->stride;
 351
 352		memset(ring->buf, 0, ring->buf_size);
 353		mlx4_en_update_rx_prod_db(ring);
 354
 355		/* Initialize all descriptors */
 356		for (i = 0; i < ring->size; i++)
 357			mlx4_en_init_rx_desc(priv, ring, i);
 358	}
 359	err = mlx4_en_fill_rx_buffers(priv);
 360	if (err)
 361		goto err_buffers;
 362
 363	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
 364		ring = priv->rx_ring[ring_ind];
 365
 366		ring->size_mask = ring->actual_size - 1;
 367		mlx4_en_update_rx_prod_db(ring);
 368	}
 369
 370	return 0;
 371
 372err_buffers:
 373	for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
 374		mlx4_en_free_rx_buf(priv, priv->rx_ring[ring_ind]);
 375
 376	ring_ind = priv->rx_ring_num - 1;
 377	while (ring_ind >= 0) {
 378		if (priv->rx_ring[ring_ind]->stride <= TXBB_SIZE)
 379			priv->rx_ring[ring_ind]->buf -= TXBB_SIZE;
 380		ring_ind--;
 381	}
 382	return err;
 383}
 384
 385/* We recover from out of memory by scheduling our napi poll
 386 * function (mlx4_en_process_cq), which tries to allocate
 387 * all missing RX buffers (call to mlx4_en_refill_rx_buffers).
 388 */
 389void mlx4_en_recover_from_oom(struct mlx4_en_priv *priv)
 390{
 391	int ring;
 392
 393	if (!priv->port_up)
 394		return;
 395
 396	for (ring = 0; ring < priv->rx_ring_num; ring++) {
 397		if (mlx4_en_is_ring_empty(priv->rx_ring[ring])) {
 398			local_bh_disable();
 399			napi_schedule(&priv->rx_cq[ring]->napi);
 400			local_bh_enable();
 401		}
 402	}
 403}
 404
 405void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
 406			     struct mlx4_en_rx_ring **pring,
 407			     u32 size, u16 stride)
 408{
 409	struct mlx4_en_dev *mdev = priv->mdev;
 410	struct mlx4_en_rx_ring *ring = *pring;
 411	struct bpf_prog *old_prog;
 412
 413	old_prog = rcu_dereference_protected(
 414					ring->xdp_prog,
 415					lockdep_is_held(&mdev->state_lock));
 416	if (old_prog)
 417		bpf_prog_put(old_prog);
 418	xdp_rxq_info_unreg(&ring->xdp_rxq);
 419	mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
 420	kvfree(ring->rx_info);
 421	page_pool_destroy(ring->pp);
 422	ring->rx_info = NULL;
 423	kfree(ring);
 424	*pring = NULL;
 425}
 426
 427void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
 428				struct mlx4_en_rx_ring *ring)
 429{
 430	mlx4_en_free_rx_buf(priv, ring);
 431	if (ring->stride <= TXBB_SIZE)
 432		ring->buf -= TXBB_SIZE;
 433}
 434
 435
 436static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
 437				    struct mlx4_en_rx_alloc *frags,
 438				    struct sk_buff *skb,
 439				    int length)
 440{
 441	const struct mlx4_en_frag_info *frag_info = priv->frag_info;
 442	unsigned int truesize = 0;
 443	bool release = true;
 444	int nr, frag_size;
 445	struct page *page;
 446	dma_addr_t dma;
 447
 448	/* Collect used fragments while replacing them in the HW descriptors */
 449	for (nr = 0;; frags++) {
 450		frag_size = min_t(int, length, frag_info->frag_size);
 451
 452		page = frags->page;
 453		if (unlikely(!page))
 454			goto fail;
 455
 456		dma = page_pool_get_dma_addr(page);
 457		dma_sync_single_range_for_cpu(priv->ddev, dma, frags->page_offset,
 458					      frag_size, priv->dma_dir);
 459
 460		__skb_fill_page_desc(skb, nr, page, frags->page_offset,
 461				     frag_size);
 462
 463		truesize += frag_info->frag_stride;
 464		if (frag_info->frag_stride == PAGE_SIZE / 2) {
 465			struct netmem_desc *desc = pp_page_to_nmdesc(page);
 466
 467			frags->page_offset ^= PAGE_SIZE / 2;
 468			release = page_count(page) != 1 ||
 469				  atomic_long_read(&desc->pp_ref_count) != 1 ||
 470				  page_is_pfmemalloc(page) ||
 471				  page_to_nid(page) != numa_mem_id();
 472		} else if (!priv->rx_headroom) {
 473			/* rx_headroom for non XDP setup is always 0.
 474			 * When XDP is set, the above condition will
 475			 * guarantee page is always released.
 476			 */
 477			u32 sz_align = ALIGN(frag_size, SMP_CACHE_BYTES);
 478
 479			frags->page_offset += sz_align;
 480			release = frags->page_offset + frag_info->frag_size > PAGE_SIZE;
 481		}
 482		if (release) {
 483			frags->page = NULL;
 484		} else {
 485			page_pool_ref_page(page);
 486		}
 487
 488		nr++;
 489		length -= frag_size;
 490		if (!length)
 491			break;
 492		frag_info++;
 493	}
 494	skb->truesize += truesize;
 495	return nr;
 496
 497fail:
 498	while (nr > 0) {
 499		nr--;
 500		__skb_frag_unref(skb_shinfo(skb)->frags + nr, false);
 501	}
 502	return 0;
 503}
 504
 505static void validate_loopback(struct mlx4_en_priv *priv, void *va)
 506{
 507	const unsigned char *data = va + ETH_HLEN;
 508	int i;
 509
 510	for (i = 0; i < MLX4_LOOPBACK_TEST_PAYLOAD; i++) {
 511		if (data[i] != (unsigned char)i)
 512			return;
 513	}
 514	/* Loopback found */
 515	priv->loopback_ok = 1;
 516}
 517
 518static void mlx4_en_refill_rx_buffers(struct mlx4_en_priv *priv,
 519				      struct mlx4_en_rx_ring *ring)
 520{
 521	u32 missing = ring->actual_size - (ring->prod - ring->cons);
 522
 523	/* Try to batch allocations, but not too much. */
 524	if (missing < 8)
 525		return;
 526	do {
 527		if (mlx4_en_prepare_rx_desc(priv, ring,
 528					    ring->prod & ring->size_mask,
 529					    GFP_ATOMIC | __GFP_MEMALLOC))
 530			break;
 531		ring->prod++;
 532	} while (likely(--missing));
 533
 534	mlx4_en_update_rx_prod_db(ring);
 535}
 536
 537/* When hardware doesn't strip the vlan, we need to calculate the checksum
 538 * over it and add it to the hardware's checksum calculation
 539 */
 540static inline __wsum get_fixed_vlan_csum(__wsum hw_checksum,
 541					 struct vlan_hdr *vlanh)
 542{
 543	return csum_add(hw_checksum, *(__wsum *)vlanh);
 544}
 545
 546/* Although the stack expects checksum which doesn't include the pseudo
 547 * header, the HW adds it. To address that, we are subtracting the pseudo
 548 * header checksum from the checksum value provided by the HW.
 549 */
 550static int get_fixed_ipv4_csum(__wsum hw_checksum, struct sk_buff *skb,
 551			       struct iphdr *iph)
 552{
 553	__u16 length_for_csum = 0;
 554	__wsum csum_pseudo_header = 0;
 555	__u8 ipproto = iph->protocol;
 556
 557	if (unlikely(ipproto == IPPROTO_SCTP))
 558		return -1;
 559
 560	length_for_csum = (be16_to_cpu(iph->tot_len) - (iph->ihl << 2));
 561	csum_pseudo_header = csum_tcpudp_nofold(iph->saddr, iph->daddr,
 562						length_for_csum, ipproto, 0);
 563	skb->csum = csum_sub(hw_checksum, csum_pseudo_header);
 564	return 0;
 565}
 566
 567#if IS_ENABLED(CONFIG_IPV6)
 568/* In IPv6 packets, hw_checksum lacks 6 bytes from IPv6 header:
 569 * 4 first bytes : priority, version, flow_lbl
 570 * and 2 additional bytes : nexthdr, hop_limit.
 571 */
 572static int get_fixed_ipv6_csum(__wsum hw_checksum, struct sk_buff *skb,
 573			       struct ipv6hdr *ipv6h)
 574{
 575	__u8 nexthdr = ipv6h->nexthdr;
 576	__wsum temp;
 577
 578	if (unlikely(nexthdr == IPPROTO_FRAGMENT ||
 579		     nexthdr == IPPROTO_HOPOPTS ||
 580		     nexthdr == IPPROTO_SCTP))
 581		return -1;
 582
 583	/* priority, version, flow_lbl */
 584	temp = csum_add(hw_checksum, *(__wsum *)ipv6h);
 585	/* nexthdr and hop_limit */
 586	skb->csum = csum_add(temp, (__force __wsum)*(__be16 *)&ipv6h->nexthdr);
 587	return 0;
 588}
 589#endif
 590
 591#define short_frame(size) ((size) <= ETH_ZLEN + ETH_FCS_LEN)
 592
 593/* We reach this function only after checking that any of
 594 * the (IPv4 | IPv6) bits are set in cqe->status.
 595 */
 596static int check_csum(struct mlx4_cqe *cqe, struct sk_buff *skb, void *va,
 597		      netdev_features_t dev_features)
 598{
 599	__wsum hw_checksum = 0;
 600	void *hdr;
 601
 602	/* CQE csum doesn't cover padding octets in short ethernet
 603	 * frames. And the pad field is appended prior to calculating
 604	 * and appending the FCS field.
 605	 *
 606	 * Detecting these padded frames requires to verify and parse
 607	 * IP headers, so we simply force all those small frames to skip
 608	 * checksum complete.
 609	 */
 610	if (short_frame(skb->len))
 611		return -EINVAL;
 612
 613	hdr = (u8 *)va + sizeof(struct ethhdr);
 614	hw_checksum = csum_unfold((__force __sum16)cqe->checksum);
 615
 616	if (cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_CVLAN_PRESENT_MASK) &&
 617	    !(dev_features & NETIF_F_HW_VLAN_CTAG_RX)) {
 618		hw_checksum = get_fixed_vlan_csum(hw_checksum, hdr);
 619		hdr += sizeof(struct vlan_hdr);
 620	}
 621
 622#if IS_ENABLED(CONFIG_IPV6)
 623	if (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPV6))
 624		return get_fixed_ipv6_csum(hw_checksum, skb, hdr);
 625#endif
 626	return get_fixed_ipv4_csum(hw_checksum, skb, hdr);
 627}
 628
 629#if IS_ENABLED(CONFIG_IPV6)
 630#define MLX4_CQE_STATUS_IP_ANY (MLX4_CQE_STATUS_IPV4 | MLX4_CQE_STATUS_IPV6)
 631#else
 632#define MLX4_CQE_STATUS_IP_ANY (MLX4_CQE_STATUS_IPV4)
 633#endif
 634
 635struct mlx4_en_xdp_buff {
 636	struct xdp_buff xdp;
 637	struct mlx4_cqe *cqe;
 638	struct mlx4_en_dev *mdev;
 639	struct mlx4_en_rx_ring *ring;
 640	struct net_device *dev;
 641};
 642
 643int mlx4_en_xdp_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp)
 644{
 645	struct mlx4_en_xdp_buff *_ctx = (void *)ctx;
 646
 647	if (unlikely(_ctx->ring->hwtstamp_rx_filter != HWTSTAMP_FILTER_ALL))
 648		return -ENODATA;
 649
 650	*timestamp = mlx4_en_get_hwtstamp(_ctx->mdev,
 651					  mlx4_en_get_cqe_ts(_ctx->cqe));
 652	return 0;
 653}
 654
 655int mlx4_en_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash,
 656			enum xdp_rss_hash_type *rss_type)
 657{
 658	struct mlx4_en_xdp_buff *_ctx = (void *)ctx;
 659	struct mlx4_cqe *cqe = _ctx->cqe;
 660	enum xdp_rss_hash_type xht = 0;
 661	__be16 status;
 662
 663	if (unlikely(!(_ctx->dev->features & NETIF_F_RXHASH)))
 664		return -ENODATA;
 665
 666	*hash = be32_to_cpu(cqe->immed_rss_invalid);
 667	status = cqe->status;
 668	if (status & cpu_to_be16(MLX4_CQE_STATUS_TCP))
 669		xht = XDP_RSS_L4_TCP;
 670	if (status & cpu_to_be16(MLX4_CQE_STATUS_UDP))
 671		xht = XDP_RSS_L4_UDP;
 672	if (status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 | MLX4_CQE_STATUS_IPV4F))
 673		xht |= XDP_RSS_L3_IPV4;
 674	if (status & cpu_to_be16(MLX4_CQE_STATUS_IPV6)) {
 675		xht |= XDP_RSS_L3_IPV6;
 676		if (cqe->ipv6_ext_mask)
 677			xht |= XDP_RSS_L3_DYNHDR;
 678	}
 679	*rss_type = xht;
 680
 681	return 0;
 682}
 683
 684int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
 685{
 686	struct mlx4_en_priv *priv = netdev_priv(dev);
 687	struct mlx4_en_xdp_buff mxbuf = {};
 688	int factor = priv->cqe_factor;
 689	struct mlx4_en_rx_ring *ring;
 690	struct bpf_prog *xdp_prog;
 691	int cq_ring = cq->ring;
 692	bool doorbell_pending;
 693	bool xdp_redir_flush;
 694	struct mlx4_cqe *cqe;
 695	int polled = 0;
 696	int index;
 697
 698	if (unlikely(!priv->port_up || budget <= 0))
 699		return 0;
 700
 701	ring = priv->rx_ring[cq_ring];
 702
 703	xdp_prog = rcu_dereference_bh(ring->xdp_prog);
 704	xdp_init_buff(&mxbuf.xdp, priv->frag_info[0].frag_stride, &ring->xdp_rxq);
 705	doorbell_pending = false;
 706	xdp_redir_flush = false;
 707
 708	/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
 709	 * descriptor offset can be deduced from the CQE index instead of
 710	 * reading 'cqe->index' */
 711	index = cq->mcq.cons_index & ring->size_mask;
 712	cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
 713
 714	/* Process all completed CQEs */
 715	while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
 716		    cq->mcq.cons_index & cq->size)) {
 717		struct mlx4_en_rx_alloc *frags;
 718		enum pkt_hash_types hash_type;
 719		struct sk_buff *skb;
 720		unsigned int length;
 721		int ip_summed;
 722		void *va;
 723		int nr;
 724
 725		frags = ring->rx_info + (index << priv->log_rx_info);
 726		va = page_address(frags[0].page) + frags[0].page_offset;
 727		net_prefetchw(va);
 728		/*
 729		 * make sure we read the CQE after we read the ownership bit
 730		 */
 731		dma_rmb();
 732
 733		/* Drop packet on bad receive or bad checksum */
 734		if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
 735						MLX4_CQE_OPCODE_ERROR)) {
 736			en_err(priv, "CQE completed in error - vendor syndrome:%d syndrome:%d\n",
 737			       ((struct mlx4_err_cqe *)cqe)->vendor_err_syndrome,
 738			       ((struct mlx4_err_cqe *)cqe)->syndrome);
 739			goto next;
 740		}
 741		if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
 742			en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
 743			goto next;
 744		}
 745
 746		/* Check if we need to drop the packet if SRIOV is not enabled
 747		 * and not performing the selftest or flb disabled
 748		 */
 749		if (priv->flags & MLX4_EN_FLAG_RX_FILTER_NEEDED) {
 750			const struct ethhdr *ethh = va;
 751			dma_addr_t dma;
 752			/* Get pointer to first fragment since we haven't
 753			 * skb yet and cast it to ethhdr struct
 754			 */
 755			dma = page_pool_get_dma_addr(frags[0].page);
 756			dma += frags[0].page_offset;
 757			dma_sync_single_for_cpu(priv->ddev, dma, sizeof(*ethh),
 758						DMA_FROM_DEVICE);
 759
 760			if (is_multicast_ether_addr(ethh->h_dest)) {
 761				struct mlx4_mac_entry *entry;
 762				struct hlist_head *bucket;
 763				unsigned int mac_hash;
 764
 765				/* Drop the packet, since HW loopback-ed it */
 766				mac_hash = ethh->h_source[MLX4_EN_MAC_HASH_IDX];
 767				bucket = &priv->mac_hash[mac_hash];
 768				hlist_for_each_entry_rcu_bh(entry, bucket, hlist) {
 769					if (ether_addr_equal_64bits(entry->mac,
 770								    ethh->h_source))
 771						goto next;
 772				}
 773			}
 774		}
 775
 776		if (unlikely(priv->validate_loopback)) {
 777			validate_loopback(priv, va);
 778			goto next;
 779		}
 780
 781		/*
 782		 * Packet is OK - process it.
 783		 */
 784		length = be32_to_cpu(cqe->byte_cnt);
 785		length -= ring->fcs_del;
 786
 787		/* A bpf program gets first chance to drop the packet. It may
 788		 * read bytes but not past the end of the frag.
 789		 */
 790		if (xdp_prog) {
 791			dma_addr_t dma;
 792			void *orig_data;
 793			u32 act;
 794
 795			dma = page_pool_get_dma_addr(frags[0].page);
 796			dma += frags[0].page_offset;
 797			dma_sync_single_for_cpu(priv->ddev, dma,
 798						priv->frag_info[0].frag_size,
 799						DMA_FROM_DEVICE);
 800
 801			xdp_prepare_buff(&mxbuf.xdp, va - frags[0].page_offset,
 802					 frags[0].page_offset, length, true);
 803			orig_data = mxbuf.xdp.data;
 804			mxbuf.cqe = cqe;
 805			mxbuf.mdev = priv->mdev;
 806			mxbuf.ring = ring;
 807			mxbuf.dev = dev;
 808
 809			act = bpf_prog_run_xdp(xdp_prog, &mxbuf.xdp);
 810
 811			length = mxbuf.xdp.data_end - mxbuf.xdp.data;
 812			if (mxbuf.xdp.data != orig_data) {
 813				frags[0].page_offset = mxbuf.xdp.data -
 814					mxbuf.xdp.data_hard_start;
 815				va = mxbuf.xdp.data;
 816			}
 817
 818			switch (act) {
 819			case XDP_PASS:
 820				break;
 821			case XDP_REDIRECT:
 822				if (likely(!xdp_do_redirect(dev, &mxbuf.xdp, xdp_prog))) {
 823					ring->xdp_redirect++;
 824					xdp_redir_flush = true;
 825					frags[0].page = NULL;
 826					goto next;
 827				}
 828				ring->xdp_redirect_fail++;
 829				trace_xdp_exception(dev, xdp_prog, act);
 830				goto xdp_drop_no_cnt;
 831			case XDP_TX:
 832				if (likely(!mlx4_en_xmit_frame(ring, frags, priv,
 833							length, cq_ring,
 834							&doorbell_pending))) {
 835					frags[0].page = NULL;
 836					goto next;
 837				}
 838				trace_xdp_exception(dev, xdp_prog, act);
 839				goto xdp_drop_no_cnt; /* Drop on xmit failure */
 840			default:
 841				bpf_warn_invalid_xdp_action(dev, xdp_prog, act);
 842				fallthrough;
 843			case XDP_ABORTED:
 844				trace_xdp_exception(dev, xdp_prog, act);
 845				fallthrough;
 846			case XDP_DROP:
 847				ring->xdp_drop++;
 848xdp_drop_no_cnt:
 849				goto next;
 850			}
 851		}
 852
 853		ring->bytes += length;
 854		ring->packets++;
 855
 856		skb = napi_get_frags(&cq->napi);
 857		if (unlikely(!skb))
 858			goto next;
 859		skb_mark_for_recycle(skb);
 860
 861		if (unlikely(ring->hwtstamp_rx_filter == HWTSTAMP_FILTER_ALL)) {
 862			u64 timestamp = mlx4_en_get_cqe_ts(cqe);
 863
 864			mlx4_en_fill_hwtstamps(priv->mdev, skb_hwtstamps(skb),
 865					       timestamp);
 866		}
 867		skb_record_rx_queue(skb, cq_ring);
 868
 869		if (likely(dev->features & NETIF_F_RXCSUM)) {
 870			/* TODO: For IP non TCP/UDP packets when csum complete is
 871			 * not an option (not supported or any other reason) we can
 872			 * actually check cqe IPOK status bit and report
 873			 * CHECKSUM_UNNECESSARY rather than CHECKSUM_NONE
 874			 */
 875			if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_TCP |
 876						       MLX4_CQE_STATUS_UDP)) &&
 877			    (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
 878			    cqe->checksum == cpu_to_be16(0xffff)) {
 879				bool l2_tunnel;
 880
 881				l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
 882					(cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
 883				ip_summed = CHECKSUM_UNNECESSARY;
 884				hash_type = PKT_HASH_TYPE_L4;
 885				if (l2_tunnel)
 886					skb->csum_level = 1;
 887				ring->csum_ok++;
 888			} else {
 889				if (!(priv->flags & MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP &&
 890				      (cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IP_ANY))))
 891					goto csum_none;
 892				if (check_csum(cqe, skb, va, dev->features))
 893					goto csum_none;
 894				ip_summed = CHECKSUM_COMPLETE;
 895				hash_type = PKT_HASH_TYPE_L3;
 896				ring->csum_complete++;
 897			}
 898		} else {
 899csum_none:
 900			ip_summed = CHECKSUM_NONE;
 901			hash_type = PKT_HASH_TYPE_L3;
 902			ring->csum_none++;
 903		}
 904		skb->ip_summed = ip_summed;
 905		if (dev->features & NETIF_F_RXHASH)
 906			skb_set_hash(skb,
 907				     be32_to_cpu(cqe->immed_rss_invalid),
 908				     hash_type);
 909
 910		if ((cqe->vlan_my_qpn &
 911		     cpu_to_be32(MLX4_CQE_CVLAN_PRESENT_MASK)) &&
 912		    (dev->features & NETIF_F_HW_VLAN_CTAG_RX))
 913			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 914					       be16_to_cpu(cqe->sl_vid));
 915		else if ((cqe->vlan_my_qpn &
 916			  cpu_to_be32(MLX4_CQE_SVLAN_PRESENT_MASK)) &&
 917			 (dev->features & NETIF_F_HW_VLAN_STAG_RX))
 918			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD),
 919					       be16_to_cpu(cqe->sl_vid));
 920
 921		nr = mlx4_en_complete_rx_desc(priv, frags, skb, length);
 922		if (likely(nr)) {
 923			skb_shinfo(skb)->nr_frags = nr;
 924			skb->len = length;
 925			skb->data_len = length;
 926			napi_gro_frags(&cq->napi);
 927		} else {
 928			__vlan_hwaccel_clear_tag(skb);
 929			skb_clear_hash(skb);
 930		}
 931next:
 932		++cq->mcq.cons_index;
 933		index = (cq->mcq.cons_index) & ring->size_mask;
 934		cqe = mlx4_en_get_cqe(cq->buf, index, priv->cqe_size) + factor;
 935		if (unlikely(++polled == budget))
 936			break;
 937	}
 938
 939	if (xdp_redir_flush)
 940		xdp_do_flush();
 941
 942	if (likely(polled)) {
 943		if (doorbell_pending) {
 944			priv->tx_cq[TX_XDP][cq_ring]->xdp_busy = true;
 945			mlx4_en_xmit_doorbell(priv->tx_ring[TX_XDP][cq_ring]);
 946		}
 947
 948		mlx4_cq_set_ci(&cq->mcq);
 949		wmb(); /* ensure HW sees CQ consumer before we post new buffers */
 950		ring->cons = cq->mcq.cons_index;
 951	}
 952
 953	mlx4_en_refill_rx_buffers(priv, ring);
 954
 955	return polled;
 956}
 957
 958
 959void mlx4_en_rx_irq(struct mlx4_cq *mcq)
 960{
 961	struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
 962	struct mlx4_en_priv *priv = netdev_priv(cq->dev);
 963
 964	if (likely(priv->port_up))
 965		napi_schedule_irqoff(&cq->napi);
 966	else
 967		mlx4_en_arm_cq(priv, cq);
 968}
 969
 970/* Rx CQ polling - called by NAPI */
 971int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
 972{
 973	struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
 974	struct net_device *dev = cq->dev;
 975	struct mlx4_en_priv *priv = netdev_priv(dev);
 976	struct mlx4_en_cq *xdp_tx_cq = NULL;
 977	bool clean_complete = true;
 978	int done;
 979
 980	if (!budget)
 981		return 0;
 982
 983	if (priv->tx_ring_num[TX_XDP]) {
 984		xdp_tx_cq = priv->tx_cq[TX_XDP][cq->ring];
 985		if (xdp_tx_cq->xdp_busy) {
 986			clean_complete = mlx4_en_process_tx_cq(dev, xdp_tx_cq,
 987							       budget) < budget;
 988			xdp_tx_cq->xdp_busy = !clean_complete;
 989		}
 990	}
 991
 992	done = mlx4_en_process_rx_cq(dev, cq, budget);
 993
 994	/* If we used up all the quota - we're probably not done yet... */
 995	if (done == budget || !clean_complete) {
 996		int cpu_curr;
 997
 998		/* in case we got here because of !clean_complete */
 999		done = budget;
1000
1001		cpu_curr = smp_processor_id();
1002
1003		if (likely(cpumask_test_cpu(cpu_curr, cq->aff_mask)))
1004			return budget;
1005
1006		/* Current cpu is not according to smp_irq_affinity -
1007		 * probably affinity changed. Need to stop this NAPI
1008		 * poll, and restart it on the right CPU.
1009		 * Try to avoid returning a too small value (like 0),
1010		 * to not fool net_rx_action() and its netdev_budget
1011		 */
1012		if (done)
1013			done--;
1014	}
1015	/* Done for now */
1016	if (likely(napi_complete_done(napi, done)))
1017		mlx4_en_arm_cq(priv, cq);
1018	return done;
1019}
1020
1021void mlx4_en_calc_rx_buf(struct net_device *dev)
1022{
1023	struct mlx4_en_priv *priv = netdev_priv(dev);
1024	int eff_mtu = MLX4_EN_EFF_MTU(dev->mtu);
1025	int i = 0;
1026
1027	/* bpf requires buffers to be set up as 1 packet per page.
1028	 * This only works when num_frags == 1.
1029	 */
1030	if (priv->tx_ring_num[TX_XDP]) {
1031		priv->frag_info[0].frag_size = eff_mtu;
1032		/* This will gain efficient xdp frame recycling at the
1033		 * expense of more costly truesize accounting
1034		 */
1035		priv->frag_info[0].frag_stride = PAGE_SIZE;
1036		priv->dma_dir = DMA_BIDIRECTIONAL;
1037		priv->rx_headroom = XDP_PACKET_HEADROOM;
1038		i = 1;
1039	} else {
1040		int frag_size_max = 2048, buf_size = 0;
1041
1042		/* should not happen, right ? */
1043		if (eff_mtu > PAGE_SIZE + (MLX4_EN_MAX_RX_FRAGS - 1) * 2048)
1044			frag_size_max = PAGE_SIZE;
1045
1046		while (buf_size < eff_mtu) {
1047			int frag_stride, frag_size = eff_mtu - buf_size;
1048			int pad, nb;
1049
1050			if (i < MLX4_EN_MAX_RX_FRAGS - 1)
1051				frag_size = min(frag_size, frag_size_max);
1052
1053			priv->frag_info[i].frag_size = frag_size;
1054			frag_stride = ALIGN(frag_size, SMP_CACHE_BYTES);
1055			/* We can only pack 2 1536-bytes frames in on 4K page
1056			 * Therefore, each frame would consume more bytes (truesize)
1057			 */
1058			nb = PAGE_SIZE / frag_stride;
1059			pad = (PAGE_SIZE - nb * frag_stride) / nb;
1060			pad &= ~(SMP_CACHE_BYTES - 1);
1061			priv->frag_info[i].frag_stride = frag_stride + pad;
1062
1063			buf_size += frag_size;
1064			i++;
1065		}
1066		priv->dma_dir = DMA_FROM_DEVICE;
1067		priv->rx_headroom = 0;
1068	}
1069
1070	priv->num_frags = i;
1071	priv->rx_skb_size = eff_mtu;
1072	priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct mlx4_en_rx_alloc));
1073
1074	en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d num_frags:%d):\n",
1075	       eff_mtu, priv->num_frags);
1076	for (i = 0; i < priv->num_frags; i++) {
1077		en_dbg(DRV,
1078		       priv,
1079		       "  frag:%d - size:%d stride:%d\n",
1080		       i,
1081		       priv->frag_info[i].frag_size,
1082		       priv->frag_info[i].frag_stride);
1083	}
1084}
1085
1086/* RSS related functions */
1087
1088static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
1089				 struct mlx4_en_rx_ring *ring,
1090				 enum mlx4_qp_state *state,
1091				 struct mlx4_qp *qp)
1092{
1093	struct mlx4_en_dev *mdev = priv->mdev;
1094	struct mlx4_qp_context *context;
1095	int err = 0;
1096
1097	context = kzalloc(sizeof(*context), GFP_KERNEL);
1098	if (!context)
1099		return -ENOMEM;
1100
1101	err = mlx4_qp_alloc(mdev->dev, qpn, qp);
1102	if (err) {
1103		en_err(priv, "Failed to allocate qp #%x\n", qpn);
1104		goto out;
1105	}
1106	qp->event = mlx4_en_sqp_event;
1107
1108	mlx4_en_fill_qp_context(priv, ring->actual_size, ring->stride, 0, 0,
1109				qpn, ring->cqn, -1, context);
1110	context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
1111
1112	/* Cancel FCS removal if FW allows */
1113	if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) {
1114		context->param3 |= cpu_to_be32(1 << 29);
1115		if (priv->dev->features & NETIF_F_RXFCS)
1116			ring->fcs_del = 0;
1117		else
1118			ring->fcs_del = ETH_FCS_LEN;
1119	} else
1120		ring->fcs_del = 0;
1121
1122	err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
1123	if (err) {
1124		mlx4_qp_remove(mdev->dev, qp);
1125		mlx4_qp_free(mdev->dev, qp);
1126	}
1127	mlx4_en_update_rx_prod_db(ring);
1128out:
1129	kfree(context);
1130	return err;
1131}
1132
1133int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
1134{
1135	int err;
1136	u32 qpn;
1137
1138	err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,
1139				    MLX4_RESERVE_A0_QP,
1140				    MLX4_RES_USAGE_DRIVER);
1141	if (err) {
1142		en_err(priv, "Failed reserving drop qpn\n");
1143		return err;
1144	}
1145	err = mlx4_qp_alloc(priv->mdev->dev, qpn, &priv->drop_qp);
1146	if (err) {
1147		en_err(priv, "Failed allocating drop qp\n");
1148		mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
1149		return err;
1150	}
1151
1152	return 0;
1153}
1154
1155void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv)
1156{
1157	u32 qpn;
1158
1159	qpn = priv->drop_qp.qpn;
1160	mlx4_qp_remove(priv->mdev->dev, &priv->drop_qp);
1161	mlx4_qp_free(priv->mdev->dev, &priv->drop_qp);
1162	mlx4_qp_release_range(priv->mdev->dev, qpn, 1);
1163}
1164
1165/* Allocate rx qp's and configure them according to rss map */
1166int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
1167{
1168	struct mlx4_en_dev *mdev = priv->mdev;
1169	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
1170	struct mlx4_qp_context context;
1171	struct mlx4_rss_context *rss_context;
1172	int rss_rings;
1173	void *ptr;
1174	u8 rss_mask = (MLX4_RSS_IPV4 | MLX4_RSS_TCP_IPV4 | MLX4_RSS_IPV6 |
1175			MLX4_RSS_TCP_IPV6);
1176	int i, qpn;
1177	int err = 0;
1178	int good_qps = 0;
1179	u8 flags;
1180
1181	en_dbg(DRV, priv, "Configuring rss steering\n");
1182
1183	flags = priv->rx_ring_num == 1 ? MLX4_RESERVE_A0_QP : 0;
1184	err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
1185				    priv->rx_ring_num,
1186				    &rss_map->base_qpn, flags,
1187				    MLX4_RES_USAGE_DRIVER);
1188	if (err) {
1189		en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
1190		return err;
1191	}
1192
1193	for (i = 0; i < priv->rx_ring_num; i++) {
1194		qpn = rss_map->base_qpn + i;
1195		err = mlx4_en_config_rss_qp(priv, qpn, priv->rx_ring[i],
1196					    &rss_map->state[i],
1197					    &rss_map->qps[i]);
1198		if (err)
1199			goto rss_err;
1200
1201		++good_qps;
1202	}
1203
1204	if (priv->rx_ring_num == 1) {
1205		rss_map->indir_qp = &rss_map->qps[0];
1206		priv->base_qpn = rss_map->indir_qp->qpn;
1207		en_info(priv, "Optimized Non-RSS steering\n");
1208		return 0;
1209	}
1210
1211	rss_map->indir_qp = kzalloc(sizeof(*rss_map->indir_qp), GFP_KERNEL);
1212	if (!rss_map->indir_qp) {
1213		err = -ENOMEM;
1214		goto rss_err;
1215	}
1216
1217	/* Configure RSS indirection qp */
1218	err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, rss_map->indir_qp);
1219	if (err) {
1220		en_err(priv, "Failed to allocate RSS indirection QP\n");
1221		goto qp_alloc_err;
1222	}
1223
1224	rss_map->indir_qp->event = mlx4_en_sqp_event;
1225	mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
1226				priv->rx_ring[0]->cqn, -1, &context);
1227
1228	if (!priv->prof->rss_rings || priv->prof->rss_rings > priv->rx_ring_num)
1229		rss_rings = priv->rx_ring_num;
1230	else
1231		rss_rings = priv->prof->rss_rings;
1232
1233	ptr = ((void *) &context) + offsetof(struct mlx4_qp_context, pri_path)
1234					+ MLX4_RSS_OFFSET_IN_QPC_PRI_PATH;
1235	rss_context = ptr;
1236	rss_context->base_qpn = cpu_to_be32(ilog2(rss_rings) << 24 |
1237					    (rss_map->base_qpn));
1238	rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
1239	if (priv->mdev->profile.udp_rss) {
1240		rss_mask |=  MLX4_RSS_UDP_IPV4 | MLX4_RSS_UDP_IPV6;
1241		rss_context->base_qpn_udp = rss_context->default_qpn;
1242	}
1243
1244	if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) {
1245		en_info(priv, "Setting RSS context tunnel type to RSS on inner headers\n");
1246		rss_mask |= MLX4_RSS_BY_INNER_HEADERS;
1247	}
1248
1249	rss_context->flags = rss_mask;
1250	rss_context->hash_fn = MLX4_RSS_HASH_TOP;
1251	if (priv->rss_hash_fn == ETH_RSS_HASH_XOR) {
1252		rss_context->hash_fn = MLX4_RSS_HASH_XOR;
1253	} else if (priv->rss_hash_fn == ETH_RSS_HASH_TOP) {
1254		rss_context->hash_fn = MLX4_RSS_HASH_TOP;
1255		memcpy(rss_context->rss_key, priv->rss_key,
1256		       MLX4_EN_RSS_KEY_SIZE);
1257	} else {
1258		en_err(priv, "Unknown RSS hash function requested\n");
1259		err = -EINVAL;
1260		goto indir_err;
1261	}
1262
1263	err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
1264			       rss_map->indir_qp, &rss_map->indir_state);
1265	if (err)
1266		goto indir_err;
1267
1268	return 0;
1269
1270indir_err:
1271	mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
1272		       MLX4_QP_STATE_RST, NULL, 0, 0, rss_map->indir_qp);
1273	mlx4_qp_remove(mdev->dev, rss_map->indir_qp);
1274	mlx4_qp_free(mdev->dev, rss_map->indir_qp);
1275qp_alloc_err:
1276	kfree(rss_map->indir_qp);
1277	rss_map->indir_qp = NULL;
1278rss_err:
1279	for (i = 0; i < good_qps; i++) {
1280		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
1281			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
1282		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
1283		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
1284	}
1285	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1286	return err;
1287}
1288
1289void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
1290{
1291	struct mlx4_en_dev *mdev = priv->mdev;
1292	struct mlx4_en_rss_map *rss_map = &priv->rss_map;
1293	int i;
1294
1295	if (priv->rx_ring_num > 1) {
1296		mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
1297			       MLX4_QP_STATE_RST, NULL, 0, 0,
1298			       rss_map->indir_qp);
1299		mlx4_qp_remove(mdev->dev, rss_map->indir_qp);
1300		mlx4_qp_free(mdev->dev, rss_map->indir_qp);
1301		kfree(rss_map->indir_qp);
1302		rss_map->indir_qp = NULL;
1303	}
1304
1305	for (i = 0; i < priv->rx_ring_num; i++) {
1306		mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
1307			       MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
1308		mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
1309		mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
1310	}
1311	mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
1312}