rx_common.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/****************************************************************************
   3 * Driver for Solarflare network controllers and boards
   4 * Copyright 2018 Solarflare Communications Inc.
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms of the GNU General Public License version 2 as published
   8 * by the Free Software Foundation, incorporated herein by reference.
   9 */
  10
  11#include "net_driver.h"
  12#include <linux/module.h>
  13#include <linux/iommu.h>
  14#include <net/rps.h>
  15#include "efx.h"
  16#include "nic.h"
  17#include "rx_common.h"
  18
  19/* This is the percentage fill level below which new RX descriptors
  20 * will be added to the RX descriptor ring.
  21 */
  22static unsigned int rx_refill_threshold;
  23module_param(rx_refill_threshold, uint, 0444);
  24MODULE_PARM_DESC(rx_refill_threshold,
  25		 "RX descriptor ring refill threshold (%)");
  26
  27/* RX maximum head room required.
  28 *
  29 * This must be at least 1 to prevent overflow, plus one packet-worth
  30 * to allow pipelined receives.
  31 */
  32#define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS)
  33
  34static void efx_unmap_rx_buffer(struct efx_nic *efx,
  35				struct efx_rx_buffer *rx_buf);
  36
  37/* Check the RX page recycle ring for a page that can be reused. */
  38static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)
  39{
  40	struct efx_nic *efx = rx_queue->efx;
  41	struct efx_rx_page_state *state;
  42	unsigned int index;
  43	struct page *page;
  44
  45	if (unlikely(!rx_queue->page_ring))
  46		return NULL;
  47	index = rx_queue->page_remove & rx_queue->page_ptr_mask;
  48	page = rx_queue->page_ring[index];
  49	if (page == NULL)
  50		return NULL;
  51
  52	rx_queue->page_ring[index] = NULL;
  53	/* page_remove cannot exceed page_add. */
  54	if (rx_queue->page_remove != rx_queue->page_add)
  55		++rx_queue->page_remove;
  56
  57	/* If page_count is 1 then we hold the only reference to this page. */
  58	if (page_count(page) == 1) {
  59		++rx_queue->page_recycle_count;
  60		return page;
  61	} else {
  62		state = page_address(page);
  63		dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
  64			       PAGE_SIZE << efx->rx_buffer_order,
  65			       DMA_FROM_DEVICE);
  66		put_page(page);
  67		++rx_queue->page_recycle_failed;
  68	}
  69
  70	return NULL;
  71}
  72
  73/* Attempt to recycle the page if there is an RX recycle ring; the page can
  74 * only be added if this is the final RX buffer, to prevent pages being used in
  75 * the descriptor ring and appearing in the recycle ring simultaneously.
  76 */
  77static void efx_recycle_rx_page(struct efx_channel *channel,
  78				struct efx_rx_buffer *rx_buf)
  79{
  80	struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
  81	struct efx_nic *efx = rx_queue->efx;
  82	struct page *page = rx_buf->page;
  83	unsigned int index;
  84
  85	/* Only recycle the page after processing the final buffer. */
  86	if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE))
  87		return;
  88
  89	index = rx_queue->page_add & rx_queue->page_ptr_mask;
  90	if (rx_queue->page_ring[index] == NULL) {
  91		unsigned int read_index = rx_queue->page_remove &
  92			rx_queue->page_ptr_mask;
  93
  94		/* The next slot in the recycle ring is available, but
  95		 * increment page_remove if the read pointer currently
  96		 * points here.
  97		 */
  98		if (read_index == index)
  99			++rx_queue->page_remove;
 100		rx_queue->page_ring[index] = page;
 101		++rx_queue->page_add;
 102		return;
 103	}
 104	++rx_queue->page_recycle_full;
 105	efx_unmap_rx_buffer(efx, rx_buf);
 106	put_page(rx_buf->page);
 107}
 108
 109/* Recycle the pages that are used by buffers that have just been received. */
 110void efx_siena_recycle_rx_pages(struct efx_channel *channel,
 111				struct efx_rx_buffer *rx_buf,
 112				unsigned int n_frags)
 113{
 114	struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
 115
 116	if (unlikely(!rx_queue->page_ring))
 117		return;
 118
 119	do {
 120		efx_recycle_rx_page(channel, rx_buf);
 121		rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
 122	} while (--n_frags);
 123}
 124
 125void efx_siena_discard_rx_packet(struct efx_channel *channel,
 126				 struct efx_rx_buffer *rx_buf,
 127				 unsigned int n_frags)
 128{
 129	struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
 130
 131	efx_siena_recycle_rx_pages(channel, rx_buf, n_frags);
 132
 133	efx_siena_free_rx_buffers(rx_queue, rx_buf, n_frags);
 134}
 135
 136static void efx_init_rx_recycle_ring(struct efx_rx_queue *rx_queue)
 137{
 138	unsigned int bufs_in_recycle_ring, page_ring_size;
 139	struct efx_nic *efx = rx_queue->efx;
 140
 141	bufs_in_recycle_ring = efx_rx_recycle_ring_size(efx);
 142	page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring /
 143					    efx->rx_bufs_per_page);
 144	rx_queue->page_ring = kcalloc(page_ring_size,
 145				      sizeof(*rx_queue->page_ring), GFP_KERNEL);
 146	if (!rx_queue->page_ring)
 147		rx_queue->page_ptr_mask = 0;
 148	else
 149		rx_queue->page_ptr_mask = page_ring_size - 1;
 150}
 151
 152static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue)
 153{
 154	struct efx_nic *efx = rx_queue->efx;
 155	int i;
 156
 157	if (unlikely(!rx_queue->page_ring))
 158		return;
 159
 160	/* Unmap and release the pages in the recycle ring. Remove the ring. */
 161	for (i = 0; i <= rx_queue->page_ptr_mask; i++) {
 162		struct page *page = rx_queue->page_ring[i];
 163		struct efx_rx_page_state *state;
 164
 165		if (page == NULL)
 166			continue;
 167
 168		state = page_address(page);
 169		dma_unmap_page(&efx->pci_dev->dev, state->dma_addr,
 170			       PAGE_SIZE << efx->rx_buffer_order,
 171			       DMA_FROM_DEVICE);
 172		put_page(page);
 173	}
 174	kfree(rx_queue->page_ring);
 175	rx_queue->page_ring = NULL;
 176}
 177
 178static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,
 179			       struct efx_rx_buffer *rx_buf)
 180{
 181	/* Release the page reference we hold for the buffer. */
 182	if (rx_buf->page)
 183		put_page(rx_buf->page);
 184
 185	/* If this is the last buffer in a page, unmap and free it. */
 186	if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) {
 187		efx_unmap_rx_buffer(rx_queue->efx, rx_buf);
 188		efx_siena_free_rx_buffers(rx_queue, rx_buf, 1);
 189	}
 190	rx_buf->page = NULL;
 191}
 192
 193int efx_siena_probe_rx_queue(struct efx_rx_queue *rx_queue)
 194{
 195	struct efx_nic *efx = rx_queue->efx;
 196	unsigned int entries;
 197	int rc;
 198
 199	/* Create the smallest power-of-two aligned ring */
 200	entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE);
 201	EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
 202	rx_queue->ptr_mask = entries - 1;
 203
 204	netif_dbg(efx, probe, efx->net_dev,
 205		  "creating RX queue %d size %#x mask %#x\n",
 206		  efx_rx_queue_index(rx_queue), efx->rxq_entries,
 207		  rx_queue->ptr_mask);
 208
 209	/* Allocate RX buffers */
 210	rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),
 211				   GFP_KERNEL);
 212	if (!rx_queue->buffer)
 213		return -ENOMEM;
 214
 215	rc = efx_nic_probe_rx(rx_queue);
 216	if (rc) {
 217		kfree(rx_queue->buffer);
 218		rx_queue->buffer = NULL;
 219	}
 220
 221	return rc;
 222}
 223
 224void efx_siena_init_rx_queue(struct efx_rx_queue *rx_queue)
 225{
 226	unsigned int max_fill, trigger, max_trigger;
 227	struct efx_nic *efx = rx_queue->efx;
 228	int rc = 0;
 229
 230	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 231		  "initialising RX queue %d\n", efx_rx_queue_index(rx_queue));
 232
 233	/* Initialise ptr fields */
 234	rx_queue->added_count = 0;
 235	rx_queue->notified_count = 0;
 236	rx_queue->removed_count = 0;
 237	rx_queue->min_fill = -1U;
 238	efx_init_rx_recycle_ring(rx_queue);
 239
 240	rx_queue->page_remove = 0;
 241	rx_queue->page_add = rx_queue->page_ptr_mask + 1;
 242	rx_queue->page_recycle_count = 0;
 243	rx_queue->page_recycle_failed = 0;
 244	rx_queue->page_recycle_full = 0;
 245
 246	/* Initialise limit fields */
 247	max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;
 248	max_trigger =
 249		max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page;
 250	if (rx_refill_threshold != 0) {
 251		trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;
 252		if (trigger > max_trigger)
 253			trigger = max_trigger;
 254	} else {
 255		trigger = max_trigger;
 256	}
 257
 258	rx_queue->max_fill = max_fill;
 259	rx_queue->fast_fill_trigger = trigger;
 260	rx_queue->refill_enabled = true;
 261
 262	/* Initialise XDP queue information */
 263	rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev,
 264			      rx_queue->core_index, 0);
 265
 266	if (rc) {
 267		netif_err(efx, rx_err, efx->net_dev,
 268			  "Failure to initialise XDP queue information rc=%d\n",
 269			  rc);
 270		efx->xdp_rxq_info_failed = true;
 271	}
 272
 273	/* Set up RX descriptor ring */
 274	efx_nic_init_rx(rx_queue);
 275}
 276
 277void efx_siena_fini_rx_queue(struct efx_rx_queue *rx_queue)
 278{
 279	struct efx_rx_buffer *rx_buf;
 280	int i;
 281
 282	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 283		  "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));
 284
 285	timer_delete_sync(&rx_queue->slow_fill);
 286
 287	/* Release RX buffers from the current read ptr to the write ptr */
 288	if (rx_queue->buffer) {
 289		for (i = rx_queue->removed_count; i < rx_queue->added_count;
 290		     i++) {
 291			unsigned int index = i & rx_queue->ptr_mask;
 292
 293			rx_buf = efx_rx_buffer(rx_queue, index);
 294			efx_fini_rx_buffer(rx_queue, rx_buf);
 295		}
 296	}
 297
 298	efx_fini_rx_recycle_ring(rx_queue);
 299
 300	if (xdp_rxq_info_is_reg(&rx_queue->xdp_rxq_info))
 301		xdp_rxq_info_unreg(&rx_queue->xdp_rxq_info);
 302}
 303
 304void efx_siena_remove_rx_queue(struct efx_rx_queue *rx_queue)
 305{
 306	netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 307		  "destroying RX queue %d\n", efx_rx_queue_index(rx_queue));
 308
 309	efx_nic_remove_rx(rx_queue);
 310
 311	kfree(rx_queue->buffer);
 312	rx_queue->buffer = NULL;
 313}
 314
 315/* Unmap a DMA-mapped page.  This function is only called for the final RX
 316 * buffer in a page.
 317 */
 318static void efx_unmap_rx_buffer(struct efx_nic *efx,
 319				struct efx_rx_buffer *rx_buf)
 320{
 321	struct page *page = rx_buf->page;
 322
 323	if (page) {
 324		struct efx_rx_page_state *state = page_address(page);
 325
 326		dma_unmap_page(&efx->pci_dev->dev,
 327			       state->dma_addr,
 328			       PAGE_SIZE << efx->rx_buffer_order,
 329			       DMA_FROM_DEVICE);
 330	}
 331}
 332
 333void efx_siena_free_rx_buffers(struct efx_rx_queue *rx_queue,
 334			       struct efx_rx_buffer *rx_buf,
 335			       unsigned int num_bufs)
 336{
 337	do {
 338		if (rx_buf->page) {
 339			put_page(rx_buf->page);
 340			rx_buf->page = NULL;
 341		}
 342		rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
 343	} while (--num_bufs);
 344}
 345
 346void efx_siena_rx_slow_fill(struct timer_list *t)
 347{
 348	struct efx_rx_queue *rx_queue = timer_container_of(rx_queue, t,
 349							   slow_fill);
 350
 351	/* Post an event to cause NAPI to run and refill the queue */
 352	efx_nic_generate_fill_event(rx_queue);
 353	++rx_queue->slow_fill_count;
 354}
 355
 356static void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
 357{
 358	mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10));
 359}
 360
 361/* efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers
 362 *
 363 * @rx_queue:		Efx RX queue
 364 *
 365 * This allocates a batch of pages, maps them for DMA, and populates
 366 * struct efx_rx_buffers for each one. Return a negative error code or
 367 * 0 on success. If a single page can be used for multiple buffers,
 368 * then the page will either be inserted fully, or not at all.
 369 */
 370static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue, bool atomic)
 371{
 372	unsigned int page_offset, index, count;
 373	struct efx_nic *efx = rx_queue->efx;
 374	struct efx_rx_page_state *state;
 375	struct efx_rx_buffer *rx_buf;
 376	dma_addr_t dma_addr;
 377	struct page *page;
 378
 379	count = 0;
 380	do {
 381		page = efx_reuse_page(rx_queue);
 382		if (page == NULL) {
 383			page = alloc_pages(__GFP_COMP |
 384					   (atomic ? GFP_ATOMIC : GFP_KERNEL),
 385					   efx->rx_buffer_order);
 386			if (unlikely(page == NULL))
 387				return -ENOMEM;
 388			dma_addr =
 389				dma_map_page(&efx->pci_dev->dev, page, 0,
 390					     PAGE_SIZE << efx->rx_buffer_order,
 391					     DMA_FROM_DEVICE);
 392			if (unlikely(dma_mapping_error(&efx->pci_dev->dev,
 393						       dma_addr))) {
 394				__free_pages(page, efx->rx_buffer_order);
 395				return -EIO;
 396			}
 397			state = page_address(page);
 398			state->dma_addr = dma_addr;
 399		} else {
 400			state = page_address(page);
 401			dma_addr = state->dma_addr;
 402		}
 403
 404		dma_addr += sizeof(struct efx_rx_page_state);
 405		page_offset = sizeof(struct efx_rx_page_state);
 406
 407		do {
 408			index = rx_queue->added_count & rx_queue->ptr_mask;
 409			rx_buf = efx_rx_buffer(rx_queue, index);
 410			rx_buf->dma_addr = dma_addr + efx->rx_ip_align +
 411					   EFX_XDP_HEADROOM;
 412			rx_buf->page = page;
 413			rx_buf->page_offset = page_offset + efx->rx_ip_align +
 414					      EFX_XDP_HEADROOM;
 415			rx_buf->len = efx->rx_dma_len;
 416			rx_buf->flags = 0;
 417			++rx_queue->added_count;
 418			get_page(page);
 419			dma_addr += efx->rx_page_buf_step;
 420			page_offset += efx->rx_page_buf_step;
 421		} while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE);
 422
 423		rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE;
 424	} while (++count < efx->rx_pages_per_batch);
 425
 426	return 0;
 427}
 428
 429void efx_siena_rx_config_page_split(struct efx_nic *efx)
 430{
 431	efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + efx->rx_ip_align +
 432				      EFX_XDP_HEADROOM + EFX_XDP_TAILROOM,
 433				      EFX_RX_BUF_ALIGNMENT);
 434	efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 :
 435		((PAGE_SIZE - sizeof(struct efx_rx_page_state)) /
 436		efx->rx_page_buf_step);
 437	efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) /
 438		efx->rx_bufs_per_page;
 439	efx->rx_pages_per_batch = DIV_ROUND_UP(EFX_RX_PREFERRED_BATCH,
 440					       efx->rx_bufs_per_page);
 441}
 442
 443/* efx_siena_fast_push_rx_descriptors - push new RX descriptors quickly
 444 * @rx_queue:		RX descriptor queue
 445 *
 446 * This will aim to fill the RX descriptor queue up to
 447 * @rx_queue->@max_fill. If there is insufficient atomic
 448 * memory to do so, a slow fill will be scheduled.
 449 *
 450 * The caller must provide serialisation (none is used here). In practise,
 451 * this means this function must run from the NAPI handler, or be called
 452 * when NAPI is disabled.
 453 */
 454void efx_siena_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue,
 455					bool atomic)
 456{
 457	struct efx_nic *efx = rx_queue->efx;
 458	unsigned int fill_level, batch_size;
 459	int space, rc = 0;
 460
 461	if (!rx_queue->refill_enabled)
 462		return;
 463
 464	/* Calculate current fill level, and exit if we don't need to fill */
 465	fill_level = (rx_queue->added_count - rx_queue->removed_count);
 466	EFX_WARN_ON_ONCE_PARANOID(fill_level > rx_queue->efx->rxq_entries);
 467	if (fill_level >= rx_queue->fast_fill_trigger)
 468		goto out;
 469
 470	/* Record minimum fill level */
 471	if (unlikely(fill_level < rx_queue->min_fill)) {
 472		if (fill_level)
 473			rx_queue->min_fill = fill_level;
 474	}
 475
 476	batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page;
 477	space = rx_queue->max_fill - fill_level;
 478	EFX_WARN_ON_ONCE_PARANOID(space < batch_size);
 479
 480	netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
 481		   "RX queue %d fast-filling descriptor ring from"
 482		   " level %d to level %d\n",
 483		   efx_rx_queue_index(rx_queue), fill_level,
 484		   rx_queue->max_fill);
 485
 486	do {
 487		rc = efx_init_rx_buffers(rx_queue, atomic);
 488		if (unlikely(rc)) {
 489			/* Ensure that we don't leave the rx queue empty */
 490			efx_schedule_slow_fill(rx_queue);
 491			goto out;
 492		}
 493	} while ((space -= batch_size) >= batch_size);
 494
 495	netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
 496		   "RX queue %d fast-filled descriptor ring "
 497		   "to level %d\n", efx_rx_queue_index(rx_queue),
 498		   rx_queue->added_count - rx_queue->removed_count);
 499
 500 out:
 501	if (rx_queue->notified_count != rx_queue->added_count)
 502		efx_nic_notify_rx_desc(rx_queue);
 503}
 504
 505/* Pass a received packet up through GRO.  GRO can handle pages
 506 * regardless of checksum state and skbs with a good checksum.
 507 */
 508void
 509efx_siena_rx_packet_gro(struct efx_channel *channel,
 510			struct efx_rx_buffer *rx_buf,
 511			unsigned int n_frags, u8 *eh, __wsum csum)
 512{
 513	struct napi_struct *napi = &channel->napi_str;
 514	struct efx_nic *efx = channel->efx;
 515	struct sk_buff *skb;
 516
 517	skb = napi_get_frags(napi);
 518	if (unlikely(!skb)) {
 519		struct efx_rx_queue *rx_queue;
 520
 521		rx_queue = efx_channel_get_rx_queue(channel);
 522		efx_siena_free_rx_buffers(rx_queue, rx_buf, n_frags);
 523		return;
 524	}
 525
 526	if (efx->net_dev->features & NETIF_F_RXHASH)
 527		skb_set_hash(skb, efx_rx_buf_hash(efx, eh),
 528			     PKT_HASH_TYPE_L3);
 529	if (csum) {
 530		skb->csum = csum;
 531		skb->ip_summed = CHECKSUM_COMPLETE;
 532	} else {
 533		skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ?
 534				  CHECKSUM_UNNECESSARY : CHECKSUM_NONE);
 535	}
 536	skb->csum_level = !!(rx_buf->flags & EFX_RX_PKT_CSUM_LEVEL);
 537
 538	for (;;) {
 539		skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 540				   rx_buf->page, rx_buf->page_offset,
 541				   rx_buf->len);
 542		rx_buf->page = NULL;
 543		skb->len += rx_buf->len;
 544		if (skb_shinfo(skb)->nr_frags == n_frags)
 545			break;
 546
 547		rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
 548	}
 549
 550	skb->data_len = skb->len;
 551	skb->truesize += n_frags * efx->rx_buffer_truesize;
 552
 553	skb_record_rx_queue(skb, channel->rx_queue.core_index);
 554
 555	napi_gro_frags(napi);
 556}
 557
 558void efx_siena_set_default_rx_indir_table(struct efx_nic *efx,
 559					  struct efx_rss_context *ctx)
 560{
 561	size_t i;
 562
 563	for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++)
 564		ctx->rx_indir_table[i] =
 565			ethtool_rxfh_indir_default(i, efx->rss_spread);
 566}
 567
 568/**
 569 * efx_siena_filter_is_mc_recipient - test whether spec is a multicast recipient
 570 * @spec: Specification to test
 571 *
 572 * Return: %true if the specification is a non-drop RX filter that
 573 * matches a local MAC address I/G bit value of 1 or matches a local
 574 * IPv4 or IPv6 address value in the respective multicast address
 575 * range.  Otherwise %false.
 576 */
 577bool efx_siena_filter_is_mc_recipient(const struct efx_filter_spec *spec)
 578{
 579	if (!(spec->flags & EFX_FILTER_FLAG_RX) ||
 580	    spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP)
 581		return false;
 582
 583	if (spec->match_flags &
 584	    (EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_MAC_IG) &&
 585	    is_multicast_ether_addr(spec->loc_mac))
 586		return true;
 587
 588	if ((spec->match_flags &
 589	     (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
 590	    (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
 591		if (spec->ether_type == htons(ETH_P_IP) &&
 592		    ipv4_is_multicast(spec->loc_host[0]))
 593			return true;
 594		if (spec->ether_type == htons(ETH_P_IPV6) &&
 595		    ((const u8 *)spec->loc_host)[0] == 0xff)
 596			return true;
 597	}
 598
 599	return false;
 600}
 601
 602bool efx_siena_filter_spec_equal(const struct efx_filter_spec *left,
 603				 const struct efx_filter_spec *right)
 604{
 605	if ((left->match_flags ^ right->match_flags) |
 606	    ((left->flags ^ right->flags) &
 607	     (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)))
 608		return false;
 609
 610	return memcmp(&left->outer_vid, &right->outer_vid,
 611		      sizeof(struct efx_filter_spec) -
 612		      offsetof(struct efx_filter_spec, outer_vid)) == 0;
 613}
 614
 615u32 efx_siena_filter_spec_hash(const struct efx_filter_spec *spec)
 616{
 617	BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3);
 618	return jhash2((const u32 *)&spec->outer_vid,
 619		      (sizeof(struct efx_filter_spec) -
 620		       offsetof(struct efx_filter_spec, outer_vid)) / 4,
 621		      0);
 622}
 623
 624#ifdef CONFIG_RFS_ACCEL
 625bool efx_siena_rps_check_rule(struct efx_arfs_rule *rule,
 626			      unsigned int filter_idx, bool *force)
 627{
 628	if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) {
 629		/* ARFS is currently updating this entry, leave it */
 630		return false;
 631	}
 632	if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) {
 633		/* ARFS tried and failed to update this, so it's probably out
 634		 * of date.  Remove the filter and the ARFS rule entry.
 635		 */
 636		rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING;
 637		*force = true;
 638		return true;
 639	} else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */
 640		/* ARFS has moved on, so old filter is not needed.  Since we did
 641		 * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will
 642		 * not be removed by efx_siena_rps_hash_del() subsequently.
 643		 */
 644		*force = true;
 645		return true;
 646	}
 647	/* Remove it iff ARFS wants to. */
 648	return true;
 649}
 650
 651static
 652struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx,
 653				       const struct efx_filter_spec *spec)
 654{
 655	u32 hash = efx_siena_filter_spec_hash(spec);
 656
 657	lockdep_assert_held(&efx->rps_hash_lock);
 658	if (!efx->rps_hash_table)
 659		return NULL;
 660	return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE];
 661}
 662
 663struct efx_arfs_rule *efx_siena_rps_hash_find(struct efx_nic *efx,
 664					const struct efx_filter_spec *spec)
 665{
 666	struct efx_arfs_rule *rule;
 667	struct hlist_head *head;
 668	struct hlist_node *node;
 669
 670	head = efx_rps_hash_bucket(efx, spec);
 671	if (!head)
 672		return NULL;
 673	hlist_for_each(node, head) {
 674		rule = container_of(node, struct efx_arfs_rule, node);
 675		if (efx_siena_filter_spec_equal(spec, &rule->spec))
 676			return rule;
 677	}
 678	return NULL;
 679}
 680
 681static struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx,
 682					const struct efx_filter_spec *spec,
 683					bool *new)
 684{
 685	struct efx_arfs_rule *rule;
 686	struct hlist_head *head;
 687	struct hlist_node *node;
 688
 689	head = efx_rps_hash_bucket(efx, spec);
 690	if (!head)
 691		return NULL;
 692	hlist_for_each(node, head) {
 693		rule = container_of(node, struct efx_arfs_rule, node);
 694		if (efx_siena_filter_spec_equal(spec, &rule->spec)) {
 695			*new = false;
 696			return rule;
 697		}
 698	}
 699	rule = kmalloc(sizeof(*rule), GFP_ATOMIC);
 700	*new = true;
 701	if (rule) {
 702		memcpy(&rule->spec, spec, sizeof(rule->spec));
 703		hlist_add_head(&rule->node, head);
 704	}
 705	return rule;
 706}
 707
 708void efx_siena_rps_hash_del(struct efx_nic *efx,
 709			    const struct efx_filter_spec *spec)
 710{
 711	struct efx_arfs_rule *rule;
 712	struct hlist_head *head;
 713	struct hlist_node *node;
 714
 715	head = efx_rps_hash_bucket(efx, spec);
 716	if (WARN_ON(!head))
 717		return;
 718	hlist_for_each(node, head) {
 719		rule = container_of(node, struct efx_arfs_rule, node);
 720		if (efx_siena_filter_spec_equal(spec, &rule->spec)) {
 721			/* Someone already reused the entry.  We know that if
 722			 * this check doesn't fire (i.e. filter_id == REMOVING)
 723			 * then the REMOVING mark was put there by our caller,
 724			 * because caller is holding a lock on filter table and
 725			 * only holders of that lock set REMOVING.
 726			 */
 727			if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING)
 728				return;
 729			hlist_del(node);
 730			kfree(rule);
 731			return;
 732		}
 733	}
 734	/* We didn't find it. */
 735	WARN_ON(1);
 736}
 737#endif
 738
 739int efx_siena_probe_filters(struct efx_nic *efx)
 740{
 741	int rc;
 742
 743	mutex_lock(&efx->mac_lock);
 744	down_write(&efx->filter_sem);
 745	rc = efx->type->filter_table_probe(efx);
 746	if (rc)
 747		goto out_unlock;
 748
 749#ifdef CONFIG_RFS_ACCEL
 750	if (efx->type->offload_features & NETIF_F_NTUPLE) {
 751		struct efx_channel *channel;
 752		int i, success = 1;
 753
 754		efx_for_each_channel(channel, efx) {
 755			channel->rps_flow_id =
 756				kcalloc(efx->type->max_rx_ip_filters,
 757					sizeof(*channel->rps_flow_id),
 758					GFP_KERNEL);
 759			if (!channel->rps_flow_id)
 760				success = 0;
 761			else
 762				for (i = 0;
 763				     i < efx->type->max_rx_ip_filters;
 764				     ++i)
 765					channel->rps_flow_id[i] =
 766						RPS_FLOW_ID_INVALID;
 767			channel->rfs_expire_index = 0;
 768			channel->rfs_filter_count = 0;
 769		}
 770
 771		if (!success) {
 772			efx_for_each_channel(channel, efx)
 773				kfree(channel->rps_flow_id);
 774			efx->type->filter_table_remove(efx);
 775			rc = -ENOMEM;
 776			goto out_unlock;
 777		}
 778	}
 779#endif
 780out_unlock:
 781	up_write(&efx->filter_sem);
 782	mutex_unlock(&efx->mac_lock);
 783	return rc;
 784}
 785
 786void efx_siena_remove_filters(struct efx_nic *efx)
 787{
 788#ifdef CONFIG_RFS_ACCEL
 789	struct efx_channel *channel;
 790
 791	efx_for_each_channel(channel, efx) {
 792		cancel_delayed_work_sync(&channel->filter_work);
 793		kfree(channel->rps_flow_id);
 794		channel->rps_flow_id = NULL;
 795	}
 796#endif
 797	down_write(&efx->filter_sem);
 798	efx->type->filter_table_remove(efx);
 799	up_write(&efx->filter_sem);
 800}
 801
 802#ifdef CONFIG_RFS_ACCEL
 803
 804static void efx_filter_rfs_work(struct work_struct *data)
 805{
 806	struct efx_async_filter_insertion *req = container_of(data, struct efx_async_filter_insertion,
 807							      work);
 808	struct efx_nic *efx = netdev_priv(req->net_dev);
 809	struct efx_channel *channel = efx_get_channel(efx, req->rxq_index);
 810	int slot_idx = req - efx->rps_slot;
 811	struct efx_arfs_rule *rule;
 812	u16 arfs_id = 0;
 813	int rc;
 814
 815	rc = efx->type->filter_insert(efx, &req->spec, true);
 816	if (rc >= 0)
 817		/* Discard 'priority' part of EF10+ filter ID (mcdi_filters) */
 818		rc %= efx->type->max_rx_ip_filters;
 819	if (efx->rps_hash_table) {
 820		spin_lock_bh(&efx->rps_hash_lock);
 821		rule = efx_siena_rps_hash_find(efx, &req->spec);
 822		/* The rule might have already gone, if someone else's request
 823		 * for the same spec was already worked and then expired before
 824		 * we got around to our work.  In that case we have nothing
 825		 * tying us to an arfs_id, meaning that as soon as the filter
 826		 * is considered for expiry it will be removed.
 827		 */
 828		if (rule) {
 829			if (rc < 0)
 830				rule->filter_id = EFX_ARFS_FILTER_ID_ERROR;
 831			else
 832				rule->filter_id = rc;
 833			arfs_id = rule->arfs_id;
 834		}
 835		spin_unlock_bh(&efx->rps_hash_lock);
 836	}
 837	if (rc >= 0) {
 838		/* Remember this so we can check whether to expire the filter
 839		 * later.
 840		 */
 841		mutex_lock(&efx->rps_mutex);
 842		if (channel->rps_flow_id[rc] == RPS_FLOW_ID_INVALID)
 843			channel->rfs_filter_count++;
 844		channel->rps_flow_id[rc] = req->flow_id;
 845		mutex_unlock(&efx->rps_mutex);
 846
 847		if (req->spec.ether_type == htons(ETH_P_IP))
 848			netif_info(efx, rx_status, efx->net_dev,
 849				   "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d id %u]\n",
 850				   (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
 851				   req->spec.rem_host, ntohs(req->spec.rem_port),
 852				   req->spec.loc_host, ntohs(req->spec.loc_port),
 853				   req->rxq_index, req->flow_id, rc, arfs_id);
 854		else
 855			netif_info(efx, rx_status, efx->net_dev,
 856				   "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d id %u]\n",
 857				   (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
 858				   req->spec.rem_host, ntohs(req->spec.rem_port),
 859				   req->spec.loc_host, ntohs(req->spec.loc_port),
 860				   req->rxq_index, req->flow_id, rc, arfs_id);
 861		channel->n_rfs_succeeded++;
 862	} else {
 863		if (req->spec.ether_type == htons(ETH_P_IP))
 864			netif_dbg(efx, rx_status, efx->net_dev,
 865				  "failed to steer %s %pI4:%u:%pI4:%u to queue %u [flow %u rc %d id %u]\n",
 866				  (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
 867				  req->spec.rem_host, ntohs(req->spec.rem_port),
 868				  req->spec.loc_host, ntohs(req->spec.loc_port),
 869				  req->rxq_index, req->flow_id, rc, arfs_id);
 870		else
 871			netif_dbg(efx, rx_status, efx->net_dev,
 872				  "failed to steer %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u rc %d id %u]\n",
 873				  (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP",
 874				  req->spec.rem_host, ntohs(req->spec.rem_port),
 875				  req->spec.loc_host, ntohs(req->spec.loc_port),
 876				  req->rxq_index, req->flow_id, rc, arfs_id);
 877		channel->n_rfs_failed++;
 878		/* We're overloading the NIC's filter tables, so let's do a
 879		 * chunk of extra expiry work.
 880		 */
 881		__efx_siena_filter_rfs_expire(channel,
 882					      min(channel->rfs_filter_count,
 883						  100u));
 884	}
 885
 886	/* Release references */
 887	clear_bit(slot_idx, &efx->rps_slot_map);
 888	netdev_put(req->net_dev, &req->net_dev_tracker);
 889}
 890
 891int efx_siena_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
 892			 u16 rxq_index, u32 flow_id)
 893{
 894	struct efx_nic *efx = netdev_priv(net_dev);
 895	struct efx_async_filter_insertion *req;
 896	struct efx_arfs_rule *rule;
 897	struct flow_keys fk;
 898	int slot_idx;
 899	bool new;
 900	int rc;
 901
 902	/* find a free slot */
 903	for (slot_idx = 0; slot_idx < EFX_RPS_MAX_IN_FLIGHT; slot_idx++)
 904		if (!test_and_set_bit(slot_idx, &efx->rps_slot_map))
 905			break;
 906	if (slot_idx >= EFX_RPS_MAX_IN_FLIGHT)
 907		return -EBUSY;
 908
 909	if (flow_id == RPS_FLOW_ID_INVALID) {
 910		rc = -EINVAL;
 911		goto out_clear;
 912	}
 913
 914	if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) {
 915		rc = -EPROTONOSUPPORT;
 916		goto out_clear;
 917	}
 918
 919	if (fk.basic.n_proto != htons(ETH_P_IP) && fk.basic.n_proto != htons(ETH_P_IPV6)) {
 920		rc = -EPROTONOSUPPORT;
 921		goto out_clear;
 922	}
 923	if (fk.control.flags & FLOW_DIS_IS_FRAGMENT) {
 924		rc = -EPROTONOSUPPORT;
 925		goto out_clear;
 926	}
 927
 928	req = efx->rps_slot + slot_idx;
 929	efx_filter_init_rx(&req->spec, EFX_FILTER_PRI_HINT,
 930			   efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0,
 931			   rxq_index);
 932	req->spec.match_flags =
 933		EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 934		EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 935		EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 936	req->spec.ether_type = fk.basic.n_proto;
 937	req->spec.ip_proto = fk.basic.ip_proto;
 938
 939	if (fk.basic.n_proto == htons(ETH_P_IP)) {
 940		req->spec.rem_host[0] = fk.addrs.v4addrs.src;
 941		req->spec.loc_host[0] = fk.addrs.v4addrs.dst;
 942	} else {
 943		memcpy(req->spec.rem_host, &fk.addrs.v6addrs.src,
 944		       sizeof(struct in6_addr));
 945		memcpy(req->spec.loc_host, &fk.addrs.v6addrs.dst,
 946		       sizeof(struct in6_addr));
 947	}
 948
 949	req->spec.rem_port = fk.ports.src;
 950	req->spec.loc_port = fk.ports.dst;
 951
 952	if (efx->rps_hash_table) {
 953		/* Add it to ARFS hash table */
 954		spin_lock(&efx->rps_hash_lock);
 955		rule = efx_rps_hash_add(efx, &req->spec, &new);
 956		if (!rule) {
 957			rc = -ENOMEM;
 958			goto out_unlock;
 959		}
 960		if (new)
 961			rule->arfs_id = efx->rps_next_id++ % RPS_NO_FILTER;
 962		rc = rule->arfs_id;
 963		/* Skip if existing or pending filter already does the right thing */
 964		if (!new && rule->rxq_index == rxq_index &&
 965		    rule->filter_id >= EFX_ARFS_FILTER_ID_PENDING)
 966			goto out_unlock;
 967		rule->rxq_index = rxq_index;
 968		rule->filter_id = EFX_ARFS_FILTER_ID_PENDING;
 969		spin_unlock(&efx->rps_hash_lock);
 970	} else {
 971		/* Without an ARFS hash table, we just use arfs_id 0 for all
 972		 * filters.  This means if multiple flows hash to the same
 973		 * flow_id, all but the most recently touched will be eligible
 974		 * for expiry.
 975		 */
 976		rc = 0;
 977	}
 978
 979	/* Queue the request */
 980	req->net_dev = net_dev;
 981	netdev_hold(req->net_dev, &req->net_dev_tracker, GFP_ATOMIC);
 982	INIT_WORK(&req->work, efx_filter_rfs_work);
 983	req->rxq_index = rxq_index;
 984	req->flow_id = flow_id;
 985	schedule_work(&req->work);
 986	return rc;
 987out_unlock:
 988	spin_unlock(&efx->rps_hash_lock);
 989out_clear:
 990	clear_bit(slot_idx, &efx->rps_slot_map);
 991	return rc;
 992}
 993
 994bool __efx_siena_filter_rfs_expire(struct efx_channel *channel,
 995				   unsigned int quota)
 996{
 997	bool (*expire_one)(struct efx_nic *efx, u32 flow_id, unsigned int index);
 998	struct efx_nic *efx = channel->efx;
 999	unsigned int index, size, start;
1000	u32 flow_id;
1001
1002	if (!mutex_trylock(&efx->rps_mutex))
1003		return false;
1004	expire_one = efx->type->filter_rfs_expire_one;
1005	index = channel->rfs_expire_index;
1006	start = index;
1007	size = efx->type->max_rx_ip_filters;
1008	while (quota) {
1009		flow_id = channel->rps_flow_id[index];
1010
1011		if (flow_id != RPS_FLOW_ID_INVALID) {
1012			quota--;
1013			if (expire_one(efx, flow_id, index)) {
1014				netif_info(efx, rx_status, efx->net_dev,
1015					   "expired filter %d [channel %u flow %u]\n",
1016					   index, channel->channel, flow_id);
1017				channel->rps_flow_id[index] = RPS_FLOW_ID_INVALID;
1018				channel->rfs_filter_count--;
1019			}
1020		}
1021		if (++index == size)
1022			index = 0;
1023		/* If we were called with a quota that exceeds the total number
1024		 * of filters in the table (which shouldn't happen, but could
1025		 * if two callers race), ensure that we don't loop forever -
1026		 * stop when we've examined every row of the table.
1027		 */
1028		if (index == start)
1029			break;
1030	}
1031
1032	channel->rfs_expire_index = index;
1033	mutex_unlock(&efx->rps_mutex);
1034	return true;
1035}
1036
1037#endif /* CONFIG_RFS_ACCEL */