ice_txrx.c

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2018, Intel Corporation. */
   3
   4/* The driver transmit and receive code */
   5
   6#include <linux/mm.h>
   7#include <linux/netdevice.h>
   8#include <linux/prefetch.h>
   9#include <linux/bpf_trace.h>
  10#include <linux/net/intel/libie/rx.h>
  11#include <net/libeth/xdp.h>
  12#include <net/dsfield.h>
  13#include <net/mpls.h>
  14#include <net/xdp.h>
  15#include "ice_txrx_lib.h"
  16#include "ice_lib.h"
  17#include "ice.h"
  18#include "ice_trace.h"
  19#include "ice_dcb_lib.h"
  20#include "ice_xsk.h"
  21#include "ice_eswitch.h"
  22
  23#define ICE_RX_HDR_SIZE		256
  24
  25#define ICE_FDIR_CLEAN_DELAY 10
  26
  27/**
  28 * ice_prgm_fdir_fltr - Program a Flow Director filter
  29 * @vsi: VSI to send dummy packet
  30 * @fdir_desc: flow director descriptor
  31 * @raw_packet: allocated buffer for flow director
  32 */
  33int
  34ice_prgm_fdir_fltr(struct ice_vsi *vsi, struct ice_fltr_desc *fdir_desc,
  35		   u8 *raw_packet)
  36{
  37	struct ice_tx_buf *tx_buf, *first;
  38	struct ice_fltr_desc *f_desc;
  39	struct ice_tx_desc *tx_desc;
  40	struct ice_tx_ring *tx_ring;
  41	struct device *dev;
  42	dma_addr_t dma;
  43	u32 td_cmd;
  44	u16 i;
  45
  46	/* VSI and Tx ring */
  47	if (!vsi)
  48		return -ENOENT;
  49	tx_ring = vsi->tx_rings[0];
  50	if (!tx_ring || !tx_ring->desc)
  51		return -ENOENT;
  52	dev = tx_ring->dev;
  53
  54	/* we are using two descriptors to add/del a filter and we can wait */
  55	for (i = ICE_FDIR_CLEAN_DELAY; ICE_DESC_UNUSED(tx_ring) < 2; i--) {
  56		if (!i)
  57			return -EAGAIN;
  58		msleep_interruptible(1);
  59	}
  60
  61	dma = dma_map_single(dev, raw_packet, ICE_FDIR_MAX_RAW_PKT_SIZE,
  62			     DMA_TO_DEVICE);
  63
  64	if (dma_mapping_error(dev, dma))
  65		return -EINVAL;
  66
  67	/* grab the next descriptor */
  68	i = tx_ring->next_to_use;
  69	first = &tx_ring->tx_buf[i];
  70	f_desc = ICE_TX_FDIRDESC(tx_ring, i);
  71	memcpy(f_desc, fdir_desc, sizeof(*f_desc));
  72
  73	i++;
  74	i = (i < tx_ring->count) ? i : 0;
  75	tx_desc = ICE_TX_DESC(tx_ring, i);
  76	tx_buf = &tx_ring->tx_buf[i];
  77
  78	i++;
  79	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
  80
  81	memset(tx_buf, 0, sizeof(*tx_buf));
  82	dma_unmap_len_set(tx_buf, len, ICE_FDIR_MAX_RAW_PKT_SIZE);
  83	dma_unmap_addr_set(tx_buf, dma, dma);
  84
  85	tx_desc->buf_addr = cpu_to_le64(dma);
  86	td_cmd = ICE_TXD_LAST_DESC_CMD | ICE_TX_DESC_CMD_DUMMY |
  87		 ICE_TX_DESC_CMD_RE;
  88
  89	tx_buf->type = ICE_TX_BUF_DUMMY;
  90	tx_buf->raw_buf = raw_packet;
  91
  92	tx_desc->cmd_type_offset_bsz =
  93		ice_build_ctob(td_cmd, 0, ICE_FDIR_MAX_RAW_PKT_SIZE, 0);
  94
  95	/* Force memory write to complete before letting h/w know
  96	 * there are new descriptors to fetch.
  97	 */
  98	wmb();
  99
 100	/* mark the data descriptor to be watched */
 101	first->next_to_watch = tx_desc;
 102
 103	writel(tx_ring->next_to_use, tx_ring->tail);
 104
 105	return 0;
 106}
 107
 108/**
 109 * ice_unmap_and_free_tx_buf - Release a Tx buffer
 110 * @ring: the ring that owns the buffer
 111 * @tx_buf: the buffer to free
 112 */
 113static void
 114ice_unmap_and_free_tx_buf(struct ice_tx_ring *ring, struct ice_tx_buf *tx_buf)
 115{
 116	if (tx_buf->type != ICE_TX_BUF_XDP_TX && dma_unmap_len(tx_buf, len))
 117		dma_unmap_page(ring->dev,
 118			       dma_unmap_addr(tx_buf, dma),
 119			       dma_unmap_len(tx_buf, len),
 120			       DMA_TO_DEVICE);
 121
 122	switch (tx_buf->type) {
 123	case ICE_TX_BUF_DUMMY:
 124		devm_kfree(ring->dev, tx_buf->raw_buf);
 125		break;
 126	case ICE_TX_BUF_SKB:
 127		dev_kfree_skb_any(tx_buf->skb);
 128		break;
 129	case ICE_TX_BUF_XDP_TX:
 130		libeth_xdp_return_va(tx_buf->raw_buf, false);
 131		break;
 132	case ICE_TX_BUF_XDP_XMIT:
 133		xdp_return_frame(tx_buf->xdpf);
 134		break;
 135	}
 136
 137	tx_buf->next_to_watch = NULL;
 138	tx_buf->type = ICE_TX_BUF_EMPTY;
 139	dma_unmap_len_set(tx_buf, len, 0);
 140	/* tx_buf must be completely set up in the transmit path */
 141}
 142
 143static struct netdev_queue *txring_txq(const struct ice_tx_ring *ring)
 144{
 145	return netdev_get_tx_queue(ring->netdev, ring->q_index);
 146}
 147
 148/**
 149 * ice_clean_tstamp_ring - clean time stamp ring
 150 * @tx_ring: Tx ring to clean the Time Stamp ring for
 151 */
 152static void ice_clean_tstamp_ring(struct ice_tx_ring *tx_ring)
 153{
 154	struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring;
 155	u32 size;
 156
 157	if (!tstamp_ring->desc)
 158		return;
 159
 160	size = ALIGN(tstamp_ring->count * sizeof(struct ice_ts_desc),
 161		     PAGE_SIZE);
 162	memset(tstamp_ring->desc, 0, size);
 163	tstamp_ring->next_to_use = 0;
 164}
 165
 166/**
 167 * ice_free_tstamp_ring - free time stamp resources per queue
 168 * @tx_ring: Tx ring to free the Time Stamp ring for
 169 */
 170void ice_free_tstamp_ring(struct ice_tx_ring *tx_ring)
 171{
 172	struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring;
 173	u32 size;
 174
 175	if (!tstamp_ring->desc)
 176		return;
 177
 178	ice_clean_tstamp_ring(tx_ring);
 179	size = ALIGN(tstamp_ring->count * sizeof(struct ice_ts_desc),
 180		     PAGE_SIZE);
 181	dmam_free_coherent(tx_ring->dev, size, tstamp_ring->desc,
 182			   tstamp_ring->dma);
 183	tstamp_ring->desc = NULL;
 184}
 185
 186/**
 187 * ice_free_tx_tstamp_ring - free time stamp resources per Tx ring
 188 * @tx_ring: Tx ring to free the Time Stamp ring for
 189 */
 190void ice_free_tx_tstamp_ring(struct ice_tx_ring *tx_ring)
 191{
 192	ice_free_tstamp_ring(tx_ring);
 193	kfree_rcu(tx_ring->tstamp_ring, rcu);
 194	tx_ring->tstamp_ring = NULL;
 195	tx_ring->flags &= ~ICE_TX_FLAGS_TXTIME;
 196}
 197
 198/**
 199 * ice_clean_tx_ring - Free any empty Tx buffers
 200 * @tx_ring: ring to be cleaned
 201 */
 202void ice_clean_tx_ring(struct ice_tx_ring *tx_ring)
 203{
 204	u32 size;
 205	u16 i;
 206
 207	if (ice_ring_is_xdp(tx_ring) && tx_ring->xsk_pool) {
 208		ice_xsk_clean_xdp_ring(tx_ring);
 209		goto tx_skip_free;
 210	}
 211
 212	/* ring already cleared, nothing to do */
 213	if (!tx_ring->tx_buf)
 214		return;
 215
 216	/* Free all the Tx ring sk_buffs */
 217	for (i = 0; i < tx_ring->count; i++)
 218		ice_unmap_and_free_tx_buf(tx_ring, &tx_ring->tx_buf[i]);
 219
 220tx_skip_free:
 221	memset(tx_ring->tx_buf, 0, sizeof(*tx_ring->tx_buf) * tx_ring->count);
 222
 223	size = ALIGN(tx_ring->count * sizeof(struct ice_tx_desc),
 224		     PAGE_SIZE);
 225	/* Zero out the descriptor ring */
 226	memset(tx_ring->desc, 0, size);
 227
 228	tx_ring->next_to_use = 0;
 229	tx_ring->next_to_clean = 0;
 230
 231	if (!tx_ring->netdev)
 232		return;
 233
 234	/* cleanup Tx queue statistics */
 235	netdev_tx_reset_queue(txring_txq(tx_ring));
 236
 237	if (ice_is_txtime_cfg(tx_ring))
 238		ice_free_tx_tstamp_ring(tx_ring);
 239}
 240
 241/**
 242 * ice_free_tx_ring - Free Tx resources per queue
 243 * @tx_ring: Tx descriptor ring for a specific queue
 244 *
 245 * Free all transmit software resources
 246 */
 247void ice_free_tx_ring(struct ice_tx_ring *tx_ring)
 248{
 249	u32 size;
 250
 251	ice_clean_tx_ring(tx_ring);
 252	devm_kfree(tx_ring->dev, tx_ring->tx_buf);
 253	tx_ring->tx_buf = NULL;
 254
 255	if (tx_ring->desc) {
 256		size = ALIGN(tx_ring->count * sizeof(struct ice_tx_desc),
 257			     PAGE_SIZE);
 258		dmam_free_coherent(tx_ring->dev, size,
 259				   tx_ring->desc, tx_ring->dma);
 260		tx_ring->desc = NULL;
 261	}
 262}
 263
 264/**
 265 * ice_clean_tx_irq - Reclaim resources after transmit completes
 266 * @tx_ring: Tx ring to clean
 267 * @napi_budget: Used to determine if we are in netpoll
 268 *
 269 * Returns true if there's any budget left (e.g. the clean is finished)
 270 */
 271static bool ice_clean_tx_irq(struct ice_tx_ring *tx_ring, int napi_budget)
 272{
 273	unsigned int total_bytes = 0, total_pkts = 0;
 274	unsigned int budget = ICE_DFLT_IRQ_WORK;
 275	struct ice_vsi *vsi = tx_ring->vsi;
 276	s16 i = tx_ring->next_to_clean;
 277	struct ice_tx_desc *tx_desc;
 278	struct ice_tx_buf *tx_buf;
 279
 280	/* get the bql data ready */
 281	netdev_txq_bql_complete_prefetchw(txring_txq(tx_ring));
 282
 283	tx_buf = &tx_ring->tx_buf[i];
 284	tx_desc = ICE_TX_DESC(tx_ring, i);
 285	i -= tx_ring->count;
 286
 287	prefetch(&vsi->state);
 288
 289	do {
 290		struct ice_tx_desc *eop_desc = tx_buf->next_to_watch;
 291
 292		/* if next_to_watch is not set then there is no work pending */
 293		if (!eop_desc)
 294			break;
 295
 296		/* follow the guidelines of other drivers */
 297		prefetchw(&tx_buf->skb->users);
 298
 299		smp_rmb();	/* prevent any other reads prior to eop_desc */
 300
 301		ice_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf);
 302		/* if the descriptor isn't done, no work yet to do */
 303		if (!(eop_desc->cmd_type_offset_bsz &
 304		      cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)))
 305			break;
 306
 307		/* clear next_to_watch to prevent false hangs */
 308		tx_buf->next_to_watch = NULL;
 309
 310		/* update the statistics for this packet */
 311		total_bytes += tx_buf->bytecount;
 312		total_pkts += tx_buf->gso_segs;
 313
 314		/* free the skb */
 315		napi_consume_skb(tx_buf->skb, napi_budget);
 316
 317		/* unmap skb header data */
 318		dma_unmap_single(tx_ring->dev,
 319				 dma_unmap_addr(tx_buf, dma),
 320				 dma_unmap_len(tx_buf, len),
 321				 DMA_TO_DEVICE);
 322
 323		/* clear tx_buf data */
 324		tx_buf->type = ICE_TX_BUF_EMPTY;
 325		dma_unmap_len_set(tx_buf, len, 0);
 326
 327		/* unmap remaining buffers */
 328		while (tx_desc != eop_desc) {
 329			ice_trace(clean_tx_irq_unmap, tx_ring, tx_desc, tx_buf);
 330			tx_buf++;
 331			tx_desc++;
 332			i++;
 333			if (unlikely(!i)) {
 334				i -= tx_ring->count;
 335				tx_buf = tx_ring->tx_buf;
 336				tx_desc = ICE_TX_DESC(tx_ring, 0);
 337			}
 338
 339			/* unmap any remaining paged data */
 340			if (dma_unmap_len(tx_buf, len)) {
 341				dma_unmap_page(tx_ring->dev,
 342					       dma_unmap_addr(tx_buf, dma),
 343					       dma_unmap_len(tx_buf, len),
 344					       DMA_TO_DEVICE);
 345				dma_unmap_len_set(tx_buf, len, 0);
 346			}
 347		}
 348		ice_trace(clean_tx_irq_unmap_eop, tx_ring, tx_desc, tx_buf);
 349
 350		/* move us one more past the eop_desc for start of next pkt */
 351		tx_buf++;
 352		tx_desc++;
 353		i++;
 354		if (unlikely(!i)) {
 355			i -= tx_ring->count;
 356			tx_buf = tx_ring->tx_buf;
 357			tx_desc = ICE_TX_DESC(tx_ring, 0);
 358		}
 359
 360		prefetch(tx_desc);
 361
 362		/* update budget accounting */
 363		budget--;
 364	} while (likely(budget));
 365
 366	i += tx_ring->count;
 367	tx_ring->next_to_clean = i;
 368
 369	ice_update_tx_ring_stats(tx_ring, total_pkts, total_bytes);
 370	netdev_tx_completed_queue(txring_txq(tx_ring), total_pkts, total_bytes);
 371
 372#define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))
 373	if (unlikely(total_pkts && netif_carrier_ok(tx_ring->netdev) &&
 374		     (ICE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 375		/* Make sure that anybody stopping the queue after this
 376		 * sees the new next_to_clean.
 377		 */
 378		smp_mb();
 379		if (netif_tx_queue_stopped(txring_txq(tx_ring)) &&
 380		    !test_bit(ICE_VSI_DOWN, vsi->state)) {
 381			netif_tx_wake_queue(txring_txq(tx_ring));
 382			++tx_ring->ring_stats->tx_stats.restart_q;
 383		}
 384	}
 385
 386	return !!budget;
 387}
 388
 389/**
 390 * ice_alloc_tstamp_ring - allocate the Time Stamp ring
 391 * @tx_ring: Tx ring to allocate the Time Stamp ring for
 392 *
 393 * Return: 0 on success, negative on error
 394 */
 395static int ice_alloc_tstamp_ring(struct ice_tx_ring *tx_ring)
 396{
 397	struct ice_tstamp_ring *tstamp_ring;
 398
 399	/* allocate with kzalloc(), free with kfree_rcu() */
 400	tstamp_ring = kzalloc(sizeof(*tstamp_ring), GFP_KERNEL);
 401	if (!tstamp_ring)
 402		return -ENOMEM;
 403
 404	tstamp_ring->tx_ring = tx_ring;
 405	tx_ring->tstamp_ring = tstamp_ring;
 406	tstamp_ring->desc = NULL;
 407	tstamp_ring->count = ice_calc_ts_ring_count(tx_ring);
 408	tx_ring->flags |= ICE_TX_FLAGS_TXTIME;
 409	return 0;
 410}
 411
 412/**
 413 * ice_setup_tstamp_ring - allocate the Time Stamp ring
 414 * @tx_ring: Tx ring to set up the Time Stamp ring for
 415 *
 416 * Return: 0 on success, negative on error
 417 */
 418static int ice_setup_tstamp_ring(struct ice_tx_ring *tx_ring)
 419{
 420	struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring;
 421	struct device *dev = tx_ring->dev;
 422	u32 size;
 423
 424	/* round up to nearest page */
 425	size = ALIGN(tstamp_ring->count * sizeof(struct ice_ts_desc),
 426		     PAGE_SIZE);
 427	tstamp_ring->desc = dmam_alloc_coherent(dev, size, &tstamp_ring->dma,
 428						GFP_KERNEL);
 429	if (!tstamp_ring->desc) {
 430		dev_err(dev, "Unable to allocate memory for Time stamp Ring, size=%d\n",
 431			size);
 432		return -ENOMEM;
 433	}
 434
 435	tstamp_ring->next_to_use = 0;
 436	return 0;
 437}
 438
 439/**
 440 * ice_alloc_setup_tstamp_ring - Allocate and setup the Time Stamp ring
 441 * @tx_ring: Tx ring to allocate and setup the Time Stamp ring for
 442 *
 443 * Return: 0 on success, negative on error
 444 */
 445int ice_alloc_setup_tstamp_ring(struct ice_tx_ring *tx_ring)
 446{
 447	struct device *dev = tx_ring->dev;
 448	int err;
 449
 450	err = ice_alloc_tstamp_ring(tx_ring);
 451	if (err) {
 452		dev_err(dev, "Unable to allocate Time stamp ring for Tx ring %d\n",
 453			tx_ring->q_index);
 454		return err;
 455	}
 456
 457	err = ice_setup_tstamp_ring(tx_ring);
 458	if (err) {
 459		dev_err(dev, "Unable to setup Time stamp ring for Tx ring %d\n",
 460			tx_ring->q_index);
 461		ice_free_tx_tstamp_ring(tx_ring);
 462		return err;
 463	}
 464	return 0;
 465}
 466
 467/**
 468 * ice_setup_tx_ring - Allocate the Tx descriptors
 469 * @tx_ring: the Tx ring to set up
 470 *
 471 * Return 0 on success, negative on error
 472 */
 473int ice_setup_tx_ring(struct ice_tx_ring *tx_ring)
 474{
 475	struct device *dev = tx_ring->dev;
 476	u32 size;
 477
 478	if (!dev)
 479		return -ENOMEM;
 480
 481	/* warn if we are about to overwrite the pointer */
 482	WARN_ON(tx_ring->tx_buf);
 483	tx_ring->tx_buf =
 484		devm_kcalloc(dev, sizeof(*tx_ring->tx_buf), tx_ring->count,
 485			     GFP_KERNEL);
 486	if (!tx_ring->tx_buf)
 487		return -ENOMEM;
 488
 489	/* round up to nearest page */
 490	size = ALIGN(tx_ring->count * sizeof(struct ice_tx_desc),
 491		     PAGE_SIZE);
 492	tx_ring->desc = dmam_alloc_coherent(dev, size, &tx_ring->dma,
 493					    GFP_KERNEL);
 494	if (!tx_ring->desc) {
 495		dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
 496			size);
 497		goto err;
 498	}
 499
 500	tx_ring->next_to_use = 0;
 501	tx_ring->next_to_clean = 0;
 502	tx_ring->ring_stats->tx_stats.prev_pkt = -1;
 503	return 0;
 504
 505err:
 506	devm_kfree(dev, tx_ring->tx_buf);
 507	tx_ring->tx_buf = NULL;
 508	return -ENOMEM;
 509}
 510
 511void ice_rxq_pp_destroy(struct ice_rx_ring *rq)
 512{
 513	struct libeth_fq fq = {
 514		.fqes	= rq->rx_fqes,
 515		.pp	= rq->pp,
 516	};
 517
 518	libeth_rx_fq_destroy(&fq);
 519	rq->rx_fqes = NULL;
 520	rq->pp = NULL;
 521
 522	if (!rq->hdr_pp)
 523		return;
 524
 525	fq.fqes = rq->hdr_fqes;
 526	fq.pp = rq->hdr_pp;
 527
 528	libeth_rx_fq_destroy(&fq);
 529	rq->hdr_fqes = NULL;
 530	rq->hdr_pp = NULL;
 531}
 532
 533/**
 534 * ice_clean_rx_ring - Free Rx buffers
 535 * @rx_ring: ring to be cleaned
 536 */
 537void ice_clean_rx_ring(struct ice_rx_ring *rx_ring)
 538{
 539	u32 size;
 540
 541	if (rx_ring->xsk_pool) {
 542		ice_xsk_clean_rx_ring(rx_ring);
 543		goto rx_skip_free;
 544	}
 545
 546	/* ring already cleared, nothing to do */
 547	if (!rx_ring->rx_fqes)
 548		return;
 549
 550	libeth_xdp_return_stash(&rx_ring->xdp);
 551
 552	/* Free all the Rx ring sk_buffs */
 553	for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) {
 554		libeth_rx_recycle_slow(rx_ring->rx_fqes[i].netmem);
 555
 556		if (rx_ring->hdr_pp)
 557			libeth_rx_recycle_slow(rx_ring->hdr_fqes[i].netmem);
 558
 559		if (unlikely(++i == rx_ring->count))
 560			i = 0;
 561	}
 562
 563	if ((rx_ring->vsi->type == ICE_VSI_PF ||
 564	     rx_ring->vsi->type == ICE_VSI_SF ||
 565	     rx_ring->vsi->type == ICE_VSI_LB) &&
 566	    xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) {
 567		xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq);
 568		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 569	}
 570
 571	ice_rxq_pp_destroy(rx_ring);
 572
 573rx_skip_free:
 574	/* Zero out the descriptor ring */
 575	size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc),
 576		     PAGE_SIZE);
 577	memset(rx_ring->desc, 0, size);
 578
 579	rx_ring->next_to_alloc = 0;
 580	rx_ring->next_to_clean = 0;
 581	rx_ring->next_to_use = 0;
 582}
 583
 584/**
 585 * ice_free_rx_ring - Free Rx resources
 586 * @rx_ring: ring to clean the resources from
 587 *
 588 * Free all receive software resources
 589 */
 590void ice_free_rx_ring(struct ice_rx_ring *rx_ring)
 591{
 592	struct device *dev = ice_pf_to_dev(rx_ring->vsi->back);
 593	u32 size;
 594
 595	ice_clean_rx_ring(rx_ring);
 596	WRITE_ONCE(rx_ring->xdp_prog, NULL);
 597	if (rx_ring->xsk_pool) {
 598		kfree(rx_ring->xdp_buf);
 599		rx_ring->xdp_buf = NULL;
 600	}
 601
 602	if (rx_ring->desc) {
 603		size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc),
 604			     PAGE_SIZE);
 605		dmam_free_coherent(dev, size, rx_ring->desc, rx_ring->dma);
 606		rx_ring->desc = NULL;
 607	}
 608}
 609
 610/**
 611 * ice_setup_rx_ring - Allocate the Rx descriptors
 612 * @rx_ring: the Rx ring to set up
 613 *
 614 * Return 0 on success, negative on error
 615 */
 616int ice_setup_rx_ring(struct ice_rx_ring *rx_ring)
 617{
 618	struct device *dev = ice_pf_to_dev(rx_ring->vsi->back);
 619	u32 size;
 620
 621	/* round up to nearest page */
 622	size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc),
 623		     PAGE_SIZE);
 624	rx_ring->desc = dmam_alloc_coherent(dev, size, &rx_ring->dma,
 625					    GFP_KERNEL);
 626	if (!rx_ring->desc) {
 627		dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
 628			size);
 629		return -ENOMEM;
 630	}
 631
 632	rx_ring->next_to_use = 0;
 633	rx_ring->next_to_clean = 0;
 634
 635	if (ice_is_xdp_ena_vsi(rx_ring->vsi))
 636		WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog);
 637
 638	return 0;
 639}
 640
 641/**
 642 * ice_run_xdp - Executes an XDP program on initialized xdp_buff
 643 * @rx_ring: Rx ring
 644 * @xdp: xdp_buff used as input to the XDP program
 645 * @xdp_prog: XDP program to run
 646 * @xdp_ring: ring to be used for XDP_TX action
 647 * @eop_desc: Last descriptor in packet to read metadata from
 648 *
 649 * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR}
 650 */
 651static u32
 652ice_run_xdp(struct ice_rx_ring *rx_ring, struct libeth_xdp_buff *xdp,
 653	    struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring,
 654	    union ice_32b_rx_flex_desc *eop_desc)
 655{
 656	unsigned int ret = ICE_XDP_PASS;
 657	u32 act;
 658
 659	if (!xdp_prog)
 660		goto exit;
 661
 662	xdp->desc = eop_desc;
 663
 664	act = bpf_prog_run_xdp(xdp_prog, &xdp->base);
 665	switch (act) {
 666	case XDP_PASS:
 667		break;
 668	case XDP_TX:
 669		if (static_branch_unlikely(&ice_xdp_locking_key))
 670			spin_lock(&xdp_ring->tx_lock);
 671		ret = __ice_xmit_xdp_ring(&xdp->base, xdp_ring, false);
 672		if (static_branch_unlikely(&ice_xdp_locking_key))
 673			spin_unlock(&xdp_ring->tx_lock);
 674		if (ret == ICE_XDP_CONSUMED)
 675			goto out_failure;
 676		break;
 677	case XDP_REDIRECT:
 678		if (xdp_do_redirect(rx_ring->netdev, &xdp->base, xdp_prog))
 679			goto out_failure;
 680		ret = ICE_XDP_REDIR;
 681		break;
 682	default:
 683		bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);
 684		fallthrough;
 685	case XDP_ABORTED:
 686out_failure:
 687		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
 688		fallthrough;
 689	case XDP_DROP:
 690		libeth_xdp_return_buff(xdp);
 691		ret = ICE_XDP_CONSUMED;
 692	}
 693
 694exit:
 695	return ret;
 696}
 697
 698/**
 699 * ice_xmit_xdp_ring - submit frame to XDP ring for transmission
 700 * @xdpf: XDP frame that will be converted to XDP buff
 701 * @xdp_ring: XDP ring for transmission
 702 */
 703static int ice_xmit_xdp_ring(const struct xdp_frame *xdpf,
 704			     struct ice_tx_ring *xdp_ring)
 705{
 706	struct xdp_buff xdp;
 707
 708	xdp.data_hard_start = (void *)xdpf;
 709	xdp.data = xdpf->data;
 710	xdp.data_end = xdp.data + xdpf->len;
 711	xdp.frame_sz = xdpf->frame_sz;
 712	xdp.flags = xdpf->flags;
 713
 714	return __ice_xmit_xdp_ring(&xdp, xdp_ring, true);
 715}
 716
 717/**
 718 * ice_xdp_xmit - submit packets to XDP ring for transmission
 719 * @dev: netdev
 720 * @n: number of XDP frames to be transmitted
 721 * @frames: XDP frames to be transmitted
 722 * @flags: transmit flags
 723 *
 724 * Returns number of frames successfully sent. Failed frames
 725 * will be free'ed by XDP core.
 726 * For error cases, a negative errno code is returned and no-frames
 727 * are transmitted (caller must handle freeing frames).
 728 */
 729int
 730ice_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
 731	     u32 flags)
 732{
 733	struct ice_netdev_priv *np = netdev_priv(dev);
 734	unsigned int queue_index = smp_processor_id();
 735	struct ice_vsi *vsi = np->vsi;
 736	struct ice_tx_ring *xdp_ring;
 737	struct ice_tx_buf *tx_buf;
 738	int nxmit = 0, i;
 739
 740	if (test_bit(ICE_VSI_DOWN, vsi->state))
 741		return -ENETDOWN;
 742
 743	if (!ice_is_xdp_ena_vsi(vsi))
 744		return -ENXIO;
 745
 746	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
 747		return -EINVAL;
 748
 749	if (static_branch_unlikely(&ice_xdp_locking_key)) {
 750		queue_index %= vsi->num_xdp_txq;
 751		xdp_ring = vsi->xdp_rings[queue_index];
 752		spin_lock(&xdp_ring->tx_lock);
 753	} else {
 754		/* Generally, should not happen */
 755		if (unlikely(queue_index >= vsi->num_xdp_txq))
 756			return -ENXIO;
 757		xdp_ring = vsi->xdp_rings[queue_index];
 758	}
 759
 760	tx_buf = &xdp_ring->tx_buf[xdp_ring->next_to_use];
 761	for (i = 0; i < n; i++) {
 762		const struct xdp_frame *xdpf = frames[i];
 763		int err;
 764
 765		err = ice_xmit_xdp_ring(xdpf, xdp_ring);
 766		if (err != ICE_XDP_TX)
 767			break;
 768		nxmit++;
 769	}
 770
 771	tx_buf->rs_idx = ice_set_rs_bit(xdp_ring);
 772	if (unlikely(flags & XDP_XMIT_FLUSH))
 773		ice_xdp_ring_update_tail(xdp_ring);
 774
 775	if (static_branch_unlikely(&ice_xdp_locking_key))
 776		spin_unlock(&xdp_ring->tx_lock);
 777
 778	return nxmit;
 779}
 780
 781/**
 782 * ice_init_ctrl_rx_descs - Initialize Rx descriptors for control vsi.
 783 * @rx_ring: ring to init descriptors on
 784 * @count: number of descriptors to initialize
 785 */
 786void ice_init_ctrl_rx_descs(struct ice_rx_ring *rx_ring, u32 count)
 787{
 788	union ice_32b_rx_flex_desc *rx_desc;
 789	u32 ntu = rx_ring->next_to_use;
 790
 791	if (!count)
 792		return;
 793
 794	rx_desc = ICE_RX_DESC(rx_ring, ntu);
 795
 796	do {
 797		rx_desc++;
 798		ntu++;
 799		if (unlikely(ntu == rx_ring->count)) {
 800			rx_desc = ICE_RX_DESC(rx_ring, 0);
 801			ntu = 0;
 802		}
 803
 804		rx_desc->wb.status_error0 = 0;
 805		count--;
 806	} while (count);
 807
 808	if (rx_ring->next_to_use != ntu)
 809		ice_release_rx_desc(rx_ring, ntu);
 810}
 811
 812/**
 813 * ice_alloc_rx_bufs - Replace used receive buffers
 814 * @rx_ring: ring to place buffers on
 815 * @cleaned_count: number of buffers to replace
 816 *
 817 * Returns false if all allocations were successful, true if any fail. Returning
 818 * true signals to the caller that we didn't replace cleaned_count buffers and
 819 * there is more work to do.
 820 *
 821 * First, try to clean "cleaned_count" Rx buffers. Then refill the cleaned Rx
 822 * buffers. Then bump tail at most one time. Grouping like this lets us avoid
 823 * multiple tail writes per call.
 824 */
 825bool ice_alloc_rx_bufs(struct ice_rx_ring *rx_ring, unsigned int cleaned_count)
 826{
 827	const struct libeth_fq_fp hdr_fq = {
 828		.pp		= rx_ring->hdr_pp,
 829		.fqes		= rx_ring->hdr_fqes,
 830		.truesize	= rx_ring->hdr_truesize,
 831		.count		= rx_ring->count,
 832	};
 833	const struct libeth_fq_fp fq = {
 834		.pp		= rx_ring->pp,
 835		.fqes		= rx_ring->rx_fqes,
 836		.truesize	= rx_ring->truesize,
 837		.count		= rx_ring->count,
 838	};
 839	union ice_32b_rx_flex_desc *rx_desc;
 840	u16 ntu = rx_ring->next_to_use;
 841
 842	/* do nothing if no valid netdev defined */
 843	if (!rx_ring->netdev || !cleaned_count)
 844		return false;
 845
 846	/* get the Rx descriptor and buffer based on next_to_use */
 847	rx_desc = ICE_RX_DESC(rx_ring, ntu);
 848
 849	do {
 850		dma_addr_t addr;
 851
 852		addr = libeth_rx_alloc(&fq, ntu);
 853		if (addr == DMA_MAPPING_ERROR) {
 854			rx_ring->ring_stats->rx_stats.alloc_page_failed++;
 855			break;
 856		}
 857
 858		/* Refresh the desc even if buffer_addrs didn't change
 859		 * because each write-back erases this info.
 860		 */
 861		rx_desc->read.pkt_addr = cpu_to_le64(addr);
 862
 863		if (!hdr_fq.pp)
 864			goto next;
 865
 866		addr = libeth_rx_alloc(&hdr_fq, ntu);
 867		if (addr == DMA_MAPPING_ERROR) {
 868			rx_ring->ring_stats->rx_stats.alloc_page_failed++;
 869
 870			libeth_rx_recycle_slow(fq.fqes[ntu].netmem);
 871			break;
 872		}
 873
 874		rx_desc->read.hdr_addr = cpu_to_le64(addr);
 875
 876next:
 877		rx_desc++;
 878		ntu++;
 879		if (unlikely(ntu == rx_ring->count)) {
 880			rx_desc = ICE_RX_DESC(rx_ring, 0);
 881			ntu = 0;
 882		}
 883
 884		/* clear the status bits for the next_to_use descriptor */
 885		rx_desc->wb.status_error0 = 0;
 886
 887		cleaned_count--;
 888	} while (cleaned_count);
 889
 890	if (rx_ring->next_to_use != ntu)
 891		ice_release_rx_desc(rx_ring, ntu);
 892
 893	return !!cleaned_count;
 894}
 895
 896/**
 897 * ice_clean_ctrl_rx_irq - Clean descriptors from flow director Rx ring
 898 * @rx_ring: Rx descriptor ring for ctrl_vsi to transact packets on
 899 *
 900 * This function cleans Rx descriptors from the ctrl_vsi Rx ring used
 901 * to set flow director rules on VFs.
 902 */
 903void ice_clean_ctrl_rx_irq(struct ice_rx_ring *rx_ring)
 904{
 905	u32 ntc = rx_ring->next_to_clean;
 906	unsigned int total_rx_pkts = 0;
 907	u32 cnt = rx_ring->count;
 908
 909	while (likely(total_rx_pkts < ICE_DFLT_IRQ_WORK)) {
 910		struct ice_vsi *ctrl_vsi = rx_ring->vsi;
 911		union ice_32b_rx_flex_desc *rx_desc;
 912		u16 stat_err_bits;
 913
 914		rx_desc = ICE_RX_DESC(rx_ring, ntc);
 915
 916		stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S);
 917		if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits))
 918			break;
 919
 920		dma_rmb();
 921
 922		if (ctrl_vsi->vf)
 923			ice_vc_fdir_irq_handler(ctrl_vsi, rx_desc);
 924
 925		if (++ntc == cnt)
 926			ntc = 0;
 927		total_rx_pkts++;
 928	}
 929
 930	rx_ring->next_to_clean = ntc;
 931	ice_init_ctrl_rx_descs(rx_ring, ICE_DESC_UNUSED(rx_ring));
 932}
 933
 934/**
 935 * ice_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
 936 * @rx_ring: Rx descriptor ring to transact packets on
 937 * @budget: Total limit on number of packets to process
 938 *
 939 * This function provides a "bounce buffer" approach to Rx interrupt
 940 * processing. The advantage to this is that on systems that have
 941 * expensive overhead for IOMMU access this provides a means of avoiding
 942 * it by maintaining the mapping of the page to the system.
 943 *
 944 * Returns amount of work completed
 945 */
 946static int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
 947{
 948	unsigned int total_rx_bytes = 0, total_rx_pkts = 0;
 949	struct ice_tx_ring *xdp_ring = NULL;
 950	struct bpf_prog *xdp_prog = NULL;
 951	u32 ntc = rx_ring->next_to_clean;
 952	LIBETH_XDP_ONSTACK_BUFF(xdp);
 953	u32 cached_ntu, xdp_verdict;
 954	u32 cnt = rx_ring->count;
 955	u32 xdp_xmit = 0;
 956	bool failure;
 957
 958	libeth_xdp_init_buff(xdp, &rx_ring->xdp, &rx_ring->xdp_rxq);
 959
 960	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 961	if (xdp_prog) {
 962		xdp_ring = rx_ring->xdp_ring;
 963		cached_ntu = xdp_ring->next_to_use;
 964	}
 965
 966	/* start the loop to process Rx packets bounded by 'budget' */
 967	while (likely(total_rx_pkts < (unsigned int)budget)) {
 968		union ice_32b_rx_flex_desc *rx_desc;
 969		struct libeth_fqe *rx_buf;
 970		struct sk_buff *skb;
 971		unsigned int size;
 972		u16 stat_err_bits;
 973		u16 vlan_tci;
 974		bool rxe;
 975
 976		/* get the Rx desc from Rx ring based on 'next_to_clean' */
 977		rx_desc = ICE_RX_DESC(rx_ring, ntc);
 978
 979		/*
 980		 * The DD bit will always be zero for unused descriptors
 981		 * because it's cleared in cleanup or when setting the DMA
 982		 * address of the header buffer, which never uses the DD bit.
 983		 * If the hardware wrote the descriptor, it will be non-zero.
 984		 */
 985		stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S);
 986		if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits))
 987			break;
 988
 989		/* This memory barrier is needed to keep us from reading
 990		 * any other fields out of the rx_desc until we know the
 991		 * DD bit is set.
 992		 */
 993		dma_rmb();
 994
 995		ice_trace(clean_rx_irq, rx_ring, rx_desc);
 996
 997		stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_HBO_S) |
 998				BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S);
 999		rxe = ice_test_staterr(rx_desc->wb.status_error0,
1000				       stat_err_bits);
1001
1002		if (!rx_ring->hdr_pp)
1003			goto payload;
1004
1005		size = le16_get_bits(rx_desc->wb.hdr_len_sph_flex_flags1,
1006				     ICE_RX_FLEX_DESC_HDR_LEN_M);
1007		if (unlikely(rxe))
1008			size = 0;
1009
1010		rx_buf = &rx_ring->hdr_fqes[ntc];
1011		libeth_xdp_process_buff(xdp, rx_buf, size);
1012		rx_buf->netmem = 0;
1013
1014payload:
1015		size = le16_to_cpu(rx_desc->wb.pkt_len) &
1016			ICE_RX_FLX_DESC_PKT_LEN_M;
1017		if (unlikely(rxe))
1018			size = 0;
1019
1020		/* retrieve a buffer from the ring */
1021		rx_buf = &rx_ring->rx_fqes[ntc];
1022		libeth_xdp_process_buff(xdp, rx_buf, size);
1023
1024		if (++ntc == cnt)
1025			ntc = 0;
1026
1027		/* skip if it is NOP desc */
1028		if (ice_is_non_eop(rx_ring, rx_desc) || unlikely(!xdp->data))
1029			continue;
1030
1031		xdp_verdict = ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_desc);
1032		if (xdp_verdict == ICE_XDP_PASS)
1033			goto construct_skb;
1034
1035		if (xdp_verdict & (ICE_XDP_TX | ICE_XDP_REDIR))
1036			xdp_xmit |= xdp_verdict;
1037		total_rx_bytes += xdp_get_buff_len(&xdp->base);
1038		total_rx_pkts++;
1039
1040		xdp->data = NULL;
1041		continue;
1042
1043construct_skb:
1044		skb = xdp_build_skb_from_buff(&xdp->base);
1045		xdp->data = NULL;
1046
1047		/* exit if we failed to retrieve a buffer */
1048		if (!skb) {
1049			libeth_xdp_return_buff_slow(xdp);
1050			rx_ring->ring_stats->rx_stats.alloc_buf_failed++;
1051			continue;
1052		}
1053
1054		vlan_tci = ice_get_vlan_tci(rx_desc);
1055
1056		/* probably a little skewed due to removing CRC */
1057		total_rx_bytes += skb->len;
1058
1059		/* populate checksum, VLAN, and protocol */
1060		ice_process_skb_fields(rx_ring, rx_desc, skb);
1061
1062		ice_trace(clean_rx_irq_indicate, rx_ring, rx_desc, skb);
1063		/* send completed skb up the stack */
1064		ice_receive_skb(rx_ring, skb, vlan_tci);
1065
1066		/* update budget accounting */
1067		total_rx_pkts++;
1068	}
1069
1070	rx_ring->next_to_clean = ntc;
1071	/* return up to cleaned_count buffers to hardware */
1072	failure = ice_alloc_rx_bufs(rx_ring, ICE_DESC_UNUSED(rx_ring));
1073
1074	if (xdp_xmit)
1075		ice_finalize_xdp_rx(xdp_ring, xdp_xmit, cached_ntu);
1076
1077	libeth_xdp_save_buff(&rx_ring->xdp, xdp);
1078
1079	if (rx_ring->ring_stats)
1080		ice_update_rx_ring_stats(rx_ring, total_rx_pkts,
1081					 total_rx_bytes);
1082
1083	/* guarantee a trip back through this routine if there was a failure */
1084	return failure ? budget : (int)total_rx_pkts;
1085}
1086
1087static void __ice_update_sample(struct ice_q_vector *q_vector,
1088				struct ice_ring_container *rc,
1089				struct dim_sample *sample,
1090				bool is_tx)
1091{
1092	u64 packets = 0, bytes = 0;
1093
1094	if (is_tx) {
1095		struct ice_tx_ring *tx_ring;
1096
1097		ice_for_each_tx_ring(tx_ring, *rc) {
1098			struct ice_ring_stats *ring_stats;
1099
1100			ring_stats = tx_ring->ring_stats;
1101			if (!ring_stats)
1102				continue;
1103			packets += ring_stats->stats.pkts;
1104			bytes += ring_stats->stats.bytes;
1105		}
1106	} else {
1107		struct ice_rx_ring *rx_ring;
1108
1109		ice_for_each_rx_ring(rx_ring, *rc) {
1110			struct ice_ring_stats *ring_stats;
1111
1112			ring_stats = rx_ring->ring_stats;
1113			if (!ring_stats)
1114				continue;
1115			packets += ring_stats->stats.pkts;
1116			bytes += ring_stats->stats.bytes;
1117		}
1118	}
1119
1120	dim_update_sample(q_vector->total_events, packets, bytes, sample);
1121	sample->comp_ctr = 0;
1122
1123	/* if dim settings get stale, like when not updated for 1
1124	 * second or longer, force it to start again. This addresses the
1125	 * frequent case of an idle queue being switched to by the
1126	 * scheduler. The 1,000 here means 1,000 milliseconds.
1127	 */
1128	if (ktime_ms_delta(sample->time, rc->dim.start_sample.time) >= 1000)
1129		rc->dim.state = DIM_START_MEASURE;
1130}
1131
1132/**
1133 * ice_net_dim - Update net DIM algorithm
1134 * @q_vector: the vector associated with the interrupt
1135 *
1136 * Create a DIM sample and notify net_dim() so that it can possibly decide
1137 * a new ITR value based on incoming packets, bytes, and interrupts.
1138 *
1139 * This function is a no-op if the ring is not configured to dynamic ITR.
1140 */
1141static void ice_net_dim(struct ice_q_vector *q_vector)
1142{
1143	struct ice_ring_container *tx = &q_vector->tx;
1144	struct ice_ring_container *rx = &q_vector->rx;
1145
1146	if (ITR_IS_DYNAMIC(tx)) {
1147		struct dim_sample dim_sample;
1148
1149		__ice_update_sample(q_vector, tx, &dim_sample, true);
1150		net_dim(&tx->dim, &dim_sample);
1151	}
1152
1153	if (ITR_IS_DYNAMIC(rx)) {
1154		struct dim_sample dim_sample;
1155
1156		__ice_update_sample(q_vector, rx, &dim_sample, false);
1157		net_dim(&rx->dim, &dim_sample);
1158	}
1159}
1160
1161/**
1162 * ice_buildreg_itr - build value for writing to the GLINT_DYN_CTL register
1163 * @itr_idx: interrupt throttling index
1164 * @itr: interrupt throttling value in usecs
1165 */
1166static u32 ice_buildreg_itr(u16 itr_idx, u16 itr)
1167{
1168	/* The ITR value is reported in microseconds, and the register value is
1169	 * recorded in 2 microsecond units. For this reason we only need to
1170	 * shift by the GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S to apply this
1171	 * granularity as a shift instead of division. The mask makes sure the
1172	 * ITR value is never odd so we don't accidentally write into the field
1173	 * prior to the ITR field.
1174	 */
1175	itr &= ICE_ITR_MASK;
1176
1177	return GLINT_DYN_CTL_INTENA_M | GLINT_DYN_CTL_CLEARPBA_M |
1178		(itr_idx << GLINT_DYN_CTL_ITR_INDX_S) |
1179		(itr << (GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S));
1180}
1181
1182/**
1183 * ice_enable_interrupt - re-enable MSI-X interrupt
1184 * @q_vector: the vector associated with the interrupt to enable
1185 *
1186 * If the VSI is down, the interrupt will not be re-enabled. Also,
1187 * when enabling the interrupt always reset the wb_on_itr to false
1188 * and trigger a software interrupt to clean out internal state.
1189 */
1190static void ice_enable_interrupt(struct ice_q_vector *q_vector)
1191{
1192	struct ice_vsi *vsi = q_vector->vsi;
1193	bool wb_en = q_vector->wb_on_itr;
1194	u32 itr_val;
1195
1196	if (test_bit(ICE_DOWN, vsi->state))
1197		return;
1198
1199	/* trigger an ITR delayed software interrupt when exiting busy poll, to
1200	 * make sure to catch any pending cleanups that might have been missed
1201	 * due to interrupt state transition. If busy poll or poll isn't
1202	 * enabled, then don't update ITR, and just enable the interrupt.
1203	 */
1204	if (!wb_en) {
1205		itr_val = ice_buildreg_itr(ICE_ITR_NONE, 0);
1206	} else {
1207		q_vector->wb_on_itr = false;
1208
1209		/* do two things here with a single write. Set up the third ITR
1210		 * index to be used for software interrupt moderation, and then
1211		 * trigger a software interrupt with a rate limit of 20K on
1212		 * software interrupts, this will help avoid high interrupt
1213		 * loads due to frequently polling and exiting polling.
1214		 */
1215		itr_val = ice_buildreg_itr(ICE_IDX_ITR2, ICE_ITR_20K);
1216		itr_val |= GLINT_DYN_CTL_SWINT_TRIG_M |
1217			   ICE_IDX_ITR2 << GLINT_DYN_CTL_SW_ITR_INDX_S |
1218			   GLINT_DYN_CTL_SW_ITR_INDX_ENA_M;
1219	}
1220	wr32(&vsi->back->hw, GLINT_DYN_CTL(q_vector->reg_idx), itr_val);
1221}
1222
1223/**
1224 * ice_set_wb_on_itr - set WB_ON_ITR for this q_vector
1225 * @q_vector: q_vector to set WB_ON_ITR on
1226 *
1227 * We need to tell hardware to write-back completed descriptors even when
1228 * interrupts are disabled. Descriptors will be written back on cache line
1229 * boundaries without WB_ON_ITR enabled, but if we don't enable WB_ON_ITR
1230 * descriptors may not be written back if they don't fill a cache line until
1231 * the next interrupt.
1232 *
1233 * This sets the write-back frequency to whatever was set previously for the
1234 * ITR indices. Also, set the INTENA_MSK bit to make sure hardware knows we
1235 * aren't meddling with the INTENA_M bit.
1236 */
1237static void ice_set_wb_on_itr(struct ice_q_vector *q_vector)
1238{
1239	struct ice_vsi *vsi = q_vector->vsi;
1240
1241	/* already in wb_on_itr mode no need to change it */
1242	if (q_vector->wb_on_itr)
1243		return;
1244
1245	/* use previously set ITR values for all of the ITR indices by
1246	 * specifying ICE_ITR_NONE, which will vary in adaptive (AIM) mode and
1247	 * be static in non-adaptive mode (user configured)
1248	 */
1249	wr32(&vsi->back->hw, GLINT_DYN_CTL(q_vector->reg_idx),
1250	     FIELD_PREP(GLINT_DYN_CTL_ITR_INDX_M, ICE_ITR_NONE) |
1251	     FIELD_PREP(GLINT_DYN_CTL_INTENA_MSK_M, 1) |
1252	     FIELD_PREP(GLINT_DYN_CTL_WB_ON_ITR_M, 1));
1253
1254	q_vector->wb_on_itr = true;
1255}
1256
1257/**
1258 * ice_napi_poll - NAPI polling Rx/Tx cleanup routine
1259 * @napi: napi struct with our devices info in it
1260 * @budget: amount of work driver is allowed to do this pass, in packets
1261 *
1262 * This function will clean all queues associated with a q_vector.
1263 *
1264 * Returns the amount of work done
1265 */
1266int ice_napi_poll(struct napi_struct *napi, int budget)
1267{
1268	struct ice_q_vector *q_vector =
1269				container_of(napi, struct ice_q_vector, napi);
1270	struct ice_tx_ring *tx_ring;
1271	struct ice_rx_ring *rx_ring;
1272	bool clean_complete = true;
1273	int budget_per_ring;
1274	int work_done = 0;
1275
1276	/* Since the actual Tx work is minimal, we can give the Tx a larger
1277	 * budget and be more aggressive about cleaning up the Tx descriptors.
1278	 */
1279	ice_for_each_tx_ring(tx_ring, q_vector->tx) {
1280		struct xsk_buff_pool *xsk_pool = READ_ONCE(tx_ring->xsk_pool);
1281		bool wd;
1282
1283		if (xsk_pool)
1284			wd = ice_xmit_zc(tx_ring, xsk_pool);
1285		else if (ice_ring_is_xdp(tx_ring))
1286			wd = true;
1287		else
1288			wd = ice_clean_tx_irq(tx_ring, budget);
1289
1290		if (!wd)
1291			clean_complete = false;
1292	}
1293
1294	/* Handle case where we are called by netpoll with a budget of 0 */
1295	if (unlikely(budget <= 0))
1296		return budget;
1297
1298	/* normally we have 1 Rx ring per q_vector */
1299	if (unlikely(q_vector->num_ring_rx > 1))
1300		/* We attempt to distribute budget to each Rx queue fairly, but
1301		 * don't allow the budget to go below 1 because that would exit
1302		 * polling early.
1303		 */
1304		budget_per_ring = max_t(int, budget / q_vector->num_ring_rx, 1);
1305	else
1306		/* Max of 1 Rx ring in this q_vector so give it the budget */
1307		budget_per_ring = budget;
1308
1309	ice_for_each_rx_ring(rx_ring, q_vector->rx) {
1310		struct xsk_buff_pool *xsk_pool = READ_ONCE(rx_ring->xsk_pool);
1311		int cleaned;
1312
1313		/* A dedicated path for zero-copy allows making a single
1314		 * comparison in the irq context instead of many inside the
1315		 * ice_clean_rx_irq function and makes the codebase cleaner.
1316		 */
1317		cleaned = rx_ring->xsk_pool ?
1318			  ice_clean_rx_irq_zc(rx_ring, xsk_pool, budget_per_ring) :
1319			  ice_clean_rx_irq(rx_ring, budget_per_ring);
1320		work_done += cleaned;
1321		/* if we clean as many as budgeted, we must not be done */
1322		if (cleaned >= budget_per_ring)
1323			clean_complete = false;
1324	}
1325
1326	/* If work not completed, return budget and polling will return */
1327	if (!clean_complete) {
1328		/* Set the writeback on ITR so partial completions of
1329		 * cache-lines will still continue even if we're polling.
1330		 */
1331		ice_set_wb_on_itr(q_vector);
1332		return budget;
1333	}
1334
1335	/* Exit the polling mode, but don't re-enable interrupts if stack might
1336	 * poll us due to busy-polling
1337	 */
1338	if (napi_complete_done(napi, work_done)) {
1339		ice_net_dim(q_vector);
1340		ice_enable_interrupt(q_vector);
1341	} else {
1342		ice_set_wb_on_itr(q_vector);
1343	}
1344
1345	return min_t(int, work_done, budget - 1);
1346}
1347
1348/**
1349 * __ice_maybe_stop_tx - 2nd level check for Tx stop conditions
1350 * @tx_ring: the ring to be checked
1351 * @size: the size buffer we want to assure is available
1352 *
1353 * Returns -EBUSY if a stop is needed, else 0
1354 */
1355static int __ice_maybe_stop_tx(struct ice_tx_ring *tx_ring, unsigned int size)
1356{
1357	netif_tx_stop_queue(txring_txq(tx_ring));
1358	/* Memory barrier before checking head and tail */
1359	smp_mb();
1360
1361	/* Check again in a case another CPU has just made room available. */
1362	if (likely(ICE_DESC_UNUSED(tx_ring) < size))
1363		return -EBUSY;
1364
1365	/* A reprieve! - use start_queue because it doesn't call schedule */
1366	netif_tx_start_queue(txring_txq(tx_ring));
1367	++tx_ring->ring_stats->tx_stats.restart_q;
1368	return 0;
1369}
1370
1371/**
1372 * ice_maybe_stop_tx - 1st level check for Tx stop conditions
1373 * @tx_ring: the ring to be checked
1374 * @size:    the size buffer we want to assure is available
1375 *
1376 * Returns 0 if stop is not needed
1377 */
1378static int ice_maybe_stop_tx(struct ice_tx_ring *tx_ring, unsigned int size)
1379{
1380	if (likely(ICE_DESC_UNUSED(tx_ring) >= size))
1381		return 0;
1382
1383	return __ice_maybe_stop_tx(tx_ring, size);
1384}
1385
1386/**
1387 * ice_tx_map - Build the Tx descriptor
1388 * @tx_ring: ring to send buffer on
1389 * @first: first buffer info buffer to use
1390 * @off: pointer to struct that holds offload parameters
1391 *
1392 * This function loops over the skb data pointed to by *first
1393 * and gets a physical address for each memory location and programs
1394 * it and the length into the transmit descriptor.
1395 */
1396static void
1397ice_tx_map(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first,
1398	   struct ice_tx_offload_params *off)
1399{
1400	u64 td_offset, td_tag, td_cmd;
1401	u16 i = tx_ring->next_to_use;
1402	unsigned int data_len, size;
1403	struct ice_tx_desc *tx_desc;
1404	struct ice_tx_buf *tx_buf;
1405	struct sk_buff *skb;
1406	skb_frag_t *frag;
1407	dma_addr_t dma;
1408	bool kick;
1409
1410	td_tag = off->td_l2tag1;
1411	td_cmd = off->td_cmd;
1412	td_offset = off->td_offset;
1413	skb = first->skb;
1414
1415	data_len = skb->data_len;
1416	size = skb_headlen(skb);
1417
1418	tx_desc = ICE_TX_DESC(tx_ring, i);
1419
1420	if (first->tx_flags & ICE_TX_FLAGS_HW_VLAN) {
1421		td_cmd |= (u64)ICE_TX_DESC_CMD_IL2TAG1;
1422		td_tag = first->vid;
1423	}
1424
1425	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1426
1427	tx_buf = first;
1428
1429	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1430		unsigned int max_data = ICE_MAX_DATA_PER_TXD_ALIGNED;
1431
1432		if (dma_mapping_error(tx_ring->dev, dma))
1433			goto dma_error;
1434
1435		/* record length, and DMA address */
1436		dma_unmap_len_set(tx_buf, len, size);
1437		dma_unmap_addr_set(tx_buf, dma, dma);
1438
1439		/* align size to end of page */
1440		max_data += -dma & (ICE_MAX_READ_REQ_SIZE - 1);
1441		tx_desc->buf_addr = cpu_to_le64(dma);
1442
1443		/* account for data chunks larger than the hardware
1444		 * can handle
1445		 */
1446		while (unlikely(size > ICE_MAX_DATA_PER_TXD)) {
1447			tx_desc->cmd_type_offset_bsz =
1448				ice_build_ctob(td_cmd, td_offset, max_data,
1449					       td_tag);
1450
1451			tx_desc++;
1452			i++;
1453
1454			if (i == tx_ring->count) {
1455				tx_desc = ICE_TX_DESC(tx_ring, 0);
1456				i = 0;
1457			}
1458
1459			dma += max_data;
1460			size -= max_data;
1461
1462			max_data = ICE_MAX_DATA_PER_TXD_ALIGNED;
1463			tx_desc->buf_addr = cpu_to_le64(dma);
1464		}
1465
1466		if (likely(!data_len))
1467			break;
1468
1469		tx_desc->cmd_type_offset_bsz = ice_build_ctob(td_cmd, td_offset,
1470							      size, td_tag);
1471
1472		tx_desc++;
1473		i++;
1474
1475		if (i == tx_ring->count) {
1476			tx_desc = ICE_TX_DESC(tx_ring, 0);
1477			i = 0;
1478		}
1479
1480		size = skb_frag_size(frag);
1481		data_len -= size;
1482
1483		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1484				       DMA_TO_DEVICE);
1485
1486		tx_buf = &tx_ring->tx_buf[i];
1487		tx_buf->type = ICE_TX_BUF_FRAG;
1488	}
1489
1490	/* record SW timestamp if HW timestamp is not available */
1491	skb_tx_timestamp(first->skb);
1492
1493	i++;
1494	if (i == tx_ring->count)
1495		i = 0;
1496
1497	/* write last descriptor with RS and EOP bits */
1498	td_cmd |= (u64)ICE_TXD_LAST_DESC_CMD;
1499	tx_desc->cmd_type_offset_bsz =
1500			ice_build_ctob(td_cmd, td_offset, size, td_tag);
1501
1502	/* Force memory writes to complete before letting h/w know there
1503	 * are new descriptors to fetch.
1504	 *
1505	 * We also use this memory barrier to make certain all of the
1506	 * status bits have been updated before next_to_watch is written.
1507	 */
1508	wmb();
1509
1510	/* set next_to_watch value indicating a packet is present */
1511	first->next_to_watch = tx_desc;
1512
1513	tx_ring->next_to_use = i;
1514
1515	ice_maybe_stop_tx(tx_ring, DESC_NEEDED);
1516
1517	/* notify HW of packet */
1518	kick = __netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount,
1519				      netdev_xmit_more());
1520	if (!kick)
1521		return;
1522
1523	if (ice_is_txtime_cfg(tx_ring)) {
1524		struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring;
1525		u32 tstamp_count = tstamp_ring->count;
1526		u32 j = tstamp_ring->next_to_use;
1527		struct ice_ts_desc *ts_desc;
1528		struct timespec64 ts;
1529		u32 tstamp;
1530
1531		ts = ktime_to_timespec64(first->skb->tstamp);
1532		tstamp = ts.tv_nsec >> ICE_TXTIME_CTX_RESOLUTION_128NS;
1533
1534		ts_desc = ICE_TS_DESC(tstamp_ring, j);
1535		ts_desc->tx_desc_idx_tstamp = ice_build_tstamp_desc(i, tstamp);
1536
1537		j++;
1538		if (j == tstamp_count) {
1539			u32 fetch = tstamp_count - tx_ring->count;
1540
1541			j = 0;
1542
1543			/* To prevent an MDD, when wrapping the tstamp ring
1544			 * create additional TS descriptors equal to the number
1545			 * of the fetch TS descriptors value. HW will merge the
1546			 * TS descriptors with the same timestamp value into a
1547			 * single descriptor.
1548			 */
1549			for (; j < fetch; j++) {
1550				ts_desc = ICE_TS_DESC(tstamp_ring, j);
1551				ts_desc->tx_desc_idx_tstamp =
1552				       ice_build_tstamp_desc(i, tstamp);
1553			}
1554		}
1555		tstamp_ring->next_to_use = j;
1556		writel_relaxed(j, tstamp_ring->tail);
1557	} else {
1558		writel_relaxed(i, tx_ring->tail);
1559	}
1560	return;
1561
1562dma_error:
1563	/* clear DMA mappings for failed tx_buf map */
1564	for (;;) {
1565		tx_buf = &tx_ring->tx_buf[i];
1566		ice_unmap_and_free_tx_buf(tx_ring, tx_buf);
1567		if (tx_buf == first)
1568			break;
1569		if (i == 0)
1570			i = tx_ring->count;
1571		i--;
1572	}
1573
1574	tx_ring->next_to_use = i;
1575}
1576
1577/**
1578 * ice_tx_csum - Enable Tx checksum offloads
1579 * @first: pointer to the first descriptor
1580 * @off: pointer to struct that holds offload parameters
1581 *
1582 * Returns 0 or error (negative) if checksum offload can't happen, 1 otherwise.
1583 */
1584static
1585int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off)
1586{
1587	const struct ice_tx_ring *tx_ring = off->tx_ring;
1588	u32 l4_len = 0, l3_len = 0, l2_len = 0;
1589	struct sk_buff *skb = first->skb;
1590	union {
1591		struct iphdr *v4;
1592		struct ipv6hdr *v6;
1593		unsigned char *hdr;
1594	} ip;
1595	union {
1596		struct tcphdr *tcp;
1597		unsigned char *hdr;
1598	} l4;
1599	__be16 frag_off, protocol;
1600	unsigned char *exthdr;
1601	u32 offset, cmd = 0;
1602	u8 l4_proto = 0;
1603
1604	if (skb->ip_summed != CHECKSUM_PARTIAL)
1605		return 0;
1606
1607	protocol = vlan_get_protocol(skb);
1608
1609	if (eth_p_mpls(protocol)) {
1610		ip.hdr = skb_inner_network_header(skb);
1611		l4.hdr = skb_checksum_start(skb);
1612	} else {
1613		ip.hdr = skb_network_header(skb);
1614		l4.hdr = skb_transport_header(skb);
1615	}
1616
1617	/* compute outer L2 header size */
1618	l2_len = ip.hdr - skb->data;
1619	offset = (l2_len / 2) << ICE_TX_DESC_LEN_MACLEN_S;
1620
1621	/* set the tx_flags to indicate the IP protocol type. this is
1622	 * required so that checksum header computation below is accurate.
1623	 */
1624	if (ip.v4->version == 4)
1625		first->tx_flags |= ICE_TX_FLAGS_IPV4;
1626	else if (ip.v6->version == 6)
1627		first->tx_flags |= ICE_TX_FLAGS_IPV6;
1628
1629	if (skb->encapsulation) {
1630		bool gso_ena = false;
1631		u32 tunnel = 0;
1632
1633		/* define outer network header type */
1634		if (first->tx_flags & ICE_TX_FLAGS_IPV4) {
1635			tunnel |= (first->tx_flags & ICE_TX_FLAGS_TSO) ?
1636				  ICE_TX_CTX_EIPT_IPV4 :
1637				  ICE_TX_CTX_EIPT_IPV4_NO_CSUM;
1638			l4_proto = ip.v4->protocol;
1639		} else if (first->tx_flags & ICE_TX_FLAGS_IPV6) {
1640			int ret;
1641
1642			tunnel |= ICE_TX_CTX_EIPT_IPV6;
1643			exthdr = ip.hdr + sizeof(*ip.v6);
1644			l4_proto = ip.v6->nexthdr;
1645			ret = ipv6_skip_exthdr(skb, exthdr - skb->data,
1646					       &l4_proto, &frag_off);
1647			if (ret < 0)
1648				return -1;
1649		}
1650
1651		/* define outer transport */
1652		switch (l4_proto) {
1653		case IPPROTO_UDP:
1654			tunnel |= ICE_TXD_CTX_UDP_TUNNELING;
1655			first->tx_flags |= ICE_TX_FLAGS_TUNNEL;
1656			break;
1657		case IPPROTO_GRE:
1658			tunnel |= ICE_TXD_CTX_GRE_TUNNELING;
1659			first->tx_flags |= ICE_TX_FLAGS_TUNNEL;
1660			break;
1661		case IPPROTO_IPIP:
1662		case IPPROTO_IPV6:
1663			first->tx_flags |= ICE_TX_FLAGS_TUNNEL;
1664			l4.hdr = skb_inner_network_header(skb);
1665			break;
1666		default:
1667			if (first->tx_flags & ICE_TX_FLAGS_TSO)
1668				return -1;
1669
1670			skb_checksum_help(skb);
1671			return 0;
1672		}
1673
1674		/* compute outer L3 header size */
1675		tunnel |= ((l4.hdr - ip.hdr) / 4) <<
1676			  ICE_TXD_CTX_QW0_EIPLEN_S;
1677
1678		/* switch IP header pointer from outer to inner header */
1679		ip.hdr = skb_inner_network_header(skb);
1680
1681		/* compute tunnel header size */
1682		tunnel |= ((ip.hdr - l4.hdr) / 2) <<
1683			   ICE_TXD_CTX_QW0_NATLEN_S;
1684
1685		gso_ena = skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL;
1686		/* indicate if we need to offload outer UDP header */
1687		if ((first->tx_flags & ICE_TX_FLAGS_TSO) && !gso_ena &&
1688		    (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
1689			tunnel |= ICE_TXD_CTX_QW0_L4T_CS_M;
1690
1691		/* record tunnel offload values */
1692		off->cd_tunnel_params |= tunnel;
1693
1694		/* set DTYP=1 to indicate that it's an Tx context descriptor
1695		 * in IPsec tunnel mode with Tx offloads in Quad word 1
1696		 */
1697		off->cd_qw1 |= (u64)ICE_TX_DESC_DTYPE_CTX;
1698
1699		/* switch L4 header pointer from outer to inner */
1700		l4.hdr = skb_inner_transport_header(skb);
1701		l4_proto = 0;
1702
1703		/* reset type as we transition from outer to inner headers */
1704		first->tx_flags &= ~(ICE_TX_FLAGS_IPV4 | ICE_TX_FLAGS_IPV6);
1705		if (ip.v4->version == 4)
1706			first->tx_flags |= ICE_TX_FLAGS_IPV4;
1707		if (ip.v6->version == 6)
1708			first->tx_flags |= ICE_TX_FLAGS_IPV6;
1709	}
1710
1711	/* Enable IP checksum offloads */
1712	if (first->tx_flags & ICE_TX_FLAGS_IPV4) {
1713		l4_proto = ip.v4->protocol;
1714		/* the stack computes the IP header already, the only time we
1715		 * need the hardware to recompute it is in the case of TSO.
1716		 */
1717		if (first->tx_flags & ICE_TX_FLAGS_TSO)
1718			cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;
1719		else
1720			cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;
1721
1722	} else if (first->tx_flags & ICE_TX_FLAGS_IPV6) {
1723		cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;
1724		exthdr = ip.hdr + sizeof(*ip.v6);
1725		l4_proto = ip.v6->nexthdr;
1726		if (l4.hdr != exthdr)
1727			ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_proto,
1728					 &frag_off);
1729	} else {
1730		return -1;
1731	}
1732
1733	/* compute inner L3 header size */
1734	l3_len = l4.hdr - ip.hdr;
1735	offset |= (l3_len / 4) << ICE_TX_DESC_LEN_IPLEN_S;
1736
1737	if ((tx_ring->netdev->features & NETIF_F_HW_CSUM) &&
1738	    !(first->tx_flags & ICE_TX_FLAGS_TSO) &&
1739	    !skb_csum_is_sctp(skb)) {
1740		/* Set GCS */
1741		u16 csum_start = (skb->csum_start - skb->mac_header) / 2;
1742		u16 csum_offset = skb->csum_offset / 2;
1743		u16 gcs_params;
1744
1745		gcs_params = FIELD_PREP(ICE_TX_GCS_DESC_START_M, csum_start) |
1746			     FIELD_PREP(ICE_TX_GCS_DESC_OFFSET_M, csum_offset) |
1747			     FIELD_PREP(ICE_TX_GCS_DESC_TYPE_M,
1748					ICE_TX_GCS_DESC_CSUM_PSH);
1749
1750		/* Unlike legacy HW checksums, GCS requires a context
1751		 * descriptor.
1752		 */
1753		off->cd_qw1 |= ICE_TX_DESC_DTYPE_CTX;
1754		off->cd_gcs_params = gcs_params;
1755		/* Fill out CSO info in data descriptors */
1756		off->td_offset |= offset;
1757		off->td_cmd |= cmd;
1758		return 1;
1759	}
1760
1761	/* Enable L4 checksum offloads */
1762	switch (l4_proto) {
1763	case IPPROTO_TCP:
1764		/* enable checksum offloads */
1765		cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;
1766		l4_len = l4.tcp->doff;
1767		offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S;
1768		break;
1769	case IPPROTO_UDP:
1770		/* enable UDP checksum offload */
1771		cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;
1772		l4_len = (sizeof(struct udphdr) >> 2);
1773		offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S;
1774		break;
1775	case IPPROTO_SCTP:
1776		/* enable SCTP checksum offload */
1777		cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;
1778		l4_len = sizeof(struct sctphdr) >> 2;
1779		offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S;
1780		break;
1781
1782	default:
1783		if (first->tx_flags & ICE_TX_FLAGS_TSO)
1784			return -1;
1785		skb_checksum_help(skb);
1786		return 0;
1787	}
1788
1789	off->td_cmd |= cmd;
1790	off->td_offset |= offset;
1791	return 1;
1792}
1793
1794/**
1795 * ice_tx_prepare_vlan_flags - prepare generic Tx VLAN tagging flags for HW
1796 * @tx_ring: ring to send buffer on
1797 * @first: pointer to struct ice_tx_buf
1798 *
1799 * Checks the skb and set up correspondingly several generic transmit flags
1800 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1801 */
1802static void
1803ice_tx_prepare_vlan_flags(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first)
1804{
1805	struct sk_buff *skb = first->skb;
1806
1807	/* nothing left to do, software offloaded VLAN */
1808	if (!skb_vlan_tag_present(skb) && eth_type_vlan(skb->protocol))
1809		return;
1810
1811	/* the VLAN ethertype/tpid is determined by VSI configuration and netdev
1812	 * feature flags, which the driver only allows either 802.1Q or 802.1ad
1813	 * VLAN offloads exclusively so we only care about the VLAN ID here
1814	 */
1815	if (skb_vlan_tag_present(skb)) {
1816		first->vid = skb_vlan_tag_get(skb);
1817		if (tx_ring->flags & ICE_TX_FLAGS_RING_VLAN_L2TAG2)
1818			first->tx_flags |= ICE_TX_FLAGS_HW_OUTER_SINGLE_VLAN;
1819		else
1820			first->tx_flags |= ICE_TX_FLAGS_HW_VLAN;
1821	}
1822
1823	ice_tx_prepare_vlan_flags_dcb(tx_ring, first);
1824}
1825
1826/**
1827 * ice_tso - computes mss and TSO length to prepare for TSO
1828 * @first: pointer to struct ice_tx_buf
1829 * @off: pointer to struct that holds offload parameters
1830 *
1831 * Returns 0 or error (negative) if TSO can't happen, 1 otherwise.
1832 */
1833static
1834int ice_tso(struct ice_tx_buf *first, struct ice_tx_offload_params *off)
1835{
1836	struct sk_buff *skb = first->skb;
1837	union {
1838		struct iphdr *v4;
1839		struct ipv6hdr *v6;
1840		unsigned char *hdr;
1841	} ip;
1842	union {
1843		struct tcphdr *tcp;
1844		struct udphdr *udp;
1845		unsigned char *hdr;
1846	} l4;
1847	u64 cd_mss, cd_tso_len;
1848	__be16 protocol;
1849	u32 paylen;
1850	u8 l4_start;
1851	int err;
1852
1853	if (skb->ip_summed != CHECKSUM_PARTIAL)
1854		return 0;
1855
1856	if (!skb_is_gso(skb))
1857		return 0;
1858
1859	err = skb_cow_head(skb, 0);
1860	if (err < 0)
1861		return err;
1862
1863	protocol = vlan_get_protocol(skb);
1864
1865	if (eth_p_mpls(protocol))
1866		ip.hdr = skb_inner_network_header(skb);
1867	else
1868		ip.hdr = skb_network_header(skb);
1869	l4.hdr = skb_checksum_start(skb);
1870
1871	/* initialize outer IP header fields */
1872	if (ip.v4->version == 4) {
1873		ip.v4->tot_len = 0;
1874		ip.v4->check = 0;
1875	} else {
1876		ip.v6->payload_len = 0;
1877	}
1878
1879	if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1880					 SKB_GSO_GRE_CSUM |
1881					 SKB_GSO_IPXIP4 |
1882					 SKB_GSO_IPXIP6 |
1883					 SKB_GSO_UDP_TUNNEL |
1884					 SKB_GSO_UDP_TUNNEL_CSUM)) {
1885		if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1886		    (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
1887			l4.udp->len = 0;
1888
1889			/* determine offset of outer transport header */
1890			l4_start = (u8)(l4.hdr - skb->data);
1891
1892			/* remove payload length from outer checksum */
1893			paylen = skb->len - l4_start;
1894			csum_replace_by_diff(&l4.udp->check,
1895					     (__force __wsum)htonl(paylen));
1896		}
1897
1898		/* reset pointers to inner headers */
1899		ip.hdr = skb_inner_network_header(skb);
1900		l4.hdr = skb_inner_transport_header(skb);
1901
1902		/* initialize inner IP header fields */
1903		if (ip.v4->version == 4) {
1904			ip.v4->tot_len = 0;
1905			ip.v4->check = 0;
1906		} else {
1907			ip.v6->payload_len = 0;
1908		}
1909	}
1910
1911	/* determine offset of transport header */
1912	l4_start = (u8)(l4.hdr - skb->data);
1913
1914	/* remove payload length from checksum */
1915	paylen = skb->len - l4_start;
1916
1917	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
1918		csum_replace_by_diff(&l4.udp->check,
1919				     (__force __wsum)htonl(paylen));
1920		/* compute length of UDP segmentation header */
1921		off->header_len = (u8)sizeof(l4.udp) + l4_start;
1922	} else {
1923		csum_replace_by_diff(&l4.tcp->check,
1924				     (__force __wsum)htonl(paylen));
1925		/* compute length of TCP segmentation header */
1926		off->header_len = (u8)((l4.tcp->doff * 4) + l4_start);
1927	}
1928
1929	/* update gso_segs and bytecount */
1930	first->gso_segs = skb_shinfo(skb)->gso_segs;
1931	first->bytecount += (first->gso_segs - 1) * off->header_len;
1932
1933	cd_tso_len = skb->len - off->header_len;
1934	cd_mss = skb_shinfo(skb)->gso_size;
1935
1936	/* record cdesc_qw1 with TSO parameters */
1937	off->cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |
1938			     (ICE_TX_CTX_DESC_TSO << ICE_TXD_CTX_QW1_CMD_S) |
1939			     (cd_tso_len << ICE_TXD_CTX_QW1_TSO_LEN_S) |
1940			     (cd_mss << ICE_TXD_CTX_QW1_MSS_S));
1941	first->tx_flags |= ICE_TX_FLAGS_TSO;
1942	return 1;
1943}
1944
1945/**
1946 * ice_txd_use_count  - estimate the number of descriptors needed for Tx
1947 * @size: transmit request size in bytes
1948 *
1949 * Due to hardware alignment restrictions (4K alignment), we need to
1950 * assume that we can have no more than 12K of data per descriptor, even
1951 * though each descriptor can take up to 16K - 1 bytes of aligned memory.
1952 * Thus, we need to divide by 12K. But division is slow! Instead,
1953 * we decompose the operation into shifts and one relatively cheap
1954 * multiply operation.
1955 *
1956 * To divide by 12K, we first divide by 4K, then divide by 3:
1957 *     To divide by 4K, shift right by 12 bits
1958 *     To divide by 3, multiply by 85, then divide by 256
1959 *     (Divide by 256 is done by shifting right by 8 bits)
1960 * Finally, we add one to round up. Because 256 isn't an exact multiple of
1961 * 3, we'll underestimate near each multiple of 12K. This is actually more
1962 * accurate as we have 4K - 1 of wiggle room that we can fit into the last
1963 * segment. For our purposes this is accurate out to 1M which is orders of
1964 * magnitude greater than our largest possible GSO size.
1965 *
1966 * This would then be implemented as:
1967 *     return (((size >> 12) * 85) >> 8) + ICE_DESCS_FOR_SKB_DATA_PTR;
1968 *
1969 * Since multiplication and division are commutative, we can reorder
1970 * operations into:
1971 *     return ((size * 85) >> 20) + ICE_DESCS_FOR_SKB_DATA_PTR;
1972 */
1973static unsigned int ice_txd_use_count(unsigned int size)
1974{
1975	return ((size * 85) >> 20) + ICE_DESCS_FOR_SKB_DATA_PTR;
1976}
1977
1978/**
1979 * ice_xmit_desc_count - calculate number of Tx descriptors needed
1980 * @skb: send buffer
1981 *
1982 * Returns number of data descriptors needed for this skb.
1983 */
1984static unsigned int ice_xmit_desc_count(struct sk_buff *skb)
1985{
1986	const skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
1987	unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1988	unsigned int count = 0, size = skb_headlen(skb);
1989
1990	for (;;) {
1991		count += ice_txd_use_count(size);
1992
1993		if (!nr_frags--)
1994			break;
1995
1996		size = skb_frag_size(frag++);
1997	}
1998
1999	return count;
2000}
2001
2002/**
2003 * __ice_chk_linearize - Check if there are more than 8 buffers per packet
2004 * @skb: send buffer
2005 *
2006 * Note: This HW can't DMA more than 8 buffers to build a packet on the wire
2007 * and so we need to figure out the cases where we need to linearize the skb.
2008 *
2009 * For TSO we need to count the TSO header and segment payload separately.
2010 * As such we need to check cases where we have 7 fragments or more as we
2011 * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for
2012 * the segment payload in the first descriptor, and another 7 for the
2013 * fragments.
2014 */
2015static bool __ice_chk_linearize(struct sk_buff *skb)
2016{
2017	const skb_frag_t *frag, *stale;
2018	int nr_frags, sum;
2019
2020	/* no need to check if number of frags is less than 7 */
2021	nr_frags = skb_shinfo(skb)->nr_frags;
2022	if (nr_frags < (ICE_MAX_BUF_TXD - 1))
2023		return false;
2024
2025	/* We need to walk through the list and validate that each group
2026	 * of 6 fragments totals at least gso_size.
2027	 */
2028	nr_frags -= ICE_MAX_BUF_TXD - 2;
2029	frag = &skb_shinfo(skb)->frags[0];
2030
2031	/* Initialize size to the negative value of gso_size minus 1. We
2032	 * use this as the worst case scenario in which the frag ahead
2033	 * of us only provides one byte which is why we are limited to 6
2034	 * descriptors for a single transmit as the header and previous
2035	 * fragment are already consuming 2 descriptors.
2036	 */
2037	sum = 1 - skb_shinfo(skb)->gso_size;
2038
2039	/* Add size of frags 0 through 4 to create our initial sum */
2040	sum += skb_frag_size(frag++);
2041	sum += skb_frag_size(frag++);
2042	sum += skb_frag_size(frag++);
2043	sum += skb_frag_size(frag++);
2044	sum += skb_frag_size(frag++);
2045
2046	/* Walk through fragments adding latest fragment, testing it, and
2047	 * then removing stale fragments from the sum.
2048	 */
2049	for (stale = &skb_shinfo(skb)->frags[0];; stale++) {
2050		int stale_size = skb_frag_size(stale);
2051
2052		sum += skb_frag_size(frag++);
2053
2054		/* The stale fragment may present us with a smaller
2055		 * descriptor than the actual fragment size. To account
2056		 * for that we need to remove all the data on the front and
2057		 * figure out what the remainder would be in the last
2058		 * descriptor associated with the fragment.
2059		 */
2060		if (stale_size > ICE_MAX_DATA_PER_TXD) {
2061			int align_pad = -(skb_frag_off(stale)) &
2062					(ICE_MAX_READ_REQ_SIZE - 1);
2063
2064			sum -= align_pad;
2065			stale_size -= align_pad;
2066
2067			do {
2068				sum -= ICE_MAX_DATA_PER_TXD_ALIGNED;
2069				stale_size -= ICE_MAX_DATA_PER_TXD_ALIGNED;
2070			} while (stale_size > ICE_MAX_DATA_PER_TXD);
2071		}
2072
2073		/* if sum is negative we failed to make sufficient progress */
2074		if (sum < 0)
2075			return true;
2076
2077		if (!nr_frags--)
2078			break;
2079
2080		sum -= stale_size;
2081	}
2082
2083	return false;
2084}
2085
2086/**
2087 * ice_chk_linearize - Check if there are more than 8 fragments per packet
2088 * @skb:      send buffer
2089 * @count:    number of buffers used
2090 *
2091 * Note: Our HW can't scatter-gather more than 8 fragments to build
2092 * a packet on the wire and so we need to figure out the cases where we
2093 * need to linearize the skb.
2094 */
2095static bool ice_chk_linearize(struct sk_buff *skb, unsigned int count)
2096{
2097	/* Both TSO and single send will work if count is less than 8 */
2098	if (likely(count < ICE_MAX_BUF_TXD))
2099		return false;
2100
2101	if (skb_is_gso(skb))
2102		return __ice_chk_linearize(skb);
2103
2104	/* we can support up to 8 data buffers for a single send */
2105	return count != ICE_MAX_BUF_TXD;
2106}
2107
2108/**
2109 * ice_tstamp - set up context descriptor for hardware timestamp
2110 * @tx_ring: pointer to the Tx ring to send buffer on
2111 * @skb: pointer to the SKB we're sending
2112 * @first: Tx buffer
2113 * @off: Tx offload parameters
2114 */
2115static void
2116ice_tstamp(struct ice_tx_ring *tx_ring, struct sk_buff *skb,
2117	   struct ice_tx_buf *first, struct ice_tx_offload_params *off)
2118{
2119	s8 idx;
2120
2121	/* only timestamp the outbound packet if the user has requested it */
2122	if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
2123		return;
2124
2125	/* Tx timestamps cannot be sampled when doing TSO */
2126	if (first->tx_flags & ICE_TX_FLAGS_TSO)
2127		return;
2128
2129	/* Grab an open timestamp slot */
2130	idx = ice_ptp_request_ts(tx_ring->tx_tstamps, skb);
2131	if (idx < 0) {
2132		tx_ring->vsi->back->ptp.tx_hwtstamp_skipped++;
2133		return;
2134	}
2135
2136	off->cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |
2137			     (ICE_TX_CTX_DESC_TSYN << ICE_TXD_CTX_QW1_CMD_S) |
2138			     ((u64)idx << ICE_TXD_CTX_QW1_TSO_LEN_S));
2139	first->tx_flags |= ICE_TX_FLAGS_TSYN;
2140}
2141
2142/**
2143 * ice_xmit_frame_ring - Sends buffer on Tx ring
2144 * @skb: send buffer
2145 * @tx_ring: ring to send buffer on
2146 *
2147 * Returns NETDEV_TX_OK if sent, else an error code
2148 */
2149static netdev_tx_t
2150ice_xmit_frame_ring(struct sk_buff *skb, struct ice_tx_ring *tx_ring)
2151{
2152	struct ice_tx_offload_params offload = { 0 };
2153	struct ice_vsi *vsi = tx_ring->vsi;
2154	struct ice_tx_buf *first;
2155	struct ethhdr *eth;
2156	unsigned int count;
2157	int tso, csum;
2158
2159	ice_trace(xmit_frame_ring, tx_ring, skb);
2160
2161	if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
2162		goto out_drop;
2163
2164	count = ice_xmit_desc_count(skb);
2165	if (ice_chk_linearize(skb, count)) {
2166		if (__skb_linearize(skb))
2167			goto out_drop;
2168		count = ice_txd_use_count(skb->len);
2169		tx_ring->ring_stats->tx_stats.tx_linearize++;
2170	}
2171
2172	/* need: 1 descriptor per page * PAGE_SIZE/ICE_MAX_DATA_PER_TXD,
2173	 *       + 1 desc for skb_head_len/ICE_MAX_DATA_PER_TXD,
2174	 *       + 4 desc gap to avoid the cache line where head is,
2175	 *       + 1 desc for context descriptor,
2176	 * otherwise try next time
2177	 */
2178	if (ice_maybe_stop_tx(tx_ring, count + ICE_DESCS_PER_CACHE_LINE +
2179			      ICE_DESCS_FOR_CTX_DESC)) {
2180		tx_ring->ring_stats->tx_stats.tx_busy++;
2181		return NETDEV_TX_BUSY;
2182	}
2183
2184	/* prefetch for bql data which is infrequently used */
2185	netdev_txq_bql_enqueue_prefetchw(txring_txq(tx_ring));
2186
2187	offload.tx_ring = tx_ring;
2188
2189	/* record the location of the first descriptor for this packet */
2190	first = &tx_ring->tx_buf[tx_ring->next_to_use];
2191	first->skb = skb;
2192	first->type = ICE_TX_BUF_SKB;
2193	first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN);
2194	first->gso_segs = 1;
2195	first->tx_flags = 0;
2196
2197	/* prepare the VLAN tagging flags for Tx */
2198	ice_tx_prepare_vlan_flags(tx_ring, first);
2199	if (first->tx_flags & ICE_TX_FLAGS_HW_OUTER_SINGLE_VLAN) {
2200		offload.cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |
2201					(ICE_TX_CTX_DESC_IL2TAG2 <<
2202					ICE_TXD_CTX_QW1_CMD_S));
2203		offload.cd_l2tag2 = first->vid;
2204	}
2205
2206	/* set up TSO offload */
2207	tso = ice_tso(first, &offload);
2208	if (tso < 0)
2209		goto out_drop;
2210
2211	/* always set up Tx checksum offload */
2212	csum = ice_tx_csum(first, &offload);
2213	if (csum < 0)
2214		goto out_drop;
2215
2216	/* allow CONTROL frames egress from main VSI if FW LLDP disabled */
2217	eth = (struct ethhdr *)skb_mac_header(skb);
2218
2219	if ((ice_is_switchdev_running(vsi->back) ||
2220	     ice_lag_is_switchdev_running(vsi->back)) &&
2221	    vsi->type != ICE_VSI_SF)
2222		ice_eswitch_set_target_vsi(skb, &offload);
2223	else if (unlikely((skb->priority == TC_PRIO_CONTROL ||
2224			   eth->h_proto == htons(ETH_P_LLDP)) &&
2225			   vsi->type == ICE_VSI_PF &&
2226			   vsi->port_info->qos_cfg.is_sw_lldp))
2227		offload.cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |
2228					ICE_TX_CTX_DESC_SWTCH_UPLINK <<
2229					ICE_TXD_CTX_QW1_CMD_S);
2230
2231	ice_tstamp(tx_ring, skb, first, &offload);
2232
2233	if (offload.cd_qw1 & ICE_TX_DESC_DTYPE_CTX) {
2234		struct ice_tx_ctx_desc *cdesc;
2235		u16 i = tx_ring->next_to_use;
2236
2237		/* grab the next descriptor */
2238		cdesc = ICE_TX_CTX_DESC(tx_ring, i);
2239		i++;
2240		tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2241
2242		/* setup context descriptor */
2243		cdesc->tunneling_params = cpu_to_le32(offload.cd_tunnel_params);
2244		cdesc->l2tag2 = cpu_to_le16(offload.cd_l2tag2);
2245		cdesc->gcs = cpu_to_le16(offload.cd_gcs_params);
2246		cdesc->qw1 = cpu_to_le64(offload.cd_qw1);
2247	}
2248
2249	ice_tx_map(tx_ring, first, &offload);
2250	return NETDEV_TX_OK;
2251
2252out_drop:
2253	ice_trace(xmit_frame_ring_drop, tx_ring, skb);
2254	dev_kfree_skb_any(skb);
2255	return NETDEV_TX_OK;
2256}
2257
2258/**
2259 * ice_start_xmit - Selects the correct VSI and Tx queue to send buffer
2260 * @skb: send buffer
2261 * @netdev: network interface device structure
2262 *
2263 * Returns NETDEV_TX_OK if sent, else an error code
2264 */
2265netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev)
2266{
2267	struct ice_netdev_priv *np = netdev_priv(netdev);
2268	struct ice_vsi *vsi = np->vsi;
2269	struct ice_tx_ring *tx_ring;
2270
2271	tx_ring = vsi->tx_rings[skb->queue_mapping];
2272
2273	/* hardware can't handle really short frames, hardware padding works
2274	 * beyond this point
2275	 */
2276	if (skb_put_padto(skb, ICE_MIN_TX_LEN))
2277		return NETDEV_TX_OK;
2278
2279	return ice_xmit_frame_ring(skb, tx_ring);
2280}
2281
2282/**
2283 * ice_get_dscp_up - return the UP/TC value for a SKB
2284 * @dcbcfg: DCB config that contains DSCP to UP/TC mapping
2285 * @skb: SKB to query for info to determine UP/TC
2286 *
2287 * This function is to only be called when the PF is in L3 DSCP PFC mode
2288 */
2289static u8 ice_get_dscp_up(struct ice_dcbx_cfg *dcbcfg, struct sk_buff *skb)
2290{
2291	u8 dscp = 0;
2292
2293	if (skb->protocol == htons(ETH_P_IP))
2294		dscp = ipv4_get_dsfield(ip_hdr(skb)) >> 2;
2295	else if (skb->protocol == htons(ETH_P_IPV6))
2296		dscp = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2;
2297
2298	return dcbcfg->dscp_map[dscp];
2299}
2300
2301u16
2302ice_select_queue(struct net_device *netdev, struct sk_buff *skb,
2303		 struct net_device *sb_dev)
2304{
2305	struct ice_pf *pf = ice_netdev_to_pf(netdev);
2306	struct ice_dcbx_cfg *dcbcfg;
2307
2308	dcbcfg = &pf->hw.port_info->qos_cfg.local_dcbx_cfg;
2309	if (dcbcfg->pfc_mode == ICE_QOS_MODE_DSCP)
2310		skb->priority = ice_get_dscp_up(dcbcfg, skb);
2311
2312	return netdev_pick_tx(netdev, skb, sb_dev);
2313}
2314
2315/**
2316 * ice_clean_ctrl_tx_irq - interrupt handler for flow director Tx queue
2317 * @tx_ring: tx_ring to clean
2318 */
2319void ice_clean_ctrl_tx_irq(struct ice_tx_ring *tx_ring)
2320{
2321	struct ice_vsi *vsi = tx_ring->vsi;
2322	s16 i = tx_ring->next_to_clean;
2323	int budget = ICE_DFLT_IRQ_WORK;
2324	struct ice_tx_desc *tx_desc;
2325	struct ice_tx_buf *tx_buf;
2326
2327	tx_buf = &tx_ring->tx_buf[i];
2328	tx_desc = ICE_TX_DESC(tx_ring, i);
2329	i -= tx_ring->count;
2330
2331	do {
2332		struct ice_tx_desc *eop_desc = tx_buf->next_to_watch;
2333
2334		/* if next_to_watch is not set then there is no pending work */
2335		if (!eop_desc)
2336			break;
2337
2338		/* prevent any other reads prior to eop_desc */
2339		smp_rmb();
2340
2341		/* if the descriptor isn't done, no work to do */
2342		if (!(eop_desc->cmd_type_offset_bsz &
2343		      cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)))
2344			break;
2345
2346		/* clear next_to_watch to prevent false hangs */
2347		tx_buf->next_to_watch = NULL;
2348		tx_desc->buf_addr = 0;
2349		tx_desc->cmd_type_offset_bsz = 0;
2350
2351		/* move past filter desc */
2352		tx_buf++;
2353		tx_desc++;
2354		i++;
2355		if (unlikely(!i)) {
2356			i -= tx_ring->count;
2357			tx_buf = tx_ring->tx_buf;
2358			tx_desc = ICE_TX_DESC(tx_ring, 0);
2359		}
2360
2361		/* unmap the data header */
2362		if (dma_unmap_len(tx_buf, len))
2363			dma_unmap_single(tx_ring->dev,
2364					 dma_unmap_addr(tx_buf, dma),
2365					 dma_unmap_len(tx_buf, len),
2366					 DMA_TO_DEVICE);
2367		if (tx_buf->type == ICE_TX_BUF_DUMMY)
2368			devm_kfree(tx_ring->dev, tx_buf->raw_buf);
2369
2370		/* clear next_to_watch to prevent false hangs */
2371		tx_buf->type = ICE_TX_BUF_EMPTY;
2372		tx_buf->tx_flags = 0;
2373		tx_buf->next_to_watch = NULL;
2374		dma_unmap_len_set(tx_buf, len, 0);
2375		tx_desc->buf_addr = 0;
2376		tx_desc->cmd_type_offset_bsz = 0;
2377
2378		/* move past eop_desc for start of next FD desc */
2379		tx_buf++;
2380		tx_desc++;
2381		i++;
2382		if (unlikely(!i)) {
2383			i -= tx_ring->count;
2384			tx_buf = tx_ring->tx_buf;
2385			tx_desc = ICE_TX_DESC(tx_ring, 0);
2386		}
2387
2388		budget--;
2389	} while (likely(budget));
2390
2391	i += tx_ring->count;
2392	tx_ring->next_to_clean = i;
2393
2394	/* re-enable interrupt if needed */
2395	ice_irq_dynamic_ena(&vsi->back->hw, vsi, vsi->q_vectors[0]);
2396}