idpf_txrx.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/* Copyright (C) 2023 Intel Corporation */
   3
   4#include "idpf.h"
   5#include "idpf_ptp.h"
   6#include "idpf_virtchnl.h"
   7#include "xdp.h"
   8#include "xsk.h"
   9
  10#define idpf_tx_buf_next(buf)		(*(u32 *)&(buf)->priv)
  11LIBETH_SQE_CHECK_PRIV(u32);
  12
  13/**
  14 * idpf_chk_linearize - Check if skb exceeds max descriptors per packet
  15 * @skb: send buffer
  16 * @max_bufs: maximum scatter gather buffers for single packet
  17 * @count: number of buffers this packet needs
  18 *
  19 * Make sure we don't exceed maximum scatter gather buffers for a single
  20 * packet.
  21 * TSO case has been handled earlier from idpf_features_check().
  22 */
  23static bool idpf_chk_linearize(const struct sk_buff *skb,
  24			       unsigned int max_bufs,
  25			       unsigned int count)
  26{
  27	if (likely(count <= max_bufs))
  28		return false;
  29
  30	if (skb_is_gso(skb))
  31		return false;
  32
  33	return true;
  34}
  35
  36/**
  37 * idpf_tx_timeout - Respond to a Tx Hang
  38 * @netdev: network interface device structure
  39 * @txqueue: TX queue
  40 */
  41void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
  42{
  43	struct idpf_adapter *adapter = idpf_netdev_to_adapter(netdev);
  44
  45	adapter->tx_timeout_count++;
  46
  47	netdev_err(netdev, "Detected Tx timeout: Count %d, Queue %d\n",
  48		   adapter->tx_timeout_count, txqueue);
  49	if (!idpf_is_reset_in_prog(adapter)) {
  50		set_bit(IDPF_HR_FUNC_RESET, adapter->flags);
  51		queue_delayed_work(adapter->vc_event_wq,
  52				   &adapter->vc_event_task,
  53				   msecs_to_jiffies(10));
  54	}
  55}
  56
  57static void idpf_tx_buf_clean(struct idpf_tx_queue *txq)
  58{
  59	struct libeth_sq_napi_stats ss = { };
  60	struct xdp_frame_bulk bq;
  61	struct libeth_cq_pp cp = {
  62		.dev	= txq->dev,
  63		.bq	= &bq,
  64		.ss	= &ss,
  65	};
  66
  67	xdp_frame_bulk_init(&bq);
  68
  69	/* Free all the Tx buffer sk_buffs */
  70	for (u32 i = 0; i < txq->buf_pool_size; i++)
  71		libeth_tx_complete_any(&txq->tx_buf[i], &cp);
  72
  73	xdp_flush_frame_bulk(&bq);
  74}
  75
  76/**
  77 * idpf_tx_buf_rel_all - Free any empty Tx buffers
  78 * @txq: queue to be cleaned
  79 */
  80static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq)
  81{
  82	/* Buffers already cleared, nothing to do */
  83	if (!txq->tx_buf)
  84		return;
  85
  86	if (idpf_queue_has(XSK, txq))
  87		idpf_xsksq_clean(txq);
  88	else
  89		idpf_tx_buf_clean(txq);
  90
  91	kfree(txq->tx_buf);
  92	txq->tx_buf = NULL;
  93}
  94
  95/**
  96 * idpf_tx_desc_rel - Free Tx resources per queue
  97 * @txq: Tx descriptor ring for a specific queue
  98 *
  99 * Free all transmit software resources
 100 */
 101static void idpf_tx_desc_rel(struct idpf_tx_queue *txq)
 102{
 103	bool xdp = idpf_queue_has(XDP, txq);
 104
 105	if (xdp)
 106		libeth_xdpsq_deinit_timer(txq->timer);
 107
 108	idpf_tx_buf_rel_all(txq);
 109
 110	if (!xdp)
 111		netdev_tx_reset_subqueue(txq->netdev, txq->idx);
 112
 113	idpf_xsk_clear_queue(txq, VIRTCHNL2_QUEUE_TYPE_TX);
 114
 115	if (!txq->desc_ring)
 116		return;
 117
 118	if (!xdp && txq->refillq)
 119		kfree(txq->refillq->ring);
 120
 121	dmam_free_coherent(txq->dev, txq->size, txq->desc_ring, txq->dma);
 122	txq->desc_ring = NULL;
 123	txq->next_to_use = 0;
 124	txq->next_to_clean = 0;
 125}
 126
 127/**
 128 * idpf_compl_desc_rel - Free completion resources per queue
 129 * @complq: completion queue
 130 *
 131 * Free all completion software resources.
 132 */
 133static void idpf_compl_desc_rel(struct idpf_compl_queue *complq)
 134{
 135	idpf_xsk_clear_queue(complq, VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION);
 136
 137	if (!complq->desc_ring)
 138		return;
 139
 140	dma_free_coherent(complq->netdev->dev.parent, complq->size,
 141			  complq->desc_ring, complq->dma);
 142	complq->desc_ring = NULL;
 143	complq->next_to_use = 0;
 144	complq->next_to_clean = 0;
 145}
 146
 147/**
 148 * idpf_tx_desc_rel_all - Free Tx Resources for All Queues
 149 * @vport: virtual port structure
 150 *
 151 * Free all transmit software resources
 152 */
 153static void idpf_tx_desc_rel_all(struct idpf_vport *vport)
 154{
 155	int i, j;
 156
 157	if (!vport->txq_grps)
 158		return;
 159
 160	for (i = 0; i < vport->num_txq_grp; i++) {
 161		struct idpf_txq_group *txq_grp = &vport->txq_grps[i];
 162
 163		for (j = 0; j < txq_grp->num_txq; j++)
 164			idpf_tx_desc_rel(txq_grp->txqs[j]);
 165
 166		if (idpf_is_queue_model_split(vport->txq_model))
 167			idpf_compl_desc_rel(txq_grp->complq);
 168	}
 169}
 170
 171/**
 172 * idpf_tx_buf_alloc_all - Allocate memory for all buffer resources
 173 * @tx_q: queue for which the buffers are allocated
 174 *
 175 * Returns 0 on success, negative on failure
 176 */
 177static int idpf_tx_buf_alloc_all(struct idpf_tx_queue *tx_q)
 178{
 179	/* Allocate book keeping buffers only. Buffers to be supplied to HW
 180	 * are allocated by kernel network stack and received as part of skb
 181	 */
 182	if (idpf_queue_has(FLOW_SCH_EN, tx_q))
 183		tx_q->buf_pool_size = U16_MAX;
 184	else
 185		tx_q->buf_pool_size = tx_q->desc_count;
 186	tx_q->tx_buf = kcalloc(tx_q->buf_pool_size, sizeof(*tx_q->tx_buf),
 187			       GFP_KERNEL);
 188	if (!tx_q->tx_buf)
 189		return -ENOMEM;
 190
 191	return 0;
 192}
 193
 194/**
 195 * idpf_tx_desc_alloc - Allocate the Tx descriptors
 196 * @vport: vport to allocate resources for
 197 * @tx_q: the tx ring to set up
 198 *
 199 * Returns 0 on success, negative on failure
 200 */
 201static int idpf_tx_desc_alloc(const struct idpf_vport *vport,
 202			      struct idpf_tx_queue *tx_q)
 203{
 204	struct device *dev = tx_q->dev;
 205	struct idpf_sw_queue *refillq;
 206	int err;
 207
 208	err = idpf_tx_buf_alloc_all(tx_q);
 209	if (err)
 210		goto err_alloc;
 211
 212	tx_q->size = tx_q->desc_count * sizeof(*tx_q->base_tx);
 213
 214	/* Allocate descriptors also round up to nearest 4K */
 215	tx_q->size = ALIGN(tx_q->size, 4096);
 216	tx_q->desc_ring = dmam_alloc_coherent(dev, tx_q->size, &tx_q->dma,
 217					      GFP_KERNEL);
 218	if (!tx_q->desc_ring) {
 219		dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
 220			tx_q->size);
 221		err = -ENOMEM;
 222		goto err_alloc;
 223	}
 224
 225	tx_q->next_to_use = 0;
 226	tx_q->next_to_clean = 0;
 227	idpf_queue_set(GEN_CHK, tx_q);
 228
 229	idpf_xsk_setup_queue(vport, tx_q, VIRTCHNL2_QUEUE_TYPE_TX);
 230
 231	if (!idpf_queue_has(FLOW_SCH_EN, tx_q))
 232		return 0;
 233
 234	refillq = tx_q->refillq;
 235	refillq->desc_count = tx_q->buf_pool_size;
 236	refillq->ring = kcalloc(refillq->desc_count, sizeof(u32),
 237				GFP_KERNEL);
 238	if (!refillq->ring) {
 239		err = -ENOMEM;
 240		goto err_alloc;
 241	}
 242
 243	for (unsigned int i = 0; i < refillq->desc_count; i++)
 244		refillq->ring[i] =
 245			FIELD_PREP(IDPF_RFL_BI_BUFID_M, i) |
 246			FIELD_PREP(IDPF_RFL_BI_GEN_M,
 247				   idpf_queue_has(GEN_CHK, refillq));
 248
 249	/* Go ahead and flip the GEN bit since this counts as filling
 250	 * up the ring, i.e. we already ring wrapped.
 251	 */
 252	idpf_queue_change(GEN_CHK, refillq);
 253
 254	tx_q->last_re = tx_q->desc_count - IDPF_TX_SPLITQ_RE_MIN_GAP;
 255
 256	return 0;
 257
 258err_alloc:
 259	idpf_tx_desc_rel(tx_q);
 260
 261	return err;
 262}
 263
 264/**
 265 * idpf_compl_desc_alloc - allocate completion descriptors
 266 * @vport: vport to allocate resources for
 267 * @complq: completion queue to set up
 268 *
 269 * Return: 0 on success, -errno on failure.
 270 */
 271static int idpf_compl_desc_alloc(const struct idpf_vport *vport,
 272				 struct idpf_compl_queue *complq)
 273{
 274	u32 desc_size;
 275
 276	desc_size = idpf_queue_has(FLOW_SCH_EN, complq) ?
 277		    sizeof(*complq->comp) : sizeof(*complq->comp_4b);
 278	complq->size = array_size(complq->desc_count, desc_size);
 279
 280	complq->desc_ring = dma_alloc_coherent(complq->netdev->dev.parent,
 281					       complq->size, &complq->dma,
 282					       GFP_KERNEL);
 283	if (!complq->desc_ring)
 284		return -ENOMEM;
 285
 286	complq->next_to_use = 0;
 287	complq->next_to_clean = 0;
 288	idpf_queue_set(GEN_CHK, complq);
 289
 290	idpf_xsk_setup_queue(vport, complq,
 291			     VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION);
 292
 293	return 0;
 294}
 295
 296/**
 297 * idpf_tx_desc_alloc_all - allocate all queues Tx resources
 298 * @vport: virtual port private structure
 299 *
 300 * Returns 0 on success, negative on failure
 301 */
 302static int idpf_tx_desc_alloc_all(struct idpf_vport *vport)
 303{
 304	int err = 0;
 305	int i, j;
 306
 307	/* Setup buffer queues. In single queue model buffer queues and
 308	 * completion queues will be same
 309	 */
 310	for (i = 0; i < vport->num_txq_grp; i++) {
 311		for (j = 0; j < vport->txq_grps[i].num_txq; j++) {
 312			struct idpf_tx_queue *txq = vport->txq_grps[i].txqs[j];
 313
 314			err = idpf_tx_desc_alloc(vport, txq);
 315			if (err) {
 316				pci_err(vport->adapter->pdev,
 317					"Allocation for Tx Queue %u failed\n",
 318					i);
 319				goto err_out;
 320			}
 321		}
 322
 323		if (!idpf_is_queue_model_split(vport->txq_model))
 324			continue;
 325
 326		/* Setup completion queues */
 327		err = idpf_compl_desc_alloc(vport, vport->txq_grps[i].complq);
 328		if (err) {
 329			pci_err(vport->adapter->pdev,
 330				"Allocation for Tx Completion Queue %u failed\n",
 331				i);
 332			goto err_out;
 333		}
 334	}
 335
 336err_out:
 337	if (err)
 338		idpf_tx_desc_rel_all(vport);
 339
 340	return err;
 341}
 342
 343/**
 344 * idpf_rx_page_rel - Release an rx buffer page
 345 * @rx_buf: the buffer to free
 346 */
 347static void idpf_rx_page_rel(struct libeth_fqe *rx_buf)
 348{
 349	if (unlikely(!rx_buf->netmem))
 350		return;
 351
 352	libeth_rx_recycle_slow(rx_buf->netmem);
 353
 354	rx_buf->netmem = 0;
 355	rx_buf->offset = 0;
 356}
 357
 358/**
 359 * idpf_rx_hdr_buf_rel_all - Release header buffer memory
 360 * @bufq: queue to use
 361 */
 362static void idpf_rx_hdr_buf_rel_all(struct idpf_buf_queue *bufq)
 363{
 364	struct libeth_fq fq = {
 365		.fqes	= bufq->hdr_buf,
 366		.pp	= bufq->hdr_pp,
 367	};
 368
 369	for (u32 i = 0; i < bufq->desc_count; i++)
 370		idpf_rx_page_rel(&bufq->hdr_buf[i]);
 371
 372	libeth_rx_fq_destroy(&fq);
 373	bufq->hdr_buf = NULL;
 374	bufq->hdr_pp = NULL;
 375}
 376
 377/**
 378 * idpf_rx_buf_rel_bufq - Free all Rx buffer resources for a buffer queue
 379 * @bufq: queue to be cleaned
 380 */
 381static void idpf_rx_buf_rel_bufq(struct idpf_buf_queue *bufq)
 382{
 383	struct libeth_fq fq = {
 384		.fqes	= bufq->buf,
 385		.pp	= bufq->pp,
 386	};
 387
 388	/* queue already cleared, nothing to do */
 389	if (!bufq->buf)
 390		return;
 391
 392	if (idpf_queue_has(XSK, bufq)) {
 393		idpf_xskfq_rel(bufq);
 394		return;
 395	}
 396
 397	/* Free all the bufs allocated and given to hw on Rx queue */
 398	for (u32 i = 0; i < bufq->desc_count; i++)
 399		idpf_rx_page_rel(&bufq->buf[i]);
 400
 401	if (idpf_queue_has(HSPLIT_EN, bufq))
 402		idpf_rx_hdr_buf_rel_all(bufq);
 403
 404	libeth_rx_fq_destroy(&fq);
 405	bufq->buf = NULL;
 406	bufq->pp = NULL;
 407}
 408
 409/**
 410 * idpf_rx_buf_rel_all - Free all Rx buffer resources for a receive queue
 411 * @rxq: queue to be cleaned
 412 */
 413static void idpf_rx_buf_rel_all(struct idpf_rx_queue *rxq)
 414{
 415	struct libeth_fq fq = {
 416		.fqes	= rxq->rx_buf,
 417		.pp	= rxq->pp,
 418	};
 419
 420	if (!rxq->rx_buf)
 421		return;
 422
 423	for (u32 i = 0; i < rxq->desc_count; i++)
 424		idpf_rx_page_rel(&rxq->rx_buf[i]);
 425
 426	libeth_rx_fq_destroy(&fq);
 427	rxq->rx_buf = NULL;
 428	rxq->pp = NULL;
 429}
 430
 431/**
 432 * idpf_rx_desc_rel - Free a specific Rx q resources
 433 * @rxq: queue to clean the resources from
 434 * @dev: device to free DMA memory
 435 * @model: single or split queue model
 436 *
 437 * Free a specific rx queue resources
 438 */
 439static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev,
 440			     u32 model)
 441{
 442	if (!rxq)
 443		return;
 444
 445	if (!idpf_queue_has(XSK, rxq))
 446		libeth_xdp_return_stash(&rxq->xdp);
 447
 448	if (!idpf_is_queue_model_split(model))
 449		idpf_rx_buf_rel_all(rxq);
 450
 451	idpf_xsk_clear_queue(rxq, VIRTCHNL2_QUEUE_TYPE_RX);
 452
 453	rxq->next_to_alloc = 0;
 454	rxq->next_to_clean = 0;
 455	rxq->next_to_use = 0;
 456	if (!rxq->desc_ring)
 457		return;
 458
 459	dmam_free_coherent(dev, rxq->size, rxq->desc_ring, rxq->dma);
 460	rxq->desc_ring = NULL;
 461}
 462
 463/**
 464 * idpf_rx_desc_rel_bufq - free buffer queue resources
 465 * @bufq: buffer queue to clean the resources from
 466 * @dev: device to free DMA memory
 467 */
 468static void idpf_rx_desc_rel_bufq(struct idpf_buf_queue *bufq,
 469				  struct device *dev)
 470{
 471	if (!bufq)
 472		return;
 473
 474	idpf_rx_buf_rel_bufq(bufq);
 475	idpf_xsk_clear_queue(bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER);
 476
 477	bufq->next_to_alloc = 0;
 478	bufq->next_to_clean = 0;
 479	bufq->next_to_use = 0;
 480
 481	if (!bufq->split_buf)
 482		return;
 483
 484	dma_free_coherent(dev, bufq->size, bufq->split_buf, bufq->dma);
 485	bufq->split_buf = NULL;
 486}
 487
 488/**
 489 * idpf_rx_desc_rel_all - Free Rx Resources for All Queues
 490 * @vport: virtual port structure
 491 *
 492 * Free all rx queues resources
 493 */
 494static void idpf_rx_desc_rel_all(struct idpf_vport *vport)
 495{
 496	struct device *dev = &vport->adapter->pdev->dev;
 497	struct idpf_rxq_group *rx_qgrp;
 498	u16 num_rxq;
 499	int i, j;
 500
 501	if (!vport->rxq_grps)
 502		return;
 503
 504	for (i = 0; i < vport->num_rxq_grp; i++) {
 505		rx_qgrp = &vport->rxq_grps[i];
 506
 507		if (!idpf_is_queue_model_split(vport->rxq_model)) {
 508			for (j = 0; j < rx_qgrp->singleq.num_rxq; j++)
 509				idpf_rx_desc_rel(rx_qgrp->singleq.rxqs[j], dev,
 510						 VIRTCHNL2_QUEUE_MODEL_SINGLE);
 511			continue;
 512		}
 513
 514		num_rxq = rx_qgrp->splitq.num_rxq_sets;
 515		for (j = 0; j < num_rxq; j++)
 516			idpf_rx_desc_rel(&rx_qgrp->splitq.rxq_sets[j]->rxq,
 517					 dev, VIRTCHNL2_QUEUE_MODEL_SPLIT);
 518
 519		if (!rx_qgrp->splitq.bufq_sets)
 520			continue;
 521
 522		for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
 523			struct idpf_bufq_set *bufq_set =
 524				&rx_qgrp->splitq.bufq_sets[j];
 525
 526			idpf_rx_desc_rel_bufq(&bufq_set->bufq, dev);
 527		}
 528	}
 529}
 530
 531/**
 532 * idpf_rx_buf_hw_update - Store the new tail and head values
 533 * @bufq: queue to bump
 534 * @val: new head index
 535 */
 536static void idpf_rx_buf_hw_update(struct idpf_buf_queue *bufq, u32 val)
 537{
 538	bufq->next_to_use = val;
 539
 540	if (unlikely(!bufq->tail))
 541		return;
 542
 543	/* writel has an implicit memory barrier */
 544	writel(val, bufq->tail);
 545}
 546
 547/**
 548 * idpf_rx_hdr_buf_alloc_all - Allocate memory for header buffers
 549 * @bufq: ring to use
 550 *
 551 * Returns 0 on success, negative on failure.
 552 */
 553static int idpf_rx_hdr_buf_alloc_all(struct idpf_buf_queue *bufq)
 554{
 555	struct libeth_fq fq = {
 556		.count	= bufq->desc_count,
 557		.type	= LIBETH_FQE_HDR,
 558		.xdp	= idpf_xdp_enabled(bufq->q_vector->vport),
 559		.nid	= idpf_q_vector_to_mem(bufq->q_vector),
 560	};
 561	int ret;
 562
 563	ret = libeth_rx_fq_create(&fq, &bufq->q_vector->napi);
 564	if (ret)
 565		return ret;
 566
 567	bufq->hdr_pp = fq.pp;
 568	bufq->hdr_buf = fq.fqes;
 569	bufq->hdr_truesize = fq.truesize;
 570	bufq->rx_hbuf_size = fq.buf_len;
 571
 572	return 0;
 573}
 574
 575/**
 576 * idpf_post_buf_refill - Post buffer id to refill queue
 577 * @refillq: refill queue to post to
 578 * @buf_id: buffer id to post
 579 */
 580static void idpf_post_buf_refill(struct idpf_sw_queue *refillq, u16 buf_id)
 581{
 582	u32 nta = refillq->next_to_use;
 583
 584	/* store the buffer ID and the SW maintained GEN bit to the refillq */
 585	refillq->ring[nta] =
 586		FIELD_PREP(IDPF_RFL_BI_BUFID_M, buf_id) |
 587		FIELD_PREP(IDPF_RFL_BI_GEN_M,
 588			   idpf_queue_has(GEN_CHK, refillq));
 589
 590	if (unlikely(++nta == refillq->desc_count)) {
 591		nta = 0;
 592		idpf_queue_change(GEN_CHK, refillq);
 593	}
 594
 595	refillq->next_to_use = nta;
 596}
 597
 598/**
 599 * idpf_rx_post_buf_desc - Post buffer to bufq descriptor ring
 600 * @bufq: buffer queue to post to
 601 * @buf_id: buffer id to post
 602 *
 603 * Returns false if buffer could not be allocated, true otherwise.
 604 */
 605static bool idpf_rx_post_buf_desc(struct idpf_buf_queue *bufq, u16 buf_id)
 606{
 607	struct virtchnl2_splitq_rx_buf_desc *splitq_rx_desc = NULL;
 608	struct libeth_fq_fp fq = {
 609		.count	= bufq->desc_count,
 610	};
 611	u16 nta = bufq->next_to_alloc;
 612	dma_addr_t addr;
 613
 614	splitq_rx_desc = &bufq->split_buf[nta];
 615
 616	if (idpf_queue_has(HSPLIT_EN, bufq)) {
 617		fq.pp = bufq->hdr_pp;
 618		fq.fqes = bufq->hdr_buf;
 619		fq.truesize = bufq->hdr_truesize;
 620
 621		addr = libeth_rx_alloc(&fq, buf_id);
 622		if (addr == DMA_MAPPING_ERROR)
 623			return false;
 624
 625		splitq_rx_desc->hdr_addr = cpu_to_le64(addr);
 626	}
 627
 628	fq.pp = bufq->pp;
 629	fq.fqes = bufq->buf;
 630	fq.truesize = bufq->truesize;
 631
 632	addr = libeth_rx_alloc(&fq, buf_id);
 633	if (addr == DMA_MAPPING_ERROR)
 634		return false;
 635
 636	splitq_rx_desc->pkt_addr = cpu_to_le64(addr);
 637	splitq_rx_desc->qword0.buf_id = cpu_to_le16(buf_id);
 638
 639	nta++;
 640	if (unlikely(nta == bufq->desc_count))
 641		nta = 0;
 642	bufq->next_to_alloc = nta;
 643
 644	return true;
 645}
 646
 647/**
 648 * idpf_rx_post_init_bufs - Post initial buffers to bufq
 649 * @bufq: buffer queue to post working set to
 650 * @working_set: number of buffers to put in working set
 651 *
 652 * Returns true if @working_set bufs were posted successfully, false otherwise.
 653 */
 654static bool idpf_rx_post_init_bufs(struct idpf_buf_queue *bufq,
 655				   u16 working_set)
 656{
 657	int i;
 658
 659	for (i = 0; i < working_set; i++) {
 660		if (!idpf_rx_post_buf_desc(bufq, i))
 661			return false;
 662	}
 663
 664	idpf_rx_buf_hw_update(bufq, ALIGN_DOWN(bufq->next_to_alloc,
 665					       IDPF_RX_BUF_STRIDE));
 666
 667	return true;
 668}
 669
 670/**
 671 * idpf_rx_buf_alloc_singleq - Allocate memory for all buffer resources
 672 * @rxq: queue for which the buffers are allocated
 673 *
 674 * Return: 0 on success, -ENOMEM on failure.
 675 */
 676static int idpf_rx_buf_alloc_singleq(struct idpf_rx_queue *rxq)
 677{
 678	if (idpf_rx_singleq_buf_hw_alloc_all(rxq, rxq->desc_count - 1))
 679		goto err;
 680
 681	return 0;
 682
 683err:
 684	idpf_rx_buf_rel_all(rxq);
 685
 686	return -ENOMEM;
 687}
 688
 689/**
 690 * idpf_rx_bufs_init_singleq - Initialize page pool and allocate Rx bufs
 691 * @rxq: buffer queue to create page pool for
 692 *
 693 * Return: 0 on success, -errno on failure.
 694 */
 695static int idpf_rx_bufs_init_singleq(struct idpf_rx_queue *rxq)
 696{
 697	struct libeth_fq fq = {
 698		.count		= rxq->desc_count,
 699		.type		= LIBETH_FQE_MTU,
 700		.buf_len	= IDPF_RX_MAX_BUF_SZ,
 701		.nid		= idpf_q_vector_to_mem(rxq->q_vector),
 702	};
 703	int ret;
 704
 705	ret = libeth_rx_fq_create(&fq, &rxq->q_vector->napi);
 706	if (ret)
 707		return ret;
 708
 709	rxq->pp = fq.pp;
 710	rxq->rx_buf = fq.fqes;
 711	rxq->truesize = fq.truesize;
 712	rxq->rx_buf_size = fq.buf_len;
 713
 714	return idpf_rx_buf_alloc_singleq(rxq);
 715}
 716
 717/**
 718 * idpf_rx_buf_alloc_all - Allocate memory for all buffer resources
 719 * @rxbufq: queue for which the buffers are allocated
 720 *
 721 * Returns 0 on success, negative on failure
 722 */
 723static int idpf_rx_buf_alloc_all(struct idpf_buf_queue *rxbufq)
 724{
 725	int err = 0;
 726
 727	if (idpf_queue_has(HSPLIT_EN, rxbufq)) {
 728		err = idpf_rx_hdr_buf_alloc_all(rxbufq);
 729		if (err)
 730			goto rx_buf_alloc_all_out;
 731	}
 732
 733	/* Allocate buffers to be given to HW.	 */
 734	if (!idpf_rx_post_init_bufs(rxbufq, IDPF_RX_BUFQ_WORKING_SET(rxbufq)))
 735		err = -ENOMEM;
 736
 737rx_buf_alloc_all_out:
 738	if (err)
 739		idpf_rx_buf_rel_bufq(rxbufq);
 740
 741	return err;
 742}
 743
 744/**
 745 * idpf_rx_bufs_init - Initialize page pool, allocate rx bufs, and post to HW
 746 * @bufq: buffer queue to create page pool for
 747 * @type: type of Rx buffers to allocate
 748 *
 749 * Returns 0 on success, negative on failure
 750 */
 751static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq,
 752			     enum libeth_fqe_type type)
 753{
 754	struct libeth_fq fq = {
 755		.truesize	= bufq->truesize,
 756		.count		= bufq->desc_count,
 757		.type		= type,
 758		.buf_len	= IDPF_RX_MAX_BUF_SZ,
 759		.hsplit		= idpf_queue_has(HSPLIT_EN, bufq),
 760		.xdp		= idpf_xdp_enabled(bufq->q_vector->vport),
 761		.nid		= idpf_q_vector_to_mem(bufq->q_vector),
 762	};
 763	int ret;
 764
 765	if (idpf_queue_has(XSK, bufq))
 766		return idpf_xskfq_init(bufq);
 767
 768	ret = libeth_rx_fq_create(&fq, &bufq->q_vector->napi);
 769	if (ret)
 770		return ret;
 771
 772	bufq->pp = fq.pp;
 773	bufq->buf = fq.fqes;
 774	bufq->truesize = fq.truesize;
 775	bufq->rx_buf_size = fq.buf_len;
 776
 777	return idpf_rx_buf_alloc_all(bufq);
 778}
 779
 780/**
 781 * idpf_rx_bufs_init_all - Initialize all RX bufs
 782 * @vport: virtual port struct
 783 *
 784 * Returns 0 on success, negative on failure
 785 */
 786int idpf_rx_bufs_init_all(struct idpf_vport *vport)
 787{
 788	bool split = idpf_is_queue_model_split(vport->rxq_model);
 789	int i, j, err;
 790
 791	idpf_xdp_copy_prog_to_rqs(vport, vport->xdp_prog);
 792
 793	for (i = 0; i < vport->num_rxq_grp; i++) {
 794		struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
 795		u32 truesize = 0;
 796
 797		/* Allocate bufs for the rxq itself in singleq */
 798		if (!split) {
 799			int num_rxq = rx_qgrp->singleq.num_rxq;
 800
 801			for (j = 0; j < num_rxq; j++) {
 802				struct idpf_rx_queue *q;
 803
 804				q = rx_qgrp->singleq.rxqs[j];
 805				err = idpf_rx_bufs_init_singleq(q);
 806				if (err)
 807					return err;
 808			}
 809
 810			continue;
 811		}
 812
 813		/* Otherwise, allocate bufs for the buffer queues */
 814		for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
 815			enum libeth_fqe_type type;
 816			struct idpf_buf_queue *q;
 817
 818			q = &rx_qgrp->splitq.bufq_sets[j].bufq;
 819			q->truesize = truesize;
 820
 821			type = truesize ? LIBETH_FQE_SHORT : LIBETH_FQE_MTU;
 822
 823			err = idpf_rx_bufs_init(q, type);
 824			if (err)
 825				return err;
 826
 827			truesize = q->truesize >> 1;
 828		}
 829	}
 830
 831	return 0;
 832}
 833
 834/**
 835 * idpf_rx_desc_alloc - Allocate queue Rx resources
 836 * @vport: vport to allocate resources for
 837 * @rxq: Rx queue for which the resources are setup
 838 *
 839 * Returns 0 on success, negative on failure
 840 */
 841static int idpf_rx_desc_alloc(const struct idpf_vport *vport,
 842			      struct idpf_rx_queue *rxq)
 843{
 844	struct device *dev = &vport->adapter->pdev->dev;
 845
 846	rxq->size = rxq->desc_count * sizeof(union virtchnl2_rx_desc);
 847
 848	/* Allocate descriptors and also round up to nearest 4K */
 849	rxq->size = ALIGN(rxq->size, 4096);
 850	rxq->desc_ring = dmam_alloc_coherent(dev, rxq->size,
 851					     &rxq->dma, GFP_KERNEL);
 852	if (!rxq->desc_ring) {
 853		dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
 854			rxq->size);
 855		return -ENOMEM;
 856	}
 857
 858	rxq->next_to_alloc = 0;
 859	rxq->next_to_clean = 0;
 860	rxq->next_to_use = 0;
 861	idpf_queue_set(GEN_CHK, rxq);
 862
 863	idpf_xsk_setup_queue(vport, rxq, VIRTCHNL2_QUEUE_TYPE_RX);
 864
 865	return 0;
 866}
 867
 868/**
 869 * idpf_bufq_desc_alloc - Allocate buffer queue descriptor ring
 870 * @vport: vport to allocate resources for
 871 * @bufq: buffer queue for which the resources are set up
 872 *
 873 * Return: 0 on success, -ENOMEM on failure.
 874 */
 875static int idpf_bufq_desc_alloc(const struct idpf_vport *vport,
 876				struct idpf_buf_queue *bufq)
 877{
 878	struct device *dev = &vport->adapter->pdev->dev;
 879
 880	bufq->size = array_size(bufq->desc_count, sizeof(*bufq->split_buf));
 881
 882	bufq->split_buf = dma_alloc_coherent(dev, bufq->size, &bufq->dma,
 883					     GFP_KERNEL);
 884	if (!bufq->split_buf)
 885		return -ENOMEM;
 886
 887	bufq->next_to_alloc = 0;
 888	bufq->next_to_clean = 0;
 889	bufq->next_to_use = 0;
 890	idpf_queue_set(GEN_CHK, bufq);
 891
 892	idpf_xsk_setup_queue(vport, bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER);
 893
 894	return 0;
 895}
 896
 897/**
 898 * idpf_rx_desc_alloc_all - allocate all RX queues resources
 899 * @vport: virtual port structure
 900 *
 901 * Returns 0 on success, negative on failure
 902 */
 903static int idpf_rx_desc_alloc_all(struct idpf_vport *vport)
 904{
 905	struct idpf_rxq_group *rx_qgrp;
 906	int i, j, err;
 907	u16 num_rxq;
 908
 909	for (i = 0; i < vport->num_rxq_grp; i++) {
 910		rx_qgrp = &vport->rxq_grps[i];
 911		if (idpf_is_queue_model_split(vport->rxq_model))
 912			num_rxq = rx_qgrp->splitq.num_rxq_sets;
 913		else
 914			num_rxq = rx_qgrp->singleq.num_rxq;
 915
 916		for (j = 0; j < num_rxq; j++) {
 917			struct idpf_rx_queue *q;
 918
 919			if (idpf_is_queue_model_split(vport->rxq_model))
 920				q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
 921			else
 922				q = rx_qgrp->singleq.rxqs[j];
 923
 924			err = idpf_rx_desc_alloc(vport, q);
 925			if (err) {
 926				pci_err(vport->adapter->pdev,
 927					"Memory allocation for Rx queue %u from queue group %u failed\n",
 928					j, i);
 929				goto err_out;
 930			}
 931		}
 932
 933		if (!idpf_is_queue_model_split(vport->rxq_model))
 934			continue;
 935
 936		for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
 937			struct idpf_buf_queue *q;
 938
 939			q = &rx_qgrp->splitq.bufq_sets[j].bufq;
 940
 941			err = idpf_bufq_desc_alloc(vport, q);
 942			if (err) {
 943				pci_err(vport->adapter->pdev,
 944					"Memory allocation for Rx Buffer Queue %u from queue group %u failed\n",
 945					j, i);
 946				goto err_out;
 947			}
 948		}
 949	}
 950
 951	return 0;
 952
 953err_out:
 954	idpf_rx_desc_rel_all(vport);
 955
 956	return err;
 957}
 958
 959static int idpf_init_queue_set(const struct idpf_queue_set *qs)
 960{
 961	const struct idpf_vport *vport = qs->vport;
 962	bool splitq;
 963	int err;
 964
 965	splitq = idpf_is_queue_model_split(vport->rxq_model);
 966
 967	for (u32 i = 0; i < qs->num; i++) {
 968		const struct idpf_queue_ptr *q = &qs->qs[i];
 969		struct idpf_buf_queue *bufq;
 970
 971		switch (q->type) {
 972		case VIRTCHNL2_QUEUE_TYPE_RX:
 973			err = idpf_rx_desc_alloc(vport, q->rxq);
 974			if (err)
 975				break;
 976
 977			err = idpf_xdp_rxq_info_init(q->rxq);
 978			if (err)
 979				break;
 980
 981			if (!splitq)
 982				err = idpf_rx_bufs_init_singleq(q->rxq);
 983
 984			break;
 985		case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER:
 986			bufq = q->bufq;
 987
 988			err = idpf_bufq_desc_alloc(vport, bufq);
 989			if (err)
 990				break;
 991
 992			for (u32 j = 0; j < bufq->q_vector->num_bufq; j++) {
 993				struct idpf_buf_queue * const *bufqs;
 994				enum libeth_fqe_type type;
 995				u32 ts;
 996
 997				bufqs = bufq->q_vector->bufq;
 998				if (bufqs[j] != bufq)
 999					continue;
1000
1001				if (j) {
1002					type = LIBETH_FQE_SHORT;
1003					ts = bufqs[j - 1]->truesize >> 1;
1004				} else {
1005					type = LIBETH_FQE_MTU;
1006					ts = 0;
1007				}
1008
1009				bufq->truesize = ts;
1010
1011				err = idpf_rx_bufs_init(bufq, type);
1012				break;
1013			}
1014
1015			break;
1016		case VIRTCHNL2_QUEUE_TYPE_TX:
1017			err = idpf_tx_desc_alloc(vport, q->txq);
1018			break;
1019		case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION:
1020			err = idpf_compl_desc_alloc(vport, q->complq);
1021			break;
1022		default:
1023			continue;
1024		}
1025
1026		if (err)
1027			return err;
1028	}
1029
1030	return 0;
1031}
1032
1033static void idpf_clean_queue_set(const struct idpf_queue_set *qs)
1034{
1035	const struct idpf_vport *vport = qs->vport;
1036	struct device *dev = vport->netdev->dev.parent;
1037
1038	for (u32 i = 0; i < qs->num; i++) {
1039		const struct idpf_queue_ptr *q = &qs->qs[i];
1040
1041		switch (q->type) {
1042		case VIRTCHNL2_QUEUE_TYPE_RX:
1043			idpf_xdp_rxq_info_deinit(q->rxq, vport->rxq_model);
1044			idpf_rx_desc_rel(q->rxq, dev, vport->rxq_model);
1045			break;
1046		case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER:
1047			idpf_rx_desc_rel_bufq(q->bufq, dev);
1048			break;
1049		case VIRTCHNL2_QUEUE_TYPE_TX:
1050			idpf_tx_desc_rel(q->txq);
1051
1052			if (idpf_queue_has(XDP, q->txq)) {
1053				q->txq->pending = 0;
1054				q->txq->xdp_tx = 0;
1055			} else {
1056				q->txq->txq_grp->num_completions_pending = 0;
1057			}
1058
1059			writel(q->txq->next_to_use, q->txq->tail);
1060			break;
1061		case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION:
1062			idpf_compl_desc_rel(q->complq);
1063			q->complq->num_completions = 0;
1064			break;
1065		default:
1066			break;
1067		}
1068	}
1069}
1070
1071static void idpf_qvec_ena_irq(struct idpf_q_vector *qv)
1072{
1073	if (qv->num_txq) {
1074		u32 itr;
1075
1076		if (IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode))
1077			itr = qv->vport->tx_itr_profile[qv->tx_dim.profile_ix];
1078		else
1079			itr = qv->tx_itr_value;
1080
1081		idpf_vport_intr_write_itr(qv, itr, true);
1082	}
1083
1084	if (qv->num_rxq) {
1085		u32 itr;
1086
1087		if (IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode))
1088			itr = qv->vport->rx_itr_profile[qv->rx_dim.profile_ix];
1089		else
1090			itr = qv->rx_itr_value;
1091
1092		idpf_vport_intr_write_itr(qv, itr, false);
1093	}
1094
1095	if (qv->num_txq || qv->num_rxq)
1096		idpf_vport_intr_update_itr_ena_irq(qv);
1097}
1098
1099/**
1100 * idpf_vector_to_queue_set - create a queue set associated with the given
1101 *			      queue vector
1102 * @qv: queue vector corresponding to the queue pair
1103 *
1104 * Returns a pointer to a dynamically allocated array of pointers to all
1105 * queues associated with a given queue vector (@qv).
1106 * Please note that the caller is responsible to free the memory allocated
1107 * by this function using kfree().
1108 *
1109 * Return: &idpf_queue_set on success, %NULL in case of error.
1110 */
1111static struct idpf_queue_set *
1112idpf_vector_to_queue_set(struct idpf_q_vector *qv)
1113{
1114	bool xdp = qv->vport->xdp_txq_offset && !qv->num_xsksq;
1115	struct idpf_vport *vport = qv->vport;
1116	struct idpf_queue_set *qs;
1117	u32 num;
1118
1119	num = qv->num_rxq + qv->num_bufq + qv->num_txq + qv->num_complq;
1120	num += xdp ? qv->num_rxq * 2 : qv->num_xsksq * 2;
1121	if (!num)
1122		return NULL;
1123
1124	qs = idpf_alloc_queue_set(vport, num);
1125	if (!qs)
1126		return NULL;
1127
1128	num = 0;
1129
1130	for (u32 i = 0; i < qv->num_bufq; i++) {
1131		qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER;
1132		qs->qs[num++].bufq = qv->bufq[i];
1133	}
1134
1135	for (u32 i = 0; i < qv->num_rxq; i++) {
1136		qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX;
1137		qs->qs[num++].rxq = qv->rx[i];
1138	}
1139
1140	for (u32 i = 0; i < qv->num_txq; i++) {
1141		qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX;
1142		qs->qs[num++].txq = qv->tx[i];
1143	}
1144
1145	for (u32 i = 0; i < qv->num_complq; i++) {
1146		qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION;
1147		qs->qs[num++].complq = qv->complq[i];
1148	}
1149
1150	if (!vport->xdp_txq_offset)
1151		goto finalize;
1152
1153	if (xdp) {
1154		for (u32 i = 0; i < qv->num_rxq; i++) {
1155			u32 idx = vport->xdp_txq_offset + qv->rx[i]->idx;
1156
1157			qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX;
1158			qs->qs[num++].txq = vport->txqs[idx];
1159
1160			qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION;
1161			qs->qs[num++].complq = vport->txqs[idx]->complq;
1162		}
1163	} else {
1164		for (u32 i = 0; i < qv->num_xsksq; i++) {
1165			qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX;
1166			qs->qs[num++].txq = qv->xsksq[i];
1167
1168			qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION;
1169			qs->qs[num++].complq = qv->xsksq[i]->complq;
1170		}
1171	}
1172
1173finalize:
1174	if (num != qs->num) {
1175		kfree(qs);
1176		return NULL;
1177	}
1178
1179	return qs;
1180}
1181
1182static int idpf_qp_enable(const struct idpf_queue_set *qs, u32 qid)
1183{
1184	struct idpf_vport *vport = qs->vport;
1185	struct idpf_q_vector *q_vector;
1186	int err;
1187
1188	q_vector = idpf_find_rxq_vec(vport, qid);
1189
1190	err = idpf_init_queue_set(qs);
1191	if (err) {
1192		netdev_err(vport->netdev, "Could not initialize queues in pair %u: %pe\n",
1193			   qid, ERR_PTR(err));
1194		return err;
1195	}
1196
1197	if (!vport->xdp_txq_offset)
1198		goto config;
1199
1200	q_vector->xsksq = kcalloc(DIV_ROUND_UP(vport->num_rxq_grp,
1201					       vport->num_q_vectors),
1202				  sizeof(*q_vector->xsksq), GFP_KERNEL);
1203	if (!q_vector->xsksq)
1204		return -ENOMEM;
1205
1206	for (u32 i = 0; i < qs->num; i++) {
1207		const struct idpf_queue_ptr *q = &qs->qs[i];
1208
1209		if (q->type != VIRTCHNL2_QUEUE_TYPE_TX)
1210			continue;
1211
1212		if (!idpf_queue_has(XSK, q->txq))
1213			continue;
1214
1215		idpf_xsk_init_wakeup(q_vector);
1216
1217		q->txq->q_vector = q_vector;
1218		q_vector->xsksq[q_vector->num_xsksq++] = q->txq;
1219	}
1220
1221config:
1222	err = idpf_send_config_queue_set_msg(qs);
1223	if (err) {
1224		netdev_err(vport->netdev, "Could not configure queues in pair %u: %pe\n",
1225			   qid, ERR_PTR(err));
1226		return err;
1227	}
1228
1229	err = idpf_send_enable_queue_set_msg(qs);
1230	if (err) {
1231		netdev_err(vport->netdev, "Could not enable queues in pair %u: %pe\n",
1232			   qid, ERR_PTR(err));
1233		return err;
1234	}
1235
1236	napi_enable(&q_vector->napi);
1237	idpf_qvec_ena_irq(q_vector);
1238
1239	netif_start_subqueue(vport->netdev, qid);
1240
1241	return 0;
1242}
1243
1244static int idpf_qp_disable(const struct idpf_queue_set *qs, u32 qid)
1245{
1246	struct idpf_vport *vport = qs->vport;
1247	struct idpf_q_vector *q_vector;
1248	int err;
1249
1250	q_vector = idpf_find_rxq_vec(vport, qid);
1251	netif_stop_subqueue(vport->netdev, qid);
1252
1253	writel(0, q_vector->intr_reg.dyn_ctl);
1254	napi_disable(&q_vector->napi);
1255
1256	err = idpf_send_disable_queue_set_msg(qs);
1257	if (err) {
1258		netdev_err(vport->netdev, "Could not disable queues in pair %u: %pe\n",
1259			   qid, ERR_PTR(err));
1260		return err;
1261	}
1262
1263	idpf_clean_queue_set(qs);
1264
1265	kfree(q_vector->xsksq);
1266	q_vector->num_xsksq = 0;
1267
1268	return 0;
1269}
1270
1271/**
1272 * idpf_qp_switch - enable or disable queues associated with queue pair
1273 * @vport: vport to switch the pair for
1274 * @qid: index of the queue pair to switch
1275 * @en: whether to enable or disable the pair
1276 *
1277 * Return: 0 on success, -errno on failure.
1278 */
1279int idpf_qp_switch(struct idpf_vport *vport, u32 qid, bool en)
1280{
1281	struct idpf_q_vector *q_vector = idpf_find_rxq_vec(vport, qid);
1282	struct idpf_queue_set *qs __free(kfree) = NULL;
1283
1284	if (idpf_find_txq_vec(vport, qid) != q_vector)
1285		return -EINVAL;
1286
1287	qs = idpf_vector_to_queue_set(q_vector);
1288	if (!qs)
1289		return -ENOMEM;
1290
1291	return en ? idpf_qp_enable(qs, qid) : idpf_qp_disable(qs, qid);
1292}
1293
1294/**
1295 * idpf_txq_group_rel - Release all resources for txq groups
1296 * @vport: vport to release txq groups on
1297 */
1298static void idpf_txq_group_rel(struct idpf_vport *vport)
1299{
1300	bool split, flow_sch_en;
1301	int i, j;
1302
1303	if (!vport->txq_grps)
1304		return;
1305
1306	split = idpf_is_queue_model_split(vport->txq_model);
1307	flow_sch_en = !idpf_is_cap_ena(vport->adapter, IDPF_OTHER_CAPS,
1308				       VIRTCHNL2_CAP_SPLITQ_QSCHED);
1309
1310	for (i = 0; i < vport->num_txq_grp; i++) {
1311		struct idpf_txq_group *txq_grp = &vport->txq_grps[i];
1312
1313		for (j = 0; j < txq_grp->num_txq; j++) {
1314			if (flow_sch_en) {
1315				kfree(txq_grp->txqs[j]->refillq);
1316				txq_grp->txqs[j]->refillq = NULL;
1317			}
1318
1319			kfree(txq_grp->txqs[j]);
1320			txq_grp->txqs[j] = NULL;
1321		}
1322
1323		if (!split)
1324			continue;
1325
1326		kfree(txq_grp->complq);
1327		txq_grp->complq = NULL;
1328	}
1329	kfree(vport->txq_grps);
1330	vport->txq_grps = NULL;
1331}
1332
1333/**
1334 * idpf_rxq_sw_queue_rel - Release software queue resources
1335 * @rx_qgrp: rx queue group with software queues
1336 */
1337static void idpf_rxq_sw_queue_rel(struct idpf_rxq_group *rx_qgrp)
1338{
1339	int i, j;
1340
1341	for (i = 0; i < rx_qgrp->vport->num_bufqs_per_qgrp; i++) {
1342		struct idpf_bufq_set *bufq_set = &rx_qgrp->splitq.bufq_sets[i];
1343
1344		for (j = 0; j < bufq_set->num_refillqs; j++) {
1345			kfree(bufq_set->refillqs[j].ring);
1346			bufq_set->refillqs[j].ring = NULL;
1347		}
1348		kfree(bufq_set->refillqs);
1349		bufq_set->refillqs = NULL;
1350	}
1351}
1352
1353/**
1354 * idpf_rxq_group_rel - Release all resources for rxq groups
1355 * @vport: vport to release rxq groups on
1356 */
1357static void idpf_rxq_group_rel(struct idpf_vport *vport)
1358{
1359	int i;
1360
1361	if (!vport->rxq_grps)
1362		return;
1363
1364	for (i = 0; i < vport->num_rxq_grp; i++) {
1365		struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
1366		u16 num_rxq;
1367		int j;
1368
1369		if (idpf_is_queue_model_split(vport->rxq_model)) {
1370			num_rxq = rx_qgrp->splitq.num_rxq_sets;
1371			for (j = 0; j < num_rxq; j++) {
1372				kfree(rx_qgrp->splitq.rxq_sets[j]);
1373				rx_qgrp->splitq.rxq_sets[j] = NULL;
1374			}
1375
1376			idpf_rxq_sw_queue_rel(rx_qgrp);
1377			kfree(rx_qgrp->splitq.bufq_sets);
1378			rx_qgrp->splitq.bufq_sets = NULL;
1379		} else {
1380			num_rxq = rx_qgrp->singleq.num_rxq;
1381			for (j = 0; j < num_rxq; j++) {
1382				kfree(rx_qgrp->singleq.rxqs[j]);
1383				rx_qgrp->singleq.rxqs[j] = NULL;
1384			}
1385		}
1386	}
1387	kfree(vport->rxq_grps);
1388	vport->rxq_grps = NULL;
1389}
1390
1391/**
1392 * idpf_vport_queue_grp_rel_all - Release all queue groups
1393 * @vport: vport to release queue groups for
1394 */
1395static void idpf_vport_queue_grp_rel_all(struct idpf_vport *vport)
1396{
1397	idpf_txq_group_rel(vport);
1398	idpf_rxq_group_rel(vport);
1399}
1400
1401/**
1402 * idpf_vport_queues_rel - Free memory for all queues
1403 * @vport: virtual port
1404 *
1405 * Free the memory allocated for queues associated to a vport
1406 */
1407void idpf_vport_queues_rel(struct idpf_vport *vport)
1408{
1409	idpf_xdp_copy_prog_to_rqs(vport, NULL);
1410
1411	idpf_tx_desc_rel_all(vport);
1412	idpf_rx_desc_rel_all(vport);
1413
1414	idpf_xdpsqs_put(vport);
1415	idpf_vport_queue_grp_rel_all(vport);
1416
1417	kfree(vport->txqs);
1418	vport->txqs = NULL;
1419}
1420
1421/**
1422 * idpf_vport_init_fast_path_txqs - Initialize fast path txq array
1423 * @vport: vport to init txqs on
1424 *
1425 * We get a queue index from skb->queue_mapping and we need a fast way to
1426 * dereference the queue from queue groups.  This allows us to quickly pull a
1427 * txq based on a queue index.
1428 *
1429 * Returns 0 on success, negative on failure
1430 */
1431static int idpf_vport_init_fast_path_txqs(struct idpf_vport *vport)
1432{
1433	struct idpf_ptp_vport_tx_tstamp_caps *caps = vport->tx_tstamp_caps;
1434	struct work_struct *tstamp_task = &vport->tstamp_task;
1435	int i, j, k = 0;
1436
1437	vport->txqs = kcalloc(vport->num_txq, sizeof(*vport->txqs),
1438			      GFP_KERNEL);
1439
1440	if (!vport->txqs)
1441		return -ENOMEM;
1442
1443	for (i = 0; i < vport->num_txq_grp; i++) {
1444		struct idpf_txq_group *tx_grp = &vport->txq_grps[i];
1445
1446		for (j = 0; j < tx_grp->num_txq; j++, k++) {
1447			vport->txqs[k] = tx_grp->txqs[j];
1448			vport->txqs[k]->idx = k;
1449
1450			if (!caps)
1451				continue;
1452
1453			vport->txqs[k]->cached_tstamp_caps = caps;
1454			vport->txqs[k]->tstamp_task = tstamp_task;
1455		}
1456	}
1457
1458	return 0;
1459}
1460
1461/**
1462 * idpf_vport_init_num_qs - Initialize number of queues
1463 * @vport: vport to initialize queues
1464 * @vport_msg: data to be filled into vport
1465 */
1466void idpf_vport_init_num_qs(struct idpf_vport *vport,
1467			    struct virtchnl2_create_vport *vport_msg)
1468{
1469	struct idpf_vport_user_config_data *config_data;
1470	u16 idx = vport->idx;
1471
1472	config_data = &vport->adapter->vport_config[idx]->user_config;
1473	vport->num_txq = le16_to_cpu(vport_msg->num_tx_q);
1474	vport->num_rxq = le16_to_cpu(vport_msg->num_rx_q);
1475	/* number of txqs and rxqs in config data will be zeros only in the
1476	 * driver load path and we dont update them there after
1477	 */
1478	if (!config_data->num_req_tx_qs && !config_data->num_req_rx_qs) {
1479		config_data->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q);
1480		config_data->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q);
1481	}
1482
1483	if (idpf_is_queue_model_split(vport->txq_model))
1484		vport->num_complq = le16_to_cpu(vport_msg->num_tx_complq);
1485	if (idpf_is_queue_model_split(vport->rxq_model))
1486		vport->num_bufq = le16_to_cpu(vport_msg->num_rx_bufq);
1487
1488	vport->xdp_prog = config_data->xdp_prog;
1489	if (idpf_xdp_enabled(vport)) {
1490		vport->xdp_txq_offset = config_data->num_req_tx_qs;
1491		vport->num_xdp_txq = le16_to_cpu(vport_msg->num_tx_q) -
1492				     vport->xdp_txq_offset;
1493		vport->xdpsq_share = libeth_xdpsq_shared(vport->num_xdp_txq);
1494	} else {
1495		vport->xdp_txq_offset = 0;
1496		vport->num_xdp_txq = 0;
1497		vport->xdpsq_share = false;
1498	}
1499
1500	/* Adjust number of buffer queues per Rx queue group. */
1501	if (!idpf_is_queue_model_split(vport->rxq_model)) {
1502		vport->num_bufqs_per_qgrp = 0;
1503
1504		return;
1505	}
1506
1507	vport->num_bufqs_per_qgrp = IDPF_MAX_BUFQS_PER_RXQ_GRP;
1508}
1509
1510/**
1511 * idpf_vport_calc_num_q_desc - Calculate number of queue groups
1512 * @vport: vport to calculate q groups for
1513 */
1514void idpf_vport_calc_num_q_desc(struct idpf_vport *vport)
1515{
1516	struct idpf_vport_user_config_data *config_data;
1517	int num_bufqs = vport->num_bufqs_per_qgrp;
1518	u32 num_req_txq_desc, num_req_rxq_desc;
1519	u16 idx = vport->idx;
1520	int i;
1521
1522	config_data =  &vport->adapter->vport_config[idx]->user_config;
1523	num_req_txq_desc = config_data->num_req_txq_desc;
1524	num_req_rxq_desc = config_data->num_req_rxq_desc;
1525
1526	vport->complq_desc_count = 0;
1527	if (num_req_txq_desc) {
1528		vport->txq_desc_count = num_req_txq_desc;
1529		if (idpf_is_queue_model_split(vport->txq_model)) {
1530			vport->complq_desc_count = num_req_txq_desc;
1531			if (vport->complq_desc_count < IDPF_MIN_TXQ_COMPLQ_DESC)
1532				vport->complq_desc_count =
1533					IDPF_MIN_TXQ_COMPLQ_DESC;
1534		}
1535	} else {
1536		vport->txq_desc_count =	IDPF_DFLT_TX_Q_DESC_COUNT;
1537		if (idpf_is_queue_model_split(vport->txq_model))
1538			vport->complq_desc_count =
1539				IDPF_DFLT_TX_COMPLQ_DESC_COUNT;
1540	}
1541
1542	if (num_req_rxq_desc)
1543		vport->rxq_desc_count = num_req_rxq_desc;
1544	else
1545		vport->rxq_desc_count = IDPF_DFLT_RX_Q_DESC_COUNT;
1546
1547	for (i = 0; i < num_bufqs; i++) {
1548		if (!vport->bufq_desc_count[i])
1549			vport->bufq_desc_count[i] =
1550				IDPF_RX_BUFQ_DESC_COUNT(vport->rxq_desc_count,
1551							num_bufqs);
1552	}
1553}
1554
1555/**
1556 * idpf_vport_calc_total_qs - Calculate total number of queues
1557 * @adapter: private data struct
1558 * @vport_idx: vport idx to retrieve vport pointer
1559 * @vport_msg: message to fill with data
1560 * @max_q: vport max queue info
1561 *
1562 * Return 0 on success, error value on failure.
1563 */
1564int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx,
1565			     struct virtchnl2_create_vport *vport_msg,
1566			     struct idpf_vport_max_q *max_q)
1567{
1568	int dflt_splitq_txq_grps = 0, dflt_singleq_txqs = 0;
1569	int dflt_splitq_rxq_grps = 0, dflt_singleq_rxqs = 0;
1570	u16 num_req_tx_qs = 0, num_req_rx_qs = 0;
1571	struct idpf_vport_user_config_data *user;
1572	struct idpf_vport_config *vport_config;
1573	u16 num_txq_grps, num_rxq_grps;
1574	u32 num_qs, num_xdpsq;
1575
1576	vport_config = adapter->vport_config[vport_idx];
1577	if (vport_config) {
1578		num_req_tx_qs = vport_config->user_config.num_req_tx_qs;
1579		num_req_rx_qs = vport_config->user_config.num_req_rx_qs;
1580	} else {
1581		u32 num_cpus = netif_get_num_default_rss_queues();
1582
1583		dflt_splitq_txq_grps = min_t(int, max_q->max_txq, num_cpus);
1584		dflt_singleq_txqs = min_t(int, max_q->max_txq, num_cpus);
1585		dflt_splitq_rxq_grps = min_t(int, max_q->max_rxq, num_cpus);
1586		dflt_singleq_rxqs = min_t(int, max_q->max_rxq, num_cpus);
1587	}
1588
1589	if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model))) {
1590		num_txq_grps = num_req_tx_qs ? num_req_tx_qs : dflt_splitq_txq_grps;
1591		vport_msg->num_tx_complq = cpu_to_le16(num_txq_grps *
1592						       IDPF_COMPLQ_PER_GROUP);
1593		vport_msg->num_tx_q = cpu_to_le16(num_txq_grps *
1594						  IDPF_DFLT_SPLITQ_TXQ_PER_GROUP);
1595	} else {
1596		num_txq_grps = IDPF_DFLT_SINGLEQ_TX_Q_GROUPS;
1597		num_qs = num_txq_grps * (num_req_tx_qs ? num_req_tx_qs :
1598					 dflt_singleq_txqs);
1599		vport_msg->num_tx_q = cpu_to_le16(num_qs);
1600		vport_msg->num_tx_complq = 0;
1601	}
1602	if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->rxq_model))) {
1603		num_rxq_grps = num_req_rx_qs ? num_req_rx_qs : dflt_splitq_rxq_grps;
1604		vport_msg->num_rx_bufq = cpu_to_le16(num_rxq_grps *
1605						     IDPF_MAX_BUFQS_PER_RXQ_GRP);
1606		vport_msg->num_rx_q = cpu_to_le16(num_rxq_grps *
1607						  IDPF_DFLT_SPLITQ_RXQ_PER_GROUP);
1608	} else {
1609		num_rxq_grps = IDPF_DFLT_SINGLEQ_RX_Q_GROUPS;
1610		num_qs = num_rxq_grps * (num_req_rx_qs ? num_req_rx_qs :
1611					 dflt_singleq_rxqs);
1612		vport_msg->num_rx_q = cpu_to_le16(num_qs);
1613		vport_msg->num_rx_bufq = 0;
1614	}
1615
1616	if (!vport_config)
1617		return 0;
1618
1619	user = &vport_config->user_config;
1620	user->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q);
1621	user->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q);
1622
1623	if (vport_config->user_config.xdp_prog)
1624		num_xdpsq = libeth_xdpsq_num(user->num_req_rx_qs,
1625					     user->num_req_tx_qs,
1626					     vport_config->max_q.max_txq);
1627	else
1628		num_xdpsq = 0;
1629
1630	vport_msg->num_tx_q = cpu_to_le16(user->num_req_tx_qs + num_xdpsq);
1631	if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model)))
1632		vport_msg->num_tx_complq = vport_msg->num_tx_q;
1633
1634	return 0;
1635}
1636
1637/**
1638 * idpf_vport_calc_num_q_groups - Calculate number of queue groups
1639 * @vport: vport to calculate q groups for
1640 */
1641void idpf_vport_calc_num_q_groups(struct idpf_vport *vport)
1642{
1643	if (idpf_is_queue_model_split(vport->txq_model))
1644		vport->num_txq_grp = vport->num_txq;
1645	else
1646		vport->num_txq_grp = IDPF_DFLT_SINGLEQ_TX_Q_GROUPS;
1647
1648	if (idpf_is_queue_model_split(vport->rxq_model))
1649		vport->num_rxq_grp = vport->num_rxq;
1650	else
1651		vport->num_rxq_grp = IDPF_DFLT_SINGLEQ_RX_Q_GROUPS;
1652}
1653
1654/**
1655 * idpf_vport_calc_numq_per_grp - Calculate number of queues per group
1656 * @vport: vport to calculate queues for
1657 * @num_txq: return parameter for number of TX queues
1658 * @num_rxq: return parameter for number of RX queues
1659 */
1660static void idpf_vport_calc_numq_per_grp(struct idpf_vport *vport,
1661					 u16 *num_txq, u16 *num_rxq)
1662{
1663	if (idpf_is_queue_model_split(vport->txq_model))
1664		*num_txq = IDPF_DFLT_SPLITQ_TXQ_PER_GROUP;
1665	else
1666		*num_txq = vport->num_txq;
1667
1668	if (idpf_is_queue_model_split(vport->rxq_model))
1669		*num_rxq = IDPF_DFLT_SPLITQ_RXQ_PER_GROUP;
1670	else
1671		*num_rxq = vport->num_rxq;
1672}
1673
1674/**
1675 * idpf_rxq_set_descids - set the descids supported by this queue
1676 * @vport: virtual port data structure
1677 * @q: rx queue for which descids are set
1678 *
1679 */
1680static void idpf_rxq_set_descids(const struct idpf_vport *vport,
1681				 struct idpf_rx_queue *q)
1682{
1683	if (idpf_is_queue_model_split(vport->rxq_model))
1684		return;
1685
1686	if (vport->base_rxd)
1687		q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M;
1688	else
1689		q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M;
1690}
1691
1692/**
1693 * idpf_txq_group_alloc - Allocate all txq group resources
1694 * @vport: vport to allocate txq groups for
1695 * @num_txq: number of txqs to allocate for each group
1696 *
1697 * Returns 0 on success, negative on failure
1698 */
1699static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq)
1700{
1701	bool split, flow_sch_en;
1702	int i;
1703
1704	vport->txq_grps = kcalloc(vport->num_txq_grp,
1705				  sizeof(*vport->txq_grps), GFP_KERNEL);
1706	if (!vport->txq_grps)
1707		return -ENOMEM;
1708
1709	split = idpf_is_queue_model_split(vport->txq_model);
1710	flow_sch_en = !idpf_is_cap_ena(vport->adapter, IDPF_OTHER_CAPS,
1711				       VIRTCHNL2_CAP_SPLITQ_QSCHED);
1712
1713	for (i = 0; i < vport->num_txq_grp; i++) {
1714		struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i];
1715		struct idpf_adapter *adapter = vport->adapter;
1716		int j;
1717
1718		tx_qgrp->vport = vport;
1719		tx_qgrp->num_txq = num_txq;
1720
1721		for (j = 0; j < tx_qgrp->num_txq; j++) {
1722			tx_qgrp->txqs[j] = kzalloc(sizeof(*tx_qgrp->txqs[j]),
1723						   GFP_KERNEL);
1724			if (!tx_qgrp->txqs[j])
1725				goto err_alloc;
1726		}
1727
1728		for (j = 0; j < tx_qgrp->num_txq; j++) {
1729			struct idpf_tx_queue *q = tx_qgrp->txqs[j];
1730
1731			q->dev = &adapter->pdev->dev;
1732			q->desc_count = vport->txq_desc_count;
1733			q->tx_max_bufs = idpf_get_max_tx_bufs(adapter);
1734			q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter);
1735			q->netdev = vport->netdev;
1736			q->txq_grp = tx_qgrp;
1737			q->rel_q_id = j;
1738
1739			if (!split) {
1740				q->clean_budget = vport->compln_clean_budget;
1741				idpf_queue_assign(CRC_EN, q,
1742						  vport->crc_enable);
1743			}
1744
1745			if (!flow_sch_en)
1746				continue;
1747
1748			idpf_queue_set(FLOW_SCH_EN, q);
1749
1750			q->refillq = kzalloc(sizeof(*q->refillq), GFP_KERNEL);
1751			if (!q->refillq)
1752				goto err_alloc;
1753
1754			idpf_queue_set(GEN_CHK, q->refillq);
1755			idpf_queue_set(RFL_GEN_CHK, q->refillq);
1756		}
1757
1758		if (!split)
1759			continue;
1760
1761		tx_qgrp->complq = kcalloc(IDPF_COMPLQ_PER_GROUP,
1762					  sizeof(*tx_qgrp->complq),
1763					  GFP_KERNEL);
1764		if (!tx_qgrp->complq)
1765			goto err_alloc;
1766
1767		tx_qgrp->complq->desc_count = vport->complq_desc_count;
1768		tx_qgrp->complq->txq_grp = tx_qgrp;
1769		tx_qgrp->complq->netdev = vport->netdev;
1770		tx_qgrp->complq->clean_budget = vport->compln_clean_budget;
1771
1772		if (flow_sch_en)
1773			idpf_queue_set(FLOW_SCH_EN, tx_qgrp->complq);
1774	}
1775
1776	return 0;
1777
1778err_alloc:
1779	idpf_txq_group_rel(vport);
1780
1781	return -ENOMEM;
1782}
1783
1784/**
1785 * idpf_rxq_group_alloc - Allocate all rxq group resources
1786 * @vport: vport to allocate rxq groups for
1787 * @num_rxq: number of rxqs to allocate for each group
1788 *
1789 * Returns 0 on success, negative on failure
1790 */
1791static int idpf_rxq_group_alloc(struct idpf_vport *vport, u16 num_rxq)
1792{
1793	int i, k, err = 0;
1794	bool hs;
1795
1796	vport->rxq_grps = kcalloc(vport->num_rxq_grp,
1797				  sizeof(struct idpf_rxq_group), GFP_KERNEL);
1798	if (!vport->rxq_grps)
1799		return -ENOMEM;
1800
1801	hs = idpf_vport_get_hsplit(vport) == ETHTOOL_TCP_DATA_SPLIT_ENABLED;
1802
1803	for (i = 0; i < vport->num_rxq_grp; i++) {
1804		struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
1805		int j;
1806
1807		rx_qgrp->vport = vport;
1808		if (!idpf_is_queue_model_split(vport->rxq_model)) {
1809			rx_qgrp->singleq.num_rxq = num_rxq;
1810			for (j = 0; j < num_rxq; j++) {
1811				rx_qgrp->singleq.rxqs[j] =
1812						kzalloc(sizeof(*rx_qgrp->singleq.rxqs[j]),
1813							GFP_KERNEL);
1814				if (!rx_qgrp->singleq.rxqs[j]) {
1815					err = -ENOMEM;
1816					goto err_alloc;
1817				}
1818			}
1819			goto skip_splitq_rx_init;
1820		}
1821		rx_qgrp->splitq.num_rxq_sets = num_rxq;
1822
1823		for (j = 0; j < num_rxq; j++) {
1824			rx_qgrp->splitq.rxq_sets[j] =
1825				kzalloc(sizeof(struct idpf_rxq_set),
1826					GFP_KERNEL);
1827			if (!rx_qgrp->splitq.rxq_sets[j]) {
1828				err = -ENOMEM;
1829				goto err_alloc;
1830			}
1831		}
1832
1833		rx_qgrp->splitq.bufq_sets = kcalloc(vport->num_bufqs_per_qgrp,
1834						    sizeof(struct idpf_bufq_set),
1835						    GFP_KERNEL);
1836		if (!rx_qgrp->splitq.bufq_sets) {
1837			err = -ENOMEM;
1838			goto err_alloc;
1839		}
1840
1841		for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
1842			struct idpf_bufq_set *bufq_set =
1843				&rx_qgrp->splitq.bufq_sets[j];
1844			int swq_size = sizeof(struct idpf_sw_queue);
1845			struct idpf_buf_queue *q;
1846
1847			q = &rx_qgrp->splitq.bufq_sets[j].bufq;
1848			q->desc_count = vport->bufq_desc_count[j];
1849			q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK;
1850
1851			idpf_queue_assign(HSPLIT_EN, q, hs);
1852
1853			bufq_set->num_refillqs = num_rxq;
1854			bufq_set->refillqs = kcalloc(num_rxq, swq_size,
1855						     GFP_KERNEL);
1856			if (!bufq_set->refillqs) {
1857				err = -ENOMEM;
1858				goto err_alloc;
1859			}
1860			for (k = 0; k < bufq_set->num_refillqs; k++) {
1861				struct idpf_sw_queue *refillq =
1862					&bufq_set->refillqs[k];
1863
1864				refillq->desc_count =
1865					vport->bufq_desc_count[j];
1866				idpf_queue_set(GEN_CHK, refillq);
1867				idpf_queue_set(RFL_GEN_CHK, refillq);
1868				refillq->ring = kcalloc(refillq->desc_count,
1869							sizeof(*refillq->ring),
1870							GFP_KERNEL);
1871				if (!refillq->ring) {
1872					err = -ENOMEM;
1873					goto err_alloc;
1874				}
1875			}
1876		}
1877
1878skip_splitq_rx_init:
1879		for (j = 0; j < num_rxq; j++) {
1880			struct idpf_rx_queue *q;
1881
1882			if (!idpf_is_queue_model_split(vport->rxq_model)) {
1883				q = rx_qgrp->singleq.rxqs[j];
1884				goto setup_rxq;
1885			}
1886			q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
1887			rx_qgrp->splitq.rxq_sets[j]->refillq[0] =
1888			      &rx_qgrp->splitq.bufq_sets[0].refillqs[j];
1889			if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP)
1890				rx_qgrp->splitq.rxq_sets[j]->refillq[1] =
1891				      &rx_qgrp->splitq.bufq_sets[1].refillqs[j];
1892
1893			idpf_queue_assign(HSPLIT_EN, q, hs);
1894
1895setup_rxq:
1896			q->desc_count = vport->rxq_desc_count;
1897			q->rx_ptype_lkup = vport->rx_ptype_lkup;
1898			q->bufq_sets = rx_qgrp->splitq.bufq_sets;
1899			q->idx = (i * num_rxq) + j;
1900			q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK;
1901			q->rx_max_pkt_size = vport->netdev->mtu +
1902							LIBETH_RX_LL_LEN;
1903			idpf_rxq_set_descids(vport, q);
1904		}
1905	}
1906
1907err_alloc:
1908	if (err)
1909		idpf_rxq_group_rel(vport);
1910
1911	return err;
1912}
1913
1914/**
1915 * idpf_vport_queue_grp_alloc_all - Allocate all queue groups/resources
1916 * @vport: vport with qgrps to allocate
1917 *
1918 * Returns 0 on success, negative on failure
1919 */
1920static int idpf_vport_queue_grp_alloc_all(struct idpf_vport *vport)
1921{
1922	u16 num_txq, num_rxq;
1923	int err;
1924
1925	idpf_vport_calc_numq_per_grp(vport, &num_txq, &num_rxq);
1926
1927	err = idpf_txq_group_alloc(vport, num_txq);
1928	if (err)
1929		goto err_out;
1930
1931	err = idpf_rxq_group_alloc(vport, num_rxq);
1932	if (err)
1933		goto err_out;
1934
1935	return 0;
1936
1937err_out:
1938	idpf_vport_queue_grp_rel_all(vport);
1939
1940	return err;
1941}
1942
1943/**
1944 * idpf_vport_queues_alloc - Allocate memory for all queues
1945 * @vport: virtual port
1946 *
1947 * Allocate memory for queues associated with a vport.  Returns 0 on success,
1948 * negative on failure.
1949 */
1950int idpf_vport_queues_alloc(struct idpf_vport *vport)
1951{
1952	int err;
1953
1954	err = idpf_vport_queue_grp_alloc_all(vport);
1955	if (err)
1956		goto err_out;
1957
1958	err = idpf_vport_init_fast_path_txqs(vport);
1959	if (err)
1960		goto err_out;
1961
1962	err = idpf_xdpsqs_get(vport);
1963	if (err)
1964		goto err_out;
1965
1966	err = idpf_tx_desc_alloc_all(vport);
1967	if (err)
1968		goto err_out;
1969
1970	err = idpf_rx_desc_alloc_all(vport);
1971	if (err)
1972		goto err_out;
1973
1974	return 0;
1975
1976err_out:
1977	idpf_vport_queues_rel(vport);
1978
1979	return err;
1980}
1981
1982/**
1983 * idpf_tx_read_tstamp - schedule a work to read Tx timestamp value
1984 * @txq: queue to read the timestamp from
1985 * @skb: socket buffer to provide Tx timestamp value
1986 *
1987 * Schedule a work to read Tx timestamp value generated once the packet is
1988 * transmitted.
1989 */
1990static void idpf_tx_read_tstamp(struct idpf_tx_queue *txq, struct sk_buff *skb)
1991{
1992	struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps;
1993	struct idpf_ptp_tx_tstamp_status *tx_tstamp_status;
1994
1995	tx_tstamp_caps = txq->cached_tstamp_caps;
1996	spin_lock_bh(&tx_tstamp_caps->status_lock);
1997
1998	for (u32 i = 0; i < tx_tstamp_caps->num_entries; i++) {
1999		tx_tstamp_status = &tx_tstamp_caps->tx_tstamp_status[i];
2000		if (tx_tstamp_status->state != IDPF_PTP_FREE)
2001			continue;
2002
2003		tx_tstamp_status->skb = skb;
2004		tx_tstamp_status->state = IDPF_PTP_REQUEST;
2005
2006		/* Fetch timestamp from completion descriptor through
2007		 * virtchnl msg to report to stack.
2008		 */
2009		queue_work(system_unbound_wq, txq->tstamp_task);
2010		break;
2011	}
2012
2013	spin_unlock_bh(&tx_tstamp_caps->status_lock);
2014}
2015
2016#define idpf_tx_splitq_clean_bump_ntc(txq, ntc, desc, buf)	\
2017do {								\
2018	if (unlikely(++(ntc) == (txq)->desc_count)) {		\
2019		ntc = 0;					\
2020		buf = (txq)->tx_buf;				\
2021		desc = &(txq)->flex_tx[0];			\
2022	} else {						\
2023		(buf)++;					\
2024		(desc)++;					\
2025	}							\
2026} while (0)
2027
2028/**
2029 * idpf_tx_splitq_clean - Reclaim resources from buffer queue
2030 * @tx_q: Tx queue to clean
2031 * @end: queue index until which it should be cleaned
2032 * @napi_budget: Used to determine if we are in netpoll
2033 * @cleaned: pointer to stats struct to track cleaned packets/bytes
2034 * @descs_only: true if queue is using flow-based scheduling and should
2035 * not clean buffers at this time
2036 *
2037 * Cleans the queue descriptor ring. If the queue is using queue-based
2038 * scheduling, the buffers will be cleaned as well. If the queue is using
2039 * flow-based scheduling, only the descriptors are cleaned at this time.
2040 * Separate packet completion events will be reported on the completion queue,
2041 * and the buffers will be cleaned separately. The stats are not updated from
2042 * this function when using flow-based scheduling.
2043 */
2044static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end,
2045				 int napi_budget,
2046				 struct libeth_sq_napi_stats *cleaned,
2047				 bool descs_only)
2048{
2049	union idpf_tx_flex_desc *next_pending_desc = NULL;
2050	union idpf_tx_flex_desc *tx_desc;
2051	u32 ntc = tx_q->next_to_clean;
2052	struct libeth_cq_pp cp = {
2053		.dev	= tx_q->dev,
2054		.ss	= cleaned,
2055		.napi	= napi_budget,
2056	};
2057	struct idpf_tx_buf *tx_buf;
2058
2059	if (descs_only) {
2060		/* Bump ring index to mark as cleaned. */
2061		tx_q->next_to_clean = end;
2062		return;
2063	}
2064
2065	tx_desc = &tx_q->flex_tx[ntc];
2066	next_pending_desc = &tx_q->flex_tx[end];
2067	tx_buf = &tx_q->tx_buf[ntc];
2068
2069	while (tx_desc != next_pending_desc) {
2070		u32 eop_idx;
2071
2072		/* If this entry in the ring was used as a context descriptor,
2073		 * it's corresponding entry in the buffer ring is reserved. We
2074		 * can skip this descriptor since there is no buffer to clean.
2075		 */
2076		if (tx_buf->type <= LIBETH_SQE_CTX)
2077			goto fetch_next_txq_desc;
2078
2079		if (unlikely(tx_buf->type != LIBETH_SQE_SKB))
2080			break;
2081
2082		eop_idx = tx_buf->rs_idx;
2083		libeth_tx_complete(tx_buf, &cp);
2084
2085		/* unmap remaining buffers */
2086		while (ntc != eop_idx) {
2087			idpf_tx_splitq_clean_bump_ntc(tx_q, ntc,
2088						      tx_desc, tx_buf);
2089
2090			/* unmap any remaining paged data */
2091			libeth_tx_complete(tx_buf, &cp);
2092		}
2093
2094fetch_next_txq_desc:
2095		idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, tx_desc, tx_buf);
2096	}
2097
2098	tx_q->next_to_clean = ntc;
2099}
2100
2101/**
2102 * idpf_tx_clean_bufs - clean flow scheduling TX queue buffers
2103 * @txq: queue to clean
2104 * @buf_id: packet's starting buffer ID, from completion descriptor
2105 * @cleaned: pointer to stats struct to track cleaned packets/bytes
2106 * @budget: Used to determine if we are in netpoll
2107 *
2108 * Clean all buffers associated with the packet starting at buf_id. Returns the
2109 * byte/segment count for the cleaned packet.
2110 */
2111static void idpf_tx_clean_bufs(struct idpf_tx_queue *txq, u32 buf_id,
2112			       struct libeth_sq_napi_stats *cleaned,
2113			       int budget)
2114{
2115	struct idpf_tx_buf *tx_buf = NULL;
2116	struct libeth_cq_pp cp = {
2117		.dev	= txq->dev,
2118		.ss	= cleaned,
2119		.napi	= budget,
2120	};
2121
2122	tx_buf = &txq->tx_buf[buf_id];
2123	if (tx_buf->type == LIBETH_SQE_SKB) {
2124		if (skb_shinfo(tx_buf->skb)->tx_flags & SKBTX_IN_PROGRESS)
2125			idpf_tx_read_tstamp(txq, tx_buf->skb);
2126
2127		libeth_tx_complete(tx_buf, &cp);
2128		idpf_post_buf_refill(txq->refillq, buf_id);
2129	}
2130
2131	while (idpf_tx_buf_next(tx_buf) != IDPF_TXBUF_NULL) {
2132		buf_id = idpf_tx_buf_next(tx_buf);
2133
2134		tx_buf = &txq->tx_buf[buf_id];
2135		libeth_tx_complete(tx_buf, &cp);
2136		idpf_post_buf_refill(txq->refillq, buf_id);
2137	}
2138}
2139
2140/**
2141 * idpf_tx_handle_rs_completion - clean a single packet and all of its buffers
2142 * whether on the buffer ring or in the hash table
2143 * @txq: Tx ring to clean
2144 * @desc: pointer to completion queue descriptor to extract completion
2145 * information from
2146 * @cleaned: pointer to stats struct to track cleaned packets/bytes
2147 * @budget: Used to determine if we are in netpoll
2148 *
2149 * Returns bytes/packets cleaned
2150 */
2151static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq,
2152					 struct idpf_splitq_tx_compl_desc *desc,
2153					 struct libeth_sq_napi_stats *cleaned,
2154					 int budget)
2155{
2156	/* RS completion contains queue head for queue based scheduling or
2157	 * completion tag for flow based scheduling.
2158	 */
2159	u16 rs_compl_val = le16_to_cpu(desc->common.q_head_compl_tag.q_head);
2160
2161	if (!idpf_queue_has(FLOW_SCH_EN, txq)) {
2162		idpf_tx_splitq_clean(txq, rs_compl_val, budget, cleaned, false);
2163		return;
2164	}
2165
2166	idpf_tx_clean_bufs(txq, rs_compl_val, cleaned, budget);
2167}
2168
2169/**
2170 * idpf_tx_clean_complq - Reclaim resources on completion queue
2171 * @complq: Tx ring to clean
2172 * @budget: Used to determine if we are in netpoll
2173 * @cleaned: returns number of packets cleaned
2174 *
2175 * Returns true if there's any budget left (e.g. the clean is finished)
2176 */
2177static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget,
2178				 int *cleaned)
2179{
2180	struct idpf_splitq_tx_compl_desc *tx_desc;
2181	s16 ntc = complq->next_to_clean;
2182	struct idpf_netdev_priv *np;
2183	unsigned int complq_budget;
2184	bool complq_ok = true;
2185	int i;
2186
2187	complq_budget = complq->clean_budget;
2188	tx_desc = &complq->comp[ntc];
2189	ntc -= complq->desc_count;
2190
2191	do {
2192		struct libeth_sq_napi_stats cleaned_stats = { };
2193		struct idpf_tx_queue *tx_q;
2194		__le16 hw_head;
2195		int rel_tx_qid;
2196		u8 ctype;	/* completion type */
2197		u16 gen;
2198
2199		/* if the descriptor isn't done, no work yet to do */
2200		gen = le16_get_bits(tx_desc->common.qid_comptype_gen,
2201				    IDPF_TXD_COMPLQ_GEN_M);
2202		if (idpf_queue_has(GEN_CHK, complq) != gen)
2203			break;
2204
2205		/* Find necessary info of TX queue to clean buffers */
2206		rel_tx_qid = le16_get_bits(tx_desc->common.qid_comptype_gen,
2207					   IDPF_TXD_COMPLQ_QID_M);
2208		if (rel_tx_qid >= complq->txq_grp->num_txq ||
2209		    !complq->txq_grp->txqs[rel_tx_qid]) {
2210			netdev_err(complq->netdev, "TxQ not found\n");
2211			goto fetch_next_desc;
2212		}
2213		tx_q = complq->txq_grp->txqs[rel_tx_qid];
2214
2215		/* Determine completion type */
2216		ctype = le16_get_bits(tx_desc->common.qid_comptype_gen,
2217				      IDPF_TXD_COMPLQ_COMPL_TYPE_M);
2218		switch (ctype) {
2219		case IDPF_TXD_COMPLT_RE:
2220			hw_head = tx_desc->common.q_head_compl_tag.q_head;
2221
2222			idpf_tx_splitq_clean(tx_q, le16_to_cpu(hw_head),
2223					     budget, &cleaned_stats, true);
2224			break;
2225		case IDPF_TXD_COMPLT_RS:
2226			idpf_tx_handle_rs_completion(tx_q, tx_desc,
2227						     &cleaned_stats, budget);
2228			break;
2229		default:
2230			netdev_err(tx_q->netdev,
2231				   "Unknown TX completion type: %d\n", ctype);
2232			goto fetch_next_desc;
2233		}
2234
2235		u64_stats_update_begin(&tx_q->stats_sync);
2236		u64_stats_add(&tx_q->q_stats.packets, cleaned_stats.packets);
2237		u64_stats_add(&tx_q->q_stats.bytes, cleaned_stats.bytes);
2238		tx_q->cleaned_pkts += cleaned_stats.packets;
2239		tx_q->cleaned_bytes += cleaned_stats.bytes;
2240		complq->num_completions++;
2241		u64_stats_update_end(&tx_q->stats_sync);
2242
2243fetch_next_desc:
2244		tx_desc++;
2245		ntc++;
2246		if (unlikely(!ntc)) {
2247			ntc -= complq->desc_count;
2248			tx_desc = &complq->comp[0];
2249			idpf_queue_change(GEN_CHK, complq);
2250		}
2251
2252		prefetch(tx_desc);
2253
2254		/* update budget accounting */
2255		complq_budget--;
2256	} while (likely(complq_budget));
2257
2258	/* Store the state of the complq to be used later in deciding if a
2259	 * TXQ can be started again
2260	 */
2261	if (unlikely(IDPF_TX_COMPLQ_PENDING(complq->txq_grp) >
2262		     IDPF_TX_COMPLQ_OVERFLOW_THRESH(complq)))
2263		complq_ok = false;
2264
2265	np = netdev_priv(complq->netdev);
2266	for (i = 0; i < complq->txq_grp->num_txq; ++i) {
2267		struct idpf_tx_queue *tx_q = complq->txq_grp->txqs[i];
2268		struct netdev_queue *nq;
2269		bool dont_wake;
2270
2271		/* We didn't clean anything on this queue, move along */
2272		if (!tx_q->cleaned_bytes)
2273			continue;
2274
2275		*cleaned += tx_q->cleaned_pkts;
2276
2277		/* Update BQL */
2278		nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx);
2279
2280		dont_wake = !complq_ok || !test_bit(IDPF_VPORT_UP, np->state) ||
2281			    !netif_carrier_ok(tx_q->netdev);
2282		/* Check if the TXQ needs to and can be restarted */
2283		__netif_txq_completed_wake(nq, tx_q->cleaned_pkts, tx_q->cleaned_bytes,
2284					   IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH,
2285					   dont_wake);
2286
2287		/* Reset cleaned stats for the next time this queue is
2288		 * cleaned
2289		 */
2290		tx_q->cleaned_bytes = 0;
2291		tx_q->cleaned_pkts = 0;
2292	}
2293
2294	ntc += complq->desc_count;
2295	complq->next_to_clean = ntc;
2296
2297	return !!complq_budget;
2298}
2299
2300/**
2301 * idpf_wait_for_sw_marker_completion - wait for SW marker of disabled Tx queue
2302 * @txq: disabled Tx queue
2303 *
2304 * When Tx queue is requested for disabling, the CP sends a special completion
2305 * descriptor called "SW marker", meaning the queue is ready to be destroyed.
2306 * If, for some reason, the marker is not received within 500 ms, break the
2307 * polling to not hang the driver.
2308 */
2309void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq)
2310{
2311	struct idpf_compl_queue *complq;
2312	unsigned long timeout;
2313	bool flow, gen_flag;
2314	u32 ntc;
2315
2316	if (!idpf_queue_has(SW_MARKER, txq))
2317		return;
2318
2319	complq = idpf_queue_has(XDP, txq) ? txq->complq : txq->txq_grp->complq;
2320	ntc = complq->next_to_clean;
2321
2322	flow = idpf_queue_has(FLOW_SCH_EN, complq);
2323	gen_flag = idpf_queue_has(GEN_CHK, complq);
2324
2325	timeout = jiffies + msecs_to_jiffies(IDPF_WAIT_FOR_MARKER_TIMEO);
2326
2327	do {
2328		struct idpf_splitq_4b_tx_compl_desc *tx_desc;
2329		struct idpf_tx_queue *target = NULL;
2330		u32 ctype_gen, id;
2331
2332		tx_desc = flow ? &complq->comp[ntc].common :
2333			  &complq->comp_4b[ntc];
2334		ctype_gen = le16_to_cpu(tx_desc->qid_comptype_gen);
2335
2336		if (!!(ctype_gen & IDPF_TXD_COMPLQ_GEN_M) != gen_flag) {
2337			usleep_range(500, 1000);
2338			continue;
2339		}
2340
2341		if (FIELD_GET(IDPF_TXD_COMPLQ_COMPL_TYPE_M, ctype_gen) !=
2342		    IDPF_TXD_COMPLT_SW_MARKER)
2343			goto next;
2344
2345		id = FIELD_GET(IDPF_TXD_COMPLQ_QID_M, ctype_gen);
2346		target = complq->txq_grp->txqs[id];
2347
2348		idpf_queue_clear(SW_MARKER, target);
2349
2350next:
2351		if (unlikely(++ntc == complq->desc_count)) {
2352			ntc = 0;
2353			gen_flag = !gen_flag;
2354		}
2355		if (target == txq)
2356			break;
2357	} while (time_before(jiffies, timeout));
2358
2359	idpf_queue_assign(GEN_CHK, complq, gen_flag);
2360	complq->next_to_clean = ntc;
2361}
2362
2363/**
2364 * idpf_tx_splitq_build_ctb - populate command tag and size for queue
2365 * based scheduling descriptors
2366 * @desc: descriptor to populate
2367 * @params: pointer to tx params struct
2368 * @td_cmd: command to be filled in desc
2369 * @size: size of buffer
2370 */
2371void idpf_tx_splitq_build_ctb(union idpf_tx_flex_desc *desc,
2372			      struct idpf_tx_splitq_params *params,
2373			      u16 td_cmd, u16 size)
2374{
2375	desc->q.qw1.cmd_dtype =
2376		le16_encode_bits(params->dtype, IDPF_FLEX_TXD_QW1_DTYPE_M);
2377	desc->q.qw1.cmd_dtype |=
2378		le16_encode_bits(td_cmd, IDPF_FLEX_TXD_QW1_CMD_M);
2379	desc->q.qw1.buf_size = cpu_to_le16(size);
2380	desc->q.qw1.l2tags.l2tag1 = cpu_to_le16(params->td_tag);
2381}
2382
2383/**
2384 * idpf_tx_splitq_build_flow_desc - populate command tag and size for flow
2385 * scheduling descriptors
2386 * @desc: descriptor to populate
2387 * @params: pointer to tx params struct
2388 * @td_cmd: command to be filled in desc
2389 * @size: size of buffer
2390 */
2391void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc,
2392				    struct idpf_tx_splitq_params *params,
2393				    u16 td_cmd, u16 size)
2394{
2395	*(u32 *)&desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd);
2396	desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size);
2397	desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag);
2398}
2399
2400/**
2401 * idpf_tx_splitq_has_room - check if enough Tx splitq resources are available
2402 * @tx_q: the queue to be checked
2403 * @descs_needed: number of descriptors required for this packet
2404 * @bufs_needed: number of Tx buffers required for this packet
2405 *
2406 * Return: 0 if no room available, 1 otherwise
2407 */
2408static int idpf_txq_has_room(struct idpf_tx_queue *tx_q, u32 descs_needed,
2409			     u32 bufs_needed)
2410{
2411	if (IDPF_DESC_UNUSED(tx_q) < descs_needed ||
2412	    IDPF_TX_COMPLQ_PENDING(tx_q->txq_grp) >
2413		IDPF_TX_COMPLQ_OVERFLOW_THRESH(tx_q->txq_grp->complq) ||
2414	    idpf_tx_splitq_get_free_bufs(tx_q->refillq) < bufs_needed)
2415		return 0;
2416	return 1;
2417}
2418
2419/**
2420 * idpf_tx_maybe_stop_splitq - 1st level check for Tx splitq stop conditions
2421 * @tx_q: the queue to be checked
2422 * @descs_needed: number of descriptors required for this packet
2423 * @bufs_needed: number of buffers needed for this packet
2424 *
2425 * Return: 0 if stop is not needed
2426 */
2427static int idpf_tx_maybe_stop_splitq(struct idpf_tx_queue *tx_q,
2428				     u32 descs_needed,
2429				     u32 bufs_needed)
2430{
2431	/* Since we have multiple resources to check for splitq, our
2432	 * start,stop_thrs becomes a boolean check instead of a count
2433	 * threshold.
2434	 */
2435	if (netif_subqueue_maybe_stop(tx_q->netdev, tx_q->idx,
2436				      idpf_txq_has_room(tx_q, descs_needed,
2437							bufs_needed),
2438				      1, 1))
2439		return 0;
2440
2441	u64_stats_update_begin(&tx_q->stats_sync);
2442	u64_stats_inc(&tx_q->q_stats.q_busy);
2443	u64_stats_update_end(&tx_q->stats_sync);
2444
2445	return -EBUSY;
2446}
2447
2448/**
2449 * idpf_tx_buf_hw_update - Store the new tail value
2450 * @tx_q: queue to bump
2451 * @val: new tail index
2452 * @xmit_more: more skb's pending
2453 *
2454 * The naming here is special in that 'hw' signals that this function is about
2455 * to do a register write to update our queue status. We know this can only
2456 * mean tail here as HW should be owning head for TX.
2457 */
2458void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val,
2459			   bool xmit_more)
2460{
2461	struct netdev_queue *nq;
2462
2463	nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx);
2464	tx_q->next_to_use = val;
2465
2466	/* Force memory writes to complete before letting h/w
2467	 * know there are new descriptors to fetch.  (Only
2468	 * applicable for weak-ordered memory model archs,
2469	 * such as IA-64).
2470	 */
2471	wmb();
2472
2473	/* notify HW of packet */
2474	if (netif_xmit_stopped(nq) || !xmit_more)
2475		writel(val, tx_q->tail);
2476}
2477
2478/**
2479 * idpf_tx_res_count_required - get number of Tx resources needed for this pkt
2480 * @txq: queue to send buffer on
2481 * @skb: send buffer
2482 * @bufs_needed: (output) number of buffers needed for this skb.
2483 *
2484 * Return: number of data descriptors and buffers needed for this skb.
2485 */
2486unsigned int idpf_tx_res_count_required(struct idpf_tx_queue *txq,
2487					struct sk_buff *skb,
2488					u32 *bufs_needed)
2489{
2490	const struct skb_shared_info *shinfo;
2491	unsigned int count = 0, i;
2492
2493	count += !!skb_headlen(skb);
2494
2495	if (!skb_is_nonlinear(skb))
2496		return count;
2497
2498	shinfo = skb_shinfo(skb);
2499	*bufs_needed += shinfo->nr_frags;
2500	for (i = 0; i < shinfo->nr_frags; i++) {
2501		unsigned int size;
2502
2503		size = skb_frag_size(&shinfo->frags[i]);
2504
2505		/* We only need to use the idpf_size_to_txd_count check if the
2506		 * fragment is going to span multiple descriptors,
2507		 * i.e. size >= 16K.
2508		 */
2509		if (size >= SZ_16K)
2510			count += idpf_size_to_txd_count(size);
2511		else
2512			count++;
2513	}
2514
2515	if (idpf_chk_linearize(skb, txq->tx_max_bufs, count)) {
2516		if (__skb_linearize(skb))
2517			return 0;
2518
2519		count = idpf_size_to_txd_count(skb->len);
2520		u64_stats_update_begin(&txq->stats_sync);
2521		u64_stats_inc(&txq->q_stats.linearize);
2522		u64_stats_update_end(&txq->stats_sync);
2523	}
2524
2525	return count;
2526}
2527
2528/**
2529 * idpf_tx_splitq_bump_ntu - adjust NTU and generation
2530 * @txq: the tx ring to wrap
2531 * @ntu: ring index to bump
2532 */
2533static unsigned int idpf_tx_splitq_bump_ntu(struct idpf_tx_queue *txq, u16 ntu)
2534{
2535	ntu++;
2536
2537	if (ntu == txq->desc_count)
2538		ntu = 0;
2539
2540	return ntu;
2541}
2542
2543/**
2544 * idpf_tx_get_free_buf_id - get a free buffer ID from the refill queue
2545 * @refillq: refill queue to get buffer ID from
2546 * @buf_id: return buffer ID
2547 *
2548 * Return: true if a buffer ID was found, false if not
2549 */
2550static bool idpf_tx_get_free_buf_id(struct idpf_sw_queue *refillq,
2551				    u32 *buf_id)
2552{
2553	u32 ntc = refillq->next_to_clean;
2554	u32 refill_desc;
2555
2556	refill_desc = refillq->ring[ntc];
2557
2558	if (unlikely(idpf_queue_has(RFL_GEN_CHK, refillq) !=
2559		     !!(refill_desc & IDPF_RFL_BI_GEN_M)))
2560		return false;
2561
2562	*buf_id = FIELD_GET(IDPF_RFL_BI_BUFID_M, refill_desc);
2563
2564	if (unlikely(++ntc == refillq->desc_count)) {
2565		idpf_queue_change(RFL_GEN_CHK, refillq);
2566		ntc = 0;
2567	}
2568
2569	refillq->next_to_clean = ntc;
2570
2571	return true;
2572}
2573
2574/**
2575 * idpf_tx_splitq_pkt_err_unmap - Unmap buffers and bump tail in case of error
2576 * @txq: Tx queue to unwind
2577 * @params: pointer to splitq params struct
2578 * @first: starting buffer for packet to unmap
2579 */
2580static void idpf_tx_splitq_pkt_err_unmap(struct idpf_tx_queue *txq,
2581					 struct idpf_tx_splitq_params *params,
2582					 struct idpf_tx_buf *first)
2583{
2584	struct idpf_sw_queue *refillq = txq->refillq;
2585	struct libeth_sq_napi_stats ss = { };
2586	struct idpf_tx_buf *tx_buf = first;
2587	struct libeth_cq_pp cp = {
2588		.dev    = txq->dev,
2589		.ss     = &ss,
2590	};
2591
2592	u64_stats_update_begin(&txq->stats_sync);
2593	u64_stats_inc(&txq->q_stats.dma_map_errs);
2594	u64_stats_update_end(&txq->stats_sync);
2595
2596	libeth_tx_complete(tx_buf, &cp);
2597	while (idpf_tx_buf_next(tx_buf) != IDPF_TXBUF_NULL) {
2598		tx_buf = &txq->tx_buf[idpf_tx_buf_next(tx_buf)];
2599		libeth_tx_complete(tx_buf, &cp);
2600	}
2601
2602	/* Update tail in case netdev_xmit_more was previously true. */
2603	idpf_tx_buf_hw_update(txq, params->prev_ntu, false);
2604
2605	if (!refillq)
2606		return;
2607
2608	/* Restore refillq state to avoid leaking tags. */
2609	if (params->prev_refill_gen != idpf_queue_has(RFL_GEN_CHK, refillq))
2610		idpf_queue_change(RFL_GEN_CHK, refillq);
2611	refillq->next_to_clean = params->prev_refill_ntc;
2612}
2613
2614/**
2615 * idpf_tx_splitq_map - Build the Tx flex descriptor
2616 * @tx_q: queue to send buffer on
2617 * @params: pointer to splitq params struct
2618 * @first: first buffer info buffer to use
2619 *
2620 * This function loops over the skb data pointed to by *first
2621 * and gets a physical address for each memory location and programs
2622 * it and the length into the transmit flex descriptor.
2623 */
2624static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q,
2625			       struct idpf_tx_splitq_params *params,
2626			       struct idpf_tx_buf *first)
2627{
2628	union idpf_tx_flex_desc *tx_desc;
2629	unsigned int data_len, size;
2630	struct idpf_tx_buf *tx_buf;
2631	u16 i = tx_q->next_to_use;
2632	struct netdev_queue *nq;
2633	struct sk_buff *skb;
2634	skb_frag_t *frag;
2635	u32 next_buf_id;
2636	u16 td_cmd = 0;
2637	dma_addr_t dma;
2638
2639	skb = first->skb;
2640
2641	td_cmd = params->offload.td_cmd;
2642
2643	data_len = skb->data_len;
2644	size = skb_headlen(skb);
2645
2646	tx_desc = &tx_q->flex_tx[i];
2647
2648	dma = dma_map_single(tx_q->dev, skb->data, size, DMA_TO_DEVICE);
2649
2650	tx_buf = first;
2651	first->nr_frags = 0;
2652
2653	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2654		unsigned int max_data = IDPF_TX_MAX_DESC_DATA_ALIGNED;
2655
2656		if (unlikely(dma_mapping_error(tx_q->dev, dma))) {
2657			idpf_tx_buf_next(tx_buf) = IDPF_TXBUF_NULL;
2658			return idpf_tx_splitq_pkt_err_unmap(tx_q, params,
2659							    first);
2660		}
2661
2662		first->nr_frags++;
2663		tx_buf->type = LIBETH_SQE_FRAG;
2664
2665		/* record length, and DMA address */
2666		dma_unmap_len_set(tx_buf, len, size);
2667		dma_unmap_addr_set(tx_buf, dma, dma);
2668
2669		/* buf_addr is in same location for both desc types */
2670		tx_desc->q.buf_addr = cpu_to_le64(dma);
2671
2672		/* The stack can send us fragments that are too large for a
2673		 * single descriptor i.e. frag size > 16K-1. We will need to
2674		 * split the fragment across multiple descriptors in this case.
2675		 * To adhere to HW alignment restrictions, the fragment needs
2676		 * to be split such that the first chunk ends on a 4K boundary
2677		 * and all subsequent chunks start on a 4K boundary. We still
2678		 * want to send as much data as possible though, so our
2679		 * intermediate descriptor chunk size will be 12K.
2680		 *
2681		 * For example, consider a 32K fragment mapped to DMA addr 2600.
2682		 * ------------------------------------------------------------
2683		 * |                    frag_size = 32K                       |
2684		 * ------------------------------------------------------------
2685		 * |2600		  |16384	    |28672
2686		 *
2687		 * 3 descriptors will be used for this fragment. The HW expects
2688		 * the descriptors to contain the following:
2689		 * ------------------------------------------------------------
2690		 * | size = 13784         | size = 12K      | size = 6696     |
2691		 * | dma = 2600           | dma = 16384     | dma = 28672     |
2692		 * ------------------------------------------------------------
2693		 *
2694		 * We need to first adjust the max_data for the first chunk so
2695		 * that it ends on a 4K boundary. By negating the value of the
2696		 * DMA address and taking only the low order bits, we're
2697		 * effectively calculating
2698		 *	4K - (DMA addr lower order bits) =
2699		 *				bytes to next boundary.
2700		 *
2701		 * Add that to our base aligned max_data (12K) and we have
2702		 * our first chunk size. In the example above,
2703		 *	13784 = 12K + (4096-2600)
2704		 *
2705		 * After guaranteeing the first chunk ends on a 4K boundary, we
2706		 * will give the intermediate descriptors 12K chunks and
2707		 * whatever is left to the final descriptor. This ensures that
2708		 * all descriptors used for the remaining chunks of the
2709		 * fragment start on a 4K boundary and we use as few
2710		 * descriptors as possible.
2711		 */
2712		max_data += -dma & (IDPF_TX_MAX_READ_REQ_SIZE - 1);
2713		while (unlikely(size > IDPF_TX_MAX_DESC_DATA)) {
2714			idpf_tx_splitq_build_desc(tx_desc, params, td_cmd,
2715						  max_data);
2716
2717			if (unlikely(++i == tx_q->desc_count)) {
2718				tx_desc = &tx_q->flex_tx[0];
2719				i = 0;
2720			} else {
2721				tx_desc++;
2722			}
2723
2724			/* Adjust the DMA offset and the remaining size of the
2725			 * fragment.  On the first iteration of this loop,
2726			 * max_data will be >= 12K and <= 16K-1.  On any
2727			 * subsequent iteration of this loop, max_data will
2728			 * always be 12K.
2729			 */
2730			dma += max_data;
2731			size -= max_data;
2732
2733			/* Reset max_data since remaining chunks will be 12K
2734			 * at most
2735			 */
2736			max_data = IDPF_TX_MAX_DESC_DATA_ALIGNED;
2737
2738			/* buf_addr is in same location for both desc types */
2739			tx_desc->q.buf_addr = cpu_to_le64(dma);
2740		}
2741
2742		if (!data_len)
2743			break;
2744
2745		idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size);
2746
2747		if (unlikely(++i == tx_q->desc_count)) {
2748			tx_desc = &tx_q->flex_tx[0];
2749			i = 0;
2750		} else {
2751			tx_desc++;
2752		}
2753
2754		if (idpf_queue_has(FLOW_SCH_EN, tx_q)) {
2755			if (unlikely(!idpf_tx_get_free_buf_id(tx_q->refillq,
2756							      &next_buf_id))) {
2757				idpf_tx_buf_next(tx_buf) = IDPF_TXBUF_NULL;
2758				return idpf_tx_splitq_pkt_err_unmap(tx_q, params,
2759								    first);
2760			}
2761		} else {
2762			next_buf_id = i;
2763		}
2764		idpf_tx_buf_next(tx_buf) = next_buf_id;
2765		tx_buf = &tx_q->tx_buf[next_buf_id];
2766
2767		size = skb_frag_size(frag);
2768		data_len -= size;
2769
2770		dma = skb_frag_dma_map(tx_q->dev, frag, 0, size,
2771				       DMA_TO_DEVICE);
2772	}
2773
2774	/* record SW timestamp if HW timestamp is not available */
2775	skb_tx_timestamp(skb);
2776
2777	first->type = LIBETH_SQE_SKB;
2778
2779	/* write last descriptor with RS and EOP bits */
2780	first->rs_idx = i;
2781	idpf_tx_buf_next(tx_buf) = IDPF_TXBUF_NULL;
2782	td_cmd |= params->eop_cmd;
2783	idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size);
2784	i = idpf_tx_splitq_bump_ntu(tx_q, i);
2785
2786	tx_q->txq_grp->num_completions_pending++;
2787
2788	/* record bytecount for BQL */
2789	nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx);
2790	netdev_tx_sent_queue(nq, first->bytes);
2791
2792	idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more());
2793}
2794
2795/**
2796 * idpf_tso - computes mss and TSO length to prepare for TSO
2797 * @skb: pointer to skb
2798 * @off: pointer to struct that holds offload parameters
2799 *
2800 * Returns error (negative) if TSO was requested but cannot be applied to the
2801 * given skb, 0 if TSO does not apply to the given skb, or 1 otherwise.
2802 */
2803int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off)
2804{
2805	const struct skb_shared_info *shinfo;
2806	union {
2807		struct iphdr *v4;
2808		struct ipv6hdr *v6;
2809		unsigned char *hdr;
2810	} ip;
2811	union {
2812		struct tcphdr *tcp;
2813		struct udphdr *udp;
2814		unsigned char *hdr;
2815	} l4;
2816	u32 paylen, l4_start;
2817	int err;
2818
2819	if (!skb_is_gso(skb))
2820		return 0;
2821
2822	err = skb_cow_head(skb, 0);
2823	if (err < 0)
2824		return err;
2825
2826	shinfo = skb_shinfo(skb);
2827
2828	ip.hdr = skb_network_header(skb);
2829	l4.hdr = skb_transport_header(skb);
2830
2831	/* initialize outer IP header fields */
2832	if (ip.v4->version == 4) {
2833		ip.v4->tot_len = 0;
2834		ip.v4->check = 0;
2835	} else if (ip.v6->version == 6) {
2836		ip.v6->payload_len = 0;
2837	}
2838
2839	l4_start = skb_transport_offset(skb);
2840
2841	/* remove payload length from checksum */
2842	paylen = skb->len - l4_start;
2843
2844	switch (shinfo->gso_type & ~SKB_GSO_DODGY) {
2845	case SKB_GSO_TCPV4:
2846	case SKB_GSO_TCPV6:
2847		csum_replace_by_diff(&l4.tcp->check,
2848				     (__force __wsum)htonl(paylen));
2849		off->tso_hdr_len = __tcp_hdrlen(l4.tcp) + l4_start;
2850		break;
2851	case SKB_GSO_UDP_L4:
2852		csum_replace_by_diff(&l4.udp->check,
2853				     (__force __wsum)htonl(paylen));
2854		/* compute length of segmentation header */
2855		off->tso_hdr_len = sizeof(struct udphdr) + l4_start;
2856		l4.udp->len = htons(shinfo->gso_size + sizeof(struct udphdr));
2857		break;
2858	default:
2859		return -EINVAL;
2860	}
2861
2862	off->tso_len = skb->len - off->tso_hdr_len;
2863	off->mss = shinfo->gso_size;
2864	off->tso_segs = shinfo->gso_segs;
2865
2866	off->tx_flags |= IDPF_TX_FLAGS_TSO;
2867
2868	return 1;
2869}
2870
2871
2872/**
2873 * idpf_tx_splitq_get_ctx_desc - grab next desc and update buffer ring
2874 * @txq: queue to put context descriptor on
2875 *
2876 * Since the TX buffer rings mimics the descriptor ring, update the tx buffer
2877 * ring entry to reflect that this index is a context descriptor
2878 */
2879static union idpf_flex_tx_ctx_desc *
2880idpf_tx_splitq_get_ctx_desc(struct idpf_tx_queue *txq)
2881{
2882	union idpf_flex_tx_ctx_desc *desc;
2883	int i = txq->next_to_use;
2884
2885	/* grab the next descriptor */
2886	desc = &txq->flex_ctx[i];
2887	txq->next_to_use = idpf_tx_splitq_bump_ntu(txq, i);
2888
2889	return desc;
2890}
2891
2892/**
2893 * idpf_tx_drop_skb - free the SKB and bump tail if necessary
2894 * @tx_q: queue to send buffer on
2895 * @skb: pointer to skb
2896 */
2897netdev_tx_t idpf_tx_drop_skb(struct idpf_tx_queue *tx_q, struct sk_buff *skb)
2898{
2899	u64_stats_update_begin(&tx_q->stats_sync);
2900	u64_stats_inc(&tx_q->q_stats.skb_drops);
2901	u64_stats_update_end(&tx_q->stats_sync);
2902
2903	idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false);
2904
2905	dev_kfree_skb(skb);
2906
2907	return NETDEV_TX_OK;
2908}
2909
2910#if (IS_ENABLED(CONFIG_PTP_1588_CLOCK))
2911/**
2912 * idpf_tx_tstamp - set up context descriptor for hardware timestamp
2913 * @tx_q: queue to send buffer on
2914 * @skb: pointer to the SKB we're sending
2915 * @off: pointer to the offload struct
2916 *
2917 * Return: Positive index number on success, negative otherwise.
2918 */
2919static int idpf_tx_tstamp(struct idpf_tx_queue *tx_q, struct sk_buff *skb,
2920			  struct idpf_tx_offload_params *off)
2921{
2922	int err, idx;
2923
2924	/* only timestamp the outbound packet if the user has requested it */
2925	if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))
2926		return -1;
2927
2928	if (!idpf_ptp_get_txq_tstamp_capability(tx_q))
2929		return -1;
2930
2931	/* Tx timestamps cannot be sampled when doing TSO */
2932	if (off->tx_flags & IDPF_TX_FLAGS_TSO)
2933		return -1;
2934
2935	/* Grab an open timestamp slot */
2936	err = idpf_ptp_request_ts(tx_q, skb, &idx);
2937	if (err) {
2938		u64_stats_update_begin(&tx_q->stats_sync);
2939		u64_stats_inc(&tx_q->q_stats.tstamp_skipped);
2940		u64_stats_update_end(&tx_q->stats_sync);
2941
2942		return -1;
2943	}
2944
2945	off->tx_flags |= IDPF_TX_FLAGS_TSYN;
2946
2947	return idx;
2948}
2949
2950/**
2951 * idpf_tx_set_tstamp_desc - Set the Tx descriptor fields needed to generate
2952 *			     PHY Tx timestamp
2953 * @ctx_desc: Context descriptor
2954 * @idx: Index of the Tx timestamp latch
2955 */
2956static void idpf_tx_set_tstamp_desc(union idpf_flex_tx_ctx_desc *ctx_desc,
2957				    u32 idx)
2958{
2959	ctx_desc->tsyn.qw1 = le64_encode_bits(IDPF_TX_DESC_DTYPE_CTX,
2960					      IDPF_TX_CTX_DTYPE_M) |
2961			     le64_encode_bits(IDPF_TX_CTX_DESC_TSYN,
2962					      IDPF_TX_CTX_CMD_M) |
2963			     le64_encode_bits(idx, IDPF_TX_CTX_TSYN_REG_M);
2964}
2965#else /* CONFIG_PTP_1588_CLOCK */
2966static int idpf_tx_tstamp(struct idpf_tx_queue *tx_q, struct sk_buff *skb,
2967			  struct idpf_tx_offload_params *off)
2968{
2969	return -1;
2970}
2971
2972static void idpf_tx_set_tstamp_desc(union idpf_flex_tx_ctx_desc *ctx_desc,
2973				    u32 idx)
2974{ }
2975#endif /* CONFIG_PTP_1588_CLOCK */
2976
2977/**
2978 * idpf_tx_splitq_need_re - check whether RE bit needs to be set
2979 * @tx_q: pointer to Tx queue
2980 *
2981 * Return: true if RE bit needs to be set, false otherwise
2982 */
2983static bool idpf_tx_splitq_need_re(struct idpf_tx_queue *tx_q)
2984{
2985	int gap = tx_q->next_to_use - tx_q->last_re;
2986
2987	gap += (gap < 0) ? tx_q->desc_count : 0;
2988
2989	return gap >= IDPF_TX_SPLITQ_RE_MIN_GAP;
2990}
2991
2992/**
2993 * idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors
2994 * @skb: send buffer
2995 * @tx_q: queue to send buffer on
2996 *
2997 * Returns NETDEV_TX_OK if sent, else an error code
2998 */
2999static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb,
3000					struct idpf_tx_queue *tx_q)
3001{
3002	struct idpf_tx_splitq_params tx_params = {
3003		.prev_ntu = tx_q->next_to_use,
3004	};
3005	union idpf_flex_tx_ctx_desc *ctx_desc;
3006	struct idpf_tx_buf *first;
3007	u32 count, buf_count = 1;
3008	int tso, idx;
3009	u32 buf_id;
3010
3011	count = idpf_tx_res_count_required(tx_q, skb, &buf_count);
3012	if (unlikely(!count))
3013		return idpf_tx_drop_skb(tx_q, skb);
3014
3015	tso = idpf_tso(skb, &tx_params.offload);
3016	if (unlikely(tso < 0))
3017		return idpf_tx_drop_skb(tx_q, skb);
3018
3019	/* Check for splitq specific TX resources */
3020	count += (IDPF_TX_DESCS_PER_CACHE_LINE + tso);
3021	if (idpf_tx_maybe_stop_splitq(tx_q, count, buf_count)) {
3022		idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false);
3023
3024		return NETDEV_TX_BUSY;
3025	}
3026
3027	if (tso) {
3028		/* If tso is needed, set up context desc */
3029		ctx_desc = idpf_tx_splitq_get_ctx_desc(tx_q);
3030
3031		ctx_desc->tso.qw1.cmd_dtype =
3032				cpu_to_le16(IDPF_TX_DESC_DTYPE_FLEX_TSO_CTX |
3033					    IDPF_TX_FLEX_CTX_DESC_CMD_TSO);
3034		ctx_desc->tso.qw0.flex_tlen =
3035				cpu_to_le32(tx_params.offload.tso_len &
3036					    IDPF_TXD_FLEX_CTX_TLEN_M);
3037		ctx_desc->tso.qw0.mss_rt =
3038				cpu_to_le16(tx_params.offload.mss &
3039					    IDPF_TXD_FLEX_CTX_MSS_RT_M);
3040		ctx_desc->tso.qw0.hdr_len = tx_params.offload.tso_hdr_len;
3041
3042		u64_stats_update_begin(&tx_q->stats_sync);
3043		u64_stats_inc(&tx_q->q_stats.lso_pkts);
3044		u64_stats_update_end(&tx_q->stats_sync);
3045	}
3046
3047	idx = idpf_tx_tstamp(tx_q, skb, &tx_params.offload);
3048	if (idx != -1) {
3049		ctx_desc = idpf_tx_splitq_get_ctx_desc(tx_q);
3050		idpf_tx_set_tstamp_desc(ctx_desc, idx);
3051	}
3052
3053	if (idpf_queue_has(FLOW_SCH_EN, tx_q)) {
3054		struct idpf_sw_queue *refillq = tx_q->refillq;
3055
3056		/* Save refillq state in case of a packet rollback.  Otherwise,
3057		 * the tags will be leaked since they will be popped from the
3058		 * refillq but never reposted during cleaning.
3059		 */
3060		tx_params.prev_refill_gen =
3061			idpf_queue_has(RFL_GEN_CHK, refillq);
3062		tx_params.prev_refill_ntc = refillq->next_to_clean;
3063
3064		if (unlikely(!idpf_tx_get_free_buf_id(tx_q->refillq,
3065						      &buf_id))) {
3066			if (tx_params.prev_refill_gen !=
3067			    idpf_queue_has(RFL_GEN_CHK, refillq))
3068				idpf_queue_change(RFL_GEN_CHK, refillq);
3069			refillq->next_to_clean = tx_params.prev_refill_ntc;
3070
3071			tx_q->next_to_use = tx_params.prev_ntu;
3072			return idpf_tx_drop_skb(tx_q, skb);
3073		}
3074		tx_params.compl_tag = buf_id;
3075
3076		tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE;
3077		tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP;
3078		/* Set the RE bit to periodically "clean" the descriptor ring.
3079		 * MIN_GAP is set to MIN_RING size to ensure it will be set at
3080		 * least once each time around the ring.
3081		 */
3082		if (idpf_tx_splitq_need_re(tx_q)) {
3083			tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE;
3084			tx_q->txq_grp->num_completions_pending++;
3085			tx_q->last_re = tx_q->next_to_use;
3086		}
3087
3088		if (skb->ip_summed == CHECKSUM_PARTIAL)
3089			tx_params.offload.td_cmd |= IDPF_TXD_FLEX_FLOW_CMD_CS_EN;
3090
3091	} else {
3092		buf_id = tx_q->next_to_use;
3093
3094		tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2;
3095		tx_params.eop_cmd = IDPF_TXD_LAST_DESC_CMD;
3096
3097		if (skb->ip_summed == CHECKSUM_PARTIAL)
3098			tx_params.offload.td_cmd |= IDPF_TX_FLEX_DESC_CMD_CS_EN;
3099	}
3100
3101	first = &tx_q->tx_buf[buf_id];
3102	first->skb = skb;
3103
3104	if (tso) {
3105		first->packets = tx_params.offload.tso_segs;
3106		first->bytes = skb->len +
3107			((first->packets - 1) * tx_params.offload.tso_hdr_len);
3108	} else {
3109		first->packets = 1;
3110		first->bytes = max_t(unsigned int, skb->len, ETH_ZLEN);
3111	}
3112
3113	idpf_tx_splitq_map(tx_q, &tx_params, first);
3114
3115	return NETDEV_TX_OK;
3116}
3117
3118/**
3119 * idpf_tx_start - Selects the right Tx queue to send buffer
3120 * @skb: send buffer
3121 * @netdev: network interface device structure
3122 *
3123 * Returns NETDEV_TX_OK if sent, else an error code
3124 */
3125netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev)
3126{
3127	const struct idpf_vport *vport = idpf_netdev_to_vport(netdev);
3128	struct idpf_tx_queue *tx_q;
3129
3130	if (unlikely(skb_get_queue_mapping(skb) >=
3131		     vport->num_txq - vport->num_xdp_txq)) {
3132		dev_kfree_skb_any(skb);
3133
3134		return NETDEV_TX_OK;
3135	}
3136
3137	tx_q = vport->txqs[skb_get_queue_mapping(skb)];
3138
3139	/* hardware can't handle really short frames, hardware padding works
3140	 * beyond this point
3141	 */
3142	if (skb_put_padto(skb, tx_q->tx_min_pkt_len)) {
3143		idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false);
3144
3145		return NETDEV_TX_OK;
3146	}
3147
3148	if (idpf_is_queue_model_split(vport->txq_model))
3149		return idpf_tx_splitq_frame(skb, tx_q);
3150	else
3151		return idpf_tx_singleq_frame(skb, tx_q);
3152}
3153
3154/**
3155 * idpf_rx_hash - set the hash value in the skb
3156 * @rxq: Rx descriptor ring packet is being transacted on
3157 * @skb: pointer to current skb being populated
3158 * @rx_desc: Receive descriptor
3159 * @decoded: Decoded Rx packet type related fields
3160 */
3161static void
3162idpf_rx_hash(const struct idpf_rx_queue *rxq, struct sk_buff *skb,
3163	     const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
3164	     struct libeth_rx_pt decoded)
3165{
3166	u32 hash;
3167
3168	if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, decoded))
3169		return;
3170
3171	hash = le16_to_cpu(rx_desc->hash1) |
3172	       (rx_desc->ff2_mirrid_hash2.hash2 << 16) |
3173	       (rx_desc->hash3 << 24);
3174
3175	libeth_rx_pt_set_hash(skb, hash, decoded);
3176}
3177
3178/**
3179 * idpf_rx_csum - Indicate in skb if checksum is good
3180 * @rxq: Rx descriptor ring packet is being transacted on
3181 * @skb: pointer to current skb being populated
3182 * @csum_bits: checksum fields extracted from the descriptor
3183 * @decoded: Decoded Rx packet type related fields
3184 *
3185 * skb->protocol must be set before this function is called
3186 */
3187static void idpf_rx_csum(struct idpf_rx_queue *rxq, struct sk_buff *skb,
3188			 struct libeth_rx_csum csum_bits,
3189			 struct libeth_rx_pt decoded)
3190{
3191	bool ipv4, ipv6;
3192
3193	/* check if Rx checksum is enabled */
3194	if (!libeth_rx_pt_has_checksum(rxq->xdp_rxq.dev, decoded))
3195		return;
3196
3197	/* check if HW has decoded the packet and checksum */
3198	if (unlikely(!csum_bits.l3l4p))
3199		return;
3200
3201	ipv4 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV4;
3202	ipv6 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV6;
3203
3204	if (unlikely(ipv4 && (csum_bits.ipe || csum_bits.eipe)))
3205		goto checksum_fail;
3206
3207	if (unlikely(ipv6 && csum_bits.ipv6exadd))
3208		return;
3209
3210	/* check for L4 errors and handle packets that were not able to be
3211	 * checksummed
3212	 */
3213	if (unlikely(csum_bits.l4e))
3214		goto checksum_fail;
3215
3216	if (!csum_bits.raw_csum_valid ||
3217	    decoded.inner_prot == LIBETH_RX_PT_INNER_SCTP) {
3218		skb->ip_summed = CHECKSUM_UNNECESSARY;
3219		return;
3220	}
3221
3222	skb->csum = csum_unfold((__force __sum16)~swab16(csum_bits.raw_csum));
3223	skb->ip_summed = CHECKSUM_COMPLETE;
3224
3225	return;
3226
3227checksum_fail:
3228	u64_stats_update_begin(&rxq->stats_sync);
3229	u64_stats_inc(&rxq->q_stats.hw_csum_err);
3230	u64_stats_update_end(&rxq->stats_sync);
3231}
3232
3233/**
3234 * idpf_rx_splitq_extract_csum_bits - Extract checksum bits from descriptor
3235 * @rx_desc: receive descriptor
3236 *
3237 * Return: parsed checksum status.
3238 **/
3239static struct libeth_rx_csum
3240idpf_rx_splitq_extract_csum_bits(const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
3241{
3242	struct libeth_rx_csum csum = { };
3243	u8 qword0, qword1;
3244
3245	qword0 = rx_desc->status_err0_qw0;
3246	qword1 = rx_desc->status_err0_qw1;
3247
3248	csum.ipe = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_IPE_M,
3249			     qword1);
3250	csum.eipe = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EIPE_M,
3251			      qword1);
3252	csum.l4e = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_L4E_M,
3253			     qword1);
3254	csum.l3l4p = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_L3L4P_M,
3255			       qword1);
3256	csum.ipv6exadd = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_IPV6EXADD_M,
3257				   qword0);
3258	csum.raw_csum_valid =
3259		!le16_get_bits(rx_desc->ptype_err_fflags0,
3260			       VIRTCHNL2_RX_FLEX_DESC_ADV_RAW_CSUM_INV_M);
3261	csum.raw_csum = le16_to_cpu(rx_desc->misc.raw_cs);
3262
3263	return csum;
3264}
3265
3266/**
3267 * idpf_rx_rsc - Set the RSC fields in the skb
3268 * @rxq : Rx descriptor ring packet is being transacted on
3269 * @skb : pointer to current skb being populated
3270 * @rx_desc: Receive descriptor
3271 * @decoded: Decoded Rx packet type related fields
3272 *
3273 * Return 0 on success and error code on failure
3274 *
3275 * Populate the skb fields with the total number of RSC segments, RSC payload
3276 * length and packet type.
3277 */
3278static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb,
3279		       const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
3280		       struct libeth_rx_pt decoded)
3281{
3282	u16 rsc_segments, rsc_seg_len;
3283	bool ipv4, ipv6;
3284	int len;
3285
3286	if (unlikely(libeth_rx_pt_get_ip_ver(decoded) ==
3287		     LIBETH_RX_PT_OUTER_L2))
3288		return -EINVAL;
3289
3290	rsc_seg_len = le16_to_cpu(rx_desc->misc.rscseglen);
3291	if (unlikely(!rsc_seg_len))
3292		return -EINVAL;
3293
3294	ipv4 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV4;
3295	ipv6 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV6;
3296
3297	if (unlikely(!(ipv4 ^ ipv6)))
3298		return -EINVAL;
3299
3300	rsc_segments = DIV_ROUND_UP(skb->data_len, rsc_seg_len);
3301
3302	NAPI_GRO_CB(skb)->count = rsc_segments;
3303	skb_shinfo(skb)->gso_size = rsc_seg_len;
3304
3305	skb_reset_network_header(skb);
3306
3307	if (ipv4) {
3308		struct iphdr *ipv4h = ip_hdr(skb);
3309
3310		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
3311
3312		/* Reset and set transport header offset in skb */
3313		skb_set_transport_header(skb, sizeof(struct iphdr));
3314		len = skb->len - skb_transport_offset(skb);
3315
3316		/* Compute the TCP pseudo header checksum*/
3317		tcp_hdr(skb)->check =
3318			~tcp_v4_check(len, ipv4h->saddr, ipv4h->daddr, 0);
3319	} else {
3320		struct ipv6hdr *ipv6h = ipv6_hdr(skb);
3321
3322		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
3323		skb_set_transport_header(skb, sizeof(struct ipv6hdr));
3324		len = skb->len - skb_transport_offset(skb);
3325		tcp_hdr(skb)->check =
3326			~tcp_v6_check(len, &ipv6h->saddr, &ipv6h->daddr, 0);
3327	}
3328
3329	tcp_gro_complete(skb);
3330
3331	u64_stats_update_begin(&rxq->stats_sync);
3332	u64_stats_inc(&rxq->q_stats.rsc_pkts);
3333	u64_stats_update_end(&rxq->stats_sync);
3334
3335	return 0;
3336}
3337
3338/**
3339 * idpf_rx_hwtstamp - check for an RX timestamp and pass up the stack
3340 * @rxq: pointer to the rx queue that receives the timestamp
3341 * @rx_desc: pointer to rx descriptor containing timestamp
3342 * @skb: skb to put timestamp in
3343 */
3344static void
3345idpf_rx_hwtstamp(const struct idpf_rx_queue *rxq,
3346		 const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
3347		 struct sk_buff *skb)
3348{
3349	u64 cached_time, ts_ns;
3350	u32 ts_high;
3351
3352	if (!(rx_desc->ts_low & VIRTCHNL2_RX_FLEX_TSTAMP_VALID))
3353		return;
3354
3355	cached_time = READ_ONCE(rxq->cached_phc_time);
3356
3357	ts_high = le32_to_cpu(rx_desc->ts_high);
3358	ts_ns = idpf_ptp_tstamp_extend_32b_to_64b(cached_time, ts_high);
3359
3360	*skb_hwtstamps(skb) = (struct skb_shared_hwtstamps) {
3361		.hwtstamp = ns_to_ktime(ts_ns),
3362	};
3363}
3364
3365/**
3366 * __idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor
3367 * @rxq: Rx descriptor ring packet is being transacted on
3368 * @skb: pointer to current skb being populated
3369 * @rx_desc: Receive descriptor
3370 *
3371 * This function checks the ring, descriptor, and packet information in
3372 * order to populate the hash, checksum, protocol, and
3373 * other fields within the skb.
3374 */
3375static int
3376__idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb,
3377			     const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
3378{
3379	struct libeth_rx_csum csum_bits;
3380	struct libeth_rx_pt decoded;
3381	u16 rx_ptype;
3382
3383	rx_ptype = le16_get_bits(rx_desc->ptype_err_fflags0,
3384				 VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_M);
3385	decoded = rxq->rx_ptype_lkup[rx_ptype];
3386
3387	/* process RSS/hash */
3388	idpf_rx_hash(rxq, skb, rx_desc, decoded);
3389
3390	if (idpf_queue_has(PTP, rxq))
3391		idpf_rx_hwtstamp(rxq, rx_desc, skb);
3392
3393	if (le16_get_bits(rx_desc->hdrlen_flags,
3394			  VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M))
3395		return idpf_rx_rsc(rxq, skb, rx_desc, decoded);
3396
3397	csum_bits = idpf_rx_splitq_extract_csum_bits(rx_desc);
3398	idpf_rx_csum(rxq, skb, csum_bits, decoded);
3399
3400	return 0;
3401}
3402
3403bool idpf_rx_process_skb_fields(struct sk_buff *skb,
3404				const struct libeth_xdp_buff *xdp,
3405				struct libeth_rq_napi_stats *rs)
3406{
3407	struct idpf_rx_queue *rxq;
3408
3409	rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq);
3410
3411	return !__idpf_rx_process_skb_fields(rxq, skb, xdp->desc);
3412}
3413
3414LIBETH_XDP_DEFINE_START();
3415LIBETH_XDP_DEFINE_RUN(static idpf_xdp_run_pass, idpf_xdp_run_prog,
3416		      idpf_xdp_tx_flush_bulk, idpf_rx_process_skb_fields);
3417LIBETH_XDP_DEFINE_FINALIZE(static idpf_xdp_finalize_rx, idpf_xdp_tx_flush_bulk,
3418			   idpf_xdp_tx_finalize);
3419LIBETH_XDP_DEFINE_END();
3420
3421/**
3422 * idpf_rx_hsplit_wa - handle header buffer overflows and split errors
3423 * @hdr: Rx buffer for the headers
3424 * @buf: Rx buffer for the payload
3425 * @data_len: number of bytes received to the payload buffer
3426 *
3427 * When a header buffer overflow occurs or the HW was unable do parse the
3428 * packet type to perform header split, the whole frame gets placed to the
3429 * payload buffer. We can't build a valid skb around a payload buffer when
3430 * the header split is active since it doesn't reserve any head- or tailroom.
3431 * In that case, copy either the whole frame when it's short or just the
3432 * Ethernet header to the header buffer to be able to build an skb and adjust
3433 * the data offset in the payload buffer, IOW emulate the header split.
3434 *
3435 * Return: number of bytes copied to the header buffer.
3436 */
3437static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr,
3438			     struct libeth_fqe *buf, u32 data_len)
3439{
3440	u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN;
3441	struct page *hdr_page, *buf_page;
3442	const void *src;
3443	void *dst;
3444
3445	if (unlikely(netmem_is_net_iov(buf->netmem)) ||
3446	    !libeth_rx_sync_for_cpu(buf, copy))
3447		return 0;
3448
3449	hdr_page = __netmem_to_page(hdr->netmem);
3450	buf_page = __netmem_to_page(buf->netmem);
3451	dst = page_address(hdr_page) + hdr->offset +
3452		pp_page_to_nmdesc(hdr_page)->pp->p.offset;
3453	src = page_address(buf_page) + buf->offset +
3454		pp_page_to_nmdesc(buf_page)->pp->p.offset;
3455
3456	memcpy(dst, src, LARGEST_ALIGN(copy));
3457	buf->offset += copy;
3458
3459	return copy;
3460}
3461
3462/**
3463 * idpf_rx_splitq_test_staterr - tests bits in Rx descriptor
3464 * status and error fields
3465 * @stat_err_field: field from descriptor to test bits in
3466 * @stat_err_bits: value to mask
3467 *
3468 */
3469static bool idpf_rx_splitq_test_staterr(const u8 stat_err_field,
3470					const u8 stat_err_bits)
3471{
3472	return !!(stat_err_field & stat_err_bits);
3473}
3474
3475/**
3476 * idpf_rx_splitq_is_eop - process handling of EOP buffers
3477 * @rx_desc: Rx descriptor for current buffer
3478 *
3479 * If the buffer is an EOP buffer, this function exits returning true,
3480 * otherwise return false indicating that this is in fact a non-EOP buffer.
3481 */
3482static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
3483{
3484	/* if we are the last buffer then there is nothing else to do */
3485	return likely(idpf_rx_splitq_test_staterr(rx_desc->status_err0_qw1,
3486						  IDPF_RXD_EOF_SPLITQ));
3487}
3488
3489/**
3490 * idpf_rx_splitq_clean - Clean completed descriptors from Rx queue
3491 * @rxq: Rx descriptor queue to retrieve receive buffer queue
3492 * @budget: Total limit on number of packets to process
3493 *
3494 * This function provides a "bounce buffer" approach to Rx interrupt
3495 * processing. The advantage to this is that on systems that have
3496 * expensive overhead for IOMMU access this provides a means of avoiding
3497 * it by maintaining the mapping of the page to the system.
3498 *
3499 * Returns amount of work completed
3500 */
3501static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
3502{
3503	struct idpf_buf_queue *rx_bufq = NULL;
3504	struct libeth_rq_napi_stats rs = { };
3505	u16 ntc = rxq->next_to_clean;
3506	LIBETH_XDP_ONSTACK_BUFF(xdp);
3507	LIBETH_XDP_ONSTACK_BULK(bq);
3508
3509	libeth_xdp_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev,
3510				rxq->xdpsqs, rxq->num_xdp_txq);
3511	libeth_xdp_init_buff(xdp, &rxq->xdp, &rxq->xdp_rxq);
3512
3513	/* Process Rx packets bounded by budget */
3514	while (likely(rs.packets < budget)) {
3515		struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc;
3516		struct libeth_fqe *hdr, *rx_buf = NULL;
3517		struct idpf_sw_queue *refillq = NULL;
3518		struct idpf_rxq_set *rxq_set = NULL;
3519		unsigned int pkt_len = 0;
3520		unsigned int hdr_len = 0;
3521		u16 gen_id, buf_id = 0;
3522		int bufq_id;
3523		u8 rxdid;
3524
3525		/* get the Rx desc from Rx queue based on 'next_to_clean' */
3526		rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb;
3527
3528		/* if the descriptor isn't done, no work yet to do */
3529		gen_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id,
3530				       VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M);
3531		if (idpf_queue_has(GEN_CHK, rxq) != gen_id)
3532			break;
3533
3534		dma_rmb();
3535
3536		rxdid = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_RXDID_M,
3537				  rx_desc->rxdid_ucast);
3538		if (rxdid != VIRTCHNL2_RXDID_2_FLEX_SPLITQ) {
3539			IDPF_RX_BUMP_NTC(rxq, ntc);
3540			u64_stats_update_begin(&rxq->stats_sync);
3541			u64_stats_inc(&rxq->q_stats.bad_descs);
3542			u64_stats_update_end(&rxq->stats_sync);
3543			continue;
3544		}
3545
3546		pkt_len = le16_get_bits(rx_desc->pktlen_gen_bufq_id,
3547					VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_M);
3548
3549		bufq_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id,
3550					VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_M);
3551
3552		rxq_set = container_of(rxq, struct idpf_rxq_set, rxq);
3553		refillq = rxq_set->refillq[bufq_id];
3554
3555		/* retrieve buffer from the rxq */
3556		rx_bufq = &rxq->bufq_sets[bufq_id].bufq;
3557
3558		buf_id = le16_to_cpu(rx_desc->buf_id);
3559
3560		rx_buf = &rx_bufq->buf[buf_id];
3561
3562		if (!rx_bufq->hdr_pp)
3563			goto payload;
3564
3565#define __HBO_BIT	VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_HBO_M
3566#define __HDR_LEN_MASK	VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_HDR_M
3567		if (likely(!(rx_desc->status_err0_qw1 & __HBO_BIT)))
3568			/* If a header buffer overflow, occurs, i.e. header is
3569			 * too large to fit in the header split buffer, HW will
3570			 * put the entire packet, including headers, in the
3571			 * data/payload buffer.
3572			 */
3573			hdr_len = le16_get_bits(rx_desc->hdrlen_flags,
3574						__HDR_LEN_MASK);
3575#undef __HDR_LEN_MASK
3576#undef __HBO_BIT
3577
3578		hdr = &rx_bufq->hdr_buf[buf_id];
3579
3580		if (unlikely(!hdr_len && !xdp->data)) {
3581			hdr_len = idpf_rx_hsplit_wa(hdr, rx_buf, pkt_len);
3582			/* If failed, drop both buffers by setting len to 0 */
3583			pkt_len -= hdr_len ? : pkt_len;
3584
3585			u64_stats_update_begin(&rxq->stats_sync);
3586			u64_stats_inc(&rxq->q_stats.hsplit_buf_ovf);
3587			u64_stats_update_end(&rxq->stats_sync);
3588		}
3589
3590		if (libeth_xdp_process_buff(xdp, hdr, hdr_len))
3591			rs.hsplit++;
3592
3593		hdr->netmem = 0;
3594
3595payload:
3596		libeth_xdp_process_buff(xdp, rx_buf, pkt_len);
3597		rx_buf->netmem = 0;
3598
3599		idpf_post_buf_refill(refillq, buf_id);
3600		IDPF_RX_BUMP_NTC(rxq, ntc);
3601
3602		/* skip if it is non EOP desc */
3603		if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!xdp->data))
3604			continue;
3605
3606		idpf_xdp_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc);
3607	}
3608
3609	idpf_xdp_finalize_rx(&bq);
3610
3611	rxq->next_to_clean = ntc;
3612	libeth_xdp_save_buff(&rxq->xdp, xdp);
3613
3614	u64_stats_update_begin(&rxq->stats_sync);
3615	u64_stats_add(&rxq->q_stats.packets, rs.packets);
3616	u64_stats_add(&rxq->q_stats.bytes, rs.bytes);
3617	u64_stats_add(&rxq->q_stats.hsplit_pkts, rs.hsplit);
3618	u64_stats_update_end(&rxq->stats_sync);
3619
3620	return rs.packets;
3621}
3622
3623/**
3624 * idpf_rx_update_bufq_desc - Update buffer queue descriptor
3625 * @bufq: Pointer to the buffer queue
3626 * @buf_id: buffer ID
3627 * @buf_desc: Buffer queue descriptor
3628 *
3629 * Return 0 on success and negative on failure.
3630 */
3631static int idpf_rx_update_bufq_desc(struct idpf_buf_queue *bufq, u32 buf_id,
3632				    struct virtchnl2_splitq_rx_buf_desc *buf_desc)
3633{
3634	struct libeth_fq_fp fq = {
3635		.pp		= bufq->pp,
3636		.fqes		= bufq->buf,
3637		.truesize	= bufq->truesize,
3638		.count		= bufq->desc_count,
3639	};
3640	dma_addr_t addr;
3641
3642	addr = libeth_rx_alloc(&fq, buf_id);
3643	if (addr == DMA_MAPPING_ERROR)
3644		return -ENOMEM;
3645
3646	buf_desc->pkt_addr = cpu_to_le64(addr);
3647	buf_desc->qword0.buf_id = cpu_to_le16(buf_id);
3648
3649	if (!idpf_queue_has(HSPLIT_EN, bufq))
3650		return 0;
3651
3652	fq.pp = bufq->hdr_pp;
3653	fq.fqes = bufq->hdr_buf;
3654	fq.truesize = bufq->hdr_truesize;
3655
3656	addr = libeth_rx_alloc(&fq, buf_id);
3657	if (addr == DMA_MAPPING_ERROR)
3658		return -ENOMEM;
3659
3660	buf_desc->hdr_addr = cpu_to_le64(addr);
3661
3662	return 0;
3663}
3664
3665/**
3666 * idpf_rx_clean_refillq - Clean refill queue buffers
3667 * @bufq: buffer queue to post buffers back to
3668 * @refillq: refill queue to clean
3669 *
3670 * This function takes care of the buffer refill management
3671 */
3672static void idpf_rx_clean_refillq(struct idpf_buf_queue *bufq,
3673				  struct idpf_sw_queue *refillq)
3674{
3675	struct virtchnl2_splitq_rx_buf_desc *buf_desc;
3676	u16 bufq_nta = bufq->next_to_alloc;
3677	u16 ntc = refillq->next_to_clean;
3678	int cleaned = 0;
3679
3680	buf_desc = &bufq->split_buf[bufq_nta];
3681
3682	/* make sure we stop at ring wrap in the unlikely case ring is full */
3683	while (likely(cleaned < refillq->desc_count)) {
3684		u32 buf_id, refill_desc = refillq->ring[ntc];
3685		bool failure;
3686
3687		if (idpf_queue_has(RFL_GEN_CHK, refillq) !=
3688		    !!(refill_desc & IDPF_RFL_BI_GEN_M))
3689			break;
3690
3691		buf_id = FIELD_GET(IDPF_RFL_BI_BUFID_M, refill_desc);
3692		failure = idpf_rx_update_bufq_desc(bufq, buf_id, buf_desc);
3693		if (failure)
3694			break;
3695
3696		if (unlikely(++ntc == refillq->desc_count)) {
3697			idpf_queue_change(RFL_GEN_CHK, refillq);
3698			ntc = 0;
3699		}
3700
3701		if (unlikely(++bufq_nta == bufq->desc_count)) {
3702			buf_desc = &bufq->split_buf[0];
3703			bufq_nta = 0;
3704		} else {
3705			buf_desc++;
3706		}
3707
3708		cleaned++;
3709	}
3710
3711	if (!cleaned)
3712		return;
3713
3714	/* We want to limit how many transactions on the bus we trigger with
3715	 * tail writes so we only do it in strides. It's also important we
3716	 * align the write to a multiple of 8 as required by HW.
3717	 */
3718	if (((bufq->next_to_use <= bufq_nta ? 0 : bufq->desc_count) +
3719	    bufq_nta - bufq->next_to_use) >= IDPF_RX_BUF_POST_STRIDE)
3720		idpf_rx_buf_hw_update(bufq, ALIGN_DOWN(bufq_nta,
3721						       IDPF_RX_BUF_POST_STRIDE));
3722
3723	/* update next to alloc since we have filled the ring */
3724	refillq->next_to_clean = ntc;
3725	bufq->next_to_alloc = bufq_nta;
3726}
3727
3728/**
3729 * idpf_rx_clean_refillq_all - Clean all refill queues
3730 * @bufq: buffer queue with refill queues
3731 * @nid: ID of the closest NUMA node with memory
3732 *
3733 * Iterates through all refill queues assigned to the buffer queue assigned to
3734 * this vector.  Returns true if clean is complete within budget, false
3735 * otherwise.
3736 */
3737static void idpf_rx_clean_refillq_all(struct idpf_buf_queue *bufq, int nid)
3738{
3739	struct idpf_bufq_set *bufq_set;
3740	int i;
3741
3742	page_pool_nid_changed(bufq->pp, nid);
3743	if (bufq->hdr_pp)
3744		page_pool_nid_changed(bufq->hdr_pp, nid);
3745
3746	bufq_set = container_of(bufq, struct idpf_bufq_set, bufq);
3747	for (i = 0; i < bufq_set->num_refillqs; i++)
3748		idpf_rx_clean_refillq(bufq, &bufq_set->refillqs[i]);
3749}
3750
3751/**
3752 * idpf_vport_intr_clean_queues - MSIX mode Interrupt Handler
3753 * @irq: interrupt number
3754 * @data: pointer to a q_vector
3755 *
3756 */
3757static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq,
3758						void *data)
3759{
3760	struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data;
3761
3762	q_vector->total_events++;
3763	napi_schedule_irqoff(&q_vector->napi);
3764
3765	return IRQ_HANDLED;
3766}
3767
3768/**
3769 * idpf_vport_intr_napi_del_all - Unregister napi for all q_vectors in vport
3770 * @vport: virtual port structure
3771 *
3772 */
3773static void idpf_vport_intr_napi_del_all(struct idpf_vport *vport)
3774{
3775	u16 v_idx;
3776
3777	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++)
3778		netif_napi_del(&vport->q_vectors[v_idx].napi);
3779}
3780
3781/**
3782 * idpf_vport_intr_napi_dis_all - Disable NAPI for all q_vectors in the vport
3783 * @vport: main vport structure
3784 */
3785static void idpf_vport_intr_napi_dis_all(struct idpf_vport *vport)
3786{
3787	int v_idx;
3788
3789	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++)
3790		napi_disable(&vport->q_vectors[v_idx].napi);
3791}
3792
3793/**
3794 * idpf_vport_intr_rel - Free memory allocated for interrupt vectors
3795 * @vport: virtual port
3796 *
3797 * Free the memory allocated for interrupt vectors  associated to a vport
3798 */
3799void idpf_vport_intr_rel(struct idpf_vport *vport)
3800{
3801	for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
3802		struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx];
3803
3804		kfree(q_vector->xsksq);
3805		q_vector->xsksq = NULL;
3806		kfree(q_vector->complq);
3807		q_vector->complq = NULL;
3808		kfree(q_vector->bufq);
3809		q_vector->bufq = NULL;
3810		kfree(q_vector->tx);
3811		q_vector->tx = NULL;
3812		kfree(q_vector->rx);
3813		q_vector->rx = NULL;
3814	}
3815
3816	kfree(vport->q_vectors);
3817	vport->q_vectors = NULL;
3818}
3819
3820static void idpf_q_vector_set_napi(struct idpf_q_vector *q_vector, bool link)
3821{
3822	struct napi_struct *napi = link ? &q_vector->napi : NULL;
3823	struct net_device *dev = q_vector->vport->netdev;
3824
3825	for (u32 i = 0; i < q_vector->num_rxq; i++)
3826		netif_queue_set_napi(dev, q_vector->rx[i]->idx,
3827				     NETDEV_QUEUE_TYPE_RX, napi);
3828
3829	for (u32 i = 0; i < q_vector->num_txq; i++)
3830		netif_queue_set_napi(dev, q_vector->tx[i]->idx,
3831				     NETDEV_QUEUE_TYPE_TX, napi);
3832}
3833
3834/**
3835 * idpf_vport_intr_rel_irq - Free the IRQ association with the OS
3836 * @vport: main vport structure
3837 */
3838static void idpf_vport_intr_rel_irq(struct idpf_vport *vport)
3839{
3840	struct idpf_adapter *adapter = vport->adapter;
3841	int vector;
3842
3843	for (vector = 0; vector < vport->num_q_vectors; vector++) {
3844		struct idpf_q_vector *q_vector = &vport->q_vectors[vector];
3845		int irq_num, vidx;
3846
3847		/* free only the irqs that were actually requested */
3848		if (!q_vector)
3849			continue;
3850
3851		vidx = vport->q_vector_idxs[vector];
3852		irq_num = adapter->msix_entries[vidx].vector;
3853
3854		idpf_q_vector_set_napi(q_vector, false);
3855		kfree(free_irq(irq_num, q_vector));
3856	}
3857}
3858
3859/**
3860 * idpf_vport_intr_dis_irq_all - Disable all interrupt
3861 * @vport: main vport structure
3862 */
3863static void idpf_vport_intr_dis_irq_all(struct idpf_vport *vport)
3864{
3865	struct idpf_q_vector *q_vector = vport->q_vectors;
3866	int q_idx;
3867
3868	writel(0, vport->noirq_dyn_ctl);
3869
3870	for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++)
3871		writel(0, q_vector[q_idx].intr_reg.dyn_ctl);
3872}
3873
3874/**
3875 * idpf_vport_intr_buildreg_itr - Enable default interrupt generation settings
3876 * @q_vector: pointer to q_vector
3877 */
3878static u32 idpf_vport_intr_buildreg_itr(struct idpf_q_vector *q_vector)
3879{
3880	u32 itr_val = q_vector->intr_reg.dyn_ctl_intena_m;
3881	int type = IDPF_NO_ITR_UPDATE_IDX;
3882	u16 itr = 0;
3883
3884	if (q_vector->wb_on_itr) {
3885		/*
3886		 * Trigger a software interrupt when exiting wb_on_itr, to make
3887		 * sure we catch any pending write backs that might have been
3888		 * missed due to interrupt state transition.
3889		 */
3890		itr_val |= q_vector->intr_reg.dyn_ctl_swint_trig_m |
3891			   q_vector->intr_reg.dyn_ctl_sw_itridx_ena_m;
3892		type = IDPF_SW_ITR_UPDATE_IDX;
3893		itr = IDPF_ITR_20K;
3894	}
3895
3896	itr &= IDPF_ITR_MASK;
3897	/* Don't clear PBA because that can cause lost interrupts that
3898	 * came in while we were cleaning/polling
3899	 */
3900	itr_val |= (type << q_vector->intr_reg.dyn_ctl_itridx_s) |
3901		   (itr << (q_vector->intr_reg.dyn_ctl_intrvl_s - 1));
3902
3903	return itr_val;
3904}
3905
3906/**
3907 * idpf_update_dim_sample - Update dim sample with packets and bytes
3908 * @q_vector: the vector associated with the interrupt
3909 * @dim_sample: dim sample to update
3910 * @dim: dim instance structure
3911 * @packets: total packets
3912 * @bytes: total bytes
3913 *
3914 * Update the dim sample with the packets and bytes which are passed to this
3915 * function. Set the dim state appropriately if the dim settings gets stale.
3916 */
3917static void idpf_update_dim_sample(struct idpf_q_vector *q_vector,
3918				   struct dim_sample *dim_sample,
3919				   struct dim *dim, u64 packets, u64 bytes)
3920{
3921	dim_update_sample(q_vector->total_events, packets, bytes, dim_sample);
3922	dim_sample->comp_ctr = 0;
3923
3924	/* if dim settings get stale, like when not updated for 1 second or
3925	 * longer, force it to start again. This addresses the frequent case
3926	 * of an idle queue being switched to by the scheduler.
3927	 */
3928	if (ktime_ms_delta(dim_sample->time, dim->start_sample.time) >= HZ)
3929		dim->state = DIM_START_MEASURE;
3930}
3931
3932/**
3933 * idpf_net_dim - Update net DIM algorithm
3934 * @q_vector: the vector associated with the interrupt
3935 *
3936 * Create a DIM sample and notify net_dim() so that it can possibly decide
3937 * a new ITR value based on incoming packets, bytes, and interrupts.
3938 *
3939 * This function is a no-op if the queue is not configured to dynamic ITR.
3940 */
3941static void idpf_net_dim(struct idpf_q_vector *q_vector)
3942{
3943	struct dim_sample dim_sample = { };
3944	u64 packets, bytes, pkts, bts;
3945	u32 i;
3946
3947	if (!IDPF_ITR_IS_DYNAMIC(q_vector->tx_intr_mode))
3948		goto check_rx_itr;
3949
3950	for (i = 0, packets = 0, bytes = 0; i < q_vector->num_txq; i++) {
3951		struct idpf_tx_queue *txq = q_vector->tx[i];
3952		unsigned int start;
3953
3954		do {
3955			start = u64_stats_fetch_begin(&txq->stats_sync);
3956			pkts = u64_stats_read(&txq->q_stats.packets);
3957			bts = u64_stats_read(&txq->q_stats.bytes);
3958		} while (u64_stats_fetch_retry(&txq->stats_sync, start));
3959
3960		packets += pkts;
3961		bytes += bts;
3962	}
3963
3964	idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->tx_dim,
3965			       packets, bytes);
3966	net_dim(&q_vector->tx_dim, &dim_sample);
3967
3968check_rx_itr:
3969	if (!IDPF_ITR_IS_DYNAMIC(q_vector->rx_intr_mode))
3970		return;
3971
3972	for (i = 0, packets = 0, bytes = 0; i < q_vector->num_rxq; i++) {
3973		struct idpf_rx_queue *rxq = q_vector->rx[i];
3974		unsigned int start;
3975
3976		do {
3977			start = u64_stats_fetch_begin(&rxq->stats_sync);
3978			pkts = u64_stats_read(&rxq->q_stats.packets);
3979			bts = u64_stats_read(&rxq->q_stats.bytes);
3980		} while (u64_stats_fetch_retry(&rxq->stats_sync, start));
3981
3982		packets += pkts;
3983		bytes += bts;
3984	}
3985
3986	idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->rx_dim,
3987			       packets, bytes);
3988	net_dim(&q_vector->rx_dim, &dim_sample);
3989}
3990
3991/**
3992 * idpf_vport_intr_update_itr_ena_irq - Update itr and re-enable MSIX interrupt
3993 * @q_vector: q_vector for which itr is being updated and interrupt enabled
3994 *
3995 * Update the net_dim() algorithm and re-enable the interrupt associated with
3996 * this vector.
3997 */
3998void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector)
3999{
4000	u32 intval;
4001
4002	/* net_dim() updates ITR out-of-band using a work item */
4003	idpf_net_dim(q_vector);
4004
4005	intval = idpf_vport_intr_buildreg_itr(q_vector);
4006	q_vector->wb_on_itr = false;
4007
4008	writel(intval, q_vector->intr_reg.dyn_ctl);
4009}
4010
4011/**
4012 * idpf_vport_intr_req_irq - get MSI-X vectors from the OS for the vport
4013 * @vport: main vport structure
4014 */
4015static int idpf_vport_intr_req_irq(struct idpf_vport *vport)
4016{
4017	struct idpf_adapter *adapter = vport->adapter;
4018	const char *drv_name, *if_name, *vec_name;
4019	int vector, err, irq_num, vidx;
4020
4021	drv_name = dev_driver_string(&adapter->pdev->dev);
4022	if_name = netdev_name(vport->netdev);
4023
4024	for (vector = 0; vector < vport->num_q_vectors; vector++) {
4025		struct idpf_q_vector *q_vector = &vport->q_vectors[vector];
4026		char *name;
4027
4028		vidx = vport->q_vector_idxs[vector];
4029		irq_num = adapter->msix_entries[vidx].vector;
4030
4031		if (q_vector->num_rxq && q_vector->num_txq)
4032			vec_name = "TxRx";
4033		else if (q_vector->num_rxq)
4034			vec_name = "Rx";
4035		else if (q_vector->num_txq)
4036			vec_name = "Tx";
4037		else
4038			continue;
4039
4040		name = kasprintf(GFP_KERNEL, "%s-%s-%s-%d", drv_name, if_name,
4041				 vec_name, vector);
4042
4043		err = request_irq(irq_num, idpf_vport_intr_clean_queues, 0,
4044				  name, q_vector);
4045		if (err) {
4046			netdev_err(vport->netdev,
4047				   "Request_irq failed, error: %d\n", err);
4048			goto free_q_irqs;
4049		}
4050
4051		idpf_q_vector_set_napi(q_vector, true);
4052	}
4053
4054	return 0;
4055
4056free_q_irqs:
4057	while (--vector >= 0) {
4058		vidx = vport->q_vector_idxs[vector];
4059		irq_num = adapter->msix_entries[vidx].vector;
4060		kfree(free_irq(irq_num, &vport->q_vectors[vector]));
4061	}
4062
4063	return err;
4064}
4065
4066/**
4067 * idpf_vport_intr_write_itr - Write ITR value to the ITR register
4068 * @q_vector: q_vector structure
4069 * @itr: Interrupt throttling rate
4070 * @tx: Tx or Rx ITR
4071 */
4072void idpf_vport_intr_write_itr(struct idpf_q_vector *q_vector, u16 itr, bool tx)
4073{
4074	struct idpf_intr_reg *intr_reg;
4075
4076	if (tx && !q_vector->tx)
4077		return;
4078	else if (!tx && !q_vector->rx)
4079		return;
4080
4081	intr_reg = &q_vector->intr_reg;
4082	writel(ITR_REG_ALIGN(itr) >> IDPF_ITR_GRAN_S,
4083	       tx ? intr_reg->tx_itr : intr_reg->rx_itr);
4084}
4085
4086/**
4087 * idpf_vport_intr_ena_irq_all - Enable IRQ for the given vport
4088 * @vport: main vport structure
4089 */
4090static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport)
4091{
4092	bool dynamic;
4093	int q_idx;
4094	u16 itr;
4095
4096	for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) {
4097		struct idpf_q_vector *qv = &vport->q_vectors[q_idx];
4098
4099		/* Set the initial ITR values */
4100		if (qv->num_txq) {
4101			dynamic = IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode);
4102			itr = vport->tx_itr_profile[qv->tx_dim.profile_ix];
4103			idpf_vport_intr_write_itr(qv, dynamic ?
4104						  itr : qv->tx_itr_value,
4105						  true);
4106		}
4107
4108		if (qv->num_rxq) {
4109			dynamic = IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode);
4110			itr = vport->rx_itr_profile[qv->rx_dim.profile_ix];
4111			idpf_vport_intr_write_itr(qv, dynamic ?
4112						  itr : qv->rx_itr_value,
4113						  false);
4114		}
4115
4116		if (qv->num_txq || qv->num_rxq)
4117			idpf_vport_intr_update_itr_ena_irq(qv);
4118	}
4119
4120	writel(vport->noirq_dyn_ctl_ena, vport->noirq_dyn_ctl);
4121}
4122
4123/**
4124 * idpf_vport_intr_deinit - Release all vector associations for the vport
4125 * @vport: main vport structure
4126 */
4127void idpf_vport_intr_deinit(struct idpf_vport *vport)
4128{
4129	idpf_vport_intr_dis_irq_all(vport);
4130	idpf_vport_intr_napi_dis_all(vport);
4131	idpf_vport_intr_napi_del_all(vport);
4132	idpf_vport_intr_rel_irq(vport);
4133}
4134
4135/**
4136 * idpf_tx_dim_work - Call back from the stack
4137 * @work: work queue structure
4138 */
4139static void idpf_tx_dim_work(struct work_struct *work)
4140{
4141	struct idpf_q_vector *q_vector;
4142	struct idpf_vport *vport;
4143	struct dim *dim;
4144	u16 itr;
4145
4146	dim = container_of(work, struct dim, work);
4147	q_vector = container_of(dim, struct idpf_q_vector, tx_dim);
4148	vport = q_vector->vport;
4149
4150	if (dim->profile_ix >= ARRAY_SIZE(vport->tx_itr_profile))
4151		dim->profile_ix = ARRAY_SIZE(vport->tx_itr_profile) - 1;
4152
4153	/* look up the values in our local table */
4154	itr = vport->tx_itr_profile[dim->profile_ix];
4155
4156	idpf_vport_intr_write_itr(q_vector, itr, true);
4157
4158	dim->state = DIM_START_MEASURE;
4159}
4160
4161/**
4162 * idpf_rx_dim_work - Call back from the stack
4163 * @work: work queue structure
4164 */
4165static void idpf_rx_dim_work(struct work_struct *work)
4166{
4167	struct idpf_q_vector *q_vector;
4168	struct idpf_vport *vport;
4169	struct dim *dim;
4170	u16 itr;
4171
4172	dim = container_of(work, struct dim, work);
4173	q_vector = container_of(dim, struct idpf_q_vector, rx_dim);
4174	vport = q_vector->vport;
4175
4176	if (dim->profile_ix >= ARRAY_SIZE(vport->rx_itr_profile))
4177		dim->profile_ix = ARRAY_SIZE(vport->rx_itr_profile) - 1;
4178
4179	/* look up the values in our local table */
4180	itr = vport->rx_itr_profile[dim->profile_ix];
4181
4182	idpf_vport_intr_write_itr(q_vector, itr, false);
4183
4184	dim->state = DIM_START_MEASURE;
4185}
4186
4187/**
4188 * idpf_init_dim - Set up dynamic interrupt moderation
4189 * @qv: q_vector structure
4190 */
4191static void idpf_init_dim(struct idpf_q_vector *qv)
4192{
4193	INIT_WORK(&qv->tx_dim.work, idpf_tx_dim_work);
4194	qv->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
4195	qv->tx_dim.profile_ix = IDPF_DIM_DEFAULT_PROFILE_IX;
4196
4197	INIT_WORK(&qv->rx_dim.work, idpf_rx_dim_work);
4198	qv->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
4199	qv->rx_dim.profile_ix = IDPF_DIM_DEFAULT_PROFILE_IX;
4200}
4201
4202/**
4203 * idpf_vport_intr_napi_ena_all - Enable NAPI for all q_vectors in the vport
4204 * @vport: main vport structure
4205 */
4206static void idpf_vport_intr_napi_ena_all(struct idpf_vport *vport)
4207{
4208	int q_idx;
4209
4210	for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) {
4211		struct idpf_q_vector *q_vector = &vport->q_vectors[q_idx];
4212
4213		idpf_init_dim(q_vector);
4214		napi_enable(&q_vector->napi);
4215	}
4216}
4217
4218/**
4219 * idpf_tx_splitq_clean_all- Clean completion queues
4220 * @q_vec: queue vector
4221 * @budget: Used to determine if we are in netpoll
4222 * @cleaned: returns number of packets cleaned
4223 *
4224 * Returns false if clean is not complete else returns true
4225 */
4226static bool idpf_tx_splitq_clean_all(struct idpf_q_vector *q_vec,
4227				     int budget, int *cleaned)
4228{
4229	u16 num_complq = q_vec->num_complq;
4230	bool clean_complete = true;
4231	int i, budget_per_q;
4232
4233	if (unlikely(!num_complq))
4234		return true;
4235
4236	budget_per_q = DIV_ROUND_UP(budget, num_complq);
4237
4238	for (i = 0; i < num_complq; i++)
4239		clean_complete &= idpf_tx_clean_complq(q_vec->complq[i],
4240						       budget_per_q, cleaned);
4241
4242	return clean_complete;
4243}
4244
4245/**
4246 * idpf_rx_splitq_clean_all- Clean completion queues
4247 * @q_vec: queue vector
4248 * @budget: Used to determine if we are in netpoll
4249 * @cleaned: returns number of packets cleaned
4250 *
4251 * Returns false if clean is not complete else returns true
4252 */
4253static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget,
4254				     int *cleaned)
4255{
4256	u16 num_rxq = q_vec->num_rxq;
4257	bool clean_complete = true;
4258	int pkts_cleaned = 0;
4259	int i, budget_per_q;
4260	int nid;
4261
4262	/* We attempt to distribute budget to each Rx queue fairly, but don't
4263	 * allow the budget to go below 1 because that would exit polling early.
4264	 */
4265	budget_per_q = num_rxq ? max(budget / num_rxq, 1) : 0;
4266	for (i = 0; i < num_rxq; i++) {
4267		struct idpf_rx_queue *rxq = q_vec->rx[i];
4268		int pkts_cleaned_per_q;
4269
4270		pkts_cleaned_per_q = idpf_queue_has(XSK, rxq) ?
4271				     idpf_xskrq_poll(rxq, budget_per_q) :
4272				     idpf_rx_splitq_clean(rxq, budget_per_q);
4273		/* if we clean as many as budgeted, we must not be done */
4274		if (pkts_cleaned_per_q >= budget_per_q)
4275			clean_complete = false;
4276		pkts_cleaned += pkts_cleaned_per_q;
4277	}
4278	*cleaned = pkts_cleaned;
4279
4280	nid = numa_mem_id();
4281
4282	for (i = 0; i < q_vec->num_bufq; i++) {
4283		if (!idpf_queue_has(XSK, q_vec->bufq[i]))
4284			idpf_rx_clean_refillq_all(q_vec->bufq[i], nid);
4285	}
4286
4287	return clean_complete;
4288}
4289
4290/**
4291 * idpf_vport_splitq_napi_poll - NAPI handler
4292 * @napi: struct from which you get q_vector
4293 * @budget: budget provided by stack
4294 */
4295static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget)
4296{
4297	struct idpf_q_vector *q_vector =
4298				container_of(napi, struct idpf_q_vector, napi);
4299	bool clean_complete = true;
4300	int work_done = 0;
4301
4302	/* Handle case where we are called by netpoll with a budget of 0 */
4303	if (unlikely(!budget)) {
4304		idpf_tx_splitq_clean_all(q_vector, budget, &work_done);
4305
4306		return 0;
4307	}
4308
4309	for (u32 i = 0; i < q_vector->num_xsksq; i++)
4310		clean_complete &= idpf_xsk_xmit(q_vector->xsksq[i]);
4311
4312	clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget,
4313						   &work_done);
4314	clean_complete &= idpf_rx_splitq_clean_all(q_vector, budget,
4315						   &work_done);
4316
4317	/* If work not completed, return budget and polling will return */
4318	if (!clean_complete) {
4319		idpf_vport_intr_set_wb_on_itr(q_vector);
4320		return budget;
4321	}
4322
4323	work_done = min_t(int, work_done, budget - 1);
4324
4325	/* Exit the polling mode, but don't re-enable interrupts if stack might
4326	 * poll us due to busy-polling
4327	 */
4328	if (napi_complete_done(napi, work_done))
4329		idpf_vport_intr_update_itr_ena_irq(q_vector);
4330	else
4331		idpf_vport_intr_set_wb_on_itr(q_vector);
4332
4333	return work_done;
4334}
4335
4336/**
4337 * idpf_vport_intr_map_vector_to_qs - Map vectors to queues
4338 * @vport: virtual port
4339 *
4340 * Mapping for vectors to queues
4341 */
4342static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport)
4343{
4344	u16 num_txq_grp = vport->num_txq_grp - vport->num_xdp_txq;
4345	bool split = idpf_is_queue_model_split(vport->rxq_model);
4346	struct idpf_rxq_group *rx_qgrp;
4347	struct idpf_txq_group *tx_qgrp;
4348	u32 i, qv_idx, q_index;
4349
4350	for (i = 0, qv_idx = 0; i < vport->num_rxq_grp; i++) {
4351		u16 num_rxq;
4352
4353		if (qv_idx >= vport->num_q_vectors)
4354			qv_idx = 0;
4355
4356		rx_qgrp = &vport->rxq_grps[i];
4357		if (split)
4358			num_rxq = rx_qgrp->splitq.num_rxq_sets;
4359		else
4360			num_rxq = rx_qgrp->singleq.num_rxq;
4361
4362		for (u32 j = 0; j < num_rxq; j++) {
4363			struct idpf_rx_queue *q;
4364
4365			if (split)
4366				q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
4367			else
4368				q = rx_qgrp->singleq.rxqs[j];
4369			q->q_vector = &vport->q_vectors[qv_idx];
4370			q_index = q->q_vector->num_rxq;
4371			q->q_vector->rx[q_index] = q;
4372			q->q_vector->num_rxq++;
4373
4374			if (split)
4375				q->napi = &q->q_vector->napi;
4376		}
4377
4378		if (split) {
4379			for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) {
4380				struct idpf_buf_queue *bufq;
4381
4382				bufq = &rx_qgrp->splitq.bufq_sets[j].bufq;
4383				bufq->q_vector = &vport->q_vectors[qv_idx];
4384				q_index = bufq->q_vector->num_bufq;
4385				bufq->q_vector->bufq[q_index] = bufq;
4386				bufq->q_vector->num_bufq++;
4387			}
4388		}
4389
4390		qv_idx++;
4391	}
4392
4393	split = idpf_is_queue_model_split(vport->txq_model);
4394
4395	for (i = 0, qv_idx = 0; i < num_txq_grp; i++) {
4396		u16 num_txq;
4397
4398		if (qv_idx >= vport->num_q_vectors)
4399			qv_idx = 0;
4400
4401		tx_qgrp = &vport->txq_grps[i];
4402		num_txq = tx_qgrp->num_txq;
4403
4404		for (u32 j = 0; j < num_txq; j++) {
4405			struct idpf_tx_queue *q;
4406
4407			q = tx_qgrp->txqs[j];
4408			q->q_vector = &vport->q_vectors[qv_idx];
4409			q->q_vector->tx[q->q_vector->num_txq++] = q;
4410		}
4411
4412		if (split) {
4413			struct idpf_compl_queue *q = tx_qgrp->complq;
4414
4415			q->q_vector = &vport->q_vectors[qv_idx];
4416			q->q_vector->complq[q->q_vector->num_complq++] = q;
4417		}
4418
4419		qv_idx++;
4420	}
4421
4422	for (i = 0; i < vport->num_xdp_txq; i++) {
4423		struct idpf_tx_queue *xdpsq;
4424		struct idpf_q_vector *qv;
4425
4426		xdpsq = vport->txqs[vport->xdp_txq_offset + i];
4427		if (!idpf_queue_has(XSK, xdpsq))
4428			continue;
4429
4430		qv = idpf_find_rxq_vec(vport, i);
4431		idpf_xsk_init_wakeup(qv);
4432
4433		xdpsq->q_vector = qv;
4434		qv->xsksq[qv->num_xsksq++] = xdpsq;
4435	}
4436}
4437
4438/**
4439 * idpf_vport_intr_init_vec_idx - Initialize the vector indexes
4440 * @vport: virtual port
4441 *
4442 * Initialize vector indexes with values returened over mailbox
4443 */
4444static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport)
4445{
4446	struct idpf_adapter *adapter = vport->adapter;
4447	struct virtchnl2_alloc_vectors *ac;
4448	u16 *vecids, total_vecs;
4449	int i;
4450
4451	ac = adapter->req_vec_chunks;
4452	if (!ac) {
4453		for (i = 0; i < vport->num_q_vectors; i++)
4454			vport->q_vectors[i].v_idx = vport->q_vector_idxs[i];
4455
4456		vport->noirq_v_idx = vport->q_vector_idxs[i];
4457
4458		return 0;
4459	}
4460
4461	total_vecs = idpf_get_reserved_vecs(adapter);
4462	vecids = kcalloc(total_vecs, sizeof(u16), GFP_KERNEL);
4463	if (!vecids)
4464		return -ENOMEM;
4465
4466	idpf_get_vec_ids(adapter, vecids, total_vecs, &ac->vchunks);
4467
4468	for (i = 0; i < vport->num_q_vectors; i++)
4469		vport->q_vectors[i].v_idx = vecids[vport->q_vector_idxs[i]];
4470
4471	vport->noirq_v_idx = vecids[vport->q_vector_idxs[i]];
4472
4473	kfree(vecids);
4474
4475	return 0;
4476}
4477
4478/**
4479 * idpf_vport_intr_napi_add_all- Register napi handler for all qvectors
4480 * @vport: virtual port structure
4481 */
4482static void idpf_vport_intr_napi_add_all(struct idpf_vport *vport)
4483{
4484	int (*napi_poll)(struct napi_struct *napi, int budget);
4485	u16 v_idx, qv_idx;
4486	int irq_num;
4487
4488	if (idpf_is_queue_model_split(vport->txq_model))
4489		napi_poll = idpf_vport_splitq_napi_poll;
4490	else
4491		napi_poll = idpf_vport_singleq_napi_poll;
4492
4493	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
4494		struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx];
4495		qv_idx = vport->q_vector_idxs[v_idx];
4496		irq_num = vport->adapter->msix_entries[qv_idx].vector;
4497
4498		netif_napi_add_config(vport->netdev, &q_vector->napi,
4499				      napi_poll, v_idx);
4500		netif_napi_set_irq(&q_vector->napi, irq_num);
4501	}
4502}
4503
4504/**
4505 * idpf_vport_intr_alloc - Allocate memory for interrupt vectors
4506 * @vport: virtual port
4507 *
4508 * We allocate one q_vector per queue interrupt. If allocation fails we
4509 * return -ENOMEM.
4510 */
4511int idpf_vport_intr_alloc(struct idpf_vport *vport)
4512{
4513	u16 txqs_per_vector, rxqs_per_vector, bufqs_per_vector;
4514	struct idpf_vport_user_config_data *user_config;
4515	struct idpf_q_vector *q_vector;
4516	struct idpf_q_coalesce *q_coal;
4517	u32 complqs_per_vector, v_idx;
4518	u16 idx = vport->idx;
4519
4520	user_config = &vport->adapter->vport_config[idx]->user_config;
4521	vport->q_vectors = kcalloc(vport->num_q_vectors,
4522				   sizeof(struct idpf_q_vector), GFP_KERNEL);
4523	if (!vport->q_vectors)
4524		return -ENOMEM;
4525
4526	txqs_per_vector = DIV_ROUND_UP(vport->num_txq_grp,
4527				       vport->num_q_vectors);
4528	rxqs_per_vector = DIV_ROUND_UP(vport->num_rxq_grp,
4529				       vport->num_q_vectors);
4530	bufqs_per_vector = vport->num_bufqs_per_qgrp *
4531			   DIV_ROUND_UP(vport->num_rxq_grp,
4532					vport->num_q_vectors);
4533	complqs_per_vector = DIV_ROUND_UP(vport->num_txq_grp,
4534					  vport->num_q_vectors);
4535
4536	for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
4537		q_vector = &vport->q_vectors[v_idx];
4538		q_coal = &user_config->q_coalesce[v_idx];
4539		q_vector->vport = vport;
4540
4541		q_vector->tx_itr_value = q_coal->tx_coalesce_usecs;
4542		q_vector->tx_intr_mode = q_coal->tx_intr_mode;
4543		q_vector->tx_itr_idx = VIRTCHNL2_ITR_IDX_1;
4544
4545		q_vector->rx_itr_value = q_coal->rx_coalesce_usecs;
4546		q_vector->rx_intr_mode = q_coal->rx_intr_mode;
4547		q_vector->rx_itr_idx = VIRTCHNL2_ITR_IDX_0;
4548
4549		q_vector->tx = kcalloc(txqs_per_vector, sizeof(*q_vector->tx),
4550				       GFP_KERNEL);
4551		if (!q_vector->tx)
4552			goto error;
4553
4554		q_vector->rx = kcalloc(rxqs_per_vector, sizeof(*q_vector->rx),
4555				       GFP_KERNEL);
4556		if (!q_vector->rx)
4557			goto error;
4558
4559		if (!idpf_is_queue_model_split(vport->rxq_model))
4560			continue;
4561
4562		q_vector->bufq = kcalloc(bufqs_per_vector,
4563					 sizeof(*q_vector->bufq),
4564					 GFP_KERNEL);
4565		if (!q_vector->bufq)
4566			goto error;
4567
4568		q_vector->complq = kcalloc(complqs_per_vector,
4569					   sizeof(*q_vector->complq),
4570					   GFP_KERNEL);
4571		if (!q_vector->complq)
4572			goto error;
4573
4574		if (!vport->xdp_txq_offset)
4575			continue;
4576
4577		q_vector->xsksq = kcalloc(rxqs_per_vector,
4578					  sizeof(*q_vector->xsksq),
4579					  GFP_KERNEL);
4580		if (!q_vector->xsksq)
4581			goto error;
4582	}
4583
4584	return 0;
4585
4586error:
4587	idpf_vport_intr_rel(vport);
4588
4589	return -ENOMEM;
4590}
4591
4592/**
4593 * idpf_vport_intr_init - Setup all vectors for the given vport
4594 * @vport: virtual port
4595 *
4596 * Returns 0 on success or negative on failure
4597 */
4598int idpf_vport_intr_init(struct idpf_vport *vport)
4599{
4600	int err;
4601
4602	err = idpf_vport_intr_init_vec_idx(vport);
4603	if (err)
4604		return err;
4605
4606	idpf_vport_intr_map_vector_to_qs(vport);
4607	idpf_vport_intr_napi_add_all(vport);
4608
4609	err = vport->adapter->dev_ops.reg_ops.intr_reg_init(vport);
4610	if (err)
4611		goto unroll_vectors_alloc;
4612
4613	err = idpf_vport_intr_req_irq(vport);
4614	if (err)
4615		goto unroll_vectors_alloc;
4616
4617	return 0;
4618
4619unroll_vectors_alloc:
4620	idpf_vport_intr_napi_del_all(vport);
4621
4622	return err;
4623}
4624
4625void idpf_vport_intr_ena(struct idpf_vport *vport)
4626{
4627	idpf_vport_intr_napi_ena_all(vport);
4628	idpf_vport_intr_ena_irq_all(vport);
4629}
4630
4631/**
4632 * idpf_config_rss - Send virtchnl messages to configure RSS
4633 * @vport: virtual port
4634 *
4635 * Return 0 on success, negative on failure
4636 */
4637int idpf_config_rss(struct idpf_vport *vport)
4638{
4639	int err;
4640
4641	err = idpf_send_get_set_rss_key_msg(vport, false);
4642	if (err)
4643		return err;
4644
4645	return idpf_send_get_set_rss_lut_msg(vport, false);
4646}
4647
4648/**
4649 * idpf_fill_dflt_rss_lut - Fill the indirection table with the default values
4650 * @vport: virtual port structure
4651 */
4652void idpf_fill_dflt_rss_lut(struct idpf_vport *vport)
4653{
4654	struct idpf_adapter *adapter = vport->adapter;
4655	u16 num_active_rxq = vport->num_rxq;
4656	struct idpf_rss_data *rss_data;
4657	int i;
4658
4659	rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4660
4661	for (i = 0; i < rss_data->rss_lut_size; i++)
4662		rss_data->rss_lut[i] = i % num_active_rxq;
4663}
4664
4665/**
4666 * idpf_init_rss_lut - Allocate and initialize RSS LUT
4667 * @vport: virtual port
4668 *
4669 * Return: 0 on success, negative on failure
4670 */
4671int idpf_init_rss_lut(struct idpf_vport *vport)
4672{
4673	struct idpf_adapter *adapter = vport->adapter;
4674	struct idpf_rss_data *rss_data;
4675
4676	rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4677	if (!rss_data->rss_lut) {
4678		u32 lut_size;
4679
4680		lut_size = rss_data->rss_lut_size * sizeof(u32);
4681		rss_data->rss_lut = kzalloc(lut_size, GFP_KERNEL);
4682		if (!rss_data->rss_lut)
4683			return -ENOMEM;
4684	}
4685
4686	/* Fill the default RSS lut values */
4687	idpf_fill_dflt_rss_lut(vport);
4688
4689	return 0;
4690}
4691
4692/**
4693 * idpf_deinit_rss_lut - Release RSS LUT
4694 * @vport: virtual port
4695 */
4696void idpf_deinit_rss_lut(struct idpf_vport *vport)
4697{
4698	struct idpf_adapter *adapter = vport->adapter;
4699	struct idpf_rss_data *rss_data;
4700
4701	rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4702	kfree(rss_data->rss_lut);
4703	rss_data->rss_lut = NULL;
4704}