gve_tx_dqo.c

   1// SPDX-License-Identifier: (GPL-2.0 OR MIT)
   2/* Google virtual Ethernet (gve) driver
   3 *
   4 * Copyright (C) 2015-2021 Google, Inc.
   5 */
   6
   7#include "gve.h"
   8#include "gve_adminq.h"
   9#include "gve_utils.h"
  10#include "gve_dqo.h"
  11#include <net/ip.h>
  12#include <linux/bpf.h>
  13#include <linux/tcp.h>
  14#include <linux/slab.h>
  15#include <linux/skbuff.h>
  16#include <net/xdp_sock_drv.h>
  17
  18/* Returns true if tx_bufs are available. */
  19static bool gve_has_free_tx_qpl_bufs(struct gve_tx_ring *tx, int count)
  20{
  21	int num_avail;
  22
  23	if (!tx->dqo.qpl)
  24		return true;
  25
  26	num_avail = tx->dqo.num_tx_qpl_bufs -
  27		(tx->dqo_tx.alloc_tx_qpl_buf_cnt -
  28		 tx->dqo_tx.free_tx_qpl_buf_cnt);
  29
  30	if (count <= num_avail)
  31		return true;
  32
  33	/* Update cached value from dqo_compl. */
  34	tx->dqo_tx.free_tx_qpl_buf_cnt =
  35		atomic_read_acquire(&tx->dqo_compl.free_tx_qpl_buf_cnt);
  36
  37	num_avail = tx->dqo.num_tx_qpl_bufs -
  38		(tx->dqo_tx.alloc_tx_qpl_buf_cnt -
  39		 tx->dqo_tx.free_tx_qpl_buf_cnt);
  40
  41	return count <= num_avail;
  42}
  43
  44static s16
  45gve_alloc_tx_qpl_buf(struct gve_tx_ring *tx)
  46{
  47	s16 index;
  48
  49	index = tx->dqo_tx.free_tx_qpl_buf_head;
  50
  51	/* No TX buffers available, try to steal the list from the
  52	 * completion handler.
  53	 */
  54	if (unlikely(index == -1)) {
  55		tx->dqo_tx.free_tx_qpl_buf_head =
  56			atomic_xchg(&tx->dqo_compl.free_tx_qpl_buf_head, -1);
  57		index = tx->dqo_tx.free_tx_qpl_buf_head;
  58
  59		if (unlikely(index == -1))
  60			return index;
  61	}
  62
  63	/* Remove TX buf from free list */
  64	tx->dqo_tx.free_tx_qpl_buf_head = tx->dqo.tx_qpl_buf_next[index];
  65
  66	return index;
  67}
  68
  69static void
  70gve_free_tx_qpl_bufs(struct gve_tx_ring *tx,
  71		     struct gve_tx_pending_packet_dqo *pkt)
  72{
  73	s16 index;
  74	int i;
  75
  76	if (!pkt->num_bufs)
  77		return;
  78
  79	index = pkt->tx_qpl_buf_ids[0];
  80	/* Create a linked list of buffers to be added to the free list */
  81	for (i = 1; i < pkt->num_bufs; i++) {
  82		tx->dqo.tx_qpl_buf_next[index] = pkt->tx_qpl_buf_ids[i];
  83		index = pkt->tx_qpl_buf_ids[i];
  84	}
  85
  86	while (true) {
  87		s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_tx_qpl_buf_head);
  88
  89		tx->dqo.tx_qpl_buf_next[index] = old_head;
  90		if (atomic_cmpxchg(&tx->dqo_compl.free_tx_qpl_buf_head,
  91				   old_head,
  92				   pkt->tx_qpl_buf_ids[0]) == old_head) {
  93			break;
  94		}
  95	}
  96
  97	atomic_add(pkt->num_bufs, &tx->dqo_compl.free_tx_qpl_buf_cnt);
  98	pkt->num_bufs = 0;
  99}
 100
 101/* Returns true if a gve_tx_pending_packet_dqo object is available. */
 102static bool gve_has_pending_packet(struct gve_tx_ring *tx)
 103{
 104	/* Check TX path's list. */
 105	if (tx->dqo_tx.free_pending_packets != -1)
 106		return true;
 107
 108	/* Check completion handler's list. */
 109	if (atomic_read_acquire(&tx->dqo_compl.free_pending_packets) != -1)
 110		return true;
 111
 112	return false;
 113}
 114
 115void gve_xdp_tx_flush_dqo(struct gve_priv *priv, u32 xdp_qid)
 116{
 117	u32 tx_qid = gve_xdp_tx_queue_id(priv, xdp_qid);
 118	struct gve_tx_ring *tx = &priv->tx[tx_qid];
 119
 120	gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail);
 121}
 122
 123static struct gve_tx_pending_packet_dqo *
 124gve_alloc_pending_packet(struct gve_tx_ring *tx)
 125{
 126	struct gve_tx_pending_packet_dqo *pending_packet;
 127	s16 index;
 128
 129	index = tx->dqo_tx.free_pending_packets;
 130
 131	/* No pending_packets available, try to steal the list from the
 132	 * completion handler.
 133	 */
 134	if (unlikely(index == -1)) {
 135		tx->dqo_tx.free_pending_packets =
 136			atomic_xchg(&tx->dqo_compl.free_pending_packets, -1);
 137		index = tx->dqo_tx.free_pending_packets;
 138
 139		if (unlikely(index == -1))
 140			return NULL;
 141	}
 142
 143	pending_packet = &tx->dqo.pending_packets[index];
 144
 145	/* Remove pending_packet from free list */
 146	tx->dqo_tx.free_pending_packets = pending_packet->next;
 147	pending_packet->state = GVE_PACKET_STATE_PENDING_DATA_COMPL;
 148
 149	return pending_packet;
 150}
 151
 152static void
 153gve_free_pending_packet(struct gve_tx_ring *tx,
 154			struct gve_tx_pending_packet_dqo *pending_packet)
 155{
 156	s16 index = pending_packet - tx->dqo.pending_packets;
 157
 158	pending_packet->state = GVE_PACKET_STATE_UNALLOCATED;
 159	while (true) {
 160		s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_pending_packets);
 161
 162		pending_packet->next = old_head;
 163		if (atomic_cmpxchg(&tx->dqo_compl.free_pending_packets,
 164				   old_head, index) == old_head) {
 165			break;
 166		}
 167	}
 168}
 169
 170static void gve_unmap_packet(struct device *dev,
 171			     struct gve_tx_pending_packet_dqo *pkt)
 172{
 173	int i;
 174
 175	if (!pkt->num_bufs)
 176		return;
 177
 178	/* SKB linear portion is guaranteed to be mapped */
 179	dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]),
 180			 dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE);
 181	for (i = 1; i < pkt->num_bufs; i++) {
 182		netmem_dma_unmap_page_attrs(dev, dma_unmap_addr(pkt, dma[i]),
 183					    dma_unmap_len(pkt, len[i]),
 184					    DMA_TO_DEVICE, 0);
 185	}
 186	pkt->num_bufs = 0;
 187}
 188
 189/* gve_tx_free_desc - Cleans up all pending tx requests and buffers.
 190 */
 191static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx)
 192{
 193	int i;
 194
 195	for (i = 0; i < tx->dqo.num_pending_packets; i++) {
 196		struct gve_tx_pending_packet_dqo *cur_state =
 197			&tx->dqo.pending_packets[i];
 198
 199		if (tx->dqo.qpl)
 200			gve_free_tx_qpl_bufs(tx, cur_state);
 201		else
 202			gve_unmap_packet(tx->dev, cur_state);
 203
 204		if (cur_state->skb) {
 205			dev_consume_skb_any(cur_state->skb);
 206			cur_state->skb = NULL;
 207		}
 208	}
 209}
 210
 211void gve_tx_stop_ring_dqo(struct gve_priv *priv, int idx)
 212{
 213	int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
 214	struct gve_tx_ring *tx = &priv->tx[idx];
 215
 216	if (!gve_tx_was_added_to_block(priv, idx))
 217		return;
 218
 219	gve_remove_napi(priv, ntfy_idx);
 220	gve_clean_tx_done_dqo(priv, tx, /*napi=*/NULL);
 221	if (tx->netdev_txq)
 222		netdev_tx_reset_queue(tx->netdev_txq);
 223	gve_tx_clean_pending_packets(tx);
 224	gve_tx_remove_from_block(priv, idx);
 225}
 226
 227static void gve_tx_free_ring_dqo(struct gve_priv *priv, struct gve_tx_ring *tx,
 228				 struct gve_tx_alloc_rings_cfg *cfg)
 229{
 230	struct device *hdev = &priv->pdev->dev;
 231	int idx = tx->q_num;
 232	size_t bytes;
 233	u32 qpl_id;
 234
 235	if (tx->q_resources) {
 236		dma_free_coherent(hdev, sizeof(*tx->q_resources),
 237				  tx->q_resources, tx->q_resources_bus);
 238		tx->q_resources = NULL;
 239	}
 240
 241	if (tx->dqo.compl_ring) {
 242		bytes = sizeof(tx->dqo.compl_ring[0]) *
 243			(tx->dqo.complq_mask + 1);
 244		dma_free_coherent(hdev, bytes, tx->dqo.compl_ring,
 245				  tx->complq_bus_dqo);
 246		tx->dqo.compl_ring = NULL;
 247	}
 248
 249	if (tx->dqo.tx_ring) {
 250		bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1);
 251		dma_free_coherent(hdev, bytes, tx->dqo.tx_ring, tx->bus);
 252		tx->dqo.tx_ring = NULL;
 253	}
 254
 255	kvfree(tx->dqo.xsk_reorder_queue);
 256	tx->dqo.xsk_reorder_queue = NULL;
 257
 258	kvfree(tx->dqo.pending_packets);
 259	tx->dqo.pending_packets = NULL;
 260
 261	kvfree(tx->dqo.tx_qpl_buf_next);
 262	tx->dqo.tx_qpl_buf_next = NULL;
 263
 264	if (tx->dqo.qpl) {
 265		qpl_id = gve_tx_qpl_id(priv, tx->q_num);
 266		gve_free_queue_page_list(priv, tx->dqo.qpl, qpl_id);
 267		tx->dqo.qpl = NULL;
 268	}
 269
 270	netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx);
 271}
 272
 273static int gve_tx_qpl_buf_init(struct gve_tx_ring *tx)
 274{
 275	int num_tx_qpl_bufs = GVE_TX_BUFS_PER_PAGE_DQO *
 276		tx->dqo.qpl->num_entries;
 277	int i;
 278
 279	tx->dqo.tx_qpl_buf_next = kvcalloc(num_tx_qpl_bufs,
 280					   sizeof(tx->dqo.tx_qpl_buf_next[0]),
 281					   GFP_KERNEL);
 282	if (!tx->dqo.tx_qpl_buf_next)
 283		return -ENOMEM;
 284
 285	tx->dqo.num_tx_qpl_bufs = num_tx_qpl_bufs;
 286
 287	/* Generate free TX buf list */
 288	for (i = 0; i < num_tx_qpl_bufs - 1; i++)
 289		tx->dqo.tx_qpl_buf_next[i] = i + 1;
 290	tx->dqo.tx_qpl_buf_next[num_tx_qpl_bufs - 1] = -1;
 291
 292	atomic_set_release(&tx->dqo_compl.free_tx_qpl_buf_head, -1);
 293	return 0;
 294}
 295
 296void gve_tx_start_ring_dqo(struct gve_priv *priv, int idx)
 297{
 298	int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
 299	struct gve_tx_ring *tx = &priv->tx[idx];
 300
 301	gve_tx_add_to_block(priv, idx);
 302
 303	if (idx < priv->tx_cfg.num_queues)
 304		tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);
 305	gve_add_napi(priv, ntfy_idx, gve_napi_poll_dqo);
 306}
 307
 308static int gve_tx_alloc_ring_dqo(struct gve_priv *priv,
 309				 struct gve_tx_alloc_rings_cfg *cfg,
 310				 struct gve_tx_ring *tx,
 311				 int idx)
 312{
 313	struct device *hdev = &priv->pdev->dev;
 314	int num_pending_packets;
 315	int qpl_page_cnt;
 316	size_t bytes;
 317	u32 qpl_id;
 318	int i;
 319
 320	memset(tx, 0, sizeof(*tx));
 321	tx->q_num = idx;
 322	tx->dev = hdev;
 323	spin_lock_init(&tx->dqo_tx.xdp_lock);
 324	atomic_set_release(&tx->dqo_compl.hw_tx_head, 0);
 325
 326	/* Queue sizes must be a power of 2 */
 327	tx->mask = cfg->ring_size - 1;
 328	tx->dqo.complq_mask = tx->mask;
 329
 330	/* The max number of pending packets determines the maximum number of
 331	 * descriptors which maybe written to the completion queue.
 332	 *
 333	 * We must set the number small enough to make sure we never overrun the
 334	 * completion queue.
 335	 */
 336	num_pending_packets = tx->dqo.complq_mask + 1;
 337
 338	/* Reserve space for descriptor completions, which will be reported at
 339	 * most every GVE_TX_MIN_RE_INTERVAL packets.
 340	 */
 341	num_pending_packets -=
 342		(tx->dqo.complq_mask + 1) / GVE_TX_MIN_RE_INTERVAL;
 343
 344	/* Each packet may have at most 2 buffer completions if it receives both
 345	 * a miss and reinjection completion.
 346	 */
 347	num_pending_packets /= 2;
 348
 349	tx->dqo.num_pending_packets = min_t(int, num_pending_packets, S16_MAX);
 350	tx->dqo.pending_packets = kvcalloc(tx->dqo.num_pending_packets,
 351					   sizeof(tx->dqo.pending_packets[0]),
 352					   GFP_KERNEL);
 353	if (!tx->dqo.pending_packets)
 354		goto err;
 355
 356	/* Set up linked list of pending packets */
 357	for (i = 0; i < tx->dqo.num_pending_packets - 1; i++)
 358		tx->dqo.pending_packets[i].next = i + 1;
 359
 360	tx->dqo.pending_packets[tx->dqo.num_pending_packets - 1].next = -1;
 361	atomic_set_release(&tx->dqo_compl.free_pending_packets, -1);
 362
 363	/* Only alloc xsk pool for XDP queues */
 364	if (idx >= cfg->qcfg->num_queues && cfg->num_xdp_rings) {
 365		tx->dqo.xsk_reorder_queue =
 366			kvcalloc(tx->dqo.complq_mask + 1,
 367				 sizeof(tx->dqo.xsk_reorder_queue[0]),
 368				 GFP_KERNEL);
 369		if (!tx->dqo.xsk_reorder_queue)
 370			goto err;
 371	}
 372
 373	tx->dqo_compl.miss_completions.head = -1;
 374	tx->dqo_compl.miss_completions.tail = -1;
 375	tx->dqo_compl.timed_out_completions.head = -1;
 376	tx->dqo_compl.timed_out_completions.tail = -1;
 377
 378	bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1);
 379	tx->dqo.tx_ring = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL);
 380	if (!tx->dqo.tx_ring)
 381		goto err;
 382
 383	bytes = sizeof(tx->dqo.compl_ring[0]) * (tx->dqo.complq_mask + 1);
 384	tx->dqo.compl_ring = dma_alloc_coherent(hdev, bytes,
 385						&tx->complq_bus_dqo,
 386						GFP_KERNEL);
 387	if (!tx->dqo.compl_ring)
 388		goto err;
 389
 390	tx->q_resources = dma_alloc_coherent(hdev, sizeof(*tx->q_resources),
 391					     &tx->q_resources_bus, GFP_KERNEL);
 392	if (!tx->q_resources)
 393		goto err;
 394
 395	if (!cfg->raw_addressing) {
 396		qpl_id = gve_tx_qpl_id(priv, tx->q_num);
 397		qpl_page_cnt = priv->tx_pages_per_qpl;
 398
 399		tx->dqo.qpl = gve_alloc_queue_page_list(priv, qpl_id,
 400							qpl_page_cnt);
 401		if (!tx->dqo.qpl)
 402			goto err;
 403
 404		if (gve_tx_qpl_buf_init(tx))
 405			goto err;
 406	}
 407
 408	return 0;
 409
 410err:
 411	gve_tx_free_ring_dqo(priv, tx, cfg);
 412	return -ENOMEM;
 413}
 414
 415int gve_tx_alloc_rings_dqo(struct gve_priv *priv,
 416			   struct gve_tx_alloc_rings_cfg *cfg)
 417{
 418	struct gve_tx_ring *tx = cfg->tx;
 419	int total_queues;
 420	int err = 0;
 421	int i, j;
 422
 423	total_queues = cfg->qcfg->num_queues + cfg->num_xdp_rings;
 424	if (total_queues > cfg->qcfg->max_queues) {
 425		netif_err(priv, drv, priv->dev,
 426			  "Cannot alloc more than the max num of Tx rings\n");
 427		return -EINVAL;
 428	}
 429
 430	tx = kvcalloc(cfg->qcfg->max_queues, sizeof(struct gve_tx_ring),
 431		      GFP_KERNEL);
 432	if (!tx)
 433		return -ENOMEM;
 434
 435	for (i = 0; i < total_queues; i++) {
 436		err = gve_tx_alloc_ring_dqo(priv, cfg, &tx[i], i);
 437		if (err) {
 438			netif_err(priv, drv, priv->dev,
 439				  "Failed to alloc tx ring=%d: err=%d\n",
 440				  i, err);
 441			goto err;
 442		}
 443	}
 444
 445	cfg->tx = tx;
 446	return 0;
 447
 448err:
 449	for (j = 0; j < i; j++)
 450		gve_tx_free_ring_dqo(priv, &tx[j], cfg);
 451	kvfree(tx);
 452	return err;
 453}
 454
 455void gve_tx_free_rings_dqo(struct gve_priv *priv,
 456			   struct gve_tx_alloc_rings_cfg *cfg)
 457{
 458	struct gve_tx_ring *tx = cfg->tx;
 459	int i;
 460
 461	if (!tx)
 462		return;
 463
 464	for (i = 0; i < cfg->qcfg->num_queues + cfg->qcfg->num_xdp_queues; i++)
 465		gve_tx_free_ring_dqo(priv, &tx[i], cfg);
 466
 467	kvfree(tx);
 468	cfg->tx = NULL;
 469}
 470
 471/* Returns the number of slots available in the ring */
 472static u32 num_avail_tx_slots(const struct gve_tx_ring *tx)
 473{
 474	u32 num_used = (tx->dqo_tx.tail - tx->dqo_tx.head) & tx->mask;
 475
 476	return tx->mask - num_used;
 477}
 478
 479/* Checks if the requested number of slots are available in the ring */
 480static bool gve_has_tx_slots_available(struct gve_tx_ring *tx, u32 slots_req)
 481{
 482	u32 num_avail = num_avail_tx_slots(tx);
 483
 484	slots_req += GVE_TX_MIN_DESC_PREVENT_CACHE_OVERLAP;
 485
 486	if (num_avail >= slots_req)
 487		return true;
 488
 489	/* Update cached TX head pointer */
 490	tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head);
 491
 492	return num_avail_tx_slots(tx) >= slots_req;
 493}
 494
 495static bool gve_has_avail_slots_tx_dqo(struct gve_tx_ring *tx,
 496				       int desc_count, int buf_count)
 497{
 498	return gve_has_pending_packet(tx) &&
 499		gve_has_tx_slots_available(tx, desc_count) &&
 500		gve_has_free_tx_qpl_bufs(tx, buf_count);
 501}
 502
 503/* Stops the queue if available descriptors is less than 'count'.
 504 * Return: 0 if stop is not required.
 505 */
 506static int gve_maybe_stop_tx_dqo(struct gve_tx_ring *tx,
 507				 int desc_count, int buf_count)
 508{
 509	if (likely(gve_has_avail_slots_tx_dqo(tx, desc_count, buf_count)))
 510		return 0;
 511
 512	/* No space, so stop the queue */
 513	tx->stop_queue++;
 514	netif_tx_stop_queue(tx->netdev_txq);
 515
 516	/* Sync with restarting queue in `gve_tx_poll_dqo()` */
 517	mb();
 518
 519	/* After stopping queue, check if we can transmit again in order to
 520	 * avoid TOCTOU bug.
 521	 */
 522	if (likely(!gve_has_avail_slots_tx_dqo(tx, desc_count, buf_count)))
 523		return -EBUSY;
 524
 525	netif_tx_start_queue(tx->netdev_txq);
 526	tx->wake_queue++;
 527	return 0;
 528}
 529
 530static void gve_extract_tx_metadata_dqo(const struct sk_buff *skb,
 531					struct gve_tx_metadata_dqo *metadata)
 532{
 533	memset(metadata, 0, sizeof(*metadata));
 534	metadata->version = GVE_TX_METADATA_VERSION_DQO;
 535
 536	if (skb->l4_hash) {
 537		u16 path_hash = skb->hash ^ (skb->hash >> 16);
 538
 539		path_hash &= (1 << 15) - 1;
 540		if (unlikely(path_hash == 0))
 541			path_hash = ~path_hash;
 542
 543		metadata->path_hash = path_hash;
 544	}
 545}
 546
 547static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx,
 548				     bool enable_csum, u32 len, u64 addr,
 549				     s16 compl_tag, bool eop, bool is_gso)
 550{
 551	while (len > 0) {
 552		struct gve_tx_pkt_desc_dqo *desc =
 553			&tx->dqo.tx_ring[*desc_idx].pkt;
 554		u32 cur_len = min_t(u32, len, GVE_TX_MAX_BUF_SIZE_DQO);
 555		bool cur_eop = eop && cur_len == len;
 556
 557		*desc = (struct gve_tx_pkt_desc_dqo){
 558			.buf_addr = cpu_to_le64(addr),
 559			.dtype = GVE_TX_PKT_DESC_DTYPE_DQO,
 560			.end_of_packet = cur_eop,
 561			.checksum_offload_enable = enable_csum,
 562			.compl_tag = cpu_to_le16(compl_tag),
 563			.buf_size = cur_len,
 564		};
 565
 566		addr += cur_len;
 567		len -= cur_len;
 568		*desc_idx = (*desc_idx + 1) & tx->mask;
 569	}
 570}
 571
 572/* Validates and prepares `skb` for TSO.
 573 *
 574 * Returns header length, or < 0 if invalid.
 575 */
 576static int gve_prep_tso(struct sk_buff *skb)
 577{
 578	struct tcphdr *tcp;
 579	int header_len;
 580	u32 paylen;
 581	int err;
 582
 583	/* Note: HW requires MSS (gso_size) to be <= 9728 and the total length
 584	 * of the TSO to be <= 262143.
 585	 *
 586	 * However, we don't validate these because:
 587	 * - Hypervisor enforces a limit of 9K MTU
 588	 * - Kernel will not produce a TSO larger than 64k
 589	 */
 590
 591	if (unlikely(skb_shinfo(skb)->gso_size < GVE_TX_MIN_TSO_MSS_DQO))
 592		return -1;
 593
 594	if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
 595		return -EINVAL;
 596
 597	/* Needed because we will modify header. */
 598	err = skb_cow_head(skb, 0);
 599	if (err < 0)
 600		return err;
 601
 602	tcp = tcp_hdr(skb);
 603	paylen = skb->len - skb_transport_offset(skb);
 604	csum_replace_by_diff(&tcp->check, (__force __wsum)htonl(paylen));
 605	header_len = skb_tcp_all_headers(skb);
 606
 607	if (unlikely(header_len > GVE_TX_MAX_HDR_SIZE_DQO))
 608		return -EINVAL;
 609
 610	return header_len;
 611}
 612
 613static void gve_tx_fill_tso_ctx_desc(struct gve_tx_tso_context_desc_dqo *desc,
 614				     const struct sk_buff *skb,
 615				     const struct gve_tx_metadata_dqo *metadata,
 616				     int header_len)
 617{
 618	*desc = (struct gve_tx_tso_context_desc_dqo){
 619		.header_len = header_len,
 620		.cmd_dtype = {
 621			.dtype = GVE_TX_TSO_CTX_DESC_DTYPE_DQO,
 622			.tso = 1,
 623		},
 624		.flex0 = metadata->bytes[0],
 625		.flex5 = metadata->bytes[5],
 626		.flex6 = metadata->bytes[6],
 627		.flex7 = metadata->bytes[7],
 628		.flex8 = metadata->bytes[8],
 629		.flex9 = metadata->bytes[9],
 630		.flex10 = metadata->bytes[10],
 631		.flex11 = metadata->bytes[11],
 632	};
 633	desc->tso_total_len = skb->len - header_len;
 634	desc->mss = skb_shinfo(skb)->gso_size;
 635}
 636
 637static void
 638gve_tx_fill_general_ctx_desc(struct gve_tx_general_context_desc_dqo *desc,
 639			     const struct gve_tx_metadata_dqo *metadata)
 640{
 641	*desc = (struct gve_tx_general_context_desc_dqo){
 642		.flex0 = metadata->bytes[0],
 643		.flex1 = metadata->bytes[1],
 644		.flex2 = metadata->bytes[2],
 645		.flex3 = metadata->bytes[3],
 646		.flex4 = metadata->bytes[4],
 647		.flex5 = metadata->bytes[5],
 648		.flex6 = metadata->bytes[6],
 649		.flex7 = metadata->bytes[7],
 650		.flex8 = metadata->bytes[8],
 651		.flex9 = metadata->bytes[9],
 652		.flex10 = metadata->bytes[10],
 653		.flex11 = metadata->bytes[11],
 654		.cmd_dtype = {.dtype = GVE_TX_GENERAL_CTX_DESC_DTYPE_DQO},
 655	};
 656}
 657
 658static void gve_tx_update_tail(struct gve_tx_ring *tx, u32 desc_idx)
 659{
 660	u32 last_desc_idx = (desc_idx - 1) & tx->mask;
 661	u32 last_report_event_interval =
 662			(last_desc_idx - tx->dqo_tx.last_re_idx) & tx->mask;
 663
 664	/* Commit the changes to our state */
 665	tx->dqo_tx.tail = desc_idx;
 666
 667	/* Request a descriptor completion on the last descriptor of the
 668	 * packet if we are allowed to by the HW enforced interval.
 669	 */
 670
 671	if (unlikely(last_report_event_interval >= GVE_TX_MIN_RE_INTERVAL)) {
 672		tx->dqo.tx_ring[last_desc_idx].pkt.report_event = true;
 673		tx->dqo_tx.last_re_idx = last_desc_idx;
 674	}
 675}
 676
 677static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
 678				      struct sk_buff *skb,
 679				      struct gve_tx_pending_packet_dqo *pkt,
 680				      s16 completion_tag,
 681				      u32 *desc_idx,
 682				      bool is_gso)
 683{
 684	bool enable_csum = skb->ip_summed == CHECKSUM_PARTIAL;
 685	const struct skb_shared_info *shinfo = skb_shinfo(skb);
 686	int i;
 687
 688	/* Note: HW requires that the size of a non-TSO packet be within the
 689	 * range of [17, 9728].
 690	 *
 691	 * We don't double check because
 692	 * - We limited `netdev->min_mtu` to ETH_MIN_MTU.
 693	 * - Hypervisor won't allow MTU larger than 9216.
 694	 */
 695
 696	pkt->num_bufs = 0;
 697	/* Map the linear portion of skb */
 698	{
 699		u32 len = skb_headlen(skb);
 700		dma_addr_t addr;
 701
 702		addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE);
 703		if (unlikely(dma_mapping_error(tx->dev, addr)))
 704			goto err;
 705
 706		dma_unmap_len_set(pkt, len[pkt->num_bufs], len);
 707		dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr);
 708		++pkt->num_bufs;
 709
 710		gve_tx_fill_pkt_desc_dqo(tx, desc_idx, enable_csum, len, addr,
 711					 completion_tag,
 712					 /*eop=*/shinfo->nr_frags == 0, is_gso);
 713	}
 714
 715	for (i = 0; i < shinfo->nr_frags; i++) {
 716		const skb_frag_t *frag = &shinfo->frags[i];
 717		bool is_eop = i == (shinfo->nr_frags - 1);
 718		u32 len = skb_frag_size(frag);
 719		dma_addr_t addr;
 720
 721		addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE);
 722		if (unlikely(dma_mapping_error(tx->dev, addr)))
 723			goto err;
 724
 725		dma_unmap_len_set(pkt, len[pkt->num_bufs], len);
 726		netmem_dma_unmap_addr_set(skb_frag_netmem(frag), pkt,
 727					  dma[pkt->num_bufs], addr);
 728		++pkt->num_bufs;
 729
 730		gve_tx_fill_pkt_desc_dqo(tx, desc_idx, enable_csum, len, addr,
 731					 completion_tag, is_eop, is_gso);
 732	}
 733
 734	return 0;
 735err:
 736	for (i = 0; i < pkt->num_bufs; i++) {
 737		if (i == 0) {
 738			dma_unmap_single(tx->dev,
 739					 dma_unmap_addr(pkt, dma[i]),
 740					 dma_unmap_len(pkt, len[i]),
 741					 DMA_TO_DEVICE);
 742		} else {
 743			dma_unmap_page(tx->dev,
 744				       dma_unmap_addr(pkt, dma[i]),
 745				       dma_unmap_len(pkt, len[i]),
 746				       DMA_TO_DEVICE);
 747		}
 748	}
 749	pkt->num_bufs = 0;
 750	return -1;
 751}
 752
 753/* Tx buffer i corresponds to
 754 * qpl_page_id = i / GVE_TX_BUFS_PER_PAGE_DQO
 755 * qpl_page_offset = (i % GVE_TX_BUFS_PER_PAGE_DQO) * GVE_TX_BUF_SIZE_DQO
 756 */
 757static void gve_tx_buf_get_addr(struct gve_tx_ring *tx,
 758				s16 index,
 759				void **va, dma_addr_t *dma_addr)
 760{
 761	int page_id = index >> (PAGE_SHIFT - GVE_TX_BUF_SHIFT_DQO);
 762	int offset = (index & (GVE_TX_BUFS_PER_PAGE_DQO - 1)) << GVE_TX_BUF_SHIFT_DQO;
 763
 764	*va = page_address(tx->dqo.qpl->pages[page_id]) + offset;
 765	*dma_addr = tx->dqo.qpl->page_buses[page_id] + offset;
 766}
 767
 768static int gve_tx_add_skb_copy_dqo(struct gve_tx_ring *tx,
 769				   struct sk_buff *skb,
 770				   struct gve_tx_pending_packet_dqo *pkt,
 771				   s16 completion_tag,
 772				   u32 *desc_idx,
 773				   bool is_gso)
 774{
 775	bool enable_csum = skb->ip_summed == CHECKSUM_PARTIAL;
 776	u32 copy_offset = 0;
 777	dma_addr_t dma_addr;
 778	u32 copy_len;
 779	s16 index;
 780	void *va;
 781
 782	/* Break the packet into buffer size chunks */
 783	pkt->num_bufs = 0;
 784	while (copy_offset < skb->len) {
 785		index = gve_alloc_tx_qpl_buf(tx);
 786		if (unlikely(index == -1))
 787			goto err;
 788
 789		gve_tx_buf_get_addr(tx, index, &va, &dma_addr);
 790		copy_len = min_t(u32, GVE_TX_BUF_SIZE_DQO,
 791				 skb->len - copy_offset);
 792		skb_copy_bits(skb, copy_offset, va, copy_len);
 793
 794		copy_offset += copy_len;
 795		dma_sync_single_for_device(tx->dev, dma_addr,
 796					   copy_len, DMA_TO_DEVICE);
 797		gve_tx_fill_pkt_desc_dqo(tx, desc_idx, enable_csum,
 798					 copy_len,
 799					 dma_addr,
 800					 completion_tag,
 801					 copy_offset == skb->len,
 802					 is_gso);
 803
 804		pkt->tx_qpl_buf_ids[pkt->num_bufs] = index;
 805		++tx->dqo_tx.alloc_tx_qpl_buf_cnt;
 806		++pkt->num_bufs;
 807	}
 808
 809	return 0;
 810err:
 811	/* Should not be here if gve_has_free_tx_qpl_bufs() check is correct */
 812	gve_free_tx_qpl_bufs(tx, pkt);
 813	return -ENOMEM;
 814}
 815
 816/* Returns 0 on success, or < 0 on error.
 817 *
 818 * Before this function is called, the caller must ensure
 819 * gve_has_pending_packet(tx) returns true.
 820 */
 821static int gve_tx_add_skb_dqo(struct gve_tx_ring *tx,
 822			      struct sk_buff *skb)
 823{
 824	const bool is_gso = skb_is_gso(skb);
 825	u32 desc_idx = tx->dqo_tx.tail;
 826	struct gve_tx_pending_packet_dqo *pkt;
 827	struct gve_tx_metadata_dqo metadata;
 828	s16 completion_tag;
 829
 830	pkt = gve_alloc_pending_packet(tx);
 831	if (!pkt)
 832		return -ENOMEM;
 833
 834	pkt->skb = skb;
 835	pkt->type = GVE_TX_PENDING_PACKET_DQO_SKB;
 836	completion_tag = pkt - tx->dqo.pending_packets;
 837
 838	gve_extract_tx_metadata_dqo(skb, &metadata);
 839	if (is_gso) {
 840		int header_len = gve_prep_tso(skb);
 841
 842		if (unlikely(header_len < 0))
 843			goto err;
 844
 845		gve_tx_fill_tso_ctx_desc(&tx->dqo.tx_ring[desc_idx].tso_ctx,
 846					 skb, &metadata, header_len);
 847		desc_idx = (desc_idx + 1) & tx->mask;
 848	}
 849
 850	gve_tx_fill_general_ctx_desc(&tx->dqo.tx_ring[desc_idx].general_ctx,
 851				     &metadata);
 852	desc_idx = (desc_idx + 1) & tx->mask;
 853
 854	if (tx->dqo.qpl) {
 855		if (gve_tx_add_skb_copy_dqo(tx, skb, pkt,
 856					    completion_tag,
 857					    &desc_idx, is_gso))
 858			goto err;
 859	}  else {
 860		if (gve_tx_add_skb_no_copy_dqo(tx, skb, pkt,
 861					       completion_tag,
 862					       &desc_idx, is_gso))
 863			goto err;
 864	}
 865
 866	tx->dqo_tx.posted_packet_desc_cnt += pkt->num_bufs;
 867
 868	gve_tx_update_tail(tx, desc_idx);
 869	return 0;
 870
 871err:
 872	pkt->skb = NULL;
 873	gve_free_pending_packet(tx, pkt);
 874
 875	return -1;
 876}
 877
 878static int gve_num_descs_per_buf(size_t size)
 879{
 880	return DIV_ROUND_UP(size, GVE_TX_MAX_BUF_SIZE_DQO);
 881}
 882
 883static int gve_num_buffer_descs_needed(const struct sk_buff *skb)
 884{
 885	const struct skb_shared_info *shinfo = skb_shinfo(skb);
 886	int num_descs;
 887	int i;
 888
 889	num_descs = gve_num_descs_per_buf(skb_headlen(skb));
 890
 891	for (i = 0; i < shinfo->nr_frags; i++) {
 892		unsigned int frag_size = skb_frag_size(&shinfo->frags[i]);
 893
 894		num_descs += gve_num_descs_per_buf(frag_size);
 895	}
 896
 897	return num_descs;
 898}
 899
 900/* Returns true if HW is capable of sending TSO represented by `skb`.
 901 *
 902 * Each segment must not span more than GVE_TX_MAX_DATA_DESCS buffers.
 903 * - The header is counted as one buffer for every single segment.
 904 * - A buffer which is split between two segments is counted for both.
 905 * - If a buffer contains both header and payload, it is counted as two buffers.
 906 */
 907static bool gve_can_send_tso(const struct sk_buff *skb)
 908{
 909	const int max_bufs_per_seg = GVE_TX_MAX_DATA_DESCS - 1;
 910	const struct skb_shared_info *shinfo = skb_shinfo(skb);
 911	const int header_len = skb_tcp_all_headers(skb);
 912	const int gso_size = shinfo->gso_size;
 913	int cur_seg_num_bufs;
 914	int prev_frag_size;
 915	int cur_seg_size;
 916	int i;
 917
 918	cur_seg_size = skb_headlen(skb) - header_len;
 919	prev_frag_size = skb_headlen(skb);
 920	cur_seg_num_bufs = cur_seg_size > 0;
 921
 922	for (i = 0; i < shinfo->nr_frags; i++) {
 923		if (cur_seg_size >= gso_size) {
 924			cur_seg_size %= gso_size;
 925			cur_seg_num_bufs = cur_seg_size > 0;
 926
 927			if (prev_frag_size > GVE_TX_MAX_BUF_SIZE_DQO) {
 928				int prev_frag_remain = prev_frag_size %
 929					GVE_TX_MAX_BUF_SIZE_DQO;
 930
 931				/* If the last descriptor of the previous frag
 932				 * is less than cur_seg_size, the segment will
 933				 * span two descriptors in the previous frag.
 934				 * Since max gso size (9728) is less than
 935				 * GVE_TX_MAX_BUF_SIZE_DQO, it is impossible
 936				 * for the segment to span more than two
 937				 * descriptors.
 938				 */
 939				if (prev_frag_remain &&
 940				    cur_seg_size > prev_frag_remain)
 941					cur_seg_num_bufs++;
 942			}
 943		}
 944
 945		if (unlikely(++cur_seg_num_bufs > max_bufs_per_seg))
 946			return false;
 947
 948		prev_frag_size = skb_frag_size(&shinfo->frags[i]);
 949		cur_seg_size += prev_frag_size;
 950	}
 951
 952	return true;
 953}
 954
 955netdev_features_t gve_features_check_dqo(struct sk_buff *skb,
 956					 struct net_device *dev,
 957					 netdev_features_t features)
 958{
 959	if (skb_is_gso(skb) && !gve_can_send_tso(skb))
 960		return features & ~NETIF_F_GSO_MASK;
 961
 962	return features;
 963}
 964
 965/* Attempt to transmit specified SKB.
 966 *
 967 * Returns 0 if the SKB was transmitted or dropped.
 968 * Returns -1 if there is not currently enough space to transmit the SKB.
 969 */
 970static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx,
 971			  struct sk_buff *skb)
 972{
 973	int num_buffer_descs;
 974	int total_num_descs;
 975
 976	if (skb_is_gso(skb) && unlikely(ipv6_hopopt_jumbo_remove(skb)))
 977		goto drop;
 978
 979	if (tx->dqo.qpl) {
 980		/* We do not need to verify the number of buffers used per
 981		 * packet or per segment in case of TSO as with 2K size buffers
 982		 * none of the TX packet rules would be violated.
 983		 *
 984		 * gve_can_send_tso() checks that each TCP segment of gso_size is
 985		 * not distributed over more than 9 SKB frags..
 986		 */
 987		num_buffer_descs = DIV_ROUND_UP(skb->len, GVE_TX_BUF_SIZE_DQO);
 988	} else {
 989		num_buffer_descs = gve_num_buffer_descs_needed(skb);
 990		if (!skb_is_gso(skb)) {
 991			if (unlikely(num_buffer_descs > GVE_TX_MAX_DATA_DESCS)) {
 992				if (unlikely(skb_linearize(skb) < 0))
 993					goto drop;
 994
 995				num_buffer_descs = 1;
 996			}
 997		}
 998	}
 999
1000	/* Metadata + (optional TSO) + data descriptors. */
1001	total_num_descs = 1 + skb_is_gso(skb) + num_buffer_descs;
1002	if (unlikely(gve_maybe_stop_tx_dqo(tx, total_num_descs,
1003					   num_buffer_descs))) {
1004		return -1;
1005	}
1006
1007	if (unlikely(gve_tx_add_skb_dqo(tx, skb) < 0))
1008		goto drop;
1009
1010	netdev_tx_sent_queue(tx->netdev_txq, skb->len);
1011	skb_tx_timestamp(skb);
1012	return 0;
1013
1014drop:
1015	u64_stats_update_begin(&tx->statss);
1016	tx->dropped_pkt++;
1017	u64_stats_update_end(&tx->statss);
1018	dev_kfree_skb_any(skb);
1019	return 0;
1020}
1021
1022static void gve_xsk_reorder_queue_push_dqo(struct gve_tx_ring *tx,
1023					   u16 completion_tag)
1024{
1025	u32 tail = atomic_read(&tx->dqo_tx.xsk_reorder_queue_tail);
1026
1027	tx->dqo.xsk_reorder_queue[tail] = completion_tag;
1028	tail = (tail + 1) & tx->dqo.complq_mask;
1029	atomic_set_release(&tx->dqo_tx.xsk_reorder_queue_tail, tail);
1030}
1031
1032static struct gve_tx_pending_packet_dqo *
1033gve_xsk_reorder_queue_head(struct gve_tx_ring *tx)
1034{
1035	u32 head = tx->dqo_compl.xsk_reorder_queue_head;
1036
1037	if (head == tx->dqo_compl.xsk_reorder_queue_tail) {
1038		tx->dqo_compl.xsk_reorder_queue_tail =
1039			atomic_read_acquire(&tx->dqo_tx.xsk_reorder_queue_tail);
1040
1041		if (head == tx->dqo_compl.xsk_reorder_queue_tail)
1042			return NULL;
1043	}
1044
1045	return &tx->dqo.pending_packets[tx->dqo.xsk_reorder_queue[head]];
1046}
1047
1048static void gve_xsk_reorder_queue_pop_dqo(struct gve_tx_ring *tx)
1049{
1050	tx->dqo_compl.xsk_reorder_queue_head++;
1051	tx->dqo_compl.xsk_reorder_queue_head &= tx->dqo.complq_mask;
1052}
1053
1054/* Transmit a given skb and ring the doorbell. */
1055netdev_tx_t gve_tx_dqo(struct sk_buff *skb, struct net_device *dev)
1056{
1057	struct gve_priv *priv = netdev_priv(dev);
1058	struct gve_tx_ring *tx;
1059
1060	tx = &priv->tx[skb_get_queue_mapping(skb)];
1061	if (unlikely(gve_try_tx_skb(priv, tx, skb) < 0)) {
1062		/* We need to ring the txq doorbell -- we have stopped the Tx
1063		 * queue for want of resources, but prior calls to gve_tx()
1064		 * may have added descriptors without ringing the doorbell.
1065		 */
1066		gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail);
1067		return NETDEV_TX_BUSY;
1068	}
1069
1070	if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more())
1071		return NETDEV_TX_OK;
1072
1073	gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail);
1074	return NETDEV_TX_OK;
1075}
1076
1077static bool gve_xsk_tx_dqo(struct gve_priv *priv, struct gve_tx_ring *tx,
1078			   int budget)
1079{
1080	struct xsk_buff_pool *pool = tx->xsk_pool;
1081	struct xdp_desc desc;
1082	bool repoll = false;
1083	int sent = 0;
1084
1085	spin_lock(&tx->dqo_tx.xdp_lock);
1086	for (; sent < budget; sent++) {
1087		struct gve_tx_pending_packet_dqo *pkt;
1088		s16 completion_tag;
1089		dma_addr_t addr;
1090		u32 desc_idx;
1091
1092		if (unlikely(!gve_has_avail_slots_tx_dqo(tx, 1, 1))) {
1093			repoll = true;
1094			break;
1095		}
1096
1097		if (!xsk_tx_peek_desc(pool, &desc))
1098			break;
1099
1100		pkt = gve_alloc_pending_packet(tx);
1101		pkt->type = GVE_TX_PENDING_PACKET_DQO_XSK;
1102		pkt->num_bufs = 0;
1103		completion_tag = pkt - tx->dqo.pending_packets;
1104
1105		addr = xsk_buff_raw_get_dma(pool, desc.addr);
1106		xsk_buff_raw_dma_sync_for_device(pool, addr, desc.len);
1107
1108		desc_idx = tx->dqo_tx.tail;
1109		gve_tx_fill_pkt_desc_dqo(tx, &desc_idx,
1110					 true, desc.len,
1111					 addr, completion_tag, true,
1112					 false);
1113		++pkt->num_bufs;
1114		gve_tx_update_tail(tx, desc_idx);
1115		tx->dqo_tx.posted_packet_desc_cnt += pkt->num_bufs;
1116		gve_xsk_reorder_queue_push_dqo(tx, completion_tag);
1117	}
1118
1119	if (sent) {
1120		gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail);
1121		xsk_tx_release(pool);
1122	}
1123
1124	spin_unlock(&tx->dqo_tx.xdp_lock);
1125
1126	u64_stats_update_begin(&tx->statss);
1127	tx->xdp_xsk_sent += sent;
1128	u64_stats_update_end(&tx->statss);
1129
1130	return (sent == budget) || repoll;
1131}
1132
1133static void add_to_list(struct gve_tx_ring *tx, struct gve_index_list *list,
1134			struct gve_tx_pending_packet_dqo *pending_packet)
1135{
1136	s16 old_tail, index;
1137
1138	index = pending_packet - tx->dqo.pending_packets;
1139	old_tail = list->tail;
1140	list->tail = index;
1141	if (old_tail == -1)
1142		list->head = index;
1143	else
1144		tx->dqo.pending_packets[old_tail].next = index;
1145
1146	pending_packet->next = -1;
1147	pending_packet->prev = old_tail;
1148}
1149
1150static void remove_from_list(struct gve_tx_ring *tx,
1151			     struct gve_index_list *list,
1152			     struct gve_tx_pending_packet_dqo *pkt)
1153{
1154	s16 prev_index, next_index;
1155
1156	prev_index = pkt->prev;
1157	next_index = pkt->next;
1158
1159	if (prev_index == -1) {
1160		/* Node is head */
1161		list->head = next_index;
1162	} else {
1163		tx->dqo.pending_packets[prev_index].next = next_index;
1164	}
1165	if (next_index == -1) {
1166		/* Node is tail */
1167		list->tail = prev_index;
1168	} else {
1169		tx->dqo.pending_packets[next_index].prev = prev_index;
1170	}
1171}
1172
1173/* Completion types and expected behavior:
1174 * No Miss compl + Packet compl = Packet completed normally.
1175 * Miss compl + Re-inject compl = Packet completed normally.
1176 * No Miss compl + Re-inject compl = Skipped i.e. packet not completed.
1177 * Miss compl + Packet compl = Skipped i.e. packet not completed.
1178 */
1179static void gve_handle_packet_completion(struct gve_priv *priv,
1180					 struct gve_tx_ring *tx, bool is_napi,
1181					 u16 compl_tag, u64 *bytes, u64 *pkts,
1182					 bool is_reinjection)
1183{
1184	struct gve_tx_pending_packet_dqo *pending_packet;
1185
1186	if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) {
1187		net_err_ratelimited("%s: Invalid TX completion tag: %d\n",
1188				    priv->dev->name, (int)compl_tag);
1189		return;
1190	}
1191
1192	pending_packet = &tx->dqo.pending_packets[compl_tag];
1193
1194	if (unlikely(is_reinjection)) {
1195		if (unlikely(pending_packet->state ==
1196			     GVE_PACKET_STATE_TIMED_OUT_COMPL)) {
1197			net_err_ratelimited("%s: Re-injection completion: %d received after timeout.\n",
1198					    priv->dev->name, (int)compl_tag);
1199			/* Packet was already completed as a result of timeout,
1200			 * so just remove from list and free pending packet.
1201			 */
1202			remove_from_list(tx,
1203					 &tx->dqo_compl.timed_out_completions,
1204					 pending_packet);
1205			gve_free_pending_packet(tx, pending_packet);
1206			return;
1207		}
1208		if (unlikely(pending_packet->state !=
1209			     GVE_PACKET_STATE_PENDING_REINJECT_COMPL)) {
1210			/* No outstanding miss completion but packet allocated
1211			 * implies packet receives a re-injection completion
1212			 * without a prior miss completion. Return without
1213			 * completing the packet.
1214			 */
1215			net_err_ratelimited("%s: Re-injection completion received without corresponding miss completion: %d\n",
1216					    priv->dev->name, (int)compl_tag);
1217			return;
1218		}
1219		remove_from_list(tx, &tx->dqo_compl.miss_completions,
1220				 pending_packet);
1221	} else {
1222		/* Packet is allocated but not a pending data completion. */
1223		if (unlikely(pending_packet->state !=
1224			     GVE_PACKET_STATE_PENDING_DATA_COMPL)) {
1225			net_err_ratelimited("%s: No pending data completion: %d\n",
1226					    priv->dev->name, (int)compl_tag);
1227			return;
1228		}
1229	}
1230	tx->dqo_tx.completed_packet_desc_cnt += pending_packet->num_bufs;
1231
1232	switch (pending_packet->type) {
1233	case GVE_TX_PENDING_PACKET_DQO_SKB:
1234		if (tx->dqo.qpl)
1235			gve_free_tx_qpl_bufs(tx, pending_packet);
1236		else
1237			gve_unmap_packet(tx->dev, pending_packet);
1238		(*pkts)++;
1239		*bytes += pending_packet->skb->len;
1240
1241		napi_consume_skb(pending_packet->skb, is_napi);
1242		pending_packet->skb = NULL;
1243		gve_free_pending_packet(tx, pending_packet);
1244		break;
1245	case GVE_TX_PENDING_PACKET_DQO_XDP_FRAME:
1246		gve_unmap_packet(tx->dev, pending_packet);
1247		(*pkts)++;
1248		*bytes += pending_packet->xdpf->len;
1249
1250		xdp_return_frame(pending_packet->xdpf);
1251		pending_packet->xdpf = NULL;
1252		gve_free_pending_packet(tx, pending_packet);
1253		break;
1254	case GVE_TX_PENDING_PACKET_DQO_XSK:
1255		pending_packet->state = GVE_PACKET_STATE_XSK_COMPLETE;
1256		break;
1257	default:
1258		WARN_ON_ONCE(1);
1259	}
1260}
1261
1262static void gve_handle_miss_completion(struct gve_priv *priv,
1263				       struct gve_tx_ring *tx, u16 compl_tag,
1264				       u64 *bytes, u64 *pkts)
1265{
1266	struct gve_tx_pending_packet_dqo *pending_packet;
1267
1268	if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) {
1269		net_err_ratelimited("%s: Invalid TX completion tag: %d\n",
1270				    priv->dev->name, (int)compl_tag);
1271		return;
1272	}
1273
1274	pending_packet = &tx->dqo.pending_packets[compl_tag];
1275	if (unlikely(pending_packet->state !=
1276				GVE_PACKET_STATE_PENDING_DATA_COMPL)) {
1277		net_err_ratelimited("%s: Unexpected packet state: %d for completion tag : %d\n",
1278				    priv->dev->name, (int)pending_packet->state,
1279				    (int)compl_tag);
1280		return;
1281	}
1282
1283	pending_packet->state = GVE_PACKET_STATE_PENDING_REINJECT_COMPL;
1284	/* jiffies can wraparound but time comparisons can handle overflows. */
1285	pending_packet->timeout_jiffies =
1286			jiffies +
1287			secs_to_jiffies(GVE_REINJECT_COMPL_TIMEOUT);
1288	add_to_list(tx, &tx->dqo_compl.miss_completions, pending_packet);
1289
1290	*bytes += pending_packet->skb->len;
1291	(*pkts)++;
1292}
1293
1294static void remove_miss_completions(struct gve_priv *priv,
1295				    struct gve_tx_ring *tx)
1296{
1297	struct gve_tx_pending_packet_dqo *pending_packet;
1298	s16 next_index;
1299
1300	next_index = tx->dqo_compl.miss_completions.head;
1301	while (next_index != -1) {
1302		pending_packet = &tx->dqo.pending_packets[next_index];
1303		next_index = pending_packet->next;
1304		/* Break early because packets should timeout in order. */
1305		if (time_is_after_jiffies(pending_packet->timeout_jiffies))
1306			break;
1307
1308		remove_from_list(tx, &tx->dqo_compl.miss_completions,
1309				 pending_packet);
1310		/* Unmap/free TX buffers and free skb but do not unallocate packet i.e.
1311		 * the completion tag is not freed to ensure that the driver
1312		 * can take appropriate action if a corresponding valid
1313		 * completion is received later.
1314		 */
1315		if (tx->dqo.qpl)
1316			gve_free_tx_qpl_bufs(tx, pending_packet);
1317		else
1318			gve_unmap_packet(tx->dev, pending_packet);
1319
1320		/* This indicates the packet was dropped. */
1321		dev_kfree_skb_any(pending_packet->skb);
1322		pending_packet->skb = NULL;
1323
1324		u64_stats_update_begin(&tx->statss);
1325		tx->dropped_pkt++;
1326		u64_stats_update_end(&tx->statss);
1327
1328		net_err_ratelimited("%s: No reinjection completion was received for: %d.\n",
1329				    priv->dev->name,
1330				    (int)(pending_packet - tx->dqo.pending_packets));
1331
1332		pending_packet->state = GVE_PACKET_STATE_TIMED_OUT_COMPL;
1333		pending_packet->timeout_jiffies =
1334				jiffies +
1335				secs_to_jiffies(GVE_DEALLOCATE_COMPL_TIMEOUT);
1336		/* Maintain pending packet in another list so the packet can be
1337		 * unallocated at a later time.
1338		 */
1339		add_to_list(tx, &tx->dqo_compl.timed_out_completions,
1340			    pending_packet);
1341	}
1342}
1343
1344static void remove_timed_out_completions(struct gve_priv *priv,
1345					 struct gve_tx_ring *tx)
1346{
1347	struct gve_tx_pending_packet_dqo *pending_packet;
1348	s16 next_index;
1349
1350	next_index = tx->dqo_compl.timed_out_completions.head;
1351	while (next_index != -1) {
1352		pending_packet = &tx->dqo.pending_packets[next_index];
1353		next_index = pending_packet->next;
1354		/* Break early because packets should timeout in order. */
1355		if (time_is_after_jiffies(pending_packet->timeout_jiffies))
1356			break;
1357
1358		remove_from_list(tx, &tx->dqo_compl.timed_out_completions,
1359				 pending_packet);
1360
1361		/* Need to count XSK packets in xsk_tx_completed. */
1362		if (pending_packet->type == GVE_TX_PENDING_PACKET_DQO_XSK)
1363			pending_packet->state = GVE_PACKET_STATE_XSK_COMPLETE;
1364		else
1365			gve_free_pending_packet(tx, pending_packet);
1366	}
1367}
1368
1369static void gve_tx_process_xsk_completions(struct gve_tx_ring *tx)
1370{
1371	u32 num_xsks = 0;
1372
1373	while (true) {
1374		struct gve_tx_pending_packet_dqo *pending_packet =
1375			gve_xsk_reorder_queue_head(tx);
1376
1377		if (!pending_packet ||
1378		    pending_packet->state != GVE_PACKET_STATE_XSK_COMPLETE)
1379			break;
1380
1381		num_xsks++;
1382		gve_xsk_reorder_queue_pop_dqo(tx);
1383		gve_free_pending_packet(tx, pending_packet);
1384	}
1385
1386	if (num_xsks)
1387		xsk_tx_completed(tx->xsk_pool, num_xsks);
1388}
1389
1390int gve_clean_tx_done_dqo(struct gve_priv *priv, struct gve_tx_ring *tx,
1391			  struct napi_struct *napi)
1392{
1393	u64 reinject_compl_bytes = 0;
1394	u64 reinject_compl_pkts = 0;
1395	int num_descs_cleaned = 0;
1396	u64 miss_compl_bytes = 0;
1397	u64 miss_compl_pkts = 0;
1398	u64 pkt_compl_bytes = 0;
1399	u64 pkt_compl_pkts = 0;
1400
1401	/* Limit in order to avoid blocking for too long */
1402	while (!napi || pkt_compl_pkts < napi->weight) {
1403		struct gve_tx_compl_desc *compl_desc =
1404			&tx->dqo.compl_ring[tx->dqo_compl.head];
1405		u16 type;
1406
1407		if (compl_desc->generation == tx->dqo_compl.cur_gen_bit)
1408			break;
1409
1410		/* Prefetch the next descriptor. */
1411		prefetch(&tx->dqo.compl_ring[(tx->dqo_compl.head + 1) &
1412				tx->dqo.complq_mask]);
1413
1414		/* Do not read data until we own the descriptor */
1415		dma_rmb();
1416		type = compl_desc->type;
1417
1418		if (type == GVE_COMPL_TYPE_DQO_DESC) {
1419			/* This is the last descriptor fetched by HW plus one */
1420			u16 tx_head = le16_to_cpu(compl_desc->tx_head);
1421
1422			atomic_set_release(&tx->dqo_compl.hw_tx_head, tx_head);
1423		} else if (type == GVE_COMPL_TYPE_DQO_PKT) {
1424			u16 compl_tag = le16_to_cpu(compl_desc->completion_tag);
1425			if (compl_tag & GVE_ALT_MISS_COMPL_BIT) {
1426				compl_tag &= ~GVE_ALT_MISS_COMPL_BIT;
1427				gve_handle_miss_completion(priv, tx, compl_tag,
1428							   &miss_compl_bytes,
1429							   &miss_compl_pkts);
1430			} else {
1431				gve_handle_packet_completion(priv, tx, !!napi,
1432							     compl_tag,
1433							     &pkt_compl_bytes,
1434							     &pkt_compl_pkts,
1435							     false);
1436			}
1437		} else if (type == GVE_COMPL_TYPE_DQO_MISS) {
1438			u16 compl_tag = le16_to_cpu(compl_desc->completion_tag);
1439
1440			gve_handle_miss_completion(priv, tx, compl_tag,
1441						   &miss_compl_bytes,
1442						   &miss_compl_pkts);
1443		} else if (type == GVE_COMPL_TYPE_DQO_REINJECTION) {
1444			u16 compl_tag = le16_to_cpu(compl_desc->completion_tag);
1445
1446			gve_handle_packet_completion(priv, tx, !!napi,
1447						     compl_tag,
1448						     &reinject_compl_bytes,
1449						     &reinject_compl_pkts,
1450						     true);
1451		}
1452
1453		tx->dqo_compl.head =
1454			(tx->dqo_compl.head + 1) & tx->dqo.complq_mask;
1455		/* Flip the generation bit when we wrap around */
1456		tx->dqo_compl.cur_gen_bit ^= tx->dqo_compl.head == 0;
1457		num_descs_cleaned++;
1458	}
1459
1460	if (tx->netdev_txq)
1461		netdev_tx_completed_queue(tx->netdev_txq,
1462					  pkt_compl_pkts + miss_compl_pkts,
1463					  pkt_compl_bytes + miss_compl_bytes);
1464
1465	remove_miss_completions(priv, tx);
1466	remove_timed_out_completions(priv, tx);
1467
1468	if (tx->xsk_pool)
1469		gve_tx_process_xsk_completions(tx);
1470
1471	u64_stats_update_begin(&tx->statss);
1472	tx->bytes_done += pkt_compl_bytes + reinject_compl_bytes;
1473	tx->pkt_done += pkt_compl_pkts + reinject_compl_pkts;
1474	u64_stats_update_end(&tx->statss);
1475	return num_descs_cleaned;
1476}
1477
1478bool gve_tx_poll_dqo(struct gve_notify_block *block, bool do_clean)
1479{
1480	struct gve_tx_compl_desc *compl_desc;
1481	struct gve_tx_ring *tx = block->tx;
1482	struct gve_priv *priv = block->priv;
1483
1484	if (do_clean) {
1485		int num_descs_cleaned = gve_clean_tx_done_dqo(priv, tx,
1486							      &block->napi);
1487
1488		/* Sync with queue being stopped in `gve_maybe_stop_tx_dqo()` */
1489		mb();
1490
1491		if (netif_tx_queue_stopped(tx->netdev_txq) &&
1492		    num_descs_cleaned > 0) {
1493			tx->wake_queue++;
1494			netif_tx_wake_queue(tx->netdev_txq);
1495		}
1496	}
1497
1498	/* Return true if we still have work. */
1499	compl_desc = &tx->dqo.compl_ring[tx->dqo_compl.head];
1500	return compl_desc->generation != tx->dqo_compl.cur_gen_bit;
1501}
1502
1503bool gve_xsk_tx_poll_dqo(struct gve_notify_block *rx_block, int budget)
1504{
1505	struct gve_rx_ring *rx = rx_block->rx;
1506	struct gve_priv *priv = rx->gve;
1507	struct gve_tx_ring *tx;
1508
1509	tx = &priv->tx[gve_xdp_tx_queue_id(priv, rx->q_num)];
1510	if (tx->xsk_pool)
1511		return gve_xsk_tx_dqo(priv, tx, budget);
1512
1513	return 0;
1514}
1515
1516bool gve_xdp_poll_dqo(struct gve_notify_block *block)
1517{
1518	struct gve_tx_compl_desc *compl_desc;
1519	struct gve_tx_ring *tx = block->tx;
1520	struct gve_priv *priv = block->priv;
1521
1522	gve_clean_tx_done_dqo(priv, tx, &block->napi);
1523
1524	/* Return true if we still have work. */
1525	compl_desc = &tx->dqo.compl_ring[tx->dqo_compl.head];
1526	return compl_desc->generation != tx->dqo_compl.cur_gen_bit;
1527}
1528
1529int gve_xdp_xmit_one_dqo(struct gve_priv *priv, struct gve_tx_ring *tx,
1530			 struct xdp_frame *xdpf)
1531{
1532	struct gve_tx_pending_packet_dqo *pkt;
1533	u32 desc_idx = tx->dqo_tx.tail;
1534	s16 completion_tag;
1535	int num_descs = 1;
1536	dma_addr_t addr;
1537	int err;
1538
1539	if (unlikely(!gve_has_tx_slots_available(tx, num_descs)))
1540		return -EBUSY;
1541
1542	pkt = gve_alloc_pending_packet(tx);
1543	if (unlikely(!pkt))
1544		return -EBUSY;
1545
1546	pkt->type = GVE_TX_PENDING_PACKET_DQO_XDP_FRAME;
1547	pkt->num_bufs = 0;
1548	pkt->xdpf = xdpf;
1549	completion_tag = pkt - tx->dqo.pending_packets;
1550
1551	/* Generate Packet Descriptor */
1552	addr = dma_map_single(tx->dev, xdpf->data, xdpf->len, DMA_TO_DEVICE);
1553	err = dma_mapping_error(tx->dev, addr);
1554	if (unlikely(err))
1555		goto err;
1556
1557	dma_unmap_len_set(pkt, len[pkt->num_bufs], xdpf->len);
1558	dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr);
1559	pkt->num_bufs++;
1560
1561	gve_tx_fill_pkt_desc_dqo(tx, &desc_idx,
1562				 false, xdpf->len,
1563				 addr, completion_tag, true,
1564				 false);
1565
1566	gve_tx_update_tail(tx, desc_idx);
1567	return 0;
1568
1569err:
1570	pkt->xdpf = NULL;
1571	pkt->num_bufs = 0;
1572	gve_free_pending_packet(tx, pkt);
1573	return err;
1574}
1575
1576int gve_xdp_xmit_dqo(struct net_device *dev, int n, struct xdp_frame **frames,
1577		     u32 flags)
1578{
1579	struct gve_priv *priv = netdev_priv(dev);
1580	struct gve_tx_ring *tx;
1581	int i, err = 0, qid;
1582
1583	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1584		return -EINVAL;
1585
1586	qid = gve_xdp_tx_queue_id(priv,
1587				  smp_processor_id() % priv->tx_cfg.num_xdp_queues);
1588
1589	tx = &priv->tx[qid];
1590
1591	spin_lock(&tx->dqo_tx.xdp_lock);
1592	for (i = 0; i < n; i++) {
1593		err = gve_xdp_xmit_one_dqo(priv, tx, frames[i]);
1594		if (err)
1595			break;
1596	}
1597
1598	if (flags & XDP_XMIT_FLUSH)
1599		gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail);
1600
1601	spin_unlock(&tx->dqo_tx.xdp_lock);
1602
1603	u64_stats_update_begin(&tx->statss);
1604	tx->xdp_xmit += n;
1605	tx->xdp_xmit_errors += n - i;
1606	u64_stats_update_end(&tx->statss);
1607
1608	return i ? i : err;
1609}