xsk.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (C) 2025 Intel Corporation */
  3
  4#include <net/libeth/xsk.h>
  5
  6#include "idpf.h"
  7#include "xdp.h"
  8#include "xsk.h"
  9
 10static void idpf_xsk_tx_timer(struct work_struct *work);
 11
 12static void idpf_xsk_setup_rxq(const struct idpf_vport *vport,
 13			       struct idpf_rx_queue *rxq)
 14{
 15	struct xsk_buff_pool *pool;
 16
 17	pool = xsk_get_pool_from_qid(vport->netdev, rxq->idx);
 18	if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1))
 19		return;
 20
 21	rxq->pool = pool;
 22
 23	idpf_queue_set(XSK, rxq);
 24}
 25
 26static void idpf_xsk_setup_bufq(const struct idpf_vport *vport,
 27				struct idpf_buf_queue *bufq)
 28{
 29	struct xsk_buff_pool *pool;
 30	u32 qid = U32_MAX;
 31
 32	for (u32 i = 0; i < vport->num_rxq_grp; i++) {
 33		const struct idpf_rxq_group *grp = &vport->rxq_grps[i];
 34
 35		for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) {
 36			if (&grp->splitq.bufq_sets[j].bufq == bufq) {
 37				qid = grp->splitq.rxq_sets[0]->rxq.idx;
 38				goto setup;
 39			}
 40		}
 41	}
 42
 43setup:
 44	pool = xsk_get_pool_from_qid(vport->netdev, qid);
 45	if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1))
 46		return;
 47
 48	bufq->pool = pool;
 49
 50	idpf_queue_set(XSK, bufq);
 51}
 52
 53static void idpf_xsk_setup_txq(const struct idpf_vport *vport,
 54			       struct idpf_tx_queue *txq)
 55{
 56	struct xsk_buff_pool *pool;
 57	u32 qid;
 58
 59	idpf_queue_clear(XSK, txq);
 60
 61	if (!idpf_queue_has(XDP, txq))
 62		return;
 63
 64	qid = txq->idx - vport->xdp_txq_offset;
 65
 66	pool = xsk_get_pool_from_qid(vport->netdev, qid);
 67	if (!pool || !pool->dev)
 68		return;
 69
 70	txq->pool = pool;
 71	libeth_xdpsq_init_timer(txq->timer, txq, &txq->xdp_lock,
 72				idpf_xsk_tx_timer);
 73
 74	idpf_queue_assign(NOIRQ, txq, xsk_uses_need_wakeup(pool));
 75	idpf_queue_set(XSK, txq);
 76}
 77
 78static void idpf_xsk_setup_complq(const struct idpf_vport *vport,
 79				  struct idpf_compl_queue *complq)
 80{
 81	const struct xsk_buff_pool *pool;
 82	u32 qid;
 83
 84	idpf_queue_clear(XSK, complq);
 85
 86	if (!idpf_queue_has(XDP, complq))
 87		return;
 88
 89	qid = complq->txq_grp->txqs[0]->idx - vport->xdp_txq_offset;
 90
 91	pool = xsk_get_pool_from_qid(vport->netdev, qid);
 92	if (!pool || !pool->dev)
 93		return;
 94
 95	idpf_queue_set(XSK, complq);
 96}
 97
 98void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q,
 99			  enum virtchnl2_queue_type type)
100{
101	if (!idpf_xdp_enabled(vport))
102		return;
103
104	switch (type) {
105	case VIRTCHNL2_QUEUE_TYPE_RX:
106		idpf_xsk_setup_rxq(vport, q);
107		break;
108	case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER:
109		idpf_xsk_setup_bufq(vport, q);
110		break;
111	case VIRTCHNL2_QUEUE_TYPE_TX:
112		idpf_xsk_setup_txq(vport, q);
113		break;
114	case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION:
115		idpf_xsk_setup_complq(vport, q);
116		break;
117	default:
118		break;
119	}
120}
121
122void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type)
123{
124	struct idpf_compl_queue *complq;
125	struct idpf_buf_queue *bufq;
126	struct idpf_rx_queue *rxq;
127	struct idpf_tx_queue *txq;
128
129	switch (type) {
130	case VIRTCHNL2_QUEUE_TYPE_RX:
131		rxq = q;
132		if (!idpf_queue_has_clear(XSK, rxq))
133			return;
134
135		rxq->pool = NULL;
136		break;
137	case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER:
138		bufq = q;
139		if (!idpf_queue_has_clear(XSK, bufq))
140			return;
141
142		bufq->pool = NULL;
143		break;
144	case VIRTCHNL2_QUEUE_TYPE_TX:
145		txq = q;
146		if (!idpf_queue_has_clear(XSK, txq))
147			return;
148
149		idpf_queue_set(NOIRQ, txq);
150		txq->dev = txq->netdev->dev.parent;
151		break;
152	case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION:
153		complq = q;
154		idpf_queue_clear(XSK, complq);
155		break;
156	default:
157		break;
158	}
159}
160
161void idpf_xsk_init_wakeup(struct idpf_q_vector *qv)
162{
163	libeth_xsk_init_wakeup(&qv->csd, &qv->napi);
164}
165
166void idpf_xsksq_clean(struct idpf_tx_queue *xdpsq)
167{
168	struct libeth_xdpsq_napi_stats ss = { };
169	u32 ntc = xdpsq->next_to_clean;
170	struct xdp_frame_bulk bq;
171	struct libeth_cq_pp cp = {
172		.dev	= xdpsq->pool->dev,
173		.bq	= &bq,
174		.xss	= &ss,
175	};
176	u32 xsk_frames = 0;
177
178	xdp_frame_bulk_init(&bq);
179
180	while (ntc != xdpsq->next_to_use) {
181		struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc];
182
183		if (sqe->type)
184			libeth_xdp_complete_tx(sqe, &cp);
185		else
186			xsk_frames++;
187
188		if (unlikely(++ntc == xdpsq->desc_count))
189			ntc = 0;
190	}
191
192	xdp_flush_frame_bulk(&bq);
193
194	if (xsk_frames)
195		xsk_tx_completed(xdpsq->pool, xsk_frames);
196}
197
198static noinline u32 idpf_xsksq_complete_slow(struct idpf_tx_queue *xdpsq,
199					     u32 done)
200{
201	struct libeth_xdpsq_napi_stats ss = { };
202	u32 ntc = xdpsq->next_to_clean;
203	u32 cnt = xdpsq->desc_count;
204	struct xdp_frame_bulk bq;
205	struct libeth_cq_pp cp = {
206		.dev	= xdpsq->pool->dev,
207		.bq	= &bq,
208		.xss	= &ss,
209		.napi	= true,
210	};
211	u32 xsk_frames = 0;
212
213	xdp_frame_bulk_init(&bq);
214
215	for (u32 i = 0; likely(i < done); i++) {
216		struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc];
217
218		if (sqe->type)
219			libeth_xdp_complete_tx(sqe, &cp);
220		else
221			xsk_frames++;
222
223		if (unlikely(++ntc == cnt))
224			ntc = 0;
225	}
226
227	xdp_flush_frame_bulk(&bq);
228
229	xdpsq->next_to_clean = ntc;
230	xdpsq->xdp_tx -= cp.xdp_tx;
231
232	return xsk_frames;
233}
234
235static __always_inline u32 idpf_xsksq_complete(void *_xdpsq, u32 budget)
236{
237	struct idpf_tx_queue *xdpsq = _xdpsq;
238	u32 tx_ntc = xdpsq->next_to_clean;
239	u32 tx_cnt = xdpsq->desc_count;
240	u32 done_frames;
241	u32 xsk_frames;
242
243	done_frames = idpf_xdpsq_poll(xdpsq, budget);
244	if (unlikely(!done_frames))
245		return 0;
246
247	if (likely(!xdpsq->xdp_tx)) {
248		tx_ntc += done_frames;
249		if (tx_ntc >= tx_cnt)
250			tx_ntc -= tx_cnt;
251
252		xdpsq->next_to_clean = tx_ntc;
253		xsk_frames = done_frames;
254
255		goto finalize;
256	}
257
258	xsk_frames = idpf_xsksq_complete_slow(xdpsq, done_frames);
259	if (xsk_frames)
260finalize:
261		xsk_tx_completed(xdpsq->pool, xsk_frames);
262
263	xdpsq->pending -= done_frames;
264
265	return done_frames;
266}
267
268static u32 idpf_xsk_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq)
269{
270	struct idpf_tx_queue *xdpsq = _xdpsq;
271	u32 free;
272
273	libeth_xdpsq_lock(&xdpsq->xdp_lock);
274
275	free = xdpsq->desc_count - xdpsq->pending;
276	if (free < xdpsq->thresh)
277		free += idpf_xsksq_complete(xdpsq, xdpsq->thresh);
278
279	*sq = (struct libeth_xdpsq){
280		.pool		= xdpsq->pool,
281		.sqes		= xdpsq->tx_buf,
282		.descs		= xdpsq->desc_ring,
283		.count		= xdpsq->desc_count,
284		.lock		= &xdpsq->xdp_lock,
285		.ntu		= &xdpsq->next_to_use,
286		.pending	= &xdpsq->pending,
287		.xdp_tx		= &xdpsq->xdp_tx,
288	};
289
290	return free;
291}
292
293static u32 idpf_xsk_xmit_prep(void *_xdpsq, struct libeth_xdpsq *sq)
294{
295	struct idpf_tx_queue *xdpsq = _xdpsq;
296
297	*sq = (struct libeth_xdpsq){
298		.pool		= xdpsq->pool,
299		.sqes		= xdpsq->tx_buf,
300		.descs		= xdpsq->desc_ring,
301		.count		= xdpsq->desc_count,
302		.lock		= &xdpsq->xdp_lock,
303		.ntu		= &xdpsq->next_to_use,
304		.pending	= &xdpsq->pending,
305	};
306
307	/*
308	 * The queue is cleaned, the budget is already known, optimize out
309	 * the second min() by passing the type limit.
310	 */
311	return U32_MAX;
312}
313
314bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq)
315{
316	u32 free;
317
318	libeth_xdpsq_lock(&xsksq->xdp_lock);
319
320	free = xsksq->desc_count - xsksq->pending;
321	if (free < xsksq->thresh)
322		free += idpf_xsksq_complete(xsksq, xsksq->thresh);
323
324	return libeth_xsk_xmit_do_bulk(xsksq->pool, xsksq,
325				       min(free - 1, xsksq->thresh),
326				       libeth_xsktmo, idpf_xsk_xmit_prep,
327				       idpf_xdp_tx_xmit, idpf_xdp_tx_finalize);
328}
329
330LIBETH_XDP_DEFINE_START();
331LIBETH_XDP_DEFINE_TIMER(static idpf_xsk_tx_timer, idpf_xsksq_complete);
332LIBETH_XSK_DEFINE_FLUSH_TX(static idpf_xsk_tx_flush_bulk, idpf_xsk_tx_prep,
333			   idpf_xdp_tx_xmit);
334LIBETH_XSK_DEFINE_RUN(static idpf_xsk_run_pass, idpf_xsk_run_prog,
335		      idpf_xsk_tx_flush_bulk, idpf_rx_process_skb_fields);
336LIBETH_XSK_DEFINE_FINALIZE(static idpf_xsk_finalize_rx, idpf_xsk_tx_flush_bulk,
337			   idpf_xdp_tx_finalize);
338LIBETH_XDP_DEFINE_END();
339
340static void idpf_xskfqe_init(const struct libeth_xskfq_fp *fq, u32 i)
341{
342	struct virtchnl2_splitq_rx_buf_desc *desc = fq->descs;
343
344	desc = &desc[i];
345#ifdef __LIBETH_WORD_ACCESS
346	*(u64 *)&desc->qword0 = i;
347#else
348	desc->qword0.buf_id = cpu_to_le16(i);
349#endif
350	desc->pkt_addr = cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i]));
351}
352
353static bool idpf_xskfq_refill_thresh(struct idpf_buf_queue *bufq, u32 count)
354{
355	struct libeth_xskfq_fp fq = {
356		.pool	= bufq->pool,
357		.fqes	= bufq->xsk_buf,
358		.descs	= bufq->split_buf,
359		.ntu	= bufq->next_to_use,
360		.count	= bufq->desc_count,
361	};
362	u32 done;
363
364	done = libeth_xskfqe_alloc(&fq, count, idpf_xskfqe_init);
365	writel(fq.ntu, bufq->tail);
366
367	bufq->next_to_use = fq.ntu;
368	bufq->pending -= done;
369
370	return done == count;
371}
372
373static bool idpf_xskfq_refill(struct idpf_buf_queue *bufq)
374{
375	u32 count, rx_thresh = bufq->thresh;
376
377	count = ALIGN_DOWN(bufq->pending - 1, rx_thresh);
378
379	for (u32 i = 0; i < count; i += rx_thresh) {
380		if (unlikely(!idpf_xskfq_refill_thresh(bufq, rx_thresh)))
381			return false;
382	}
383
384	return true;
385}
386
387int idpf_xskfq_init(struct idpf_buf_queue *bufq)
388{
389	struct libeth_xskfq fq = {
390		.pool	= bufq->pool,
391		.count	= bufq->desc_count,
392		.nid	= idpf_q_vector_to_mem(bufq->q_vector),
393	};
394	int ret;
395
396	ret = libeth_xskfq_create(&fq);
397	if (ret)
398		return ret;
399
400	bufq->xsk_buf = fq.fqes;
401	bufq->pending = fq.pending;
402	bufq->thresh = fq.thresh;
403	bufq->rx_buf_size = fq.buf_len;
404
405	if (!idpf_xskfq_refill(bufq))
406		netdev_err(bufq->pool->netdev,
407			   "failed to allocate XSk buffers for qid %d\n",
408			   bufq->pool->queue_id);
409
410	bufq->next_to_alloc = bufq->next_to_use;
411
412	idpf_queue_clear(HSPLIT_EN, bufq);
413	bufq->rx_hbuf_size = 0;
414
415	return 0;
416}
417
418void idpf_xskfq_rel(struct idpf_buf_queue *bufq)
419{
420	struct libeth_xskfq fq = {
421		.fqes	= bufq->xsk_buf,
422	};
423
424	libeth_xskfq_destroy(&fq);
425
426	bufq->rx_buf_size = fq.buf_len;
427	bufq->thresh = fq.thresh;
428	bufq->pending = fq.pending;
429}
430
431struct idpf_xskfq_refill_set {
432	struct {
433		struct idpf_buf_queue	*q;
434		u32			buf_id;
435		u32			pending;
436	} bufqs[IDPF_MAX_BUFQS_PER_RXQ_GRP];
437};
438
439static bool idpf_xskfq_refill_set(const struct idpf_xskfq_refill_set *set)
440{
441	bool ret = true;
442
443	for (u32 i = 0; i < ARRAY_SIZE(set->bufqs); i++) {
444		struct idpf_buf_queue *bufq = set->bufqs[i].q;
445		u32 ntc;
446
447		if (!bufq)
448			continue;
449
450		ntc = set->bufqs[i].buf_id;
451		if (unlikely(++ntc == bufq->desc_count))
452			ntc = 0;
453
454		bufq->next_to_clean = ntc;
455		bufq->pending += set->bufqs[i].pending;
456
457		if (bufq->pending > bufq->thresh)
458			ret &= idpf_xskfq_refill(bufq);
459	}
460
461	return ret;
462}
463
464int idpf_xskrq_poll(struct idpf_rx_queue *rxq, u32 budget)
465{
466	struct idpf_xskfq_refill_set set = { };
467	struct libeth_rq_napi_stats rs = { };
468	bool wake, gen, fail = false;
469	u32 ntc = rxq->next_to_clean;
470	struct libeth_xdp_buff *xdp;
471	LIBETH_XDP_ONSTACK_BULK(bq);
472	u32 cnt = rxq->desc_count;
473
474	wake = xsk_uses_need_wakeup(rxq->pool);
475	if (wake)
476		xsk_clear_rx_need_wakeup(rxq->pool);
477
478	gen = idpf_queue_has(GEN_CHK, rxq);
479
480	libeth_xsk_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev,
481				rxq->xdpsqs, rxq->num_xdp_txq);
482	xdp = rxq->xsk;
483
484	while (likely(rs.packets < budget)) {
485		const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc;
486		struct idpf_xdp_rx_desc desc __uninitialized;
487		struct idpf_buf_queue *bufq;
488		u32 bufq_id, buf_id;
489
490		rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb;
491
492		idpf_xdp_get_qw0(&desc, rx_desc);
493		if (idpf_xdp_rx_gen(&desc) != gen)
494			break;
495
496		dma_rmb();
497
498		bufq_id = idpf_xdp_rx_bufq(&desc);
499		bufq = set.bufqs[bufq_id].q;
500		if (!bufq) {
501			bufq = &rxq->bufq_sets[bufq_id].bufq;
502			set.bufqs[bufq_id].q = bufq;
503		}
504
505		idpf_xdp_get_qw1(&desc, rx_desc);
506		buf_id = idpf_xdp_rx_buf(&desc);
507
508		set.bufqs[bufq_id].buf_id = buf_id;
509		set.bufqs[bufq_id].pending++;
510
511		xdp = libeth_xsk_process_buff(xdp, bufq->xsk_buf[buf_id],
512					      idpf_xdp_rx_len(&desc));
513
514		if (unlikely(++ntc == cnt)) {
515			ntc = 0;
516			gen = !gen;
517			idpf_queue_change(GEN_CHK, rxq);
518		}
519
520		if (!idpf_xdp_rx_eop(&desc) || unlikely(!xdp))
521			continue;
522
523		fail = !idpf_xsk_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc);
524		xdp = NULL;
525
526		if (fail)
527			break;
528	}
529
530	idpf_xsk_finalize_rx(&bq);
531
532	rxq->next_to_clean = ntc;
533	rxq->xsk = xdp;
534
535	fail |= !idpf_xskfq_refill_set(&set);
536
537	u64_stats_update_begin(&rxq->stats_sync);
538	u64_stats_add(&rxq->q_stats.packets, rs.packets);
539	u64_stats_add(&rxq->q_stats.bytes, rs.bytes);
540	u64_stats_update_end(&rxq->stats_sync);
541
542	if (!wake)
543		return unlikely(fail) ? budget : rs.packets;
544
545	if (unlikely(fail))
546		xsk_set_rx_need_wakeup(rxq->pool);
547
548	return rs.packets;
549}
550
551int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *bpf)
552{
553	struct xsk_buff_pool *pool = bpf->xsk.pool;
554	u32 qid = bpf->xsk.queue_id;
555	bool restart;
556	int ret;
557
558	if (pool && !IS_ALIGNED(xsk_pool_get_rx_frame_size(pool),
559				LIBETH_RX_BUF_STRIDE)) {
560		NL_SET_ERR_MSG_FMT_MOD(bpf->extack,
561				       "%s: HW doesn't support frames sizes not aligned to %u (qid %u: %u)",
562				       netdev_name(vport->netdev),
563				       LIBETH_RX_BUF_STRIDE, qid,
564				       xsk_pool_get_rx_frame_size(pool));
565		return -EINVAL;
566	}
567
568	restart = idpf_xdp_enabled(vport) && netif_running(vport->netdev);
569	if (!restart)
570		goto pool;
571
572	ret = idpf_qp_switch(vport, qid, false);
573	if (ret) {
574		NL_SET_ERR_MSG_FMT_MOD(bpf->extack,
575				       "%s: failed to disable queue pair %u: %pe",
576				       netdev_name(vport->netdev), qid,
577				       ERR_PTR(ret));
578		return ret;
579	}
580
581pool:
582	ret = libeth_xsk_setup_pool(vport->netdev, qid, pool);
583	if (ret) {
584		NL_SET_ERR_MSG_FMT_MOD(bpf->extack,
585				       "%s: failed to configure XSk pool for pair %u: %pe",
586				       netdev_name(vport->netdev), qid,
587				       ERR_PTR(ret));
588		return ret;
589	}
590
591	if (!restart)
592		return 0;
593
594	ret = idpf_qp_switch(vport, qid, true);
595	if (ret) {
596		NL_SET_ERR_MSG_FMT_MOD(bpf->extack,
597				       "%s: failed to enable queue pair %u: %pe",
598				       netdev_name(vport->netdev), qid,
599				       ERR_PTR(ret));
600		goto err_dis;
601	}
602
603	return 0;
604
605err_dis:
606	libeth_xsk_setup_pool(vport->netdev, qid, false);
607
608	return ret;
609}
610
611int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
612{
613	const struct idpf_netdev_priv *np = netdev_priv(dev);
614	const struct idpf_vport *vport = np->vport;
615	struct idpf_q_vector *q_vector;
616
617	if (unlikely(idpf_vport_ctrl_is_locked(dev)))
618		return -EBUSY;
619
620	if (unlikely(!vport->link_up))
621		return -ENETDOWN;
622
623	if (unlikely(!vport->num_xdp_txq))
624		return -ENXIO;
625
626	q_vector = idpf_find_rxq_vec(vport, qid);
627	if (unlikely(!q_vector->xsksq))
628		return -ENXIO;
629
630	libeth_xsk_wakeup(&q_vector->csd, qid);
631
632	return 0;
633}