xsk.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (C) 2025 Intel Corporation */
  3
  4#define DEFAULT_SYMBOL_NAMESPACE	"LIBETH_XDP"
  5
  6#include <linux/export.h>
  7
  8#include <net/libeth/xsk.h>
  9
 10#include "priv.h"
 11
 12/* ``XDP_TX`` bulking */
 13
 14void __cold libeth_xsk_tx_return_bulk(const struct libeth_xdp_tx_frame *bq,
 15				      u32 count)
 16{
 17	for (u32 i = 0; i < count; i++)
 18		libeth_xsk_buff_free_slow(bq[i].xsk);
 19}
 20
 21/* XSk TMO */
 22
 23const struct xsk_tx_metadata_ops libeth_xsktmo_slow = {
 24	.tmo_request_checksum		= libeth_xsktmo_req_csum,
 25};
 26
 27/* Rx polling path */
 28
 29/**
 30 * libeth_xsk_buff_free_slow - free an XSk Rx buffer
 31 * @xdp: buffer to free
 32 *
 33 * Slowpath version of xsk_buff_free() to be used on exceptions, cleanups etc.
 34 * to avoid unwanted inlining.
 35 */
 36void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp)
 37{
 38	xsk_buff_free(&xdp->base);
 39}
 40EXPORT_SYMBOL_GPL(libeth_xsk_buff_free_slow);
 41
 42/**
 43 * libeth_xsk_buff_add_frag - add frag to XSk Rx buffer
 44 * @head: head buffer
 45 * @xdp: frag buffer
 46 *
 47 * External helper used by libeth_xsk_process_buff(), do not call directly.
 48 * Frees both main and frag buffers on error.
 49 *
 50 * Return: main buffer with attached frag on success, %NULL on error (no space
 51 * for a new frag).
 52 */
 53struct libeth_xdp_buff *libeth_xsk_buff_add_frag(struct libeth_xdp_buff *head,
 54						 struct libeth_xdp_buff *xdp)
 55{
 56	if (!xsk_buff_add_frag(&head->base, &xdp->base))
 57		goto free;
 58
 59	return head;
 60
 61free:
 62	libeth_xsk_buff_free_slow(xdp);
 63	libeth_xsk_buff_free_slow(head);
 64
 65	return NULL;
 66}
 67EXPORT_SYMBOL_GPL(libeth_xsk_buff_add_frag);
 68
 69/**
 70 * libeth_xsk_buff_stats_frags - update onstack RQ stats with XSk frags info
 71 * @rs: onstack stats to update
 72 * @xdp: buffer to account
 73 *
 74 * External helper used by __libeth_xsk_run_pass(), do not call directly.
 75 * Adds buffer's frags count and total len to the onstack stats.
 76 */
 77void libeth_xsk_buff_stats_frags(struct libeth_rq_napi_stats *rs,
 78				 const struct libeth_xdp_buff *xdp)
 79{
 80	libeth_xdp_buff_stats_frags(rs, xdp);
 81}
 82EXPORT_SYMBOL_GPL(libeth_xsk_buff_stats_frags);
 83
 84/**
 85 * __libeth_xsk_run_prog_slow - process the non-``XDP_REDIRECT`` verdicts
 86 * @xdp: buffer to process
 87 * @bq: Tx bulk for queueing on ``XDP_TX``
 88 * @act: verdict to process
 89 * @ret: error code if ``XDP_REDIRECT`` failed
 90 *
 91 * External helper used by __libeth_xsk_run_prog(), do not call directly.
 92 * ``XDP_REDIRECT`` is the most common and hottest verdict on XSk, thus
 93 * it is processed inline. The rest goes here for out-of-line processing,
 94 * together with redirect errors.
 95 *
 96 * Return: libeth_xdp XDP prog verdict.
 97 */
 98u32 __libeth_xsk_run_prog_slow(struct libeth_xdp_buff *xdp,
 99			       const struct libeth_xdp_tx_bulk *bq,
100			       enum xdp_action act, int ret)
101{
102	switch (act) {
103	case XDP_DROP:
104		xsk_buff_free(&xdp->base);
105
106		return LIBETH_XDP_DROP;
107	case XDP_TX:
108		return LIBETH_XDP_TX;
109	case XDP_PASS:
110		return LIBETH_XDP_PASS;
111	default:
112		break;
113	}
114
115	return libeth_xdp_prog_exception(bq, xdp, act, ret);
116}
117EXPORT_SYMBOL_GPL(__libeth_xsk_run_prog_slow);
118
119/**
120 * libeth_xsk_prog_exception - handle XDP prog exceptions on XSk
121 * @xdp: buffer to process
122 * @act: verdict returned by the prog
123 * @ret: error code if ``XDP_REDIRECT`` failed
124 *
125 * Internal. Frees the buffer and, if the queue uses XSk wakeups, stop the
126 * current NAPI poll when there are no free buffers left.
127 *
128 * Return: libeth_xdp's XDP prog verdict.
129 */
130u32 __cold libeth_xsk_prog_exception(struct libeth_xdp_buff *xdp,
131				     enum xdp_action act, int ret)
132{
133	const struct xdp_buff_xsk *xsk;
134	u32 __ret = LIBETH_XDP_DROP;
135
136	if (act != XDP_REDIRECT)
137		goto drop;
138
139	xsk = container_of(&xdp->base, typeof(*xsk), xdp);
140	if (xsk_uses_need_wakeup(xsk->pool) && ret == -ENOBUFS)
141		__ret = LIBETH_XDP_ABORTED;
142
143drop:
144	libeth_xsk_buff_free_slow(xdp);
145
146	return __ret;
147}
148
149/* Refill */
150
151/**
152 * libeth_xskfq_create - create an XSkFQ
153 * @fq: fill queue to initialize
154 *
155 * Allocates the FQEs and initializes the fields used by libeth_xdp: number
156 * of buffers to refill, refill threshold and buffer len.
157 *
158 * Return: %0 on success, -errno otherwise.
159 */
160int libeth_xskfq_create(struct libeth_xskfq *fq)
161{
162	fq->fqes = kvcalloc_node(fq->count, sizeof(*fq->fqes), GFP_KERNEL,
163				 fq->nid);
164	if (!fq->fqes)
165		return -ENOMEM;
166
167	fq->pending = fq->count;
168	fq->thresh = libeth_xdp_queue_threshold(fq->count);
169	fq->buf_len = xsk_pool_get_rx_frame_size(fq->pool);
170
171	return 0;
172}
173EXPORT_SYMBOL_GPL(libeth_xskfq_create);
174
175/**
176 * libeth_xskfq_destroy - destroy an XSkFQ
177 * @fq: fill queue to destroy
178 *
179 * Zeroes the used fields and frees the FQEs array.
180 */
181void libeth_xskfq_destroy(struct libeth_xskfq *fq)
182{
183	fq->buf_len = 0;
184	fq->thresh = 0;
185	fq->pending = 0;
186
187	kvfree(fq->fqes);
188}
189EXPORT_SYMBOL_GPL(libeth_xskfq_destroy);
190
191/* .ndo_xsk_wakeup */
192
193static void libeth_xsk_napi_sched(void *info)
194{
195	__napi_schedule_irqoff(info);
196}
197
198/**
199 * libeth_xsk_init_wakeup - initialize libeth XSk wakeup structure
200 * @csd: struct to initialize
201 * @napi: NAPI corresponding to this queue
202 *
203 * libeth_xdp uses inter-processor interrupts to perform XSk wakeups. In order
204 * to do that, the corresponding CSDs must be initialized when creating the
205 * queues.
206 */
207void libeth_xsk_init_wakeup(call_single_data_t *csd, struct napi_struct *napi)
208{
209	INIT_CSD(csd, libeth_xsk_napi_sched, napi);
210}
211EXPORT_SYMBOL_GPL(libeth_xsk_init_wakeup);
212
213/**
214 * libeth_xsk_wakeup - perform an XSk wakeup
215 * @csd: CSD corresponding to the queue
216 * @qid: the stack queue index
217 *
218 * Try to mark the NAPI as missed first, so that it could be rescheduled.
219 * If it's not, schedule it on the corresponding CPU using IPIs (or directly
220 * if already running on it).
221 */
222void libeth_xsk_wakeup(call_single_data_t *csd, u32 qid)
223{
224	struct napi_struct *napi = csd->info;
225
226	if (napi_if_scheduled_mark_missed(napi) ||
227	    unlikely(!napi_schedule_prep(napi)))
228		return;
229
230	if (unlikely(qid >= nr_cpu_ids))
231		qid %= nr_cpu_ids;
232
233	if (qid != raw_smp_processor_id() && cpu_online(qid))
234		smp_call_function_single_async(qid, csd);
235	else
236		__napi_schedule(napi);
237}
238EXPORT_SYMBOL_GPL(libeth_xsk_wakeup);
239
240/* Pool setup */
241
242#define LIBETH_XSK_DMA_ATTR					\
243	(DMA_ATTR_WEAK_ORDERING | DMA_ATTR_SKIP_CPU_SYNC)
244
245/**
246 * libeth_xsk_setup_pool - setup or destroy an XSk pool for a queue
247 * @dev: target &net_device
248 * @qid: stack queue index to configure
249 * @enable: whether to enable or disable the pool
250 *
251 * Check that @qid is valid and then map or unmap the pool.
252 *
253 * Return: %0 on success, -errno otherwise.
254 */
255int libeth_xsk_setup_pool(struct net_device *dev, u32 qid, bool enable)
256{
257	struct xsk_buff_pool *pool;
258
259	pool = xsk_get_pool_from_qid(dev, qid);
260	if (!pool)
261		return -EINVAL;
262
263	if (enable)
264		return xsk_pool_dma_map(pool, dev->dev.parent,
265					LIBETH_XSK_DMA_ATTR);
266	else
267		xsk_pool_dma_unmap(pool, LIBETH_XSK_DMA_ATTR);
268
269	return 0;
270}
271EXPORT_SYMBOL_GPL(libeth_xsk_setup_pool);