xdp.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (C) 2025 Intel Corporation */
  3
  4#include "idpf.h"
  5#include "idpf_virtchnl.h"
  6#include "xdp.h"
  7#include "xsk.h"
  8
  9static int idpf_rxq_for_each(const struct idpf_vport *vport,
 10			     int (*fn)(struct idpf_rx_queue *rxq, void *arg),
 11			     void *arg)
 12{
 13	bool splitq = idpf_is_queue_model_split(vport->rxq_model);
 14
 15	if (!vport->rxq_grps)
 16		return -ENETDOWN;
 17
 18	for (u32 i = 0; i < vport->num_rxq_grp; i++) {
 19		const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
 20		u32 num_rxq;
 21
 22		if (splitq)
 23			num_rxq = rx_qgrp->splitq.num_rxq_sets;
 24		else
 25			num_rxq = rx_qgrp->singleq.num_rxq;
 26
 27		for (u32 j = 0; j < num_rxq; j++) {
 28			struct idpf_rx_queue *q;
 29			int err;
 30
 31			if (splitq)
 32				q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
 33			else
 34				q = rx_qgrp->singleq.rxqs[j];
 35
 36			err = fn(q, arg);
 37			if (err)
 38				return err;
 39		}
 40	}
 41
 42	return 0;
 43}
 44
 45static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg)
 46{
 47	const struct idpf_vport *vport = rxq->q_vector->vport;
 48	bool split = idpf_is_queue_model_split(vport->rxq_model);
 49	int err;
 50
 51	err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx,
 52				 rxq->q_vector->napi.napi_id,
 53				 rxq->rx_buf_size);
 54	if (err)
 55		return err;
 56
 57	if (idpf_queue_has(XSK, rxq)) {
 58		err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq,
 59						 MEM_TYPE_XSK_BUFF_POOL,
 60						 rxq->pool);
 61		if (err)
 62			goto unreg;
 63	} else {
 64		const struct page_pool *pp;
 65
 66		pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp;
 67		xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp);
 68	}
 69
 70	if (!split)
 71		return 0;
 72
 73	rxq->xdpsqs = &vport->txqs[vport->xdp_txq_offset];
 74	rxq->num_xdp_txq = vport->num_xdp_txq;
 75
 76	return 0;
 77
 78unreg:
 79	xdp_rxq_info_unreg(&rxq->xdp_rxq);
 80
 81	return err;
 82}
 83
 84int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq)
 85{
 86	return __idpf_xdp_rxq_info_init(rxq, NULL);
 87}
 88
 89int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport)
 90{
 91	return idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_init, NULL);
 92}
 93
 94static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg)
 95{
 96	if (idpf_is_queue_model_split((size_t)arg)) {
 97		rxq->xdpsqs = NULL;
 98		rxq->num_xdp_txq = 0;
 99	}
100
101	if (!idpf_queue_has(XSK, rxq))
102		xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq);
103
104	xdp_rxq_info_unreg(&rxq->xdp_rxq);
105
106	return 0;
107}
108
109void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model)
110{
111	__idpf_xdp_rxq_info_deinit(rxq, (void *)(size_t)model);
112}
113
114void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport)
115{
116	idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_deinit,
117			  (void *)(size_t)vport->rxq_model);
118}
119
120static int idpf_xdp_rxq_assign_prog(struct idpf_rx_queue *rxq, void *arg)
121{
122	struct bpf_prog *prog = arg;
123	struct bpf_prog *old;
124
125	if (prog)
126		bpf_prog_inc(prog);
127
128	old = rcu_replace_pointer(rxq->xdp_prog, prog, lockdep_rtnl_is_held());
129	if (old)
130		bpf_prog_put(old);
131
132	return 0;
133}
134
135void idpf_xdp_copy_prog_to_rqs(const struct idpf_vport *vport,
136			       struct bpf_prog *xdp_prog)
137{
138	idpf_rxq_for_each(vport, idpf_xdp_rxq_assign_prog, xdp_prog);
139}
140
141static void idpf_xdp_tx_timer(struct work_struct *work);
142
143int idpf_xdpsqs_get(const struct idpf_vport *vport)
144{
145	struct libeth_xdpsq_timer **timers __free(kvfree) = NULL;
146	struct net_device *dev;
147	u32 sqs;
148
149	if (!idpf_xdp_enabled(vport))
150		return 0;
151
152	timers = kvcalloc(vport->num_xdp_txq, sizeof(*timers), GFP_KERNEL);
153	if (!timers)
154		return -ENOMEM;
155
156	for (u32 i = 0; i < vport->num_xdp_txq; i++) {
157		timers[i] = kzalloc_node(sizeof(*timers[i]), GFP_KERNEL,
158					 cpu_to_mem(i));
159		if (!timers[i]) {
160			for (int j = i - 1; j >= 0; j--)
161				kfree(timers[j]);
162
163			return -ENOMEM;
164		}
165	}
166
167	dev = vport->netdev;
168	sqs = vport->xdp_txq_offset;
169
170	for (u32 i = sqs; i < vport->num_txq; i++) {
171		struct idpf_tx_queue *xdpsq = vport->txqs[i];
172
173		xdpsq->complq = xdpsq->txq_grp->complq;
174		kfree(xdpsq->refillq);
175		xdpsq->refillq = NULL;
176
177		idpf_queue_clear(FLOW_SCH_EN, xdpsq);
178		idpf_queue_clear(FLOW_SCH_EN, xdpsq->complq);
179		idpf_queue_set(NOIRQ, xdpsq);
180		idpf_queue_set(XDP, xdpsq);
181		idpf_queue_set(XDP, xdpsq->complq);
182
183		xdpsq->timer = timers[i - sqs];
184		libeth_xdpsq_get(&xdpsq->xdp_lock, dev, vport->xdpsq_share);
185		libeth_xdpsq_init_timer(xdpsq->timer, xdpsq, &xdpsq->xdp_lock,
186					idpf_xdp_tx_timer);
187
188		xdpsq->pending = 0;
189		xdpsq->xdp_tx = 0;
190		xdpsq->thresh = libeth_xdp_queue_threshold(xdpsq->desc_count);
191	}
192
193	return 0;
194}
195
196void idpf_xdpsqs_put(const struct idpf_vport *vport)
197{
198	struct net_device *dev;
199	u32 sqs;
200
201	if (!idpf_xdp_enabled(vport))
202		return;
203
204	dev = vport->netdev;
205	sqs = vport->xdp_txq_offset;
206
207	for (u32 i = sqs; i < vport->num_txq; i++) {
208		struct idpf_tx_queue *xdpsq = vport->txqs[i];
209
210		if (!idpf_queue_has_clear(XDP, xdpsq))
211			continue;
212
213		libeth_xdpsq_deinit_timer(xdpsq->timer);
214		libeth_xdpsq_put(&xdpsq->xdp_lock, dev);
215
216		kfree(xdpsq->timer);
217		xdpsq->refillq = NULL;
218		idpf_queue_clear(NOIRQ, xdpsq);
219	}
220}
221
222static int idpf_xdp_parse_cqe(const struct idpf_splitq_4b_tx_compl_desc *desc,
223			      bool gen)
224{
225	u32 val;
226
227#ifdef __LIBETH_WORD_ACCESS
228	val = *(const u32 *)desc;
229#else
230	val = ((u32)le16_to_cpu(desc->q_head_compl_tag.q_head) << 16) |
231	      le16_to_cpu(desc->qid_comptype_gen);
232#endif
233	if (!!(val & IDPF_TXD_COMPLQ_GEN_M) != gen)
234		return -ENODATA;
235
236	if (unlikely((val & GENMASK(IDPF_TXD_COMPLQ_GEN_S - 1, 0)) !=
237		     FIELD_PREP(IDPF_TXD_COMPLQ_COMPL_TYPE_M,
238				IDPF_TXD_COMPLT_RS)))
239		return -EINVAL;
240
241	return upper_16_bits(val);
242}
243
244u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget)
245{
246	struct idpf_compl_queue *cq = xdpsq->complq;
247	u32 tx_ntc = xdpsq->next_to_clean;
248	u32 tx_cnt = xdpsq->desc_count;
249	u32 ntc = cq->next_to_clean;
250	u32 cnt = cq->desc_count;
251	u32 done_frames;
252	bool gen;
253
254	gen = idpf_queue_has(GEN_CHK, cq);
255
256	for (done_frames = 0; done_frames < budget; ) {
257		int ret;
258
259		ret = idpf_xdp_parse_cqe(&cq->comp_4b[ntc], gen);
260		if (ret >= 0) {
261			done_frames = ret > tx_ntc ? ret - tx_ntc :
262						     ret + tx_cnt - tx_ntc;
263			goto next;
264		}
265
266		switch (ret) {
267		case -ENODATA:
268			goto out;
269		case -EINVAL:
270			break;
271		}
272
273next:
274		if (unlikely(++ntc == cnt)) {
275			ntc = 0;
276			gen = !gen;
277			idpf_queue_change(GEN_CHK, cq);
278		}
279	}
280
281out:
282	cq->next_to_clean = ntc;
283
284	return done_frames;
285}
286
287static u32 idpf_xdpsq_complete(void *_xdpsq, u32 budget)
288{
289	struct libeth_xdpsq_napi_stats ss = { };
290	struct idpf_tx_queue *xdpsq = _xdpsq;
291	u32 tx_ntc = xdpsq->next_to_clean;
292	u32 tx_cnt = xdpsq->desc_count;
293	struct xdp_frame_bulk bq;
294	struct libeth_cq_pp cp = {
295		.dev	= xdpsq->dev,
296		.bq	= &bq,
297		.xss	= &ss,
298		.napi	= true,
299	};
300	u32 done_frames;
301
302	done_frames = idpf_xdpsq_poll(xdpsq, budget);
303	if (unlikely(!done_frames))
304		return 0;
305
306	xdp_frame_bulk_init(&bq);
307
308	for (u32 i = 0; likely(i < done_frames); i++) {
309		libeth_xdp_complete_tx(&xdpsq->tx_buf[tx_ntc], &cp);
310
311		if (unlikely(++tx_ntc == tx_cnt))
312			tx_ntc = 0;
313	}
314
315	xdp_flush_frame_bulk(&bq);
316
317	xdpsq->next_to_clean = tx_ntc;
318	xdpsq->pending -= done_frames;
319	xdpsq->xdp_tx -= cp.xdp_tx;
320
321	return done_frames;
322}
323
324static u32 idpf_xdp_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq)
325{
326	struct idpf_tx_queue *xdpsq = _xdpsq;
327	u32 free;
328
329	libeth_xdpsq_lock(&xdpsq->xdp_lock);
330
331	free = xdpsq->desc_count - xdpsq->pending;
332	if (free < xdpsq->thresh)
333		free += idpf_xdpsq_complete(xdpsq, xdpsq->thresh);
334
335	*sq = (struct libeth_xdpsq){
336		.sqes		= xdpsq->tx_buf,
337		.descs		= xdpsq->desc_ring,
338		.count		= xdpsq->desc_count,
339		.lock		= &xdpsq->xdp_lock,
340		.ntu		= &xdpsq->next_to_use,
341		.pending	= &xdpsq->pending,
342		.xdp_tx		= &xdpsq->xdp_tx,
343	};
344
345	return free;
346}
347
348LIBETH_XDP_DEFINE_START();
349LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_xdpsq_complete);
350LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep,
351			   idpf_xdp_tx_xmit);
352LIBETH_XDP_DEFINE_FLUSH_XMIT(static idpf_xdp_xmit_flush_bulk, idpf_xdp_tx_prep,
353			     idpf_xdp_tx_xmit);
354LIBETH_XDP_DEFINE_END();
355
356int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
357		  u32 flags)
358{
359	const struct idpf_netdev_priv *np = netdev_priv(dev);
360	const struct idpf_vport *vport = np->vport;
361
362	if (unlikely(!netif_carrier_ok(dev) || !vport->link_up))
363		return -ENETDOWN;
364
365	return libeth_xdp_xmit_do_bulk(dev, n, frames, flags,
366				       &vport->txqs[vport->xdp_txq_offset],
367				       vport->num_xdp_txq,
368				       idpf_xdp_xmit_flush_bulk,
369				       idpf_xdp_tx_finalize);
370}
371
372static int idpf_xdpmo_rx_hash(const struct xdp_md *ctx, u32 *hash,
373			      enum xdp_rss_hash_type *rss_type)
374{
375	const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx;
376	struct idpf_xdp_rx_desc desc __uninitialized;
377	const struct idpf_rx_queue *rxq;
378	struct libeth_rx_pt pt;
379
380	rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq);
381
382	idpf_xdp_get_qw0(&desc, xdp->desc);
383
384	pt = rxq->rx_ptype_lkup[idpf_xdp_rx_pt(&desc)];
385	if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, pt))
386		return -ENODATA;
387
388	idpf_xdp_get_qw2(&desc, xdp->desc);
389
390	return libeth_xdpmo_rx_hash(hash, rss_type, idpf_xdp_rx_hash(&desc),
391				    pt);
392}
393
394static const struct xdp_metadata_ops idpf_xdpmo = {
395	.xmo_rx_hash		= idpf_xdpmo_rx_hash,
396};
397
398void idpf_xdp_set_features(const struct idpf_vport *vport)
399{
400	if (!idpf_is_queue_model_split(vport->rxq_model))
401		return;
402
403	libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo,
404					idpf_get_max_tx_bufs(vport->adapter),
405					libeth_xsktmo);
406}
407
408static int idpf_xdp_setup_prog(struct idpf_vport *vport,
409			       const struct netdev_bpf *xdp)
410{
411	const struct idpf_netdev_priv *np = netdev_priv(vport->netdev);
412	struct bpf_prog *old, *prog = xdp->prog;
413	struct idpf_vport_config *cfg;
414	int ret;
415
416	cfg = vport->adapter->vport_config[vport->idx];
417
418	if (test_bit(IDPF_REMOVE_IN_PROG, vport->adapter->flags) ||
419	    !test_bit(IDPF_VPORT_REG_NETDEV, cfg->flags) ||
420	    !!vport->xdp_prog == !!prog) {
421		if (test_bit(IDPF_VPORT_UP, np->state))
422			idpf_xdp_copy_prog_to_rqs(vport, prog);
423
424		old = xchg(&vport->xdp_prog, prog);
425		if (old)
426			bpf_prog_put(old);
427
428		cfg->user_config.xdp_prog = prog;
429
430		return 0;
431	}
432
433	if (!vport->num_xdp_txq && vport->num_txq == cfg->max_q.max_txq) {
434		NL_SET_ERR_MSG_MOD(xdp->extack,
435				   "No Tx queues available for XDP, please decrease the number of regular SQs");
436		return -ENOSPC;
437	}
438
439	old = cfg->user_config.xdp_prog;
440	cfg->user_config.xdp_prog = prog;
441
442	ret = idpf_initiate_soft_reset(vport, IDPF_SR_Q_CHANGE);
443	if (ret) {
444		NL_SET_ERR_MSG_MOD(xdp->extack,
445				   "Could not reopen the vport after XDP setup");
446
447		cfg->user_config.xdp_prog = old;
448		old = prog;
449	}
450
451	if (old)
452		bpf_prog_put(old);
453
454	libeth_xdp_set_redirect(vport->netdev, vport->xdp_prog);
455
456	return ret;
457}
458
459int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp)
460{
461	struct idpf_vport *vport;
462	int ret;
463
464	idpf_vport_ctrl_lock(dev);
465	vport = idpf_netdev_to_vport(dev);
466
467	if (!idpf_is_queue_model_split(vport->txq_model))
468		goto notsupp;
469
470	switch (xdp->command) {
471	case XDP_SETUP_PROG:
472		ret = idpf_xdp_setup_prog(vport, xdp);
473		break;
474	case XDP_SETUP_XSK_POOL:
475		ret = idpf_xsk_pool_setup(vport, xdp);
476		break;
477	default:
478notsupp:
479		ret = -EOPNOTSUPP;
480		break;
481	}
482
483	idpf_vport_ctrl_unlock(dev);
484
485	return ret;
486}