xdp.h

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/* Copyright (C) 2025 Intel Corporation */
   3
   4#ifndef __LIBETH_XDP_H
   5#define __LIBETH_XDP_H
   6
   7#include <linux/bpf_trace.h>
   8#include <linux/unroll.h>
   9
  10#include <net/libeth/rx.h>
  11#include <net/libeth/tx.h>
  12#include <net/xsk_buff_pool.h>
  13
  14/*
  15 * Defined as bits to be able to use them as a mask on Rx.
  16 * Also used as internal return values on Tx.
  17 */
  18enum {
  19	LIBETH_XDP_PASS			= 0U,
  20	LIBETH_XDP_DROP			= BIT(0),
  21	LIBETH_XDP_ABORTED		= BIT(1),
  22	LIBETH_XDP_TX			= BIT(2),
  23	LIBETH_XDP_REDIRECT		= BIT(3),
  24};
  25
  26/*
  27 * &xdp_buff_xsk is the largest structure &libeth_xdp_buff gets casted to,
  28 * pick maximum pointer-compatible alignment.
  29 */
  30#define __LIBETH_XDP_BUFF_ALIGN						      \
  31	(IS_ALIGNED(sizeof(struct xdp_buff_xsk), 16) ? 16 :		      \
  32	 IS_ALIGNED(sizeof(struct xdp_buff_xsk), 8) ? 8 :		      \
  33	 sizeof(long))
  34
  35/**
  36 * struct libeth_xdp_buff - libeth extension over &xdp_buff
  37 * @base: main &xdp_buff
  38 * @data: shortcut for @base.data
  39 * @desc: RQ descriptor containing metadata for this buffer
  40 * @priv: driver-private scratchspace
  41 *
  42 * The main reason for this is to have a pointer to the descriptor to be able
  43 * to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks
  44 * (as well as bigger alignment).
  45 * Pointer/layout-compatible with &xdp_buff and &xdp_buff_xsk.
  46 */
  47struct libeth_xdp_buff {
  48	union {
  49		struct xdp_buff		base;
  50		void			*data;
  51	};
  52
  53	const void			*desc;
  54	unsigned long			priv[]
  55					__aligned(__LIBETH_XDP_BUFF_ALIGN);
  56} __aligned(__LIBETH_XDP_BUFF_ALIGN);
  57static_assert(offsetof(struct libeth_xdp_buff, data) ==
  58	      offsetof(struct xdp_buff_xsk, xdp.data));
  59static_assert(offsetof(struct libeth_xdp_buff, desc) ==
  60	      offsetof(struct xdp_buff_xsk, cb));
  61static_assert(IS_ALIGNED(sizeof(struct xdp_buff_xsk),
  62			 __alignof(struct libeth_xdp_buff)));
  63
  64/**
  65 * __LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
  66 * @name: name of the variable to declare
  67 * @...: sizeof() of the driver-private data
  68 */
  69#define __LIBETH_XDP_ONSTACK_BUFF(name, ...)				      \
  70	___LIBETH_XDP_ONSTACK_BUFF(name, ##__VA_ARGS__)
  71/**
  72 * LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
  73 * @name: name of the variable to declare
  74 * @...: type or variable name of the driver-private data
  75 */
  76#define LIBETH_XDP_ONSTACK_BUFF(name, ...)				      \
  77	__LIBETH_XDP_ONSTACK_BUFF(name, __libeth_xdp_priv_sz(__VA_ARGS__))
  78
  79#define ___LIBETH_XDP_ONSTACK_BUFF(name, ...)				      \
  80	__DEFINE_FLEX(struct libeth_xdp_buff, name, priv,		      \
  81		      LIBETH_XDP_PRIV_SZ(__VA_ARGS__ + 0),		      \
  82		      __uninitialized);					      \
  83	LIBETH_XDP_ASSERT_PRIV_SZ(__VA_ARGS__ + 0)
  84
  85#define __libeth_xdp_priv_sz(...)					      \
  86	CONCATENATE(__libeth_xdp_psz, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)
  87
  88#define __libeth_xdp_psz0(...)
  89#define __libeth_xdp_psz1(...)		sizeof(__VA_ARGS__)
  90
  91#define LIBETH_XDP_PRIV_SZ(sz)						      \
  92	(ALIGN(sz, __alignof(struct libeth_xdp_buff)) / sizeof(long))
  93
  94/* Performs XSK_CHECK_PRIV_TYPE() */
  95#define LIBETH_XDP_ASSERT_PRIV_SZ(sz)					      \
  96	static_assert(offsetofend(struct xdp_buff_xsk, cb) >=		      \
  97		      struct_size_t(struct libeth_xdp_buff, priv,	      \
  98				    LIBETH_XDP_PRIV_SZ(sz)))
  99
 100/* XDPSQ sharing */
 101
 102DECLARE_STATIC_KEY_FALSE(libeth_xdpsq_share);
 103
 104/**
 105 * libeth_xdpsq_num - calculate optimal number of XDPSQs for this device + sys
 106 * @rxq: current number of active Rx queues
 107 * @txq: current number of active Tx queues
 108 * @max: maximum number of Tx queues
 109 *
 110 * Each RQ must have its own XDPSQ for XSk pairs, each CPU must have own XDPSQ
 111 * for lockless sending (``XDP_TX``, .ndo_xdp_xmit()). Cap the maximum of these
 112 * two with the number of SQs the device can have (minus used ones).
 113 *
 114 * Return: number of XDP Tx queues the device needs to use.
 115 */
 116static inline u32 libeth_xdpsq_num(u32 rxq, u32 txq, u32 max)
 117{
 118	return min(max(nr_cpu_ids, rxq), max - txq);
 119}
 120
 121/**
 122 * libeth_xdpsq_shared - whether XDPSQs can be shared between several CPUs
 123 * @num: number of active XDPSQs
 124 *
 125 * Return: true if there's no 1:1 XDPSQ/CPU association, false otherwise.
 126 */
 127static inline bool libeth_xdpsq_shared(u32 num)
 128{
 129	return num < nr_cpu_ids;
 130}
 131
 132/**
 133 * libeth_xdpsq_id - get XDPSQ index corresponding to this CPU
 134 * @num: number of active XDPSQs
 135 *
 136 * Helper for libeth_xdp routines, do not use in drivers directly.
 137 *
 138 * Return: XDPSQ index needs to be used on this CPU.
 139 */
 140static inline u32 libeth_xdpsq_id(u32 num)
 141{
 142	u32 ret = raw_smp_processor_id();
 143
 144	if (static_branch_unlikely(&libeth_xdpsq_share) &&
 145	    libeth_xdpsq_shared(num))
 146		ret %= num;
 147
 148	return ret;
 149}
 150
 151void __libeth_xdpsq_get(struct libeth_xdpsq_lock *lock,
 152			const struct net_device *dev);
 153void __libeth_xdpsq_put(struct libeth_xdpsq_lock *lock,
 154			const struct net_device *dev);
 155
 156/**
 157 * libeth_xdpsq_get - initialize &libeth_xdpsq_lock
 158 * @lock: lock to initialize
 159 * @dev: netdev which this lock belongs to
 160 * @share: whether XDPSQs can be shared
 161 *
 162 * Tracks the current XDPSQ association and enables the static lock
 163 * if needed.
 164 */
 165static inline void libeth_xdpsq_get(struct libeth_xdpsq_lock *lock,
 166				    const struct net_device *dev,
 167				    bool share)
 168{
 169	if (unlikely(share))
 170		__libeth_xdpsq_get(lock, dev);
 171}
 172
 173/**
 174 * libeth_xdpsq_put - deinitialize &libeth_xdpsq_lock
 175 * @lock: lock to deinitialize
 176 * @dev: netdev which this lock belongs to
 177 *
 178 * Tracks the current XDPSQ association and disables the static lock
 179 * if needed.
 180 */
 181static inline void libeth_xdpsq_put(struct libeth_xdpsq_lock *lock,
 182				    const struct net_device *dev)
 183{
 184	if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
 185		__libeth_xdpsq_put(lock, dev);
 186}
 187
 188void __libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock);
 189void __libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock);
 190
 191/**
 192 * libeth_xdpsq_lock - grab &libeth_xdpsq_lock if needed
 193 * @lock: lock to take
 194 *
 195 * Touches the underlying spinlock only if the static key is enabled
 196 * and the queue itself is marked as shareable.
 197 */
 198static inline void libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock)
 199{
 200	if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
 201		__libeth_xdpsq_lock(lock);
 202}
 203
 204/**
 205 * libeth_xdpsq_unlock - free &libeth_xdpsq_lock if needed
 206 * @lock: lock to free
 207 *
 208 * Touches the underlying spinlock only if the static key is enabled
 209 * and the queue itself is marked as shareable.
 210 */
 211static inline void libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock)
 212{
 213	if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
 214		__libeth_xdpsq_unlock(lock);
 215}
 216
 217/* XDPSQ clean-up timers */
 218
 219void libeth_xdpsq_init_timer(struct libeth_xdpsq_timer *timer, void *xdpsq,
 220			     struct libeth_xdpsq_lock *lock,
 221			     void (*poll)(struct work_struct *work));
 222
 223/**
 224 * libeth_xdpsq_deinit_timer - deinitialize &libeth_xdpsq_timer
 225 * @timer: timer to deinitialize
 226 *
 227 * Flush and disable the underlying workqueue.
 228 */
 229static inline void libeth_xdpsq_deinit_timer(struct libeth_xdpsq_timer *timer)
 230{
 231	cancel_delayed_work_sync(&timer->dwork);
 232}
 233
 234/**
 235 * libeth_xdpsq_queue_timer - run &libeth_xdpsq_timer
 236 * @timer: timer to queue
 237 *
 238 * Should be called after the queue was filled and the transmission was run
 239 * to complete the pending buffers if no further sending will be done in a
 240 * second (-> lazy cleaning won't happen).
 241 * If the timer was already run, it will be requeued back to one second
 242 * timeout again.
 243 */
 244static inline void libeth_xdpsq_queue_timer(struct libeth_xdpsq_timer *timer)
 245{
 246	mod_delayed_work_on(raw_smp_processor_id(), system_bh_highpri_wq,
 247			    &timer->dwork, HZ);
 248}
 249
 250/**
 251 * libeth_xdpsq_run_timer - wrapper to run a queue clean-up on a timer event
 252 * @work: workqueue belonging to the corresponding timer
 253 * @poll: driver-specific completion queue poll function
 254 *
 255 * Run the polling function on the locked queue and requeue the timer if
 256 * there's more work to do.
 257 * Designed to be used via LIBETH_XDP_DEFINE_TIMER() below.
 258 */
 259static __always_inline void
 260libeth_xdpsq_run_timer(struct work_struct *work,
 261		       u32 (*poll)(void *xdpsq, u32 budget))
 262{
 263	struct libeth_xdpsq_timer *timer = container_of(work, typeof(*timer),
 264							dwork.work);
 265
 266	libeth_xdpsq_lock(timer->lock);
 267
 268	if (poll(timer->xdpsq, U32_MAX))
 269		libeth_xdpsq_queue_timer(timer);
 270
 271	libeth_xdpsq_unlock(timer->lock);
 272}
 273
 274/* Common Tx bits */
 275
 276/**
 277 * enum - libeth_xdp internal Tx flags
 278 * @LIBETH_XDP_TX_BULK: one bulk size at which it will be flushed to the queue
 279 * @LIBETH_XDP_TX_BATCH: batch size for which the queue fill loop is unrolled
 280 * @LIBETH_XDP_TX_DROP: indicates the send function must drop frames not sent
 281 * @LIBETH_XDP_TX_NDO: whether the send function is called from .ndo_xdp_xmit()
 282 * @LIBETH_XDP_TX_XSK: whether the function is called for ``XDP_TX`` for XSk
 283 */
 284enum {
 285	LIBETH_XDP_TX_BULK		= DEV_MAP_BULK_SIZE,
 286	LIBETH_XDP_TX_BATCH		= 8,
 287
 288	LIBETH_XDP_TX_DROP		= BIT(0),
 289	LIBETH_XDP_TX_NDO		= BIT(1),
 290	LIBETH_XDP_TX_XSK		= BIT(2),
 291};
 292
 293/**
 294 * enum - &libeth_xdp_tx_frame and &libeth_xdp_tx_desc flags
 295 * @LIBETH_XDP_TX_LEN: only for ``XDP_TX``, [15:0] of ::len_fl is actual length
 296 * @LIBETH_XDP_TX_CSUM: for XSk xmit, enable checksum offload
 297 * @LIBETH_XDP_TX_XSKMD: for XSk xmit, mask of the metadata bits
 298 * @LIBETH_XDP_TX_FIRST: indicates the frag is the first one of the frame
 299 * @LIBETH_XDP_TX_LAST: whether the frag is the last one of the frame
 300 * @LIBETH_XDP_TX_MULTI: whether the frame contains several frags
 301 * @LIBETH_XDP_TX_FLAGS: only for ``XDP_TX``, [31:16] of ::len_fl is flags
 302 */
 303enum {
 304	LIBETH_XDP_TX_LEN		= GENMASK(15, 0),
 305
 306	LIBETH_XDP_TX_CSUM		= XDP_TXMD_FLAGS_CHECKSUM,
 307	LIBETH_XDP_TX_XSKMD		= LIBETH_XDP_TX_LEN,
 308
 309	LIBETH_XDP_TX_FIRST		= BIT(16),
 310	LIBETH_XDP_TX_LAST		= BIT(17),
 311	LIBETH_XDP_TX_MULTI		= BIT(18),
 312
 313	LIBETH_XDP_TX_FLAGS		= GENMASK(31, 16),
 314};
 315
 316/**
 317 * struct libeth_xdp_tx_frame - represents one XDP Tx element
 318 * @data: frame start pointer for ``XDP_TX``
 319 * @len_fl: ``XDP_TX``, combined flags [31:16] and len [15:0] field for speed
 320 * @soff: ``XDP_TX``, offset from @data to the start of &skb_shared_info
 321 * @frag: one (non-head) frag for ``XDP_TX``
 322 * @xdpf: &xdp_frame for the head frag for .ndo_xdp_xmit()
 323 * @dma: DMA address of the non-head frag for .ndo_xdp_xmit()
 324 * @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag
 325 * @len: frag length for XSk ``XDP_TX`` and .ndo_xdp_xmit()
 326 * @flags: Tx flags for the above
 327 * @opts: combined @len + @flags for the above for speed
 328 * @desc: XSk xmit descriptor for direct casting
 329 */
 330struct libeth_xdp_tx_frame {
 331	union {
 332		/* ``XDP_TX`` */
 333		struct {
 334			void				*data;
 335			u32				len_fl;
 336			u32				soff;
 337		};
 338
 339		/* ``XDP_TX`` frag */
 340		skb_frag_t			frag;
 341
 342		/* .ndo_xdp_xmit(), XSk ``XDP_TX`` */
 343		struct {
 344			union {
 345				struct xdp_frame		*xdpf;
 346				dma_addr_t			dma;
 347
 348				struct libeth_xdp_buff		*xsk;
 349			};
 350			union {
 351				struct {
 352					u32				len;
 353					u32				flags;
 354				};
 355				aligned_u64			opts;
 356			};
 357		};
 358
 359		/* XSk xmit */
 360		struct xdp_desc			desc;
 361	};
 362} __aligned(sizeof(struct xdp_desc));
 363static_assert(offsetof(struct libeth_xdp_tx_frame, frag.len) ==
 364	      offsetof(struct libeth_xdp_tx_frame, len_fl));
 365static_assert(sizeof(struct libeth_xdp_tx_frame) == sizeof(struct xdp_desc));
 366
 367/**
 368 * struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending
 369 * @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit()
 370 * @dev: &net_device which the frames are transmitted on
 371 * @xdpsq: shortcut to the corresponding driver-specific XDPSQ structure
 372 * @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session
 373 * @count: current number of frames in @bulk
 374 * @bulk: array of queued frames for bulk Tx
 375 *
 376 * All XDP Tx operations except XSk xmit queue each frame to the bulk first
 377 * and flush it when @count reaches the array end. Bulk is always placed on
 378 * the stack for performance. One bulk element contains all the data necessary
 379 * for sending a frame and then freeing it on completion.
 380 * For XSk xmit, Tx descriptor array from &xsk_buff_pool is casted directly
 381 * to &libeth_xdp_tx_frame as they are compatible and the bulk structure is
 382 * not used.
 383 */
 384struct libeth_xdp_tx_bulk {
 385	const struct bpf_prog		*prog;
 386	struct net_device		*dev;
 387	void				*xdpsq;
 388
 389	u32				act_mask;
 390	u32				count;
 391	struct libeth_xdp_tx_frame	bulk[LIBETH_XDP_TX_BULK];
 392} __aligned(sizeof(struct libeth_xdp_tx_frame));
 393
 394/**
 395 * LIBETH_XDP_ONSTACK_BULK - declare &libeth_xdp_tx_bulk on the stack
 396 * @bq: name of the variable to declare
 397 *
 398 * Helper to declare a bulk on the stack with a compiler hint that it should
 399 * not be initialized automatically (with `CONFIG_INIT_STACK_ALL_*`) for
 400 * performance reasons.
 401 */
 402#define LIBETH_XDP_ONSTACK_BULK(bq)					      \
 403	struct libeth_xdp_tx_bulk bq __uninitialized
 404
 405/**
 406 * struct libeth_xdpsq - abstraction for an XDPSQ
 407 * @pool: XSk buffer pool for XSk ``XDP_TX`` and xmit
 408 * @sqes: array of Tx buffers from the actual queue struct
 409 * @descs: opaque pointer to the HW descriptor array
 410 * @ntu: pointer to the next free descriptor index
 411 * @count: number of descriptors on that queue
 412 * @pending: pointer to the number of sent-not-completed descs on that queue
 413 * @xdp_tx: pointer to the above, but only for non-XSk-xmit frames
 414 * @lock: corresponding XDPSQ lock
 415 *
 416 * Abstraction for driver-independent implementation of Tx. Placed on the stack
 417 * and filled by the driver before the transmission, so that the generic
 418 * functions can access and modify driver-specific resources.
 419 */
 420struct libeth_xdpsq {
 421	struct xsk_buff_pool		*pool;
 422	struct libeth_sqe		*sqes;
 423	void				*descs;
 424
 425	u32				*ntu;
 426	u32				count;
 427
 428	u32				*pending;
 429	u32				*xdp_tx;
 430	struct libeth_xdpsq_lock	*lock;
 431};
 432
 433/**
 434 * struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor
 435 * @addr: DMA address of the frame
 436 * @len: length of the frame
 437 * @flags: XDP Tx flags
 438 * @opts: combined @len + @flags for speed
 439 *
 440 * Filled by the generic functions and then passed to driver-specific functions
 441 * to fill a HW Tx descriptor, always placed on the [function] stack.
 442 */
 443struct libeth_xdp_tx_desc {
 444	dma_addr_t			addr;
 445	union {
 446		struct {
 447			u32				len;
 448			u32				flags;
 449		};
 450		aligned_u64			opts;
 451	};
 452} __aligned_largest;
 453
 454/**
 455 * libeth_xdp_ptr_to_priv - convert pointer to a libeth_xdp u64 priv
 456 * @ptr: pointer to convert
 457 *
 458 * The main sending function passes private data as the largest scalar, u64.
 459 * Use this helper when you want to pass a pointer there.
 460 */
 461#define libeth_xdp_ptr_to_priv(ptr) ({					      \
 462	typecheck_pointer(ptr);						      \
 463	((u64)(uintptr_t)(ptr));					      \
 464})
 465/**
 466 * libeth_xdp_priv_to_ptr - convert libeth_xdp u64 priv to a pointer
 467 * @priv: private data to convert
 468 *
 469 * The main sending function passes private data as the largest scalar, u64.
 470 * Use this helper when your callback takes this u64 and you want to convert
 471 * it back to a pointer.
 472 */
 473#define libeth_xdp_priv_to_ptr(priv) ({					      \
 474	static_assert(__same_type(priv, u64));				      \
 475	((const void *)(uintptr_t)(priv));				      \
 476})
 477
 478/*
 479 * On 64-bit systems, assigning one u64 is faster than two u32s. When ::len
 480 * occupies lowest 32 bits (LE), whole ::opts can be assigned directly instead.
 481 */
 482#ifdef __LITTLE_ENDIAN
 483#define __LIBETH_WORD_ACCESS		1
 484#endif
 485#ifdef __LIBETH_WORD_ACCESS
 486#define __libeth_xdp_tx_len(flen, ...)					      \
 487	.opts = ((flen) | FIELD_PREP(GENMASK_ULL(63, 32), (__VA_ARGS__ + 0)))
 488#else
 489#define __libeth_xdp_tx_len(flen, ...)					      \
 490	.len = (flen), .flags = (__VA_ARGS__ + 0)
 491#endif
 492
 493/**
 494 * libeth_xdp_tx_xmit_bulk - main XDP Tx function
 495 * @bulk: array of frames to send
 496 * @xdpsq: pointer to the driver-specific XDPSQ struct
 497 * @n: number of frames to send
 498 * @unroll: whether to unroll the queue filling loop for speed
 499 * @priv: driver-specific private data
 500 * @prep: callback for cleaning the queue and filling abstract &libeth_xdpsq
 501 * @fill: internal callback for filling &libeth_sqe and &libeth_xdp_tx_desc
 502 * @xmit: callback for filling a HW descriptor with the frame info
 503 *
 504 * Internal abstraction for placing @n XDP Tx frames on the HW XDPSQ. Used for
 505 * all types of frames: ``XDP_TX``, .ndo_xdp_xmit(), XSk ``XDP_TX``, and XSk
 506 * xmit.
 507 * @prep must lock the queue as this function releases it at the end. @unroll
 508 * greatly increases the object code size, but also greatly increases XSk xmit
 509 * performance; for other types of frames, it's not enabled.
 510 * The compilers inline all those onstack abstractions to direct data accesses.
 511 *
 512 * Return: number of frames actually placed on the queue, <= @n. The function
 513 * can't fail, but can send less frames if there's no enough free descriptors
 514 * available. The actual free space is returned by @prep from the driver.
 515 */
 516static __always_inline __nocfi_generic u32
 517libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame *bulk, void *xdpsq,
 518			u32 n, bool unroll, u64 priv,
 519			u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq),
 520			struct libeth_xdp_tx_desc
 521			(*fill)(struct libeth_xdp_tx_frame frm, u32 i,
 522				const struct libeth_xdpsq *sq, u64 priv),
 523			void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
 524				     const struct libeth_xdpsq *sq, u64 priv))
 525{
 526	struct libeth_xdpsq sq __uninitialized;
 527	u32 this, batched, off = 0;
 528	u32 ntu, i = 0;
 529
 530	n = min(n, prep(xdpsq, &sq));
 531	if (unlikely(!n))
 532		goto unlock;
 533
 534	ntu = *sq.ntu;
 535
 536	this = sq.count - ntu;
 537	if (likely(this > n))
 538		this = n;
 539
 540again:
 541	if (!unroll)
 542		goto linear;
 543
 544	batched = ALIGN_DOWN(this, LIBETH_XDP_TX_BATCH);
 545
 546	for ( ; i < off + batched; i += LIBETH_XDP_TX_BATCH) {
 547		u32 base = ntu + i - off;
 548
 549		unrolled_count(LIBETH_XDP_TX_BATCH)
 550		for (u32 j = 0; j < LIBETH_XDP_TX_BATCH; j++)
 551			xmit(fill(bulk[i + j], base + j, &sq, priv),
 552			     base + j, &sq, priv);
 553	}
 554
 555	if (batched < this) {
 556linear:
 557		for ( ; i < off + this; i++)
 558			xmit(fill(bulk[i], ntu + i - off, &sq, priv),
 559			     ntu + i - off, &sq, priv);
 560	}
 561
 562	ntu += this;
 563	if (likely(ntu < sq.count))
 564		goto out;
 565
 566	ntu = 0;
 567
 568	if (i < n) {
 569		this = n - i;
 570		off = i;
 571
 572		goto again;
 573	}
 574
 575out:
 576	*sq.ntu = ntu;
 577	*sq.pending += n;
 578	if (sq.xdp_tx)
 579		*sq.xdp_tx += n;
 580
 581unlock:
 582	libeth_xdpsq_unlock(sq.lock);
 583
 584	return n;
 585}
 586
 587/* ``XDP_TX`` bulking */
 588
 589void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp);
 590
 591/**
 592 * libeth_xdp_tx_queue_head - internal helper for queueing one ``XDP_TX`` head
 593 * @bq: XDP Tx bulk to queue the head frag to
 594 * @xdp: XDP buffer with the head to queue
 595 *
 596 * Return: false if it's the only frag of the frame, true if it's an S/G frame.
 597 */
 598static inline bool libeth_xdp_tx_queue_head(struct libeth_xdp_tx_bulk *bq,
 599					    const struct libeth_xdp_buff *xdp)
 600{
 601	const struct xdp_buff *base = &xdp->base;
 602
 603	bq->bulk[bq->count++] = (typeof(*bq->bulk)){
 604		.data	= xdp->data,
 605		.len_fl	= (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST,
 606		.soff	= xdp_data_hard_end(base) - xdp->data,
 607	};
 608
 609	if (!xdp_buff_has_frags(base))
 610		return false;
 611
 612	bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI;
 613
 614	return true;
 615}
 616
 617/**
 618 * libeth_xdp_tx_queue_frag - internal helper for queueing one ``XDP_TX`` frag
 619 * @bq: XDP Tx bulk to queue the frag to
 620 * @frag: frag to queue
 621 */
 622static inline void libeth_xdp_tx_queue_frag(struct libeth_xdp_tx_bulk *bq,
 623					    const skb_frag_t *frag)
 624{
 625	bq->bulk[bq->count++].frag = *frag;
 626}
 627
 628/**
 629 * libeth_xdp_tx_queue_bulk - internal helper for queueing one ``XDP_TX`` frame
 630 * @bq: XDP Tx bulk to queue the frame to
 631 * @xdp: XDP buffer to queue
 632 * @flush_bulk: driver callback to flush the bulk to the HW queue
 633 *
 634 * Return: true on success, false on flush error.
 635 */
 636static __always_inline bool
 637libeth_xdp_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq,
 638			 struct libeth_xdp_buff *xdp,
 639			 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
 640					    u32 flags))
 641{
 642	const struct skb_shared_info *sinfo;
 643	bool ret = true;
 644	u32 nr_frags;
 645
 646	if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
 647	    unlikely(!flush_bulk(bq, 0))) {
 648		libeth_xdp_return_buff_slow(xdp);
 649		return false;
 650	}
 651
 652	if (!libeth_xdp_tx_queue_head(bq, xdp))
 653		goto out;
 654
 655	sinfo = xdp_get_shared_info_from_buff(&xdp->base);
 656	nr_frags = sinfo->nr_frags;
 657
 658	for (u32 i = 0; i < nr_frags; i++) {
 659		if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
 660		    unlikely(!flush_bulk(bq, 0))) {
 661			ret = false;
 662			break;
 663		}
 664
 665		libeth_xdp_tx_queue_frag(bq, &sinfo->frags[i]);
 666	}
 667
 668out:
 669	bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_LAST;
 670	xdp->data = NULL;
 671
 672	return ret;
 673}
 674
 675/**
 676 * libeth_xdp_tx_fill_stats - fill &libeth_sqe with ``XDP_TX`` frame stats
 677 * @sqe: SQ element to fill
 678 * @desc: libeth_xdp Tx descriptor
 679 * @sinfo: &skb_shared_info for this frame
 680 *
 681 * Internal helper for filling an SQE with the frame stats, do not use in
 682 * drivers. Fills the number of frags and bytes for this frame.
 683 */
 684#define libeth_xdp_tx_fill_stats(sqe, desc, sinfo)			      \
 685	__libeth_xdp_tx_fill_stats(sqe, desc, sinfo, __UNIQUE_ID(sqe_),	      \
 686				   __UNIQUE_ID(desc_), __UNIQUE_ID(sinfo_))
 687
 688#define __libeth_xdp_tx_fill_stats(sqe, desc, sinfo, ue, ud, us) do {	      \
 689	const struct libeth_xdp_tx_desc *ud = (desc);			      \
 690	const struct skb_shared_info *us;				      \
 691	struct libeth_sqe *ue = (sqe);					      \
 692									      \
 693	ue->nr_frags = 1;						      \
 694	ue->bytes = ud->len;						      \
 695									      \
 696	if (ud->flags & LIBETH_XDP_TX_MULTI) {				      \
 697		us = (sinfo);						      \
 698		ue->nr_frags += us->nr_frags;				      \
 699		ue->bytes += us->xdp_frags_size;			      \
 700	}								      \
 701} while (0)
 702
 703/**
 704 * libeth_xdp_tx_fill_buf - internal helper to fill one ``XDP_TX`` &libeth_sqe
 705 * @frm: XDP Tx frame from the bulk
 706 * @i: index on the HW queue
 707 * @sq: XDPSQ abstraction for the queue
 708 * @priv: private data
 709 *
 710 * Return: XDP Tx descriptor with the synced DMA and other info to pass to
 711 * the driver callback.
 712 */
 713static inline struct libeth_xdp_tx_desc
 714libeth_xdp_tx_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
 715		       const struct libeth_xdpsq *sq, u64 priv)
 716{
 717	struct libeth_xdp_tx_desc desc;
 718	struct skb_shared_info *sinfo;
 719	skb_frag_t *frag = &frm.frag;
 720	struct libeth_sqe *sqe;
 721	netmem_ref netmem;
 722
 723	if (frm.len_fl & LIBETH_XDP_TX_FIRST) {
 724		sinfo = frm.data + frm.soff;
 725		skb_frag_fill_netmem_desc(frag, virt_to_netmem(frm.data),
 726					  offset_in_page(frm.data),
 727					  frm.len_fl);
 728	} else {
 729		sinfo = NULL;
 730	}
 731
 732	netmem = skb_frag_netmem(frag);
 733	desc = (typeof(desc)){
 734		.addr	= page_pool_get_dma_addr_netmem(netmem) +
 735			  skb_frag_off(frag),
 736		.len	= skb_frag_size(frag) & LIBETH_XDP_TX_LEN,
 737		.flags	= skb_frag_size(frag) & LIBETH_XDP_TX_FLAGS,
 738	};
 739
 740	dma_sync_single_for_device(__netmem_get_pp(netmem)->p.dev, desc.addr,
 741				   desc.len, DMA_BIDIRECTIONAL);
 742
 743	if (!sinfo)
 744		return desc;
 745
 746	sqe = &sq->sqes[i];
 747	sqe->type = LIBETH_SQE_XDP_TX;
 748	sqe->sinfo = sinfo;
 749	libeth_xdp_tx_fill_stats(sqe, &desc, sinfo);
 750
 751	return desc;
 752}
 753
 754void libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent,
 755			     u32 flags);
 756
 757/**
 758 * __libeth_xdp_tx_flush_bulk - internal helper to flush one XDP Tx bulk
 759 * @bq: bulk to flush
 760 * @flags: XDP TX flags (.ndo_xdp_xmit(), XSk etc.)
 761 * @prep: driver-specific callback to prepare the queue for sending
 762 * @fill: libeth_xdp callback to fill &libeth_sqe and &libeth_xdp_tx_desc
 763 * @xmit: driver callback to fill a HW descriptor
 764 *
 765 * Internal abstraction to create bulk flush functions for drivers. Used for
 766 * everything except XSk xmit.
 767 *
 768 * Return: true if anything was sent, false otherwise.
 769 */
 770static __always_inline bool
 771__libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags,
 772			   u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq),
 773			   struct libeth_xdp_tx_desc
 774			   (*fill)(struct libeth_xdp_tx_frame frm, u32 i,
 775				   const struct libeth_xdpsq *sq, u64 priv),
 776			   void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
 777					const struct libeth_xdpsq *sq,
 778					u64 priv))
 779{
 780	u32 sent, drops;
 781	int err = 0;
 782
 783	sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq,
 784				       min(bq->count, LIBETH_XDP_TX_BULK),
 785				       false, 0, prep, fill, xmit);
 786	drops = bq->count - sent;
 787
 788	if (unlikely(drops)) {
 789		libeth_xdp_tx_exception(bq, sent, flags);
 790		err = -ENXIO;
 791	} else {
 792		bq->count = 0;
 793	}
 794
 795	trace_xdp_bulk_tx(bq->dev, sent, drops, err);
 796
 797	return likely(sent);
 798}
 799
 800/**
 801 * libeth_xdp_tx_flush_bulk - wrapper to define flush of one ``XDP_TX`` bulk
 802 * @bq: bulk to flush
 803 * @flags: Tx flags, see above
 804 * @prep: driver callback to prepare the queue
 805 * @xmit: driver callback to fill a HW descriptor
 806 *
 807 * Use via LIBETH_XDP_DEFINE_FLUSH_TX() to define an ``XDP_TX`` driver
 808 * callback.
 809 */
 810#define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit)			      \
 811	__libeth_xdp_tx_flush_bulk(bq, flags, prep, libeth_xdp_tx_fill_buf,   \
 812				   xmit)
 813
 814/* .ndo_xdp_xmit() implementation */
 815
 816/**
 817 * libeth_xdp_xmit_init_bulk - internal helper to initialize bulk for XDP xmit
 818 * @bq: bulk to initialize
 819 * @dev: target &net_device
 820 * @xdpsqs: array of driver-specific XDPSQ structs
 821 * @num: number of active XDPSQs (the above array length)
 822 */
 823#define libeth_xdp_xmit_init_bulk(bq, dev, xdpsqs, num)			      \
 824	__libeth_xdp_xmit_init_bulk(bq, dev, (xdpsqs)[libeth_xdpsq_id(num)])
 825
 826static inline void __libeth_xdp_xmit_init_bulk(struct libeth_xdp_tx_bulk *bq,
 827					       struct net_device *dev,
 828					       void *xdpsq)
 829{
 830	bq->dev = dev;
 831	bq->xdpsq = xdpsq;
 832	bq->count = 0;
 833}
 834
 835/**
 836 * libeth_xdp_xmit_frame_dma - internal helper to access DMA of an &xdp_frame
 837 * @xf: pointer to the XDP frame
 838 *
 839 * There's no place in &libeth_xdp_tx_frame to store DMA address for an
 840 * &xdp_frame head. The headroom is used then, the address is placed right
 841 * after the frame struct, naturally aligned.
 842 *
 843 * Return: pointer to the DMA address to use.
 844 */
 845#define libeth_xdp_xmit_frame_dma(xf)					      \
 846	_Generic((xf),							      \
 847		 const struct xdp_frame *:				      \
 848			(const dma_addr_t *)__libeth_xdp_xmit_frame_dma(xf),  \
 849		 struct xdp_frame *:					      \
 850			(dma_addr_t *)__libeth_xdp_xmit_frame_dma(xf)	      \
 851	)
 852
 853static inline void *__libeth_xdp_xmit_frame_dma(const struct xdp_frame *xdpf)
 854{
 855	void *addr = (void *)(xdpf + 1);
 856
 857	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
 858	    __alignof(*xdpf) < sizeof(dma_addr_t))
 859		addr = PTR_ALIGN(addr, sizeof(dma_addr_t));
 860
 861	return addr;
 862}
 863
 864/**
 865 * libeth_xdp_xmit_queue_head - internal helper for queueing one XDP xmit head
 866 * @bq: XDP Tx bulk to queue the head frag to
 867 * @xdpf: XDP frame with the head to queue
 868 * @dev: device to perform DMA mapping
 869 *
 870 * Return: ``LIBETH_XDP_DROP`` on DMA mapping error,
 871 *	   ``LIBETH_XDP_PASS`` if it's the only frag in the frame,
 872 *	   ``LIBETH_XDP_TX`` if it's an S/G frame.
 873 */
 874static inline u32 libeth_xdp_xmit_queue_head(struct libeth_xdp_tx_bulk *bq,
 875					     struct xdp_frame *xdpf,
 876					     struct device *dev)
 877{
 878	dma_addr_t dma;
 879
 880	dma = dma_map_single(dev, xdpf->data, xdpf->len, DMA_TO_DEVICE);
 881	if (dma_mapping_error(dev, dma))
 882		return LIBETH_XDP_DROP;
 883
 884	*libeth_xdp_xmit_frame_dma(xdpf) = dma;
 885
 886	bq->bulk[bq->count++] = (typeof(*bq->bulk)){
 887		.xdpf	= xdpf,
 888		__libeth_xdp_tx_len(xdpf->len, LIBETH_XDP_TX_FIRST),
 889	};
 890
 891	if (!xdp_frame_has_frags(xdpf))
 892		return LIBETH_XDP_PASS;
 893
 894	bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI;
 895
 896	return LIBETH_XDP_TX;
 897}
 898
 899/**
 900 * libeth_xdp_xmit_queue_frag - internal helper for queueing one XDP xmit frag
 901 * @bq: XDP Tx bulk to queue the frag to
 902 * @frag: frag to queue
 903 * @dev: device to perform DMA mapping
 904 *
 905 * Return: true on success, false on DMA mapping error.
 906 */
 907static inline bool libeth_xdp_xmit_queue_frag(struct libeth_xdp_tx_bulk *bq,
 908					      const skb_frag_t *frag,
 909					      struct device *dev)
 910{
 911	dma_addr_t dma;
 912
 913	dma = skb_frag_dma_map(dev, frag);
 914	if (dma_mapping_error(dev, dma))
 915		return false;
 916
 917	bq->bulk[bq->count++] = (typeof(*bq->bulk)){
 918		.dma	= dma,
 919		__libeth_xdp_tx_len(skb_frag_size(frag)),
 920	};
 921
 922	return true;
 923}
 924
 925/**
 926 * libeth_xdp_xmit_queue_bulk - internal helper for queueing one XDP xmit frame
 927 * @bq: XDP Tx bulk to queue the frame to
 928 * @xdpf: XDP frame to queue
 929 * @flush_bulk: driver callback to flush the bulk to the HW queue
 930 *
 931 * Return: ``LIBETH_XDP_TX`` on success,
 932 *	   ``LIBETH_XDP_DROP`` if the frame should be dropped by the stack,
 933 *	   ``LIBETH_XDP_ABORTED`` if the frame will be dropped by libeth_xdp.
 934 */
 935static __always_inline u32
 936libeth_xdp_xmit_queue_bulk(struct libeth_xdp_tx_bulk *bq,
 937			   struct xdp_frame *xdpf,
 938			   bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
 939					      u32 flags))
 940{
 941	u32 head, nr_frags, i, ret = LIBETH_XDP_TX;
 942	struct device *dev = bq->dev->dev.parent;
 943	const struct skb_shared_info *sinfo;
 944
 945	if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
 946	    unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO)))
 947		return LIBETH_XDP_DROP;
 948
 949	head = libeth_xdp_xmit_queue_head(bq, xdpf, dev);
 950	if (head == LIBETH_XDP_PASS)
 951		goto out;
 952	else if (head == LIBETH_XDP_DROP)
 953		return LIBETH_XDP_DROP;
 954
 955	sinfo = xdp_get_shared_info_from_frame(xdpf);
 956	nr_frags = sinfo->nr_frags;
 957
 958	for (i = 0; i < nr_frags; i++) {
 959		if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
 960		    unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO)))
 961			break;
 962
 963		if (!libeth_xdp_xmit_queue_frag(bq, &sinfo->frags[i], dev))
 964			break;
 965	}
 966
 967	if (unlikely(i < nr_frags))
 968		ret = LIBETH_XDP_ABORTED;
 969
 970out:
 971	bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST;
 972
 973	return ret;
 974}
 975
 976/**
 977 * libeth_xdp_xmit_fill_buf - internal helper to fill one XDP xmit &libeth_sqe
 978 * @frm: XDP Tx frame from the bulk
 979 * @i: index on the HW queue
 980 * @sq: XDPSQ abstraction for the queue
 981 * @priv: private data
 982 *
 983 * Return: XDP Tx descriptor with the mapped DMA and other info to pass to
 984 * the driver callback.
 985 */
 986static inline struct libeth_xdp_tx_desc
 987libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
 988			 const struct libeth_xdpsq *sq, u64 priv)
 989{
 990	struct libeth_xdp_tx_desc desc;
 991	struct libeth_sqe *sqe;
 992	struct xdp_frame *xdpf;
 993
 994	if (frm.flags & LIBETH_XDP_TX_FIRST) {
 995		xdpf = frm.xdpf;
 996		desc.addr = *libeth_xdp_xmit_frame_dma(xdpf);
 997	} else {
 998		xdpf = NULL;
 999		desc.addr = frm.dma;
1000	}
1001	desc.opts = frm.opts;
1002
1003	sqe = &sq->sqes[i];
1004	dma_unmap_addr_set(sqe, dma, desc.addr);
1005	dma_unmap_len_set(sqe, len, desc.len);
1006
1007	if (!xdpf) {
1008		sqe->type = LIBETH_SQE_XDP_XMIT_FRAG;
1009		return desc;
1010	}
1011
1012	sqe->type = LIBETH_SQE_XDP_XMIT;
1013	sqe->xdpf = xdpf;
1014	libeth_xdp_tx_fill_stats(sqe, &desc,
1015				 xdp_get_shared_info_from_frame(xdpf));
1016
1017	return desc;
1018}
1019
1020/**
1021 * libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk
1022 * @bq: bulk to flush
1023 * @flags: Tx flags, see __libeth_xdp_tx_flush_bulk()
1024 * @prep: driver callback to prepare the queue
1025 * @xmit: driver callback to fill a HW descriptor
1026 *
1027 * Use via LIBETH_XDP_DEFINE_FLUSH_XMIT() to define an XDP xmit driver
1028 * callback.
1029 */
1030#define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit)		      \
1031	__libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_NDO, prep,     \
1032				   libeth_xdp_xmit_fill_buf, xmit)
1033
1034u32 libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq,
1035				u32 count, const struct net_device *dev);
1036
1037/**
1038 * __libeth_xdp_xmit_do_bulk - internal function to implement .ndo_xdp_xmit()
1039 * @bq: XDP Tx bulk to queue frames to
1040 * @frames: XDP frames passed by the stack
1041 * @n: number of frames
1042 * @flags: flags passed by the stack
1043 * @flush_bulk: driver callback to flush an XDP xmit bulk
1044 * @finalize: driver callback to finalize sending XDP Tx frames on the queue
1045 *
1046 * Perform common checks, map the frags and queue them to the bulk, then flush
1047 * the bulk to the XDPSQ. If requested by the stack, finalize the queue.
1048 *
1049 * Return: number of frames send or -errno on error.
1050 */
1051static __always_inline int
1052__libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq,
1053			  struct xdp_frame **frames, u32 n, u32 flags,
1054			  bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1055					     u32 flags),
1056			  void (*finalize)(void *xdpsq, bool sent, bool flush))
1057{
1058	u32 nxmit = 0;
1059
1060	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1061		return -EINVAL;
1062
1063	for (u32 i = 0; likely(i < n); i++) {
1064		u32 ret;
1065
1066		ret = libeth_xdp_xmit_queue_bulk(bq, frames[i], flush_bulk);
1067		if (unlikely(ret != LIBETH_XDP_TX)) {
1068			nxmit += ret == LIBETH_XDP_ABORTED;
1069			break;
1070		}
1071
1072		nxmit++;
1073	}
1074
1075	if (bq->count) {
1076		flush_bulk(bq, LIBETH_XDP_TX_NDO);
1077		if (unlikely(bq->count))
1078			nxmit -= libeth_xdp_xmit_return_bulk(bq->bulk,
1079							     bq->count,
1080							     bq->dev);
1081	}
1082
1083	finalize(bq->xdpsq, nxmit, flags & XDP_XMIT_FLUSH);
1084
1085	return nxmit;
1086}
1087
1088/**
1089 * libeth_xdp_xmit_do_bulk - implement full .ndo_xdp_xmit() in driver
1090 * @dev: target &net_device
1091 * @n: number of frames to send
1092 * @fr: XDP frames to send
1093 * @f: flags passed by the stack
1094 * @xqs: array of XDPSQs driver structs
1095 * @nqs: number of active XDPSQs, the above array length
1096 * @fl: driver callback to flush an XDP xmit bulk
1097 * @fin: driver cabback to finalize the queue
1098 *
1099 * If the driver has active XDPSQs, perform common checks and send the frames.
1100 * Finalize the queue, if requested.
1101 *
1102 * Return: number of frames sent or -errno on error.
1103 */
1104#define libeth_xdp_xmit_do_bulk(dev, n, fr, f, xqs, nqs, fl, fin)	      \
1105	_libeth_xdp_xmit_do_bulk(dev, n, fr, f, xqs, nqs, fl, fin,	      \
1106				 __UNIQUE_ID(bq_), __UNIQUE_ID(ret_),	      \
1107				 __UNIQUE_ID(nqs_))
1108
1109#define _libeth_xdp_xmit_do_bulk(d, n, fr, f, xqs, nqs, fl, fin, ub, ur, un)  \
1110({									      \
1111	u32 un = (nqs);							      \
1112	int ur;								      \
1113									      \
1114	if (likely(un)) {						      \
1115		LIBETH_XDP_ONSTACK_BULK(ub);				      \
1116									      \
1117		libeth_xdp_xmit_init_bulk(&ub, d, xqs, un);		      \
1118		ur = __libeth_xdp_xmit_do_bulk(&ub, fr, n, f, fl, fin);	      \
1119	} else {							      \
1120		ur = -ENXIO;						      \
1121	}								      \
1122									      \
1123	ur;								      \
1124})
1125
1126/* Rx polling path */
1127
1128/**
1129 * libeth_xdp_tx_init_bulk - initialize an XDP Tx bulk for Rx NAPI poll
1130 * @bq: bulk to initialize
1131 * @prog: RCU pointer to the XDP program (can be %NULL)
1132 * @dev: target &net_device
1133 * @xdpsqs: array of driver XDPSQ structs
1134 * @num: number of active XDPSQs, the above array length
1135 *
1136 * Should be called on an onstack XDP Tx bulk before the NAPI polling loop.
1137 * Initializes all the needed fields to run libeth_xdp functions. If @num == 0,
1138 * assumes XDP is not enabled.
1139 * Do not use for XSk, it has its own optimized helper.
1140 */
1141#define libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num)		      \
1142	__libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false,	      \
1143				  __UNIQUE_ID(bq_), __UNIQUE_ID(nqs_))
1144
1145#define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, ub, un) do {   \
1146	typeof(bq) ub = (bq);						      \
1147	u32 un = (num);							      \
1148									      \
1149	rcu_read_lock();						      \
1150									      \
1151	if (un || (xsk)) {						      \
1152		ub->prog = rcu_dereference(pr);				      \
1153		ub->dev = (d);						      \
1154		ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un)];		      \
1155	} else {							      \
1156		ub->prog = NULL;					      \
1157	}								      \
1158									      \
1159	ub->act_mask = 0;						      \
1160	ub->count = 0;							      \
1161} while (0)
1162
1163void libeth_xdp_load_stash(struct libeth_xdp_buff *dst,
1164			   const struct libeth_xdp_buff_stash *src);
1165void libeth_xdp_save_stash(struct libeth_xdp_buff_stash *dst,
1166			   const struct libeth_xdp_buff *src);
1167void __libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash);
1168
1169/**
1170 * libeth_xdp_init_buff - initialize a &libeth_xdp_buff for Rx NAPI poll
1171 * @dst: onstack buffer to initialize
1172 * @src: XDP buffer stash placed on the queue
1173 * @rxq: registered &xdp_rxq_info corresponding to this queue
1174 *
1175 * Should be called before the main NAPI polling loop. Loads the content of
1176 * the previously saved stash or initializes the buffer from scratch.
1177 * Do not use for XSk.
1178 */
1179static inline void
1180libeth_xdp_init_buff(struct libeth_xdp_buff *dst,
1181		     const struct libeth_xdp_buff_stash *src,
1182		     struct xdp_rxq_info *rxq)
1183{
1184	if (likely(!src->data))
1185		dst->data = NULL;
1186	else
1187		libeth_xdp_load_stash(dst, src);
1188
1189	dst->base.rxq = rxq;
1190}
1191
1192/**
1193 * libeth_xdp_save_buff - save a partially built buffer on a queue
1194 * @dst: XDP buffer stash placed on the queue
1195 * @src: onstack buffer to save
1196 *
1197 * Should be called after the main NAPI polling loop. If the loop exited before
1198 * the buffer was finished, saves its content on the queue, so that it can be
1199 * completed during the next poll. Otherwise, clears the stash.
1200 */
1201static inline void libeth_xdp_save_buff(struct libeth_xdp_buff_stash *dst,
1202					const struct libeth_xdp_buff *src)
1203{
1204	if (likely(!src->data))
1205		dst->data = NULL;
1206	else
1207		libeth_xdp_save_stash(dst, src);
1208}
1209
1210/**
1211 * libeth_xdp_return_stash - free an XDP buffer stash from a queue
1212 * @stash: stash to free
1213 *
1214 * If the queue is about to be destroyed, but it still has an incompleted
1215 * buffer stash, this helper should be called to free it.
1216 */
1217static inline void libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash)
1218{
1219	if (stash->data)
1220		__libeth_xdp_return_stash(stash);
1221}
1222
1223static inline void libeth_xdp_return_va(const void *data, bool napi)
1224{
1225	netmem_ref netmem = virt_to_netmem(data);
1226
1227	page_pool_put_full_netmem(__netmem_get_pp(netmem), netmem, napi);
1228}
1229
1230static inline void libeth_xdp_return_frags(const struct skb_shared_info *sinfo,
1231					   bool napi)
1232{
1233	for (u32 i = 0; i < sinfo->nr_frags; i++) {
1234		netmem_ref netmem = skb_frag_netmem(&sinfo->frags[i]);
1235
1236		page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, napi);
1237	}
1238}
1239
1240/**
1241 * libeth_xdp_return_buff - free/recycle &libeth_xdp_buff
1242 * @xdp: buffer to free
1243 *
1244 * Hotpath helper to free &libeth_xdp_buff. Comparing to xdp_return_buff(),
1245 * it's faster as it gets inlined and always assumes order-0 pages and safe
1246 * direct recycling. Zeroes @xdp->data to avoid UAFs.
1247 */
1248#define libeth_xdp_return_buff(xdp)	__libeth_xdp_return_buff(xdp, true)
1249
1250static inline void __libeth_xdp_return_buff(struct libeth_xdp_buff *xdp,
1251					    bool napi)
1252{
1253	if (!xdp_buff_has_frags(&xdp->base))
1254		goto out;
1255
1256	libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base),
1257				napi);
1258
1259out:
1260	libeth_xdp_return_va(xdp->data, napi);
1261	xdp->data = NULL;
1262}
1263
1264bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp,
1265			      const struct libeth_fqe *fqe,
1266			      u32 len);
1267
1268/**
1269 * libeth_xdp_prepare_buff - fill &libeth_xdp_buff with head FQE data
1270 * @xdp: XDP buffer to attach the head to
1271 * @fqe: FQE containing the head buffer
1272 * @len: buffer len passed from HW
1273 *
1274 * Internal, use libeth_xdp_process_buff() instead. Initializes XDP buffer
1275 * head with the Rx buffer data: data pointer, length, headroom, and
1276 * truesize/tailroom. Zeroes the flags.
1277 */
1278static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp,
1279					   const struct libeth_fqe *fqe,
1280					   u32 len)
1281{
1282	const struct page *page = __netmem_to_page(fqe->netmem);
1283
1284	xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset,
1285			 pp_page_to_nmdesc(page)->pp->p.offset, len, true);
1286	xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq);
1287}
1288
1289/**
1290 * libeth_xdp_process_buff - attach Rx buffer to &libeth_xdp_buff
1291 * @xdp: XDP buffer to attach the Rx buffer to
1292 * @fqe: Rx buffer to process
1293 * @len: received data length from the descriptor
1294 *
1295 * If the XDP buffer is empty, attaches the Rx buffer as head and initializes
1296 * the required fields. Otherwise, attaches the buffer as a frag.
1297 * Already performs DMA sync-for-CPU and frame start prefetch
1298 * (for head buffers only).
1299 *
1300 * Return: true on success, false if the descriptor must be skipped (empty or
1301 * no space for a new frag).
1302 */
1303static inline bool libeth_xdp_process_buff(struct libeth_xdp_buff *xdp,
1304					   const struct libeth_fqe *fqe,
1305					   u32 len)
1306{
1307	if (!libeth_rx_sync_for_cpu(fqe, len))
1308		return false;
1309
1310	if (xdp->data)
1311		return libeth_xdp_buff_add_frag(xdp, fqe, len);
1312
1313	libeth_xdp_prepare_buff(xdp, fqe, len);
1314
1315	prefetch(xdp->data);
1316
1317	return true;
1318}
1319
1320/**
1321 * libeth_xdp_buff_stats_frags - update onstack RQ stats with XDP frags info
1322 * @ss: onstack stats to update
1323 * @xdp: buffer to account
1324 *
1325 * Internal helper used by __libeth_xdp_run_pass(), do not call directly.
1326 * Adds buffer's frags count and total len to the onstack stats.
1327 */
1328static inline void
1329libeth_xdp_buff_stats_frags(struct libeth_rq_napi_stats *ss,
1330			    const struct libeth_xdp_buff *xdp)
1331{
1332	const struct skb_shared_info *sinfo;
1333
1334	sinfo = xdp_get_shared_info_from_buff(&xdp->base);
1335	ss->bytes += sinfo->xdp_frags_size;
1336	ss->fragments += sinfo->nr_frags + 1;
1337}
1338
1339u32 libeth_xdp_prog_exception(const struct libeth_xdp_tx_bulk *bq,
1340			      struct libeth_xdp_buff *xdp,
1341			      enum xdp_action act, int ret);
1342
1343/**
1344 * __libeth_xdp_run_prog - run XDP program on an XDP buffer
1345 * @xdp: XDP buffer to run the prog on
1346 * @bq: buffer bulk for ``XDP_TX`` queueing
1347 *
1348 * Internal inline abstraction to run XDP program. Handles ``XDP_DROP``
1349 * and ``XDP_REDIRECT`` only, the rest is processed levels up.
1350 * Reports an XDP prog exception on errors.
1351 *
1352 * Return: libeth_xdp prog verdict depending on the prog's verdict.
1353 */
1354static __always_inline u32
1355__libeth_xdp_run_prog(struct libeth_xdp_buff *xdp,
1356		      const struct libeth_xdp_tx_bulk *bq)
1357{
1358	enum xdp_action act;
1359
1360	act = bpf_prog_run_xdp(bq->prog, &xdp->base);
1361	if (unlikely(act < XDP_DROP || act > XDP_REDIRECT))
1362		goto out;
1363
1364	switch (act) {
1365	case XDP_PASS:
1366		return LIBETH_XDP_PASS;
1367	case XDP_DROP:
1368		libeth_xdp_return_buff(xdp);
1369
1370		return LIBETH_XDP_DROP;
1371	case XDP_TX:
1372		return LIBETH_XDP_TX;
1373	case XDP_REDIRECT:
1374		if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog)))
1375			break;
1376
1377		xdp->data = NULL;
1378
1379		return LIBETH_XDP_REDIRECT;
1380	default:
1381		break;
1382	}
1383
1384out:
1385	return libeth_xdp_prog_exception(bq, xdp, act, 0);
1386}
1387
1388/**
1389 * __libeth_xdp_run_flush - run XDP program and handle ``XDP_TX`` verdict
1390 * @xdp: XDP buffer to run the prog on
1391 * @bq: buffer bulk for ``XDP_TX`` queueing
1392 * @run: internal callback for running XDP program
1393 * @queue: internal callback for queuing ``XDP_TX`` frame
1394 * @flush_bulk: driver callback for flushing a bulk
1395 *
1396 * Internal inline abstraction to run XDP program and additionally handle
1397 * ``XDP_TX`` verdict. Used by both XDP and XSk, hence @run and @queue.
1398 * Do not use directly.
1399 *
1400 * Return: libeth_xdp prog verdict depending on the prog's verdict.
1401 */
1402static __always_inline u32
1403__libeth_xdp_run_flush(struct libeth_xdp_buff *xdp,
1404		       struct libeth_xdp_tx_bulk *bq,
1405		       u32 (*run)(struct libeth_xdp_buff *xdp,
1406				  const struct libeth_xdp_tx_bulk *bq),
1407		       bool (*queue)(struct libeth_xdp_tx_bulk *bq,
1408				     struct libeth_xdp_buff *xdp,
1409				     bool (*flush_bulk)
1410					  (struct libeth_xdp_tx_bulk *bq,
1411					   u32 flags)),
1412		       bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1413					  u32 flags))
1414{
1415	u32 act;
1416
1417	act = run(xdp, bq);
1418	if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk)))
1419		act = LIBETH_XDP_DROP;
1420
1421	bq->act_mask |= act;
1422
1423	return act;
1424}
1425
1426/**
1427 * libeth_xdp_run_prog - run XDP program (non-XSk path) and handle all verdicts
1428 * @xdp: XDP buffer to process
1429 * @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
1430 * @fl: driver ``XDP_TX`` bulk flush callback
1431 *
1432 * Run the attached XDP program and handle all possible verdicts. XSk has its
1433 * own version.
1434 * Prefer using it via LIBETH_XDP_DEFINE_RUN{,_PASS,_PROG}().
1435 *
1436 * Return: true if the buffer should be passed up the stack, false if the poll
1437 * should go to the next buffer.
1438 */
1439#define libeth_xdp_run_prog(xdp, bq, fl)				      \
1440	(__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog,		      \
1441				libeth_xdp_tx_queue_bulk,		      \
1442				fl) == LIBETH_XDP_PASS)
1443
1444/**
1445 * __libeth_xdp_run_pass - helper to run XDP program and handle the result
1446 * @xdp: XDP buffer to process
1447 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1448 * @napi: NAPI to build an skb and pass it up the stack
1449 * @rs: onstack libeth RQ stats
1450 * @md: metadata that should be filled to the XDP buffer
1451 * @prep: callback for filling the metadata
1452 * @run: driver wrapper to run XDP program
1453 * @populate: driver callback to populate an skb with the HW descriptor data
1454 *
1455 * Inline abstraction that does the following (non-XSk path):
1456 * 1) adds frame size and frag number (if needed) to the onstack stats;
1457 * 2) fills the descriptor metadata to the onstack &libeth_xdp_buff
1458 * 3) runs XDP program if present;
1459 * 4) handles all possible verdicts;
1460 * 5) on ``XDP_PASS`, builds an skb from the buffer;
1461 * 6) populates it with the descriptor metadata;
1462 * 7) passes it up the stack.
1463 *
1464 * In most cases, number 2 means just writing the pointer to the HW descriptor
1465 * to the XDP buffer. If so, please use LIBETH_XDP_DEFINE_RUN{,_PASS}()
1466 * wrappers to build a driver function.
1467 */
1468static __always_inline void
1469__libeth_xdp_run_pass(struct libeth_xdp_buff *xdp,
1470		      struct libeth_xdp_tx_bulk *bq, struct napi_struct *napi,
1471		      struct libeth_rq_napi_stats *rs, const void *md,
1472		      void (*prep)(struct libeth_xdp_buff *xdp,
1473				   const void *md),
1474		      bool (*run)(struct libeth_xdp_buff *xdp,
1475				  struct libeth_xdp_tx_bulk *bq),
1476		      bool (*populate)(struct sk_buff *skb,
1477				       const struct libeth_xdp_buff *xdp,
1478				       struct libeth_rq_napi_stats *rs))
1479{
1480	struct sk_buff *skb;
1481
1482	rs->bytes += xdp->base.data_end - xdp->data;
1483	rs->packets++;
1484
1485	if (xdp_buff_has_frags(&xdp->base))
1486		libeth_xdp_buff_stats_frags(rs, xdp);
1487
1488	if (prep && (!__builtin_constant_p(!!md) || md))
1489		prep(xdp, md);
1490
1491	if (!bq || !run || !bq->prog)
1492		goto build;
1493
1494	if (!run(xdp, bq))
1495		return;
1496
1497build:
1498	skb = xdp_build_skb_from_buff(&xdp->base);
1499	if (unlikely(!skb)) {
1500		libeth_xdp_return_buff_slow(xdp);
1501		return;
1502	}
1503
1504	xdp->data = NULL;
1505
1506	if (unlikely(!populate(skb, xdp, rs))) {
1507		napi_consume_skb(skb, true);
1508		return;
1509	}
1510
1511	napi_gro_receive(napi, skb);
1512}
1513
1514static inline void libeth_xdp_prep_desc(struct libeth_xdp_buff *xdp,
1515					const void *desc)
1516{
1517	xdp->desc = desc;
1518}
1519
1520/**
1521 * libeth_xdp_run_pass - helper to run XDP program and handle the result
1522 * @xdp: XDP buffer to process
1523 * @bq: XDP Tx bulk to queue ``XDP_TX`` frames
1524 * @napi: NAPI to build an skb and pass it up the stack
1525 * @ss: onstack libeth RQ stats
1526 * @desc: pointer to the HW descriptor for that frame
1527 * @run: driver wrapper to run XDP program
1528 * @populate: driver callback to populate an skb with the HW descriptor data
1529 *
1530 * Wrapper around the underscored version when "fill the descriptor metadata"
1531 * means just writing the pointer to the HW descriptor as @xdp->desc.
1532 */
1533#define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate)	      \
1534	__libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc,  \
1535			      run, populate)
1536
1537/**
1538 * libeth_xdp_finalize_rx - finalize XDPSQ after a NAPI polling loop (non-XSk)
1539 * @bq: ``XDP_TX`` frame bulk
1540 * @flush: driver callback to flush the bulk
1541 * @finalize: driver callback to start sending the frames and run the timer
1542 *
1543 * Flush the bulk if there are frames left to send, kick the queue and flush
1544 * the XDP maps.
1545 */
1546#define libeth_xdp_finalize_rx(bq, flush, finalize)			      \
1547	__libeth_xdp_finalize_rx(bq, 0, flush, finalize)
1548
1549static __always_inline void
1550__libeth_xdp_finalize_rx(struct libeth_xdp_tx_bulk *bq, u32 flags,
1551			 bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
1552					    u32 flags),
1553			 void (*finalize)(void *xdpsq, bool sent, bool flush))
1554{
1555	if (bq->act_mask & LIBETH_XDP_TX) {
1556		if (bq->count)
1557			flush_bulk(bq, flags | LIBETH_XDP_TX_DROP);
1558		finalize(bq->xdpsq, true, true);
1559	}
1560	if (bq->act_mask & LIBETH_XDP_REDIRECT)
1561		xdp_do_flush();
1562
1563	rcu_read_unlock();
1564}
1565
1566/*
1567 * Helpers to reduce boilerplate code in drivers.
1568 *
1569 * Typical driver Rx flow would be (excl. bulk and buff init, frag attach):
1570 *
1571 * LIBETH_XDP_DEFINE_START();
1572 * LIBETH_XDP_DEFINE_FLUSH_TX(static driver_xdp_flush_tx, driver_xdp_tx_prep,
1573 *			      driver_xdp_xmit);
1574 * LIBETH_XDP_DEFINE_RUN(static driver_xdp_run, driver_xdp_run_prog,
1575 *			 driver_xdp_flush_tx, driver_populate_skb);
1576 * LIBETH_XDP_DEFINE_FINALIZE(static driver_xdp_finalize_rx,
1577 *			      driver_xdp_flush_tx, driver_xdp_finalize_sq);
1578 * LIBETH_XDP_DEFINE_END();
1579 *
1580 * This will build a set of 4 static functions. The compiler is free to decide
1581 * whether to inline them.
1582 * Then, in the NAPI polling function:
1583 *
1584 *	while (packets < budget) {
1585 *		// ...
1586 *		driver_xdp_run(xdp, &bq, napi, &rs, desc);
1587 *	}
1588 *	driver_xdp_finalize_rx(&bq);
1589 */
1590
1591#define LIBETH_XDP_DEFINE_START()					      \
1592	__diag_push();							      \
1593	__diag_ignore(GCC, 8, "-Wold-style-declaration",		      \
1594		      "Allow specifying \'static\' after the return type")
1595
1596/**
1597 * LIBETH_XDP_DEFINE_TIMER - define a driver XDPSQ cleanup timer callback
1598 * @name: name of the function to define
1599 * @poll: Tx polling/completion function
1600 */
1601#define LIBETH_XDP_DEFINE_TIMER(name, poll)				      \
1602void name(struct work_struct *work)					      \
1603{									      \
1604	libeth_xdpsq_run_timer(work, poll);				      \
1605}
1606
1607/**
1608 * LIBETH_XDP_DEFINE_FLUSH_TX - define a driver ``XDP_TX`` bulk flush function
1609 * @name: name of the function to define
1610 * @prep: driver callback to clean an XDPSQ
1611 * @xmit: driver callback to write a HW Tx descriptor
1612 */
1613#define LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit)			      \
1614	__LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp)
1615
1616#define __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, pfx)		      \
1617bool name(struct libeth_xdp_tx_bulk *bq, u32 flags)			      \
1618{									      \
1619	return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit);	      \
1620}
1621
1622/**
1623 * LIBETH_XDP_DEFINE_FLUSH_XMIT - define a driver XDP xmit bulk flush function
1624 * @name: name of the function to define
1625 * @prep: driver callback to clean an XDPSQ
1626 * @xmit: driver callback to write a HW Tx descriptor
1627 */
1628#define LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit)			      \
1629bool name(struct libeth_xdp_tx_bulk *bq, u32 flags)			      \
1630{									      \
1631	return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit);	      \
1632}
1633
1634/**
1635 * LIBETH_XDP_DEFINE_RUN_PROG - define a driver XDP program run function
1636 * @name: name of the function to define
1637 * @flush: driver callback to flush an ``XDP_TX`` bulk
1638 */
1639#define LIBETH_XDP_DEFINE_RUN_PROG(name, flush)				      \
1640	bool __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xdp)
1641
1642#define __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, pfx)			      \
1643name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq)	      \
1644{									      \
1645	return libeth_##pfx##_run_prog(xdp, bq, flush);			      \
1646}
1647
1648/**
1649 * LIBETH_XDP_DEFINE_RUN_PASS - define a driver buffer process + pass function
1650 * @name: name of the function to define
1651 * @run: driver callback to run XDP program (above)
1652 * @populate: driver callback to fill an skb with HW descriptor info
1653 */
1654#define LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate)			      \
1655	void __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xdp)
1656
1657#define __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, pfx)		      \
1658name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq,	      \
1659     struct napi_struct *napi, struct libeth_rq_napi_stats *ss,		      \
1660     const void *desc)							      \
1661{									      \
1662	return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run,	      \
1663				       populate);			      \
1664}
1665
1666/**
1667 * LIBETH_XDP_DEFINE_RUN - define a driver buffer process, run + pass function
1668 * @name: name of the function to define
1669 * @run: name of the XDP prog run function to define
1670 * @flush: driver callback to flush an ``XDP_TX`` bulk
1671 * @populate: driver callback to fill an skb with HW descriptor info
1672 */
1673#define LIBETH_XDP_DEFINE_RUN(name, run, flush, populate)		      \
1674	__LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, XDP)
1675
1676#define __LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, pfx)	      \
1677	LIBETH_##pfx##_DEFINE_RUN_PROG(static run, flush);		      \
1678	LIBETH_##pfx##_DEFINE_RUN_PASS(name, run, populate)
1679
1680/**
1681 * LIBETH_XDP_DEFINE_FINALIZE - define a driver Rx NAPI poll finalize function
1682 * @name: name of the function to define
1683 * @flush: driver callback to flush an ``XDP_TX`` bulk
1684 * @finalize: driver callback to finalize an XDPSQ and run the timer
1685 */
1686#define LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize)		      \
1687	__LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xdp)
1688
1689#define __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, pfx)	      \
1690void name(struct libeth_xdp_tx_bulk *bq)				      \
1691{									      \
1692	libeth_##pfx##_finalize_rx(bq, flush, finalize);		      \
1693}
1694
1695#define LIBETH_XDP_DEFINE_END()		__diag_pop()
1696
1697/* XMO */
1698
1699/**
1700 * libeth_xdp_buff_to_rq - get RQ pointer from an XDP buffer pointer
1701 * @xdp: &libeth_xdp_buff corresponding to the queue
1702 * @type: typeof() of the driver Rx queue structure
1703 * @member: name of &xdp_rxq_info inside @type
1704 *
1705 * Often times, pointer to the RQ is needed when reading/filling metadata from
1706 * HW descriptors. The helper can be used to quickly jump from an XDP buffer
1707 * to the queue corresponding to its &xdp_rxq_info without introducing
1708 * additional fields (&libeth_xdp_buff is precisely 1 cacheline long on x64).
1709 */
1710#define libeth_xdp_buff_to_rq(xdp, type, member)			      \
1711	container_of_const((xdp)->base.rxq, type, member)
1712
1713/**
1714 * libeth_xdpmo_rx_hash - convert &libeth_rx_pt to an XDP RSS hash metadata
1715 * @hash: pointer to the variable to write the hash to
1716 * @rss_type: pointer to the variable to write the hash type to
1717 * @val: hash value from the HW descriptor
1718 * @pt: libeth parsed packet type
1719 *
1720 * Handle zeroed/non-available hash and convert libeth parsed packet type to
1721 * the corresponding XDP RSS hash type. To be called at the end of
1722 * xdp_metadata_ops idpf_xdpmo::xmo_rx_hash() implementation.
1723 * Note that if the driver doesn't use a constant packet type lookup table but
1724 * generates it at runtime, it must call libeth_rx_pt_gen_hash_type(pt) to
1725 * generate XDP RSS hash type for each packet type.
1726 *
1727 * Return: 0 on success, -ENODATA when the hash is not available.
1728 */
1729static inline int libeth_xdpmo_rx_hash(u32 *hash,
1730				       enum xdp_rss_hash_type *rss_type,
1731				       u32 val, struct libeth_rx_pt pt)
1732{
1733	if (unlikely(!val))
1734		return -ENODATA;
1735
1736	*hash = val;
1737	*rss_type = pt.hash_type;
1738
1739	return 0;
1740}
1741
1742/* Tx buffer completion */
1743
1744void libeth_xdp_return_buff_bulk(const struct skb_shared_info *sinfo,
1745				 struct xdp_frame_bulk *bq, bool frags);
1746void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp);
1747
1748/**
1749 * __libeth_xdp_complete_tx - complete sent XDPSQE
1750 * @sqe: SQ element / Tx buffer to complete
1751 * @cp: Tx polling/completion params
1752 * @bulk: internal callback to bulk-free ``XDP_TX`` buffers
1753 * @xsk: internal callback to free XSk ``XDP_TX`` buffers
1754 *
1755 * Use the non-underscored version in drivers instead. This one is shared
1756 * internally with libeth_tx_complete_any().
1757 * Complete an XDPSQE of any type of XDP frame. This includes DMA unmapping
1758 * when needed, buffer freeing, stats update, and SQE invalidation.
1759 */
1760static __always_inline void
1761__libeth_xdp_complete_tx(struct libeth_sqe *sqe, struct libeth_cq_pp *cp,
1762			 typeof(libeth_xdp_return_buff_bulk) bulk,
1763			 typeof(libeth_xsk_buff_free_slow) xsk)
1764{
1765	enum libeth_sqe_type type = sqe->type;
1766
1767	switch (type) {
1768	case LIBETH_SQE_EMPTY:
1769		return;
1770	case LIBETH_SQE_XDP_XMIT:
1771	case LIBETH_SQE_XDP_XMIT_FRAG:
1772		dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma),
1773			       dma_unmap_len(sqe, len), DMA_TO_DEVICE);
1774		break;
1775	default:
1776		break;
1777	}
1778
1779	switch (type) {
1780	case LIBETH_SQE_XDP_TX:
1781		bulk(sqe->sinfo, cp->bq, sqe->nr_frags != 1);
1782		break;
1783	case LIBETH_SQE_XDP_XMIT:
1784		xdp_return_frame_bulk(sqe->xdpf, cp->bq);
1785		break;
1786	case LIBETH_SQE_XSK_TX:
1787	case LIBETH_SQE_XSK_TX_FRAG:
1788		xsk(sqe->xsk);
1789		break;
1790	default:
1791		break;
1792	}
1793
1794	switch (type) {
1795	case LIBETH_SQE_XDP_TX:
1796	case LIBETH_SQE_XDP_XMIT:
1797	case LIBETH_SQE_XSK_TX:
1798		cp->xdp_tx -= sqe->nr_frags;
1799
1800		cp->xss->packets++;
1801		cp->xss->bytes += sqe->bytes;
1802		break;
1803	default:
1804		break;
1805	}
1806
1807	sqe->type = LIBETH_SQE_EMPTY;
1808}
1809
1810static inline void libeth_xdp_complete_tx(struct libeth_sqe *sqe,
1811					  struct libeth_cq_pp *cp)
1812{
1813	__libeth_xdp_complete_tx(sqe, cp, libeth_xdp_return_buff_bulk,
1814				 libeth_xsk_buff_free_slow);
1815}
1816
1817/* Misc */
1818
1819u32 libeth_xdp_queue_threshold(u32 count);
1820
1821void __libeth_xdp_set_features(struct net_device *dev,
1822			       const struct xdp_metadata_ops *xmo,
1823			       u32 zc_segs,
1824			       const struct xsk_tx_metadata_ops *tmo);
1825void libeth_xdp_set_redirect(struct net_device *dev, bool enable);
1826
1827/**
1828 * libeth_xdp_set_features - set XDP features for netdev
1829 * @dev: &net_device to configure
1830 * @...: optional params, see __libeth_xdp_set_features()
1831 *
1832 * Set all the features libeth_xdp supports, including .ndo_xdp_xmit(). That
1833 * said, it should be used only when XDPSQs are always available regardless
1834 * of whether an XDP prog is attached to @dev.
1835 */
1836#define libeth_xdp_set_features(dev, ...)				      \
1837	CONCATENATE(__libeth_xdp_feat,					      \
1838		    COUNT_ARGS(__VA_ARGS__))(dev, ##__VA_ARGS__)
1839
1840#define __libeth_xdp_feat0(dev)						      \
1841	__libeth_xdp_set_features(dev, NULL, 0, NULL)
1842#define __libeth_xdp_feat1(dev, xmo)					      \
1843	__libeth_xdp_set_features(dev, xmo, 0, NULL)
1844#define __libeth_xdp_feat2(dev, xmo, zc_segs)				      \
1845	__libeth_xdp_set_features(dev, xmo, zc_segs, NULL)
1846#define __libeth_xdp_feat3(dev, xmo, zc_segs, tmo)			      \
1847	__libeth_xdp_set_features(dev, xmo, zc_segs, tmo)
1848
1849/**
1850 * libeth_xdp_set_features_noredir - enable all libeth_xdp features w/o redir
1851 * @dev: target &net_device
1852 * @...: optional params, see __libeth_xdp_set_features()
1853 *
1854 * Enable everything except the .ndo_xdp_xmit() feature, use when XDPSQs are
1855 * not available right after netdev registration.
1856 */
1857#define libeth_xdp_set_features_noredir(dev, ...)			      \
1858	__libeth_xdp_set_features_noredir(dev, __UNIQUE_ID(dev_),	      \
1859					  ##__VA_ARGS__)
1860
1861#define __libeth_xdp_set_features_noredir(dev, ud, ...) do {		      \
1862	struct net_device *ud = (dev);					      \
1863									      \
1864	libeth_xdp_set_features(ud, ##__VA_ARGS__);			      \
1865	libeth_xdp_set_redirect(ud, false);				      \
1866} while (0)
1867
1868#define libeth_xsktmo			((const void *)GOLDEN_RATIO_PRIME)
1869
1870#endif /* __LIBETH_XDP_H */