xdp_sock.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2/* AF_XDP internal functions
  3 * Copyright(c) 2018 Intel Corporation.
  4 */
  5
  6#ifndef _LINUX_XDP_SOCK_H
  7#define _LINUX_XDP_SOCK_H
  8
  9#include <linux/bpf.h>
 10#include <linux/workqueue.h>
 11#include <linux/if_xdp.h>
 12#include <linux/mutex.h>
 13#include <linux/spinlock.h>
 14#include <linux/mm.h>
 15#include <net/sock.h>
 16
 17#define XDP_UMEM_SG_FLAG BIT(3)
 18
 19struct net_device;
 20struct xsk_queue;
 21struct xdp_buff;
 22
 23struct xdp_umem {
 24	void *addrs;
 25	u64 size;
 26	u32 headroom;
 27	u32 chunk_size;
 28	u32 chunks;
 29	u32 npgs;
 30	struct user_struct *user;
 31	refcount_t users;
 32	u8 flags;
 33	u8 tx_metadata_len;
 34	bool zc;
 35	struct page **pgs;
 36	int id;
 37	struct list_head xsk_dma_list;
 38	struct work_struct work;
 39};
 40
 41struct xsk_map {
 42	struct bpf_map map;
 43	spinlock_t lock; /* Synchronize map updates */
 44	atomic_t count;
 45	struct xdp_sock __rcu *xsk_map[];
 46};
 47
 48struct xdp_sock {
 49	/* struct sock must be the first member of struct xdp_sock */
 50	struct sock sk;
 51	struct xsk_queue *rx ____cacheline_aligned_in_smp;
 52	struct net_device *dev;
 53	struct xdp_umem *umem;
 54	struct list_head flush_node;
 55	struct xsk_buff_pool *pool;
 56	u16 queue_id;
 57	bool zc;
 58	bool sg;
 59	enum {
 60		XSK_READY = 0,
 61		XSK_BOUND,
 62		XSK_UNBOUND,
 63	} state;
 64
 65	struct xsk_queue *tx ____cacheline_aligned_in_smp;
 66	struct list_head tx_list;
 67	/* record the number of tx descriptors sent by this xsk and
 68	 * when it exceeds MAX_PER_SOCKET_BUDGET, an opportunity needs
 69	 * to be given to other xsks for sending tx descriptors, thereby
 70	 * preventing other XSKs from being starved.
 71	 */
 72	u32 tx_budget_spent;
 73
 74	/* Statistics */
 75	u64 rx_dropped;
 76	u64 rx_queue_full;
 77
 78	/* When __xsk_generic_xmit() must return before it sees the EOP descriptor for the current
 79	 * packet, the partially built skb is saved here so that packet building can resume in next
 80	 * call of __xsk_generic_xmit().
 81	 */
 82	struct sk_buff *skb;
 83
 84	struct list_head map_list;
 85	/* Protects map_list */
 86	spinlock_t map_list_lock;
 87	u32 max_tx_budget;
 88	/* Protects multiple processes in the control path */
 89	struct mutex mutex;
 90	struct xsk_queue *fq_tmp; /* Only as tmp storage before bind */
 91	struct xsk_queue *cq_tmp; /* Only as tmp storage before bind */
 92};
 93
 94/*
 95 * AF_XDP TX metadata hooks for network devices.
 96 * The following hooks can be defined; unless noted otherwise, they are
 97 * optional and can be filled with a null pointer.
 98 *
 99 * void (*tmo_request_timestamp)(void *priv)
100 *     Called when AF_XDP frame requested egress timestamp.
101 *
102 * u64 (*tmo_fill_timestamp)(void *priv)
103 *     Called when AF_XDP frame, that had requested egress timestamp,
104 *     received a completion. The hook needs to return the actual HW timestamp.
105 *
106 * void (*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv)
107 *     Called when AF_XDP frame requested HW checksum offload. csum_start
108 *     indicates position where checksumming should start.
109 *     csum_offset indicates position where checksum should be stored.
110 *
111 * void (*tmo_request_launch_time)(u64 launch_time, void *priv)
112 *     Called when AF_XDP frame requested launch time HW offload support.
113 *     launch_time indicates the PTP time at which the device can schedule the
114 *     packet for transmission.
115 */
116struct xsk_tx_metadata_ops {
117	void	(*tmo_request_timestamp)(void *priv);
118	u64	(*tmo_fill_timestamp)(void *priv);
119	void	(*tmo_request_checksum)(u16 csum_start, u16 csum_offset, void *priv);
120	void	(*tmo_request_launch_time)(u64 launch_time, void *priv);
121};
122
123#ifdef CONFIG_XDP_SOCKETS
124
125int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
126int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp);
127void __xsk_map_flush(struct list_head *flush_list);
128INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *));
129
130/**
131 *  xsk_tx_metadata_to_compl - Save enough relevant metadata information
132 *  to perform tx completion in the future.
133 *  @meta: pointer to AF_XDP metadata area
134 *  @compl: pointer to output struct xsk_tx_metadata_to_compl
135 *
136 *  This function should be called by the networking device when
137 *  it prepares AF_XDP egress packet. The value of @compl should be stored
138 *  and passed to xsk_tx_metadata_complete upon TX completion.
139 */
140static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
141					    struct xsk_tx_metadata_compl *compl)
142{
143	if (!meta)
144		return;
145
146	if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
147		compl->tx_timestamp = &meta->completion.tx_timestamp;
148	else
149		compl->tx_timestamp = NULL;
150}
151
152/**
153 *  xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission
154 *  and call appropriate xsk_tx_metadata_ops operation.
155 *  @meta: pointer to AF_XDP metadata area
156 *  @ops: pointer to struct xsk_tx_metadata_ops
157 *  @priv: pointer to driver-private aread
158 *
159 *  This function should be called by the networking device when
160 *  it prepares AF_XDP egress packet.
161 */
162static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta,
163					   const struct xsk_tx_metadata_ops *ops,
164					   void *priv)
165{
166	if (!meta)
167		return;
168
169	if (ops->tmo_request_launch_time)
170		if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
171			ops->tmo_request_launch_time(meta->request.launch_time,
172						     priv);
173
174	if (ops->tmo_request_timestamp)
175		if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP)
176			ops->tmo_request_timestamp(priv);
177
178	if (ops->tmo_request_checksum)
179		if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM)
180			ops->tmo_request_checksum(meta->request.csum_start,
181						  meta->request.csum_offset, priv);
182}
183
184/**
185 *  xsk_tx_metadata_complete - Evaluate AF_XDP TX metadata at completion
186 *  and call appropriate xsk_tx_metadata_ops operation.
187 *  @compl: pointer to completion metadata produced from xsk_tx_metadata_to_compl
188 *  @ops: pointer to struct xsk_tx_metadata_ops
189 *  @priv: pointer to driver-private aread
190 *
191 *  This function should be called by the networking device upon
192 *  AF_XDP egress completion.
193 */
194static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
195					    const struct xsk_tx_metadata_ops *ops,
196					    void *priv)
197{
198	if (!compl)
199		return;
200	if (!compl->tx_timestamp)
201		return;
202
203	*compl->tx_timestamp = ops->tmo_fill_timestamp(priv);
204}
205
206#else
207
208static inline int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
209{
210	return -ENOTSUPP;
211}
212
213static inline int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
214{
215	return -EOPNOTSUPP;
216}
217
218static inline void __xsk_map_flush(struct list_head *flush_list)
219{
220}
221
222#ifdef CONFIG_MITIGATION_RETPOLINE
223static inline void xsk_destruct_skb(struct sk_buff *skb)
224{
225}
226#endif
227
228static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta,
229					    struct xsk_tx_metadata_compl *compl)
230{
231}
232
233static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta,
234					   const struct xsk_tx_metadata_ops *ops,
235					   void *priv)
236{
237}
238
239static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl,
240					    const struct xsk_tx_metadata_ops *ops,
241					    void *priv)
242{
243}
244
245#endif /* CONFIG_XDP_SOCKETS */
246#endif /* _LINUX_XDP_SOCK_H */