netmem.h

  1/* SPDX-License-Identifier: GPL-2.0
  2 *
  3 *	Network memory
  4 *
  5 *	Author:	Mina Almasry <almasrymina@google.com>
  6 */
  7
  8#ifndef _NET_NETMEM_H
  9#define _NET_NETMEM_H
 10
 11#include <linux/dma-mapping.h>
 12#include <linux/mm.h>
 13#include <net/net_debug.h>
 14
 15/* These fields in struct page are used by the page_pool and net stack:
 16 *
 17 *        struct {
 18 *                unsigned long pp_magic;
 19 *                struct page_pool *pp;
 20 *                unsigned long _pp_mapping_pad;
 21 *                unsigned long dma_addr;
 22 *                atomic_long_t pp_ref_count;
 23 *        };
 24 *
 25 * We mirror the page_pool fields here so the page_pool can access these
 26 * fields without worrying whether the underlying fields belong to a
 27 * page or netmem_desc.
 28 *
 29 * CAUTION: Do not update the fields in netmem_desc without also
 30 * updating the anonymous aliasing union in struct net_iov.
 31 */
 32struct netmem_desc {
 33	unsigned long _flags;
 34	unsigned long pp_magic;
 35	struct page_pool *pp;
 36	unsigned long _pp_mapping_pad;
 37	unsigned long dma_addr;
 38	atomic_long_t pp_ref_count;
 39};
 40
 41#define NETMEM_DESC_ASSERT_OFFSET(pg, desc)        \
 42	static_assert(offsetof(struct page, pg) == \
 43		      offsetof(struct netmem_desc, desc))
 44NETMEM_DESC_ASSERT_OFFSET(flags, _flags);
 45NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic);
 46NETMEM_DESC_ASSERT_OFFSET(pp, pp);
 47NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad);
 48NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr);
 49NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
 50#undef NETMEM_DESC_ASSERT_OFFSET
 51
 52/*
 53 * Since struct netmem_desc uses the space in struct page, the size
 54 * should be checked, until struct netmem_desc has its own instance from
 55 * slab, to avoid conflicting with other members within struct page.
 56 */
 57static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount));
 58
 59/* net_iov */
 60
 61DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers);
 62
 63/*  We overload the LSB of the struct page pointer to indicate whether it's
 64 *  a page or net_iov.
 65 */
 66#define NET_IOV 0x01UL
 67
 68enum net_iov_type {
 69	NET_IOV_DMABUF,
 70	NET_IOV_IOURING,
 71};
 72
 73/* A memory descriptor representing abstract networking I/O vectors,
 74 * generally for non-pages memory that doesn't have its corresponding
 75 * struct page and needs to be explicitly allocated through slab.
 76 *
 77 * net_iovs are allocated and used by networking code, and the size of
 78 * the chunk is PAGE_SIZE.
 79 *
 80 * This memory can be any form of non-struct paged memory.  Examples
 81 * include imported dmabuf memory and imported io_uring memory.  See
 82 * net_iov_type for all the supported types.
 83 *
 84 * @pp_magic:	pp field, similar to the one in struct page/struct
 85 *		netmem_desc.
 86 * @pp:		the pp this net_iov belongs to, if any.
 87 * @dma_addr:	the dma addrs of the net_iov. Needed for the network
 88 *		card to send/receive this net_iov.
 89 * @pp_ref_count: the pp ref count of this net_iov, exactly the same
 90 *		usage as struct page/struct netmem_desc.
 91 * @owner:	the net_iov_area this net_iov belongs to, if any.
 92 * @type:	the type of the memory.  Different types of net_iovs are
 93 *		supported.
 94 */
 95struct net_iov {
 96	union {
 97		struct netmem_desc desc;
 98
 99		/* XXX: The following part should be removed once all
100		 * the references to them are converted so as to be
101		 * accessed via netmem_desc e.g. niov->desc.pp instead
102		 * of niov->pp.
103		 */
104		struct {
105			unsigned long _flags;
106			unsigned long pp_magic;
107			struct page_pool *pp;
108			unsigned long _pp_mapping_pad;
109			unsigned long dma_addr;
110			atomic_long_t pp_ref_count;
111		};
112	};
113	struct net_iov_area *owner;
114	enum net_iov_type type;
115};
116
117struct net_iov_area {
118	/* Array of net_iovs for this area. */
119	struct net_iov *niovs;
120	size_t num_niovs;
121
122	/* Offset into the dma-buf where this chunk starts.  */
123	unsigned long base_virtual;
124};
125
126/* net_iov is union'ed with struct netmem_desc mirroring struct page, so
127 * the page_pool can access these fields without worrying whether the
128 * underlying fields are accessed via netmem_desc or directly via
129 * net_iov, until all the references to them are converted so as to be
130 * accessed via netmem_desc e.g. niov->desc.pp instead of niov->pp.
131 *
132 * The non-net stack fields of struct page are private to the mm stack
133 * and must never be mirrored to net_iov.
134 */
135#define NET_IOV_ASSERT_OFFSET(desc, iov)                    \
136	static_assert(offsetof(struct netmem_desc, desc) == \
137		      offsetof(struct net_iov, iov))
138NET_IOV_ASSERT_OFFSET(_flags, _flags);
139NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic);
140NET_IOV_ASSERT_OFFSET(pp, pp);
141NET_IOV_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad);
142NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr);
143NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count);
144#undef NET_IOV_ASSERT_OFFSET
145
146static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov)
147{
148	return niov->owner;
149}
150
151static inline unsigned int net_iov_idx(const struct net_iov *niov)
152{
153	return niov - net_iov_owner(niov)->niovs;
154}
155
156/* netmem */
157
158/**
159 * typedef netmem_ref - a nonexistent type marking a reference to generic
160 * network memory.
161 *
162 * A netmem_ref can be a struct page* or a struct net_iov* underneath.
163 *
164 * Use the supplied helpers to obtain the underlying memory pointer and fields.
165 */
166typedef unsigned long __bitwise netmem_ref;
167
168static inline bool netmem_is_net_iov(const netmem_ref netmem)
169{
170	return (__force unsigned long)netmem & NET_IOV;
171}
172
173/**
174 * __netmem_to_page - unsafely get pointer to the &page backing @netmem
175 * @netmem: netmem reference to convert
176 *
177 * Unsafe version of netmem_to_page(). When @netmem is always page-backed,
178 * e.g. when it's a header buffer, performs faster and generates smaller
179 * object code (no check for the LSB, no WARN). When @netmem points to IOV,
180 * provokes undefined behaviour.
181 *
182 * Return: pointer to the &page (garbage if @netmem is not page-backed).
183 */
184static inline struct page *__netmem_to_page(netmem_ref netmem)
185{
186	return (__force struct page *)netmem;
187}
188
189static inline struct page *netmem_to_page(netmem_ref netmem)
190{
191	if (WARN_ON_ONCE(netmem_is_net_iov(netmem)))
192		return NULL;
193
194	return __netmem_to_page(netmem);
195}
196
197static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem)
198{
199	if (netmem_is_net_iov(netmem))
200		return (struct net_iov *)((__force unsigned long)netmem &
201					  ~NET_IOV);
202
203	DEBUG_NET_WARN_ON_ONCE(true);
204	return NULL;
205}
206
207static inline netmem_ref net_iov_to_netmem(struct net_iov *niov)
208{
209	return (__force netmem_ref)((unsigned long)niov | NET_IOV);
210}
211
212#define page_to_netmem(p)	(_Generic((p),			\
213	const struct page * :	(__force const netmem_ref)(p),	\
214	struct page * :		(__force netmem_ref)(p)))
215
216/**
217 * virt_to_netmem - convert virtual memory pointer to a netmem reference
218 * @data: host memory pointer to convert
219 *
220 * Return: netmem reference to the &page backing this virtual address.
221 */
222static inline netmem_ref virt_to_netmem(const void *data)
223{
224	return page_to_netmem(virt_to_page(data));
225}
226
227static inline int netmem_ref_count(netmem_ref netmem)
228{
229	/* The non-pp refcount of net_iov is always 1. On net_iov, we only
230	 * support pp refcounting which uses the pp_ref_count field.
231	 */
232	if (netmem_is_net_iov(netmem))
233		return 1;
234
235	return page_ref_count(netmem_to_page(netmem));
236}
237
238static inline unsigned long netmem_pfn_trace(netmem_ref netmem)
239{
240	if (netmem_is_net_iov(netmem))
241		return 0;
242
243	return page_to_pfn(netmem_to_page(netmem));
244}
245
246/* XXX: How to extract netmem_desc from page must be changed, once
247 * netmem_desc no longer overlays on page and will be allocated through
248 * slab.
249 */
250#define __pp_page_to_nmdesc(p)	(_Generic((p),				\
251	const struct page * :	(const struct netmem_desc *)(p),	\
252	struct page * :		(struct netmem_desc *)(p)))
253
254/* CAUTION: Check if the page is a pp page before calling this helper or
255 * know it's a pp page.
256 */
257#define pp_page_to_nmdesc(p)						\
258({									\
259	DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p));		\
260	__pp_page_to_nmdesc(p);						\
261})
262
263/**
264 * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing
265 * @netmem
266 * @netmem: netmem reference to convert
267 *
268 * Unsafe version that can be used only when @netmem is always backed by
269 * system memory, performs faster and generates smaller object code (no
270 * check for the LSB, no WARN). When @netmem points to IOV, provokes
271 * undefined behaviour.
272 *
273 * Return: pointer to the &netmem_desc (garbage if @netmem is not backed
274 * by system memory).
275 */
276static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem)
277{
278	return (__force struct netmem_desc *)netmem;
279}
280
281/* netmem_to_nmdesc - convert netmem_ref to struct netmem_desc * for
282 * access to common fields.
283 * @netmem: netmem reference to get netmem_desc.
284 *
285 * All the sub types of netmem_ref (netmem_desc, net_iov) have the same
286 * pp, pp_magic, dma_addr, and pp_ref_count fields via netmem_desc.
287 *
288 * Return: the pointer to struct netmem_desc * regardless of its
289 * underlying type.
290 */
291static inline struct netmem_desc *netmem_to_nmdesc(netmem_ref netmem)
292{
293	void *p = (void *)((__force unsigned long)netmem & ~NET_IOV);
294
295	if (netmem_is_net_iov(netmem))
296		return &((struct net_iov *)p)->desc;
297
298	return __pp_page_to_nmdesc((struct page *)p);
299}
300
301/**
302 * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem
303 * @netmem: netmem reference to get the pointer from
304 *
305 * Unsafe version of netmem_get_pp(). When @netmem is always page-backed,
306 * e.g. when it's a header buffer, performs faster and generates smaller
307 * object code (avoids clearing the LSB). When @netmem points to IOV,
308 * provokes invalid memory access.
309 *
310 * Return: pointer to the &page_pool (garbage if @netmem is not page-backed).
311 */
312static inline struct page_pool *__netmem_get_pp(netmem_ref netmem)
313{
314	return __netmem_to_nmdesc(netmem)->pp;
315}
316
317static inline struct page_pool *netmem_get_pp(netmem_ref netmem)
318{
319	return netmem_to_nmdesc(netmem)->pp;
320}
321
322static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem)
323{
324	return &netmem_to_nmdesc(netmem)->pp_ref_count;
325}
326
327static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid)
328{
329	/* NUMA node preference only makes sense if we're allocating
330	 * system memory. Memory providers (which give us net_iovs)
331	 * choose for us.
332	 */
333	if (netmem_is_net_iov(netmem))
334		return true;
335
336	return page_to_nid(netmem_to_page(netmem)) == pref_nid;
337}
338
339static inline netmem_ref netmem_compound_head(netmem_ref netmem)
340{
341	/* niov are never compounded */
342	if (netmem_is_net_iov(netmem))
343		return netmem;
344
345	return page_to_netmem(compound_head(netmem_to_page(netmem)));
346}
347
348/**
349 * __netmem_address - unsafely get pointer to the memory backing @netmem
350 * @netmem: netmem reference to get the pointer for
351 *
352 * Unsafe version of netmem_address(). When @netmem is always page-backed,
353 * e.g. when it's a header buffer, performs faster and generates smaller
354 * object code (no check for the LSB). When @netmem points to IOV, provokes
355 * undefined behaviour.
356 *
357 * Return: pointer to the memory (garbage if @netmem is not page-backed).
358 */
359static inline void *__netmem_address(netmem_ref netmem)
360{
361	return page_address(__netmem_to_page(netmem));
362}
363
364static inline void *netmem_address(netmem_ref netmem)
365{
366	if (netmem_is_net_iov(netmem))
367		return NULL;
368
369	return __netmem_address(netmem);
370}
371
372/**
373 * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure
374 * @netmem: netmem reference to check
375 *
376 * Return: true if @netmem is page-backed and the page was allocated under
377 * memory pressure, false otherwise.
378 */
379static inline bool netmem_is_pfmemalloc(netmem_ref netmem)
380{
381	if (netmem_is_net_iov(netmem))
382		return false;
383
384	return page_is_pfmemalloc(netmem_to_page(netmem));
385}
386
387static inline unsigned long netmem_get_dma_addr(netmem_ref netmem)
388{
389	return netmem_to_nmdesc(netmem)->dma_addr;
390}
391
392void get_netmem(netmem_ref netmem);
393void put_netmem(netmem_ref netmem);
394
395#define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL)   \
396	do {                                                     \
397		if (!netmem_is_net_iov(NETMEM))                  \
398			dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \
399		else                                             \
400			dma_unmap_addr_set(PTR, ADDR_NAME, 0);   \
401	} while (0)
402
403static inline void netmem_dma_unmap_page_attrs(struct device *dev,
404					       dma_addr_t addr, size_t size,
405					       enum dma_data_direction dir,
406					       unsigned long attrs)
407{
408	if (!addr)
409		return;
410
411	dma_unmap_page_attrs(dev, addr, size, dir, attrs);
412}
413
414#endif /* _NET_NETMEM_H */