page_pool.c

   1/* SPDX-License-Identifier: GPL-2.0
   2 *
   3 * page_pool.c
   4 *	Author:	Jesper Dangaard Brouer <netoptimizer@brouer.com>
   5 *	Copyright (C) 2016 Red Hat, Inc.
   6 */
   7
   8#include <linux/error-injection.h>
   9#include <linux/types.h>
  10#include <linux/kernel.h>
  11#include <linux/slab.h>
  12#include <linux/device.h>
  13
  14#include <net/netdev_lock.h>
  15#include <net/netdev_rx_queue.h>
  16#include <net/page_pool/helpers.h>
  17#include <net/page_pool/memory_provider.h>
  18#include <net/xdp.h>
  19
  20#include <linux/dma-direction.h>
  21#include <linux/dma-mapping.h>
  22#include <linux/page-flags.h>
  23#include <linux/mm.h> /* for put_page() */
  24#include <linux/poison.h>
  25#include <linux/ethtool.h>
  26#include <linux/netdevice.h>
  27
  28#include <trace/events/page_pool.h>
  29
  30#include "dev.h"
  31#include "mp_dmabuf_devmem.h"
  32#include "netmem_priv.h"
  33#include "page_pool_priv.h"
  34
  35DEFINE_STATIC_KEY_FALSE(page_pool_mem_providers);
  36
  37#define DEFER_TIME (msecs_to_jiffies(1000))
  38#define DEFER_WARN_INTERVAL (60 * HZ)
  39
  40#define BIAS_MAX	(LONG_MAX >> 1)
  41
  42#ifdef CONFIG_PAGE_POOL_STATS
  43static DEFINE_PER_CPU(struct page_pool_recycle_stats, pp_system_recycle_stats);
  44
  45/* alloc_stat_inc is intended to be used in softirq context */
  46#define alloc_stat_inc(pool, __stat)	(pool->alloc_stats.__stat++)
  47/* recycle_stat_inc is safe to use when preemption is possible. */
  48#define recycle_stat_inc(pool, __stat)							\
  49	do {										\
  50		struct page_pool_recycle_stats __percpu *s = pool->recycle_stats;	\
  51		this_cpu_inc(s->__stat);						\
  52	} while (0)
  53
  54#define recycle_stat_add(pool, __stat, val)						\
  55	do {										\
  56		struct page_pool_recycle_stats __percpu *s = pool->recycle_stats;	\
  57		this_cpu_add(s->__stat, val);						\
  58	} while (0)
  59
  60static const char pp_stats[][ETH_GSTRING_LEN] = {
  61	"rx_pp_alloc_fast",
  62	"rx_pp_alloc_slow",
  63	"rx_pp_alloc_slow_ho",
  64	"rx_pp_alloc_empty",
  65	"rx_pp_alloc_refill",
  66	"rx_pp_alloc_waive",
  67	"rx_pp_recycle_cached",
  68	"rx_pp_recycle_cache_full",
  69	"rx_pp_recycle_ring",
  70	"rx_pp_recycle_ring_full",
  71	"rx_pp_recycle_released_ref",
  72};
  73
  74/**
  75 * page_pool_get_stats() - fetch page pool stats
  76 * @pool:	pool from which page was allocated
  77 * @stats:	struct page_pool_stats to fill in
  78 *
  79 * Retrieve statistics about the page_pool. This API is only available
  80 * if the kernel has been configured with ``CONFIG_PAGE_POOL_STATS=y``.
  81 * A pointer to a caller allocated struct page_pool_stats structure
  82 * is passed to this API which is filled in. The caller can then report
  83 * those stats to the user (perhaps via ethtool, debugfs, etc.).
  84 */
  85bool page_pool_get_stats(const struct page_pool *pool,
  86			 struct page_pool_stats *stats)
  87{
  88	int cpu = 0;
  89
  90	if (!stats)
  91		return false;
  92
  93	/* The caller is responsible to initialize stats. */
  94	stats->alloc_stats.fast += pool->alloc_stats.fast;
  95	stats->alloc_stats.slow += pool->alloc_stats.slow;
  96	stats->alloc_stats.slow_high_order += pool->alloc_stats.slow_high_order;
  97	stats->alloc_stats.empty += pool->alloc_stats.empty;
  98	stats->alloc_stats.refill += pool->alloc_stats.refill;
  99	stats->alloc_stats.waive += pool->alloc_stats.waive;
 100
 101	for_each_possible_cpu(cpu) {
 102		const struct page_pool_recycle_stats *pcpu =
 103			per_cpu_ptr(pool->recycle_stats, cpu);
 104
 105		stats->recycle_stats.cached += pcpu->cached;
 106		stats->recycle_stats.cache_full += pcpu->cache_full;
 107		stats->recycle_stats.ring += pcpu->ring;
 108		stats->recycle_stats.ring_full += pcpu->ring_full;
 109		stats->recycle_stats.released_refcnt += pcpu->released_refcnt;
 110	}
 111
 112	return true;
 113}
 114EXPORT_SYMBOL(page_pool_get_stats);
 115
 116u8 *page_pool_ethtool_stats_get_strings(u8 *data)
 117{
 118	int i;
 119
 120	for (i = 0; i < ARRAY_SIZE(pp_stats); i++) {
 121		memcpy(data, pp_stats[i], ETH_GSTRING_LEN);
 122		data += ETH_GSTRING_LEN;
 123	}
 124
 125	return data;
 126}
 127EXPORT_SYMBOL(page_pool_ethtool_stats_get_strings);
 128
 129int page_pool_ethtool_stats_get_count(void)
 130{
 131	return ARRAY_SIZE(pp_stats);
 132}
 133EXPORT_SYMBOL(page_pool_ethtool_stats_get_count);
 134
 135u64 *page_pool_ethtool_stats_get(u64 *data, const void *stats)
 136{
 137	const struct page_pool_stats *pool_stats = stats;
 138
 139	*data++ = pool_stats->alloc_stats.fast;
 140	*data++ = pool_stats->alloc_stats.slow;
 141	*data++ = pool_stats->alloc_stats.slow_high_order;
 142	*data++ = pool_stats->alloc_stats.empty;
 143	*data++ = pool_stats->alloc_stats.refill;
 144	*data++ = pool_stats->alloc_stats.waive;
 145	*data++ = pool_stats->recycle_stats.cached;
 146	*data++ = pool_stats->recycle_stats.cache_full;
 147	*data++ = pool_stats->recycle_stats.ring;
 148	*data++ = pool_stats->recycle_stats.ring_full;
 149	*data++ = pool_stats->recycle_stats.released_refcnt;
 150
 151	return data;
 152}
 153EXPORT_SYMBOL(page_pool_ethtool_stats_get);
 154
 155#else
 156#define alloc_stat_inc(...)	do { } while (0)
 157#define recycle_stat_inc(...)	do { } while (0)
 158#define recycle_stat_add(...)	do { } while (0)
 159#endif
 160
 161static bool page_pool_producer_lock(struct page_pool *pool)
 162	__acquires(&pool->ring.producer_lock)
 163{
 164	bool in_softirq = in_softirq();
 165
 166	if (in_softirq)
 167		spin_lock(&pool->ring.producer_lock);
 168	else
 169		spin_lock_bh(&pool->ring.producer_lock);
 170
 171	return in_softirq;
 172}
 173
 174static void page_pool_producer_unlock(struct page_pool *pool,
 175				      bool in_softirq)
 176	__releases(&pool->ring.producer_lock)
 177{
 178	if (in_softirq)
 179		spin_unlock(&pool->ring.producer_lock);
 180	else
 181		spin_unlock_bh(&pool->ring.producer_lock);
 182}
 183
 184static void page_pool_struct_check(void)
 185{
 186	CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_users);
 187	CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_page);
 188	CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_offset);
 189	CACHELINE_ASSERT_GROUP_SIZE(struct page_pool, frag,
 190				    PAGE_POOL_FRAG_GROUP_ALIGN);
 191}
 192
 193static int page_pool_init(struct page_pool *pool,
 194			  const struct page_pool_params *params,
 195			  int cpuid)
 196{
 197	unsigned int ring_qsize = 1024; /* Default */
 198	struct netdev_rx_queue *rxq;
 199	int err;
 200
 201	page_pool_struct_check();
 202
 203	memcpy(&pool->p, &params->fast, sizeof(pool->p));
 204	memcpy(&pool->slow, &params->slow, sizeof(pool->slow));
 205
 206	pool->cpuid = cpuid;
 207	pool->dma_sync_for_cpu = true;
 208
 209	/* Validate only known flags were used */
 210	if (pool->slow.flags & ~PP_FLAG_ALL)
 211		return -EINVAL;
 212
 213	if (pool->p.pool_size)
 214		ring_qsize = min(pool->p.pool_size, 16384);
 215
 216	/* DMA direction is either DMA_FROM_DEVICE or DMA_BIDIRECTIONAL.
 217	 * DMA_BIDIRECTIONAL is for allowing page used for DMA sending,
 218	 * which is the XDP_TX use-case.
 219	 */
 220	if (pool->slow.flags & PP_FLAG_DMA_MAP) {
 221		if ((pool->p.dma_dir != DMA_FROM_DEVICE) &&
 222		    (pool->p.dma_dir != DMA_BIDIRECTIONAL))
 223			return -EINVAL;
 224
 225		pool->dma_map = true;
 226	}
 227
 228	if (pool->slow.flags & PP_FLAG_DMA_SYNC_DEV) {
 229		/* In order to request DMA-sync-for-device the page
 230		 * needs to be mapped
 231		 */
 232		if (!(pool->slow.flags & PP_FLAG_DMA_MAP))
 233			return -EINVAL;
 234
 235		if (!pool->p.max_len)
 236			return -EINVAL;
 237
 238		pool->dma_sync = true;
 239
 240		/* pool->p.offset has to be set according to the address
 241		 * offset used by the DMA engine to start copying rx data
 242		 */
 243	}
 244
 245	pool->has_init_callback = !!pool->slow.init_callback;
 246
 247#ifdef CONFIG_PAGE_POOL_STATS
 248	if (!(pool->slow.flags & PP_FLAG_SYSTEM_POOL)) {
 249		pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats);
 250		if (!pool->recycle_stats)
 251			return -ENOMEM;
 252	} else {
 253		/* For system page pool instance we use a singular stats object
 254		 * instead of allocating a separate percpu variable for each
 255		 * (also percpu) page pool instance.
 256		 */
 257		pool->recycle_stats = &pp_system_recycle_stats;
 258		pool->system = true;
 259	}
 260#endif
 261
 262	if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) {
 263#ifdef CONFIG_PAGE_POOL_STATS
 264		if (!pool->system)
 265			free_percpu(pool->recycle_stats);
 266#endif
 267		return -ENOMEM;
 268	}
 269
 270	atomic_set(&pool->pages_state_release_cnt, 0);
 271
 272	/* Driver calling page_pool_create() also call page_pool_destroy() */
 273	refcount_set(&pool->user_cnt, 1);
 274
 275	xa_init_flags(&pool->dma_mapped, XA_FLAGS_ALLOC1);
 276
 277	if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) {
 278		netdev_assert_locked(pool->slow.netdev);
 279		rxq = __netif_get_rx_queue(pool->slow.netdev,
 280					   pool->slow.queue_idx);
 281		pool->mp_priv = rxq->mp_params.mp_priv;
 282		pool->mp_ops = rxq->mp_params.mp_ops;
 283	}
 284
 285	if (pool->mp_ops) {
 286		if (!pool->dma_map || !pool->dma_sync) {
 287			err = -EOPNOTSUPP;
 288			goto free_ptr_ring;
 289		}
 290
 291		if (WARN_ON(!is_kernel_rodata((unsigned long)pool->mp_ops))) {
 292			err = -EFAULT;
 293			goto free_ptr_ring;
 294		}
 295
 296		err = pool->mp_ops->init(pool);
 297		if (err) {
 298			pr_warn("%s() mem-provider init failed %d\n", __func__,
 299				err);
 300			goto free_ptr_ring;
 301		}
 302
 303		static_branch_inc(&page_pool_mem_providers);
 304	} else if (pool->p.order > MAX_PAGE_ORDER) {
 305		err = -EINVAL;
 306		goto free_ptr_ring;
 307	}
 308
 309	return 0;
 310
 311free_ptr_ring:
 312	ptr_ring_cleanup(&pool->ring, NULL);
 313	xa_destroy(&pool->dma_mapped);
 314#ifdef CONFIG_PAGE_POOL_STATS
 315	if (!pool->system)
 316		free_percpu(pool->recycle_stats);
 317#endif
 318	return err;
 319}
 320
 321static void page_pool_uninit(struct page_pool *pool)
 322{
 323	ptr_ring_cleanup(&pool->ring, NULL);
 324	xa_destroy(&pool->dma_mapped);
 325
 326#ifdef CONFIG_PAGE_POOL_STATS
 327	if (!pool->system)
 328		free_percpu(pool->recycle_stats);
 329#endif
 330}
 331
 332/**
 333 * page_pool_create_percpu() - create a page pool for a given cpu.
 334 * @params: parameters, see struct page_pool_params
 335 * @cpuid: cpu identifier
 336 */
 337struct page_pool *
 338page_pool_create_percpu(const struct page_pool_params *params, int cpuid)
 339{
 340	struct page_pool *pool;
 341	int err;
 342
 343	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid);
 344	if (!pool)
 345		return ERR_PTR(-ENOMEM);
 346
 347	err = page_pool_init(pool, params, cpuid);
 348	if (err < 0)
 349		goto err_free;
 350
 351	err = page_pool_list(pool);
 352	if (err)
 353		goto err_uninit;
 354
 355	return pool;
 356
 357err_uninit:
 358	page_pool_uninit(pool);
 359err_free:
 360	pr_warn("%s() gave up with errno %d\n", __func__, err);
 361	kfree(pool);
 362	return ERR_PTR(err);
 363}
 364EXPORT_SYMBOL(page_pool_create_percpu);
 365
 366/**
 367 * page_pool_create() - create a page pool
 368 * @params: parameters, see struct page_pool_params
 369 */
 370struct page_pool *page_pool_create(const struct page_pool_params *params)
 371{
 372	return page_pool_create_percpu(params, -1);
 373}
 374EXPORT_SYMBOL(page_pool_create);
 375
 376static void page_pool_return_netmem(struct page_pool *pool, netmem_ref netmem);
 377
 378static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)
 379{
 380	struct ptr_ring *r = &pool->ring;
 381	netmem_ref netmem;
 382	int pref_nid; /* preferred NUMA node */
 383
 384	/* Quicker fallback, avoid locks when ring is empty */
 385	if (__ptr_ring_empty(r)) {
 386		alloc_stat_inc(pool, empty);
 387		return 0;
 388	}
 389
 390	/* Softirq guarantee CPU and thus NUMA node is stable. This,
 391	 * assumes CPU refilling driver RX-ring will also run RX-NAPI.
 392	 */
 393#ifdef CONFIG_NUMA
 394	pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid;
 395#else
 396	/* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */
 397	pref_nid = numa_mem_id(); /* will be zero like page_to_nid() */
 398#endif
 399
 400	/* Refill alloc array, but only if NUMA match */
 401	do {
 402		netmem = (__force netmem_ref)__ptr_ring_consume(r);
 403		if (unlikely(!netmem))
 404			break;
 405
 406		if (likely(netmem_is_pref_nid(netmem, pref_nid))) {
 407			pool->alloc.cache[pool->alloc.count++] = netmem;
 408		} else {
 409			/* NUMA mismatch;
 410			 * (1) release 1 page to page-allocator and
 411			 * (2) break out to fallthrough to alloc_pages_node.
 412			 * This limit stress on page buddy alloactor.
 413			 */
 414			page_pool_return_netmem(pool, netmem);
 415			alloc_stat_inc(pool, waive);
 416			netmem = 0;
 417			break;
 418		}
 419	} while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);
 420
 421	/* Return last page */
 422	if (likely(pool->alloc.count > 0)) {
 423		netmem = pool->alloc.cache[--pool->alloc.count];
 424		alloc_stat_inc(pool, refill);
 425	}
 426
 427	return netmem;
 428}
 429
 430/* fast path */
 431static netmem_ref __page_pool_get_cached(struct page_pool *pool)
 432{
 433	netmem_ref netmem;
 434
 435	/* Caller MUST guarantee safe non-concurrent access, e.g. softirq */
 436	if (likely(pool->alloc.count)) {
 437		/* Fast-path */
 438		netmem = pool->alloc.cache[--pool->alloc.count];
 439		alloc_stat_inc(pool, fast);
 440	} else {
 441		netmem = page_pool_refill_alloc_cache(pool);
 442	}
 443
 444	return netmem;
 445}
 446
 447static void __page_pool_dma_sync_for_device(const struct page_pool *pool,
 448					    netmem_ref netmem,
 449					    u32 dma_sync_size)
 450{
 451#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)
 452	dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem);
 453
 454	dma_sync_size = min(dma_sync_size, pool->p.max_len);
 455	__dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset,
 456				     dma_sync_size, pool->p.dma_dir);
 457#endif
 458}
 459
 460static __always_inline void
 461page_pool_dma_sync_for_device(const struct page_pool *pool,
 462			      netmem_ref netmem,
 463			      u32 dma_sync_size)
 464{
 465	if (pool->dma_sync && dma_dev_need_sync(pool->p.dev)) {
 466		rcu_read_lock();
 467		/* re-check under rcu_read_lock() to sync with page_pool_scrub() */
 468		if (pool->dma_sync)
 469			__page_pool_dma_sync_for_device(pool, netmem,
 470							dma_sync_size);
 471		rcu_read_unlock();
 472	}
 473}
 474
 475static int page_pool_register_dma_index(struct page_pool *pool,
 476					netmem_ref netmem, gfp_t gfp)
 477{
 478	int err = 0;
 479	u32 id;
 480
 481	if (unlikely(!PP_DMA_INDEX_BITS))
 482		goto out;
 483
 484	if (in_softirq())
 485		err = xa_alloc(&pool->dma_mapped, &id, netmem_to_page(netmem),
 486			       PP_DMA_INDEX_LIMIT, gfp);
 487	else
 488		err = xa_alloc_bh(&pool->dma_mapped, &id, netmem_to_page(netmem),
 489				  PP_DMA_INDEX_LIMIT, gfp);
 490	if (err) {
 491		WARN_ONCE(err != -ENOMEM, "couldn't track DMA mapping, please report to netdev@");
 492		goto out;
 493	}
 494
 495	netmem_set_dma_index(netmem, id);
 496out:
 497	return err;
 498}
 499
 500static int page_pool_release_dma_index(struct page_pool *pool,
 501				       netmem_ref netmem)
 502{
 503	struct page *old, *page = netmem_to_page(netmem);
 504	unsigned long id;
 505
 506	if (unlikely(!PP_DMA_INDEX_BITS))
 507		return 0;
 508
 509	id = netmem_get_dma_index(netmem);
 510	if (!id)
 511		return -1;
 512
 513	if (in_softirq())
 514		old = xa_cmpxchg(&pool->dma_mapped, id, page, NULL, 0);
 515	else
 516		old = xa_cmpxchg_bh(&pool->dma_mapped, id, page, NULL, 0);
 517	if (old != page)
 518		return -1;
 519
 520	netmem_set_dma_index(netmem, 0);
 521
 522	return 0;
 523}
 524
 525static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem, gfp_t gfp)
 526{
 527	dma_addr_t dma;
 528	int err;
 529
 530	/* Setup DMA mapping: use 'struct page' area for storing DMA-addr
 531	 * since dma_addr_t can be either 32 or 64 bits and does not always fit
 532	 * into page private data (i.e 32bit cpu with 64bit DMA caps)
 533	 * This mapping is kept for lifetime of page, until leaving pool.
 534	 */
 535	dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0,
 536				 (PAGE_SIZE << pool->p.order), pool->p.dma_dir,
 537				 DMA_ATTR_SKIP_CPU_SYNC |
 538					 DMA_ATTR_WEAK_ORDERING);
 539	if (dma_mapping_error(pool->p.dev, dma))
 540		return false;
 541
 542	if (page_pool_set_dma_addr_netmem(netmem, dma)) {
 543		WARN_ONCE(1, "unexpected DMA address, please report to netdev@");
 544		goto unmap_failed;
 545	}
 546
 547	err = page_pool_register_dma_index(pool, netmem, gfp);
 548	if (err)
 549		goto unset_failed;
 550
 551	page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len);
 552
 553	return true;
 554
 555unset_failed:
 556	page_pool_set_dma_addr_netmem(netmem, 0);
 557unmap_failed:
 558	dma_unmap_page_attrs(pool->p.dev, dma,
 559			     PAGE_SIZE << pool->p.order, pool->p.dma_dir,
 560			     DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
 561	return false;
 562}
 563
 564static struct page *__page_pool_alloc_page_order(struct page_pool *pool,
 565						 gfp_t gfp)
 566{
 567	struct page *page;
 568
 569	gfp |= __GFP_COMP;
 570	page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
 571	if (unlikely(!page))
 572		return NULL;
 573
 574	if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page_to_netmem(page), gfp))) {
 575		put_page(page);
 576		return NULL;
 577	}
 578
 579	alloc_stat_inc(pool, slow_high_order);
 580	page_pool_set_pp_info(pool, page_to_netmem(page));
 581
 582	/* Track how many pages are held 'in-flight' */
 583	pool->pages_state_hold_cnt++;
 584	trace_page_pool_state_hold(pool, page_to_netmem(page),
 585				   pool->pages_state_hold_cnt);
 586	return page;
 587}
 588
 589/* slow path */
 590static noinline netmem_ref __page_pool_alloc_netmems_slow(struct page_pool *pool,
 591							  gfp_t gfp)
 592{
 593	const int bulk = PP_ALLOC_CACHE_REFILL;
 594	unsigned int pp_order = pool->p.order;
 595	bool dma_map = pool->dma_map;
 596	netmem_ref netmem;
 597	int i, nr_pages;
 598
 599	/* Unconditionally set NOWARN if allocating from NAPI.
 600	 * Drivers forget to set it, and OOM reports on packet Rx are useless.
 601	 */
 602	if ((gfp & GFP_ATOMIC) == GFP_ATOMIC)
 603		gfp |= __GFP_NOWARN;
 604
 605	/* Don't support bulk alloc for high-order pages */
 606	if (unlikely(pp_order))
 607		return page_to_netmem(__page_pool_alloc_page_order(pool, gfp));
 608
 609	/* Unnecessary as alloc cache is empty, but guarantees zero count */
 610	if (unlikely(pool->alloc.count > 0))
 611		return pool->alloc.cache[--pool->alloc.count];
 612
 613	/* Mark empty alloc.cache slots "empty" for alloc_pages_bulk */
 614	memset(&pool->alloc.cache, 0, sizeof(void *) * bulk);
 615
 616	nr_pages = alloc_pages_bulk_node(gfp, pool->p.nid, bulk,
 617					 (struct page **)pool->alloc.cache);
 618	if (unlikely(!nr_pages))
 619		return 0;
 620
 621	/* Pages have been filled into alloc.cache array, but count is zero and
 622	 * page element have not been (possibly) DMA mapped.
 623	 */
 624	for (i = 0; i < nr_pages; i++) {
 625		netmem = pool->alloc.cache[i];
 626		if (dma_map && unlikely(!page_pool_dma_map(pool, netmem, gfp))) {
 627			put_page(netmem_to_page(netmem));
 628			continue;
 629		}
 630
 631		page_pool_set_pp_info(pool, netmem);
 632		pool->alloc.cache[pool->alloc.count++] = netmem;
 633		/* Track how many pages are held 'in-flight' */
 634		pool->pages_state_hold_cnt++;
 635		trace_page_pool_state_hold(pool, netmem,
 636					   pool->pages_state_hold_cnt);
 637	}
 638
 639	/* Return last page */
 640	if (likely(pool->alloc.count > 0)) {
 641		netmem = pool->alloc.cache[--pool->alloc.count];
 642		alloc_stat_inc(pool, slow);
 643	} else {
 644		netmem = 0;
 645	}
 646
 647	/* When page just alloc'ed is should/must have refcnt 1. */
 648	return netmem;
 649}
 650
 651/* For using page_pool replace: alloc_pages() API calls, but provide
 652 * synchronization guarantee for allocation side.
 653 */
 654netmem_ref page_pool_alloc_netmems(struct page_pool *pool, gfp_t gfp)
 655{
 656	netmem_ref netmem;
 657
 658	/* Fast-path: Get a page from cache */
 659	netmem = __page_pool_get_cached(pool);
 660	if (netmem)
 661		return netmem;
 662
 663	/* Slow-path: cache empty, do real allocation */
 664	if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
 665		netmem = pool->mp_ops->alloc_netmems(pool, gfp);
 666	else
 667		netmem = __page_pool_alloc_netmems_slow(pool, gfp);
 668	return netmem;
 669}
 670EXPORT_SYMBOL(page_pool_alloc_netmems);
 671ALLOW_ERROR_INJECTION(page_pool_alloc_netmems, NULL);
 672
 673struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
 674{
 675	return netmem_to_page(page_pool_alloc_netmems(pool, gfp));
 676}
 677EXPORT_SYMBOL(page_pool_alloc_pages);
 678
 679/* Calculate distance between two u32 values, valid if distance is below 2^(31)
 680 *  https://en.wikipedia.org/wiki/Serial_number_arithmetic#General_Solution
 681 */
 682#define _distance(a, b)	(s32)((a) - (b))
 683
 684s32 page_pool_inflight(const struct page_pool *pool, bool strict)
 685{
 686	u32 release_cnt = atomic_read(&pool->pages_state_release_cnt);
 687	u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt);
 688	s32 inflight;
 689
 690	inflight = _distance(hold_cnt, release_cnt);
 691
 692	if (strict) {
 693		trace_page_pool_release(pool, inflight, hold_cnt, release_cnt);
 694		WARN(inflight < 0, "Negative(%d) inflight packet-pages",
 695		     inflight);
 696	} else {
 697		inflight = max(0, inflight);
 698	}
 699
 700	return inflight;
 701}
 702
 703void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
 704{
 705	netmem_set_pp(netmem, pool);
 706	netmem_or_pp_magic(netmem, PP_SIGNATURE);
 707
 708	/* Ensuring all pages have been split into one fragment initially:
 709	 * page_pool_set_pp_info() is only called once for every page when it
 710	 * is allocated from the page allocator and page_pool_fragment_page()
 711	 * is dirtying the same cache line as the page->pp_magic above, so
 712	 * the overhead is negligible.
 713	 */
 714	page_pool_fragment_netmem(netmem, 1);
 715	if (pool->has_init_callback)
 716		pool->slow.init_callback(netmem, pool->slow.init_arg);
 717}
 718
 719void page_pool_clear_pp_info(netmem_ref netmem)
 720{
 721	netmem_clear_pp_magic(netmem);
 722	netmem_set_pp(netmem, NULL);
 723}
 724
 725static __always_inline void __page_pool_release_netmem_dma(struct page_pool *pool,
 726							   netmem_ref netmem)
 727{
 728	dma_addr_t dma;
 729
 730	if (!pool->dma_map)
 731		/* Always account for inflight pages, even if we didn't
 732		 * map them
 733		 */
 734		return;
 735
 736	if (page_pool_release_dma_index(pool, netmem))
 737		return;
 738
 739	dma = page_pool_get_dma_addr_netmem(netmem);
 740
 741	/* When page is unmapped, it cannot be returned to our pool */
 742	dma_unmap_page_attrs(pool->p.dev, dma,
 743			     PAGE_SIZE << pool->p.order, pool->p.dma_dir,
 744			     DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
 745	page_pool_set_dma_addr_netmem(netmem, 0);
 746}
 747
 748/* Disconnects a page (from a page_pool).  API users can have a need
 749 * to disconnect a page (from a page_pool), to allow it to be used as
 750 * a regular page (that will eventually be returned to the normal
 751 * page-allocator via put_page).
 752 */
 753static void page_pool_return_netmem(struct page_pool *pool, netmem_ref netmem)
 754{
 755	int count;
 756	bool put;
 757
 758	put = true;
 759	if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_ops)
 760		put = pool->mp_ops->release_netmem(pool, netmem);
 761	else
 762		__page_pool_release_netmem_dma(pool, netmem);
 763
 764	/* This may be the last page returned, releasing the pool, so
 765	 * it is not safe to reference pool afterwards.
 766	 */
 767	count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt);
 768	trace_page_pool_state_release(pool, netmem, count);
 769
 770	if (put) {
 771		page_pool_clear_pp_info(netmem);
 772		put_page(netmem_to_page(netmem));
 773	}
 774	/* An optimization would be to call __free_pages(page, pool->p.order)
 775	 * knowing page is not part of page-cache (thus avoiding a
 776	 * __page_cache_release() call).
 777	 */
 778}
 779
 780static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem)
 781{
 782	bool in_softirq, ret;
 783
 784	/* BH protection not needed if current is softirq */
 785	in_softirq = page_pool_producer_lock(pool);
 786	ret = !__ptr_ring_produce(&pool->ring, (__force void *)netmem);
 787	if (ret)
 788		recycle_stat_inc(pool, ring);
 789	page_pool_producer_unlock(pool, in_softirq);
 790
 791	return ret;
 792}
 793
 794/* Only allow direct recycling in special circumstances, into the
 795 * alloc side cache.  E.g. during RX-NAPI processing for XDP_DROP use-case.
 796 *
 797 * Caller must provide appropriate safe context.
 798 */
 799static bool page_pool_recycle_in_cache(netmem_ref netmem,
 800				       struct page_pool *pool)
 801{
 802	if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) {
 803		recycle_stat_inc(pool, cache_full);
 804		return false;
 805	}
 806
 807	/* Caller MUST have verified/know (page_ref_count(page) == 1) */
 808	pool->alloc.cache[pool->alloc.count++] = netmem;
 809	recycle_stat_inc(pool, cached);
 810	return true;
 811}
 812
 813static bool __page_pool_page_can_be_recycled(netmem_ref netmem)
 814{
 815	return netmem_is_net_iov(netmem) ||
 816	       (page_ref_count(netmem_to_page(netmem)) == 1 &&
 817		!page_is_pfmemalloc(netmem_to_page(netmem)));
 818}
 819
 820/* If the page refcnt == 1, this will try to recycle the page.
 821 * If pool->dma_sync is set, we'll try to sync the DMA area for
 822 * the configured size min(dma_sync_size, pool->max_len).
 823 * If the page refcnt != 1, then the page will be returned to memory
 824 * subsystem.
 825 */
 826static __always_inline netmem_ref
 827__page_pool_put_page(struct page_pool *pool, netmem_ref netmem,
 828		     unsigned int dma_sync_size, bool allow_direct)
 829{
 830	lockdep_assert_no_hardirq();
 831
 832	/* This allocator is optimized for the XDP mode that uses
 833	 * one-frame-per-page, but have fallbacks that act like the
 834	 * regular page allocator APIs.
 835	 *
 836	 * refcnt == 1 means page_pool owns page, and can recycle it.
 837	 *
 838	 * page is NOT reusable when allocated when system is under
 839	 * some pressure. (page_is_pfmemalloc)
 840	 */
 841	if (likely(__page_pool_page_can_be_recycled(netmem))) {
 842		/* Read barrier done in page_ref_count / READ_ONCE */
 843
 844		page_pool_dma_sync_for_device(pool, netmem, dma_sync_size);
 845
 846		if (allow_direct && page_pool_recycle_in_cache(netmem, pool))
 847			return 0;
 848
 849		/* Page found as candidate for recycling */
 850		return netmem;
 851	}
 852
 853	/* Fallback/non-XDP mode: API user have elevated refcnt.
 854	 *
 855	 * Many drivers split up the page into fragments, and some
 856	 * want to keep doing this to save memory and do refcnt based
 857	 * recycling. Support this use case too, to ease drivers
 858	 * switching between XDP/non-XDP.
 859	 *
 860	 * In-case page_pool maintains the DMA mapping, API user must
 861	 * call page_pool_put_page once.  In this elevated refcnt
 862	 * case, the DMA is unmapped/released, as driver is likely
 863	 * doing refcnt based recycle tricks, meaning another process
 864	 * will be invoking put_page.
 865	 */
 866	recycle_stat_inc(pool, released_refcnt);
 867	page_pool_return_netmem(pool, netmem);
 868
 869	return 0;
 870}
 871
 872static bool page_pool_napi_local(const struct page_pool *pool)
 873{
 874	const struct napi_struct *napi;
 875	u32 cpuid;
 876
 877	/* On PREEMPT_RT the softirq can be preempted by the consumer */
 878	if (IS_ENABLED(CONFIG_PREEMPT_RT))
 879		return false;
 880
 881	if (unlikely(!in_softirq()))
 882		return false;
 883
 884	/* Allow direct recycle if we have reasons to believe that we are
 885	 * in the same context as the consumer would run, so there's
 886	 * no possible race.
 887	 * __page_pool_put_page() makes sure we're not in hardirq context
 888	 * and interrupts are enabled prior to accessing the cache.
 889	 */
 890	cpuid = smp_processor_id();
 891	if (READ_ONCE(pool->cpuid) == cpuid)
 892		return true;
 893
 894	napi = READ_ONCE(pool->p.napi);
 895
 896	return napi && READ_ONCE(napi->list_owner) == cpuid;
 897}
 898
 899void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
 900				  unsigned int dma_sync_size, bool allow_direct)
 901{
 902	if (!allow_direct)
 903		allow_direct = page_pool_napi_local(pool);
 904
 905	netmem = __page_pool_put_page(pool, netmem, dma_sync_size,
 906				      allow_direct);
 907	if (netmem && !page_pool_recycle_in_ring(pool, netmem)) {
 908		/* Cache full, fallback to free pages */
 909		recycle_stat_inc(pool, ring_full);
 910		page_pool_return_netmem(pool, netmem);
 911	}
 912}
 913EXPORT_SYMBOL(page_pool_put_unrefed_netmem);
 914
 915void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
 916				unsigned int dma_sync_size, bool allow_direct)
 917{
 918	page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size,
 919				     allow_direct);
 920}
 921EXPORT_SYMBOL(page_pool_put_unrefed_page);
 922
 923static void page_pool_recycle_ring_bulk(struct page_pool *pool,
 924					netmem_ref *bulk,
 925					u32 bulk_len)
 926{
 927	bool in_softirq;
 928	u32 i;
 929
 930	/* Bulk produce into ptr_ring page_pool cache */
 931	in_softirq = page_pool_producer_lock(pool);
 932
 933	for (i = 0; i < bulk_len; i++) {
 934		if (__ptr_ring_produce(&pool->ring, (__force void *)bulk[i])) {
 935			/* ring full */
 936			recycle_stat_inc(pool, ring_full);
 937			break;
 938		}
 939	}
 940
 941	page_pool_producer_unlock(pool, in_softirq);
 942	recycle_stat_add(pool, ring, i);
 943
 944	/* Hopefully all pages were returned into ptr_ring */
 945	if (likely(i == bulk_len))
 946		return;
 947
 948	/*
 949	 * ptr_ring cache is full, free remaining pages outside producer lock
 950	 * since put_page() with refcnt == 1 can be an expensive operation.
 951	 */
 952	for (; i < bulk_len; i++)
 953		page_pool_return_netmem(pool, bulk[i]);
 954}
 955
 956/**
 957 * page_pool_put_netmem_bulk() - release references on multiple netmems
 958 * @data:	array holding netmem references
 959 * @count:	number of entries in @data
 960 *
 961 * Tries to refill a number of netmems into the ptr_ring cache holding ptr_ring
 962 * producer lock. If the ptr_ring is full, page_pool_put_netmem_bulk()
 963 * will release leftover netmems to the memory provider.
 964 * page_pool_put_netmem_bulk() is suitable to be run inside the driver NAPI tx
 965 * completion loop for the XDP_REDIRECT use case.
 966 *
 967 * Please note the caller must not use data area after running
 968 * page_pool_put_netmem_bulk(), as this function overwrites it.
 969 */
 970void page_pool_put_netmem_bulk(netmem_ref *data, u32 count)
 971{
 972	u32 bulk_len = 0;
 973
 974	for (u32 i = 0; i < count; i++) {
 975		netmem_ref netmem = netmem_compound_head(data[i]);
 976
 977		if (page_pool_unref_and_test(netmem))
 978			data[bulk_len++] = netmem;
 979	}
 980
 981	count = bulk_len;
 982	while (count) {
 983		netmem_ref bulk[XDP_BULK_QUEUE_SIZE];
 984		struct page_pool *pool = NULL;
 985		bool allow_direct;
 986		u32 foreign = 0;
 987
 988		bulk_len = 0;
 989
 990		for (u32 i = 0; i < count; i++) {
 991			struct page_pool *netmem_pp;
 992			netmem_ref netmem = data[i];
 993
 994			netmem_pp = netmem_get_pp(netmem);
 995			if (unlikely(!pool)) {
 996				pool = netmem_pp;
 997				allow_direct = page_pool_napi_local(pool);
 998			} else if (netmem_pp != pool) {
 999				/*
1000				 * If the netmem belongs to a different
1001				 * page_pool, save it for another round.
1002				 */
1003				data[foreign++] = netmem;
1004				continue;
1005			}
1006
1007			netmem = __page_pool_put_page(pool, netmem, -1,
1008						      allow_direct);
1009			/* Approved for bulk recycling in ptr_ring cache */
1010			if (netmem)
1011				bulk[bulk_len++] = netmem;
1012		}
1013
1014		if (bulk_len)
1015			page_pool_recycle_ring_bulk(pool, bulk, bulk_len);
1016
1017		count = foreign;
1018	}
1019}
1020EXPORT_SYMBOL(page_pool_put_netmem_bulk);
1021
1022static netmem_ref page_pool_drain_frag(struct page_pool *pool,
1023				       netmem_ref netmem)
1024{
1025	long drain_count = BIAS_MAX - pool->frag_users;
1026
1027	/* Some user is still using the page frag */
1028	if (likely(page_pool_unref_netmem(netmem, drain_count)))
1029		return 0;
1030
1031	if (__page_pool_page_can_be_recycled(netmem)) {
1032		page_pool_dma_sync_for_device(pool, netmem, -1);
1033		return netmem;
1034	}
1035
1036	page_pool_return_netmem(pool, netmem);
1037	return 0;
1038}
1039
1040static void page_pool_free_frag(struct page_pool *pool)
1041{
1042	long drain_count = BIAS_MAX - pool->frag_users;
1043	netmem_ref netmem = pool->frag_page;
1044
1045	pool->frag_page = 0;
1046
1047	if (!netmem || page_pool_unref_netmem(netmem, drain_count))
1048		return;
1049
1050	page_pool_return_netmem(pool, netmem);
1051}
1052
1053netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,
1054				       unsigned int *offset, unsigned int size,
1055				       gfp_t gfp)
1056{
1057	unsigned int max_size = PAGE_SIZE << pool->p.order;
1058	netmem_ref netmem = pool->frag_page;
1059
1060	if (WARN_ON(size > max_size))
1061		return 0;
1062
1063	size = ALIGN(size, dma_get_cache_alignment());
1064	*offset = pool->frag_offset;
1065
1066	if (netmem && *offset + size > max_size) {
1067		netmem = page_pool_drain_frag(pool, netmem);
1068		if (netmem) {
1069			recycle_stat_inc(pool, cached);
1070			alloc_stat_inc(pool, fast);
1071			goto frag_reset;
1072		}
1073	}
1074
1075	if (!netmem) {
1076		netmem = page_pool_alloc_netmems(pool, gfp);
1077		if (unlikely(!netmem)) {
1078			pool->frag_page = 0;
1079			return 0;
1080		}
1081
1082		pool->frag_page = netmem;
1083
1084frag_reset:
1085		pool->frag_users = 1;
1086		*offset = 0;
1087		pool->frag_offset = size;
1088		page_pool_fragment_netmem(netmem, BIAS_MAX);
1089		return netmem;
1090	}
1091
1092	pool->frag_users++;
1093	pool->frag_offset = *offset + size;
1094	return netmem;
1095}
1096EXPORT_SYMBOL(page_pool_alloc_frag_netmem);
1097
1098struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
1099				  unsigned int size, gfp_t gfp)
1100{
1101	return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size,
1102							  gfp));
1103}
1104EXPORT_SYMBOL(page_pool_alloc_frag);
1105
1106static void page_pool_empty_ring(struct page_pool *pool)
1107{
1108	netmem_ref netmem;
1109
1110	/* Empty recycle ring */
1111	while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) {
1112		/* Verify the refcnt invariant of cached pages */
1113		if (!(netmem_ref_count(netmem) == 1))
1114			pr_crit("%s() page_pool refcnt %d violation\n",
1115				__func__, netmem_ref_count(netmem));
1116
1117		page_pool_return_netmem(pool, netmem);
1118	}
1119}
1120
1121static void __page_pool_destroy(struct page_pool *pool)
1122{
1123	if (pool->disconnect)
1124		pool->disconnect(pool);
1125
1126	page_pool_unlist(pool);
1127	page_pool_uninit(pool);
1128
1129	if (pool->mp_ops) {
1130		pool->mp_ops->destroy(pool);
1131		static_branch_dec(&page_pool_mem_providers);
1132	}
1133
1134	kfree(pool);
1135}
1136
1137static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
1138{
1139	netmem_ref netmem;
1140
1141	if (pool->destroy_cnt)
1142		return;
1143
1144	/* Empty alloc cache, assume caller made sure this is
1145	 * no-longer in use, and page_pool_alloc_pages() cannot be
1146	 * call concurrently.
1147	 */
1148	while (pool->alloc.count) {
1149		netmem = pool->alloc.cache[--pool->alloc.count];
1150		page_pool_return_netmem(pool, netmem);
1151	}
1152}
1153
1154static void page_pool_scrub(struct page_pool *pool)
1155{
1156	unsigned long id;
1157	void *ptr;
1158
1159	page_pool_empty_alloc_cache_once(pool);
1160	if (!pool->destroy_cnt++ && pool->dma_map) {
1161		if (pool->dma_sync) {
1162			/* Disable page_pool_dma_sync_for_device() */
1163			pool->dma_sync = false;
1164
1165			/* Make sure all concurrent returns that may see the old
1166			 * value of dma_sync (and thus perform a sync) have
1167			 * finished before doing the unmapping below. Skip the
1168			 * wait if the device doesn't actually need syncing, or
1169			 * if there are no outstanding mapped pages.
1170			 */
1171			if (dma_dev_need_sync(pool->p.dev) &&
1172			    !xa_empty(&pool->dma_mapped))
1173				synchronize_net();
1174		}
1175
1176		xa_for_each(&pool->dma_mapped, id, ptr)
1177			__page_pool_release_netmem_dma(pool, page_to_netmem((struct page *)ptr));
1178	}
1179
1180	/* No more consumers should exist, but producers could still
1181	 * be in-flight.
1182	 */
1183	page_pool_empty_ring(pool);
1184}
1185
1186static int page_pool_release(struct page_pool *pool)
1187{
1188	bool in_softirq;
1189	int inflight;
1190
1191	page_pool_scrub(pool);
1192	inflight = page_pool_inflight(pool, true);
1193	/* Acquire producer lock to make sure producers have exited. */
1194	in_softirq = page_pool_producer_lock(pool);
1195	page_pool_producer_unlock(pool, in_softirq);
1196	if (!inflight)
1197		__page_pool_destroy(pool);
1198
1199	return inflight;
1200}
1201
1202static void page_pool_release_retry(struct work_struct *wq)
1203{
1204	struct delayed_work *dwq = to_delayed_work(wq);
1205	struct page_pool *pool = container_of(dwq, typeof(*pool), release_dw);
1206	void *netdev;
1207	int inflight;
1208
1209	inflight = page_pool_release(pool);
1210	/* In rare cases, a driver bug may cause inflight to go negative.
1211	 * Don't reschedule release if inflight is 0 or negative.
1212	 * - If 0, the page_pool has been destroyed
1213	 * - if negative, we will never recover
1214	 * in both cases no reschedule is necessary.
1215	 */
1216	if (inflight <= 0)
1217		return;
1218
1219	/* Periodic warning for page pools the user can't see */
1220	netdev = READ_ONCE(pool->slow.netdev);
1221	if (time_after_eq(jiffies, pool->defer_warn) &&
1222	    (!netdev || netdev == NET_PTR_POISON)) {
1223		int sec = (s32)((u32)jiffies - (u32)pool->defer_start) / HZ;
1224
1225		pr_warn("%s() stalled pool shutdown: id %u, %d inflight %d sec\n",
1226			__func__, pool->user.id, inflight, sec);
1227		pool->defer_warn = jiffies + DEFER_WARN_INTERVAL;
1228	}
1229
1230	/* Still not ready to be disconnected, retry later */
1231	schedule_delayed_work(&pool->release_dw, DEFER_TIME);
1232}
1233
1234void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *),
1235			   const struct xdp_mem_info *mem)
1236{
1237	refcount_inc(&pool->user_cnt);
1238	pool->disconnect = disconnect;
1239	pool->xdp_mem_id = mem->id;
1240}
1241
1242/**
1243 * page_pool_enable_direct_recycling() - mark page pool as owned by NAPI
1244 * @pool: page pool to modify
1245 * @napi: NAPI instance to associate the page pool with
1246 *
1247 * Associate a page pool with a NAPI instance for lockless page recycling.
1248 * This is useful when a new page pool has to be added to a NAPI instance
1249 * without disabling that NAPI instance, to mark the point at which control
1250 * path "hands over" the page pool to the NAPI instance. In most cases driver
1251 * can simply set the @napi field in struct page_pool_params, and does not
1252 * have to call this helper.
1253 *
1254 * The function is idempotent, but does not implement any refcounting.
1255 * Single page_pool_disable_direct_recycling() will disable recycling,
1256 * no matter how many times enable was called.
1257 */
1258void page_pool_enable_direct_recycling(struct page_pool *pool,
1259				       struct napi_struct *napi)
1260{
1261	if (READ_ONCE(pool->p.napi) == napi)
1262		return;
1263	WARN_ON(!napi || pool->p.napi);
1264
1265	mutex_lock(&page_pools_lock);
1266	WRITE_ONCE(pool->p.napi, napi);
1267	mutex_unlock(&page_pools_lock);
1268}
1269EXPORT_SYMBOL(page_pool_enable_direct_recycling);
1270
1271void page_pool_disable_direct_recycling(struct page_pool *pool)
1272{
1273	/* Disable direct recycling based on pool->cpuid.
1274	 * Paired with READ_ONCE() in page_pool_napi_local().
1275	 */
1276	WRITE_ONCE(pool->cpuid, -1);
1277
1278	if (!pool->p.napi)
1279		return;
1280
1281	napi_assert_will_not_race(pool->p.napi);
1282
1283	mutex_lock(&page_pools_lock);
1284	WRITE_ONCE(pool->p.napi, NULL);
1285	mutex_unlock(&page_pools_lock);
1286}
1287EXPORT_SYMBOL(page_pool_disable_direct_recycling);
1288
1289void page_pool_destroy(struct page_pool *pool)
1290{
1291	if (!pool)
1292		return;
1293
1294	if (!page_pool_put(pool))
1295		return;
1296
1297	page_pool_disable_direct_recycling(pool);
1298	page_pool_free_frag(pool);
1299
1300	if (!page_pool_release(pool))
1301		return;
1302
1303	page_pool_detached(pool);
1304	pool->defer_start = jiffies;
1305	pool->defer_warn  = jiffies + DEFER_WARN_INTERVAL;
1306
1307	INIT_DELAYED_WORK(&pool->release_dw, page_pool_release_retry);
1308	schedule_delayed_work(&pool->release_dw, DEFER_TIME);
1309}
1310EXPORT_SYMBOL(page_pool_destroy);
1311
1312/* Caller must provide appropriate safe context, e.g. NAPI. */
1313void page_pool_update_nid(struct page_pool *pool, int new_nid)
1314{
1315	netmem_ref netmem;
1316
1317	trace_page_pool_update_nid(pool, new_nid);
1318	pool->p.nid = new_nid;
1319
1320	/* Flush pool alloc cache, as refill will check NUMA node */
1321	while (pool->alloc.count) {
1322		netmem = pool->alloc.cache[--pool->alloc.count];
1323		page_pool_return_netmem(pool, netmem);
1324	}
1325}
1326EXPORT_SYMBOL(page_pool_update_nid);
1327
1328bool net_mp_niov_set_dma_addr(struct net_iov *niov, dma_addr_t addr)
1329{
1330	return page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), addr);
1331}
1332
1333/* Associate a niov with a page pool. Should follow with a matching
1334 * net_mp_niov_clear_page_pool()
1335 */
1336void net_mp_niov_set_page_pool(struct page_pool *pool, struct net_iov *niov)
1337{
1338	netmem_ref netmem = net_iov_to_netmem(niov);
1339
1340	page_pool_set_pp_info(pool, netmem);
1341
1342	pool->pages_state_hold_cnt++;
1343	trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
1344}
1345
1346/* Disassociate a niov from a page pool. Should only be used in the
1347 * ->release_netmem() path.
1348 */
1349void net_mp_niov_clear_page_pool(struct net_iov *niov)
1350{
1351	netmem_ref netmem = net_iov_to_netmem(niov);
1352
1353	page_pool_clear_pp_info(netmem);
1354}