page_alloc.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/mm/page_alloc.c
   4 *
   5 *  Manages the free list, the system allocates free pages here.
   6 *  Note that kmalloc() lives in slab.c
   7 *
   8 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   9 *  Swap reorganised 29.12.95, Stephen Tweedie
  10 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  11 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  12 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  13 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  14 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  15 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  16 */
  17
  18#include <linux/stddef.h>
  19#include <linux/mm.h>
  20#include <linux/highmem.h>
  21#include <linux/interrupt.h>
  22#include <linux/jiffies.h>
  23#include <linux/compiler.h>
  24#include <linux/kernel.h>
  25#include <linux/kasan.h>
  26#include <linux/kmsan.h>
  27#include <linux/module.h>
  28#include <linux/suspend.h>
  29#include <linux/ratelimit.h>
  30#include <linux/oom.h>
  31#include <linux/topology.h>
  32#include <linux/sysctl.h>
  33#include <linux/cpu.h>
  34#include <linux/cpuset.h>
  35#include <linux/pagevec.h>
  36#include <linux/memory_hotplug.h>
  37#include <linux/nodemask.h>
  38#include <linux/vmstat.h>
  39#include <linux/fault-inject.h>
  40#include <linux/compaction.h>
  41#include <trace/events/kmem.h>
  42#include <trace/events/oom.h>
  43#include <linux/prefetch.h>
  44#include <linux/mm_inline.h>
  45#include <linux/mmu_notifier.h>
  46#include <linux/migrate.h>
  47#include <linux/sched/mm.h>
  48#include <linux/page_owner.h>
  49#include <linux/page_table_check.h>
  50#include <linux/memcontrol.h>
  51#include <linux/ftrace.h>
  52#include <linux/lockdep.h>
  53#include <linux/psi.h>
  54#include <linux/khugepaged.h>
  55#include <linux/delayacct.h>
  56#include <linux/cacheinfo.h>
  57#include <linux/pgalloc_tag.h>
  58#include <asm/div64.h>
  59#include "internal.h"
  60#include "shuffle.h"
  61#include "page_reporting.h"
  62
  63/* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
  64typedef int __bitwise fpi_t;
  65
  66/* No special request */
  67#define FPI_NONE		((__force fpi_t)0)
  68
  69/*
  70 * Skip free page reporting notification for the (possibly merged) page.
  71 * This does not hinder free page reporting from grabbing the page,
  72 * reporting it and marking it "reported" -  it only skips notifying
  73 * the free page reporting infrastructure about a newly freed page. For
  74 * example, used when temporarily pulling a page from a freelist and
  75 * putting it back unmodified.
  76 */
  77#define FPI_SKIP_REPORT_NOTIFY	((__force fpi_t)BIT(0))
  78
  79/*
  80 * Place the (possibly merged) page to the tail of the freelist. Will ignore
  81 * page shuffling (relevant code - e.g., memory onlining - is expected to
  82 * shuffle the whole zone).
  83 *
  84 * Note: No code should rely on this flag for correctness - it's purely
  85 *       to allow for optimizations when handing back either fresh pages
  86 *       (memory onlining) or untouched pages (page isolation, free page
  87 *       reporting).
  88 */
  89#define FPI_TO_TAIL		((__force fpi_t)BIT(1))
  90
  91/* Free the page without taking locks. Rely on trylock only. */
  92#define FPI_TRYLOCK		((__force fpi_t)BIT(2))
  93
  94/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
  95static DEFINE_MUTEX(pcp_batch_high_lock);
  96#define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)
  97
  98#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
  99/*
 100 * On SMP, spin_trylock is sufficient protection.
 101 * On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP.
 102 * Pass flags to a no-op inline function to typecheck and silence the unused
 103 * variable warning.
 104 */
 105static inline void __pcp_trylock_noop(unsigned long *flags) { }
 106#define pcp_trylock_prepare(flags)	__pcp_trylock_noop(&(flags))
 107#define pcp_trylock_finish(flags)	__pcp_trylock_noop(&(flags))
 108#else
 109
 110/* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */
 111#define pcp_trylock_prepare(flags)	local_irq_save(flags)
 112#define pcp_trylock_finish(flags)	local_irq_restore(flags)
 113#endif
 114
 115/*
 116 * Locking a pcp requires a PCP lookup followed by a spinlock. To avoid
 117 * a migration causing the wrong PCP to be locked and remote memory being
 118 * potentially allocated, pin the task to the CPU for the lookup+lock.
 119 * preempt_disable is used on !RT because it is faster than migrate_disable.
 120 * migrate_disable is used on RT because otherwise RT spinlock usage is
 121 * interfered with and a high priority task cannot preempt the allocator.
 122 */
 123#ifndef CONFIG_PREEMPT_RT
 124#define pcpu_task_pin()		preempt_disable()
 125#define pcpu_task_unpin()	preempt_enable()
 126#else
 127#define pcpu_task_pin()		migrate_disable()
 128#define pcpu_task_unpin()	migrate_enable()
 129#endif
 130
 131/*
 132 * Generic helper to lookup and a per-cpu variable with an embedded spinlock.
 133 * Return value should be used with equivalent unlock helper.
 134 */
 135#define pcpu_spin_trylock(type, member, ptr)				\
 136({									\
 137	type *_ret;							\
 138	pcpu_task_pin();						\
 139	_ret = this_cpu_ptr(ptr);					\
 140	if (!spin_trylock(&_ret->member)) {				\
 141		pcpu_task_unpin();					\
 142		_ret = NULL;						\
 143	}								\
 144	_ret;								\
 145})
 146
 147#define pcpu_spin_unlock(member, ptr)					\
 148({									\
 149	spin_unlock(&ptr->member);					\
 150	pcpu_task_unpin();						\
 151})
 152
 153/* struct per_cpu_pages specific helpers. */
 154#define pcp_spin_trylock(ptr, UP_flags)					\
 155({									\
 156	struct per_cpu_pages *__ret;					\
 157	pcp_trylock_prepare(UP_flags);					\
 158	__ret = pcpu_spin_trylock(struct per_cpu_pages, lock, ptr);	\
 159	if (!__ret)							\
 160		pcp_trylock_finish(UP_flags);				\
 161	__ret;								\
 162})
 163
 164#define pcp_spin_unlock(ptr, UP_flags)					\
 165({									\
 166	pcpu_spin_unlock(lock, ptr);					\
 167	pcp_trylock_finish(UP_flags);					\
 168})
 169
 170/*
 171 * With the UP spinlock implementation, when we spin_lock(&pcp->lock) (for i.e.
 172 * a potentially remote cpu drain) and get interrupted by an operation that
 173 * attempts pcp_spin_trylock(), we can't rely on the trylock failure due to UP
 174 * spinlock assumptions making the trylock a no-op. So we have to turn that
 175 * spin_lock() to a spin_lock_irqsave(). This works because on UP there are no
 176 * remote cpu's so we can only be locking the only existing local one.
 177 */
 178#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
 179static inline void __flags_noop(unsigned long *flags) { }
 180#define pcp_spin_lock_maybe_irqsave(ptr, flags)		\
 181({							\
 182	 __flags_noop(&(flags));			\
 183	 spin_lock(&(ptr)->lock);			\
 184})
 185#define pcp_spin_unlock_maybe_irqrestore(ptr, flags)	\
 186({							\
 187	 spin_unlock(&(ptr)->lock);			\
 188	 __flags_noop(&(flags));			\
 189})
 190#else
 191#define pcp_spin_lock_maybe_irqsave(ptr, flags)		\
 192		spin_lock_irqsave(&(ptr)->lock, flags)
 193#define pcp_spin_unlock_maybe_irqrestore(ptr, flags)	\
 194		spin_unlock_irqrestore(&(ptr)->lock, flags)
 195#endif
 196
 197#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
 198DEFINE_PER_CPU(int, numa_node);
 199EXPORT_PER_CPU_SYMBOL(numa_node);
 200#endif
 201
 202DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
 203
 204#ifdef CONFIG_HAVE_MEMORYLESS_NODES
 205/*
 206 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 207 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 208 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 209 * defined in <linux/topology.h>.
 210 */
 211DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */
 212EXPORT_PER_CPU_SYMBOL(_numa_mem_);
 213#endif
 214
 215static DEFINE_MUTEX(pcpu_drain_mutex);
 216
 217#ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
 218volatile unsigned long latent_entropy __latent_entropy;
 219EXPORT_SYMBOL(latent_entropy);
 220#endif
 221
 222/*
 223 * Array of node states.
 224 */
 225nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
 226	[N_POSSIBLE] = NODE_MASK_ALL,
 227	[N_ONLINE] = { { [0] = 1UL } },
 228#ifndef CONFIG_NUMA
 229	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
 230#ifdef CONFIG_HIGHMEM
 231	[N_HIGH_MEMORY] = { { [0] = 1UL } },
 232#endif
 233	[N_MEMORY] = { { [0] = 1UL } },
 234	[N_CPU] = { { [0] = 1UL } },
 235#endif	/* NUMA */
 236};
 237EXPORT_SYMBOL(node_states);
 238
 239gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
 240
 241#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
 242unsigned int pageblock_order __read_mostly;
 243#endif
 244
 245static void __free_pages_ok(struct page *page, unsigned int order,
 246			    fpi_t fpi_flags);
 247
 248/*
 249 * results with 256, 32 in the lowmem_reserve sysctl:
 250 *	1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 251 *	1G machine -> (16M dma, 784M normal, 224M high)
 252 *	NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 253 *	HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
 254 *	HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
 255 *
 256 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 257 * don't need any ZONE_NORMAL reservation
 258 */
 259static int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
 260#ifdef CONFIG_ZONE_DMA
 261	[ZONE_DMA] = 256,
 262#endif
 263#ifdef CONFIG_ZONE_DMA32
 264	[ZONE_DMA32] = 256,
 265#endif
 266	[ZONE_NORMAL] = 32,
 267#ifdef CONFIG_HIGHMEM
 268	[ZONE_HIGHMEM] = 0,
 269#endif
 270	[ZONE_MOVABLE] = 0,
 271};
 272
 273char * const zone_names[MAX_NR_ZONES] = {
 274#ifdef CONFIG_ZONE_DMA
 275	 "DMA",
 276#endif
 277#ifdef CONFIG_ZONE_DMA32
 278	 "DMA32",
 279#endif
 280	 "Normal",
 281#ifdef CONFIG_HIGHMEM
 282	 "HighMem",
 283#endif
 284	 "Movable",
 285#ifdef CONFIG_ZONE_DEVICE
 286	 "Device",
 287#endif
 288};
 289
 290const char * const migratetype_names[MIGRATE_TYPES] = {
 291	"Unmovable",
 292	"Movable",
 293	"Reclaimable",
 294	"HighAtomic",
 295#ifdef CONFIG_CMA
 296	"CMA",
 297#endif
 298#ifdef CONFIG_MEMORY_ISOLATION
 299	"Isolate",
 300#endif
 301};
 302
 303int min_free_kbytes = 1024;
 304int user_min_free_kbytes = -1;
 305static int watermark_boost_factor __read_mostly = 15000;
 306static int watermark_scale_factor = 10;
 307int defrag_mode;
 308
 309/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
 310int movable_zone;
 311EXPORT_SYMBOL(movable_zone);
 312
 313#if MAX_NUMNODES > 1
 314unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
 315unsigned int nr_online_nodes __read_mostly = 1;
 316EXPORT_SYMBOL(nr_node_ids);
 317EXPORT_SYMBOL(nr_online_nodes);
 318#endif
 319
 320static bool page_contains_unaccepted(struct page *page, unsigned int order);
 321static bool cond_accept_memory(struct zone *zone, unsigned int order,
 322			       int alloc_flags);
 323static bool __free_unaccepted(struct page *page);
 324
 325int page_group_by_mobility_disabled __read_mostly;
 326
 327#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
 328/*
 329 * During boot we initialize deferred pages on-demand, as needed, but once
 330 * page_alloc_init_late() has finished, the deferred pages are all initialized,
 331 * and we can permanently disable that path.
 332 */
 333DEFINE_STATIC_KEY_TRUE(deferred_pages);
 334
 335static inline bool deferred_pages_enabled(void)
 336{
 337	return static_branch_unlikely(&deferred_pages);
 338}
 339
 340/*
 341 * deferred_grow_zone() is __init, but it is called from
 342 * get_page_from_freelist() during early boot until deferred_pages permanently
 343 * disables this call. This is why we have refdata wrapper to avoid warning,
 344 * and to ensure that the function body gets unloaded.
 345 */
 346static bool __ref
 347_deferred_grow_zone(struct zone *zone, unsigned int order)
 348{
 349	return deferred_grow_zone(zone, order);
 350}
 351#else
 352static inline bool deferred_pages_enabled(void)
 353{
 354	return false;
 355}
 356
 357static inline bool _deferred_grow_zone(struct zone *zone, unsigned int order)
 358{
 359	return false;
 360}
 361#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
 362
 363/* Return a pointer to the bitmap storing bits affecting a block of pages */
 364static inline unsigned long *get_pageblock_bitmap(const struct page *page,
 365							unsigned long pfn)
 366{
 367#ifdef CONFIG_SPARSEMEM
 368	return section_to_usemap(__pfn_to_section(pfn));
 369#else
 370	return page_zone(page)->pageblock_flags;
 371#endif /* CONFIG_SPARSEMEM */
 372}
 373
 374static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn)
 375{
 376#ifdef CONFIG_SPARSEMEM
 377	pfn &= (PAGES_PER_SECTION-1);
 378#else
 379	pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn);
 380#endif /* CONFIG_SPARSEMEM */
 381	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
 382}
 383
 384static __always_inline bool is_standalone_pb_bit(enum pageblock_bits pb_bit)
 385{
 386	return pb_bit >= PB_compact_skip && pb_bit < __NR_PAGEBLOCK_BITS;
 387}
 388
 389static __always_inline void
 390get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn,
 391			   unsigned long **bitmap_word, unsigned long *bitidx)
 392{
 393	unsigned long *bitmap;
 394	unsigned long word_bitidx;
 395
 396#ifdef CONFIG_MEMORY_ISOLATION
 397	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 8);
 398#else
 399	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
 400#endif
 401	BUILD_BUG_ON(__MIGRATE_TYPE_END > MIGRATETYPE_MASK);
 402	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
 403
 404	bitmap = get_pageblock_bitmap(page, pfn);
 405	*bitidx = pfn_to_bitidx(page, pfn);
 406	word_bitidx = *bitidx / BITS_PER_LONG;
 407	*bitidx &= (BITS_PER_LONG - 1);
 408	*bitmap_word = &bitmap[word_bitidx];
 409}
 410
 411
 412/**
 413 * __get_pfnblock_flags_mask - Return the requested group of flags for
 414 * a pageblock_nr_pages block of pages
 415 * @page: The page within the block of interest
 416 * @pfn: The target page frame number
 417 * @mask: mask of bits that the caller is interested in
 418 *
 419 * Return: pageblock_bits flags
 420 */
 421static unsigned long __get_pfnblock_flags_mask(const struct page *page,
 422					       unsigned long pfn,
 423					       unsigned long mask)
 424{
 425	unsigned long *bitmap_word;
 426	unsigned long bitidx;
 427	unsigned long word;
 428
 429	get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx);
 430	/*
 431	 * This races, without locks, with set_pfnblock_migratetype(). Ensure
 432	 * a consistent read of the memory array, so that results, even though
 433	 * racy, are not corrupted.
 434	 */
 435	word = READ_ONCE(*bitmap_word);
 436	return (word >> bitidx) & mask;
 437}
 438
 439/**
 440 * get_pfnblock_bit - Check if a standalone bit of a pageblock is set
 441 * @page: The page within the block of interest
 442 * @pfn: The target page frame number
 443 * @pb_bit: pageblock bit to check
 444 *
 445 * Return: true if the bit is set, otherwise false
 446 */
 447bool get_pfnblock_bit(const struct page *page, unsigned long pfn,
 448		      enum pageblock_bits pb_bit)
 449{
 450	unsigned long *bitmap_word;
 451	unsigned long bitidx;
 452
 453	if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit)))
 454		return false;
 455
 456	get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx);
 457
 458	return test_bit(bitidx + pb_bit, bitmap_word);
 459}
 460
 461/**
 462 * get_pfnblock_migratetype - Return the migratetype of a pageblock
 463 * @page: The page within the block of interest
 464 * @pfn: The target page frame number
 465 *
 466 * Return: The migratetype of the pageblock
 467 *
 468 * Use get_pfnblock_migratetype() if caller already has both @page and @pfn
 469 * to save a call to page_to_pfn().
 470 */
 471__always_inline enum migratetype
 472get_pfnblock_migratetype(const struct page *page, unsigned long pfn)
 473{
 474	unsigned long mask = MIGRATETYPE_AND_ISO_MASK;
 475	unsigned long flags;
 476
 477	flags = __get_pfnblock_flags_mask(page, pfn, mask);
 478
 479#ifdef CONFIG_MEMORY_ISOLATION
 480	if (flags & BIT(PB_migrate_isolate))
 481		return MIGRATE_ISOLATE;
 482#endif
 483	return flags & MIGRATETYPE_MASK;
 484}
 485
 486/**
 487 * __set_pfnblock_flags_mask - Set the requested group of flags for
 488 * a pageblock_nr_pages block of pages
 489 * @page: The page within the block of interest
 490 * @pfn: The target page frame number
 491 * @flags: The flags to set
 492 * @mask: mask of bits that the caller is interested in
 493 */
 494static void __set_pfnblock_flags_mask(struct page *page, unsigned long pfn,
 495				      unsigned long flags, unsigned long mask)
 496{
 497	unsigned long *bitmap_word;
 498	unsigned long bitidx;
 499	unsigned long word;
 500
 501	get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx);
 502
 503	mask <<= bitidx;
 504	flags <<= bitidx;
 505
 506	word = READ_ONCE(*bitmap_word);
 507	do {
 508	} while (!try_cmpxchg(bitmap_word, &word, (word & ~mask) | flags));
 509}
 510
 511/**
 512 * set_pfnblock_bit - Set a standalone bit of a pageblock
 513 * @page: The page within the block of interest
 514 * @pfn: The target page frame number
 515 * @pb_bit: pageblock bit to set
 516 */
 517void set_pfnblock_bit(const struct page *page, unsigned long pfn,
 518		      enum pageblock_bits pb_bit)
 519{
 520	unsigned long *bitmap_word;
 521	unsigned long bitidx;
 522
 523	if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit)))
 524		return;
 525
 526	get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx);
 527
 528	set_bit(bitidx + pb_bit, bitmap_word);
 529}
 530
 531/**
 532 * clear_pfnblock_bit - Clear a standalone bit of a pageblock
 533 * @page: The page within the block of interest
 534 * @pfn: The target page frame number
 535 * @pb_bit: pageblock bit to clear
 536 */
 537void clear_pfnblock_bit(const struct page *page, unsigned long pfn,
 538			enum pageblock_bits pb_bit)
 539{
 540	unsigned long *bitmap_word;
 541	unsigned long bitidx;
 542
 543	if (WARN_ON_ONCE(!is_standalone_pb_bit(pb_bit)))
 544		return;
 545
 546	get_pfnblock_bitmap_bitidx(page, pfn, &bitmap_word, &bitidx);
 547
 548	clear_bit(bitidx + pb_bit, bitmap_word);
 549}
 550
 551/**
 552 * set_pageblock_migratetype - Set the migratetype of a pageblock
 553 * @page: The page within the block of interest
 554 * @migratetype: migratetype to set
 555 */
 556static void set_pageblock_migratetype(struct page *page,
 557				      enum migratetype migratetype)
 558{
 559	if (unlikely(page_group_by_mobility_disabled &&
 560		     migratetype < MIGRATE_PCPTYPES))
 561		migratetype = MIGRATE_UNMOVABLE;
 562
 563#ifdef CONFIG_MEMORY_ISOLATION
 564	if (migratetype == MIGRATE_ISOLATE) {
 565		VM_WARN_ONCE(1,
 566			"Use set_pageblock_isolate() for pageblock isolation");
 567		return;
 568	}
 569	VM_WARN_ONCE(get_pageblock_isolate(page),
 570		     "Use clear_pageblock_isolate() to unisolate pageblock");
 571	/* MIGRATETYPE_AND_ISO_MASK clears PB_migrate_isolate if it is set */
 572#endif
 573	__set_pfnblock_flags_mask(page, page_to_pfn(page),
 574				  (unsigned long)migratetype,
 575				  MIGRATETYPE_AND_ISO_MASK);
 576}
 577
 578void __meminit init_pageblock_migratetype(struct page *page,
 579					  enum migratetype migratetype,
 580					  bool isolate)
 581{
 582	unsigned long flags;
 583
 584	if (unlikely(page_group_by_mobility_disabled &&
 585		     migratetype < MIGRATE_PCPTYPES))
 586		migratetype = MIGRATE_UNMOVABLE;
 587
 588	flags = migratetype;
 589
 590#ifdef CONFIG_MEMORY_ISOLATION
 591	if (migratetype == MIGRATE_ISOLATE) {
 592		VM_WARN_ONCE(
 593			1,
 594			"Set isolate=true to isolate pageblock with a migratetype");
 595		return;
 596	}
 597	if (isolate)
 598		flags |= BIT(PB_migrate_isolate);
 599#endif
 600	__set_pfnblock_flags_mask(page, page_to_pfn(page), flags,
 601				  MIGRATETYPE_AND_ISO_MASK);
 602}
 603
 604#ifdef CONFIG_DEBUG_VM
 605static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
 606{
 607	int ret;
 608	unsigned seq;
 609	unsigned long pfn = page_to_pfn(page);
 610	unsigned long sp, start_pfn;
 611
 612	do {
 613		seq = zone_span_seqbegin(zone);
 614		start_pfn = zone->zone_start_pfn;
 615		sp = zone->spanned_pages;
 616		ret = !zone_spans_pfn(zone, pfn);
 617	} while (zone_span_seqretry(zone, seq));
 618
 619	if (ret)
 620		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
 621			pfn, zone_to_nid(zone), zone->name,
 622			start_pfn, start_pfn + sp);
 623
 624	return ret;
 625}
 626
 627/*
 628 * Temporary debugging check for pages not lying within a given zone.
 629 */
 630static bool __maybe_unused bad_range(struct zone *zone, struct page *page)
 631{
 632	if (page_outside_zone_boundaries(zone, page))
 633		return true;
 634	if (zone != page_zone(page))
 635		return true;
 636
 637	return false;
 638}
 639#else
 640static inline bool __maybe_unused bad_range(struct zone *zone, struct page *page)
 641{
 642	return false;
 643}
 644#endif
 645
 646static void bad_page(struct page *page, const char *reason)
 647{
 648	static unsigned long resume;
 649	static unsigned long nr_shown;
 650	static unsigned long nr_unshown;
 651
 652	/*
 653	 * Allow a burst of 60 reports, then keep quiet for that minute;
 654	 * or allow a steady drip of one report per second.
 655	 */
 656	if (nr_shown == 60) {
 657		if (time_before(jiffies, resume)) {
 658			nr_unshown++;
 659			goto out;
 660		}
 661		if (nr_unshown) {
 662			pr_alert(
 663			      "BUG: Bad page state: %lu messages suppressed\n",
 664				nr_unshown);
 665			nr_unshown = 0;
 666		}
 667		nr_shown = 0;
 668	}
 669	if (nr_shown++ == 0)
 670		resume = jiffies + 60 * HZ;
 671
 672	pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
 673		current->comm, page_to_pfn(page));
 674	dump_page(page, reason);
 675
 676	print_modules();
 677	dump_stack();
 678out:
 679	/* Leave bad fields for debug, except PageBuddy could make trouble */
 680	if (PageBuddy(page))
 681		__ClearPageBuddy(page);
 682	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 683}
 684
 685static inline unsigned int order_to_pindex(int migratetype, int order)
 686{
 687
 688#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 689	bool movable;
 690	if (order > PAGE_ALLOC_COSTLY_ORDER) {
 691		VM_BUG_ON(order != HPAGE_PMD_ORDER);
 692
 693		movable = migratetype == MIGRATE_MOVABLE;
 694
 695		return NR_LOWORDER_PCP_LISTS + movable;
 696	}
 697#else
 698	VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
 699#endif
 700
 701	return (MIGRATE_PCPTYPES * order) + migratetype;
 702}
 703
 704static inline int pindex_to_order(unsigned int pindex)
 705{
 706	int order = pindex / MIGRATE_PCPTYPES;
 707
 708#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 709	if (pindex >= NR_LOWORDER_PCP_LISTS)
 710		order = HPAGE_PMD_ORDER;
 711#else
 712	VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
 713#endif
 714
 715	return order;
 716}
 717
 718static inline bool pcp_allowed_order(unsigned int order)
 719{
 720	if (order <= PAGE_ALLOC_COSTLY_ORDER)
 721		return true;
 722#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 723	if (order == HPAGE_PMD_ORDER)
 724		return true;
 725#endif
 726	return false;
 727}
 728
 729/*
 730 * Higher-order pages are called "compound pages".  They are structured thusly:
 731 *
 732 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
 733 *
 734 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
 735 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
 736 *
 737 * The first tail page's ->compound_order holds the order of allocation.
 738 * This usage means that zero-order pages may not be compound.
 739 */
 740
 741void prep_compound_page(struct page *page, unsigned int order)
 742{
 743	int i;
 744	int nr_pages = 1 << order;
 745
 746	__SetPageHead(page);
 747	for (i = 1; i < nr_pages; i++)
 748		prep_compound_tail(page, i);
 749
 750	prep_compound_head(page, order);
 751}
 752
 753static inline void set_buddy_order(struct page *page, unsigned int order)
 754{
 755	set_page_private(page, order);
 756	__SetPageBuddy(page);
 757}
 758
 759#ifdef CONFIG_COMPACTION
 760static inline struct capture_control *task_capc(struct zone *zone)
 761{
 762	struct capture_control *capc = current->capture_control;
 763
 764	return unlikely(capc) &&
 765		!(current->flags & PF_KTHREAD) &&
 766		!capc->page &&
 767		capc->cc->zone == zone ? capc : NULL;
 768}
 769
 770static inline bool
 771compaction_capture(struct capture_control *capc, struct page *page,
 772		   int order, int migratetype)
 773{
 774	if (!capc || order != capc->cc->order)
 775		return false;
 776
 777	/* Do not accidentally pollute CMA or isolated regions*/
 778	if (is_migrate_cma(migratetype) ||
 779	    is_migrate_isolate(migratetype))
 780		return false;
 781
 782	/*
 783	 * Do not let lower order allocations pollute a movable pageblock
 784	 * unless compaction is also requesting movable pages.
 785	 * This might let an unmovable request use a reclaimable pageblock
 786	 * and vice-versa but no more than normal fallback logic which can
 787	 * have trouble finding a high-order free page.
 788	 */
 789	if (order < pageblock_order && migratetype == MIGRATE_MOVABLE &&
 790	    capc->cc->migratetype != MIGRATE_MOVABLE)
 791		return false;
 792
 793	if (migratetype != capc->cc->migratetype)
 794		trace_mm_page_alloc_extfrag(page, capc->cc->order, order,
 795					    capc->cc->migratetype, migratetype);
 796
 797	capc->page = page;
 798	return true;
 799}
 800
 801#else
 802static inline struct capture_control *task_capc(struct zone *zone)
 803{
 804	return NULL;
 805}
 806
 807static inline bool
 808compaction_capture(struct capture_control *capc, struct page *page,
 809		   int order, int migratetype)
 810{
 811	return false;
 812}
 813#endif /* CONFIG_COMPACTION */
 814
 815static inline void account_freepages(struct zone *zone, int nr_pages,
 816				     int migratetype)
 817{
 818	lockdep_assert_held(&zone->lock);
 819
 820	if (is_migrate_isolate(migratetype))
 821		return;
 822
 823	__mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
 824
 825	if (is_migrate_cma(migratetype))
 826		__mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
 827	else if (migratetype == MIGRATE_HIGHATOMIC)
 828		WRITE_ONCE(zone->nr_free_highatomic,
 829			   zone->nr_free_highatomic + nr_pages);
 830}
 831
 832/* Used for pages not on another list */
 833static inline void __add_to_free_list(struct page *page, struct zone *zone,
 834				      unsigned int order, int migratetype,
 835				      bool tail)
 836{
 837	struct free_area *area = &zone->free_area[order];
 838	int nr_pages = 1 << order;
 839
 840	VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype,
 841		     "page type is %d, passed migratetype is %d (nr=%d)\n",
 842		     get_pageblock_migratetype(page), migratetype, nr_pages);
 843
 844	if (tail)
 845		list_add_tail(&page->buddy_list, &area->free_list[migratetype]);
 846	else
 847		list_add(&page->buddy_list, &area->free_list[migratetype]);
 848	area->nr_free++;
 849
 850	if (order >= pageblock_order && !is_migrate_isolate(migratetype))
 851		__mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, nr_pages);
 852}
 853
 854/*
 855 * Used for pages which are on another list. Move the pages to the tail
 856 * of the list - so the moved pages won't immediately be considered for
 857 * allocation again (e.g., optimization for memory onlining).
 858 */
 859static inline void move_to_free_list(struct page *page, struct zone *zone,
 860				     unsigned int order, int old_mt, int new_mt)
 861{
 862	struct free_area *area = &zone->free_area[order];
 863	int nr_pages = 1 << order;
 864
 865	/* Free page moving can fail, so it happens before the type update */
 866	VM_WARN_ONCE(get_pageblock_migratetype(page) != old_mt,
 867		     "page type is %d, passed migratetype is %d (nr=%d)\n",
 868		     get_pageblock_migratetype(page), old_mt, nr_pages);
 869
 870	list_move_tail(&page->buddy_list, &area->free_list[new_mt]);
 871
 872	account_freepages(zone, -nr_pages, old_mt);
 873	account_freepages(zone, nr_pages, new_mt);
 874
 875	if (order >= pageblock_order &&
 876	    is_migrate_isolate(old_mt) != is_migrate_isolate(new_mt)) {
 877		if (!is_migrate_isolate(old_mt))
 878			nr_pages = -nr_pages;
 879		__mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, nr_pages);
 880	}
 881}
 882
 883static inline void __del_page_from_free_list(struct page *page, struct zone *zone,
 884					     unsigned int order, int migratetype)
 885{
 886	int nr_pages = 1 << order;
 887
 888        VM_WARN_ONCE(get_pageblock_migratetype(page) != migratetype,
 889		     "page type is %d, passed migratetype is %d (nr=%d)\n",
 890		     get_pageblock_migratetype(page), migratetype, nr_pages);
 891
 892	/* clear reported state and update reported page count */
 893	if (page_reported(page))
 894		__ClearPageReported(page);
 895
 896	list_del(&page->buddy_list);
 897	__ClearPageBuddy(page);
 898	set_page_private(page, 0);
 899	zone->free_area[order].nr_free--;
 900
 901	if (order >= pageblock_order && !is_migrate_isolate(migratetype))
 902		__mod_zone_page_state(zone, NR_FREE_PAGES_BLOCKS, -nr_pages);
 903}
 904
 905static inline void del_page_from_free_list(struct page *page, struct zone *zone,
 906					   unsigned int order, int migratetype)
 907{
 908	__del_page_from_free_list(page, zone, order, migratetype);
 909	account_freepages(zone, -(1 << order), migratetype);
 910}
 911
 912static inline struct page *get_page_from_free_area(struct free_area *area,
 913					    int migratetype)
 914{
 915	return list_first_entry_or_null(&area->free_list[migratetype],
 916					struct page, buddy_list);
 917}
 918
 919/*
 920 * If this is less than the 2nd largest possible page, check if the buddy
 921 * of the next-higher order is free. If it is, it's possible
 922 * that pages are being freed that will coalesce soon. In case,
 923 * that is happening, add the free page to the tail of the list
 924 * so it's less likely to be used soon and more likely to be merged
 925 * as a 2-level higher order page
 926 */
 927static inline bool
 928buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
 929		   struct page *page, unsigned int order)
 930{
 931	unsigned long higher_page_pfn;
 932	struct page *higher_page;
 933
 934	if (order >= MAX_PAGE_ORDER - 1)
 935		return false;
 936
 937	higher_page_pfn = buddy_pfn & pfn;
 938	higher_page = page + (higher_page_pfn - pfn);
 939
 940	return find_buddy_page_pfn(higher_page, higher_page_pfn, order + 1,
 941			NULL) != NULL;
 942}
 943
 944static void change_pageblock_range(struct page *pageblock_page,
 945				   int start_order, int migratetype)
 946{
 947	int nr_pageblocks = 1 << (start_order - pageblock_order);
 948
 949	while (nr_pageblocks--) {
 950		set_pageblock_migratetype(pageblock_page, migratetype);
 951		pageblock_page += pageblock_nr_pages;
 952	}
 953}
 954
 955/*
 956 * Freeing function for a buddy system allocator.
 957 *
 958 * The concept of a buddy system is to maintain direct-mapped table
 959 * (containing bit values) for memory blocks of various "orders".
 960 * The bottom level table contains the map for the smallest allocatable
 961 * units of memory (here, pages), and each level above it describes
 962 * pairs of units from the levels below, hence, "buddies".
 963 * At a high level, all that happens here is marking the table entry
 964 * at the bottom level available, and propagating the changes upward
 965 * as necessary, plus some accounting needed to play nicely with other
 966 * parts of the VM system.
 967 * At each level, we keep a list of pages, which are heads of continuous
 968 * free pages of length of (1 << order) and marked with PageBuddy.
 969 * Page's order is recorded in page_private(page) field.
 970 * So when we are allocating or freeing one, we can derive the state of the
 971 * other.  That is, if we allocate a small block, and both were
 972 * free, the remainder of the region must be split into blocks.
 973 * If a block is freed, and its buddy is also free, then this
 974 * triggers coalescing into a block of larger size.
 975 *
 976 * -- nyc
 977 */
 978
 979static inline void __free_one_page(struct page *page,
 980		unsigned long pfn,
 981		struct zone *zone, unsigned int order,
 982		int migratetype, fpi_t fpi_flags)
 983{
 984	struct capture_control *capc = task_capc(zone);
 985	unsigned long buddy_pfn = 0;
 986	unsigned long combined_pfn;
 987	struct page *buddy;
 988	bool to_tail;
 989
 990	VM_BUG_ON(!zone_is_initialized(zone));
 991	VM_BUG_ON_PAGE(page->flags.f & PAGE_FLAGS_CHECK_AT_PREP, page);
 992
 993	VM_BUG_ON(migratetype == -1);
 994	VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
 995	VM_BUG_ON_PAGE(bad_range(zone, page), page);
 996
 997	account_freepages(zone, 1 << order, migratetype);
 998
 999	while (order < MAX_PAGE_ORDER) {
1000		int buddy_mt = migratetype;
1001
1002		if (compaction_capture(capc, page, order, migratetype)) {
1003			account_freepages(zone, -(1 << order), migratetype);
1004			return;
1005		}
1006
1007		buddy = find_buddy_page_pfn(page, pfn, order, &buddy_pfn);
1008		if (!buddy)
1009			goto done_merging;
1010
1011		if (unlikely(order >= pageblock_order)) {
1012			/*
1013			 * We want to prevent merge between freepages on pageblock
1014			 * without fallbacks and normal pageblock. Without this,
1015			 * pageblock isolation could cause incorrect freepage or CMA
1016			 * accounting or HIGHATOMIC accounting.
1017			 */
1018			buddy_mt = get_pfnblock_migratetype(buddy, buddy_pfn);
1019
1020			if (migratetype != buddy_mt &&
1021			    (!migratetype_is_mergeable(migratetype) ||
1022			     !migratetype_is_mergeable(buddy_mt)))
1023				goto done_merging;
1024		}
1025
1026		/*
1027		 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
1028		 * merge with it and move up one order.
1029		 */
1030		if (page_is_guard(buddy))
1031			clear_page_guard(zone, buddy, order);
1032		else
1033			__del_page_from_free_list(buddy, zone, order, buddy_mt);
1034
1035		if (unlikely(buddy_mt != migratetype)) {
1036			/*
1037			 * Match buddy type. This ensures that an
1038			 * expand() down the line puts the sub-blocks
1039			 * on the right freelists.
1040			 */
1041			change_pageblock_range(buddy, order, migratetype);
1042		}
1043
1044		combined_pfn = buddy_pfn & pfn;
1045		page = page + (combined_pfn - pfn);
1046		pfn = combined_pfn;
1047		order++;
1048	}
1049
1050done_merging:
1051	set_buddy_order(page, order);
1052
1053	if (fpi_flags & FPI_TO_TAIL)
1054		to_tail = true;
1055	else if (is_shuffle_order(order))
1056		to_tail = shuffle_pick_tail();
1057	else
1058		to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
1059
1060	__add_to_free_list(page, zone, order, migratetype, to_tail);
1061
1062	/* Notify page reporting subsystem of freed page */
1063	if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
1064		page_reporting_notify_free(order);
1065}
1066
1067/*
1068 * A bad page could be due to a number of fields. Instead of multiple branches,
1069 * try and check multiple fields with one check. The caller must do a detailed
1070 * check if necessary.
1071 */
1072static inline bool page_expected_state(struct page *page,
1073					unsigned long check_flags)
1074{
1075	if (unlikely(atomic_read(&page->_mapcount) != -1))
1076		return false;
1077
1078	if (unlikely((unsigned long)page->mapping |
1079			page_ref_count(page) |
1080#ifdef CONFIG_MEMCG
1081			page->memcg_data |
1082#endif
1083			page_pool_page_is_pp(page) |
1084			(page->flags.f & check_flags)))
1085		return false;
1086
1087	return true;
1088}
1089
1090static const char *page_bad_reason(struct page *page, unsigned long flags)
1091{
1092	const char *bad_reason = NULL;
1093
1094	if (unlikely(atomic_read(&page->_mapcount) != -1))
1095		bad_reason = "nonzero mapcount";
1096	if (unlikely(page->mapping != NULL))
1097		bad_reason = "non-NULL mapping";
1098	if (unlikely(page_ref_count(page) != 0))
1099		bad_reason = "nonzero _refcount";
1100	if (unlikely(page->flags.f & flags)) {
1101		if (flags == PAGE_FLAGS_CHECK_AT_PREP)
1102			bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
1103		else
1104			bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
1105	}
1106#ifdef CONFIG_MEMCG
1107	if (unlikely(page->memcg_data))
1108		bad_reason = "page still charged to cgroup";
1109#endif
1110	if (unlikely(page_pool_page_is_pp(page)))
1111		bad_reason = "page_pool leak";
1112	return bad_reason;
1113}
1114
1115static inline bool free_page_is_bad(struct page *page)
1116{
1117	if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
1118		return false;
1119
1120	/* Something has gone sideways, find it */
1121	bad_page(page, page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
1122	return true;
1123}
1124
1125static inline bool is_check_pages_enabled(void)
1126{
1127	return static_branch_unlikely(&check_pages_enabled);
1128}
1129
1130static int free_tail_page_prepare(struct page *head_page, struct page *page)
1131{
1132	struct folio *folio = (struct folio *)head_page;
1133	int ret = 1;
1134
1135	/*
1136	 * We rely page->lru.next never has bit 0 set, unless the page
1137	 * is PageTail(). Let's make sure that's true even for poisoned ->lru.
1138	 */
1139	BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
1140
1141	if (!is_check_pages_enabled()) {
1142		ret = 0;
1143		goto out;
1144	}
1145	switch (page - head_page) {
1146	case 1:
1147		/* the first tail page: these may be in place of ->mapping */
1148		if (unlikely(folio_large_mapcount(folio))) {
1149			bad_page(page, "nonzero large_mapcount");
1150			goto out;
1151		}
1152		if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT) &&
1153		    unlikely(atomic_read(&folio->_nr_pages_mapped))) {
1154			bad_page(page, "nonzero nr_pages_mapped");
1155			goto out;
1156		}
1157		if (IS_ENABLED(CONFIG_MM_ID)) {
1158			if (unlikely(folio->_mm_id_mapcount[0] != -1)) {
1159				bad_page(page, "nonzero mm mapcount 0");
1160				goto out;
1161			}
1162			if (unlikely(folio->_mm_id_mapcount[1] != -1)) {
1163				bad_page(page, "nonzero mm mapcount 1");
1164				goto out;
1165			}
1166		}
1167		if (IS_ENABLED(CONFIG_64BIT)) {
1168			if (unlikely(atomic_read(&folio->_entire_mapcount) + 1)) {
1169				bad_page(page, "nonzero entire_mapcount");
1170				goto out;
1171			}
1172			if (unlikely(atomic_read(&folio->_pincount))) {
1173				bad_page(page, "nonzero pincount");
1174				goto out;
1175			}
1176		}
1177		break;
1178	case 2:
1179		/* the second tail page: deferred_list overlaps ->mapping */
1180		if (unlikely(!list_empty(&folio->_deferred_list))) {
1181			bad_page(page, "on deferred list");
1182			goto out;
1183		}
1184		if (!IS_ENABLED(CONFIG_64BIT)) {
1185			if (unlikely(atomic_read(&folio->_entire_mapcount) + 1)) {
1186				bad_page(page, "nonzero entire_mapcount");
1187				goto out;
1188			}
1189			if (unlikely(atomic_read(&folio->_pincount))) {
1190				bad_page(page, "nonzero pincount");
1191				goto out;
1192			}
1193		}
1194		break;
1195	case 3:
1196		/* the third tail page: hugetlb specifics overlap ->mappings */
1197		if (IS_ENABLED(CONFIG_HUGETLB_PAGE))
1198			break;
1199		fallthrough;
1200	default:
1201		if (page->mapping != TAIL_MAPPING) {
1202			bad_page(page, "corrupted mapping in tail page");
1203			goto out;
1204		}
1205		break;
1206	}
1207	if (unlikely(!PageTail(page))) {
1208		bad_page(page, "PageTail not set");
1209		goto out;
1210	}
1211	if (unlikely(compound_head(page) != head_page)) {
1212		bad_page(page, "compound_head not consistent");
1213		goto out;
1214	}
1215	ret = 0;
1216out:
1217	page->mapping = NULL;
1218	clear_compound_head(page);
1219	return ret;
1220}
1221
1222/*
1223 * Skip KASAN memory poisoning when either:
1224 *
1225 * 1. For generic KASAN: deferred memory initialization has not yet completed.
1226 *    Tag-based KASAN modes skip pages freed via deferred memory initialization
1227 *    using page tags instead (see below).
1228 * 2. For tag-based KASAN modes: the page has a match-all KASAN tag, indicating
1229 *    that error detection is disabled for accesses via the page address.
1230 *
1231 * Pages will have match-all tags in the following circumstances:
1232 *
1233 * 1. Pages are being initialized for the first time, including during deferred
1234 *    memory init; see the call to page_kasan_tag_reset in __init_single_page.
1235 * 2. The allocation was not unpoisoned due to __GFP_SKIP_KASAN, with the
1236 *    exception of pages unpoisoned by kasan_unpoison_vmalloc.
1237 * 3. The allocation was excluded from being checked due to sampling,
1238 *    see the call to kasan_unpoison_pages.
1239 *
1240 * Poisoning pages during deferred memory init will greatly lengthen the
1241 * process and cause problem in large memory systems as the deferred pages
1242 * initialization is done with interrupt disabled.
1243 *
1244 * Assuming that there will be no reference to those newly initialized
1245 * pages before they are ever allocated, this should have no effect on
1246 * KASAN memory tracking as the poison will be properly inserted at page
1247 * allocation time. The only corner case is when pages are allocated by
1248 * on-demand allocation and then freed again before the deferred pages
1249 * initialization is done, but this is not likely to happen.
1250 */
1251static inline bool should_skip_kasan_poison(struct page *page)
1252{
1253	if (IS_ENABLED(CONFIG_KASAN_GENERIC))
1254		return deferred_pages_enabled();
1255
1256	return page_kasan_tag(page) == KASAN_TAG_KERNEL;
1257}
1258
1259static void kernel_init_pages(struct page *page, int numpages)
1260{
1261	int i;
1262
1263	/* s390's use of memset() could override KASAN redzones. */
1264	kasan_disable_current();
1265	for (i = 0; i < numpages; i++)
1266		clear_highpage_kasan_tagged(page + i);
1267	kasan_enable_current();
1268}
1269
1270#ifdef CONFIG_MEM_ALLOC_PROFILING
1271
1272/* Should be called only if mem_alloc_profiling_enabled() */
1273void __clear_page_tag_ref(struct page *page)
1274{
1275	union pgtag_ref_handle handle;
1276	union codetag_ref ref;
1277
1278	if (get_page_tag_ref(page, &ref, &handle)) {
1279		set_codetag_empty(&ref);
1280		update_page_tag_ref(handle, &ref);
1281		put_page_tag_ref(handle);
1282	}
1283}
1284
1285/* Should be called only if mem_alloc_profiling_enabled() */
1286static noinline
1287void __pgalloc_tag_add(struct page *page, struct task_struct *task,
1288		       unsigned int nr)
1289{
1290	union pgtag_ref_handle handle;
1291	union codetag_ref ref;
1292
1293	if (get_page_tag_ref(page, &ref, &handle)) {
1294		alloc_tag_add(&ref, task->alloc_tag, PAGE_SIZE * nr);
1295		update_page_tag_ref(handle, &ref);
1296		put_page_tag_ref(handle);
1297	}
1298}
1299
1300static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
1301				   unsigned int nr)
1302{
1303	if (mem_alloc_profiling_enabled())
1304		__pgalloc_tag_add(page, task, nr);
1305}
1306
1307/* Should be called only if mem_alloc_profiling_enabled() */
1308static noinline
1309void __pgalloc_tag_sub(struct page *page, unsigned int nr)
1310{
1311	union pgtag_ref_handle handle;
1312	union codetag_ref ref;
1313
1314	if (get_page_tag_ref(page, &ref, &handle)) {
1315		alloc_tag_sub(&ref, PAGE_SIZE * nr);
1316		update_page_tag_ref(handle, &ref);
1317		put_page_tag_ref(handle);
1318	}
1319}
1320
1321static inline void pgalloc_tag_sub(struct page *page, unsigned int nr)
1322{
1323	if (mem_alloc_profiling_enabled())
1324		__pgalloc_tag_sub(page, nr);
1325}
1326
1327/* When tag is not NULL, assuming mem_alloc_profiling_enabled */
1328static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
1329{
1330	if (tag)
1331		this_cpu_sub(tag->counters->bytes, PAGE_SIZE * nr);
1332}
1333
1334#else /* CONFIG_MEM_ALLOC_PROFILING */
1335
1336static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
1337				   unsigned int nr) {}
1338static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
1339static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
1340
1341#endif /* CONFIG_MEM_ALLOC_PROFILING */
1342
1343__always_inline bool __free_pages_prepare(struct page *page,
1344					  unsigned int order, fpi_t fpi_flags)
1345{
1346	int bad = 0;
1347	bool skip_kasan_poison = should_skip_kasan_poison(page);
1348	bool init = want_init_on_free();
1349	bool compound = PageCompound(page);
1350	struct folio *folio = page_folio(page);
1351
1352	VM_BUG_ON_PAGE(PageTail(page), page);
1353
1354	trace_mm_page_free(page, order);
1355	kmsan_free_page(page, order);
1356
1357	if (memcg_kmem_online() && PageMemcgKmem(page))
1358		__memcg_kmem_uncharge_page(page, order);
1359
1360	/*
1361	 * In rare cases, when truncation or holepunching raced with
1362	 * munlock after VM_LOCKED was cleared, Mlocked may still be
1363	 * found set here.  This does not indicate a problem, unless
1364	 * "unevictable_pgs_cleared" appears worryingly large.
1365	 */
1366	if (unlikely(folio_test_mlocked(folio))) {
1367		long nr_pages = folio_nr_pages(folio);
1368
1369		__folio_clear_mlocked(folio);
1370		zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages);
1371		count_vm_events(UNEVICTABLE_PGCLEARED, nr_pages);
1372	}
1373
1374	if (unlikely(PageHWPoison(page)) && !order) {
1375		/* Do not let hwpoison pages hit pcplists/buddy */
1376		reset_page_owner(page, order);
1377		page_table_check_free(page, order);
1378		pgalloc_tag_sub(page, 1 << order);
1379
1380		/*
1381		 * The page is isolated and accounted for.
1382		 * Mark the codetag as empty to avoid accounting error
1383		 * when the page is freed by unpoison_memory().
1384		 */
1385		clear_page_tag_ref(page);
1386		return false;
1387	}
1388
1389	VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
1390
1391	/*
1392	 * Check tail pages before head page information is cleared to
1393	 * avoid checking PageCompound for order-0 pages.
1394	 */
1395	if (unlikely(order)) {
1396		int i;
1397
1398		if (compound) {
1399			page[1].flags.f &= ~PAGE_FLAGS_SECOND;
1400#ifdef NR_PAGES_IN_LARGE_FOLIO
1401			folio->_nr_pages = 0;
1402#endif
1403		}
1404		for (i = 1; i < (1 << order); i++) {
1405			if (compound)
1406				bad += free_tail_page_prepare(page, page + i);
1407			if (is_check_pages_enabled()) {
1408				if (free_page_is_bad(page + i)) {
1409					bad++;
1410					continue;
1411				}
1412			}
1413			(page + i)->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
1414		}
1415	}
1416	if (folio_test_anon(folio)) {
1417		mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1);
1418		folio->mapping = NULL;
1419	}
1420	if (unlikely(page_has_type(page)))
1421		/* Reset the page_type (which overlays _mapcount) */
1422		page->page_type = UINT_MAX;
1423
1424	if (is_check_pages_enabled()) {
1425		if (free_page_is_bad(page))
1426			bad++;
1427		if (bad)
1428			return false;
1429	}
1430
1431	page_cpupid_reset_last(page);
1432	page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
1433	page->private = 0;
1434	reset_page_owner(page, order);
1435	page_table_check_free(page, order);
1436	pgalloc_tag_sub(page, 1 << order);
1437
1438	if (!PageHighMem(page) && !(fpi_flags & FPI_TRYLOCK)) {
1439		debug_check_no_locks_freed(page_address(page),
1440					   PAGE_SIZE << order);
1441		debug_check_no_obj_freed(page_address(page),
1442					   PAGE_SIZE << order);
1443	}
1444
1445	kernel_poison_pages(page, 1 << order);
1446
1447	/*
1448	 * As memory initialization might be integrated into KASAN,
1449	 * KASAN poisoning and memory initialization code must be
1450	 * kept together to avoid discrepancies in behavior.
1451	 *
1452	 * With hardware tag-based KASAN, memory tags must be set before the
1453	 * page becomes unavailable via debug_pagealloc or arch_free_page.
1454	 */
1455	if (!skip_kasan_poison) {
1456		kasan_poison_pages(page, order, init);
1457
1458		/* Memory is already initialized if KASAN did it internally. */
1459		if (kasan_has_integrated_init())
1460			init = false;
1461	}
1462	if (init)
1463		kernel_init_pages(page, 1 << order);
1464
1465	/*
1466	 * arch_free_page() can make the page's contents inaccessible.  s390
1467	 * does this.  So nothing which can access the page's contents should
1468	 * happen after this.
1469	 */
1470	arch_free_page(page, order);
1471
1472	debug_pagealloc_unmap_pages(page, 1 << order);
1473
1474	return true;
1475}
1476
1477bool free_pages_prepare(struct page *page, unsigned int order)
1478{
1479	return __free_pages_prepare(page, order, FPI_NONE);
1480}
1481
1482/*
1483 * Frees a number of pages from the PCP lists
1484 * Assumes all pages on list are in same zone.
1485 * count is the number of pages to free.
1486 */
1487static void free_pcppages_bulk(struct zone *zone, int count,
1488					struct per_cpu_pages *pcp,
1489					int pindex)
1490{
1491	unsigned long flags;
1492	unsigned int order;
1493	struct page *page;
1494
1495	/*
1496	 * Ensure proper count is passed which otherwise would stuck in the
1497	 * below while (list_empty(list)) loop.
1498	 */
1499	count = min(pcp->count, count);
1500
1501	/* Ensure requested pindex is drained first. */
1502	pindex = pindex - 1;
1503
1504	spin_lock_irqsave(&zone->lock, flags);
1505
1506	while (count > 0) {
1507		struct list_head *list;
1508		int nr_pages;
1509
1510		/* Remove pages from lists in a round-robin fashion. */
1511		do {
1512			if (++pindex > NR_PCP_LISTS - 1)
1513				pindex = 0;
1514			list = &pcp->lists[pindex];
1515		} while (list_empty(list));
1516
1517		order = pindex_to_order(pindex);
1518		nr_pages = 1 << order;
1519		do {
1520			unsigned long pfn;
1521			int mt;
1522
1523			page = list_last_entry(list, struct page, pcp_list);
1524			pfn = page_to_pfn(page);
1525			mt = get_pfnblock_migratetype(page, pfn);
1526
1527			/* must delete to avoid corrupting pcp list */
1528			list_del(&page->pcp_list);
1529			count -= nr_pages;
1530			pcp->count -= nr_pages;
1531
1532			__free_one_page(page, pfn, zone, order, mt, FPI_NONE);
1533			trace_mm_page_pcpu_drain(page, order, mt);
1534		} while (count > 0 && !list_empty(list));
1535	}
1536
1537	spin_unlock_irqrestore(&zone->lock, flags);
1538}
1539
1540/* Split a multi-block free page into its individual pageblocks. */
1541static void split_large_buddy(struct zone *zone, struct page *page,
1542			      unsigned long pfn, int order, fpi_t fpi)
1543{
1544	unsigned long end = pfn + (1 << order);
1545
1546	VM_WARN_ON_ONCE(!IS_ALIGNED(pfn, 1 << order));
1547	/* Caller removed page from freelist, buddy info cleared! */
1548	VM_WARN_ON_ONCE(PageBuddy(page));
1549
1550	if (order > pageblock_order)
1551		order = pageblock_order;
1552
1553	do {
1554		int mt = get_pfnblock_migratetype(page, pfn);
1555
1556		__free_one_page(page, pfn, zone, order, mt, fpi);
1557		pfn += 1 << order;
1558		if (pfn == end)
1559			break;
1560		page = pfn_to_page(pfn);
1561	} while (1);
1562}
1563
1564static void add_page_to_zone_llist(struct zone *zone, struct page *page,
1565				   unsigned int order)
1566{
1567	/* Remember the order */
1568	page->private = order;
1569	/* Add the page to the free list */
1570	llist_add(&page->pcp_llist, &zone->trylock_free_pages);
1571}
1572
1573static void free_one_page(struct zone *zone, struct page *page,
1574			  unsigned long pfn, unsigned int order,
1575			  fpi_t fpi_flags)
1576{
1577	struct llist_head *llhead;
1578	unsigned long flags;
1579
1580	if (unlikely(fpi_flags & FPI_TRYLOCK)) {
1581		if (!spin_trylock_irqsave(&zone->lock, flags)) {
1582			add_page_to_zone_llist(zone, page, order);
1583			return;
1584		}
1585	} else {
1586		spin_lock_irqsave(&zone->lock, flags);
1587	}
1588
1589	/* The lock succeeded. Process deferred pages. */
1590	llhead = &zone->trylock_free_pages;
1591	if (unlikely(!llist_empty(llhead) && !(fpi_flags & FPI_TRYLOCK))) {
1592		struct llist_node *llnode;
1593		struct page *p, *tmp;
1594
1595		llnode = llist_del_all(llhead);
1596		llist_for_each_entry_safe(p, tmp, llnode, pcp_llist) {
1597			unsigned int p_order = p->private;
1598
1599			split_large_buddy(zone, p, page_to_pfn(p), p_order, fpi_flags);
1600			__count_vm_events(PGFREE, 1 << p_order);
1601		}
1602	}
1603	split_large_buddy(zone, page, pfn, order, fpi_flags);
1604	spin_unlock_irqrestore(&zone->lock, flags);
1605
1606	__count_vm_events(PGFREE, 1 << order);
1607}
1608
1609static void __free_pages_ok(struct page *page, unsigned int order,
1610			    fpi_t fpi_flags)
1611{
1612	unsigned long pfn = page_to_pfn(page);
1613	struct zone *zone = page_zone(page);
1614
1615	if (__free_pages_prepare(page, order, fpi_flags))
1616		free_one_page(zone, page, pfn, order, fpi_flags);
1617}
1618
1619void __meminit __free_pages_core(struct page *page, unsigned int order,
1620		enum meminit_context context)
1621{
1622	unsigned int nr_pages = 1 << order;
1623	struct page *p = page;
1624	unsigned int loop;
1625
1626	/*
1627	 * When initializing the memmap, __init_single_page() sets the refcount
1628	 * of all pages to 1 ("allocated"/"not free"). We have to set the
1629	 * refcount of all involved pages to 0.
1630	 *
1631	 * Note that hotplugged memory pages are initialized to PageOffline().
1632	 * Pages freed from memblock might be marked as reserved.
1633	 */
1634	if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG) &&
1635	    unlikely(context == MEMINIT_HOTPLUG)) {
1636		for (loop = 0; loop < nr_pages; loop++, p++) {
1637			VM_WARN_ON_ONCE(PageReserved(p));
1638			__ClearPageOffline(p);
1639			set_page_count(p, 0);
1640		}
1641
1642		adjust_managed_page_count(page, nr_pages);
1643	} else {
1644		for (loop = 0; loop < nr_pages; loop++, p++) {
1645			__ClearPageReserved(p);
1646			set_page_count(p, 0);
1647		}
1648
1649		/* memblock adjusts totalram_pages() manually. */
1650		atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
1651	}
1652
1653	if (page_contains_unaccepted(page, order)) {
1654		if (order == MAX_PAGE_ORDER && __free_unaccepted(page))
1655			return;
1656
1657		accept_memory(page_to_phys(page), PAGE_SIZE << order);
1658	}
1659
1660	/*
1661	 * Bypass PCP and place fresh pages right to the tail, primarily
1662	 * relevant for memory onlining.
1663	 */
1664	__free_pages_ok(page, order, FPI_TO_TAIL);
1665}
1666
1667/*
1668 * Check that the whole (or subset of) a pageblock given by the interval of
1669 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
1670 * with the migration of free compaction scanner.
1671 *
1672 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
1673 *
1674 * It's possible on some configurations to have a setup like node0 node1 node0
1675 * i.e. it's possible that all pages within a zones range of pages do not
1676 * belong to a single zone. We assume that a border between node0 and node1
1677 * can occur within a single pageblock, but not a node0 node1 node0
1678 * interleaving within a single pageblock. It is therefore sufficient to check
1679 * the first and last page of a pageblock and avoid checking each individual
1680 * page in a pageblock.
1681 *
1682 * Note: the function may return non-NULL struct page even for a page block
1683 * which contains a memory hole (i.e. there is no physical memory for a subset
1684 * of the pfn range). For example, if the pageblock order is MAX_PAGE_ORDER, which
1685 * will fall into 2 sub-sections, and the end pfn of the pageblock may be hole
1686 * even though the start pfn is online and valid. This should be safe most of
1687 * the time because struct pages are still initialized via init_unavailable_range()
1688 * and pfn walkers shouldn't touch any physical memory range for which they do
1689 * not recognize any specific metadata in struct pages.
1690 */
1691struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
1692				     unsigned long end_pfn, struct zone *zone)
1693{
1694	struct page *start_page;
1695	struct page *end_page;
1696
1697	/* end_pfn is one past the range we are checking */
1698	end_pfn--;
1699
1700	if (!pfn_valid(end_pfn))
1701		return NULL;
1702
1703	start_page = pfn_to_online_page(start_pfn);
1704	if (!start_page)
1705		return NULL;
1706
1707	if (page_zone(start_page) != zone)
1708		return NULL;
1709
1710	end_page = pfn_to_page(end_pfn);
1711
1712	/* This gives a shorter code than deriving page_zone(end_page) */
1713	if (page_zone_id(start_page) != page_zone_id(end_page))
1714		return NULL;
1715
1716	return start_page;
1717}
1718
1719/*
1720 * The order of subdivision here is critical for the IO subsystem.
1721 * Please do not alter this order without good reasons and regression
1722 * testing. Specifically, as large blocks of memory are subdivided,
1723 * the order in which smaller blocks are delivered depends on the order
1724 * they're subdivided in this function. This is the primary factor
1725 * influencing the order in which pages are delivered to the IO
1726 * subsystem according to empirical testing, and this is also justified
1727 * by considering the behavior of a buddy system containing a single
1728 * large block of memory acted on by a series of small allocations.
1729 * This behavior is a critical factor in sglist merging's success.
1730 *
1731 * -- nyc
1732 */
1733static inline unsigned int expand(struct zone *zone, struct page *page, int low,
1734				  int high, int migratetype)
1735{
1736	unsigned int size = 1 << high;
1737	unsigned int nr_added = 0;
1738
1739	while (high > low) {
1740		high--;
1741		size >>= 1;
1742		VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
1743
1744		/*
1745		 * Mark as guard pages (or page), that will allow to
1746		 * merge back to allocator when buddy will be freed.
1747		 * Corresponding page table entries will not be touched,
1748		 * pages will stay not present in virtual address space
1749		 */
1750		if (set_page_guard(zone, &page[size], high))
1751			continue;
1752
1753		__add_to_free_list(&page[size], zone, high, migratetype, false);
1754		set_buddy_order(&page[size], high);
1755		nr_added += size;
1756	}
1757
1758	return nr_added;
1759}
1760
1761static __always_inline void page_del_and_expand(struct zone *zone,
1762						struct page *page, int low,
1763						int high, int migratetype)
1764{
1765	int nr_pages = 1 << high;
1766
1767	__del_page_from_free_list(page, zone, high, migratetype);
1768	nr_pages -= expand(zone, page, low, high, migratetype);
1769	account_freepages(zone, -nr_pages, migratetype);
1770}
1771
1772static void check_new_page_bad(struct page *page)
1773{
1774	if (unlikely(PageHWPoison(page))) {
1775		/* Don't complain about hwpoisoned pages */
1776		if (PageBuddy(page))
1777			__ClearPageBuddy(page);
1778		return;
1779	}
1780
1781	bad_page(page,
1782		 page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
1783}
1784
1785/*
1786 * This page is about to be returned from the page allocator
1787 */
1788static bool check_new_page(struct page *page)
1789{
1790	if (likely(page_expected_state(page,
1791				PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
1792		return false;
1793
1794	check_new_page_bad(page);
1795	return true;
1796}
1797
1798static inline bool check_new_pages(struct page *page, unsigned int order)
1799{
1800	if (is_check_pages_enabled()) {
1801		for (int i = 0; i < (1 << order); i++) {
1802			struct page *p = page + i;
1803
1804			if (check_new_page(p))
1805				return true;
1806		}
1807	}
1808
1809	return false;
1810}
1811
1812static inline bool should_skip_kasan_unpoison(gfp_t flags)
1813{
1814	/* Don't skip if a software KASAN mode is enabled. */
1815	if (IS_ENABLED(CONFIG_KASAN_GENERIC) ||
1816	    IS_ENABLED(CONFIG_KASAN_SW_TAGS))
1817		return false;
1818
1819	/* Skip, if hardware tag-based KASAN is not enabled. */
1820	if (!kasan_hw_tags_enabled())
1821		return true;
1822
1823	/*
1824	 * With hardware tag-based KASAN enabled, skip if this has been
1825	 * requested via __GFP_SKIP_KASAN.
1826	 */
1827	return flags & __GFP_SKIP_KASAN;
1828}
1829
1830static inline bool should_skip_init(gfp_t flags)
1831{
1832	/* Don't skip, if hardware tag-based KASAN is not enabled. */
1833	if (!kasan_hw_tags_enabled())
1834		return false;
1835
1836	/* For hardware tag-based KASAN, skip if requested. */
1837	return (flags & __GFP_SKIP_ZERO);
1838}
1839
1840inline void post_alloc_hook(struct page *page, unsigned int order,
1841				gfp_t gfp_flags)
1842{
1843	bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
1844			!should_skip_init(gfp_flags);
1845	bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS);
1846	int i;
1847
1848	set_page_private(page, 0);
1849
1850	arch_alloc_page(page, order);
1851	debug_pagealloc_map_pages(page, 1 << order);
1852
1853	/*
1854	 * Page unpoisoning must happen before memory initialization.
1855	 * Otherwise, the poison pattern will be overwritten for __GFP_ZERO
1856	 * allocations and the page unpoisoning code will complain.
1857	 */
1858	kernel_unpoison_pages(page, 1 << order);
1859
1860	/*
1861	 * As memory initialization might be integrated into KASAN,
1862	 * KASAN unpoisoning and memory initializion code must be
1863	 * kept together to avoid discrepancies in behavior.
1864	 */
1865
1866	/*
1867	 * If memory tags should be zeroed
1868	 * (which happens only when memory should be initialized as well).
1869	 */
1870	if (zero_tags)
1871		init = !tag_clear_highpages(page, 1 << order);
1872
1873	if (!should_skip_kasan_unpoison(gfp_flags) &&
1874	    kasan_unpoison_pages(page, order, init)) {
1875		/* Take note that memory was initialized by KASAN. */
1876		if (kasan_has_integrated_init())
1877			init = false;
1878	} else {
1879		/*
1880		 * If memory tags have not been set by KASAN, reset the page
1881		 * tags to ensure page_address() dereferencing does not fault.
1882		 */
1883		for (i = 0; i != 1 << order; ++i)
1884			page_kasan_tag_reset(page + i);
1885	}
1886	/* If memory is still not initialized, initialize it now. */
1887	if (init)
1888		kernel_init_pages(page, 1 << order);
1889
1890	set_page_owner(page, order, gfp_flags);
1891	page_table_check_alloc(page, order);
1892	pgalloc_tag_add(page, current, 1 << order);
1893}
1894
1895static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
1896							unsigned int alloc_flags)
1897{
1898	post_alloc_hook(page, order, gfp_flags);
1899
1900	if (order && (gfp_flags & __GFP_COMP))
1901		prep_compound_page(page, order);
1902
1903	/*
1904	 * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
1905	 * allocate the page. The expectation is that the caller is taking
1906	 * steps that will free more memory. The caller should avoid the page
1907	 * being used for !PFMEMALLOC purposes.
1908	 */
1909	if (alloc_flags & ALLOC_NO_WATERMARKS)
1910		set_page_pfmemalloc(page);
1911	else
1912		clear_page_pfmemalloc(page);
1913}
1914
1915/*
1916 * Go through the free lists for the given migratetype and remove
1917 * the smallest available page from the freelists
1918 */
1919static __always_inline
1920struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
1921						int migratetype)
1922{
1923	unsigned int current_order;
1924	struct free_area *area;
1925	struct page *page;
1926
1927	/* Find a page of the appropriate size in the preferred list */
1928	for (current_order = order; current_order < NR_PAGE_ORDERS; ++current_order) {
1929		area = &(zone->free_area[current_order]);
1930		page = get_page_from_free_area(area, migratetype);
1931		if (!page)
1932			continue;
1933
1934		page_del_and_expand(zone, page, order, current_order,
1935				    migratetype);
1936		trace_mm_page_alloc_zone_locked(page, order, migratetype,
1937				pcp_allowed_order(order) &&
1938				migratetype < MIGRATE_PCPTYPES);
1939		return page;
1940	}
1941
1942	return NULL;
1943}
1944
1945
1946/*
1947 * This array describes the order lists are fallen back to when
1948 * the free lists for the desirable migrate type are depleted
1949 *
1950 * The other migratetypes do not have fallbacks.
1951 */
1952static int fallbacks[MIGRATE_PCPTYPES][MIGRATE_PCPTYPES - 1] = {
1953	[MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE   },
1954	[MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE },
1955	[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE   },
1956};
1957
1958#ifdef CONFIG_CMA
1959static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
1960					unsigned int order)
1961{
1962	return __rmqueue_smallest(zone, order, MIGRATE_CMA);
1963}
1964#else
1965static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
1966					unsigned int order) { return NULL; }
1967#endif
1968
1969/*
1970 * Move all free pages of a block to new type's freelist. Caller needs to
1971 * change the block type.
1972 */
1973static int __move_freepages_block(struct zone *zone, unsigned long start_pfn,
1974				  int old_mt, int new_mt)
1975{
1976	struct page *page;
1977	unsigned long pfn, end_pfn;
1978	unsigned int order;
1979	int pages_moved = 0;
1980
1981	VM_WARN_ON(start_pfn & (pageblock_nr_pages - 1));
1982	end_pfn = pageblock_end_pfn(start_pfn);
1983
1984	for (pfn = start_pfn; pfn < end_pfn;) {
1985		page = pfn_to_page(pfn);
1986		if (!PageBuddy(page)) {
1987			pfn++;
1988			continue;
1989		}
1990
1991		/* Make sure we are not inadvertently changing nodes */
1992		VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
1993		VM_BUG_ON_PAGE(page_zone(page) != zone, page);
1994
1995		order = buddy_order(page);
1996
1997		move_to_free_list(page, zone, order, old_mt, new_mt);
1998
1999		pfn += 1 << order;
2000		pages_moved += 1 << order;
2001	}
2002
2003	return pages_moved;
2004}
2005
2006static bool prep_move_freepages_block(struct zone *zone, struct page *page,
2007				      unsigned long *start_pfn,
2008				      int *num_free, int *num_movable)
2009{
2010	unsigned long pfn, start, end;
2011
2012	pfn = page_to_pfn(page);
2013	start = pageblock_start_pfn(pfn);
2014	end = pageblock_end_pfn(pfn);
2015
2016	/*
2017	 * The caller only has the lock for @zone, don't touch ranges
2018	 * that straddle into other zones. While we could move part of
2019	 * the range that's inside the zone, this call is usually
2020	 * accompanied by other operations such as migratetype updates
2021	 * which also should be locked.
2022	 */
2023	if (!zone_spans_pfn(zone, start))
2024		return false;
2025	if (!zone_spans_pfn(zone, end - 1))
2026		return false;
2027
2028	*start_pfn = start;
2029
2030	if (num_free) {
2031		*num_free = 0;
2032		*num_movable = 0;
2033		for (pfn = start; pfn < end;) {
2034			page = pfn_to_page(pfn);
2035			if (PageBuddy(page)) {
2036				int nr = 1 << buddy_order(page);
2037
2038				*num_free += nr;
2039				pfn += nr;
2040				continue;
2041			}
2042			/*
2043			 * We assume that pages that could be isolated for
2044			 * migration are movable. But we don't actually try
2045			 * isolating, as that would be expensive.
2046			 */
2047			if (PageLRU(page) || page_has_movable_ops(page))
2048				(*num_movable)++;
2049			pfn++;
2050		}
2051	}
2052
2053	return true;
2054}
2055
2056static int move_freepages_block(struct zone *zone, struct page *page,
2057				int old_mt, int new_mt)
2058{
2059	unsigned long start_pfn;
2060	int res;
2061
2062	if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))
2063		return -1;
2064
2065	res = __move_freepages_block(zone, start_pfn, old_mt, new_mt);
2066	set_pageblock_migratetype(pfn_to_page(start_pfn), new_mt);
2067
2068	return res;
2069
2070}
2071
2072#ifdef CONFIG_MEMORY_ISOLATION
2073/* Look for a buddy that straddles start_pfn */
2074static unsigned long find_large_buddy(unsigned long start_pfn)
2075{
2076	/*
2077	 * If start_pfn is not an order-0 PageBuddy, next PageBuddy containing
2078	 * start_pfn has minimal order of __ffs(start_pfn) + 1. Start checking
2079	 * the order with __ffs(start_pfn). If start_pfn is order-0 PageBuddy,
2080	 * the starting order does not matter.
2081	 */
2082	int order = start_pfn ? __ffs(start_pfn) : MAX_PAGE_ORDER;
2083	struct page *page;
2084	unsigned long pfn = start_pfn;
2085
2086	while (!PageBuddy(page = pfn_to_page(pfn))) {
2087		/* Nothing found */
2088		if (++order > MAX_PAGE_ORDER)
2089			return start_pfn;
2090		pfn &= ~0UL << order;
2091	}
2092
2093	/*
2094	 * Found a preceding buddy, but does it straddle?
2095	 */
2096	if (pfn + (1 << buddy_order(page)) > start_pfn)
2097		return pfn;
2098
2099	/* Nothing found */
2100	return start_pfn;
2101}
2102
2103static inline void toggle_pageblock_isolate(struct page *page, bool isolate)
2104{
2105	if (isolate)
2106		set_pageblock_isolate(page);
2107	else
2108		clear_pageblock_isolate(page);
2109}
2110
2111/**
2112 * __move_freepages_block_isolate - move free pages in block for page isolation
2113 * @zone: the zone
2114 * @page: the pageblock page
2115 * @isolate: to isolate the given pageblock or unisolate it
2116 *
2117 * This is similar to move_freepages_block(), but handles the special
2118 * case encountered in page isolation, where the block of interest
2119 * might be part of a larger buddy spanning multiple pageblocks.
2120 *
2121 * Unlike the regular page allocator path, which moves pages while
2122 * stealing buddies off the freelist, page isolation is interested in
2123 * arbitrary pfn ranges that may have overlapping buddies on both ends.
2124 *
2125 * This function handles that. Straddling buddies are split into
2126 * individual pageblocks. Only the block of interest is moved.
2127 *
2128 * Returns %true if pages could be moved, %false otherwise.
2129 */
2130static bool __move_freepages_block_isolate(struct zone *zone,
2131		struct page *page, bool isolate)
2132{
2133	unsigned long start_pfn, buddy_pfn;
2134	int from_mt;
2135	int to_mt;
2136	struct page *buddy;
2137
2138	if (isolate == get_pageblock_isolate(page)) {
2139		VM_WARN_ONCE(1, "%s a pageblock that is already in that state",
2140			     isolate ? "Isolate" : "Unisolate");
2141		return false;
2142	}
2143
2144	if (!prep_move_freepages_block(zone, page, &start_pfn, NULL, NULL))
2145		return false;
2146
2147	/* No splits needed if buddies can't span multiple blocks */
2148	if (pageblock_order == MAX_PAGE_ORDER)
2149		goto move;
2150
2151	buddy_pfn = find_large_buddy(start_pfn);
2152	buddy = pfn_to_page(buddy_pfn);
2153	/* We're a part of a larger buddy */
2154	if (PageBuddy(buddy) && buddy_order(buddy) > pageblock_order) {
2155		int order = buddy_order(buddy);
2156
2157		del_page_from_free_list(buddy, zone, order,
2158					get_pfnblock_migratetype(buddy, buddy_pfn));
2159		toggle_pageblock_isolate(page, isolate);
2160		split_large_buddy(zone, buddy, buddy_pfn, order, FPI_NONE);
2161		return true;
2162	}
2163
2164move:
2165	/* Use MIGRATETYPE_MASK to get non-isolate migratetype */
2166	if (isolate) {
2167		from_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),
2168						    MIGRATETYPE_MASK);
2169		to_mt = MIGRATE_ISOLATE;
2170	} else {
2171		from_mt = MIGRATE_ISOLATE;
2172		to_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),
2173						  MIGRATETYPE_MASK);
2174	}
2175
2176	__move_freepages_block(zone, start_pfn, from_mt, to_mt);
2177	toggle_pageblock_isolate(pfn_to_page(start_pfn), isolate);
2178
2179	return true;
2180}
2181
2182bool pageblock_isolate_and_move_free_pages(struct zone *zone, struct page *page)
2183{
2184	return __move_freepages_block_isolate(zone, page, true);
2185}
2186
2187bool pageblock_unisolate_and_move_free_pages(struct zone *zone, struct page *page)
2188{
2189	return __move_freepages_block_isolate(zone, page, false);
2190}
2191
2192#endif /* CONFIG_MEMORY_ISOLATION */
2193
2194static inline bool boost_watermark(struct zone *zone)
2195{
2196	unsigned long max_boost;
2197
2198	if (!watermark_boost_factor)
2199		return false;
2200	/*
2201	 * Don't bother in zones that are unlikely to produce results.
2202	 * On small machines, including kdump capture kernels running
2203	 * in a small area, boosting the watermark can cause an out of
2204	 * memory situation immediately.
2205	 */
2206	if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
2207		return false;
2208
2209	max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
2210			watermark_boost_factor, 10000);
2211
2212	/*
2213	 * high watermark may be uninitialised if fragmentation occurs
2214	 * very early in boot so do not boost. We do not fall
2215	 * through and boost by pageblock_nr_pages as failing
2216	 * allocations that early means that reclaim is not going
2217	 * to help and it may even be impossible to reclaim the
2218	 * boosted watermark resulting in a hang.
2219	 */
2220	if (!max_boost)
2221		return false;
2222
2223	max_boost = max(pageblock_nr_pages, max_boost);
2224
2225	zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
2226		max_boost);
2227
2228	return true;
2229}
2230
2231/*
2232 * When we are falling back to another migratetype during allocation, should we
2233 * try to claim an entire block to satisfy further allocations, instead of
2234 * polluting multiple pageblocks?
2235 */
2236static bool should_try_claim_block(unsigned int order, int start_mt)
2237{
2238	/*
2239	 * Leaving this order check is intended, although there is
2240	 * relaxed order check in next check. The reason is that
2241	 * we can actually claim the whole pageblock if this condition met,
2242	 * but, below check doesn't guarantee it and that is just heuristic
2243	 * so could be changed anytime.
2244	 */
2245	if (order >= pageblock_order)
2246		return true;
2247
2248	/*
2249	 * Above a certain threshold, always try to claim, as it's likely there
2250	 * will be more free pages in the pageblock.
2251	 */
2252	if (order >= pageblock_order / 2)
2253		return true;
2254
2255	/*
2256	 * Unmovable/reclaimable allocations would cause permanent
2257	 * fragmentations if they fell back to allocating from a movable block
2258	 * (polluting it), so we try to claim the whole block regardless of the
2259	 * allocation size. Later movable allocations can always steal from this
2260	 * block, which is less problematic.
2261	 */
2262	if (start_mt == MIGRATE_RECLAIMABLE || start_mt == MIGRATE_UNMOVABLE)
2263		return true;
2264
2265	if (page_group_by_mobility_disabled)
2266		return true;
2267
2268	/*
2269	 * Movable pages won't cause permanent fragmentation, so when you alloc
2270	 * small pages, we just need to temporarily steal unmovable or
2271	 * reclaimable pages that are closest to the request size. After a
2272	 * while, memory compaction may occur to form large contiguous pages,
2273	 * and the next movable allocation may not need to steal.
2274	 */
2275	return false;
2276}
2277
2278/*
2279 * Check whether there is a suitable fallback freepage with requested order.
2280 * If claimable is true, this function returns fallback_mt only if
2281 * we would do this whole-block claiming. This would help to reduce
2282 * fragmentation due to mixed migratetype pages in one pageblock.
2283 */
2284int find_suitable_fallback(struct free_area *area, unsigned int order,
2285			   int migratetype, bool claimable)
2286{
2287	int i;
2288
2289	if (claimable && !should_try_claim_block(order, migratetype))
2290		return -2;
2291
2292	if (area->nr_free == 0)
2293		return -1;
2294
2295	for (i = 0; i < MIGRATE_PCPTYPES - 1 ; i++) {
2296		int fallback_mt = fallbacks[migratetype][i];
2297
2298		if (!free_area_empty(area, fallback_mt))
2299			return fallback_mt;
2300	}
2301
2302	return -1;
2303}
2304
2305/*
2306 * This function implements actual block claiming behaviour. If order is large
2307 * enough, we can claim the whole pageblock for the requested migratetype. If
2308 * not, we check the pageblock for constituent pages; if at least half of the
2309 * pages are free or compatible, we can still claim the whole block, so pages
2310 * freed in the future will be put on the correct free list.
2311 */
2312static struct page *
2313try_to_claim_block(struct zone *zone, struct page *page,
2314		   int current_order, int order, int start_type,
2315		   int block_type, unsigned int alloc_flags)
2316{
2317	int free_pages, movable_pages, alike_pages;
2318	unsigned long start_pfn;
2319
2320	/* Take ownership for orders >= pageblock_order */
2321	if (current_order >= pageblock_order) {
2322		unsigned int nr_added;
2323
2324		del_page_from_free_list(page, zone, current_order, block_type);
2325		change_pageblock_range(page, current_order, start_type);
2326		nr_added = expand(zone, page, order, current_order, start_type);
2327		account_freepages(zone, nr_added, start_type);
2328		return page;
2329	}
2330
2331	/*
2332	 * Boost watermarks to increase reclaim pressure to reduce the
2333	 * likelihood of future fallbacks. Wake kswapd now as the node
2334	 * may be balanced overall and kswapd will not wake naturally.
2335	 */
2336	if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
2337		set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
2338
2339	/* moving whole block can fail due to zone boundary conditions */
2340	if (!prep_move_freepages_block(zone, page, &start_pfn, &free_pages,
2341				       &movable_pages))
2342		return NULL;
2343
2344	/*
2345	 * Determine how many pages are compatible with our allocation.
2346	 * For movable allocation, it's the number of movable pages which
2347	 * we just obtained. For other types it's a bit more tricky.
2348	 */
2349	if (start_type == MIGRATE_MOVABLE) {
2350		alike_pages = movable_pages;
2351	} else {
2352		/*
2353		 * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
2354		 * to MOVABLE pageblock, consider all non-movable pages as
2355		 * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
2356		 * vice versa, be conservative since we can't distinguish the
2357		 * exact migratetype of non-movable pages.
2358		 */
2359		if (block_type == MIGRATE_MOVABLE)
2360			alike_pages = pageblock_nr_pages
2361						- (free_pages + movable_pages);
2362		else
2363			alike_pages = 0;
2364	}
2365	/*
2366	 * If a sufficient number of pages in the block are either free or of
2367	 * compatible migratability as our allocation, claim the whole block.
2368	 */
2369	if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
2370			page_group_by_mobility_disabled) {
2371		__move_freepages_block(zone, start_pfn, block_type, start_type);
2372		set_pageblock_migratetype(pfn_to_page(start_pfn), start_type);
2373		return __rmqueue_smallest(zone, order, start_type);
2374	}
2375
2376	return NULL;
2377}
2378
2379/*
2380 * Try to allocate from some fallback migratetype by claiming the entire block,
2381 * i.e. converting it to the allocation's start migratetype.
2382 *
2383 * The use of signed ints for order and current_order is a deliberate
2384 * deviation from the rest of this file, to make the for loop
2385 * condition simpler.
2386 */
2387static __always_inline struct page *
2388__rmqueue_claim(struct zone *zone, int order, int start_migratetype,
2389						unsigned int alloc_flags)
2390{
2391	struct free_area *area;
2392	int current_order;
2393	int min_order = order;
2394	struct page *page;
2395	int fallback_mt;
2396
2397	/*
2398	 * Do not steal pages from freelists belonging to other pageblocks
2399	 * i.e. orders < pageblock_order. If there are no local zones free,
2400	 * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
2401	 */
2402	if (order < pageblock_order && alloc_flags & ALLOC_NOFRAGMENT)
2403		min_order = pageblock_order;
2404
2405	/*
2406	 * Find the largest available free page in the other list. This roughly
2407	 * approximates finding the pageblock with the most free pages, which
2408	 * would be too costly to do exactly.
2409	 */
2410	for (current_order = MAX_PAGE_ORDER; current_order >= min_order;
2411				--current_order) {
2412		area = &(zone->free_area[current_order]);
2413		fallback_mt = find_suitable_fallback(area, current_order,
2414						     start_migratetype, true);
2415
2416		/* No block in that order */
2417		if (fallback_mt == -1)
2418			continue;
2419
2420		/* Advanced into orders too low to claim, abort */
2421		if (fallback_mt == -2)
2422			break;
2423
2424		page = get_page_from_free_area(area, fallback_mt);
2425		page = try_to_claim_block(zone, page, current_order, order,
2426					  start_migratetype, fallback_mt,
2427					  alloc_flags);
2428		if (page) {
2429			trace_mm_page_alloc_extfrag(page, order, current_order,
2430						    start_migratetype, fallback_mt);
2431			return page;
2432		}
2433	}
2434
2435	return NULL;
2436}
2437
2438/*
2439 * Try to steal a single page from some fallback migratetype. Leave the rest of
2440 * the block as its current migratetype, potentially causing fragmentation.
2441 */
2442static __always_inline struct page *
2443__rmqueue_steal(struct zone *zone, int order, int start_migratetype)
2444{
2445	struct free_area *area;
2446	int current_order;
2447	struct page *page;
2448	int fallback_mt;
2449
2450	for (current_order = order; current_order < NR_PAGE_ORDERS; current_order++) {
2451		area = &(zone->free_area[current_order]);
2452		fallback_mt = find_suitable_fallback(area, current_order,
2453						     start_migratetype, false);
2454		if (fallback_mt == -1)
2455			continue;
2456
2457		page = get_page_from_free_area(area, fallback_mt);
2458		page_del_and_expand(zone, page, order, current_order, fallback_mt);
2459		trace_mm_page_alloc_extfrag(page, order, current_order,
2460					    start_migratetype, fallback_mt);
2461		return page;
2462	}
2463
2464	return NULL;
2465}
2466
2467enum rmqueue_mode {
2468	RMQUEUE_NORMAL,
2469	RMQUEUE_CMA,
2470	RMQUEUE_CLAIM,
2471	RMQUEUE_STEAL,
2472};
2473
2474/*
2475 * Do the hard work of removing an element from the buddy allocator.
2476 * Call me with the zone->lock already held.
2477 */
2478static __always_inline struct page *
2479__rmqueue(struct zone *zone, unsigned int order, int migratetype,
2480	  unsigned int alloc_flags, enum rmqueue_mode *mode)
2481{
2482	struct page *page;
2483
2484	if (IS_ENABLED(CONFIG_CMA)) {
2485		/*
2486		 * Balance movable allocations between regular and CMA areas by
2487		 * allocating from CMA when over half of the zone's free memory
2488		 * is in the CMA area.
2489		 */
2490		if (alloc_flags & ALLOC_CMA &&
2491		    zone_page_state(zone, NR_FREE_CMA_PAGES) >
2492		    zone_page_state(zone, NR_FREE_PAGES) / 2) {
2493			page = __rmqueue_cma_fallback(zone, order);
2494			if (page)
2495				return page;
2496		}
2497	}
2498
2499	/*
2500	 * First try the freelists of the requested migratetype, then try
2501	 * fallbacks modes with increasing levels of fragmentation risk.
2502	 *
2503	 * The fallback logic is expensive and rmqueue_bulk() calls in
2504	 * a loop with the zone->lock held, meaning the freelists are
2505	 * not subject to any outside changes. Remember in *mode where
2506	 * we found pay dirt, to save us the search on the next call.
2507	 */
2508	switch (*mode) {
2509	case RMQUEUE_NORMAL:
2510		page = __rmqueue_smallest(zone, order, migratetype);
2511		if (page)
2512			return page;
2513		fallthrough;
2514	case RMQUEUE_CMA:
2515		if (alloc_flags & ALLOC_CMA) {
2516			page = __rmqueue_cma_fallback(zone, order);
2517			if (page) {
2518				*mode = RMQUEUE_CMA;
2519				return page;
2520			}
2521		}
2522		fallthrough;
2523	case RMQUEUE_CLAIM:
2524		page = __rmqueue_claim(zone, order, migratetype, alloc_flags);
2525		if (page) {
2526			/* Replenished preferred freelist, back to normal mode. */
2527			*mode = RMQUEUE_NORMAL;
2528			return page;
2529		}
2530		fallthrough;
2531	case RMQUEUE_STEAL:
2532		if (!(alloc_flags & ALLOC_NOFRAGMENT)) {
2533			page = __rmqueue_steal(zone, order, migratetype);
2534			if (page) {
2535				*mode = RMQUEUE_STEAL;
2536				return page;
2537			}
2538		}
2539	}
2540	return NULL;
2541}
2542
2543/*
2544 * Obtain a specified number of elements from the buddy allocator, all under
2545 * a single hold of the lock, for efficiency.  Add them to the supplied list.
2546 * Returns the number of new pages which were placed at *list.
2547 */
2548static int rmqueue_bulk(struct zone *zone, unsigned int order,
2549			unsigned long count, struct list_head *list,
2550			int migratetype, unsigned int alloc_flags)
2551{
2552	enum rmqueue_mode rmqm = RMQUEUE_NORMAL;
2553	unsigned long flags;
2554	int i;
2555
2556	if (unlikely(alloc_flags & ALLOC_TRYLOCK)) {
2557		if (!spin_trylock_irqsave(&zone->lock, flags))
2558			return 0;
2559	} else {
2560		spin_lock_irqsave(&zone->lock, flags);
2561	}
2562	for (i = 0; i < count; ++i) {
2563		struct page *page = __rmqueue(zone, order, migratetype,
2564					      alloc_flags, &rmqm);
2565		if (unlikely(page == NULL))
2566			break;
2567
2568		/*
2569		 * Split buddy pages returned by expand() are received here in
2570		 * physical page order. The page is added to the tail of
2571		 * caller's list. From the callers perspective, the linked list
2572		 * is ordered by page number under some conditions. This is
2573		 * useful for IO devices that can forward direction from the
2574		 * head, thus also in the physical page order. This is useful
2575		 * for IO devices that can merge IO requests if the physical
2576		 * pages are ordered properly.
2577		 */
2578		list_add_tail(&page->pcp_list, list);
2579	}
2580	spin_unlock_irqrestore(&zone->lock, flags);
2581
2582	return i;
2583}
2584
2585/*
2586 * Called from the vmstat counter updater to decay the PCP high.
2587 * Return whether there are addition works to do.
2588 */
2589bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp)
2590{
2591	int high_min, to_drain, to_drain_batched, batch;
2592	unsigned long UP_flags;
2593	bool todo = false;
2594
2595	high_min = READ_ONCE(pcp->high_min);
2596	batch = READ_ONCE(pcp->batch);
2597	/*
2598	 * Decrease pcp->high periodically to try to free possible
2599	 * idle PCP pages.  And, avoid to free too many pages to
2600	 * control latency.  This caps pcp->high decrement too.
2601	 */
2602	if (pcp->high > high_min) {
2603		pcp->high = max3(pcp->count - (batch << CONFIG_PCP_BATCH_SCALE_MAX),
2604				 pcp->high - (pcp->high >> 3), high_min);
2605		if (pcp->high > high_min)
2606			todo = true;
2607	}
2608
2609	to_drain = pcp->count - pcp->high;
2610	while (to_drain > 0) {
2611		to_drain_batched = min(to_drain, batch);
2612		pcp_spin_lock_maybe_irqsave(pcp, UP_flags);
2613		free_pcppages_bulk(zone, to_drain_batched, pcp, 0);
2614		pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags);
2615		todo = true;
2616
2617		to_drain -= to_drain_batched;
2618	}
2619
2620	return todo;
2621}
2622
2623#ifdef CONFIG_NUMA
2624/*
2625 * Called from the vmstat counter updater to drain pagesets of this
2626 * currently executing processor on remote nodes after they have
2627 * expired.
2628 */
2629void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
2630{
2631	unsigned long UP_flags;
2632	int to_drain, batch;
2633
2634	batch = READ_ONCE(pcp->batch);
2635	to_drain = min(pcp->count, batch);
2636	if (to_drain > 0) {
2637		pcp_spin_lock_maybe_irqsave(pcp, UP_flags);
2638		free_pcppages_bulk(zone, to_drain, pcp, 0);
2639		pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags);
2640	}
2641}
2642#endif
2643
2644/*
2645 * Drain pcplists of the indicated processor and zone.
2646 */
2647static void drain_pages_zone(unsigned int cpu, struct zone *zone)
2648{
2649	struct per_cpu_pages *pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
2650	unsigned long UP_flags;
2651	int count;
2652
2653	do {
2654		pcp_spin_lock_maybe_irqsave(pcp, UP_flags);
2655		count = pcp->count;
2656		if (count) {
2657			int to_drain = min(count,
2658				pcp->batch << CONFIG_PCP_BATCH_SCALE_MAX);
2659
2660			free_pcppages_bulk(zone, to_drain, pcp, 0);
2661			count -= to_drain;
2662		}
2663		pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags);
2664	} while (count);
2665}
2666
2667/*
2668 * Drain pcplists of all zones on the indicated processor.
2669 */
2670static void drain_pages(unsigned int cpu)
2671{
2672	struct zone *zone;
2673
2674	for_each_populated_zone(zone) {
2675		drain_pages_zone(cpu, zone);
2676	}
2677}
2678
2679/*
2680 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
2681 */
2682void drain_local_pages(struct zone *zone)
2683{
2684	int cpu = smp_processor_id();
2685
2686	if (zone)
2687		drain_pages_zone(cpu, zone);
2688	else
2689		drain_pages(cpu);
2690}
2691
2692/*
2693 * The implementation of drain_all_pages(), exposing an extra parameter to
2694 * drain on all cpus.
2695 *
2696 * drain_all_pages() is optimized to only execute on cpus where pcplists are
2697 * not empty. The check for non-emptiness can however race with a free to
2698 * pcplist that has not yet increased the pcp->count from 0 to 1. Callers
2699 * that need the guarantee that every CPU has drained can disable the
2700 * optimizing racy check.
2701 */
2702static void __drain_all_pages(struct zone *zone, bool force_all_cpus)
2703{
2704	int cpu;
2705
2706	/*
2707	 * Allocate in the BSS so we won't require allocation in
2708	 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
2709	 */
2710	static cpumask_t cpus_with_pcps;
2711
2712	/*
2713	 * Do not drain if one is already in progress unless it's specific to
2714	 * a zone. Such callers are primarily CMA and memory hotplug and need
2715	 * the drain to be complete when the call returns.
2716	 */
2717	if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
2718		if (!zone)
2719			return;
2720		mutex_lock(&pcpu_drain_mutex);
2721	}
2722
2723	/*
2724	 * We don't care about racing with CPU hotplug event
2725	 * as offline notification will cause the notified
2726	 * cpu to drain that CPU pcps and on_each_cpu_mask
2727	 * disables preemption as part of its processing
2728	 */
2729	for_each_online_cpu(cpu) {
2730		struct per_cpu_pages *pcp;
2731		struct zone *z;
2732		bool has_pcps = false;
2733
2734		if (force_all_cpus) {
2735			/*
2736			 * The pcp.count check is racy, some callers need a
2737			 * guarantee that no cpu is missed.
2738			 */
2739			has_pcps = true;
2740		} else if (zone) {
2741			pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
2742			if (pcp->count)
2743				has_pcps = true;
2744		} else {
2745			for_each_populated_zone(z) {
2746				pcp = per_cpu_ptr(z->per_cpu_pageset, cpu);
2747				if (pcp->count) {
2748					has_pcps = true;
2749					break;
2750				}
2751			}
2752		}
2753
2754		if (has_pcps)
2755			cpumask_set_cpu(cpu, &cpus_with_pcps);
2756		else
2757			cpumask_clear_cpu(cpu, &cpus_with_pcps);
2758	}
2759
2760	for_each_cpu(cpu, &cpus_with_pcps) {
2761		if (zone)
2762			drain_pages_zone(cpu, zone);
2763		else
2764			drain_pages(cpu);
2765	}
2766
2767	mutex_unlock(&pcpu_drain_mutex);
2768}
2769
2770/*
2771 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
2772 *
2773 * When zone parameter is non-NULL, spill just the single zone's pages.
2774 */
2775void drain_all_pages(struct zone *zone)
2776{
2777	__drain_all_pages(zone, false);
2778}
2779
2780static int nr_pcp_free(struct per_cpu_pages *pcp, int batch, int high, bool free_high)
2781{
2782	int min_nr_free, max_nr_free;
2783
2784	/* Free as much as possible if batch freeing high-order pages. */
2785	if (unlikely(free_high))
2786		return min(pcp->count, batch << CONFIG_PCP_BATCH_SCALE_MAX);
2787
2788	/* Check for PCP disabled or boot pageset */
2789	if (unlikely(high < batch))
2790		return 1;
2791
2792	/* Leave at least pcp->batch pages on the list */
2793	min_nr_free = batch;
2794	max_nr_free = high - batch;
2795
2796	/*
2797	 * Increase the batch number to the number of the consecutive
2798	 * freed pages to reduce zone lock contention.
2799	 */
2800	batch = clamp_t(int, pcp->free_count, min_nr_free, max_nr_free);
2801
2802	return batch;
2803}
2804
2805static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone,
2806		       int batch, bool free_high)
2807{
2808	int high, high_min, high_max;
2809
2810	high_min = READ_ONCE(pcp->high_min);
2811	high_max = READ_ONCE(pcp->high_max);
2812	high = pcp->high = clamp(pcp->high, high_min, high_max);
2813
2814	if (unlikely(!high))
2815		return 0;
2816
2817	if (unlikely(free_high)) {
2818		pcp->high = max(high - (batch << CONFIG_PCP_BATCH_SCALE_MAX),
2819				high_min);
2820		return 0;
2821	}
2822
2823	/*
2824	 * If reclaim is active, limit the number of pages that can be
2825	 * stored on pcp lists
2826	 */
2827	if (test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags)) {
2828		int free_count = max_t(int, pcp->free_count, batch);
2829
2830		pcp->high = max(high - free_count, high_min);
2831		return min(batch << 2, pcp->high);
2832	}
2833
2834	if (high_min == high_max)
2835		return high;
2836
2837	if (test_bit(ZONE_BELOW_HIGH, &zone->flags)) {
2838		int free_count = max_t(int, pcp->free_count, batch);
2839
2840		pcp->high = max(high - free_count, high_min);
2841		high = max(pcp->count, high_min);
2842	} else if (pcp->count >= high) {
2843		int need_high = pcp->free_count + batch;
2844
2845		/* pcp->high should be large enough to hold batch freed pages */
2846		if (pcp->high < need_high)
2847			pcp->high = clamp(need_high, high_min, high_max);
2848	}
2849
2850	return high;
2851}
2852
2853/*
2854 * Tune pcp alloc factor and adjust count & free_count. Free pages to bring the
2855 * pcp's watermarks below high.
2856 *
2857 * May return a freed pcp, if during page freeing the pcp spinlock cannot be
2858 * reacquired. Return true if pcp is locked, false otherwise.
2859 */
2860static bool free_frozen_page_commit(struct zone *zone,
2861		struct per_cpu_pages *pcp, struct page *page, int migratetype,
2862		unsigned int order, fpi_t fpi_flags, unsigned long *UP_flags)
2863{
2864	int high, batch;
2865	int to_free, to_free_batched;
2866	int pindex;
2867	int cpu = smp_processor_id();
2868	int ret = true;
2869	bool free_high = false;
2870
2871	/*
2872	 * On freeing, reduce the number of pages that are batch allocated.
2873	 * See nr_pcp_alloc() where alloc_factor is increased for subsequent
2874	 * allocations.
2875	 */
2876	pcp->alloc_factor >>= 1;
2877	__count_vm_events(PGFREE, 1 << order);
2878	pindex = order_to_pindex(migratetype, order);
2879	list_add(&page->pcp_list, &pcp->lists[pindex]);
2880	pcp->count += 1 << order;
2881
2882	batch = READ_ONCE(pcp->batch);
2883	/*
2884	 * As high-order pages other than THP's stored on PCP can contribute
2885	 * to fragmentation, limit the number stored when PCP is heavily
2886	 * freeing without allocation. The remainder after bulk freeing
2887	 * stops will be drained from vmstat refresh context.
2888	 */
2889	if (order && order <= PAGE_ALLOC_COSTLY_ORDER) {
2890		free_high = (pcp->free_count >= (batch + pcp->high_min / 2) &&
2891			     (pcp->flags & PCPF_PREV_FREE_HIGH_ORDER) &&
2892			     (!(pcp->flags & PCPF_FREE_HIGH_BATCH) ||
2893			      pcp->count >= batch));
2894		pcp->flags |= PCPF_PREV_FREE_HIGH_ORDER;
2895	} else if (pcp->flags & PCPF_PREV_FREE_HIGH_ORDER) {
2896		pcp->flags &= ~PCPF_PREV_FREE_HIGH_ORDER;
2897	}
2898	if (pcp->free_count < (batch << CONFIG_PCP_BATCH_SCALE_MAX))
2899		pcp->free_count += (1 << order);
2900
2901	if (unlikely(fpi_flags & FPI_TRYLOCK)) {
2902		/*
2903		 * Do not attempt to take a zone lock. Let pcp->count get
2904		 * over high mark temporarily.
2905		 */
2906		return true;
2907	}
2908
2909	high = nr_pcp_high(pcp, zone, batch, free_high);
2910	if (pcp->count < high)
2911		return true;
2912
2913	to_free = nr_pcp_free(pcp, batch, high, free_high);
2914	while (to_free > 0 && pcp->count > 0) {
2915		to_free_batched = min(to_free, batch);
2916		free_pcppages_bulk(zone, to_free_batched, pcp, pindex);
2917		to_free -= to_free_batched;
2918
2919		if (to_free == 0 || pcp->count == 0)
2920			break;
2921
2922		pcp_spin_unlock(pcp, *UP_flags);
2923
2924		pcp = pcp_spin_trylock(zone->per_cpu_pageset, *UP_flags);
2925		if (!pcp) {
2926			ret = false;
2927			break;
2928		}
2929
2930		/*
2931		 * Check if this thread has been migrated to a different CPU.
2932		 * If that is the case, give up and indicate that the pcp is
2933		 * returned in an unlocked state.
2934		 */
2935		if (smp_processor_id() != cpu) {
2936			pcp_spin_unlock(pcp, *UP_flags);
2937			ret = false;
2938			break;
2939		}
2940	}
2941
2942	if (test_bit(ZONE_BELOW_HIGH, &zone->flags) &&
2943	    zone_watermark_ok(zone, 0, high_wmark_pages(zone),
2944			      ZONE_MOVABLE, 0)) {
2945		struct pglist_data *pgdat = zone->zone_pgdat;
2946		clear_bit(ZONE_BELOW_HIGH, &zone->flags);
2947
2948		/*
2949		 * Assume that memory pressure on this node is gone and may be
2950		 * in a reclaimable state. If a memory fallback node exists,
2951		 * direct reclaim may not have been triggered, causing a
2952		 * 'hopeless node' to stay in that state for a while.  Let
2953		 * kswapd work again by resetting kswapd_failures.
2954		 */
2955		if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES &&
2956		    next_memory_node(pgdat->node_id) < MAX_NUMNODES)
2957			atomic_set(&pgdat->kswapd_failures, 0);
2958	}
2959	return ret;
2960}
2961
2962/*
2963 * Free a pcp page
2964 */
2965static void __free_frozen_pages(struct page *page, unsigned int order,
2966				fpi_t fpi_flags)
2967{
2968	unsigned long UP_flags;
2969	struct per_cpu_pages *pcp;
2970	struct zone *zone;
2971	unsigned long pfn = page_to_pfn(page);
2972	int migratetype;
2973
2974	if (!pcp_allowed_order(order)) {
2975		__free_pages_ok(page, order, fpi_flags);
2976		return;
2977	}
2978
2979	if (!__free_pages_prepare(page, order, fpi_flags))
2980		return;
2981
2982	/*
2983	 * We only track unmovable, reclaimable and movable on pcp lists.
2984	 * Place ISOLATE pages on the isolated list because they are being
2985	 * offlined but treat HIGHATOMIC and CMA as movable pages so we can
2986	 * get those areas back if necessary. Otherwise, we may have to free
2987	 * excessively into the page allocator
2988	 */
2989	zone = page_zone(page);
2990	migratetype = get_pfnblock_migratetype(page, pfn);
2991	if (unlikely(migratetype >= MIGRATE_PCPTYPES)) {
2992		if (unlikely(is_migrate_isolate(migratetype))) {
2993			free_one_page(zone, page, pfn, order, fpi_flags);
2994			return;
2995		}
2996		migratetype = MIGRATE_MOVABLE;
2997	}
2998
2999	if (unlikely((fpi_flags & FPI_TRYLOCK) && IS_ENABLED(CONFIG_PREEMPT_RT)
3000		     && (in_nmi() || in_hardirq()))) {
3001		add_page_to_zone_llist(zone, page, order);
3002		return;
3003	}
3004	pcp = pcp_spin_trylock(zone->per_cpu_pageset, UP_flags);
3005	if (pcp) {
3006		if (!free_frozen_page_commit(zone, pcp, page, migratetype,
3007						order, fpi_flags, &UP_flags))
3008			return;
3009		pcp_spin_unlock(pcp, UP_flags);
3010	} else {
3011		free_one_page(zone, page, pfn, order, fpi_flags);
3012	}
3013}
3014
3015void free_frozen_pages(struct page *page, unsigned int order)
3016{
3017	__free_frozen_pages(page, order, FPI_NONE);
3018}
3019
3020/*
3021 * Free a batch of folios
3022 */
3023void free_unref_folios(struct folio_batch *folios)
3024{
3025	unsigned long UP_flags;
3026	struct per_cpu_pages *pcp = NULL;
3027	struct zone *locked_zone = NULL;
3028	int i, j;
3029
3030	/* Prepare folios for freeing */
3031	for (i = 0, j = 0; i < folios->nr; i++) {
3032		struct folio *folio = folios->folios[i];
3033		unsigned long pfn = folio_pfn(folio);
3034		unsigned int order = folio_order(folio);
3035
3036		if (!__free_pages_prepare(&folio->page, order, FPI_NONE))
3037			continue;
3038		/*
3039		 * Free orders not handled on the PCP directly to the
3040		 * allocator.
3041		 */
3042		if (!pcp_allowed_order(order)) {
3043			free_one_page(folio_zone(folio), &folio->page,
3044				      pfn, order, FPI_NONE);
3045			continue;
3046		}
3047		folio->private = (void *)(unsigned long)order;
3048		if (j != i)
3049			folios->folios[j] = folio;
3050		j++;
3051	}
3052	folios->nr = j;
3053
3054	for (i = 0; i < folios->nr; i++) {
3055		struct folio *folio = folios->folios[i];
3056		struct zone *zone = folio_zone(folio);
3057		unsigned long pfn = folio_pfn(folio);
3058		unsigned int order = (unsigned long)folio->private;
3059		int migratetype;
3060
3061		folio->private = NULL;
3062		migratetype = get_pfnblock_migratetype(&folio->page, pfn);
3063
3064		/* Different zone requires a different pcp lock */
3065		if (zone != locked_zone ||
3066		    is_migrate_isolate(migratetype)) {
3067			if (pcp) {
3068				pcp_spin_unlock(pcp, UP_flags);
3069				locked_zone = NULL;
3070				pcp = NULL;
3071			}
3072
3073			/*
3074			 * Free isolated pages directly to the
3075			 * allocator, see comment in free_frozen_pages.
3076			 */
3077			if (is_migrate_isolate(migratetype)) {
3078				free_one_page(zone, &folio->page, pfn,
3079					      order, FPI_NONE);
3080				continue;
3081			}
3082
3083			/*
3084			 * trylock is necessary as folios may be getting freed
3085			 * from IRQ or SoftIRQ context after an IO completion.
3086			 */
3087			pcp = pcp_spin_trylock(zone->per_cpu_pageset, UP_flags);
3088			if (unlikely(!pcp)) {
3089				free_one_page(zone, &folio->page, pfn,
3090					      order, FPI_NONE);
3091				continue;
3092			}
3093			locked_zone = zone;
3094		}
3095
3096		/*
3097		 * Non-isolated types over MIGRATE_PCPTYPES get added
3098		 * to the MIGRATE_MOVABLE pcp list.
3099		 */
3100		if (unlikely(migratetype >= MIGRATE_PCPTYPES))
3101			migratetype = MIGRATE_MOVABLE;
3102
3103		trace_mm_page_free_batched(&folio->page);
3104		if (!free_frozen_page_commit(zone, pcp, &folio->page,
3105				migratetype, order, FPI_NONE, &UP_flags)) {
3106			pcp = NULL;
3107			locked_zone = NULL;
3108		}
3109	}
3110
3111	if (pcp)
3112		pcp_spin_unlock(pcp, UP_flags);
3113	folio_batch_reinit(folios);
3114}
3115
3116/*
3117 * split_page takes a non-compound higher-order page, and splits it into
3118 * n (1<<order) sub-pages: page[0..n]
3119 * Each sub-page must be freed individually.
3120 *
3121 * Note: this is probably too low level an operation for use in drivers.
3122 * Please consult with lkml before using this in your driver.
3123 */
3124void split_page(struct page *page, unsigned int order)
3125{
3126	int i;
3127
3128	VM_BUG_ON_PAGE(PageCompound(page), page);
3129	VM_BUG_ON_PAGE(!page_count(page), page);
3130
3131	for (i = 1; i < (1 << order); i++)
3132		set_page_refcounted(page + i);
3133	split_page_owner(page, order, 0);
3134	pgalloc_tag_split(page_folio(page), order, 0);
3135	split_page_memcg(page, order);
3136}
3137EXPORT_SYMBOL_GPL(split_page);
3138
3139int __isolate_free_page(struct page *page, unsigned int order)
3140{
3141	struct zone *zone = page_zone(page);
3142	int mt = get_pageblock_migratetype(page);
3143
3144	if (!is_migrate_isolate(mt)) {
3145		unsigned long watermark;
3146		/*
3147		 * Obey watermarks as if the page was being allocated. We can
3148		 * emulate a high-order watermark check with a raised order-0
3149		 * watermark, because we already know our high-order page
3150		 * exists.
3151		 */
3152		watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
3153		if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
3154			return 0;
3155	}
3156
3157	del_page_from_free_list(page, zone, order, mt);
3158
3159	/*
3160	 * Set the pageblock if the isolated page is at least half of a
3161	 * pageblock
3162	 */
3163	if (order >= pageblock_order - 1) {
3164		struct page *endpage = page + (1 << order) - 1;
3165		for (; page < endpage; page += pageblock_nr_pages) {
3166			int mt = get_pageblock_migratetype(page);
3167			/*
3168			 * Only change normal pageblocks (i.e., they can merge
3169			 * with others)
3170			 */
3171			if (migratetype_is_mergeable(mt))
3172				move_freepages_block(zone, page, mt,
3173						     MIGRATE_MOVABLE);
3174		}
3175	}
3176
3177	return 1UL << order;
3178}
3179
3180/**
3181 * __putback_isolated_page - Return a now-isolated page back where we got it
3182 * @page: Page that was isolated
3183 * @order: Order of the isolated page
3184 * @mt: The page's pageblock's migratetype
3185 *
3186 * This function is meant to return a page pulled from the free lists via
3187 * __isolate_free_page back to the free lists they were pulled from.
3188 */
3189void __putback_isolated_page(struct page *page, unsigned int order, int mt)
3190{
3191	struct zone *zone = page_zone(page);
3192
3193	/* zone lock should be held when this function is called */
3194	lockdep_assert_held(&zone->lock);
3195
3196	/* Return isolated page to tail of freelist. */
3197	__free_one_page(page, page_to_pfn(page), zone, order, mt,
3198			FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
3199}
3200
3201/*
3202 * Update NUMA hit/miss statistics
3203 */
3204static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,
3205				   long nr_account)
3206{
3207#ifdef CONFIG_NUMA
3208	enum numa_stat_item local_stat = NUMA_LOCAL;
3209
3210	/* skip numa counters update if numa stats is disabled */
3211	if (!static_branch_likely(&vm_numa_stat_key))
3212		return;
3213
3214	if (zone_to_nid(z) != numa_node_id())
3215		local_stat = NUMA_OTHER;
3216
3217	if (zone_to_nid(z) == zone_to_nid(preferred_zone))
3218		__count_numa_events(z, NUMA_HIT, nr_account);
3219	else {
3220		__count_numa_events(z, NUMA_MISS, nr_account);
3221		__count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account);
3222	}
3223	__count_numa_events(z, local_stat, nr_account);
3224#endif
3225}
3226
3227static __always_inline
3228struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,
3229			   unsigned int order, unsigned int alloc_flags,
3230			   int migratetype)
3231{
3232	struct page *page;
3233	unsigned long flags;
3234
3235	do {
3236		page = NULL;
3237		if (unlikely(alloc_flags & ALLOC_TRYLOCK)) {
3238			if (!spin_trylock_irqsave(&zone->lock, flags))
3239				return NULL;
3240		} else {
3241			spin_lock_irqsave(&zone->lock, flags);
3242		}
3243		if (alloc_flags & ALLOC_HIGHATOMIC)
3244			page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3245		if (!page) {
3246			enum rmqueue_mode rmqm = RMQUEUE_NORMAL;
3247
3248			page = __rmqueue(zone, order, migratetype, alloc_flags, &rmqm);
3249
3250			/*
3251			 * If the allocation fails, allow OOM handling and
3252			 * order-0 (atomic) allocs access to HIGHATOMIC
3253			 * reserves as failing now is worse than failing a
3254			 * high-order atomic allocation in the future.
3255			 */
3256			if (!page && (alloc_flags & (ALLOC_OOM|ALLOC_NON_BLOCK)))
3257				page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
3258
3259			if (!page) {
3260				spin_unlock_irqrestore(&zone->lock, flags);
3261				return NULL;
3262			}
3263		}
3264		spin_unlock_irqrestore(&zone->lock, flags);
3265	} while (check_new_pages(page, order));
3266
3267	__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3268	zone_statistics(preferred_zone, zone, 1);
3269
3270	return page;
3271}
3272
3273static int nr_pcp_alloc(struct per_cpu_pages *pcp, struct zone *zone, int order)
3274{
3275	int high, base_batch, batch, max_nr_alloc;
3276	int high_max, high_min;
3277
3278	base_batch = READ_ONCE(pcp->batch);
3279	high_min = READ_ONCE(pcp->high_min);
3280	high_max = READ_ONCE(pcp->high_max);
3281	high = pcp->high = clamp(pcp->high, high_min, high_max);
3282
3283	/* Check for PCP disabled or boot pageset */
3284	if (unlikely(high < base_batch))
3285		return 1;
3286
3287	if (order)
3288		batch = base_batch;
3289	else
3290		batch = (base_batch << pcp->alloc_factor);
3291
3292	/*
3293	 * If we had larger pcp->high, we could avoid to allocate from
3294	 * zone.
3295	 */
3296	if (high_min != high_max && !test_bit(ZONE_BELOW_HIGH, &zone->flags))
3297		high = pcp->high = min(high + batch, high_max);
3298
3299	if (!order) {
3300		max_nr_alloc = max(high - pcp->count - base_batch, base_batch);
3301		/*
3302		 * Double the number of pages allocated each time there is
3303		 * subsequent allocation of order-0 pages without any freeing.
3304		 */
3305		if (batch <= max_nr_alloc &&
3306		    pcp->alloc_factor < CONFIG_PCP_BATCH_SCALE_MAX)
3307			pcp->alloc_factor++;
3308		batch = min(batch, max_nr_alloc);
3309	}
3310
3311	/*
3312	 * Scale batch relative to order if batch implies free pages
3313	 * can be stored on the PCP. Batch can be 1 for small zones or
3314	 * for boot pagesets which should never store free pages as
3315	 * the pages may belong to arbitrary zones.
3316	 */
3317	if (batch > 1)
3318		batch = max(batch >> order, 2);
3319
3320	return batch;
3321}
3322
3323/* Remove page from the per-cpu list, caller must protect the list */
3324static inline
3325struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,
3326			int migratetype,
3327			unsigned int alloc_flags,
3328			struct per_cpu_pages *pcp,
3329			struct list_head *list)
3330{
3331	struct page *page;
3332
3333	do {
3334		if (list_empty(list)) {
3335			int batch = nr_pcp_alloc(pcp, zone, order);
3336			int alloced;
3337
3338			alloced = rmqueue_bulk(zone, order,
3339					batch, list,
3340					migratetype, alloc_flags);
3341
3342			pcp->count += alloced << order;
3343			if (unlikely(list_empty(list)))
3344				return NULL;
3345		}
3346
3347		page = list_first_entry(list, struct page, pcp_list);
3348		list_del(&page->pcp_list);
3349		pcp->count -= 1 << order;
3350	} while (check_new_pages(page, order));
3351
3352	return page;
3353}
3354
3355/* Lock and remove page from the per-cpu list */
3356static struct page *rmqueue_pcplist(struct zone *preferred_zone,
3357			struct zone *zone, unsigned int order,
3358			int migratetype, unsigned int alloc_flags)
3359{
3360	struct per_cpu_pages *pcp;
3361	struct list_head *list;
3362	struct page *page;
3363	unsigned long UP_flags;
3364
3365	/* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */
3366	pcp = pcp_spin_trylock(zone->per_cpu_pageset, UP_flags);
3367	if (!pcp)
3368		return NULL;
3369
3370	/*
3371	 * On allocation, reduce the number of pages that are batch freed.
3372	 * See nr_pcp_free() where free_factor is increased for subsequent
3373	 * frees.
3374	 */
3375	pcp->free_count >>= 1;
3376	list = &pcp->lists[order_to_pindex(migratetype, order)];
3377	page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp, list);
3378	pcp_spin_unlock(pcp, UP_flags);
3379	if (page) {
3380		__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
3381		zone_statistics(preferred_zone, zone, 1);
3382	}
3383	return page;
3384}
3385
3386/*
3387 * Allocate a page from the given zone.
3388 * Use pcplists for THP or "cheap" high-order allocations.
3389 */
3390
3391/*
3392 * Do not instrument rmqueue() with KMSAN. This function may call
3393 * __msan_poison_alloca() through a call to set_pfnblock_migratetype().
3394 * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it
3395 * may call rmqueue() again, which will result in a deadlock.
3396 */
3397__no_sanitize_memory
3398static inline
3399struct page *rmqueue(struct zone *preferred_zone,
3400			struct zone *zone, unsigned int order,
3401			gfp_t gfp_flags, unsigned int alloc_flags,
3402			int migratetype)
3403{
3404	struct page *page;
3405
3406	if (likely(pcp_allowed_order(order))) {
3407		page = rmqueue_pcplist(preferred_zone, zone, order,
3408				       migratetype, alloc_flags);
3409		if (likely(page))
3410			goto out;
3411	}
3412
3413	page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags,
3414							migratetype);
3415
3416out:
3417	/* Separate test+clear to avoid unnecessary atomics */
3418	if ((alloc_flags & ALLOC_KSWAPD) &&
3419	    unlikely(test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags))) {
3420		clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
3421		wakeup_kswapd(zone, 0, 0, zone_idx(zone));
3422	}
3423
3424	VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
3425	return page;
3426}
3427
3428/*
3429 * Reserve the pageblock(s) surrounding an allocation request for
3430 * exclusive use of high-order atomic allocations if there are no
3431 * empty page blocks that contain a page with a suitable order
3432 */
3433static void reserve_highatomic_pageblock(struct page *page, int order,
3434					 struct zone *zone)
3435{
3436	int mt;
3437	unsigned long max_managed, flags;
3438
3439	/*
3440	 * The number reserved as: minimum is 1 pageblock, maximum is
3441	 * roughly 1% of a zone. But if 1% of a zone falls below a
3442	 * pageblock size, then don't reserve any pageblocks.
3443	 * Check is race-prone but harmless.
3444	 */
3445	if ((zone_managed_pages(zone) / 100) < pageblock_nr_pages)
3446		return;
3447	max_managed = ALIGN((zone_managed_pages(zone) / 100), pageblock_nr_pages);
3448	if (zone->nr_reserved_highatomic >= max_managed)
3449		return;
3450
3451	spin_lock_irqsave(&zone->lock, flags);
3452
3453	/* Recheck the nr_reserved_highatomic limit under the lock */
3454	if (zone->nr_reserved_highatomic >= max_managed)
3455		goto out_unlock;
3456
3457	/* Yoink! */
3458	mt = get_pageblock_migratetype(page);
3459	/* Only reserve normal pageblocks (i.e., they can merge with others) */
3460	if (!migratetype_is_mergeable(mt))
3461		goto out_unlock;
3462
3463	if (order < pageblock_order) {
3464		if (move_freepages_block(zone, page, mt, MIGRATE_HIGHATOMIC) == -1)
3465			goto out_unlock;
3466		zone->nr_reserved_highatomic += pageblock_nr_pages;
3467	} else {
3468		change_pageblock_range(page, order, MIGRATE_HIGHATOMIC);
3469		zone->nr_reserved_highatomic += 1 << order;
3470	}
3471
3472out_unlock:
3473	spin_unlock_irqrestore(&zone->lock, flags);
3474}
3475
3476/*
3477 * Used when an allocation is about to fail under memory pressure. This
3478 * potentially hurts the reliability of high-order allocations when under
3479 * intense memory pressure but failed atomic allocations should be easier
3480 * to recover from than an OOM.
3481 *
3482 * If @force is true, try to unreserve pageblocks even though highatomic
3483 * pageblock is exhausted.
3484 */
3485static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
3486						bool force)
3487{
3488	struct zonelist *zonelist = ac->zonelist;
3489	unsigned long flags;
3490	struct zoneref *z;
3491	struct zone *zone;
3492	struct page *page;
3493	int order;
3494	int ret;
3495
3496	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
3497								ac->nodemask) {
3498		/*
3499		 * Preserve at least one pageblock unless memory pressure
3500		 * is really high.
3501		 */
3502		if (!force && zone->nr_reserved_highatomic <=
3503					pageblock_nr_pages)
3504			continue;
3505
3506		spin_lock_irqsave(&zone->lock, flags);
3507		for (order = 0; order < NR_PAGE_ORDERS; order++) {
3508			struct free_area *area = &(zone->free_area[order]);
3509			unsigned long size;
3510
3511			page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
3512			if (!page)
3513				continue;
3514
3515			size = max(pageblock_nr_pages, 1UL << order);
3516			/*
3517			 * It should never happen but changes to
3518			 * locking could inadvertently allow a per-cpu
3519			 * drain to add pages to MIGRATE_HIGHATOMIC
3520			 * while unreserving so be safe and watch for
3521			 * underflows.
3522			 */
3523			if (WARN_ON_ONCE(size > zone->nr_reserved_highatomic))
3524				size = zone->nr_reserved_highatomic;
3525			zone->nr_reserved_highatomic -= size;
3526
3527			/*
3528			 * Convert to ac->migratetype and avoid the normal
3529			 * pageblock stealing heuristics. Minimally, the caller
3530			 * is doing the work and needs the pages. More
3531			 * importantly, if the block was always converted to
3532			 * MIGRATE_UNMOVABLE or another type then the number
3533			 * of pageblocks that cannot be completely freed
3534			 * may increase.
3535			 */
3536			if (order < pageblock_order)
3537				ret = move_freepages_block(zone, page,
3538							   MIGRATE_HIGHATOMIC,
3539							   ac->migratetype);
3540			else {
3541				move_to_free_list(page, zone, order,
3542						  MIGRATE_HIGHATOMIC,
3543						  ac->migratetype);
3544				change_pageblock_range(page, order,
3545						       ac->migratetype);
3546				ret = 1;
3547			}
3548			/*
3549			 * Reserving the block(s) already succeeded,
3550			 * so this should not fail on zone boundaries.
3551			 */
3552			WARN_ON_ONCE(ret == -1);
3553			if (ret > 0) {
3554				spin_unlock_irqrestore(&zone->lock, flags);
3555				return ret;
3556			}
3557		}
3558		spin_unlock_irqrestore(&zone->lock, flags);
3559	}
3560
3561	return false;
3562}
3563
3564static inline long __zone_watermark_unusable_free(struct zone *z,
3565				unsigned int order, unsigned int alloc_flags)
3566{
3567	long unusable_free = (1 << order) - 1;
3568
3569	/*
3570	 * If the caller does not have rights to reserves below the min
3571	 * watermark then subtract the free pages reserved for highatomic.
3572	 */
3573	if (likely(!(alloc_flags & ALLOC_RESERVES)))
3574		unusable_free += READ_ONCE(z->nr_free_highatomic);
3575
3576#ifdef CONFIG_CMA
3577	/* If allocation can't use CMA areas don't use free CMA pages */
3578	if (!(alloc_flags & ALLOC_CMA))
3579		unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
3580#endif
3581
3582	return unusable_free;
3583}
3584
3585/*
3586 * Return true if free base pages are above 'mark'. For high-order checks it
3587 * will return true of the order-0 watermark is reached and there is at least
3588 * one free page of a suitable size. Checking now avoids taking the zone lock
3589 * to check in the allocation paths if no pages are free.
3590 */
3591bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3592			 int highest_zoneidx, unsigned int alloc_flags,
3593			 long free_pages)
3594{
3595	long min = mark;
3596	int o;
3597
3598	/* free_pages may go negative - that's OK */
3599	free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
3600
3601	if (unlikely(alloc_flags & ALLOC_RESERVES)) {
3602		/*
3603		 * __GFP_HIGH allows access to 50% of the min reserve as well
3604		 * as OOM.
3605		 */
3606		if (alloc_flags & ALLOC_MIN_RESERVE) {
3607			min -= min / 2;
3608
3609			/*
3610			 * Non-blocking allocations (e.g. GFP_ATOMIC) can
3611			 * access more reserves than just __GFP_HIGH. Other
3612			 * non-blocking allocations requests such as GFP_NOWAIT
3613			 * or (GFP_KERNEL & ~__GFP_DIRECT_RECLAIM) do not get
3614			 * access to the min reserve.
3615			 */
3616			if (alloc_flags & ALLOC_NON_BLOCK)
3617				min -= min / 4;
3618		}
3619
3620		/*
3621		 * OOM victims can try even harder than the normal reserve
3622		 * users on the grounds that it's definitely going to be in
3623		 * the exit path shortly and free memory. Any allocation it
3624		 * makes during the free path will be small and short-lived.
3625		 */
3626		if (alloc_flags & ALLOC_OOM)
3627			min -= min / 2;
3628	}
3629
3630	/*
3631	 * Check watermarks for an order-0 allocation request. If these
3632	 * are not met, then a high-order request also cannot go ahead
3633	 * even if a suitable page happened to be free.
3634	 */
3635	if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
3636		return false;
3637
3638	/* If this is an order-0 request then the watermark is fine */
3639	if (!order)
3640		return true;
3641
3642	/* For a high-order request, check at least one suitable page is free */
3643	for (o = order; o < NR_PAGE_ORDERS; o++) {
3644		struct free_area *area = &z->free_area[o];
3645		int mt;
3646
3647		if (!area->nr_free)
3648			continue;
3649
3650		for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
3651			if (!free_area_empty(area, mt))
3652				return true;
3653		}
3654
3655#ifdef CONFIG_CMA
3656		if ((alloc_flags & ALLOC_CMA) &&
3657		    !free_area_empty(area, MIGRATE_CMA)) {
3658			return true;
3659		}
3660#endif
3661		if ((alloc_flags & (ALLOC_HIGHATOMIC|ALLOC_OOM)) &&
3662		    !free_area_empty(area, MIGRATE_HIGHATOMIC)) {
3663			return true;
3664		}
3665	}
3666	return false;
3667}
3668
3669bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
3670		      int highest_zoneidx, unsigned int alloc_flags)
3671{
3672	return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3673					zone_page_state(z, NR_FREE_PAGES));
3674}
3675
3676static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
3677				unsigned long mark, int highest_zoneidx,
3678				unsigned int alloc_flags, gfp_t gfp_mask)
3679{
3680	long free_pages;
3681
3682	free_pages = zone_page_state(z, NR_FREE_PAGES);
3683
3684	/*
3685	 * Fast check for order-0 only. If this fails then the reserves
3686	 * need to be calculated.
3687	 */
3688	if (!order) {
3689		long usable_free;
3690		long reserved;
3691
3692		usable_free = free_pages;
3693		reserved = __zone_watermark_unusable_free(z, 0, alloc_flags);
3694
3695		/* reserved may over estimate high-atomic reserves. */
3696		usable_free -= min(usable_free, reserved);
3697		if (usable_free > mark + z->lowmem_reserve[highest_zoneidx])
3698			return true;
3699	}
3700
3701	if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
3702					free_pages))
3703		return true;
3704
3705	/*
3706	 * Ignore watermark boosting for __GFP_HIGH order-0 allocations
3707	 * when checking the min watermark. The min watermark is the
3708	 * point where boosting is ignored so that kswapd is woken up
3709	 * when below the low watermark.
3710	 */
3711	if (unlikely(!order && (alloc_flags & ALLOC_MIN_RESERVE) && z->watermark_boost
3712		&& ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
3713		mark = z->_watermark[WMARK_MIN];
3714		return __zone_watermark_ok(z, order, mark, highest_zoneidx,
3715					alloc_flags, free_pages);
3716	}
3717
3718	return false;
3719}
3720
3721#ifdef CONFIG_NUMA
3722int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE;
3723
3724static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3725{
3726	return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
3727				node_reclaim_distance;
3728}
3729#else	/* CONFIG_NUMA */
3730static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
3731{
3732	return true;
3733}
3734#endif	/* CONFIG_NUMA */
3735
3736/*
3737 * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
3738 * fragmentation is subtle. If the preferred zone was HIGHMEM then
3739 * premature use of a lower zone may cause lowmem pressure problems that
3740 * are worse than fragmentation. If the next zone is ZONE_DMA then it is
3741 * probably too small. It only makes sense to spread allocations to avoid
3742 * fragmentation between the Normal and DMA32 zones.
3743 */
3744static inline unsigned int
3745alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
3746{
3747	unsigned int alloc_flags;
3748
3749	/*
3750	 * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
3751	 * to save a branch.
3752	 */
3753	alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
3754
3755	if (defrag_mode) {
3756		alloc_flags |= ALLOC_NOFRAGMENT;
3757		return alloc_flags;
3758	}
3759
3760#ifdef CONFIG_ZONE_DMA32
3761	if (!zone)
3762		return alloc_flags;
3763
3764	if (zone_idx(zone) != ZONE_NORMAL)
3765		return alloc_flags;
3766
3767	/*
3768	 * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
3769	 * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
3770	 * on UMA that if Normal is populated then so is DMA32.
3771	 */
3772	BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
3773	if (nr_online_nodes > 1 && !populated_zone(--zone))
3774		return alloc_flags;
3775
3776	alloc_flags |= ALLOC_NOFRAGMENT;
3777#endif /* CONFIG_ZONE_DMA32 */
3778	return alloc_flags;
3779}
3780
3781/* Must be called after current_gfp_context() which can change gfp_mask */
3782static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask,
3783						  unsigned int alloc_flags)
3784{
3785#ifdef CONFIG_CMA
3786	if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE)
3787		alloc_flags |= ALLOC_CMA;
3788#endif
3789	return alloc_flags;
3790}
3791
3792/*
3793 * get_page_from_freelist goes through the zonelist trying to allocate
3794 * a page.
3795 */
3796static struct page *
3797get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
3798						const struct alloc_context *ac)
3799{
3800	struct zoneref *z;
3801	struct zone *zone;
3802	struct pglist_data *last_pgdat = NULL;
3803	bool last_pgdat_dirty_ok = false;
3804	bool no_fallback;
3805	bool skip_kswapd_nodes = nr_online_nodes > 1;
3806	bool skipped_kswapd_nodes = false;
3807
3808retry:
3809	/*
3810	 * Scan zonelist, looking for a zone with enough free.
3811	 * See also cpuset_current_node_allowed() comment in kernel/cgroup/cpuset.c.
3812	 */
3813	no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
3814	z = ac->preferred_zoneref;
3815	for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
3816					ac->nodemask) {
3817		struct page *page;
3818		unsigned long mark;
3819
3820		if (cpusets_enabled() &&
3821			(alloc_flags & ALLOC_CPUSET) &&
3822			!__cpuset_zone_allowed(zone, gfp_mask))
3823				continue;
3824		/*
3825		 * When allocating a page cache page for writing, we
3826		 * want to get it from a node that is within its dirty
3827		 * limit, such that no single node holds more than its
3828		 * proportional share of globally allowed dirty pages.
3829		 * The dirty limits take into account the node's
3830		 * lowmem reserves and high watermark so that kswapd
3831		 * should be able to balance it without having to
3832		 * write pages from its LRU list.
3833		 *
3834		 * XXX: For now, allow allocations to potentially
3835		 * exceed the per-node dirty limit in the slowpath
3836		 * (spread_dirty_pages unset) before going into reclaim,
3837		 * which is important when on a NUMA setup the allowed
3838		 * nodes are together not big enough to reach the
3839		 * global limit.  The proper fix for these situations
3840		 * will require awareness of nodes in the
3841		 * dirty-throttling and the flusher threads.
3842		 */
3843		if (ac->spread_dirty_pages) {
3844			if (last_pgdat != zone->zone_pgdat) {
3845				last_pgdat = zone->zone_pgdat;
3846				last_pgdat_dirty_ok = node_dirty_ok(zone->zone_pgdat);
3847			}
3848
3849			if (!last_pgdat_dirty_ok)
3850				continue;
3851		}
3852
3853		if (no_fallback && !defrag_mode && nr_online_nodes > 1 &&
3854		    zone != zonelist_zone(ac->preferred_zoneref)) {
3855			int local_nid;
3856
3857			/*
3858			 * If moving to a remote node, retry but allow
3859			 * fragmenting fallbacks. Locality is more important
3860			 * than fragmentation avoidance.
3861			 */
3862			local_nid = zonelist_node_idx(ac->preferred_zoneref);
3863			if (zone_to_nid(zone) != local_nid) {
3864				alloc_flags &= ~ALLOC_NOFRAGMENT;
3865				goto retry;
3866			}
3867		}
3868
3869		/*
3870		 * If kswapd is already active on a node, keep looking
3871		 * for other nodes that might be idle. This can happen
3872		 * if another process has NUMA bindings and is causing
3873		 * kswapd wakeups on only some nodes. Avoid accidental
3874		 * "node_reclaim_mode"-like behavior in this case.
3875		 */
3876		if (skip_kswapd_nodes &&
3877		    !waitqueue_active(&zone->zone_pgdat->kswapd_wait)) {
3878			skipped_kswapd_nodes = true;
3879			continue;
3880		}
3881
3882		cond_accept_memory(zone, order, alloc_flags);
3883
3884		/*
3885		 * Detect whether the number of free pages is below high
3886		 * watermark.  If so, we will decrease pcp->high and free
3887		 * PCP pages in free path to reduce the possibility of
3888		 * premature page reclaiming.  Detection is done here to
3889		 * avoid to do that in hotter free path.
3890		 */
3891		if (test_bit(ZONE_BELOW_HIGH, &zone->flags))
3892			goto check_alloc_wmark;
3893
3894		mark = high_wmark_pages(zone);
3895		if (zone_watermark_fast(zone, order, mark,
3896					ac->highest_zoneidx, alloc_flags,
3897					gfp_mask))
3898			goto try_this_zone;
3899		else
3900			set_bit(ZONE_BELOW_HIGH, &zone->flags);
3901
3902check_alloc_wmark:
3903		mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
3904		if (!zone_watermark_fast(zone, order, mark,
3905				       ac->highest_zoneidx, alloc_flags,
3906				       gfp_mask)) {
3907			int ret;
3908
3909			if (cond_accept_memory(zone, order, alloc_flags))
3910				goto try_this_zone;
3911
3912			/*
3913			 * Watermark failed for this zone, but see if we can
3914			 * grow this zone if it contains deferred pages.
3915			 */
3916			if (deferred_pages_enabled()) {
3917				if (_deferred_grow_zone(zone, order))
3918					goto try_this_zone;
3919			}
3920			/* Checked here to keep the fast path fast */
3921			BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
3922			if (alloc_flags & ALLOC_NO_WATERMARKS)
3923				goto try_this_zone;
3924
3925			if (!node_reclaim_enabled() ||
3926			    !zone_allows_reclaim(zonelist_zone(ac->preferred_zoneref), zone))
3927				continue;
3928
3929			ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
3930			switch (ret) {
3931			case NODE_RECLAIM_NOSCAN:
3932				/* did not scan */
3933				continue;
3934			case NODE_RECLAIM_FULL:
3935				/* scanned but unreclaimable */
3936				continue;
3937			default:
3938				/* did we reclaim enough */
3939				if (zone_watermark_ok(zone, order, mark,
3940					ac->highest_zoneidx, alloc_flags))
3941					goto try_this_zone;
3942
3943				continue;
3944			}
3945		}
3946
3947try_this_zone:
3948		page = rmqueue(zonelist_zone(ac->preferred_zoneref), zone, order,
3949				gfp_mask, alloc_flags, ac->migratetype);
3950		if (page) {
3951			prep_new_page(page, order, gfp_mask, alloc_flags);
3952
3953			/*
3954			 * If this is a high-order atomic allocation then check
3955			 * if the pageblock should be reserved for the future
3956			 */
3957			if (unlikely(alloc_flags & ALLOC_HIGHATOMIC))
3958				reserve_highatomic_pageblock(page, order, zone);
3959
3960			return page;
3961		} else {
3962			if (cond_accept_memory(zone, order, alloc_flags))
3963				goto try_this_zone;
3964
3965			/* Try again if zone has deferred pages */
3966			if (deferred_pages_enabled()) {
3967				if (_deferred_grow_zone(zone, order))
3968					goto try_this_zone;
3969			}
3970		}
3971	}
3972
3973	/*
3974	 * If we skipped over nodes with active kswapds and found no
3975	 * idle nodes, retry and place anywhere the watermarks permit.
3976	 */
3977	if (skip_kswapd_nodes && skipped_kswapd_nodes) {
3978		skip_kswapd_nodes = false;
3979		goto retry;
3980	}
3981
3982	/*
3983	 * It's possible on a UMA machine to get through all zones that are
3984	 * fragmented. If avoiding fragmentation, reset and try again.
3985	 */
3986	if (no_fallback && !defrag_mode) {
3987		alloc_flags &= ~ALLOC_NOFRAGMENT;
3988		goto retry;
3989	}
3990
3991	return NULL;
3992}
3993
3994static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
3995{
3996	unsigned int filter = SHOW_MEM_FILTER_NODES;
3997
3998	/*
3999	 * This documents exceptions given to allocations in certain
4000	 * contexts that are allowed to allocate outside current's set
4001	 * of allowed nodes.
4002	 */
4003	if (!(gfp_mask & __GFP_NOMEMALLOC))
4004		if (tsk_is_oom_victim(current) ||
4005		    (current->flags & (PF_MEMALLOC | PF_EXITING)))
4006			filter &= ~SHOW_MEM_FILTER_NODES;
4007	if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
4008		filter &= ~SHOW_MEM_FILTER_NODES;
4009
4010	__show_mem(filter, nodemask, gfp_zone(gfp_mask));
4011	mem_cgroup_show_protected_memory(NULL);
4012}
4013
4014void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
4015{
4016	struct va_format vaf;
4017	va_list args;
4018	static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
4019
4020	if ((gfp_mask & __GFP_NOWARN) ||
4021	     !__ratelimit(&nopage_rs) ||
4022	     ((gfp_mask & __GFP_DMA) && !has_managed_dma()))
4023		return;
4024
4025	va_start(args, fmt);
4026	vaf.fmt = fmt;
4027	vaf.va = &args;
4028	pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
4029			current->comm, &vaf, gfp_mask, &gfp_mask,
4030			nodemask_pr_args(nodemask));
4031	va_end(args);
4032
4033	cpuset_print_current_mems_allowed();
4034	pr_cont("\n");
4035	dump_stack();
4036	warn_alloc_show_mem(gfp_mask, nodemask);
4037}
4038
4039static inline struct page *
4040__alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
4041			      unsigned int alloc_flags,
4042			      const struct alloc_context *ac)
4043{
4044	struct page *page;
4045
4046	page = get_page_from_freelist(gfp_mask, order,
4047			alloc_flags|ALLOC_CPUSET, ac);
4048	/*
4049	 * fallback to ignore cpuset restriction if our nodes
4050	 * are depleted
4051	 */
4052	if (!page)
4053		page = get_page_from_freelist(gfp_mask, order,
4054				alloc_flags, ac);
4055	return page;
4056}
4057
4058static inline struct page *
4059__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
4060	const struct alloc_context *ac, unsigned long *did_some_progress)
4061{
4062	struct oom_control oc = {
4063		.zonelist = ac->zonelist,
4064		.nodemask = ac->nodemask,
4065		.memcg = NULL,
4066		.gfp_mask = gfp_mask,
4067		.order = order,
4068	};
4069	struct page *page;
4070
4071	*did_some_progress = 0;
4072
4073	/*
4074	 * Acquire the oom lock.  If that fails, somebody else is
4075	 * making progress for us.
4076	 */
4077	if (!mutex_trylock(&oom_lock)) {
4078		*did_some_progress = 1;
4079		schedule_timeout_uninterruptible(1);
4080		return NULL;
4081	}
4082
4083	/*
4084	 * Go through the zonelist yet one more time, keep very high watermark
4085	 * here, this is only to catch a parallel oom killing, we must fail if
4086	 * we're still under heavy pressure. But make sure that this reclaim
4087	 * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
4088	 * allocation which will never fail due to oom_lock already held.
4089	 */
4090	page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
4091				      ~__GFP_DIRECT_RECLAIM, order,
4092				      ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
4093	if (page)
4094		goto out;
4095
4096	/* Coredumps can quickly deplete all memory reserves */
4097	if (current->flags & PF_DUMPCORE)
4098		goto out;
4099	/* The OOM killer will not help higher order allocs */
4100	if (order > PAGE_ALLOC_COSTLY_ORDER)
4101		goto out;
4102	/*
4103	 * We have already exhausted all our reclaim opportunities without any
4104	 * success so it is time to admit defeat. We will skip the OOM killer
4105	 * because it is very likely that the caller has a more reasonable
4106	 * fallback than shooting a random task.
4107	 *
4108	 * The OOM killer may not free memory on a specific node.
4109	 */
4110	if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))
4111		goto out;
4112	/* The OOM killer does not needlessly kill tasks for lowmem */
4113	if (ac->highest_zoneidx < ZONE_NORMAL)
4114		goto out;
4115	if (pm_suspended_storage())
4116		goto out;
4117	/*
4118	 * XXX: GFP_NOFS allocations should rather fail than rely on
4119	 * other request to make a forward progress.
4120	 * We are in an unfortunate situation where out_of_memory cannot
4121	 * do much for this context but let's try it to at least get
4122	 * access to memory reserved if the current task is killed (see
4123	 * out_of_memory). Once filesystems are ready to handle allocation
4124	 * failures more gracefully we should just bail out here.
4125	 */
4126
4127	/* Exhausted what can be done so it's blame time */
4128	if (out_of_memory(&oc) ||
4129	    WARN_ON_ONCE_GFP(gfp_mask & __GFP_NOFAIL, gfp_mask)) {
4130		*did_some_progress = 1;
4131
4132		/*
4133		 * Help non-failing allocations by giving them access to memory
4134		 * reserves
4135		 */
4136		if (gfp_mask & __GFP_NOFAIL)
4137			page = __alloc_pages_cpuset_fallback(gfp_mask, order,
4138					ALLOC_NO_WATERMARKS, ac);
4139	}
4140out:
4141	mutex_unlock(&oom_lock);
4142	return page;
4143}
4144
4145/*
4146 * Maximum number of compaction retries with a progress before OOM
4147 * killer is consider as the only way to move forward.
4148 */
4149#define MAX_COMPACT_RETRIES 16
4150
4151#ifdef CONFIG_COMPACTION
4152/* Try memory compaction for high-order allocations before reclaim */
4153static struct page *
4154__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
4155		unsigned int alloc_flags, const struct alloc_context *ac,
4156		enum compact_priority prio, enum compact_result *compact_result)
4157{
4158	struct page *page = NULL;
4159	unsigned long pflags;
4160	unsigned int noreclaim_flag;
4161
4162	if (!order)
4163		return NULL;
4164
4165	psi_memstall_enter(&pflags);
4166	delayacct_compact_start();
4167	noreclaim_flag = memalloc_noreclaim_save();
4168
4169	*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
4170								prio, &page);
4171
4172	memalloc_noreclaim_restore(noreclaim_flag);
4173	psi_memstall_leave(&pflags);
4174	delayacct_compact_end();
4175
4176	if (*compact_result == COMPACT_SKIPPED)
4177		return NULL;
4178	/*
4179	 * At least in one zone compaction wasn't deferred or skipped, so let's
4180	 * count a compaction stall
4181	 */
4182	count_vm_event(COMPACTSTALL);
4183
4184	/* Prep a captured page if available */
4185	if (page)
4186		prep_new_page(page, order, gfp_mask, alloc_flags);
4187
4188	/* Try get a page from the freelist if available */
4189	if (!page)
4190		page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4191
4192	if (page) {
4193		struct zone *zone = page_zone(page);
4194
4195		zone->compact_blockskip_flush = false;
4196		compaction_defer_reset(zone, order, true);
4197		count_vm_event(COMPACTSUCCESS);
4198		return page;
4199	}
4200
4201	/*
4202	 * It's bad if compaction run occurs and fails. The most likely reason
4203	 * is that pages exist, but not enough to satisfy watermarks.
4204	 */
4205	count_vm_event(COMPACTFAIL);
4206
4207	cond_resched();
4208
4209	return NULL;
4210}
4211
4212static inline bool
4213should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
4214		     enum compact_result compact_result,
4215		     enum compact_priority *compact_priority,
4216		     int *compaction_retries)
4217{
4218	int max_retries = MAX_COMPACT_RETRIES;
4219	int min_priority;
4220	bool ret = false;
4221	int retries = *compaction_retries;
4222	enum compact_priority priority = *compact_priority;
4223
4224	if (!order)
4225		return false;
4226
4227	if (fatal_signal_pending(current))
4228		return false;
4229
4230	/*
4231	 * Compaction was skipped due to a lack of free order-0
4232	 * migration targets. Continue if reclaim can help.
4233	 */
4234	if (compact_result == COMPACT_SKIPPED) {
4235		ret = compaction_zonelist_suitable(ac, order, alloc_flags);
4236		goto out;
4237	}
4238
4239	/*
4240	 * Compaction managed to coalesce some page blocks, but the
4241	 * allocation failed presumably due to a race. Retry some.
4242	 */
4243	if (compact_result == COMPACT_SUCCESS) {
4244		/*
4245		 * !costly requests are much more important than
4246		 * __GFP_RETRY_MAYFAIL costly ones because they are de
4247		 * facto nofail and invoke OOM killer to move on while
4248		 * costly can fail and users are ready to cope with
4249		 * that. 1/4 retries is rather arbitrary but we would
4250		 * need much more detailed feedback from compaction to
4251		 * make a better decision.
4252		 */
4253		if (order > PAGE_ALLOC_COSTLY_ORDER)
4254			max_retries /= 4;
4255
4256		if (++(*compaction_retries) <= max_retries) {
4257			ret = true;
4258			goto out;
4259		}
4260	}
4261
4262	/*
4263	 * Compaction failed. Retry with increasing priority.
4264	 */
4265	min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
4266			MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
4267
4268	if (*compact_priority > min_priority) {
4269		(*compact_priority)--;
4270		*compaction_retries = 0;
4271		ret = true;
4272	}
4273out:
4274	trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
4275	return ret;
4276}
4277#else
4278static inline struct page *
4279__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
4280		unsigned int alloc_flags, const struct alloc_context *ac,
4281		enum compact_priority prio, enum compact_result *compact_result)
4282{
4283	*compact_result = COMPACT_SKIPPED;
4284	return NULL;
4285}
4286
4287static inline bool
4288should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
4289		     enum compact_result compact_result,
4290		     enum compact_priority *compact_priority,
4291		     int *compaction_retries)
4292{
4293	struct zone *zone;
4294	struct zoneref *z;
4295
4296	if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
4297		return false;
4298
4299	/*
4300	 * There are setups with compaction disabled which would prefer to loop
4301	 * inside the allocator rather than hit the oom killer prematurely.
4302	 * Let's give them a good hope and keep retrying while the order-0
4303	 * watermarks are OK.
4304	 */
4305	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4306				ac->highest_zoneidx, ac->nodemask) {
4307		if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
4308					ac->highest_zoneidx, alloc_flags))
4309			return true;
4310	}
4311	return false;
4312}
4313#endif /* CONFIG_COMPACTION */
4314
4315#ifdef CONFIG_LOCKDEP
4316static struct lockdep_map __fs_reclaim_map =
4317	STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
4318
4319static bool __need_reclaim(gfp_t gfp_mask)
4320{
4321	/* no reclaim without waiting on it */
4322	if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
4323		return false;
4324
4325	/* this guy won't enter reclaim */
4326	if (current->flags & PF_MEMALLOC)
4327		return false;
4328
4329	if (gfp_mask & __GFP_NOLOCKDEP)
4330		return false;
4331
4332	return true;
4333}
4334
4335void __fs_reclaim_acquire(unsigned long ip)
4336{
4337	lock_acquire_exclusive(&__fs_reclaim_map, 0, 0, NULL, ip);
4338}
4339
4340void __fs_reclaim_release(unsigned long ip)
4341{
4342	lock_release(&__fs_reclaim_map, ip);
4343}
4344
4345void fs_reclaim_acquire(gfp_t gfp_mask)
4346{
4347	gfp_mask = current_gfp_context(gfp_mask);
4348
4349	if (__need_reclaim(gfp_mask)) {
4350		if (gfp_mask & __GFP_FS)
4351			__fs_reclaim_acquire(_RET_IP_);
4352
4353#ifdef CONFIG_MMU_NOTIFIER
4354		lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
4355		lock_map_release(&__mmu_notifier_invalidate_range_start_map);
4356#endif
4357
4358	}
4359}
4360EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
4361
4362void fs_reclaim_release(gfp_t gfp_mask)
4363{
4364	gfp_mask = current_gfp_context(gfp_mask);
4365
4366	if (__need_reclaim(gfp_mask)) {
4367		if (gfp_mask & __GFP_FS)
4368			__fs_reclaim_release(_RET_IP_);
4369	}
4370}
4371EXPORT_SYMBOL_GPL(fs_reclaim_release);
4372#endif
4373
4374/*
4375 * Zonelists may change due to hotplug during allocation. Detect when zonelists
4376 * have been rebuilt so allocation retries. Reader side does not lock and
4377 * retries the allocation if zonelist changes. Writer side is protected by the
4378 * embedded spin_lock.
4379 */
4380static DEFINE_SEQLOCK(zonelist_update_seq);
4381
4382static unsigned int zonelist_iter_begin(void)
4383{
4384	if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
4385		return read_seqbegin(&zonelist_update_seq);
4386
4387	return 0;
4388}
4389
4390static unsigned int check_retry_zonelist(unsigned int seq)
4391{
4392	if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
4393		return read_seqretry(&zonelist_update_seq, seq);
4394
4395	return seq;
4396}
4397
4398/* Perform direct synchronous page reclaim */
4399static unsigned long
4400__perform_reclaim(gfp_t gfp_mask, unsigned int order,
4401					const struct alloc_context *ac)
4402{
4403	unsigned int noreclaim_flag;
4404	unsigned long progress;
4405
4406	cond_resched();
4407
4408	/* We now go into synchronous reclaim */
4409	cpuset_memory_pressure_bump();
4410	fs_reclaim_acquire(gfp_mask);
4411	noreclaim_flag = memalloc_noreclaim_save();
4412
4413	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
4414								ac->nodemask);
4415
4416	memalloc_noreclaim_restore(noreclaim_flag);
4417	fs_reclaim_release(gfp_mask);
4418
4419	cond_resched();
4420
4421	return progress;
4422}
4423
4424/* The really slow allocator path where we enter direct reclaim */
4425static inline struct page *
4426__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
4427		unsigned int alloc_flags, const struct alloc_context *ac,
4428		unsigned long *did_some_progress)
4429{
4430	struct page *page = NULL;
4431	unsigned long pflags;
4432	bool drained = false;
4433
4434	psi_memstall_enter(&pflags);
4435	*did_some_progress = __perform_reclaim(gfp_mask, order, ac);
4436	if (unlikely(!(*did_some_progress)))
4437		goto out;
4438
4439retry:
4440	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4441
4442	/*
4443	 * If an allocation failed after direct reclaim, it could be because
4444	 * pages are pinned on the per-cpu lists or in high alloc reserves.
4445	 * Shrink them and try again
4446	 */
4447	if (!page && !drained) {
4448		unreserve_highatomic_pageblock(ac, false);
4449		drain_all_pages(NULL);
4450		drained = true;
4451		goto retry;
4452	}
4453out:
4454	psi_memstall_leave(&pflags);
4455
4456	return page;
4457}
4458
4459static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
4460			     const struct alloc_context *ac)
4461{
4462	struct zoneref *z;
4463	struct zone *zone;
4464	pg_data_t *last_pgdat = NULL;
4465	enum zone_type highest_zoneidx = ac->highest_zoneidx;
4466	unsigned int reclaim_order;
4467
4468	if (defrag_mode)
4469		reclaim_order = max(order, pageblock_order);
4470	else
4471		reclaim_order = order;
4472
4473	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,
4474					ac->nodemask) {
4475		if (!managed_zone(zone))
4476			continue;
4477		if (last_pgdat == zone->zone_pgdat)
4478			continue;
4479		wakeup_kswapd(zone, gfp_mask, reclaim_order, highest_zoneidx);
4480		last_pgdat = zone->zone_pgdat;
4481	}
4482}
4483
4484static inline unsigned int
4485gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
4486{
4487	unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
4488
4489	/*
4490	 * __GFP_HIGH is assumed to be the same as ALLOC_MIN_RESERVE
4491	 * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
4492	 * to save two branches.
4493	 */
4494	BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_MIN_RESERVE);
4495	BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
4496
4497	/*
4498	 * The caller may dip into page reserves a bit more if the caller
4499	 * cannot run direct reclaim, or if the caller has realtime scheduling
4500	 * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
4501	 * set both ALLOC_NON_BLOCK and ALLOC_MIN_RESERVE(__GFP_HIGH).
4502	 */
4503	alloc_flags |= (__force int)
4504		(gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
4505
4506	if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
4507		/*
4508		 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
4509		 * if it can't schedule.
4510		 */
4511		if (!(gfp_mask & __GFP_NOMEMALLOC)) {
4512			alloc_flags |= ALLOC_NON_BLOCK;
4513
4514			if (order > 0 && (alloc_flags & ALLOC_MIN_RESERVE))
4515				alloc_flags |= ALLOC_HIGHATOMIC;
4516		}
4517
4518		/*
4519		 * Ignore cpuset mems for non-blocking __GFP_HIGH (probably
4520		 * GFP_ATOMIC) rather than fail, see the comment for
4521		 * cpuset_current_node_allowed().
4522		 */
4523		if (alloc_flags & ALLOC_MIN_RESERVE)
4524			alloc_flags &= ~ALLOC_CPUSET;
4525	} else if (unlikely(rt_or_dl_task(current)) && in_task())
4526		alloc_flags |= ALLOC_MIN_RESERVE;
4527
4528	alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);
4529
4530	if (defrag_mode)
4531		alloc_flags |= ALLOC_NOFRAGMENT;
4532
4533	return alloc_flags;
4534}
4535
4536static bool oom_reserves_allowed(struct task_struct *tsk)
4537{
4538	if (!tsk_is_oom_victim(tsk))
4539		return false;
4540
4541	/*
4542	 * !MMU doesn't have oom reaper so give access to memory reserves
4543	 * only to the thread with TIF_MEMDIE set
4544	 */
4545	if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
4546		return false;
4547
4548	return true;
4549}
4550
4551/*
4552 * Distinguish requests which really need access to full memory
4553 * reserves from oom victims which can live with a portion of it
4554 */
4555static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
4556{
4557	if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
4558		return 0;
4559	if (gfp_mask & __GFP_MEMALLOC)
4560		return ALLOC_NO_WATERMARKS;
4561	if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
4562		return ALLOC_NO_WATERMARKS;
4563	if (!in_interrupt()) {
4564		if (current->flags & PF_MEMALLOC)
4565			return ALLOC_NO_WATERMARKS;
4566		else if (oom_reserves_allowed(current))
4567			return ALLOC_OOM;
4568	}
4569
4570	return 0;
4571}
4572
4573bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
4574{
4575	return !!__gfp_pfmemalloc_flags(gfp_mask);
4576}
4577
4578/*
4579 * Checks whether it makes sense to retry the reclaim to make a forward progress
4580 * for the given allocation request.
4581 *
4582 * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
4583 * without success, or when we couldn't even meet the watermark if we
4584 * reclaimed all remaining pages on the LRU lists.
4585 *
4586 * Returns true if a retry is viable or false to enter the oom path.
4587 */
4588static inline bool
4589should_reclaim_retry(gfp_t gfp_mask, unsigned order,
4590		     struct alloc_context *ac, int alloc_flags,
4591		     bool did_some_progress, int *no_progress_loops)
4592{
4593	struct zone *zone;
4594	struct zoneref *z;
4595	bool ret = false;
4596
4597	/*
4598	 * Costly allocations might have made a progress but this doesn't mean
4599	 * their order will become available due to high fragmentation so
4600	 * always increment the no progress counter for them
4601	 */
4602	if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
4603		*no_progress_loops = 0;
4604	else
4605		(*no_progress_loops)++;
4606
4607	if (*no_progress_loops > MAX_RECLAIM_RETRIES)
4608		goto out;
4609
4610
4611	/*
4612	 * Keep reclaiming pages while there is a chance this will lead
4613	 * somewhere.  If none of the target zones can satisfy our allocation
4614	 * request even if all reclaimable pages are considered then we are
4615	 * screwed and have to go OOM.
4616	 */
4617	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
4618				ac->highest_zoneidx, ac->nodemask) {
4619		unsigned long available;
4620		unsigned long reclaimable;
4621		unsigned long min_wmark = min_wmark_pages(zone);
4622		bool wmark;
4623
4624		if (cpusets_enabled() &&
4625			(alloc_flags & ALLOC_CPUSET) &&
4626			!__cpuset_zone_allowed(zone, gfp_mask))
4627				continue;
4628
4629		available = reclaimable = zone_reclaimable_pages(zone);
4630		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
4631
4632		/*
4633		 * Would the allocation succeed if we reclaimed all
4634		 * reclaimable pages?
4635		 */
4636		wmark = __zone_watermark_ok(zone, order, min_wmark,
4637				ac->highest_zoneidx, alloc_flags, available);
4638		trace_reclaim_retry_zone(z, order, reclaimable,
4639				available, min_wmark, *no_progress_loops, wmark);
4640		if (wmark) {
4641			ret = true;
4642			break;
4643		}
4644	}
4645
4646	/*
4647	 * Memory allocation/reclaim might be called from a WQ context and the
4648	 * current implementation of the WQ concurrency control doesn't
4649	 * recognize that a particular WQ is congested if the worker thread is
4650	 * looping without ever sleeping. Therefore we have to do a short sleep
4651	 * here rather than calling cond_resched().
4652	 */
4653	if (current->flags & PF_WQ_WORKER)
4654		schedule_timeout_uninterruptible(1);
4655	else
4656		cond_resched();
4657out:
4658	/* Before OOM, exhaust highatomic_reserve */
4659	if (!ret)
4660		return unreserve_highatomic_pageblock(ac, true);
4661
4662	return ret;
4663}
4664
4665static inline bool
4666check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
4667{
4668	/*
4669	 * It's possible that cpuset's mems_allowed and the nodemask from
4670	 * mempolicy don't intersect. This should be normally dealt with by
4671	 * policy_nodemask(), but it's possible to race with cpuset update in
4672	 * such a way the check therein was true, and then it became false
4673	 * before we got our cpuset_mems_cookie here.
4674	 * This assumes that for all allocations, ac->nodemask can come only
4675	 * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
4676	 * when it does not intersect with the cpuset restrictions) or the
4677	 * caller can deal with a violated nodemask.
4678	 */
4679	if (cpusets_enabled() && ac->nodemask &&
4680			!cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
4681		ac->nodemask = NULL;
4682		return true;
4683	}
4684
4685	/*
4686	 * When updating a task's mems_allowed or mempolicy nodemask, it is
4687	 * possible to race with parallel threads in such a way that our
4688	 * allocation can fail while the mask is being updated. If we are about
4689	 * to fail, check if the cpuset changed during allocation and if so,
4690	 * retry.
4691	 */
4692	if (read_mems_allowed_retry(cpuset_mems_cookie))
4693		return true;
4694
4695	return false;
4696}
4697
4698static inline struct page *
4699__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
4700						struct alloc_context *ac)
4701{
4702	bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
4703	bool can_compact = gfp_compaction_allowed(gfp_mask);
4704	bool nofail = gfp_mask & __GFP_NOFAIL;
4705	const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
4706	struct page *page = NULL;
4707	unsigned int alloc_flags;
4708	unsigned long did_some_progress;
4709	enum compact_priority compact_priority;
4710	enum compact_result compact_result;
4711	int compaction_retries;
4712	int no_progress_loops;
4713	unsigned int cpuset_mems_cookie;
4714	unsigned int zonelist_iter_cookie;
4715	int reserve_flags;
4716
4717	if (unlikely(nofail)) {
4718		/*
4719		 * Also we don't support __GFP_NOFAIL without __GFP_DIRECT_RECLAIM,
4720		 * otherwise, we may result in lockup.
4721		 */
4722		WARN_ON_ONCE(!can_direct_reclaim);
4723		/*
4724		 * PF_MEMALLOC request from this context is rather bizarre
4725		 * because we cannot reclaim anything and only can loop waiting
4726		 * for somebody to do a work for us.
4727		 */
4728		WARN_ON_ONCE(current->flags & PF_MEMALLOC);
4729	}
4730
4731restart:
4732	compaction_retries = 0;
4733	no_progress_loops = 0;
4734	compact_result = COMPACT_SKIPPED;
4735	compact_priority = DEF_COMPACT_PRIORITY;
4736	cpuset_mems_cookie = read_mems_allowed_begin();
4737	zonelist_iter_cookie = zonelist_iter_begin();
4738
4739	/*
4740	 * The fast path uses conservative alloc_flags to succeed only until
4741	 * kswapd needs to be woken up, and to avoid the cost of setting up
4742	 * alloc_flags precisely. So we do that now.
4743	 */
4744	alloc_flags = gfp_to_alloc_flags(gfp_mask, order);
4745
4746	/*
4747	 * We need to recalculate the starting point for the zonelist iterator
4748	 * because we might have used different nodemask in the fast path, or
4749	 * there was a cpuset modification and we are retrying - otherwise we
4750	 * could end up iterating over non-eligible zones endlessly.
4751	 */
4752	ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4753					ac->highest_zoneidx, ac->nodemask);
4754	if (!zonelist_zone(ac->preferred_zoneref))
4755		goto nopage;
4756
4757	/*
4758	 * Check for insane configurations where the cpuset doesn't contain
4759	 * any suitable zone to satisfy the request - e.g. non-movable
4760	 * GFP_HIGHUSER allocations from MOVABLE nodes only.
4761	 */
4762	if (cpusets_insane_config() && (gfp_mask & __GFP_HARDWALL)) {
4763		struct zoneref *z = first_zones_zonelist(ac->zonelist,
4764					ac->highest_zoneidx,
4765					&cpuset_current_mems_allowed);
4766		if (!zonelist_zone(z))
4767			goto nopage;
4768	}
4769
4770	if (alloc_flags & ALLOC_KSWAPD)
4771		wake_all_kswapds(order, gfp_mask, ac);
4772
4773	/*
4774	 * The adjusted alloc_flags might result in immediate success, so try
4775	 * that first
4776	 */
4777	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4778	if (page)
4779		goto got_pg;
4780
4781	/*
4782	 * For costly allocations, try direct compaction first, as it's likely
4783	 * that we have enough base pages and don't need to reclaim. For non-
4784	 * movable high-order allocations, do that as well, as compaction will
4785	 * try prevent permanent fragmentation by migrating from blocks of the
4786	 * same migratetype.
4787	 * Don't try this for allocations that are allowed to ignore
4788	 * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
4789	 */
4790	if (can_direct_reclaim && can_compact &&
4791			(costly_order ||
4792			   (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
4793			&& !gfp_pfmemalloc_allowed(gfp_mask)) {
4794		page = __alloc_pages_direct_compact(gfp_mask, order,
4795						alloc_flags, ac,
4796						INIT_COMPACT_PRIORITY,
4797						&compact_result);
4798		if (page)
4799			goto got_pg;
4800
4801		/*
4802		 * Checks for costly allocations with __GFP_NORETRY, which
4803		 * includes some THP page fault allocations
4804		 */
4805		if (costly_order && (gfp_mask & __GFP_NORETRY)) {
4806			/*
4807			 * If allocating entire pageblock(s) and compaction
4808			 * failed because all zones are below low watermarks
4809			 * or is prohibited because it recently failed at this
4810			 * order, fail immediately unless the allocator has
4811			 * requested compaction and reclaim retry.
4812			 *
4813			 * Reclaim is
4814			 *  - potentially very expensive because zones are far
4815			 *    below their low watermarks or this is part of very
4816			 *    bursty high order allocations,
4817			 *  - not guaranteed to help because isolate_freepages()
4818			 *    may not iterate over freed pages as part of its
4819			 *    linear scan, and
4820			 *  - unlikely to make entire pageblocks free on its
4821			 *    own.
4822			 */
4823			if (compact_result == COMPACT_SKIPPED ||
4824			    compact_result == COMPACT_DEFERRED)
4825				goto nopage;
4826
4827			/*
4828			 * THP page faults may attempt local node only first,
4829			 * but are then allowed to only compact, not reclaim,
4830			 * see alloc_pages_mpol().
4831			 *
4832			 * Compaction can fail for other reasons than those
4833			 * checked above and we don't want such THP allocations
4834			 * to put reclaim pressure on a single node in a
4835			 * situation where other nodes might have plenty of
4836			 * available memory.
4837			 */
4838			if (gfp_mask & __GFP_THISNODE)
4839				goto nopage;
4840
4841			/*
4842			 * Looks like reclaim/compaction is worth trying, but
4843			 * sync compaction could be very expensive, so keep
4844			 * using async compaction.
4845			 */
4846			compact_priority = INIT_COMPACT_PRIORITY;
4847		}
4848	}
4849
4850retry:
4851	/*
4852	 * Deal with possible cpuset update races or zonelist updates to avoid
4853	 * infinite retries.
4854	 */
4855	if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
4856	    check_retry_zonelist(zonelist_iter_cookie))
4857		goto restart;
4858
4859	/* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
4860	if (alloc_flags & ALLOC_KSWAPD)
4861		wake_all_kswapds(order, gfp_mask, ac);
4862
4863	reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
4864	if (reserve_flags)
4865		alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) |
4866					  (alloc_flags & ALLOC_KSWAPD);
4867
4868	/*
4869	 * Reset the nodemask and zonelist iterators if memory policies can be
4870	 * ignored. These allocations are high priority and system rather than
4871	 * user oriented.
4872	 */
4873	if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
4874		ac->nodemask = NULL;
4875		ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
4876					ac->highest_zoneidx, ac->nodemask);
4877	}
4878
4879	/* Attempt with potentially adjusted zonelist and alloc_flags */
4880	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
4881	if (page)
4882		goto got_pg;
4883
4884	/* Caller is not willing to reclaim, we can't balance anything */
4885	if (!can_direct_reclaim)
4886		goto nopage;
4887
4888	/* Avoid recursion of direct reclaim */
4889	if (current->flags & PF_MEMALLOC)
4890		goto nopage;
4891
4892	/* Try direct reclaim and then allocating */
4893	page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
4894							&did_some_progress);
4895	if (page)
4896		goto got_pg;
4897
4898	/* Try direct compaction and then allocating */
4899	page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
4900					compact_priority, &compact_result);
4901	if (page)
4902		goto got_pg;
4903
4904	/* Do not loop if specifically requested */
4905	if (gfp_mask & __GFP_NORETRY)
4906		goto nopage;
4907
4908	/*
4909	 * Do not retry costly high order allocations unless they are
4910	 * __GFP_RETRY_MAYFAIL and we can compact
4911	 */
4912	if (costly_order && (!can_compact ||
4913			     !(gfp_mask & __GFP_RETRY_MAYFAIL)))
4914		goto nopage;
4915
4916	if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
4917				 did_some_progress > 0, &no_progress_loops))
4918		goto retry;
4919
4920	/*
4921	 * It doesn't make any sense to retry for the compaction if the order-0
4922	 * reclaim is not able to make any progress because the current
4923	 * implementation of the compaction depends on the sufficient amount
4924	 * of free memory (see __compaction_suitable)
4925	 */
4926	if (did_some_progress > 0 && can_compact &&
4927			should_compact_retry(ac, order, alloc_flags,
4928				compact_result, &compact_priority,
4929				&compaction_retries))
4930		goto retry;
4931
4932	/* Reclaim/compaction failed to prevent the fallback */
4933	if (defrag_mode && (alloc_flags & ALLOC_NOFRAGMENT)) {
4934		alloc_flags &= ~ALLOC_NOFRAGMENT;
4935		goto retry;
4936	}
4937
4938	/*
4939	 * Deal with possible cpuset update races or zonelist updates to avoid
4940	 * a unnecessary OOM kill.
4941	 */
4942	if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
4943	    check_retry_zonelist(zonelist_iter_cookie))
4944		goto restart;
4945
4946	/* Reclaim has failed us, start killing things */
4947	page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
4948	if (page)
4949		goto got_pg;
4950
4951	/* Avoid allocations with no watermarks from looping endlessly */
4952	if (tsk_is_oom_victim(current) &&
4953	    (alloc_flags & ALLOC_OOM ||
4954	     (gfp_mask & __GFP_NOMEMALLOC)))
4955		goto nopage;
4956
4957	/* Retry as long as the OOM killer is making progress */
4958	if (did_some_progress) {
4959		no_progress_loops = 0;
4960		goto retry;
4961	}
4962
4963nopage:
4964	/*
4965	 * Deal with possible cpuset update races or zonelist updates to avoid
4966	 * a unnecessary OOM kill.
4967	 */
4968	if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
4969	    check_retry_zonelist(zonelist_iter_cookie))
4970		goto restart;
4971
4972	/*
4973	 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
4974	 * we always retry
4975	 */
4976	if (unlikely(nofail)) {
4977		/*
4978		 * Lacking direct_reclaim we can't do anything to reclaim memory,
4979		 * we disregard these unreasonable nofail requests and still
4980		 * return NULL
4981		 */
4982		if (!can_direct_reclaim)
4983			goto fail;
4984
4985		/*
4986		 * Help non-failing allocations by giving some access to memory
4987		 * reserves normally used for high priority non-blocking
4988		 * allocations but do not use ALLOC_NO_WATERMARKS because this
4989		 * could deplete whole memory reserves which would just make
4990		 * the situation worse.
4991		 */
4992		page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_MIN_RESERVE, ac);
4993		if (page)
4994			goto got_pg;
4995
4996		cond_resched();
4997		goto retry;
4998	}
4999fail:
5000	warn_alloc(gfp_mask, ac->nodemask,
5001			"page allocation failure: order:%u", order);
5002got_pg:
5003	return page;
5004}
5005
5006static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
5007		int preferred_nid, nodemask_t *nodemask,
5008		struct alloc_context *ac, gfp_t *alloc_gfp,
5009		unsigned int *alloc_flags)
5010{
5011	ac->highest_zoneidx = gfp_zone(gfp_mask);
5012	ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
5013	ac->nodemask = nodemask;
5014	ac->migratetype = gfp_migratetype(gfp_mask);
5015
5016	if (cpusets_enabled()) {
5017		*alloc_gfp |= __GFP_HARDWALL;
5018		/*
5019		 * When we are in the interrupt context, it is irrelevant
5020		 * to the current task context. It means that any node ok.
5021		 */
5022		if (in_task() && !ac->nodemask)
5023			ac->nodemask = &cpuset_current_mems_allowed;
5024		else
5025			*alloc_flags |= ALLOC_CPUSET;
5026	}
5027
5028	might_alloc(gfp_mask);
5029
5030	/*
5031	 * Don't invoke should_fail logic, since it may call
5032	 * get_random_u32() and printk() which need to spin_lock.
5033	 */
5034	if (!(*alloc_flags & ALLOC_TRYLOCK) &&
5035	    should_fail_alloc_page(gfp_mask, order))
5036		return false;
5037
5038	*alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags);
5039
5040	/* Dirty zone balancing only done in the fast path */
5041	ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
5042
5043	/*
5044	 * The preferred zone is used for statistics but crucially it is
5045	 * also used as the starting point for the zonelist iterator. It
5046	 * may get reset for allocations that ignore memory policies.
5047	 */
5048	ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
5049					ac->highest_zoneidx, ac->nodemask);
5050
5051	return true;
5052}
5053
5054/*
5055 * __alloc_pages_bulk - Allocate a number of order-0 pages to an array
5056 * @gfp: GFP flags for the allocation
5057 * @preferred_nid: The preferred NUMA node ID to allocate from
5058 * @nodemask: Set of nodes to allocate from, may be NULL
5059 * @nr_pages: The number of pages desired in the array
5060 * @page_array: Array to store the pages
5061 *
5062 * This is a batched version of the page allocator that attempts to allocate
5063 * @nr_pages quickly.  Pages are added to @page_array.
5064 *
5065 * Note that only the elements in @page_array that were cleared to %NULL on
5066 * entry are populated with newly allocated pages. @nr_pages is the maximum
5067 * number of pages that will be stored in the array.
5068 *
5069 * Returns the number of pages in @page_array, including ones already
5070 * allocated on entry.  This can be less than the number requested in @nr_pages,
5071 * but all empty slots are filled from the beginning.  I.e., if all slots in
5072 * @page_array were set to %NULL on entry, the slots from 0 to the return value
5073 * - 1 will be filled.
5074 */
5075unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
5076			nodemask_t *nodemask, int nr_pages,
5077			struct page **page_array)
5078{
5079	struct page *page;
5080	unsigned long UP_flags;
5081	struct zone *zone;
5082	struct zoneref *z;
5083	struct per_cpu_pages *pcp;
5084	struct list_head *pcp_list;
5085	struct alloc_context ac;
5086	gfp_t alloc_gfp;
5087	unsigned int alloc_flags = ALLOC_WMARK_LOW;
5088	int nr_populated = 0, nr_account = 0;
5089
5090	/*
5091	 * Skip populated array elements to determine if any pages need
5092	 * to be allocated before disabling IRQs.
5093	 */
5094	while (nr_populated < nr_pages && page_array[nr_populated])
5095		nr_populated++;
5096
5097	/* No pages requested? */
5098	if (unlikely(nr_pages <= 0))
5099		goto out;
5100
5101	/* Already populated array? */
5102	if (unlikely(nr_pages - nr_populated == 0))
5103		goto out;
5104
5105	/* Bulk allocator does not support memcg accounting. */
5106	if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT))
5107		goto failed;
5108
5109	/* Use the single page allocator for one page. */
5110	if (nr_pages - nr_populated == 1)
5111		goto failed;
5112
5113#ifdef CONFIG_PAGE_OWNER
5114	/*
5115	 * PAGE_OWNER may recurse into the allocator to allocate space to
5116	 * save the stack with pagesets.lock held. Releasing/reacquiring
5117	 * removes much of the performance benefit of bulk allocation so
5118	 * force the caller to allocate one page at a time as it'll have
5119	 * similar performance to added complexity to the bulk allocator.
5120	 */
5121	if (static_branch_unlikely(&page_owner_inited))
5122		goto failed;
5123#endif
5124
5125	/* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */
5126	gfp &= gfp_allowed_mask;
5127	alloc_gfp = gfp;
5128	if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags))
5129		goto out;
5130	gfp = alloc_gfp;
5131
5132	/* Find an allowed local zone that meets the low watermark. */
5133	z = ac.preferred_zoneref;
5134	for_next_zone_zonelist_nodemask(zone, z, ac.highest_zoneidx, ac.nodemask) {
5135		unsigned long mark;
5136
5137		if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) &&
5138		    !__cpuset_zone_allowed(zone, gfp)) {
5139			continue;
5140		}
5141
5142		if (nr_online_nodes > 1 && zone != zonelist_zone(ac.preferred_zoneref) &&
5143		    zone_to_nid(zone) != zonelist_node_idx(ac.preferred_zoneref)) {
5144			goto failed;
5145		}
5146
5147		cond_accept_memory(zone, 0, alloc_flags);
5148retry_this_zone:
5149		mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages;
5150		if (zone_watermark_fast(zone, 0,  mark,
5151				zonelist_zone_idx(ac.preferred_zoneref),
5152				alloc_flags, gfp)) {
5153			break;
5154		}
5155
5156		if (cond_accept_memory(zone, 0, alloc_flags))
5157			goto retry_this_zone;
5158
5159		/* Try again if zone has deferred pages */
5160		if (deferred_pages_enabled()) {
5161			if (_deferred_grow_zone(zone, 0))
5162				goto retry_this_zone;
5163		}
5164	}
5165
5166	/*
5167	 * If there are no allowed local zones that meets the watermarks then
5168	 * try to allocate a single page and reclaim if necessary.
5169	 */
5170	if (unlikely(!zone))
5171		goto failed;
5172
5173	/* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */
5174	pcp = pcp_spin_trylock(zone->per_cpu_pageset, UP_flags);
5175	if (!pcp)
5176		goto failed;
5177
5178	/* Attempt the batch allocation */
5179	pcp_list = &pcp->lists[order_to_pindex(ac.migratetype, 0)];
5180	while (nr_populated < nr_pages) {
5181
5182		/* Skip existing pages */
5183		if (page_array[nr_populated]) {
5184			nr_populated++;
5185			continue;
5186		}
5187
5188		page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags,
5189								pcp, pcp_list);
5190		if (unlikely(!page)) {
5191			/* Try and allocate at least one page */
5192			if (!nr_account) {
5193				pcp_spin_unlock(pcp, UP_flags);
5194				goto failed;
5195			}
5196			break;
5197		}
5198		nr_account++;
5199
5200		prep_new_page(page, 0, gfp, 0);
5201		set_page_refcounted(page);
5202		page_array[nr_populated++] = page;
5203	}
5204
5205	pcp_spin_unlock(pcp, UP_flags);
5206
5207	__count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account);
5208	zone_statistics(zonelist_zone(ac.preferred_zoneref), zone, nr_account);
5209
5210out:
5211	return nr_populated;
5212
5213failed:
5214	page = __alloc_pages_noprof(gfp, 0, preferred_nid, nodemask);
5215	if (page)
5216		page_array[nr_populated++] = page;
5217	goto out;
5218}
5219EXPORT_SYMBOL_GPL(alloc_pages_bulk_noprof);
5220
5221/*
5222 * This is the 'heart' of the zoned buddy allocator.
5223 */
5224struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
5225		int preferred_nid, nodemask_t *nodemask)
5226{
5227	struct page *page;
5228	unsigned int alloc_flags = ALLOC_WMARK_LOW;
5229	gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
5230	struct alloc_context ac = { };
5231
5232	/*
5233	 * There are several places where we assume that the order value is sane
5234	 * so bail out early if the request is out of bound.
5235	 */
5236	if (WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER, gfp))
5237		return NULL;
5238
5239	gfp &= gfp_allowed_mask;
5240	/*
5241	 * Apply scoped allocation constraints. This is mainly about GFP_NOFS
5242	 * resp. GFP_NOIO which has to be inherited for all allocation requests
5243	 * from a particular context which has been marked by
5244	 * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures
5245	 * movable zones are not used during allocation.
5246	 */
5247	gfp = current_gfp_context(gfp);
5248	alloc_gfp = gfp;
5249	if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac,
5250			&alloc_gfp, &alloc_flags))
5251		return NULL;
5252
5253	/*
5254	 * Forbid the first pass from falling back to types that fragment
5255	 * memory until all local zones are considered.
5256	 */
5257	alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp);
5258
5259	/* First allocation attempt */
5260	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
5261	if (likely(page))
5262		goto out;
5263
5264	alloc_gfp = gfp;
5265	ac.spread_dirty_pages = false;
5266
5267	/*
5268	 * Restore the original nodemask if it was potentially replaced with
5269	 * &cpuset_current_mems_allowed to optimize the fast-path attempt.
5270	 */
5271	ac.nodemask = nodemask;
5272
5273	page = __alloc_pages_slowpath(alloc_gfp, order, &ac);
5274
5275out:
5276	if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page &&
5277	    unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) {
5278		free_frozen_pages(page, order);
5279		page = NULL;
5280	}
5281
5282	trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
5283	kmsan_alloc_page(page, order, alloc_gfp);
5284
5285	return page;
5286}
5287EXPORT_SYMBOL(__alloc_frozen_pages_noprof);
5288
5289struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order,
5290		int preferred_nid, nodemask_t *nodemask)
5291{
5292	struct page *page;
5293
5294	page = __alloc_frozen_pages_noprof(gfp, order, preferred_nid, nodemask);
5295	if (page)
5296		set_page_refcounted(page);
5297	return page;
5298}
5299EXPORT_SYMBOL(__alloc_pages_noprof);
5300
5301struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid,
5302		nodemask_t *nodemask)
5303{
5304	struct page *page = __alloc_pages_noprof(gfp | __GFP_COMP, order,
5305					preferred_nid, nodemask);
5306	return page_rmappable_folio(page);
5307}
5308EXPORT_SYMBOL(__folio_alloc_noprof);
5309
5310/*
5311 * Common helper functions. Never use with __GFP_HIGHMEM because the returned
5312 * address cannot represent highmem pages. Use alloc_pages and then kmap if
5313 * you need to access high mem.
5314 */
5315unsigned long get_free_pages_noprof(gfp_t gfp_mask, unsigned int order)
5316{
5317	struct page *page;
5318
5319	page = alloc_pages_noprof(gfp_mask & ~__GFP_HIGHMEM, order);
5320	if (!page)
5321		return 0;
5322	return (unsigned long) page_address(page);
5323}
5324EXPORT_SYMBOL(get_free_pages_noprof);
5325
5326unsigned long get_zeroed_page_noprof(gfp_t gfp_mask)
5327{
5328	return get_free_pages_noprof(gfp_mask | __GFP_ZERO, 0);
5329}
5330EXPORT_SYMBOL(get_zeroed_page_noprof);
5331
5332static void ___free_pages(struct page *page, unsigned int order,
5333			  fpi_t fpi_flags)
5334{
5335	/* get PageHead before we drop reference */
5336	int head = PageHead(page);
5337	/* get alloc tag in case the page is released by others */
5338	struct alloc_tag *tag = pgalloc_tag_get(page);
5339
5340	if (put_page_testzero(page))
5341		__free_frozen_pages(page, order, fpi_flags);
5342	else if (!head) {
5343		pgalloc_tag_sub_pages(tag, (1 << order) - 1);
5344		while (order-- > 0) {
5345			/*
5346			 * The "tail" pages of this non-compound high-order
5347			 * page will have no code tags, so to avoid warnings
5348			 * mark them as empty.
5349			 */
5350			clear_page_tag_ref(page + (1 << order));
5351			__free_frozen_pages(page + (1 << order), order,
5352					    fpi_flags);
5353		}
5354	}
5355}
5356
5357/**
5358 * __free_pages - Free pages allocated with alloc_pages().
5359 * @page: The page pointer returned from alloc_pages().
5360 * @order: The order of the allocation.
5361 *
5362 * This function can free multi-page allocations that are not compound
5363 * pages.  It does not check that the @order passed in matches that of
5364 * the allocation, so it is easy to leak memory.  Freeing more memory
5365 * than was allocated will probably emit a warning.
5366 *
5367 * If the last reference to this page is speculative, it will be released
5368 * by put_page() which only frees the first page of a non-compound
5369 * allocation.  To prevent the remaining pages from being leaked, we free
5370 * the subsequent pages here.  If you want to use the page's reference
5371 * count to decide when to free the allocation, you should allocate a
5372 * compound page, and use put_page() instead of __free_pages().
5373 *
5374 * Context: May be called in interrupt context or while holding a normal
5375 * spinlock, but not in NMI context or while holding a raw spinlock.
5376 */
5377void __free_pages(struct page *page, unsigned int order)
5378{
5379	___free_pages(page, order, FPI_NONE);
5380}
5381EXPORT_SYMBOL(__free_pages);
5382
5383/*
5384 * Can be called while holding raw_spin_lock or from IRQ and NMI for any
5385 * page type (not only those that came from alloc_pages_nolock)
5386 */
5387void free_pages_nolock(struct page *page, unsigned int order)
5388{
5389	___free_pages(page, order, FPI_TRYLOCK);
5390}
5391
5392/**
5393 * free_pages - Free pages allocated with __get_free_pages().
5394 * @addr: The virtual address tied to a page returned from __get_free_pages().
5395 * @order: The order of the allocation.
5396 *
5397 * This function behaves the same as __free_pages(). Use this function
5398 * to free pages when you only have a valid virtual address. If you have
5399 * the page, call __free_pages() instead.
5400 */
5401void free_pages(unsigned long addr, unsigned int order)
5402{
5403	if (addr != 0) {
5404		VM_BUG_ON(!virt_addr_valid((void *)addr));
5405		__free_pages(virt_to_page((void *)addr), order);
5406	}
5407}
5408
5409EXPORT_SYMBOL(free_pages);
5410
5411static void *make_alloc_exact(unsigned long addr, unsigned int order,
5412		size_t size)
5413{
5414	if (addr) {
5415		unsigned long nr = DIV_ROUND_UP(size, PAGE_SIZE);
5416		struct page *page = virt_to_page((void *)addr);
5417		struct page *last = page + nr;
5418
5419		split_page_owner(page, order, 0);
5420		pgalloc_tag_split(page_folio(page), order, 0);
5421		split_page_memcg(page, order);
5422		while (page < --last)
5423			set_page_refcounted(last);
5424
5425		last = page + (1UL << order);
5426		for (page += nr; page < last; page++)
5427			__free_pages_ok(page, 0, FPI_TO_TAIL);
5428	}
5429	return (void *)addr;
5430}
5431
5432/**
5433 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
5434 * @size: the number of bytes to allocate
5435 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
5436 *
5437 * This function is similar to alloc_pages(), except that it allocates the
5438 * minimum number of pages to satisfy the request.  alloc_pages() can only
5439 * allocate memory in power-of-two pages.
5440 *
5441 * This function is also limited by MAX_PAGE_ORDER.
5442 *
5443 * Memory allocated by this function must be released by free_pages_exact().
5444 *
5445 * Return: pointer to the allocated area or %NULL in case of error.
5446 */
5447void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask)
5448{
5449	unsigned int order = get_order(size);
5450	unsigned long addr;
5451
5452	if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM)))
5453		gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM);
5454
5455	addr = get_free_pages_noprof(gfp_mask, order);
5456	return make_alloc_exact(addr, order, size);
5457}
5458EXPORT_SYMBOL(alloc_pages_exact_noprof);
5459
5460/**
5461 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
5462 *			   pages on a node.
5463 * @nid: the preferred node ID where memory should be allocated
5464 * @size: the number of bytes to allocate
5465 * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
5466 *
5467 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
5468 * back.
5469 *
5470 * Return: pointer to the allocated area or %NULL in case of error.
5471 */
5472void * __meminit alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask)
5473{
5474	unsigned int order = get_order(size);
5475	struct page *p;
5476
5477	if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM)))
5478		gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM);
5479
5480	p = alloc_pages_node_noprof(nid, gfp_mask, order);
5481	if (!p)
5482		return NULL;
5483	return make_alloc_exact((unsigned long)page_address(p), order, size);
5484}
5485
5486/**
5487 * free_pages_exact - release memory allocated via alloc_pages_exact()
5488 * @virt: the value returned by alloc_pages_exact.
5489 * @size: size of allocation, same value as passed to alloc_pages_exact().
5490 *
5491 * Release the memory allocated by a previous call to alloc_pages_exact.
5492 */
5493void free_pages_exact(void *virt, size_t size)
5494{
5495	unsigned long addr = (unsigned long)virt;
5496	unsigned long end = addr + PAGE_ALIGN(size);
5497
5498	while (addr < end) {
5499		free_page(addr);
5500		addr += PAGE_SIZE;
5501	}
5502}
5503EXPORT_SYMBOL(free_pages_exact);
5504
5505/**
5506 * nr_free_zone_pages - count number of pages beyond high watermark
5507 * @offset: The zone index of the highest zone
5508 *
5509 * nr_free_zone_pages() counts the number of pages which are beyond the
5510 * high watermark within all zones at or below a given zone index.  For each
5511 * zone, the number of pages is calculated as:
5512 *
5513 *     nr_free_zone_pages = managed_pages - high_pages
5514 *
5515 * Return: number of pages beyond high watermark.
5516 */
5517static unsigned long nr_free_zone_pages(int offset)
5518{
5519	struct zoneref *z;
5520	struct zone *zone;
5521
5522	/* Just pick one node, since fallback list is circular */
5523	unsigned long sum = 0;
5524
5525	struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
5526
5527	for_each_zone_zonelist(zone, z, zonelist, offset) {
5528		unsigned long size = zone_managed_pages(zone);
5529		unsigned long high = high_wmark_pages(zone);
5530		if (size > high)
5531			sum += size - high;
5532	}
5533
5534	return sum;
5535}
5536
5537/**
5538 * nr_free_buffer_pages - count number of pages beyond high watermark
5539 *
5540 * nr_free_buffer_pages() counts the number of pages which are beyond the high
5541 * watermark within ZONE_DMA and ZONE_NORMAL.
5542 *
5543 * Return: number of pages beyond high watermark within ZONE_DMA and
5544 * ZONE_NORMAL.
5545 */
5546unsigned long nr_free_buffer_pages(void)
5547{
5548	return nr_free_zone_pages(gfp_zone(GFP_USER));
5549}
5550EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
5551
5552static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
5553{
5554	zoneref->zone = zone;
5555	zoneref->zone_idx = zone_idx(zone);
5556}
5557
5558/*
5559 * Builds allocation fallback zone lists.
5560 *
5561 * Add all populated zones of a node to the zonelist.
5562 */
5563static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
5564{
5565	struct zone *zone;
5566	enum zone_type zone_type = MAX_NR_ZONES;
5567	int nr_zones = 0;
5568
5569	do {
5570		zone_type--;
5571		zone = pgdat->node_zones + zone_type;
5572		if (populated_zone(zone)) {
5573			zoneref_set_zone(zone, &zonerefs[nr_zones++]);
5574			check_highest_zone(zone_type);
5575		}
5576	} while (zone_type);
5577
5578	return nr_zones;
5579}
5580
5581#ifdef CONFIG_NUMA
5582
5583static int __parse_numa_zonelist_order(char *s)
5584{
5585	/*
5586	 * We used to support different zonelists modes but they turned
5587	 * out to be just not useful. Let's keep the warning in place
5588	 * if somebody still use the cmd line parameter so that we do
5589	 * not fail it silently
5590	 */
5591	if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
5592		pr_warn("Ignoring unsupported numa_zonelist_order value:  %s\n", s);
5593		return -EINVAL;
5594	}
5595	return 0;
5596}
5597
5598static char numa_zonelist_order[] = "Node";
5599#define NUMA_ZONELIST_ORDER_LEN	16
5600/*
5601 * sysctl handler for numa_zonelist_order
5602 */
5603static int numa_zonelist_order_handler(const struct ctl_table *table, int write,
5604		void *buffer, size_t *length, loff_t *ppos)
5605{
5606	if (write)
5607		return __parse_numa_zonelist_order(buffer);
5608	return proc_dostring(table, write, buffer, length, ppos);
5609}
5610
5611static int node_load[MAX_NUMNODES];
5612
5613/**
5614 * find_next_best_node - find the next node that should appear in a given node's fallback list
5615 * @node: node whose fallback list we're appending
5616 * @used_node_mask: nodemask_t of already used nodes
5617 *
5618 * We use a number of factors to determine which is the next node that should
5619 * appear on a given node's fallback list.  The node should not have appeared
5620 * already in @node's fallback list, and it should be the next closest node
5621 * according to the distance array (which contains arbitrary distance values
5622 * from each node to each node in the system), and should also prefer nodes
5623 * with no CPUs, since presumably they'll have very little allocation pressure
5624 * on them otherwise.
5625 *
5626 * Return: node id of the found node or %NUMA_NO_NODE if no node is found.
5627 */
5628int find_next_best_node(int node, nodemask_t *used_node_mask)
5629{
5630	int n, val;
5631	int min_val = INT_MAX;
5632	int best_node = NUMA_NO_NODE;
5633
5634	/*
5635	 * Use the local node if we haven't already, but for memoryless local
5636	 * node, we should skip it and fall back to other nodes.
5637	 */
5638	if (!node_isset(node, *used_node_mask) && node_state(node, N_MEMORY)) {
5639		node_set(node, *used_node_mask);
5640		return node;
5641	}
5642
5643	for_each_node_state(n, N_MEMORY) {
5644
5645		/* Don't want a node to appear more than once */
5646		if (node_isset(n, *used_node_mask))
5647			continue;
5648
5649		/* Use the distance array to find the distance */
5650		val = node_distance(node, n);
5651
5652		/* Penalize nodes under us ("prefer the next node") */
5653		val += (n < node);
5654
5655		/* Give preference to headless and unused nodes */
5656		if (!cpumask_empty(cpumask_of_node(n)))
5657			val += PENALTY_FOR_NODE_WITH_CPUS;
5658
5659		/* Slight preference for less loaded node */
5660		val *= MAX_NUMNODES;
5661		val += node_load[n];
5662
5663		if (val < min_val) {
5664			min_val = val;
5665			best_node = n;
5666		}
5667	}
5668
5669	if (best_node >= 0)
5670		node_set(best_node, *used_node_mask);
5671
5672	return best_node;
5673}
5674
5675
5676/*
5677 * Build zonelists ordered by node and zones within node.
5678 * This results in maximum locality--normal zone overflows into local
5679 * DMA zone, if any--but risks exhausting DMA zone.
5680 */
5681static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
5682		unsigned nr_nodes)
5683{
5684	struct zoneref *zonerefs;
5685	int i;
5686
5687	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5688
5689	for (i = 0; i < nr_nodes; i++) {
5690		int nr_zones;
5691
5692		pg_data_t *node = NODE_DATA(node_order[i]);
5693
5694		nr_zones = build_zonerefs_node(node, zonerefs);
5695		zonerefs += nr_zones;
5696	}
5697	zonerefs->zone = NULL;
5698	zonerefs->zone_idx = 0;
5699}
5700
5701/*
5702 * Build __GFP_THISNODE zonelists
5703 */
5704static void build_thisnode_zonelists(pg_data_t *pgdat)
5705{
5706	struct zoneref *zonerefs;
5707	int nr_zones;
5708
5709	zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
5710	nr_zones = build_zonerefs_node(pgdat, zonerefs);
5711	zonerefs += nr_zones;
5712	zonerefs->zone = NULL;
5713	zonerefs->zone_idx = 0;
5714}
5715
5716static void build_zonelists(pg_data_t *pgdat)
5717{
5718	static int node_order[MAX_NUMNODES];
5719	int node, nr_nodes = 0;
5720	nodemask_t used_mask = NODE_MASK_NONE;
5721	int local_node, prev_node;
5722
5723	/* NUMA-aware ordering of nodes */
5724	local_node = pgdat->node_id;
5725	prev_node = local_node;
5726
5727	memset(node_order, 0, sizeof(node_order));
5728	while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
5729		/*
5730		 * We don't want to pressure a particular node.
5731		 * So adding penalty to the first node in same
5732		 * distance group to make it round-robin.
5733		 */
5734		if (node_distance(local_node, node) !=
5735		    node_distance(local_node, prev_node))
5736			node_load[node] += 1;
5737
5738		node_order[nr_nodes++] = node;
5739		prev_node = node;
5740	}
5741
5742	build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
5743	build_thisnode_zonelists(pgdat);
5744	pr_info("Fallback order for Node %d: ", local_node);
5745	for (node = 0; node < nr_nodes; node++)
5746		pr_cont("%d ", node_order[node]);
5747	pr_cont("\n");
5748}
5749
5750#ifdef CONFIG_HAVE_MEMORYLESS_NODES
5751/*
5752 * Return node id of node used for "local" allocations.
5753 * I.e., first node id of first zone in arg node's generic zonelist.
5754 * Used for initializing percpu 'numa_mem', which is used primarily
5755 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
5756 */
5757int local_memory_node(int node)
5758{
5759	struct zoneref *z;
5760
5761	z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
5762				   gfp_zone(GFP_KERNEL),
5763				   NULL);
5764	return zonelist_node_idx(z);
5765}
5766#endif
5767
5768static void setup_min_unmapped_ratio(void);
5769static void setup_min_slab_ratio(void);
5770#else	/* CONFIG_NUMA */
5771
5772static void build_zonelists(pg_data_t *pgdat)
5773{
5774	struct zoneref *zonerefs;
5775	int nr_zones;
5776
5777	zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
5778	nr_zones = build_zonerefs_node(pgdat, zonerefs);
5779	zonerefs += nr_zones;
5780
5781	zonerefs->zone = NULL;
5782	zonerefs->zone_idx = 0;
5783}
5784
5785#endif	/* CONFIG_NUMA */
5786
5787/*
5788 * Boot pageset table. One per cpu which is going to be used for all
5789 * zones and all nodes. The parameters will be set in such a way
5790 * that an item put on a list will immediately be handed over to
5791 * the buddy list. This is safe since pageset manipulation is done
5792 * with interrupts disabled.
5793 *
5794 * The boot_pagesets must be kept even after bootup is complete for
5795 * unused processors and/or zones. They do play a role for bootstrapping
5796 * hotplugged processors.
5797 *
5798 * zoneinfo_show() and maybe other functions do
5799 * not check if the processor is online before following the pageset pointer.
5800 * Other parts of the kernel may not check if the zone is available.
5801 */
5802static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats);
5803/* These effectively disable the pcplists in the boot pageset completely */
5804#define BOOT_PAGESET_HIGH	0
5805#define BOOT_PAGESET_BATCH	1
5806static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset);
5807static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats);
5808
5809static void __build_all_zonelists(void *data)
5810{
5811	int nid;
5812	int __maybe_unused cpu;
5813	pg_data_t *self = data;
5814	unsigned long flags;
5815
5816	/*
5817	 * The zonelist_update_seq must be acquired with irqsave because the
5818	 * reader can be invoked from IRQ with GFP_ATOMIC.
5819	 */
5820	write_seqlock_irqsave(&zonelist_update_seq, flags);
5821	/*
5822	 * Also disable synchronous printk() to prevent any printk() from
5823	 * trying to hold port->lock, for
5824	 * tty_insert_flip_string_and_push_buffer() on other CPU might be
5825	 * calling kmalloc(GFP_ATOMIC | __GFP_NOWARN) with port->lock held.
5826	 */
5827	printk_deferred_enter();
5828
5829#ifdef CONFIG_NUMA
5830	memset(node_load, 0, sizeof(node_load));
5831#endif
5832
5833	/*
5834	 * This node is hotadded and no memory is yet present.   So just
5835	 * building zonelists is fine - no need to touch other nodes.
5836	 */
5837	if (self && !node_online(self->node_id)) {
5838		build_zonelists(self);
5839	} else {
5840		/*
5841		 * All possible nodes have pgdat preallocated
5842		 * in free_area_init
5843		 */
5844		for_each_node(nid) {
5845			pg_data_t *pgdat = NODE_DATA(nid);
5846
5847			build_zonelists(pgdat);
5848		}
5849
5850#ifdef CONFIG_HAVE_MEMORYLESS_NODES
5851		/*
5852		 * We now know the "local memory node" for each node--
5853		 * i.e., the node of the first zone in the generic zonelist.
5854		 * Set up numa_mem percpu variable for on-line cpus.  During
5855		 * boot, only the boot cpu should be on-line;  we'll init the
5856		 * secondary cpus' numa_mem as they come on-line.  During
5857		 * node/memory hotplug, we'll fixup all on-line cpus.
5858		 */
5859		for_each_online_cpu(cpu)
5860			set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
5861#endif
5862	}
5863
5864	printk_deferred_exit();
5865	write_sequnlock_irqrestore(&zonelist_update_seq, flags);
5866}
5867
5868static noinline void __init
5869build_all_zonelists_init(void)
5870{
5871	int cpu;
5872
5873	__build_all_zonelists(NULL);
5874
5875	/*
5876	 * Initialize the boot_pagesets that are going to be used
5877	 * for bootstrapping processors. The real pagesets for
5878	 * each zone will be allocated later when the per cpu
5879	 * allocator is available.
5880	 *
5881	 * boot_pagesets are used also for bootstrapping offline
5882	 * cpus if the system is already booted because the pagesets
5883	 * are needed to initialize allocators on a specific cpu too.
5884	 * F.e. the percpu allocator needs the page allocator which
5885	 * needs the percpu allocator in order to allocate its pagesets
5886	 * (a chicken-egg dilemma).
5887	 */
5888	for_each_possible_cpu(cpu)
5889		per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu));
5890
5891	mminit_verify_zonelist();
5892	cpuset_init_current_mems_allowed();
5893}
5894
5895/*
5896 * unless system_state == SYSTEM_BOOTING.
5897 *
5898 * __ref due to call of __init annotated helper build_all_zonelists_init
5899 * [protected by SYSTEM_BOOTING].
5900 */
5901void __ref build_all_zonelists(pg_data_t *pgdat)
5902{
5903	unsigned long vm_total_pages;
5904
5905	if (system_state == SYSTEM_BOOTING) {
5906		build_all_zonelists_init();
5907	} else {
5908		__build_all_zonelists(pgdat);
5909		/* cpuset refresh routine should be here */
5910	}
5911	/* Get the number of free pages beyond high watermark in all zones. */
5912	vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
5913	/*
5914	 * Disable grouping by mobility if the number of pages in the
5915	 * system is too low to allow the mechanism to work. It would be
5916	 * more accurate, but expensive to check per-zone. This check is
5917	 * made on memory-hotadd so a system can start with mobility
5918	 * disabled and enable it later
5919	 */
5920	if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
5921		page_group_by_mobility_disabled = 1;
5922	else
5923		page_group_by_mobility_disabled = 0;
5924
5925	pr_info("Built %u zonelists, mobility grouping %s.  Total pages: %ld\n",
5926		nr_online_nodes,
5927		str_off_on(page_group_by_mobility_disabled),
5928		vm_total_pages);
5929#ifdef CONFIG_NUMA
5930	pr_info("Policy zone: %s\n", zone_names[policy_zone]);
5931#endif
5932}
5933
5934static int zone_batchsize(struct zone *zone)
5935{
5936#ifdef CONFIG_MMU
5937	int batch;
5938
5939	/*
5940	 * The number of pages to batch allocate is either ~0.025%
5941	 * of the zone or 256KB, whichever is smaller. The batch
5942	 * size is striking a balance between allocation latency
5943	 * and zone lock contention.
5944	 */
5945	batch = min(zone_managed_pages(zone) >> 12, SZ_256K / PAGE_SIZE);
5946	if (batch <= 1)
5947		return 1;
5948
5949	/*
5950	 * Clamp the batch to a 2^n - 1 value. Having a power
5951	 * of 2 value was found to be more likely to have
5952	 * suboptimal cache aliasing properties in some cases.
5953	 *
5954	 * For example if 2 tasks are alternately allocating
5955	 * batches of pages, one task can end up with a lot
5956	 * of pages of one half of the possible page colors
5957	 * and the other with pages of the other colors.
5958	 */
5959	batch = rounddown_pow_of_two(batch + batch/2) - 1;
5960
5961	return batch;
5962
5963#else
5964	/* The deferral and batching of frees should be suppressed under NOMMU
5965	 * conditions.
5966	 *
5967	 * The problem is that NOMMU needs to be able to allocate large chunks
5968	 * of contiguous memory as there's no hardware page translation to
5969	 * assemble apparent contiguous memory from discontiguous pages.
5970	 *
5971	 * Queueing large contiguous runs of pages for batching, however,
5972	 * causes the pages to actually be freed in smaller chunks.  As there
5973	 * can be a significant delay between the individual batches being
5974	 * recycled, this leads to the once large chunks of space being
5975	 * fragmented and becoming unavailable for high-order allocations.
5976	 */
5977	return 1;
5978#endif
5979}
5980
5981static int percpu_pagelist_high_fraction;
5982static int zone_highsize(struct zone *zone, int batch, int cpu_online,
5983			 int high_fraction)
5984{
5985#ifdef CONFIG_MMU
5986	int high;
5987	int nr_split_cpus;
5988	unsigned long total_pages;
5989
5990	if (!high_fraction) {
5991		/*
5992		 * By default, the high value of the pcp is based on the zone
5993		 * low watermark so that if they are full then background
5994		 * reclaim will not be started prematurely.
5995		 */
5996		total_pages = low_wmark_pages(zone);
5997	} else {
5998		/*
5999		 * If percpu_pagelist_high_fraction is configured, the high
6000		 * value is based on a fraction of the managed pages in the
6001		 * zone.
6002		 */
6003		total_pages = zone_managed_pages(zone) / high_fraction;
6004	}
6005
6006	/*
6007	 * Split the high value across all online CPUs local to the zone. Note
6008	 * that early in boot that CPUs may not be online yet and that during
6009	 * CPU hotplug that the cpumask is not yet updated when a CPU is being
6010	 * onlined. For memory nodes that have no CPUs, split the high value
6011	 * across all online CPUs to mitigate the risk that reclaim is triggered
6012	 * prematurely due to pages stored on pcp lists.
6013	 */
6014	nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online;
6015	if (!nr_split_cpus)
6016		nr_split_cpus = num_online_cpus();
6017	high = total_pages / nr_split_cpus;
6018
6019	/*
6020	 * Ensure high is at least batch*4. The multiple is based on the
6021	 * historical relationship between high and batch.
6022	 */
6023	high = max(high, batch << 2);
6024
6025	return high;
6026#else
6027	return 0;
6028#endif
6029}
6030
6031/*
6032 * pcp->high and pcp->batch values are related and generally batch is lower
6033 * than high. They are also related to pcp->count such that count is lower
6034 * than high, and as soon as it reaches high, the pcplist is flushed.
6035 *
6036 * However, guaranteeing these relations at all times would require e.g. write
6037 * barriers here but also careful usage of read barriers at the read side, and
6038 * thus be prone to error and bad for performance. Thus the update only prevents
6039 * store tearing. Any new users of pcp->batch, pcp->high_min and pcp->high_max
6040 * should ensure they can cope with those fields changing asynchronously, and
6041 * fully trust only the pcp->count field on the local CPU with interrupts
6042 * disabled.
6043 *
6044 * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
6045 * outside of boot time (or some other assurance that no concurrent updaters
6046 * exist).
6047 */
6048static void pageset_update(struct per_cpu_pages *pcp, unsigned long high_min,
6049			   unsigned long high_max, unsigned long batch)
6050{
6051	WRITE_ONCE(pcp->batch, batch);
6052	WRITE_ONCE(pcp->high_min, high_min);
6053	WRITE_ONCE(pcp->high_max, high_max);
6054}
6055
6056static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats)
6057{
6058	int pindex;
6059
6060	memset(pcp, 0, sizeof(*pcp));
6061	memset(pzstats, 0, sizeof(*pzstats));
6062
6063	spin_lock_init(&pcp->lock);
6064	for (pindex = 0; pindex < NR_PCP_LISTS; pindex++)
6065		INIT_LIST_HEAD(&pcp->lists[pindex]);
6066
6067	/*
6068	 * Set batch and high values safe for a boot pageset. A true percpu
6069	 * pageset's initialization will update them subsequently. Here we don't
6070	 * need to be as careful as pageset_update() as nobody can access the
6071	 * pageset yet.
6072	 */
6073	pcp->high_min = BOOT_PAGESET_HIGH;
6074	pcp->high_max = BOOT_PAGESET_HIGH;
6075	pcp->batch = BOOT_PAGESET_BATCH;
6076}
6077
6078static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high_min,
6079					      unsigned long high_max, unsigned long batch)
6080{
6081	struct per_cpu_pages *pcp;
6082	int cpu;
6083
6084	for_each_possible_cpu(cpu) {
6085		pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
6086		pageset_update(pcp, high_min, high_max, batch);
6087	}
6088}
6089
6090/*
6091 * Calculate and set new high and batch values for all per-cpu pagesets of a
6092 * zone based on the zone's size.
6093 */
6094static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online)
6095{
6096	int new_high_min, new_high_max, new_batch;
6097
6098	new_batch = zone_batchsize(zone);
6099	if (percpu_pagelist_high_fraction) {
6100		new_high_min = zone_highsize(zone, new_batch, cpu_online,
6101					     percpu_pagelist_high_fraction);
6102		/*
6103		 * PCP high is tuned manually, disable auto-tuning via
6104		 * setting high_min and high_max to the manual value.
6105		 */
6106		new_high_max = new_high_min;
6107	} else {
6108		new_high_min = zone_highsize(zone, new_batch, cpu_online, 0);
6109		new_high_max = zone_highsize(zone, new_batch, cpu_online,
6110					     MIN_PERCPU_PAGELIST_HIGH_FRACTION);
6111	}
6112
6113	if (zone->pageset_high_min == new_high_min &&
6114	    zone->pageset_high_max == new_high_max &&
6115	    zone->pageset_batch == new_batch)
6116		return;
6117
6118	zone->pageset_high_min = new_high_min;
6119	zone->pageset_high_max = new_high_max;
6120	zone->pageset_batch = new_batch;
6121
6122	__zone_set_pageset_high_and_batch(zone, new_high_min, new_high_max,
6123					  new_batch);
6124}
6125
6126void __meminit setup_zone_pageset(struct zone *zone)
6127{
6128	int cpu;
6129
6130	/* Size may be 0 on !SMP && !NUMA */
6131	if (sizeof(struct per_cpu_zonestat) > 0)
6132		zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat);
6133
6134	zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages);
6135	for_each_possible_cpu(cpu) {
6136		struct per_cpu_pages *pcp;
6137		struct per_cpu_zonestat *pzstats;
6138
6139		pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
6140		pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
6141		per_cpu_pages_init(pcp, pzstats);
6142	}
6143
6144	zone_set_pageset_high_and_batch(zone, 0);
6145}
6146
6147/*
6148 * The zone indicated has a new number of managed_pages; batch sizes and percpu
6149 * page high values need to be recalculated.
6150 */
6151static void zone_pcp_update(struct zone *zone, int cpu_online)
6152{
6153	mutex_lock(&pcp_batch_high_lock);
6154	zone_set_pageset_high_and_batch(zone, cpu_online);
6155	mutex_unlock(&pcp_batch_high_lock);
6156}
6157
6158static void zone_pcp_update_cacheinfo(struct zone *zone, unsigned int cpu)
6159{
6160	struct per_cpu_pages *pcp;
6161	struct cpu_cacheinfo *cci;
6162	unsigned long UP_flags;
6163
6164	pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
6165	cci = get_cpu_cacheinfo(cpu);
6166	/*
6167	 * If data cache slice of CPU is large enough, "pcp->batch"
6168	 * pages can be preserved in PCP before draining PCP for
6169	 * consecutive high-order pages freeing without allocation.
6170	 * This can reduce zone lock contention without hurting
6171	 * cache-hot pages sharing.
6172	 */
6173	pcp_spin_lock_maybe_irqsave(pcp, UP_flags);
6174	if ((cci->per_cpu_data_slice_size >> PAGE_SHIFT) > 3 * pcp->batch)
6175		pcp->flags |= PCPF_FREE_HIGH_BATCH;
6176	else
6177		pcp->flags &= ~PCPF_FREE_HIGH_BATCH;
6178	pcp_spin_unlock_maybe_irqrestore(pcp, UP_flags);
6179}
6180
6181void setup_pcp_cacheinfo(unsigned int cpu)
6182{
6183	struct zone *zone;
6184
6185	for_each_populated_zone(zone)
6186		zone_pcp_update_cacheinfo(zone, cpu);
6187}
6188
6189/*
6190 * Allocate per cpu pagesets and initialize them.
6191 * Before this call only boot pagesets were available.
6192 */
6193void __init setup_per_cpu_pageset(void)
6194{
6195	struct pglist_data *pgdat;
6196	struct zone *zone;
6197	int __maybe_unused cpu;
6198
6199	for_each_populated_zone(zone)
6200		setup_zone_pageset(zone);
6201
6202#ifdef CONFIG_NUMA
6203	/*
6204	 * Unpopulated zones continue using the boot pagesets.
6205	 * The numa stats for these pagesets need to be reset.
6206	 * Otherwise, they will end up skewing the stats of
6207	 * the nodes these zones are associated with.
6208	 */
6209	for_each_possible_cpu(cpu) {
6210		struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu);
6211		memset(pzstats->vm_numa_event, 0,
6212		       sizeof(pzstats->vm_numa_event));
6213	}
6214#endif
6215
6216	for_each_online_pgdat(pgdat)
6217		pgdat->per_cpu_nodestats =
6218			alloc_percpu(struct per_cpu_nodestat);
6219}
6220
6221__meminit void zone_pcp_init(struct zone *zone)
6222{
6223	/*
6224	 * per cpu subsystem is not up at this point. The following code
6225	 * relies on the ability of the linker to provide the
6226	 * offset of a (static) per cpu variable into the per cpu area.
6227	 */
6228	zone->per_cpu_pageset = &boot_pageset;
6229	zone->per_cpu_zonestats = &boot_zonestats;
6230	zone->pageset_high_min = BOOT_PAGESET_HIGH;
6231	zone->pageset_high_max = BOOT_PAGESET_HIGH;
6232	zone->pageset_batch = BOOT_PAGESET_BATCH;
6233
6234	if (populated_zone(zone))
6235		pr_debug("  %s zone: %lu pages, LIFO batch:%u\n", zone->name,
6236			 zone->present_pages, zone_batchsize(zone));
6237}
6238
6239static void setup_per_zone_lowmem_reserve(void);
6240
6241void adjust_managed_page_count(struct page *page, long count)
6242{
6243	atomic_long_add(count, &page_zone(page)->managed_pages);
6244	totalram_pages_add(count);
6245	setup_per_zone_lowmem_reserve();
6246}
6247EXPORT_SYMBOL(adjust_managed_page_count);
6248
6249unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
6250{
6251	void *pos;
6252	unsigned long pages = 0;
6253
6254	start = (void *)PAGE_ALIGN((unsigned long)start);
6255	end = (void *)((unsigned long)end & PAGE_MASK);
6256	for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
6257		struct page *page = virt_to_page(pos);
6258		void *direct_map_addr;
6259
6260		/*
6261		 * 'direct_map_addr' might be different from 'pos'
6262		 * because some architectures' virt_to_page()
6263		 * work with aliases.  Getting the direct map
6264		 * address ensures that we get a _writeable_
6265		 * alias for the memset().
6266		 */
6267		direct_map_addr = page_address(page);
6268		/*
6269		 * Perform a kasan-unchecked memset() since this memory
6270		 * has not been initialized.
6271		 */
6272		direct_map_addr = kasan_reset_tag(direct_map_addr);
6273		if ((unsigned int)poison <= 0xFF)
6274			memset(direct_map_addr, poison, PAGE_SIZE);
6275
6276		free_reserved_page(page);
6277	}
6278
6279	if (pages && s)
6280		pr_info("Freeing %s memory: %ldK\n", s, K(pages));
6281
6282	return pages;
6283}
6284
6285void free_reserved_page(struct page *page)
6286{
6287	clear_page_tag_ref(page);
6288	ClearPageReserved(page);
6289	init_page_count(page);
6290	__free_page(page);
6291	adjust_managed_page_count(page, 1);
6292}
6293EXPORT_SYMBOL(free_reserved_page);
6294
6295static int page_alloc_cpu_dead(unsigned int cpu)
6296{
6297	struct zone *zone;
6298
6299	lru_add_drain_cpu(cpu);
6300	mlock_drain_remote(cpu);
6301	drain_pages(cpu);
6302
6303	/*
6304	 * Spill the event counters of the dead processor
6305	 * into the current processors event counters.
6306	 * This artificially elevates the count of the current
6307	 * processor.
6308	 */
6309	vm_events_fold_cpu(cpu);
6310
6311	/*
6312	 * Zero the differential counters of the dead processor
6313	 * so that the vm statistics are consistent.
6314	 *
6315	 * This is only okay since the processor is dead and cannot
6316	 * race with what we are doing.
6317	 */
6318	cpu_vm_stats_fold(cpu);
6319
6320	for_each_populated_zone(zone)
6321		zone_pcp_update(zone, 0);
6322
6323	return 0;
6324}
6325
6326static int page_alloc_cpu_online(unsigned int cpu)
6327{
6328	struct zone *zone;
6329
6330	for_each_populated_zone(zone)
6331		zone_pcp_update(zone, 1);
6332	return 0;
6333}
6334
6335void __init page_alloc_init_cpuhp(void)
6336{
6337	int ret;
6338
6339	ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC,
6340					"mm/page_alloc:pcp",
6341					page_alloc_cpu_online,
6342					page_alloc_cpu_dead);
6343	WARN_ON(ret < 0);
6344}
6345
6346/*
6347 * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
6348 *	or min_free_kbytes changes.
6349 */
6350static void calculate_totalreserve_pages(void)
6351{
6352	struct pglist_data *pgdat;
6353	unsigned long reserve_pages = 0;
6354	enum zone_type i, j;
6355
6356	for_each_online_pgdat(pgdat) {
6357
6358		pgdat->totalreserve_pages = 0;
6359
6360		for (i = 0; i < MAX_NR_ZONES; i++) {
6361			struct zone *zone = pgdat->node_zones + i;
6362			long max = 0;
6363			unsigned long managed_pages = zone_managed_pages(zone);
6364
6365			/*
6366			 * lowmem_reserve[j] is monotonically non-decreasing
6367			 * in j for a given zone (see
6368			 * setup_per_zone_lowmem_reserve()). The maximum
6369			 * valid reserve lives at the highest index with a
6370			 * non-zero value, so scan backwards and stop at the
6371			 * first hit.
6372			 */
6373			for (j = MAX_NR_ZONES - 1; j > i; j--) {
6374				if (!zone->lowmem_reserve[j])
6375					continue;
6376
6377				max = zone->lowmem_reserve[j];
6378				break;
6379			}
6380			/* we treat the high watermark as reserved pages. */
6381			max += high_wmark_pages(zone);
6382
6383			max = min_t(unsigned long, max, managed_pages);
6384
6385			pgdat->totalreserve_pages += max;
6386
6387			reserve_pages += max;
6388		}
6389	}
6390	totalreserve_pages = reserve_pages;
6391	trace_mm_calculate_totalreserve_pages(totalreserve_pages);
6392}
6393
6394/*
6395 * setup_per_zone_lowmem_reserve - called whenever
6396 *	sysctl_lowmem_reserve_ratio changes.  Ensures that each zone
6397 *	has a correct pages reserved value, so an adequate number of
6398 *	pages are left in the zone after a successful __alloc_pages().
6399 */
6400static void setup_per_zone_lowmem_reserve(void)
6401{
6402	struct pglist_data *pgdat;
6403	enum zone_type i, j;
6404	/*
6405	 * For a given zone node_zones[i], lowmem_reserve[j] (j > i)
6406	 * represents how many pages in zone i must effectively be kept
6407	 * in reserve when deciding whether an allocation class that is
6408	 * allowed to allocate from zones up to j may fall back into
6409	 * zone i.
6410	 *
6411	 * As j increases, the allocation class can use a strictly larger
6412	 * set of fallback zones and therefore must not be allowed to
6413	 * deplete low zones more aggressively than a less flexible one.
6414	 * As a result, lowmem_reserve[j] is required to be monotonically
6415	 * non-decreasing in j for each zone i. Callers such as
6416	 * calculate_totalreserve_pages() rely on this monotonicity when
6417	 * selecting the maximum reserve entry.
6418	 */
6419	for_each_online_pgdat(pgdat) {
6420		for (i = 0; i < MAX_NR_ZONES - 1; i++) {
6421			struct zone *zone = &pgdat->node_zones[i];
6422			int ratio = sysctl_lowmem_reserve_ratio[i];
6423			bool clear = !ratio || !zone_managed_pages(zone);
6424			unsigned long managed_pages = 0;
6425
6426			for (j = i + 1; j < MAX_NR_ZONES; j++) {
6427				struct zone *upper_zone = &pgdat->node_zones[j];
6428
6429				managed_pages += zone_managed_pages(upper_zone);
6430
6431				if (clear)
6432					zone->lowmem_reserve[j] = 0;
6433				else
6434					zone->lowmem_reserve[j] = managed_pages / ratio;
6435				trace_mm_setup_per_zone_lowmem_reserve(zone, upper_zone,
6436								       zone->lowmem_reserve[j]);
6437			}
6438		}
6439	}
6440
6441	/* update totalreserve_pages */
6442	calculate_totalreserve_pages();
6443}
6444
6445static void __setup_per_zone_wmarks(void)
6446{
6447	unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
6448	unsigned long lowmem_pages = 0;
6449	struct zone *zone;
6450	unsigned long flags;
6451
6452	/* Calculate total number of !ZONE_HIGHMEM and !ZONE_MOVABLE pages */
6453	for_each_zone(zone) {
6454		if (!is_highmem(zone) && zone_idx(zone) != ZONE_MOVABLE)
6455			lowmem_pages += zone_managed_pages(zone);
6456	}
6457
6458	for_each_zone(zone) {
6459		u64 tmp;
6460
6461		spin_lock_irqsave(&zone->lock, flags);
6462		tmp = (u64)pages_min * zone_managed_pages(zone);
6463		tmp = div64_ul(tmp, lowmem_pages);
6464		if (is_highmem(zone) || zone_idx(zone) == ZONE_MOVABLE) {
6465			/*
6466			 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
6467			 * need highmem and movable zones pages, so cap pages_min
6468			 * to a small  value here.
6469			 *
6470			 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
6471			 * deltas control async page reclaim, and so should
6472			 * not be capped for highmem and movable zones.
6473			 */
6474			unsigned long min_pages;
6475
6476			min_pages = zone_managed_pages(zone) / 1024;
6477			min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
6478			zone->_watermark[WMARK_MIN] = min_pages;
6479		} else {
6480			/*
6481			 * If it's a lowmem zone, reserve a number of pages
6482			 * proportionate to the zone's size.
6483			 */
6484			zone->_watermark[WMARK_MIN] = tmp;
6485		}
6486
6487		/*
6488		 * Set the kswapd watermarks distance according to the
6489		 * scale factor in proportion to available memory, but
6490		 * ensure a minimum size on small systems.
6491		 */
6492		tmp = max_t(u64, tmp >> 2,
6493			    mult_frac(zone_managed_pages(zone),
6494				      watermark_scale_factor, 10000));
6495
6496		zone->watermark_boost = 0;
6497		zone->_watermark[WMARK_LOW]  = min_wmark_pages(zone) + tmp;
6498		zone->_watermark[WMARK_HIGH] = low_wmark_pages(zone) + tmp;
6499		zone->_watermark[WMARK_PROMO] = high_wmark_pages(zone) + tmp;
6500		trace_mm_setup_per_zone_wmarks(zone);
6501
6502		spin_unlock_irqrestore(&zone->lock, flags);
6503	}
6504
6505	/* update totalreserve_pages */
6506	calculate_totalreserve_pages();
6507}
6508
6509/**
6510 * setup_per_zone_wmarks - called when min_free_kbytes changes
6511 * or when memory is hot-{added|removed}
6512 *
6513 * Ensures that the watermark[min,low,high] values for each zone are set
6514 * correctly with respect to min_free_kbytes.
6515 */
6516void setup_per_zone_wmarks(void)
6517{
6518	struct zone *zone;
6519	static DEFINE_SPINLOCK(lock);
6520
6521	spin_lock(&lock);
6522	__setup_per_zone_wmarks();
6523	spin_unlock(&lock);
6524
6525	/*
6526	 * The watermark size have changed so update the pcpu batch
6527	 * and high limits or the limits may be inappropriate.
6528	 */
6529	for_each_zone(zone)
6530		zone_pcp_update(zone, 0);
6531}
6532
6533/*
6534 * Initialise min_free_kbytes.
6535 *
6536 * For small machines we want it small (128k min).  For large machines
6537 * we want it large (256MB max).  But it is not linear, because network
6538 * bandwidth does not increase linearly with machine size.  We use
6539 *
6540 *	min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
6541 *	min_free_kbytes = sqrt(lowmem_kbytes * 16)
6542 *
6543 * which yields
6544 *
6545 * 16MB:	512k
6546 * 32MB:	724k
6547 * 64MB:	1024k
6548 * 128MB:	1448k
6549 * 256MB:	2048k
6550 * 512MB:	2896k
6551 * 1024MB:	4096k
6552 * 2048MB:	5792k
6553 * 4096MB:	8192k
6554 * 8192MB:	11584k
6555 * 16384MB:	16384k
6556 */
6557void calculate_min_free_kbytes(void)
6558{
6559	unsigned long lowmem_kbytes;
6560	int new_min_free_kbytes;
6561
6562	lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
6563	new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
6564
6565	if (new_min_free_kbytes > user_min_free_kbytes)
6566		min_free_kbytes = clamp(new_min_free_kbytes, 128, 262144);
6567	else
6568		pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
6569				new_min_free_kbytes, user_min_free_kbytes);
6570
6571}
6572
6573int __meminit init_per_zone_wmark_min(void)
6574{
6575	calculate_min_free_kbytes();
6576	setup_per_zone_wmarks();
6577	refresh_zone_stat_thresholds();
6578	setup_per_zone_lowmem_reserve();
6579
6580#ifdef CONFIG_NUMA
6581	setup_min_unmapped_ratio();
6582	setup_min_slab_ratio();
6583#endif
6584
6585	khugepaged_min_free_kbytes_update();
6586
6587	return 0;
6588}
6589postcore_initcall(init_per_zone_wmark_min)
6590
6591/*
6592 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
6593 *	that we can call two helper functions whenever min_free_kbytes
6594 *	changes.
6595 */
6596static int min_free_kbytes_sysctl_handler(const struct ctl_table *table, int write,
6597		void *buffer, size_t *length, loff_t *ppos)
6598{
6599	int rc;
6600
6601	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6602	if (rc)
6603		return rc;
6604
6605	if (write) {
6606		user_min_free_kbytes = min_free_kbytes;
6607		setup_per_zone_wmarks();
6608	}
6609	return 0;
6610}
6611
6612static int watermark_scale_factor_sysctl_handler(const struct ctl_table *table, int write,
6613		void *buffer, size_t *length, loff_t *ppos)
6614{
6615	int rc;
6616
6617	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6618	if (rc)
6619		return rc;
6620
6621	if (write)
6622		setup_per_zone_wmarks();
6623
6624	return 0;
6625}
6626
6627#ifdef CONFIG_NUMA
6628static void setup_min_unmapped_ratio(void)
6629{
6630	pg_data_t *pgdat;
6631	struct zone *zone;
6632
6633	for_each_online_pgdat(pgdat)
6634		pgdat->min_unmapped_pages = 0;
6635
6636	for_each_zone(zone)
6637		zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
6638						         sysctl_min_unmapped_ratio) / 100;
6639}
6640
6641
6642static int sysctl_min_unmapped_ratio_sysctl_handler(const struct ctl_table *table, int write,
6643		void *buffer, size_t *length, loff_t *ppos)
6644{
6645	int rc;
6646
6647	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6648	if (rc)
6649		return rc;
6650
6651	setup_min_unmapped_ratio();
6652
6653	return 0;
6654}
6655
6656static void setup_min_slab_ratio(void)
6657{
6658	pg_data_t *pgdat;
6659	struct zone *zone;
6660
6661	for_each_online_pgdat(pgdat)
6662		pgdat->min_slab_pages = 0;
6663
6664	for_each_zone(zone)
6665		zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
6666						     sysctl_min_slab_ratio) / 100;
6667}
6668
6669static int sysctl_min_slab_ratio_sysctl_handler(const struct ctl_table *table, int write,
6670		void *buffer, size_t *length, loff_t *ppos)
6671{
6672	int rc;
6673
6674	rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
6675	if (rc)
6676		return rc;
6677
6678	setup_min_slab_ratio();
6679
6680	return 0;
6681}
6682#endif
6683
6684/*
6685 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
6686 *	proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
6687 *	whenever sysctl_lowmem_reserve_ratio changes.
6688 *
6689 * The reserve ratio obviously has absolutely no relation with the
6690 * minimum watermarks. The lowmem reserve ratio can only make sense
6691 * if in function of the boot time zone sizes.
6692 */
6693static int lowmem_reserve_ratio_sysctl_handler(const struct ctl_table *table,
6694		int write, void *buffer, size_t *length, loff_t *ppos)
6695{
6696	int i;
6697
6698	proc_dointvec_minmax(table, write, buffer, length, ppos);
6699
6700	for (i = 0; i < MAX_NR_ZONES; i++) {
6701		if (sysctl_lowmem_reserve_ratio[i] < 1)
6702			sysctl_lowmem_reserve_ratio[i] = 0;
6703	}
6704
6705	setup_per_zone_lowmem_reserve();
6706	return 0;
6707}
6708
6709/*
6710 * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each
6711 * cpu. It is the fraction of total pages in each zone that a hot per cpu
6712 * pagelist can have before it gets flushed back to buddy allocator.
6713 */
6714static int percpu_pagelist_high_fraction_sysctl_handler(const struct ctl_table *table,
6715		int write, void *buffer, size_t *length, loff_t *ppos)
6716{
6717	struct zone *zone;
6718	int old_percpu_pagelist_high_fraction;
6719	int ret;
6720
6721	/*
6722	 * Avoid using pcp_batch_high_lock for reads as the value is read
6723	 * atomically and a race with offlining is harmless.
6724	 */
6725
6726	if (!write)
6727		return proc_dointvec_minmax(table, write, buffer, length, ppos);
6728
6729	mutex_lock(&pcp_batch_high_lock);
6730	old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction;
6731
6732	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
6733	if (ret < 0)
6734		goto out;
6735
6736	/* Sanity checking to avoid pcp imbalance */
6737	if (percpu_pagelist_high_fraction &&
6738	    percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) {
6739		percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction;
6740		ret = -EINVAL;
6741		goto out;
6742	}
6743
6744	/* No change? */
6745	if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction)
6746		goto out;
6747
6748	for_each_populated_zone(zone)
6749		zone_set_pageset_high_and_batch(zone, 0);
6750out:
6751	mutex_unlock(&pcp_batch_high_lock);
6752	return ret;
6753}
6754
6755static const struct ctl_table page_alloc_sysctl_table[] = {
6756	{
6757		.procname	= "min_free_kbytes",
6758		.data		= &min_free_kbytes,
6759		.maxlen		= sizeof(min_free_kbytes),
6760		.mode		= 0644,
6761		.proc_handler	= min_free_kbytes_sysctl_handler,
6762		.extra1		= SYSCTL_ZERO,
6763	},
6764	{
6765		.procname	= "watermark_boost_factor",
6766		.data		= &watermark_boost_factor,
6767		.maxlen		= sizeof(watermark_boost_factor),
6768		.mode		= 0644,
6769		.proc_handler	= proc_dointvec_minmax,
6770		.extra1		= SYSCTL_ZERO,
6771	},
6772	{
6773		.procname	= "watermark_scale_factor",
6774		.data		= &watermark_scale_factor,
6775		.maxlen		= sizeof(watermark_scale_factor),
6776		.mode		= 0644,
6777		.proc_handler	= watermark_scale_factor_sysctl_handler,
6778		.extra1		= SYSCTL_ONE,
6779		.extra2		= SYSCTL_THREE_THOUSAND,
6780	},
6781	{
6782		.procname	= "defrag_mode",
6783		.data		= &defrag_mode,
6784		.maxlen		= sizeof(defrag_mode),
6785		.mode		= 0644,
6786		.proc_handler	= proc_dointvec_minmax,
6787		.extra1		= SYSCTL_ZERO,
6788		.extra2		= SYSCTL_ONE,
6789	},
6790	{
6791		.procname	= "percpu_pagelist_high_fraction",
6792		.data		= &percpu_pagelist_high_fraction,
6793		.maxlen		= sizeof(percpu_pagelist_high_fraction),
6794		.mode		= 0644,
6795		.proc_handler	= percpu_pagelist_high_fraction_sysctl_handler,
6796		.extra1		= SYSCTL_ZERO,
6797	},
6798	{
6799		.procname	= "lowmem_reserve_ratio",
6800		.data		= &sysctl_lowmem_reserve_ratio,
6801		.maxlen		= sizeof(sysctl_lowmem_reserve_ratio),
6802		.mode		= 0644,
6803		.proc_handler	= lowmem_reserve_ratio_sysctl_handler,
6804	},
6805#ifdef CONFIG_NUMA
6806	{
6807		.procname	= "numa_zonelist_order",
6808		.data		= &numa_zonelist_order,
6809		.maxlen		= NUMA_ZONELIST_ORDER_LEN,
6810		.mode		= 0644,
6811		.proc_handler	= numa_zonelist_order_handler,
6812	},
6813	{
6814		.procname	= "min_unmapped_ratio",
6815		.data		= &sysctl_min_unmapped_ratio,
6816		.maxlen		= sizeof(sysctl_min_unmapped_ratio),
6817		.mode		= 0644,
6818		.proc_handler	= sysctl_min_unmapped_ratio_sysctl_handler,
6819		.extra1		= SYSCTL_ZERO,
6820		.extra2		= SYSCTL_ONE_HUNDRED,
6821	},
6822	{
6823		.procname	= "min_slab_ratio",
6824		.data		= &sysctl_min_slab_ratio,
6825		.maxlen		= sizeof(sysctl_min_slab_ratio),
6826		.mode		= 0644,
6827		.proc_handler	= sysctl_min_slab_ratio_sysctl_handler,
6828		.extra1		= SYSCTL_ZERO,
6829		.extra2		= SYSCTL_ONE_HUNDRED,
6830	},
6831#endif
6832};
6833
6834void __init page_alloc_sysctl_init(void)
6835{
6836	register_sysctl_init("vm", page_alloc_sysctl_table);
6837}
6838
6839#ifdef CONFIG_CONTIG_ALLOC
6840/* Usage: See admin-guide/dynamic-debug-howto.rst */
6841static void alloc_contig_dump_pages(struct list_head *page_list)
6842{
6843	DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure");
6844
6845	if (DYNAMIC_DEBUG_BRANCH(descriptor)) {
6846		struct page *page;
6847
6848		dump_stack();
6849		list_for_each_entry(page, page_list, lru)
6850			dump_page(page, "migration failure");
6851	}
6852}
6853
6854/* [start, end) must belong to a single zone. */
6855static int __alloc_contig_migrate_range(struct compact_control *cc,
6856					unsigned long start, unsigned long end)
6857{
6858	/* This function is based on compact_zone() from compaction.c. */
6859	unsigned int nr_reclaimed;
6860	unsigned long pfn = start;
6861	unsigned int tries = 0;
6862	int ret = 0;
6863	struct migration_target_control mtc = {
6864		.nid = zone_to_nid(cc->zone),
6865		.gfp_mask = cc->gfp_mask,
6866		.reason = MR_CONTIG_RANGE,
6867	};
6868
6869	lru_cache_disable();
6870
6871	while (pfn < end || !list_empty(&cc->migratepages)) {
6872		if (fatal_signal_pending(current)) {
6873			ret = -EINTR;
6874			break;
6875		}
6876
6877		if (list_empty(&cc->migratepages)) {
6878			cc->nr_migratepages = 0;
6879			ret = isolate_migratepages_range(cc, pfn, end);
6880			if (ret && ret != -EAGAIN)
6881				break;
6882			pfn = cc->migrate_pfn;
6883			tries = 0;
6884		} else if (++tries == 5) {
6885			ret = -EBUSY;
6886			break;
6887		}
6888
6889		nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
6890							&cc->migratepages);
6891		cc->nr_migratepages -= nr_reclaimed;
6892
6893		ret = migrate_pages(&cc->migratepages, alloc_migration_target,
6894			NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL);
6895
6896		/*
6897		 * On -ENOMEM, migrate_pages() bails out right away. It is pointless
6898		 * to retry again over this error, so do the same here.
6899		 */
6900		if (ret == -ENOMEM)
6901			break;
6902	}
6903
6904	lru_cache_enable();
6905	if (ret < 0) {
6906		if (!(cc->gfp_mask & __GFP_NOWARN) && ret == -EBUSY)
6907			alloc_contig_dump_pages(&cc->migratepages);
6908		putback_movable_pages(&cc->migratepages);
6909	}
6910
6911	return (ret < 0) ? ret : 0;
6912}
6913
6914static void split_free_pages(struct list_head *list, gfp_t gfp_mask)
6915{
6916	int order;
6917
6918	for (order = 0; order < NR_PAGE_ORDERS; order++) {
6919		struct page *page, *next;
6920		int nr_pages = 1 << order;
6921
6922		list_for_each_entry_safe(page, next, &list[order], lru) {
6923			int i;
6924
6925			post_alloc_hook(page, order, gfp_mask);
6926			set_page_refcounted(page);
6927			if (!order)
6928				continue;
6929
6930			split_page(page, order);
6931
6932			/* Add all subpages to the order-0 head, in sequence. */
6933			list_del(&page->lru);
6934			for (i = 0; i < nr_pages; i++)
6935				list_add_tail(&page[i].lru, &list[0]);
6936		}
6937	}
6938}
6939
6940static int __alloc_contig_verify_gfp_mask(gfp_t gfp_mask, gfp_t *gfp_cc_mask)
6941{
6942	const gfp_t reclaim_mask = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
6943	const gfp_t action_mask = __GFP_COMP | __GFP_RETRY_MAYFAIL | __GFP_NOWARN |
6944				  __GFP_ZERO | __GFP_ZEROTAGS | __GFP_SKIP_ZERO |
6945				  __GFP_SKIP_KASAN;
6946	const gfp_t cc_action_mask = __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
6947
6948	/*
6949	 * We are given the range to allocate; node, mobility and placement
6950	 * hints are irrelevant at this point. We'll simply ignore them.
6951	 */
6952	gfp_mask &= ~(GFP_ZONEMASK | __GFP_RECLAIMABLE | __GFP_WRITE |
6953		      __GFP_HARDWALL | __GFP_THISNODE | __GFP_MOVABLE);
6954
6955	/*
6956	 * We only support most reclaim flags (but not NOFAIL/NORETRY), and
6957	 * selected action flags.
6958	 */
6959	if (gfp_mask & ~(reclaim_mask | action_mask))
6960		return -EINVAL;
6961
6962	/*
6963	 * Flags to control page compaction/migration/reclaim, to free up our
6964	 * page range. Migratable pages are movable, __GFP_MOVABLE is implied
6965	 * for them.
6966	 *
6967	 * Traditionally we always had __GFP_RETRY_MAYFAIL set, keep doing that
6968	 * to not degrade callers.
6969	 */
6970	*gfp_cc_mask = (gfp_mask & (reclaim_mask | cc_action_mask)) |
6971			__GFP_MOVABLE | __GFP_RETRY_MAYFAIL;
6972	return 0;
6973}
6974
6975/**
6976 * alloc_contig_range() -- tries to allocate given range of pages
6977 * @start:	start PFN to allocate
6978 * @end:	one-past-the-last PFN to allocate
6979 * @alloc_flags:	allocation information
6980 * @gfp_mask:	GFP mask. Node/zone/placement hints are ignored; only some
6981 *		action and reclaim modifiers are supported. Reclaim modifiers
6982 *		control allocation behavior during compaction/migration/reclaim.
6983 *
6984 * The PFN range does not have to be pageblock aligned. The PFN range must
6985 * belong to a single zone.
6986 *
6987 * The first thing this routine does is attempt to MIGRATE_ISOLATE all
6988 * pageblocks in the range.  Once isolated, the pageblocks should not
6989 * be modified by others.
6990 *
6991 * Return: zero on success or negative error code.  On success all
6992 * pages which PFN is in [start, end) are allocated for the caller and
6993 * need to be freed with free_contig_range().
6994 */
6995int alloc_contig_range_noprof(unsigned long start, unsigned long end,
6996			      acr_flags_t alloc_flags, gfp_t gfp_mask)
6997{
6998	const unsigned int order = ilog2(end - start);
6999	unsigned long outer_start, outer_end;
7000	int ret = 0;
7001
7002	struct compact_control cc = {
7003		.nr_migratepages = 0,
7004		.order = -1,
7005		.zone = page_zone(pfn_to_page(start)),
7006		.mode = MIGRATE_SYNC,
7007		.ignore_skip_hint = true,
7008		.no_set_skip_hint = true,
7009		.alloc_contig = true,
7010	};
7011	INIT_LIST_HEAD(&cc.migratepages);
7012	enum pb_isolate_mode mode = (alloc_flags & ACR_FLAGS_CMA) ?
7013					    PB_ISOLATE_MODE_CMA_ALLOC :
7014					    PB_ISOLATE_MODE_OTHER;
7015
7016	/*
7017	 * In contrast to the buddy, we allow for orders here that exceed
7018	 * MAX_PAGE_ORDER, so we must manually make sure that we are not
7019	 * exceeding the maximum folio order.
7020	 */
7021	if (WARN_ON_ONCE((gfp_mask & __GFP_COMP) && order > MAX_FOLIO_ORDER))
7022		return -EINVAL;
7023
7024	gfp_mask = current_gfp_context(gfp_mask);
7025	if (__alloc_contig_verify_gfp_mask(gfp_mask, (gfp_t *)&cc.gfp_mask))
7026		return -EINVAL;
7027
7028	/*
7029	 * What we do here is we mark all pageblocks in range as
7030	 * MIGRATE_ISOLATE.  Because pageblock and max order pages may
7031	 * have different sizes, and due to the way page allocator
7032	 * work, start_isolate_page_range() has special handlings for this.
7033	 *
7034	 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
7035	 * migrate the pages from an unaligned range (ie. pages that
7036	 * we are interested in). This will put all the pages in
7037	 * range back to page allocator as MIGRATE_ISOLATE.
7038	 *
7039	 * When this is done, we take the pages in range from page
7040	 * allocator removing them from the buddy system.  This way
7041	 * page allocator will never consider using them.
7042	 *
7043	 * This lets us mark the pageblocks back as
7044	 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
7045	 * aligned range but not in the unaligned, original range are
7046	 * put back to page allocator so that buddy can use them.
7047	 */
7048
7049	ret = start_isolate_page_range(start, end, mode);
7050	if (ret)
7051		goto done;
7052
7053	drain_all_pages(cc.zone);
7054
7055	/*
7056	 * In case of -EBUSY, we'd like to know which page causes problem.
7057	 * So, just fall through. test_pages_isolated() has a tracepoint
7058	 * which will report the busy page.
7059	 *
7060	 * It is possible that busy pages could become available before
7061	 * the call to test_pages_isolated, and the range will actually be
7062	 * allocated.  So, if we fall through be sure to clear ret so that
7063	 * -EBUSY is not accidentally used or returned to caller.
7064	 */
7065	ret = __alloc_contig_migrate_range(&cc, start, end);
7066	if (ret && ret != -EBUSY)
7067		goto done;
7068
7069	/*
7070	 * When in-use hugetlb pages are migrated, they may simply be released
7071	 * back into the free hugepage pool instead of being returned to the
7072	 * buddy system.  After the migration of in-use huge pages is completed,
7073	 * we will invoke replace_free_hugepage_folios() to ensure that these
7074	 * hugepages are properly released to the buddy system.
7075	 */
7076	ret = replace_free_hugepage_folios(start, end);
7077	if (ret)
7078		goto done;
7079
7080	/*
7081	 * Pages from [start, end) are within a pageblock_nr_pages
7082	 * aligned blocks that are marked as MIGRATE_ISOLATE.  What's
7083	 * more, all pages in [start, end) are free in page allocator.
7084	 * What we are going to do is to allocate all pages from
7085	 * [start, end) (that is remove them from page allocator).
7086	 *
7087	 * The only problem is that pages at the beginning and at the
7088	 * end of interesting range may be not aligned with pages that
7089	 * page allocator holds, ie. they can be part of higher order
7090	 * pages.  Because of this, we reserve the bigger range and
7091	 * once this is done free the pages we are not interested in.
7092	 *
7093	 * We don't have to hold zone->lock here because the pages are
7094	 * isolated thus they won't get removed from buddy.
7095	 */
7096	outer_start = find_large_buddy(start);
7097
7098	/* Make sure the range is really isolated. */
7099	if (test_pages_isolated(outer_start, end, mode)) {
7100		ret = -EBUSY;
7101		goto done;
7102	}
7103
7104	/* Grab isolated pages from freelists. */
7105	outer_end = isolate_freepages_range(&cc, outer_start, end);
7106	if (!outer_end) {
7107		ret = -EBUSY;
7108		goto done;
7109	}
7110
7111	if (!(gfp_mask & __GFP_COMP)) {
7112		split_free_pages(cc.freepages, gfp_mask);
7113
7114		/* Free head and tail (if any) */
7115		if (start != outer_start)
7116			free_contig_range(outer_start, start - outer_start);
7117		if (end != outer_end)
7118			free_contig_range(end, outer_end - end);
7119	} else if (start == outer_start && end == outer_end && is_power_of_2(end - start)) {
7120		struct page *head = pfn_to_page(start);
7121
7122		check_new_pages(head, order);
7123		prep_new_page(head, order, gfp_mask, 0);
7124		set_page_refcounted(head);
7125	} else {
7126		ret = -EINVAL;
7127		WARN(true, "PFN range: requested [%lu, %lu), allocated [%lu, %lu)\n",
7128		     start, end, outer_start, outer_end);
7129	}
7130done:
7131	undo_isolate_page_range(start, end);
7132	return ret;
7133}
7134EXPORT_SYMBOL(alloc_contig_range_noprof);
7135
7136static int __alloc_contig_pages(unsigned long start_pfn,
7137				unsigned long nr_pages, gfp_t gfp_mask)
7138{
7139	unsigned long end_pfn = start_pfn + nr_pages;
7140
7141	return alloc_contig_range_noprof(start_pfn, end_pfn, ACR_FLAGS_NONE,
7142					 gfp_mask);
7143}
7144
7145static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,
7146				   unsigned long nr_pages)
7147{
7148	unsigned long i, end_pfn = start_pfn + nr_pages;
7149	struct page *page;
7150
7151	for (i = start_pfn; i < end_pfn; i++) {
7152		page = pfn_to_online_page(i);
7153		if (!page)
7154			return false;
7155
7156		if (page_zone(page) != z)
7157			return false;
7158
7159		if (PageReserved(page))
7160			return false;
7161
7162		if (PageHuge(page))
7163			return false;
7164	}
7165	return true;
7166}
7167
7168static bool zone_spans_last_pfn(const struct zone *zone,
7169				unsigned long start_pfn, unsigned long nr_pages)
7170{
7171	unsigned long last_pfn = start_pfn + nr_pages - 1;
7172
7173	return zone_spans_pfn(zone, last_pfn);
7174}
7175
7176/**
7177 * alloc_contig_pages() -- tries to find and allocate contiguous range of pages
7178 * @nr_pages:	Number of contiguous pages to allocate
7179 * @gfp_mask:	GFP mask. Node/zone/placement hints limit the search; only some
7180 *		action and reclaim modifiers are supported. Reclaim modifiers
7181 *		control allocation behavior during compaction/migration/reclaim.
7182 * @nid:	Target node
7183 * @nodemask:	Mask for other possible nodes
7184 *
7185 * This routine is a wrapper around alloc_contig_range(). It scans over zones
7186 * on an applicable zonelist to find a contiguous pfn range which can then be
7187 * tried for allocation with alloc_contig_range(). This routine is intended
7188 * for allocation requests which can not be fulfilled with the buddy allocator.
7189 *
7190 * The allocated memory is always aligned to a page boundary. If nr_pages is a
7191 * power of two, then allocated range is also guaranteed to be aligned to same
7192 * nr_pages (e.g. 1GB request would be aligned to 1GB).
7193 *
7194 * Allocated pages can be freed with free_contig_range() or by manually calling
7195 * __free_page() on each allocated page.
7196 *
7197 * Return: pointer to contiguous pages on success, or NULL if not successful.
7198 */
7199struct page *alloc_contig_pages_noprof(unsigned long nr_pages, gfp_t gfp_mask,
7200				 int nid, nodemask_t *nodemask)
7201{
7202	unsigned long ret, pfn, flags;
7203	struct zonelist *zonelist;
7204	struct zone *zone;
7205	struct zoneref *z;
7206
7207	zonelist = node_zonelist(nid, gfp_mask);
7208	for_each_zone_zonelist_nodemask(zone, z, zonelist,
7209					gfp_zone(gfp_mask), nodemask) {
7210		spin_lock_irqsave(&zone->lock, flags);
7211
7212		pfn = ALIGN(zone->zone_start_pfn, nr_pages);
7213		while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
7214			if (pfn_range_valid_contig(zone, pfn, nr_pages)) {
7215				/*
7216				 * We release the zone lock here because
7217				 * alloc_contig_range() will also lock the zone
7218				 * at some point. If there's an allocation
7219				 * spinning on this lock, it may win the race
7220				 * and cause alloc_contig_range() to fail...
7221				 */
7222				spin_unlock_irqrestore(&zone->lock, flags);
7223				ret = __alloc_contig_pages(pfn, nr_pages,
7224							gfp_mask);
7225				if (!ret)
7226					return pfn_to_page(pfn);
7227				spin_lock_irqsave(&zone->lock, flags);
7228			}
7229			pfn += nr_pages;
7230		}
7231		spin_unlock_irqrestore(&zone->lock, flags);
7232	}
7233	return NULL;
7234}
7235#endif /* CONFIG_CONTIG_ALLOC */
7236
7237void free_contig_range(unsigned long pfn, unsigned long nr_pages)
7238{
7239	unsigned long count = 0;
7240	struct folio *folio = pfn_folio(pfn);
7241
7242	if (folio_test_large(folio)) {
7243		int expected = folio_nr_pages(folio);
7244
7245		if (nr_pages == expected)
7246			folio_put(folio);
7247		else
7248			WARN(true, "PFN %lu: nr_pages %lu != expected %d\n",
7249			     pfn, nr_pages, expected);
7250		return;
7251	}
7252
7253	for (; nr_pages--; pfn++) {
7254		struct page *page = pfn_to_page(pfn);
7255
7256		count += page_count(page) != 1;
7257		__free_page(page);
7258	}
7259	WARN(count != 0, "%lu pages are still in use!\n", count);
7260}
7261EXPORT_SYMBOL(free_contig_range);
7262
7263/*
7264 * Effectively disable pcplists for the zone by setting the high limit to 0
7265 * and draining all cpus. A concurrent page freeing on another CPU that's about
7266 * to put the page on pcplist will either finish before the drain and the page
7267 * will be drained, or observe the new high limit and skip the pcplist.
7268 *
7269 * Must be paired with a call to zone_pcp_enable().
7270 */
7271void zone_pcp_disable(struct zone *zone)
7272{
7273	mutex_lock(&pcp_batch_high_lock);
7274	__zone_set_pageset_high_and_batch(zone, 0, 0, 1);
7275	__drain_all_pages(zone, true);
7276}
7277
7278void zone_pcp_enable(struct zone *zone)
7279{
7280	__zone_set_pageset_high_and_batch(zone, zone->pageset_high_min,
7281		zone->pageset_high_max, zone->pageset_batch);
7282	mutex_unlock(&pcp_batch_high_lock);
7283}
7284
7285void zone_pcp_reset(struct zone *zone)
7286{
7287	int cpu;
7288	struct per_cpu_zonestat *pzstats;
7289
7290	if (zone->per_cpu_pageset != &boot_pageset) {
7291		for_each_online_cpu(cpu) {
7292			pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
7293			drain_zonestat(zone, pzstats);
7294		}
7295		free_percpu(zone->per_cpu_pageset);
7296		zone->per_cpu_pageset = &boot_pageset;
7297		if (zone->per_cpu_zonestats != &boot_zonestats) {
7298			free_percpu(zone->per_cpu_zonestats);
7299			zone->per_cpu_zonestats = &boot_zonestats;
7300		}
7301	}
7302}
7303
7304#ifdef CONFIG_MEMORY_HOTREMOVE
7305/*
7306 * All pages in the range must be in a single zone, must not contain holes,
7307 * must span full sections, and must be isolated before calling this function.
7308 *
7309 * Returns the number of managed (non-PageOffline()) pages in the range: the
7310 * number of pages for which memory offlining code must adjust managed page
7311 * counters using adjust_managed_page_count().
7312 */
7313unsigned long __offline_isolated_pages(unsigned long start_pfn,
7314		unsigned long end_pfn)
7315{
7316	unsigned long already_offline = 0, flags;
7317	unsigned long pfn = start_pfn;
7318	struct page *page;
7319	struct zone *zone;
7320	unsigned int order;
7321
7322	offline_mem_sections(pfn, end_pfn);
7323	zone = page_zone(pfn_to_page(pfn));
7324	spin_lock_irqsave(&zone->lock, flags);
7325	while (pfn < end_pfn) {
7326		page = pfn_to_page(pfn);
7327		/*
7328		 * The HWPoisoned page may be not in buddy system, and
7329		 * page_count() is not 0.
7330		 */
7331		if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
7332			pfn++;
7333			continue;
7334		}
7335		/*
7336		 * At this point all remaining PageOffline() pages have a
7337		 * reference count of 0 and can simply be skipped.
7338		 */
7339		if (PageOffline(page)) {
7340			BUG_ON(page_count(page));
7341			BUG_ON(PageBuddy(page));
7342			already_offline++;
7343			pfn++;
7344			continue;
7345		}
7346
7347		BUG_ON(page_count(page));
7348		BUG_ON(!PageBuddy(page));
7349		VM_WARN_ON(get_pageblock_migratetype(page) != MIGRATE_ISOLATE);
7350		order = buddy_order(page);
7351		del_page_from_free_list(page, zone, order, MIGRATE_ISOLATE);
7352		pfn += (1 << order);
7353	}
7354	spin_unlock_irqrestore(&zone->lock, flags);
7355
7356	return end_pfn - start_pfn - already_offline;
7357}
7358#endif
7359
7360/*
7361 * This function returns a stable result only if called under zone lock.
7362 */
7363bool is_free_buddy_page(const struct page *page)
7364{
7365	unsigned long pfn = page_to_pfn(page);
7366	unsigned int order;
7367
7368	for (order = 0; order < NR_PAGE_ORDERS; order++) {
7369		const struct page *head = page - (pfn & ((1 << order) - 1));
7370
7371		if (PageBuddy(head) &&
7372		    buddy_order_unsafe(head) >= order)
7373			break;
7374	}
7375
7376	return order <= MAX_PAGE_ORDER;
7377}
7378EXPORT_SYMBOL(is_free_buddy_page);
7379
7380#ifdef CONFIG_MEMORY_FAILURE
7381static inline void add_to_free_list(struct page *page, struct zone *zone,
7382				    unsigned int order, int migratetype,
7383				    bool tail)
7384{
7385	__add_to_free_list(page, zone, order, migratetype, tail);
7386	account_freepages(zone, 1 << order, migratetype);
7387}
7388
7389/*
7390 * Break down a higher-order page in sub-pages, and keep our target out of
7391 * buddy allocator.
7392 */
7393static void break_down_buddy_pages(struct zone *zone, struct page *page,
7394				   struct page *target, int low, int high,
7395				   int migratetype)
7396{
7397	unsigned long size = 1 << high;
7398	struct page *current_buddy;
7399
7400	while (high > low) {
7401		high--;
7402		size >>= 1;
7403
7404		if (target >= &page[size]) {
7405			current_buddy = page;
7406			page = page + size;
7407		} else {
7408			current_buddy = page + size;
7409		}
7410
7411		if (set_page_guard(zone, current_buddy, high))
7412			continue;
7413
7414		add_to_free_list(current_buddy, zone, high, migratetype, false);
7415		set_buddy_order(current_buddy, high);
7416	}
7417}
7418
7419/*
7420 * Take a page that will be marked as poisoned off the buddy allocator.
7421 */
7422bool take_page_off_buddy(struct page *page)
7423{
7424	struct zone *zone = page_zone(page);
7425	unsigned long pfn = page_to_pfn(page);
7426	unsigned long flags;
7427	unsigned int order;
7428	bool ret = false;
7429
7430	spin_lock_irqsave(&zone->lock, flags);
7431	for (order = 0; order < NR_PAGE_ORDERS; order++) {
7432		struct page *page_head = page - (pfn & ((1 << order) - 1));
7433		int page_order = buddy_order(page_head);
7434
7435		if (PageBuddy(page_head) && page_order >= order) {
7436			unsigned long pfn_head = page_to_pfn(page_head);
7437			int migratetype = get_pfnblock_migratetype(page_head,
7438								   pfn_head);
7439
7440			del_page_from_free_list(page_head, zone, page_order,
7441						migratetype);
7442			break_down_buddy_pages(zone, page_head, page, 0,
7443						page_order, migratetype);
7444			SetPageHWPoisonTakenOff(page);
7445			ret = true;
7446			break;
7447		}
7448		if (page_count(page_head) > 0)
7449			break;
7450	}
7451	spin_unlock_irqrestore(&zone->lock, flags);
7452	return ret;
7453}
7454
7455/*
7456 * Cancel takeoff done by take_page_off_buddy().
7457 */
7458bool put_page_back_buddy(struct page *page)
7459{
7460	struct zone *zone = page_zone(page);
7461	unsigned long flags;
7462	bool ret = false;
7463
7464	spin_lock_irqsave(&zone->lock, flags);
7465	if (put_page_testzero(page)) {
7466		unsigned long pfn = page_to_pfn(page);
7467		int migratetype = get_pfnblock_migratetype(page, pfn);
7468
7469		ClearPageHWPoisonTakenOff(page);
7470		__free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE);
7471		if (TestClearPageHWPoison(page)) {
7472			ret = true;
7473		}
7474	}
7475	spin_unlock_irqrestore(&zone->lock, flags);
7476
7477	return ret;
7478}
7479#endif
7480
7481bool has_managed_zone(enum zone_type zone)
7482{
7483	struct pglist_data *pgdat;
7484
7485	for_each_online_pgdat(pgdat) {
7486		if (managed_zone(&pgdat->node_zones[zone]))
7487			return true;
7488	}
7489	return false;
7490}
7491
7492#ifdef CONFIG_UNACCEPTED_MEMORY
7493
7494static bool lazy_accept = true;
7495
7496static int __init accept_memory_parse(char *p)
7497{
7498	if (!strcmp(p, "lazy")) {
7499		lazy_accept = true;
7500		return 0;
7501	} else if (!strcmp(p, "eager")) {
7502		lazy_accept = false;
7503		return 0;
7504	} else {
7505		return -EINVAL;
7506	}
7507}
7508early_param("accept_memory", accept_memory_parse);
7509
7510static bool page_contains_unaccepted(struct page *page, unsigned int order)
7511{
7512	phys_addr_t start = page_to_phys(page);
7513
7514	return range_contains_unaccepted_memory(start, PAGE_SIZE << order);
7515}
7516
7517static void __accept_page(struct zone *zone, unsigned long *flags,
7518			  struct page *page)
7519{
7520	list_del(&page->lru);
7521	account_freepages(zone, -MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE);
7522	__mod_zone_page_state(zone, NR_UNACCEPTED, -MAX_ORDER_NR_PAGES);
7523	__ClearPageUnaccepted(page);
7524	spin_unlock_irqrestore(&zone->lock, *flags);
7525
7526	accept_memory(page_to_phys(page), PAGE_SIZE << MAX_PAGE_ORDER);
7527
7528	__free_pages_ok(page, MAX_PAGE_ORDER, FPI_TO_TAIL);
7529}
7530
7531void accept_page(struct page *page)
7532{
7533	struct zone *zone = page_zone(page);
7534	unsigned long flags;
7535
7536	spin_lock_irqsave(&zone->lock, flags);
7537	if (!PageUnaccepted(page)) {
7538		spin_unlock_irqrestore(&zone->lock, flags);
7539		return;
7540	}
7541
7542	/* Unlocks zone->lock */
7543	__accept_page(zone, &flags, page);
7544}
7545
7546static bool try_to_accept_memory_one(struct zone *zone)
7547{
7548	unsigned long flags;
7549	struct page *page;
7550
7551	spin_lock_irqsave(&zone->lock, flags);
7552	page = list_first_entry_or_null(&zone->unaccepted_pages,
7553					struct page, lru);
7554	if (!page) {
7555		spin_unlock_irqrestore(&zone->lock, flags);
7556		return false;
7557	}
7558
7559	/* Unlocks zone->lock */
7560	__accept_page(zone, &flags, page);
7561
7562	return true;
7563}
7564
7565static bool cond_accept_memory(struct zone *zone, unsigned int order,
7566			       int alloc_flags)
7567{
7568	long to_accept, wmark;
7569	bool ret = false;
7570
7571	if (list_empty(&zone->unaccepted_pages))
7572		return false;
7573
7574	/* Bailout, since try_to_accept_memory_one() needs to take a lock */
7575	if (alloc_flags & ALLOC_TRYLOCK)
7576		return false;
7577
7578	wmark = promo_wmark_pages(zone);
7579
7580	/*
7581	 * Watermarks have not been initialized yet.
7582	 *
7583	 * Accepting one MAX_ORDER page to ensure progress.
7584	 */
7585	if (!wmark)
7586		return try_to_accept_memory_one(zone);
7587
7588	/* How much to accept to get to promo watermark? */
7589	to_accept = wmark -
7590		    (zone_page_state(zone, NR_FREE_PAGES) -
7591		    __zone_watermark_unusable_free(zone, order, 0) -
7592		    zone_page_state(zone, NR_UNACCEPTED));
7593
7594	while (to_accept > 0) {
7595		if (!try_to_accept_memory_one(zone))
7596			break;
7597		ret = true;
7598		to_accept -= MAX_ORDER_NR_PAGES;
7599	}
7600
7601	return ret;
7602}
7603
7604static bool __free_unaccepted(struct page *page)
7605{
7606	struct zone *zone = page_zone(page);
7607	unsigned long flags;
7608
7609	if (!lazy_accept)
7610		return false;
7611
7612	spin_lock_irqsave(&zone->lock, flags);
7613	list_add_tail(&page->lru, &zone->unaccepted_pages);
7614	account_freepages(zone, MAX_ORDER_NR_PAGES, MIGRATE_MOVABLE);
7615	__mod_zone_page_state(zone, NR_UNACCEPTED, MAX_ORDER_NR_PAGES);
7616	__SetPageUnaccepted(page);
7617	spin_unlock_irqrestore(&zone->lock, flags);
7618
7619	return true;
7620}
7621
7622#else
7623
7624static bool page_contains_unaccepted(struct page *page, unsigned int order)
7625{
7626	return false;
7627}
7628
7629static bool cond_accept_memory(struct zone *zone, unsigned int order,
7630			       int alloc_flags)
7631{
7632	return false;
7633}
7634
7635static bool __free_unaccepted(struct page *page)
7636{
7637	BUILD_BUG();
7638	return false;
7639}
7640
7641#endif /* CONFIG_UNACCEPTED_MEMORY */
7642
7643struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order)
7644{
7645	/*
7646	 * Do not specify __GFP_DIRECT_RECLAIM, since direct claim is not allowed.
7647	 * Do not specify __GFP_KSWAPD_RECLAIM either, since wake up of kswapd
7648	 * is not safe in arbitrary context.
7649	 *
7650	 * These two are the conditions for gfpflags_allow_spinning() being true.
7651	 *
7652	 * Specify __GFP_NOWARN since failing alloc_pages_nolock() is not a reason
7653	 * to warn. Also warn would trigger printk() which is unsafe from
7654	 * various contexts. We cannot use printk_deferred_enter() to mitigate,
7655	 * since the running context is unknown.
7656	 *
7657	 * Specify __GFP_ZERO to make sure that call to kmsan_alloc_page() below
7658	 * is safe in any context. Also zeroing the page is mandatory for
7659	 * BPF use cases.
7660	 *
7661	 * Though __GFP_NOMEMALLOC is not checked in the code path below,
7662	 * specify it here to highlight that alloc_pages_nolock()
7663	 * doesn't want to deplete reserves.
7664	 */
7665	gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP
7666			| gfp_flags;
7667	unsigned int alloc_flags = ALLOC_TRYLOCK;
7668	struct alloc_context ac = { };
7669	struct page *page;
7670
7671	VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT);
7672	/*
7673	 * In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
7674	 * unsafe in NMI. If spin_trylock() is called from hard IRQ the current
7675	 * task may be waiting for one rt_spin_lock, but rt_spin_trylock() will
7676	 * mark the task as the owner of another rt_spin_lock which will
7677	 * confuse PI logic, so return immediately if called form hard IRQ or
7678	 * NMI.
7679	 *
7680	 * Note, irqs_disabled() case is ok. This function can be called
7681	 * from raw_spin_lock_irqsave region.
7682	 */
7683	if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq()))
7684		return NULL;
7685	if (!pcp_allowed_order(order))
7686		return NULL;
7687
7688	/* Bailout, since _deferred_grow_zone() needs to take a lock */
7689	if (deferred_pages_enabled())
7690		return NULL;
7691
7692	if (nid == NUMA_NO_NODE)
7693		nid = numa_node_id();
7694
7695	prepare_alloc_pages(alloc_gfp, order, nid, NULL, &ac,
7696			    &alloc_gfp, &alloc_flags);
7697
7698	/*
7699	 * Best effort allocation from percpu free list.
7700	 * If it's empty attempt to spin_trylock zone->lock.
7701	 */
7702	page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
7703
7704	/* Unlike regular alloc_pages() there is no __alloc_pages_slowpath(). */
7705
7706	if (memcg_kmem_online() && page && (gfp_flags & __GFP_ACCOUNT) &&
7707	    unlikely(__memcg_kmem_charge_page(page, alloc_gfp, order) != 0)) {
7708		__free_frozen_pages(page, order, FPI_TRYLOCK);
7709		page = NULL;
7710	}
7711	trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
7712	kmsan_alloc_page(page, order, alloc_gfp);
7713	return page;
7714}
7715/**
7716 * alloc_pages_nolock - opportunistic reentrant allocation from any context
7717 * @gfp_flags: GFP flags. Only __GFP_ACCOUNT allowed.
7718 * @nid: node to allocate from
7719 * @order: allocation order size
7720 *
7721 * Allocates pages of a given order from the given node. This is safe to
7722 * call from any context (from atomic, NMI, and also reentrant
7723 * allocator -> tracepoint -> alloc_pages_nolock_noprof).
7724 * Allocation is best effort and to be expected to fail easily so nobody should
7725 * rely on the success. Failures are not reported via warn_alloc().
7726 * See always fail conditions below.
7727 *
7728 * Return: allocated page or NULL on failure. NULL does not mean EBUSY or EAGAIN.
7729 * It means ENOMEM. There is no reason to call it again and expect !NULL.
7730 */
7731struct page *alloc_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order)
7732{
7733	struct page *page;
7734
7735	page = alloc_frozen_pages_nolock_noprof(gfp_flags, nid, order);
7736	if (page)
7737		set_page_refcounted(page);
7738	return page;
7739}
7740EXPORT_SYMBOL_GPL(alloc_pages_nolock_noprof);