mempolicy.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Simple NUMA memory policy for the Linux kernel.
   4 *
   5 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
   6 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
   7 *
   8 * NUMA policy allows the user to give hints in which node(s) memory should
   9 * be allocated.
  10 *
  11 * Support six policies per VMA and per process:
  12 *
  13 * The VMA policy has priority over the process policy for a page fault.
  14 *
  15 * interleave     Allocate memory interleaved over a set of nodes,
  16 *                with normal fallback if it fails.
  17 *                For VMA based allocations this interleaves based on the
  18 *                offset into the backing object or offset into the mapping
  19 *                for anonymous memory. For process policy an process counter
  20 *                is used.
  21 *
  22 * weighted interleave
  23 *                Allocate memory interleaved over a set of nodes based on
  24 *                a set of weights (per-node), with normal fallback if it
  25 *                fails.  Otherwise operates the same as interleave.
  26 *                Example: nodeset(0,1) & weights (2,1) - 2 pages allocated
  27 *                on node 0 for every 1 page allocated on node 1.
  28 *
  29 * bind           Only allocate memory on a specific set of nodes,
  30 *                no fallback.
  31 *                FIXME: memory is allocated starting with the first node
  32 *                to the last. It would be better if bind would truly restrict
  33 *                the allocation to memory nodes instead
  34 *
  35 * preferred      Try a specific node first before normal fallback.
  36 *                As a special case NUMA_NO_NODE here means do the allocation
  37 *                on the local CPU. This is normally identical to default,
  38 *                but useful to set in a VMA when you have a non default
  39 *                process policy.
  40 *
  41 * preferred many Try a set of nodes first before normal fallback. This is
  42 *                similar to preferred without the special case.
  43 *
  44 * default        Allocate on the local node first, or when on a VMA
  45 *                use the process policy. This is what Linux always did
  46 *		  in a NUMA aware kernel and still does by, ahem, default.
  47 *
  48 * The process policy is applied for most non interrupt memory allocations
  49 * in that process' context. Interrupts ignore the policies and always
  50 * try to allocate on the local CPU. The VMA policy is only applied for memory
  51 * allocations for a VMA in the VM.
  52 *
  53 * Currently there are a few corner cases in swapping where the policy
  54 * is not applied, but the majority should be handled. When process policy
  55 * is used it is not remembered over swap outs/swap ins.
  56 *
  57 * Only the highest zone in the zone hierarchy gets policied. Allocations
  58 * requesting a lower zone just use default policy. This implies that
  59 * on systems with highmem kernel lowmem allocation don't get policied.
  60 * Same with GFP_DMA allocations.
  61 *
  62 * For shmem/tmpfs shared memory the policy is shared between
  63 * all users and remembered even when nobody has memory mapped.
  64 */
  65
  66/* Notebook:
  67   fix mmap readahead to honour policy and enable policy for any page cache
  68   object
  69   statistics for bigpages
  70   global policy for page cache? currently it uses process policy. Requires
  71   first item above.
  72   handle mremap for shared memory (currently ignored for the policy)
  73   grows down?
  74   make bind policy root only? It can trigger oom much faster and the
  75   kernel is not always grateful with that.
  76*/
  77
  78#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  79
  80#include <linux/mempolicy.h>
  81#include <linux/pagewalk.h>
  82#include <linux/highmem.h>
  83#include <linux/hugetlb.h>
  84#include <linux/kernel.h>
  85#include <linux/sched.h>
  86#include <linux/sched/mm.h>
  87#include <linux/sched/numa_balancing.h>
  88#include <linux/sched/sysctl.h>
  89#include <linux/sched/task.h>
  90#include <linux/nodemask.h>
  91#include <linux/cpuset.h>
  92#include <linux/slab.h>
  93#include <linux/string.h>
  94#include <linux/export.h>
  95#include <linux/nsproxy.h>
  96#include <linux/interrupt.h>
  97#include <linux/init.h>
  98#include <linux/compat.h>
  99#include <linux/ptrace.h>
 100#include <linux/swap.h>
 101#include <linux/seq_file.h>
 102#include <linux/proc_fs.h>
 103#include <linux/memory-tiers.h>
 104#include <linux/migrate.h>
 105#include <linux/ksm.h>
 106#include <linux/rmap.h>
 107#include <linux/security.h>
 108#include <linux/syscalls.h>
 109#include <linux/ctype.h>
 110#include <linux/mm_inline.h>
 111#include <linux/mmu_notifier.h>
 112#include <linux/printk.h>
 113#include <linux/leafops.h>
 114#include <linux/gcd.h>
 115
 116#include <asm/tlbflush.h>
 117#include <asm/tlb.h>
 118#include <linux/uaccess.h>
 119#include <linux/memory.h>
 120
 121#include "internal.h"
 122
 123/* Internal flags */
 124#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
 125#define MPOL_MF_INVERT       (MPOL_MF_INTERNAL << 1)	/* Invert check for nodemask */
 126#define MPOL_MF_WRLOCK       (MPOL_MF_INTERNAL << 2)	/* Write-lock walked vmas */
 127
 128static struct kmem_cache *policy_cache;
 129static struct kmem_cache *sn_cache;
 130
 131/* Highest zone. An specific allocation for a zone below that is not
 132   policied. */
 133enum zone_type policy_zone = 0;
 134
 135/*
 136 * run-time system-wide default policy => local allocation
 137 */
 138static struct mempolicy default_policy = {
 139	.refcnt = ATOMIC_INIT(1), /* never free it */
 140	.mode = MPOL_LOCAL,
 141};
 142
 143static struct mempolicy preferred_node_policy[MAX_NUMNODES];
 144
 145/*
 146 * weightiness balances the tradeoff between small weights (cycles through nodes
 147 * faster, more fair/even distribution) and large weights (smaller errors
 148 * between actual bandwidth ratios and weight ratios). 32 is a number that has
 149 * been found to perform at a reasonable compromise between the two goals.
 150 */
 151static const int weightiness = 32;
 152
 153/*
 154 * A null weighted_interleave_state is interpreted as having .mode="auto",
 155 * and .iw_table is interpreted as an array of 1s with length nr_node_ids.
 156 */
 157struct weighted_interleave_state {
 158	bool mode_auto;
 159	u8 iw_table[];
 160};
 161static struct weighted_interleave_state __rcu *wi_state;
 162static unsigned int *node_bw_table;
 163
 164/*
 165 * wi_state_lock protects both wi_state and node_bw_table.
 166 * node_bw_table is only used by writers to update wi_state.
 167 */
 168static DEFINE_MUTEX(wi_state_lock);
 169
 170static u8 get_il_weight(int node)
 171{
 172	struct weighted_interleave_state *state;
 173	u8 weight = 1;
 174
 175	rcu_read_lock();
 176	state = rcu_dereference(wi_state);
 177	if (state)
 178		weight = state->iw_table[node];
 179	rcu_read_unlock();
 180	return weight;
 181}
 182
 183/*
 184 * Convert bandwidth values into weighted interleave weights.
 185 * Call with wi_state_lock.
 186 */
 187static void reduce_interleave_weights(unsigned int *bw, u8 *new_iw)
 188{
 189	u64 sum_bw = 0;
 190	unsigned int cast_sum_bw, scaling_factor = 1, iw_gcd = 0;
 191	int nid;
 192
 193	for_each_node_state(nid, N_MEMORY)
 194		sum_bw += bw[nid];
 195
 196	/* Scale bandwidths to whole numbers in the range [1, weightiness] */
 197	for_each_node_state(nid, N_MEMORY) {
 198		/*
 199		 * Try not to perform 64-bit division.
 200		 * If sum_bw < scaling_factor, then sum_bw < U32_MAX.
 201		 * If sum_bw > scaling_factor, then round the weight up to 1.
 202		 */
 203		scaling_factor = weightiness * bw[nid];
 204		if (bw[nid] && sum_bw < scaling_factor) {
 205			cast_sum_bw = (unsigned int)sum_bw;
 206			new_iw[nid] = scaling_factor / cast_sum_bw;
 207		} else {
 208			new_iw[nid] = 1;
 209		}
 210		if (!iw_gcd)
 211			iw_gcd = new_iw[nid];
 212		iw_gcd = gcd(iw_gcd, new_iw[nid]);
 213	}
 214
 215	/* 1:2 is strictly better than 16:32. Reduce by the weights' GCD. */
 216	for_each_node_state(nid, N_MEMORY)
 217		new_iw[nid] /= iw_gcd;
 218}
 219
 220int mempolicy_set_node_perf(unsigned int node, struct access_coordinate *coords)
 221{
 222	struct weighted_interleave_state *new_wi_state, *old_wi_state = NULL;
 223	unsigned int *old_bw, *new_bw;
 224	unsigned int bw_val;
 225	int i;
 226
 227	bw_val = min(coords->read_bandwidth, coords->write_bandwidth);
 228	new_bw = kcalloc(nr_node_ids, sizeof(unsigned int), GFP_KERNEL);
 229	if (!new_bw)
 230		return -ENOMEM;
 231
 232	new_wi_state = kmalloc(struct_size(new_wi_state, iw_table, nr_node_ids),
 233			       GFP_KERNEL);
 234	if (!new_wi_state) {
 235		kfree(new_bw);
 236		return -ENOMEM;
 237	}
 238	new_wi_state->mode_auto = true;
 239	for (i = 0; i < nr_node_ids; i++)
 240		new_wi_state->iw_table[i] = 1;
 241
 242	/*
 243	 * Update bandwidth info, even in manual mode. That way, when switching
 244	 * to auto mode in the future, iw_table can be overwritten using
 245	 * accurate bw data.
 246	 */
 247	mutex_lock(&wi_state_lock);
 248
 249	old_bw = node_bw_table;
 250	if (old_bw)
 251		memcpy(new_bw, old_bw, nr_node_ids * sizeof(*old_bw));
 252	new_bw[node] = bw_val;
 253	node_bw_table = new_bw;
 254
 255	old_wi_state = rcu_dereference_protected(wi_state,
 256					lockdep_is_held(&wi_state_lock));
 257	if (old_wi_state && !old_wi_state->mode_auto) {
 258		/* Manual mode; skip reducing weights and updating wi_state */
 259		mutex_unlock(&wi_state_lock);
 260		kfree(new_wi_state);
 261		goto out;
 262	}
 263
 264	/* NULL wi_state assumes auto=true; reduce weights and update wi_state*/
 265	reduce_interleave_weights(new_bw, new_wi_state->iw_table);
 266	rcu_assign_pointer(wi_state, new_wi_state);
 267
 268	mutex_unlock(&wi_state_lock);
 269	if (old_wi_state) {
 270		synchronize_rcu();
 271		kfree(old_wi_state);
 272	}
 273out:
 274	kfree(old_bw);
 275	return 0;
 276}
 277
 278/**
 279 * numa_nearest_node - Find nearest node by state
 280 * @node: Node id to start the search
 281 * @state: State to filter the search
 282 *
 283 * Lookup the closest node by distance if @nid is not in state.
 284 *
 285 * Return: this @node if it is in state, otherwise the closest node by distance
 286 */
 287int numa_nearest_node(int node, unsigned int state)
 288{
 289	int min_dist = INT_MAX, dist, n, min_node;
 290
 291	if (state >= NR_NODE_STATES)
 292		return -EINVAL;
 293
 294	if (node == NUMA_NO_NODE || node_state(node, state))
 295		return node;
 296
 297	min_node = node;
 298	for_each_node_state(n, state) {
 299		dist = node_distance(node, n);
 300		if (dist < min_dist) {
 301			min_dist = dist;
 302			min_node = n;
 303		}
 304	}
 305
 306	return min_node;
 307}
 308EXPORT_SYMBOL_GPL(numa_nearest_node);
 309
 310/**
 311 * nearest_node_nodemask - Find the node in @mask at the nearest distance
 312 *			   from @node.
 313 *
 314 * @node: a valid node ID to start the search from.
 315 * @mask: a pointer to a nodemask representing the allowed nodes.
 316 *
 317 * This function iterates over all nodes in @mask and calculates the
 318 * distance from the starting @node, then it returns the node ID that is
 319 * the closest to @node, or MAX_NUMNODES if no node is found.
 320 *
 321 * Note that @node must be a valid node ID usable with node_distance(),
 322 * providing an invalid node ID (e.g., NUMA_NO_NODE) may result in crashes
 323 * or unexpected behavior.
 324 */
 325int nearest_node_nodemask(int node, nodemask_t *mask)
 326{
 327	int dist, n, min_dist = INT_MAX, min_node = MAX_NUMNODES;
 328
 329	for_each_node_mask(n, *mask) {
 330		dist = node_distance(node, n);
 331		if (dist < min_dist) {
 332			min_dist = dist;
 333			min_node = n;
 334		}
 335	}
 336
 337	return min_node;
 338}
 339EXPORT_SYMBOL_GPL(nearest_node_nodemask);
 340
 341struct mempolicy *get_task_policy(struct task_struct *p)
 342{
 343	struct mempolicy *pol = p->mempolicy;
 344	int node;
 345
 346	if (pol)
 347		return pol;
 348
 349	node = numa_node_id();
 350	if (node != NUMA_NO_NODE) {
 351		pol = &preferred_node_policy[node];
 352		/* preferred_node_policy is not initialised early in boot */
 353		if (pol->mode)
 354			return pol;
 355	}
 356
 357	return &default_policy;
 358}
 359EXPORT_SYMBOL_FOR_MODULES(get_task_policy, "kvm");
 360
 361static const struct mempolicy_operations {
 362	int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
 363	void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
 364} mpol_ops[MPOL_MAX];
 365
 366static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
 367{
 368	return pol->flags & MPOL_MODE_FLAGS;
 369}
 370
 371static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
 372				   const nodemask_t *rel)
 373{
 374	nodemask_t tmp;
 375	nodes_fold(tmp, *orig, nodes_weight(*rel));
 376	nodes_onto(*ret, tmp, *rel);
 377}
 378
 379static int mpol_new_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
 380{
 381	if (nodes_empty(*nodes))
 382		return -EINVAL;
 383	pol->nodes = *nodes;
 384	return 0;
 385}
 386
 387static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
 388{
 389	if (nodes_empty(*nodes))
 390		return -EINVAL;
 391
 392	nodes_clear(pol->nodes);
 393	node_set(first_node(*nodes), pol->nodes);
 394	return 0;
 395}
 396
 397/*
 398 * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
 399 * any, for the new policy.  mpol_new() has already validated the nodes
 400 * parameter with respect to the policy mode and flags.
 401 *
 402 * Must be called holding task's alloc_lock to protect task's mems_allowed
 403 * and mempolicy.  May also be called holding the mmap_lock for write.
 404 */
 405static int mpol_set_nodemask(struct mempolicy *pol,
 406		     const nodemask_t *nodes, struct nodemask_scratch *nsc)
 407{
 408	int ret;
 409
 410	/*
 411	 * Default (pol==NULL) resp. local memory policies are not a
 412	 * subject of any remapping. They also do not need any special
 413	 * constructor.
 414	 */
 415	if (!pol || pol->mode == MPOL_LOCAL)
 416		return 0;
 417
 418	/* Check N_MEMORY */
 419	nodes_and(nsc->mask1,
 420		  cpuset_current_mems_allowed, node_states[N_MEMORY]);
 421
 422	VM_BUG_ON(!nodes);
 423
 424	if (pol->flags & MPOL_F_RELATIVE_NODES)
 425		mpol_relative_nodemask(&nsc->mask2, nodes, &nsc->mask1);
 426	else
 427		nodes_and(nsc->mask2, *nodes, nsc->mask1);
 428
 429	if (mpol_store_user_nodemask(pol))
 430		pol->w.user_nodemask = *nodes;
 431	else
 432		pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed;
 433
 434	ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
 435	return ret;
 436}
 437
 438/*
 439 * This function just creates a new policy, does some check and simple
 440 * initialization. You must invoke mpol_set_nodemask() to set nodes.
 441 */
 442static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
 443				  nodemask_t *nodes)
 444{
 445	struct mempolicy *policy;
 446
 447	if (mode == MPOL_DEFAULT) {
 448		if (nodes && !nodes_empty(*nodes))
 449			return ERR_PTR(-EINVAL);
 450		return NULL;
 451	}
 452	VM_BUG_ON(!nodes);
 453
 454	/*
 455	 * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
 456	 * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
 457	 * All other modes require a valid pointer to a non-empty nodemask.
 458	 */
 459	if (mode == MPOL_PREFERRED) {
 460		if (nodes_empty(*nodes)) {
 461			if (((flags & MPOL_F_STATIC_NODES) ||
 462			     (flags & MPOL_F_RELATIVE_NODES)))
 463				return ERR_PTR(-EINVAL);
 464
 465			mode = MPOL_LOCAL;
 466		}
 467	} else if (mode == MPOL_LOCAL) {
 468		if (!nodes_empty(*nodes) ||
 469		    (flags & MPOL_F_STATIC_NODES) ||
 470		    (flags & MPOL_F_RELATIVE_NODES))
 471			return ERR_PTR(-EINVAL);
 472	} else if (nodes_empty(*nodes))
 473		return ERR_PTR(-EINVAL);
 474
 475	policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
 476	if (!policy)
 477		return ERR_PTR(-ENOMEM);
 478	atomic_set(&policy->refcnt, 1);
 479	policy->mode = mode;
 480	policy->flags = flags;
 481	policy->home_node = NUMA_NO_NODE;
 482
 483	return policy;
 484}
 485
 486/* Slow path of a mpol destructor. */
 487void __mpol_put(struct mempolicy *pol)
 488{
 489	if (!atomic_dec_and_test(&pol->refcnt))
 490		return;
 491	/*
 492	 * Required to allow mmap_lock_speculative*() access, see for example
 493	 * futex_key_to_node_opt(). All accesses are serialized by mmap_lock,
 494	 * however the speculative lock section unbound by the normal lock
 495	 * boundaries, requiring RCU freeing.
 496	 */
 497	kfree_rcu(pol, rcu);
 498}
 499EXPORT_SYMBOL_FOR_MODULES(__mpol_put, "kvm");
 500
 501static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
 502{
 503}
 504
 505static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
 506{
 507	nodemask_t tmp;
 508
 509	if (pol->flags & MPOL_F_STATIC_NODES)
 510		nodes_and(tmp, pol->w.user_nodemask, *nodes);
 511	else if (pol->flags & MPOL_F_RELATIVE_NODES)
 512		mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
 513	else {
 514		nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed,
 515								*nodes);
 516		pol->w.cpuset_mems_allowed = *nodes;
 517	}
 518
 519	if (nodes_empty(tmp))
 520		tmp = *nodes;
 521
 522	pol->nodes = tmp;
 523}
 524
 525static void mpol_rebind_preferred(struct mempolicy *pol,
 526						const nodemask_t *nodes)
 527{
 528	pol->w.cpuset_mems_allowed = *nodes;
 529}
 530
 531/*
 532 * mpol_rebind_policy - Migrate a policy to a different set of nodes
 533 *
 534 * Per-vma policies are protected by mmap_lock. Allocations using per-task
 535 * policies are protected by task->mems_allowed_seq to prevent a premature
 536 * OOM/allocation failure due to parallel nodemask modification.
 537 */
 538static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
 539{
 540	if (!pol || pol->mode == MPOL_LOCAL)
 541		return;
 542	if (!mpol_store_user_nodemask(pol) &&
 543	    nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
 544		return;
 545
 546	mpol_ops[pol->mode].rebind(pol, newmask);
 547}
 548
 549/*
 550 * Wrapper for mpol_rebind_policy() that just requires task
 551 * pointer, and updates task mempolicy.
 552 *
 553 * Called with task's alloc_lock held.
 554 */
 555void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
 556{
 557	mpol_rebind_policy(tsk->mempolicy, new);
 558}
 559
 560/*
 561 * Rebind each vma in mm to new nodemask.
 562 *
 563 * Call holding a reference to mm.  Takes mm->mmap_lock during call.
 564 */
 565void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
 566{
 567	struct vm_area_struct *vma;
 568	VMA_ITERATOR(vmi, mm, 0);
 569
 570	mmap_write_lock(mm);
 571	for_each_vma(vmi, vma) {
 572		vma_start_write(vma);
 573		mpol_rebind_policy(vma->vm_policy, new);
 574	}
 575	mmap_write_unlock(mm);
 576}
 577
 578static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
 579	[MPOL_DEFAULT] = {
 580		.rebind = mpol_rebind_default,
 581	},
 582	[MPOL_INTERLEAVE] = {
 583		.create = mpol_new_nodemask,
 584		.rebind = mpol_rebind_nodemask,
 585	},
 586	[MPOL_PREFERRED] = {
 587		.create = mpol_new_preferred,
 588		.rebind = mpol_rebind_preferred,
 589	},
 590	[MPOL_BIND] = {
 591		.create = mpol_new_nodemask,
 592		.rebind = mpol_rebind_nodemask,
 593	},
 594	[MPOL_LOCAL] = {
 595		.rebind = mpol_rebind_default,
 596	},
 597	[MPOL_PREFERRED_MANY] = {
 598		.create = mpol_new_nodemask,
 599		.rebind = mpol_rebind_preferred,
 600	},
 601	[MPOL_WEIGHTED_INTERLEAVE] = {
 602		.create = mpol_new_nodemask,
 603		.rebind = mpol_rebind_nodemask,
 604	},
 605};
 606
 607static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
 608				unsigned long flags);
 609static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol,
 610				pgoff_t ilx, int *nid);
 611
 612static bool strictly_unmovable(unsigned long flags)
 613{
 614	/*
 615	 * STRICT without MOVE flags lets do_mbind() fail immediately with -EIO
 616	 * if any misplaced page is found.
 617	 */
 618	return (flags & (MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ==
 619			 MPOL_MF_STRICT;
 620}
 621
 622struct migration_mpol {		/* for alloc_migration_target_by_mpol() */
 623	struct mempolicy *pol;
 624	pgoff_t ilx;
 625};
 626
 627struct queue_pages {
 628	struct list_head *pagelist;
 629	unsigned long flags;
 630	nodemask_t *nmask;
 631	unsigned long start;
 632	unsigned long end;
 633	struct vm_area_struct *first;
 634	struct folio *large;		/* note last large folio encountered */
 635	long nr_failed;			/* could not be isolated at this time */
 636};
 637
 638/*
 639 * Check if the folio's nid is in qp->nmask.
 640 *
 641 * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
 642 * in the invert of qp->nmask.
 643 */
 644static inline bool queue_folio_required(struct folio *folio,
 645					struct queue_pages *qp)
 646{
 647	int nid = folio_nid(folio);
 648	unsigned long flags = qp->flags;
 649
 650	return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
 651}
 652
 653static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
 654{
 655	struct folio *folio;
 656	struct queue_pages *qp = walk->private;
 657
 658	if (unlikely(pmd_is_migration_entry(*pmd))) {
 659		qp->nr_failed++;
 660		return;
 661	}
 662	folio = pmd_folio(*pmd);
 663	if (is_huge_zero_folio(folio)) {
 664		walk->action = ACTION_CONTINUE;
 665		return;
 666	}
 667	if (!queue_folio_required(folio, qp))
 668		return;
 669	if (!(qp->flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
 670	    !vma_migratable(walk->vma) ||
 671	    !migrate_folio_add(folio, qp->pagelist, qp->flags))
 672		qp->nr_failed++;
 673}
 674
 675/*
 676 * Scan through folios, checking if they satisfy the required conditions,
 677 * moving them from LRU to local pagelist for migration if they do (or not).
 678 *
 679 * queue_folios_pte_range() has two possible return values:
 680 * 0 - continue walking to scan for more, even if an existing folio on the
 681 *     wrong node could not be isolated and queued for migration.
 682 * -EIO - only MPOL_MF_STRICT was specified, without MPOL_MF_MOVE or ..._ALL,
 683 *        and an existing folio was on a node that does not follow the policy.
 684 */
 685static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
 686			unsigned long end, struct mm_walk *walk)
 687{
 688	struct vm_area_struct *vma = walk->vma;
 689	struct folio *folio;
 690	struct queue_pages *qp = walk->private;
 691	unsigned long flags = qp->flags;
 692	pte_t *pte, *mapped_pte;
 693	pte_t ptent;
 694	spinlock_t *ptl;
 695	int max_nr, nr;
 696
 697	ptl = pmd_trans_huge_lock(pmd, vma);
 698	if (ptl) {
 699		queue_folios_pmd(pmd, walk);
 700		spin_unlock(ptl);
 701		goto out;
 702	}
 703
 704	mapped_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
 705	if (!pte) {
 706		walk->action = ACTION_AGAIN;
 707		return 0;
 708	}
 709	for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) {
 710		max_nr = (end - addr) >> PAGE_SHIFT;
 711		nr = 1;
 712		ptent = ptep_get(pte);
 713		if (pte_none(ptent))
 714			continue;
 715		if (!pte_present(ptent)) {
 716			const softleaf_t entry = softleaf_from_pte(ptent);
 717
 718			if (softleaf_is_migration(entry))
 719				qp->nr_failed++;
 720			continue;
 721		}
 722		folio = vm_normal_folio(vma, addr, ptent);
 723		if (!folio || folio_is_zone_device(folio))
 724			continue;
 725		if (folio_test_large(folio) && max_nr != 1)
 726			nr = folio_pte_batch(folio, pte, ptent, max_nr);
 727		/*
 728		 * vm_normal_folio() filters out zero pages, but there might
 729		 * still be reserved folios to skip, perhaps in a VDSO.
 730		 */
 731		if (folio_test_reserved(folio))
 732			continue;
 733		if (!queue_folio_required(folio, qp))
 734			continue;
 735		if (folio_test_large(folio)) {
 736			/*
 737			 * A large folio can only be isolated from LRU once,
 738			 * but may be mapped by many PTEs (and Copy-On-Write may
 739			 * intersperse PTEs of other, order 0, folios).  This is
 740			 * a common case, so don't mistake it for failure (but
 741			 * there can be other cases of multi-mapped pages which
 742			 * this quick check does not help to filter out - and a
 743			 * search of the pagelist might grow to be prohibitive).
 744			 *
 745			 * migrate_pages(&pagelist) returns nr_failed folios, so
 746			 * check "large" now so that queue_pages_range() returns
 747			 * a comparable nr_failed folios.  This does imply that
 748			 * if folio could not be isolated for some racy reason
 749			 * at its first PTE, later PTEs will not give it another
 750			 * chance of isolation; but keeps the accounting simple.
 751			 */
 752			if (folio == qp->large)
 753				continue;
 754			qp->large = folio;
 755		}
 756		if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
 757		    !vma_migratable(vma) ||
 758		    !migrate_folio_add(folio, qp->pagelist, flags)) {
 759			qp->nr_failed += nr;
 760			if (strictly_unmovable(flags))
 761				break;
 762		}
 763	}
 764	pte_unmap_unlock(mapped_pte, ptl);
 765	cond_resched();
 766out:
 767	if (qp->nr_failed && strictly_unmovable(flags))
 768		return -EIO;
 769	return 0;
 770}
 771
 772static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask,
 773			       unsigned long addr, unsigned long end,
 774			       struct mm_walk *walk)
 775{
 776#ifdef CONFIG_HUGETLB_PAGE
 777	struct queue_pages *qp = walk->private;
 778	unsigned long flags = qp->flags;
 779	struct folio *folio;
 780	spinlock_t *ptl;
 781	pte_t ptep;
 782
 783	ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
 784	ptep = huge_ptep_get(walk->mm, addr, pte);
 785	if (!pte_present(ptep)) {
 786		if (!huge_pte_none(ptep)) {
 787			const softleaf_t entry = softleaf_from_pte(ptep);
 788
 789			if (unlikely(softleaf_is_migration(entry)))
 790				qp->nr_failed++;
 791		}
 792
 793		goto unlock;
 794	}
 795	folio = pfn_folio(pte_pfn(ptep));
 796	if (!queue_folio_required(folio, qp))
 797		goto unlock;
 798	if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
 799	    !vma_migratable(walk->vma)) {
 800		qp->nr_failed++;
 801		goto unlock;
 802	}
 803	/*
 804	 * Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio.
 805	 * Choosing not to migrate a shared folio is not counted as a failure.
 806	 *
 807	 * See folio_maybe_mapped_shared() on possible imprecision when we
 808	 * cannot easily detect if a folio is shared.
 809	 */
 810	if ((flags & MPOL_MF_MOVE_ALL) ||
 811	    (!folio_maybe_mapped_shared(folio) && !hugetlb_pmd_shared(pte)))
 812		if (!folio_isolate_hugetlb(folio, qp->pagelist))
 813			qp->nr_failed++;
 814unlock:
 815	spin_unlock(ptl);
 816	if (qp->nr_failed && strictly_unmovable(flags))
 817		return -EIO;
 818#endif
 819	return 0;
 820}
 821
 822#ifdef CONFIG_NUMA_BALANCING
 823/**
 824 * folio_can_map_prot_numa() - check whether the folio can map prot numa
 825 * @folio: The folio whose mapping considered for being made NUMA hintable
 826 * @vma: The VMA that the folio belongs to.
 827 * @is_private_single_threaded: Is this a single-threaded private VMA or not
 828 *
 829 * This function checks to see if the folio actually indicates that
 830 * we need to make the mapping one which causes a NUMA hinting fault,
 831 * as there are cases where it's simply unnecessary, and the folio's
 832 * access time is adjusted for memory tiering if prot numa needed.
 833 *
 834 * Return: True if the mapping of the folio needs to be changed, false otherwise.
 835 */
 836bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma,
 837		bool is_private_single_threaded)
 838{
 839	int nid;
 840
 841	if (!folio || folio_is_zone_device(folio) || folio_test_ksm(folio))
 842		return false;
 843
 844	/* Also skip shared copy-on-write folios */
 845	if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio))
 846		return false;
 847
 848	/* Folios are pinned and can't be migrated */
 849	if (folio_maybe_dma_pinned(folio))
 850		return false;
 851
 852	/*
 853	 * While migration can move some dirty folios,
 854	 * it cannot move them all from MIGRATE_ASYNC
 855	 * context.
 856	 */
 857	if (folio_is_file_lru(folio) && folio_test_dirty(folio))
 858		return false;
 859
 860	/*
 861	 * Don't mess with PTEs if folio is already on the node
 862	 * a single-threaded process is running on.
 863	 */
 864	nid = folio_nid(folio);
 865	if (is_private_single_threaded && (nid == numa_node_id()))
 866		return false;
 867
 868	/*
 869	 * Skip scanning top tier node if normal numa
 870	 * balancing is disabled
 871	 */
 872	if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
 873	    node_is_toptier(nid))
 874		return false;
 875
 876	if (folio_use_access_time(folio))
 877		folio_xchg_access_time(folio, jiffies_to_msecs(jiffies));
 878
 879	return true;
 880}
 881
 882/*
 883 * This is used to mark a range of virtual addresses to be inaccessible.
 884 * These are later cleared by a NUMA hinting fault. Depending on these
 885 * faults, pages may be migrated for better NUMA placement.
 886 *
 887 * This is assuming that NUMA faults are handled using PROT_NONE. If
 888 * an architecture makes a different choice, it will need further
 889 * changes to the core.
 890 */
 891unsigned long change_prot_numa(struct vm_area_struct *vma,
 892			unsigned long addr, unsigned long end)
 893{
 894	struct mmu_gather tlb;
 895	long nr_updated;
 896
 897	tlb_gather_mmu(&tlb, vma->vm_mm);
 898
 899	nr_updated = change_protection(&tlb, vma, addr, end, MM_CP_PROT_NUMA);
 900	if (nr_updated > 0) {
 901		count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
 902		count_memcg_events_mm(vma->vm_mm, NUMA_PTE_UPDATES, nr_updated);
 903	}
 904
 905	tlb_finish_mmu(&tlb);
 906
 907	return nr_updated;
 908}
 909#endif /* CONFIG_NUMA_BALANCING */
 910
 911static int queue_pages_test_walk(unsigned long start, unsigned long end,
 912				struct mm_walk *walk)
 913{
 914	struct vm_area_struct *next, *vma = walk->vma;
 915	struct queue_pages *qp = walk->private;
 916	unsigned long flags = qp->flags;
 917
 918	/* range check first */
 919	VM_BUG_ON_VMA(!range_in_vma(vma, start, end), vma);
 920
 921	if (!qp->first) {
 922		qp->first = vma;
 923		if (!(flags & MPOL_MF_DISCONTIG_OK) &&
 924			(qp->start < vma->vm_start))
 925			/* hole at head side of range */
 926			return -EFAULT;
 927	}
 928	next = find_vma(vma->vm_mm, vma->vm_end);
 929	if (!(flags & MPOL_MF_DISCONTIG_OK) &&
 930		((vma->vm_end < qp->end) &&
 931		(!next || vma->vm_end < next->vm_start)))
 932		/* hole at middle or tail of range */
 933		return -EFAULT;
 934
 935	/*
 936	 * Need check MPOL_MF_STRICT to return -EIO if possible
 937	 * regardless of vma_migratable
 938	 */
 939	if (!vma_migratable(vma) &&
 940	    !(flags & MPOL_MF_STRICT))
 941		return 1;
 942
 943	/*
 944	 * Check page nodes, and queue pages to move, in the current vma.
 945	 * But if no moving, and no strict checking, the scan can be skipped.
 946	 */
 947	if (flags & (MPOL_MF_STRICT | MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
 948		return 0;
 949	return 1;
 950}
 951
 952static const struct mm_walk_ops queue_pages_walk_ops = {
 953	.hugetlb_entry		= queue_folios_hugetlb,
 954	.pmd_entry		= queue_folios_pte_range,
 955	.test_walk		= queue_pages_test_walk,
 956	.walk_lock		= PGWALK_RDLOCK,
 957};
 958
 959static const struct mm_walk_ops queue_pages_lock_vma_walk_ops = {
 960	.hugetlb_entry		= queue_folios_hugetlb,
 961	.pmd_entry		= queue_folios_pte_range,
 962	.test_walk		= queue_pages_test_walk,
 963	.walk_lock		= PGWALK_WRLOCK,
 964};
 965
 966/*
 967 * Walk through page tables and collect pages to be migrated.
 968 *
 969 * If pages found in a given range are not on the required set of @nodes,
 970 * and migration is allowed, they are isolated and queued to @pagelist.
 971 *
 972 * queue_pages_range() may return:
 973 * 0 - all pages already on the right node, or successfully queued for moving
 974 *     (or neither strict checking nor moving requested: only range checking).
 975 * >0 - this number of misplaced folios could not be queued for moving
 976 *      (a hugetlbfs page or a transparent huge page being counted as 1).
 977 * -EIO - a misplaced page found, when MPOL_MF_STRICT specified without MOVEs.
 978 * -EFAULT - a hole in the memory range, when MPOL_MF_DISCONTIG_OK unspecified.
 979 */
 980static long
 981queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
 982		nodemask_t *nodes, unsigned long flags,
 983		struct list_head *pagelist)
 984{
 985	int err;
 986	struct queue_pages qp = {
 987		.pagelist = pagelist,
 988		.flags = flags,
 989		.nmask = nodes,
 990		.start = start,
 991		.end = end,
 992		.first = NULL,
 993	};
 994	const struct mm_walk_ops *ops = (flags & MPOL_MF_WRLOCK) ?
 995			&queue_pages_lock_vma_walk_ops : &queue_pages_walk_ops;
 996
 997	err = walk_page_range(mm, start, end, ops, &qp);
 998
 999	if (!qp.first)
1000		/* whole range in hole */
1001		err = -EFAULT;
1002
1003	return err ? : qp.nr_failed;
1004}
1005
1006/*
1007 * Apply policy to a single VMA
1008 * This must be called with the mmap_lock held for writing.
1009 */
1010static int vma_replace_policy(struct vm_area_struct *vma,
1011				struct mempolicy *pol)
1012{
1013	int err;
1014	struct mempolicy *old;
1015	struct mempolicy *new;
1016
1017	vma_assert_write_locked(vma);
1018
1019	new = mpol_dup(pol);
1020	if (IS_ERR(new))
1021		return PTR_ERR(new);
1022
1023	if (vma->vm_ops && vma->vm_ops->set_policy) {
1024		err = vma->vm_ops->set_policy(vma, new);
1025		if (err)
1026			goto err_out;
1027	}
1028
1029	old = vma->vm_policy;
1030	WRITE_ONCE(vma->vm_policy, new); /* protected by mmap_lock */
1031	mpol_put(old);
1032
1033	return 0;
1034 err_out:
1035	mpol_put(new);
1036	return err;
1037}
1038
1039/* Split or merge the VMA (if required) and apply the new policy */
1040static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma,
1041		struct vm_area_struct **prev, unsigned long start,
1042		unsigned long end, struct mempolicy *new_pol)
1043{
1044	unsigned long vmstart, vmend;
1045
1046	vmend = min(end, vma->vm_end);
1047	if (start > vma->vm_start) {
1048		*prev = vma;
1049		vmstart = start;
1050	} else {
1051		vmstart = vma->vm_start;
1052	}
1053
1054	if (mpol_equal(vma->vm_policy, new_pol)) {
1055		*prev = vma;
1056		return 0;
1057	}
1058
1059	vma =  vma_modify_policy(vmi, *prev, vma, vmstart, vmend, new_pol);
1060	if (IS_ERR(vma))
1061		return PTR_ERR(vma);
1062
1063	*prev = vma;
1064	return vma_replace_policy(vma, new_pol);
1065}
1066
1067/* Set the process memory policy */
1068static long do_set_mempolicy(unsigned short mode, unsigned short flags,
1069			     nodemask_t *nodes)
1070{
1071	struct mempolicy *new, *old;
1072	NODEMASK_SCRATCH(scratch);
1073	int ret;
1074
1075	if (!scratch)
1076		return -ENOMEM;
1077
1078	new = mpol_new(mode, flags, nodes);
1079	if (IS_ERR(new)) {
1080		ret = PTR_ERR(new);
1081		goto out;
1082	}
1083
1084	task_lock(current);
1085	ret = mpol_set_nodemask(new, nodes, scratch);
1086	if (ret) {
1087		task_unlock(current);
1088		mpol_put(new);
1089		goto out;
1090	}
1091
1092	old = current->mempolicy;
1093	current->mempolicy = new;
1094	if (new && (new->mode == MPOL_INTERLEAVE ||
1095		    new->mode == MPOL_WEIGHTED_INTERLEAVE)) {
1096		current->il_prev = MAX_NUMNODES-1;
1097		current->il_weight = 0;
1098	}
1099	task_unlock(current);
1100	mpol_put(old);
1101	ret = 0;
1102out:
1103	NODEMASK_SCRATCH_FREE(scratch);
1104	return ret;
1105}
1106
1107/*
1108 * Return nodemask for policy for get_mempolicy() query
1109 *
1110 * Called with task's alloc_lock held
1111 */
1112static void get_policy_nodemask(struct mempolicy *pol, nodemask_t *nodes)
1113{
1114	nodes_clear(*nodes);
1115	if (pol == &default_policy)
1116		return;
1117
1118	switch (pol->mode) {
1119	case MPOL_BIND:
1120	case MPOL_INTERLEAVE:
1121	case MPOL_PREFERRED:
1122	case MPOL_PREFERRED_MANY:
1123	case MPOL_WEIGHTED_INTERLEAVE:
1124		*nodes = pol->nodes;
1125		break;
1126	case MPOL_LOCAL:
1127		/* return empty node mask for local allocation */
1128		break;
1129	default:
1130		BUG();
1131	}
1132}
1133
1134static int lookup_node(struct mm_struct *mm, unsigned long addr)
1135{
1136	struct page *p = NULL;
1137	int ret;
1138
1139	ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p);
1140	if (ret > 0) {
1141		ret = page_to_nid(p);
1142		put_page(p);
1143	}
1144	return ret;
1145}
1146
1147/* Retrieve NUMA policy */
1148static long do_get_mempolicy(int *policy, nodemask_t *nmask,
1149			     unsigned long addr, unsigned long flags)
1150{
1151	int err;
1152	struct mm_struct *mm = current->mm;
1153	struct vm_area_struct *vma = NULL;
1154	struct mempolicy *pol = current->mempolicy, *pol_refcount = NULL;
1155
1156	if (flags &
1157		~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
1158		return -EINVAL;
1159
1160	if (flags & MPOL_F_MEMS_ALLOWED) {
1161		if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
1162			return -EINVAL;
1163		*policy = 0;	/* just so it's initialized */
1164		task_lock(current);
1165		*nmask  = cpuset_current_mems_allowed;
1166		task_unlock(current);
1167		return 0;
1168	}
1169
1170	if (flags & MPOL_F_ADDR) {
1171		pgoff_t ilx;		/* ignored here */
1172		/*
1173		 * Do NOT fall back to task policy if the
1174		 * vma/shared policy at addr is NULL.  We
1175		 * want to return MPOL_DEFAULT in this case.
1176		 */
1177		mmap_read_lock(mm);
1178		vma = vma_lookup(mm, addr);
1179		if (!vma) {
1180			mmap_read_unlock(mm);
1181			return -EFAULT;
1182		}
1183		pol = __get_vma_policy(vma, addr, &ilx);
1184	} else if (addr)
1185		return -EINVAL;
1186
1187	if (!pol)
1188		pol = &default_policy;	/* indicates default behavior */
1189
1190	if (flags & MPOL_F_NODE) {
1191		if (flags & MPOL_F_ADDR) {
1192			/*
1193			 * Take a refcount on the mpol, because we are about to
1194			 * drop the mmap_lock, after which only "pol" remains
1195			 * valid, "vma" is stale.
1196			 */
1197			pol_refcount = pol;
1198			vma = NULL;
1199			mpol_get(pol);
1200			mmap_read_unlock(mm);
1201			err = lookup_node(mm, addr);
1202			if (err < 0)
1203				goto out;
1204			*policy = err;
1205		} else if (pol == current->mempolicy &&
1206				pol->mode == MPOL_INTERLEAVE) {
1207			*policy = next_node_in(current->il_prev, pol->nodes);
1208		} else if (pol == current->mempolicy &&
1209				pol->mode == MPOL_WEIGHTED_INTERLEAVE) {
1210			if (current->il_weight)
1211				*policy = current->il_prev;
1212			else
1213				*policy = next_node_in(current->il_prev,
1214						       pol->nodes);
1215		} else {
1216			err = -EINVAL;
1217			goto out;
1218		}
1219	} else {
1220		*policy = pol == &default_policy ? MPOL_DEFAULT :
1221						pol->mode;
1222		/*
1223		 * Internal mempolicy flags must be masked off before exposing
1224		 * the policy to userspace.
1225		 */
1226		*policy |= (pol->flags & MPOL_MODE_FLAGS);
1227	}
1228
1229	err = 0;
1230	if (nmask) {
1231		if (mpol_store_user_nodemask(pol)) {
1232			*nmask = pol->w.user_nodemask;
1233		} else {
1234			task_lock(current);
1235			get_policy_nodemask(pol, nmask);
1236			task_unlock(current);
1237		}
1238	}
1239
1240 out:
1241	mpol_cond_put(pol);
1242	if (vma)
1243		mmap_read_unlock(mm);
1244	if (pol_refcount)
1245		mpol_put(pol_refcount);
1246	return err;
1247}
1248
1249#ifdef CONFIG_MIGRATION
1250static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
1251				unsigned long flags)
1252{
1253	/*
1254	 * Unless MPOL_MF_MOVE_ALL, we try to avoid migrating a shared folio.
1255	 * Choosing not to migrate a shared folio is not counted as a failure.
1256	 *
1257	 * See folio_maybe_mapped_shared() on possible imprecision when we
1258	 * cannot easily detect if a folio is shared.
1259	 */
1260	if ((flags & MPOL_MF_MOVE_ALL) || !folio_maybe_mapped_shared(folio)) {
1261		if (folio_isolate_lru(folio)) {
1262			list_add_tail(&folio->lru, foliolist);
1263			node_stat_mod_folio(folio,
1264				NR_ISOLATED_ANON + folio_is_file_lru(folio),
1265				folio_nr_pages(folio));
1266		} else {
1267			/*
1268			 * Non-movable folio may reach here.  And, there may be
1269			 * temporary off LRU folios or non-LRU movable folios.
1270			 * Treat them as unmovable folios since they can't be
1271			 * isolated, so they can't be moved at the moment.
1272			 */
1273			return false;
1274		}
1275	}
1276	return true;
1277}
1278
1279/*
1280 * Migrate pages from one node to a target node.
1281 * Returns error or the number of pages not migrated.
1282 */
1283static long migrate_to_node(struct mm_struct *mm, int source, int dest,
1284			    int flags)
1285{
1286	nodemask_t nmask;
1287	struct vm_area_struct *vma;
1288	LIST_HEAD(pagelist);
1289	long nr_failed;
1290	long err = 0;
1291	struct migration_target_control mtc = {
1292		.nid = dest,
1293		.gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
1294		.reason = MR_SYSCALL,
1295	};
1296
1297	nodes_clear(nmask);
1298	node_set(source, nmask);
1299
1300	VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
1301
1302	mmap_read_lock(mm);
1303	vma = find_vma(mm, 0);
1304	if (unlikely(!vma)) {
1305		mmap_read_unlock(mm);
1306		return 0;
1307	}
1308
1309	/*
1310	 * This does not migrate the range, but isolates all pages that
1311	 * need migration.  Between passing in the full user address
1312	 * space range and MPOL_MF_DISCONTIG_OK, this call cannot fail,
1313	 * but passes back the count of pages which could not be isolated.
1314	 */
1315	nr_failed = queue_pages_range(mm, vma->vm_start, mm->task_size, &nmask,
1316				      flags | MPOL_MF_DISCONTIG_OK, &pagelist);
1317	mmap_read_unlock(mm);
1318
1319	if (!list_empty(&pagelist)) {
1320		err = migrate_pages(&pagelist, alloc_migration_target, NULL,
1321			(unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL, NULL);
1322		if (err)
1323			putback_movable_pages(&pagelist);
1324	}
1325
1326	if (err >= 0)
1327		err += nr_failed;
1328	return err;
1329}
1330
1331/*
1332 * Move pages between the two nodesets so as to preserve the physical
1333 * layout as much as possible.
1334 *
1335 * Returns the number of page that could not be moved.
1336 */
1337int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1338		     const nodemask_t *to, int flags)
1339{
1340	long nr_failed = 0;
1341	long err = 0;
1342	nodemask_t tmp;
1343
1344	lru_cache_disable();
1345
1346	/*
1347	 * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
1348	 * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
1349	 * bit in 'tmp', and return that <source, dest> pair for migration.
1350	 * The pair of nodemasks 'to' and 'from' define the map.
1351	 *
1352	 * If no pair of bits is found that way, fallback to picking some
1353	 * pair of 'source' and 'dest' bits that are not the same.  If the
1354	 * 'source' and 'dest' bits are the same, this represents a node
1355	 * that will be migrating to itself, so no pages need move.
1356	 *
1357	 * If no bits are left in 'tmp', or if all remaining bits left
1358	 * in 'tmp' correspond to the same bit in 'to', return false
1359	 * (nothing left to migrate).
1360	 *
1361	 * This lets us pick a pair of nodes to migrate between, such that
1362	 * if possible the dest node is not already occupied by some other
1363	 * source node, minimizing the risk of overloading the memory on a
1364	 * node that would happen if we migrated incoming memory to a node
1365	 * before migrating outgoing memory source that same node.
1366	 *
1367	 * A single scan of tmp is sufficient.  As we go, we remember the
1368	 * most recent <s, d> pair that moved (s != d).  If we find a pair
1369	 * that not only moved, but what's better, moved to an empty slot
1370	 * (d is not set in tmp), then we break out then, with that pair.
1371	 * Otherwise when we finish scanning from_tmp, we at least have the
1372	 * most recent <s, d> pair that moved.  If we get all the way through
1373	 * the scan of tmp without finding any node that moved, much less
1374	 * moved to an empty node, then there is nothing left worth migrating.
1375	 */
1376
1377	tmp = *from;
1378	while (!nodes_empty(tmp)) {
1379		int s, d;
1380		int source = NUMA_NO_NODE;
1381		int dest = 0;
1382
1383		for_each_node_mask(s, tmp) {
1384
1385			/*
1386			 * do_migrate_pages() tries to maintain the relative
1387			 * node relationship of the pages established between
1388			 * threads and memory areas.
1389                         *
1390			 * However if the number of source nodes is not equal to
1391			 * the number of destination nodes we can not preserve
1392			 * this node relative relationship.  In that case, skip
1393			 * copying memory from a node that is in the destination
1394			 * mask.
1395			 *
1396			 * Example: [2,3,4] -> [3,4,5] moves everything.
1397			 *          [0-7] - > [3,4,5] moves only 0,1,2,6,7.
1398			 */
1399
1400			if ((nodes_weight(*from) != nodes_weight(*to)) &&
1401						(node_isset(s, *to)))
1402				continue;
1403
1404			d = node_remap(s, *from, *to);
1405			if (s == d)
1406				continue;
1407
1408			source = s;	/* Node moved. Memorize */
1409			dest = d;
1410
1411			/* dest not in remaining from nodes? */
1412			if (!node_isset(dest, tmp))
1413				break;
1414		}
1415		if (source == NUMA_NO_NODE)
1416			break;
1417
1418		node_clear(source, tmp);
1419		err = migrate_to_node(mm, source, dest, flags);
1420		if (err > 0)
1421			nr_failed += err;
1422		if (err < 0)
1423			break;
1424	}
1425
1426	lru_cache_enable();
1427	if (err < 0)
1428		return err;
1429	return (nr_failed < INT_MAX) ? nr_failed : INT_MAX;
1430}
1431
1432/*
1433 * Allocate a new folio for page migration, according to NUMA mempolicy.
1434 */
1435static struct folio *alloc_migration_target_by_mpol(struct folio *src,
1436						    unsigned long private)
1437{
1438	struct migration_mpol *mmpol = (struct migration_mpol *)private;
1439	struct mempolicy *pol = mmpol->pol;
1440	pgoff_t ilx = mmpol->ilx;
1441	unsigned int order;
1442	int nid = numa_node_id();
1443	gfp_t gfp;
1444
1445	order = folio_order(src);
1446	ilx += src->index >> order;
1447
1448	if (folio_test_hugetlb(src)) {
1449		nodemask_t *nodemask;
1450		struct hstate *h;
1451
1452		h = folio_hstate(src);
1453		gfp = htlb_alloc_mask(h);
1454		nodemask = policy_nodemask(gfp, pol, ilx, &nid);
1455		return alloc_hugetlb_folio_nodemask(h, nid, nodemask, gfp,
1456				htlb_allow_alloc_fallback(MR_MEMPOLICY_MBIND));
1457	}
1458
1459	if (folio_test_large(src))
1460		gfp = GFP_TRANSHUGE;
1461	else
1462		gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL | __GFP_COMP;
1463
1464	return folio_alloc_mpol(gfp, order, pol, ilx, nid);
1465}
1466#else
1467
1468static bool migrate_folio_add(struct folio *folio, struct list_head *foliolist,
1469				unsigned long flags)
1470{
1471	return false;
1472}
1473
1474int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
1475		     const nodemask_t *to, int flags)
1476{
1477	return -ENOSYS;
1478}
1479
1480static struct folio *alloc_migration_target_by_mpol(struct folio *src,
1481						    unsigned long private)
1482{
1483	return NULL;
1484}
1485#endif
1486
1487static long do_mbind(unsigned long start, unsigned long len,
1488		     unsigned short mode, unsigned short mode_flags,
1489		     nodemask_t *nmask, unsigned long flags)
1490{
1491	struct mm_struct *mm = current->mm;
1492	struct vm_area_struct *vma, *prev;
1493	struct vma_iterator vmi;
1494	struct migration_mpol mmpol;
1495	struct mempolicy *new;
1496	unsigned long end;
1497	long err;
1498	long nr_failed;
1499	LIST_HEAD(pagelist);
1500
1501	if (flags & ~(unsigned long)MPOL_MF_VALID)
1502		return -EINVAL;
1503	if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1504		return -EPERM;
1505
1506	if (start & ~PAGE_MASK)
1507		return -EINVAL;
1508
1509	if (mode == MPOL_DEFAULT)
1510		flags &= ~MPOL_MF_STRICT;
1511
1512	len = PAGE_ALIGN(len);
1513	end = start + len;
1514
1515	if (end < start)
1516		return -EINVAL;
1517	if (end == start)
1518		return 0;
1519
1520	new = mpol_new(mode, mode_flags, nmask);
1521	if (IS_ERR(new))
1522		return PTR_ERR(new);
1523
1524	/*
1525	 * If we are using the default policy then operation
1526	 * on discontinuous address spaces is okay after all
1527	 */
1528	if (!new)
1529		flags |= MPOL_MF_DISCONTIG_OK;
1530
1531	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
1532		lru_cache_disable();
1533	{
1534		NODEMASK_SCRATCH(scratch);
1535		if (scratch) {
1536			mmap_write_lock(mm);
1537			err = mpol_set_nodemask(new, nmask, scratch);
1538			if (err)
1539				mmap_write_unlock(mm);
1540		} else
1541			err = -ENOMEM;
1542		NODEMASK_SCRATCH_FREE(scratch);
1543	}
1544	if (err)
1545		goto mpol_out;
1546
1547	/*
1548	 * Lock the VMAs before scanning for pages to migrate,
1549	 * to ensure we don't miss a concurrently inserted page.
1550	 */
1551	nr_failed = queue_pages_range(mm, start, end, nmask,
1552			flags | MPOL_MF_INVERT | MPOL_MF_WRLOCK, &pagelist);
1553
1554	if (nr_failed < 0) {
1555		err = nr_failed;
1556		nr_failed = 0;
1557	} else {
1558		vma_iter_init(&vmi, mm, start);
1559		prev = vma_prev(&vmi);
1560		for_each_vma_range(vmi, vma, end) {
1561			err = mbind_range(&vmi, vma, &prev, start, end, new);
1562			if (err)
1563				break;
1564		}
1565	}
1566
1567	if (!err && !list_empty(&pagelist)) {
1568		/* Convert MPOL_DEFAULT's NULL to task or default policy */
1569		if (!new) {
1570			new = get_task_policy(current);
1571			mpol_get(new);
1572		}
1573		mmpol.pol = new;
1574		mmpol.ilx = 0;
1575
1576		/*
1577		 * In the interleaved case, attempt to allocate on exactly the
1578		 * targeted nodes, for the first VMA to be migrated; for later
1579		 * VMAs, the nodes will still be interleaved from the targeted
1580		 * nodemask, but one by one may be selected differently.
1581		 */
1582		if (new->mode == MPOL_INTERLEAVE ||
1583		    new->mode == MPOL_WEIGHTED_INTERLEAVE) {
1584			struct folio *folio;
1585			unsigned int order;
1586			unsigned long addr = -EFAULT;
1587
1588			list_for_each_entry(folio, &pagelist, lru) {
1589				if (!folio_test_ksm(folio))
1590					break;
1591			}
1592			if (!list_entry_is_head(folio, &pagelist, lru)) {
1593				vma_iter_init(&vmi, mm, start);
1594				for_each_vma_range(vmi, vma, end) {
1595					addr = page_address_in_vma(folio,
1596						folio_page(folio, 0), vma);
1597					if (addr != -EFAULT)
1598						break;
1599				}
1600			}
1601			if (addr != -EFAULT) {
1602				order = folio_order(folio);
1603				/* We already know the pol, but not the ilx */
1604				mpol_cond_put(get_vma_policy(vma, addr, order,
1605							     &mmpol.ilx));
1606				/* Set base from which to increment by index */
1607				mmpol.ilx -= folio->index >> order;
1608			}
1609		}
1610	}
1611
1612	mmap_write_unlock(mm);
1613
1614	if (!err && !list_empty(&pagelist)) {
1615		nr_failed |= migrate_pages(&pagelist,
1616				alloc_migration_target_by_mpol, NULL,
1617				(unsigned long)&mmpol, MIGRATE_SYNC,
1618				MR_MEMPOLICY_MBIND, NULL);
1619	}
1620
1621	if (nr_failed && (flags & MPOL_MF_STRICT))
1622		err = -EIO;
1623	if (!list_empty(&pagelist))
1624		putback_movable_pages(&pagelist);
1625mpol_out:
1626	mpol_put(new);
1627	if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
1628		lru_cache_enable();
1629	return err;
1630}
1631
1632/*
1633 * User space interface with variable sized bitmaps for nodelists.
1634 */
1635static int get_bitmap(unsigned long *mask, const unsigned long __user *nmask,
1636		      unsigned long maxnode)
1637{
1638	unsigned long nlongs = BITS_TO_LONGS(maxnode);
1639	int ret;
1640
1641	if (in_compat_syscall())
1642		ret = compat_get_bitmap(mask,
1643					(const compat_ulong_t __user *)nmask,
1644					maxnode);
1645	else
1646		ret = copy_from_user(mask, nmask,
1647				     nlongs * sizeof(unsigned long));
1648
1649	if (ret)
1650		return -EFAULT;
1651
1652	if (maxnode % BITS_PER_LONG)
1653		mask[nlongs - 1] &= (1UL << (maxnode % BITS_PER_LONG)) - 1;
1654
1655	return 0;
1656}
1657
1658/* Copy a node mask from user space. */
1659static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
1660		     unsigned long maxnode)
1661{
1662	--maxnode;
1663	nodes_clear(*nodes);
1664	if (maxnode == 0 || !nmask)
1665		return 0;
1666	if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
1667		return -EINVAL;
1668
1669	/*
1670	 * When the user specified more nodes than supported just check
1671	 * if the non supported part is all zero, one word at a time,
1672	 * starting at the end.
1673	 */
1674	while (maxnode > MAX_NUMNODES) {
1675		unsigned long bits = min_t(unsigned long, maxnode, BITS_PER_LONG);
1676		unsigned long t;
1677
1678		if (get_bitmap(&t, &nmask[(maxnode - 1) / BITS_PER_LONG], bits))
1679			return -EFAULT;
1680
1681		if (maxnode - bits >= MAX_NUMNODES) {
1682			maxnode -= bits;
1683		} else {
1684			maxnode = MAX_NUMNODES;
1685			t &= ~((1UL << (MAX_NUMNODES % BITS_PER_LONG)) - 1);
1686		}
1687		if (t)
1688			return -EINVAL;
1689	}
1690
1691	return get_bitmap(nodes_addr(*nodes), nmask, maxnode);
1692}
1693
1694/* Copy a kernel node mask to user space */
1695static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
1696			      nodemask_t *nodes)
1697{
1698	unsigned long copy = ALIGN(maxnode-1, 64) / 8;
1699	unsigned int nbytes = BITS_TO_LONGS(nr_node_ids) * sizeof(long);
1700	bool compat = in_compat_syscall();
1701
1702	if (compat)
1703		nbytes = BITS_TO_COMPAT_LONGS(nr_node_ids) * sizeof(compat_long_t);
1704
1705	if (copy > nbytes) {
1706		if (copy > PAGE_SIZE)
1707			return -EINVAL;
1708		if (clear_user((char __user *)mask + nbytes, copy - nbytes))
1709			return -EFAULT;
1710		copy = nbytes;
1711		maxnode = nr_node_ids;
1712	}
1713
1714	if (compat)
1715		return compat_put_bitmap((compat_ulong_t __user *)mask,
1716					 nodes_addr(*nodes), maxnode);
1717
1718	return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
1719}
1720
1721/* Basic parameter sanity check used by both mbind() and set_mempolicy() */
1722static inline int sanitize_mpol_flags(int *mode, unsigned short *flags)
1723{
1724	*flags = *mode & MPOL_MODE_FLAGS;
1725	*mode &= ~MPOL_MODE_FLAGS;
1726
1727	if ((unsigned int)(*mode) >=  MPOL_MAX)
1728		return -EINVAL;
1729	if ((*flags & MPOL_F_STATIC_NODES) && (*flags & MPOL_F_RELATIVE_NODES))
1730		return -EINVAL;
1731	if (*flags & MPOL_F_NUMA_BALANCING) {
1732		if (*mode == MPOL_BIND || *mode == MPOL_PREFERRED_MANY)
1733			*flags |= (MPOL_F_MOF | MPOL_F_MORON);
1734		else
1735			return -EINVAL;
1736	}
1737	return 0;
1738}
1739
1740static long kernel_mbind(unsigned long start, unsigned long len,
1741			 unsigned long mode, const unsigned long __user *nmask,
1742			 unsigned long maxnode, unsigned int flags)
1743{
1744	unsigned short mode_flags;
1745	nodemask_t nodes;
1746	int lmode = mode;
1747	int err;
1748
1749	start = untagged_addr(start);
1750	err = sanitize_mpol_flags(&lmode, &mode_flags);
1751	if (err)
1752		return err;
1753
1754	err = get_nodes(&nodes, nmask, maxnode);
1755	if (err)
1756		return err;
1757
1758	return do_mbind(start, len, lmode, mode_flags, &nodes, flags);
1759}
1760
1761SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, len,
1762		unsigned long, home_node, unsigned long, flags)
1763{
1764	struct mm_struct *mm = current->mm;
1765	struct vm_area_struct *vma, *prev;
1766	struct mempolicy *new, *old;
1767	unsigned long end;
1768	int err = -ENOENT;
1769	VMA_ITERATOR(vmi, mm, start);
1770
1771	start = untagged_addr(start);
1772	if (start & ~PAGE_MASK)
1773		return -EINVAL;
1774	/*
1775	 * flags is used for future extension if any.
1776	 */
1777	if (flags != 0)
1778		return -EINVAL;
1779
1780	/*
1781	 * Check home_node is online to avoid accessing uninitialized
1782	 * NODE_DATA.
1783	 */
1784	if (home_node >= MAX_NUMNODES || !node_online(home_node))
1785		return -EINVAL;
1786
1787	len = PAGE_ALIGN(len);
1788	end = start + len;
1789
1790	if (end < start)
1791		return -EINVAL;
1792	if (end == start)
1793		return 0;
1794	mmap_write_lock(mm);
1795	prev = vma_prev(&vmi);
1796	for_each_vma_range(vmi, vma, end) {
1797		/*
1798		 * If any vma in the range got policy other than MPOL_BIND
1799		 * or MPOL_PREFERRED_MANY we return error. We don't reset
1800		 * the home node for vmas we already updated before.
1801		 */
1802		old = vma_policy(vma);
1803		if (!old) {
1804			prev = vma;
1805			continue;
1806		}
1807		if (old->mode != MPOL_BIND && old->mode != MPOL_PREFERRED_MANY) {
1808			err = -EOPNOTSUPP;
1809			break;
1810		}
1811		new = mpol_dup(old);
1812		if (IS_ERR(new)) {
1813			err = PTR_ERR(new);
1814			break;
1815		}
1816
1817		vma_start_write(vma);
1818		new->home_node = home_node;
1819		err = mbind_range(&vmi, vma, &prev, start, end, new);
1820		mpol_put(new);
1821		if (err)
1822			break;
1823	}
1824	mmap_write_unlock(mm);
1825	return err;
1826}
1827
1828SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
1829		unsigned long, mode, const unsigned long __user *, nmask,
1830		unsigned long, maxnode, unsigned int, flags)
1831{
1832	return kernel_mbind(start, len, mode, nmask, maxnode, flags);
1833}
1834
1835/* Set the process memory policy */
1836static long kernel_set_mempolicy(int mode, const unsigned long __user *nmask,
1837				 unsigned long maxnode)
1838{
1839	unsigned short mode_flags;
1840	nodemask_t nodes;
1841	int lmode = mode;
1842	int err;
1843
1844	err = sanitize_mpol_flags(&lmode, &mode_flags);
1845	if (err)
1846		return err;
1847
1848	err = get_nodes(&nodes, nmask, maxnode);
1849	if (err)
1850		return err;
1851
1852	return do_set_mempolicy(lmode, mode_flags, &nodes);
1853}
1854
1855SYSCALL_DEFINE3(set_mempolicy, int, mode, const unsigned long __user *, nmask,
1856		unsigned long, maxnode)
1857{
1858	return kernel_set_mempolicy(mode, nmask, maxnode);
1859}
1860
1861static int kernel_migrate_pages(pid_t pid, unsigned long maxnode,
1862				const unsigned long __user *old_nodes,
1863				const unsigned long __user *new_nodes)
1864{
1865	struct mm_struct *mm = NULL;
1866	struct task_struct *task;
1867	nodemask_t task_nodes;
1868	int err;
1869	nodemask_t *old;
1870	nodemask_t *new;
1871	NODEMASK_SCRATCH(scratch);
1872
1873	if (!scratch)
1874		return -ENOMEM;
1875
1876	old = &scratch->mask1;
1877	new = &scratch->mask2;
1878
1879	err = get_nodes(old, old_nodes, maxnode);
1880	if (err)
1881		goto out;
1882
1883	err = get_nodes(new, new_nodes, maxnode);
1884	if (err)
1885		goto out;
1886
1887	/* Find the mm_struct */
1888	rcu_read_lock();
1889	task = pid ? find_task_by_vpid(pid) : current;
1890	if (!task) {
1891		rcu_read_unlock();
1892		err = -ESRCH;
1893		goto out;
1894	}
1895	get_task_struct(task);
1896
1897	err = -EINVAL;
1898
1899	/*
1900	 * Check if this process has the right to modify the specified process.
1901	 * Use the regular "ptrace_may_access()" checks.
1902	 */
1903	if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1904		rcu_read_unlock();
1905		err = -EPERM;
1906		goto out_put;
1907	}
1908	rcu_read_unlock();
1909
1910	task_nodes = cpuset_mems_allowed(task);
1911	/* Is the user allowed to access the target nodes? */
1912	if (!nodes_subset(*new, task_nodes) && !capable(CAP_SYS_NICE)) {
1913		err = -EPERM;
1914		goto out_put;
1915	}
1916
1917	task_nodes = cpuset_mems_allowed(current);
1918	nodes_and(*new, *new, task_nodes);
1919	if (nodes_empty(*new))
1920		goto out_put;
1921
1922	err = security_task_movememory(task);
1923	if (err)
1924		goto out_put;
1925
1926	mm = get_task_mm(task);
1927	put_task_struct(task);
1928
1929	if (!mm) {
1930		err = -EINVAL;
1931		goto out;
1932	}
1933
1934	err = do_migrate_pages(mm, old, new,
1935		capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);
1936
1937	mmput(mm);
1938out:
1939	NODEMASK_SCRATCH_FREE(scratch);
1940
1941	return err;
1942
1943out_put:
1944	put_task_struct(task);
1945	goto out;
1946}
1947
1948SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
1949		const unsigned long __user *, old_nodes,
1950		const unsigned long __user *, new_nodes)
1951{
1952	return kernel_migrate_pages(pid, maxnode, old_nodes, new_nodes);
1953}
1954
1955/* Retrieve NUMA policy */
1956static int kernel_get_mempolicy(int __user *policy,
1957				unsigned long __user *nmask,
1958				unsigned long maxnode,
1959				unsigned long addr,
1960				unsigned long flags)
1961{
1962	int err;
1963	int pval;
1964	nodemask_t nodes;
1965
1966	if (nmask != NULL && maxnode < nr_node_ids)
1967		return -EINVAL;
1968
1969	addr = untagged_addr(addr);
1970
1971	err = do_get_mempolicy(&pval, &nodes, addr, flags);
1972
1973	if (err)
1974		return err;
1975
1976	if (policy && put_user(pval, policy))
1977		return -EFAULT;
1978
1979	if (nmask)
1980		err = copy_nodes_to_user(nmask, maxnode, &nodes);
1981
1982	return err;
1983}
1984
1985SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
1986		unsigned long __user *, nmask, unsigned long, maxnode,
1987		unsigned long, addr, unsigned long, flags)
1988{
1989	return kernel_get_mempolicy(policy, nmask, maxnode, addr, flags);
1990}
1991
1992bool vma_migratable(struct vm_area_struct *vma)
1993{
1994	if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1995		return false;
1996
1997	/*
1998	 * DAX device mappings require predictable access latency, so avoid
1999	 * incurring periodic faults.
2000	 */
2001	if (vma_is_dax(vma))
2002		return false;
2003
2004	if (is_vm_hugetlb_page(vma) &&
2005		!hugepage_migration_supported(hstate_vma(vma)))
2006		return false;
2007
2008	/*
2009	 * Migration allocates pages in the highest zone. If we cannot
2010	 * do so then migration (at least from node to node) is not
2011	 * possible.
2012	 */
2013	if (vma->vm_file &&
2014		gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping))
2015			< policy_zone)
2016		return false;
2017	return true;
2018}
2019
2020struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
2021				   unsigned long addr, pgoff_t *ilx)
2022{
2023	*ilx = 0;
2024	return (vma->vm_ops && vma->vm_ops->get_policy) ?
2025		vma->vm_ops->get_policy(vma, addr, ilx) : vma->vm_policy;
2026}
2027
2028/*
2029 * get_vma_policy(@vma, @addr, @order, @ilx)
2030 * @vma: virtual memory area whose policy is sought
2031 * @addr: address in @vma for shared policy lookup
2032 * @order: 0, or appropriate huge_page_order for interleaving
2033 * @ilx: interleave index (output), for use only when MPOL_INTERLEAVE or
2034 *       MPOL_WEIGHTED_INTERLEAVE
2035 *
2036 * Returns effective policy for a VMA at specified address.
2037 * Falls back to current->mempolicy or system default policy, as necessary.
2038 * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
2039 * count--added by the get_policy() vm_op, as appropriate--to protect against
2040 * freeing by another task.  It is the caller's responsibility to free the
2041 * extra reference for shared policies.
2042 */
2043struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
2044				 unsigned long addr, int order, pgoff_t *ilx)
2045{
2046	struct mempolicy *pol;
2047
2048	pol = __get_vma_policy(vma, addr, ilx);
2049	if (!pol)
2050		pol = get_task_policy(current);
2051	if (pol->mode == MPOL_INTERLEAVE ||
2052	    pol->mode == MPOL_WEIGHTED_INTERLEAVE) {
2053		*ilx += vma->vm_pgoff >> order;
2054		*ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order);
2055	}
2056	return pol;
2057}
2058
2059bool vma_policy_mof(struct vm_area_struct *vma)
2060{
2061	struct mempolicy *pol;
2062
2063	if (vma->vm_ops && vma->vm_ops->get_policy) {
2064		bool ret = false;
2065		pgoff_t ilx;		/* ignored here */
2066
2067		pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx);
2068		if (pol && (pol->flags & MPOL_F_MOF))
2069			ret = true;
2070		mpol_cond_put(pol);
2071
2072		return ret;
2073	}
2074
2075	pol = vma->vm_policy;
2076	if (!pol)
2077		pol = get_task_policy(current);
2078
2079	return pol->flags & MPOL_F_MOF;
2080}
2081
2082bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone)
2083{
2084	enum zone_type dynamic_policy_zone = policy_zone;
2085
2086	BUG_ON(dynamic_policy_zone == ZONE_MOVABLE);
2087
2088	/*
2089	 * if policy->nodes has movable memory only,
2090	 * we apply policy when gfp_zone(gfp) = ZONE_MOVABLE only.
2091	 *
2092	 * policy->nodes is intersect with node_states[N_MEMORY].
2093	 * so if the following test fails, it implies
2094	 * policy->nodes has movable memory only.
2095	 */
2096	if (!nodes_intersects(policy->nodes, node_states[N_HIGH_MEMORY]))
2097		dynamic_policy_zone = ZONE_MOVABLE;
2098
2099	return zone >= dynamic_policy_zone;
2100}
2101
2102static unsigned int weighted_interleave_nodes(struct mempolicy *policy)
2103{
2104	unsigned int node;
2105	unsigned int cpuset_mems_cookie;
2106
2107retry:
2108	/* to prevent miscount use tsk->mems_allowed_seq to detect rebind */
2109	cpuset_mems_cookie = read_mems_allowed_begin();
2110	node = current->il_prev;
2111	if (!current->il_weight || !node_isset(node, policy->nodes)) {
2112		node = next_node_in(node, policy->nodes);
2113		if (read_mems_allowed_retry(cpuset_mems_cookie))
2114			goto retry;
2115		if (node == MAX_NUMNODES)
2116			return node;
2117		current->il_prev = node;
2118		current->il_weight = get_il_weight(node);
2119	}
2120	current->il_weight--;
2121	return node;
2122}
2123
2124/* Do dynamic interleaving for a process */
2125static unsigned int interleave_nodes(struct mempolicy *policy)
2126{
2127	unsigned int nid;
2128	unsigned int cpuset_mems_cookie;
2129
2130	/* to prevent miscount, use tsk->mems_allowed_seq to detect rebind */
2131	do {
2132		cpuset_mems_cookie = read_mems_allowed_begin();
2133		nid = next_node_in(current->il_prev, policy->nodes);
2134	} while (read_mems_allowed_retry(cpuset_mems_cookie));
2135
2136	if (nid < MAX_NUMNODES)
2137		current->il_prev = nid;
2138	return nid;
2139}
2140
2141/*
2142 * Depending on the memory policy provide a node from which to allocate the
2143 * next slab entry.
2144 */
2145unsigned int mempolicy_slab_node(void)
2146{
2147	struct mempolicy *policy;
2148	int node = numa_mem_id();
2149
2150	if (!in_task())
2151		return node;
2152
2153	policy = current->mempolicy;
2154	if (!policy)
2155		return node;
2156
2157	switch (policy->mode) {
2158	case MPOL_PREFERRED:
2159		return first_node(policy->nodes);
2160
2161	case MPOL_INTERLEAVE:
2162		return interleave_nodes(policy);
2163
2164	case MPOL_WEIGHTED_INTERLEAVE:
2165		return weighted_interleave_nodes(policy);
2166
2167	case MPOL_BIND:
2168	case MPOL_PREFERRED_MANY:
2169	{
2170		struct zoneref *z;
2171
2172		/*
2173		 * Follow bind policy behavior and start allocation at the
2174		 * first node.
2175		 */
2176		struct zonelist *zonelist;
2177		enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
2178		zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK];
2179		z = first_zones_zonelist(zonelist, highest_zoneidx,
2180							&policy->nodes);
2181		return zonelist_zone(z) ? zonelist_node_idx(z) : node;
2182	}
2183	case MPOL_LOCAL:
2184		return node;
2185
2186	default:
2187		BUG();
2188	}
2189}
2190
2191static unsigned int read_once_policy_nodemask(struct mempolicy *pol,
2192					      nodemask_t *mask)
2193{
2194	/*
2195	 * barrier stabilizes the nodemask locally so that it can be iterated
2196	 * over safely without concern for changes. Allocators validate node
2197	 * selection does not violate mems_allowed, so this is safe.
2198	 */
2199	barrier();
2200	memcpy(mask, &pol->nodes, sizeof(nodemask_t));
2201	barrier();
2202	return nodes_weight(*mask);
2203}
2204
2205static unsigned int weighted_interleave_nid(struct mempolicy *pol, pgoff_t ilx)
2206{
2207	struct weighted_interleave_state *state;
2208	nodemask_t nodemask;
2209	unsigned int target, nr_nodes;
2210	u8 *table = NULL;
2211	unsigned int weight_total = 0;
2212	u8 weight;
2213	int nid = 0;
2214
2215	nr_nodes = read_once_policy_nodemask(pol, &nodemask);
2216	if (!nr_nodes)
2217		return numa_node_id();
2218
2219	rcu_read_lock();
2220
2221	state = rcu_dereference(wi_state);
2222	/* Uninitialized wi_state means we should assume all weights are 1 */
2223	if (state)
2224		table = state->iw_table;
2225
2226	/* calculate the total weight */
2227	for_each_node_mask(nid, nodemask)
2228		weight_total += table ? table[nid] : 1;
2229
2230	/* Calculate the node offset based on totals */
2231	target = ilx % weight_total;
2232	nid = first_node(nodemask);
2233	while (target) {
2234		/* detect system default usage */
2235		weight = table ? table[nid] : 1;
2236		if (target < weight)
2237			break;
2238		target -= weight;
2239		nid = next_node_in(nid, nodemask);
2240	}
2241	rcu_read_unlock();
2242	return nid;
2243}
2244
2245/*
2246 * Do static interleaving for interleave index @ilx.  Returns the ilx'th
2247 * node in pol->nodes (starting from ilx=0), wrapping around if ilx
2248 * exceeds the number of present nodes.
2249 */
2250static unsigned int interleave_nid(struct mempolicy *pol, pgoff_t ilx)
2251{
2252	nodemask_t nodemask;
2253	unsigned int target, nnodes;
2254	int i;
2255	int nid;
2256
2257	nnodes = read_once_policy_nodemask(pol, &nodemask);
2258	if (!nnodes)
2259		return numa_node_id();
2260	target = ilx % nnodes;
2261	nid = first_node(nodemask);
2262	for (i = 0; i < target; i++)
2263		nid = next_node(nid, nodemask);
2264	return nid;
2265}
2266
2267/*
2268 * Return a nodemask representing a mempolicy for filtering nodes for
2269 * page allocation, together with preferred node id (or the input node id).
2270 */
2271static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol,
2272				   pgoff_t ilx, int *nid)
2273{
2274	nodemask_t *nodemask = NULL;
2275
2276	switch (pol->mode) {
2277	case MPOL_PREFERRED:
2278		/* Override input node id */
2279		*nid = first_node(pol->nodes);
2280		break;
2281	case MPOL_PREFERRED_MANY:
2282		nodemask = &pol->nodes;
2283		if (pol->home_node != NUMA_NO_NODE)
2284			*nid = pol->home_node;
2285		break;
2286	case MPOL_BIND:
2287		/* Restrict to nodemask (but not on lower zones) */
2288		if (apply_policy_zone(pol, gfp_zone(gfp)) &&
2289		    cpuset_nodemask_valid_mems_allowed(&pol->nodes))
2290			nodemask = &pol->nodes;
2291		if (pol->home_node != NUMA_NO_NODE)
2292			*nid = pol->home_node;
2293		/*
2294		 * __GFP_THISNODE shouldn't even be used with the bind policy
2295		 * because we might easily break the expectation to stay on the
2296		 * requested node and not break the policy.
2297		 */
2298		WARN_ON_ONCE(gfp & __GFP_THISNODE);
2299		break;
2300	case MPOL_INTERLEAVE:
2301		/* Override input node id */
2302		*nid = (ilx == NO_INTERLEAVE_INDEX) ?
2303			interleave_nodes(pol) : interleave_nid(pol, ilx);
2304		break;
2305	case MPOL_WEIGHTED_INTERLEAVE:
2306		*nid = (ilx == NO_INTERLEAVE_INDEX) ?
2307			weighted_interleave_nodes(pol) :
2308			weighted_interleave_nid(pol, ilx);
2309		break;
2310	}
2311
2312	return nodemask;
2313}
2314
2315#ifdef CONFIG_HUGETLBFS
2316/*
2317 * huge_node(@vma, @addr, @gfp_flags, @mpol)
2318 * @vma: virtual memory area whose policy is sought
2319 * @addr: address in @vma for shared policy lookup and interleave policy
2320 * @gfp_flags: for requested zone
2321 * @mpol: pointer to mempolicy pointer for reference counted mempolicy
2322 * @nodemask: pointer to nodemask pointer for 'bind' and 'prefer-many' policy
2323 *
2324 * Returns a nid suitable for a huge page allocation and a pointer
2325 * to the struct mempolicy for conditional unref after allocation.
2326 * If the effective policy is 'bind' or 'prefer-many', returns a pointer
2327 * to the mempolicy's @nodemask for filtering the zonelist.
2328 */
2329int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags,
2330		struct mempolicy **mpol, nodemask_t **nodemask)
2331{
2332	pgoff_t ilx;
2333	int nid;
2334
2335	nid = numa_node_id();
2336	*mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx);
2337	*nodemask = policy_nodemask(gfp_flags, *mpol, ilx, &nid);
2338	return nid;
2339}
2340
2341/*
2342 * init_nodemask_of_mempolicy
2343 *
2344 * If the current task's mempolicy is "default" [NULL], return 'false'
2345 * to indicate default policy.  Otherwise, extract the policy nodemask
2346 * for 'bind' or 'interleave' policy into the argument nodemask, or
2347 * initialize the argument nodemask to contain the single node for
2348 * 'preferred' or 'local' policy and return 'true' to indicate presence
2349 * of non-default mempolicy.
2350 *
2351 * We don't bother with reference counting the mempolicy [mpol_get/put]
2352 * because the current task is examining it's own mempolicy and a task's
2353 * mempolicy is only ever changed by the task itself.
2354 *
2355 * N.B., it is the caller's responsibility to free a returned nodemask.
2356 */
2357bool init_nodemask_of_mempolicy(nodemask_t *mask)
2358{
2359	struct mempolicy *mempolicy;
2360
2361	if (!(mask && current->mempolicy))
2362		return false;
2363
2364	task_lock(current);
2365	mempolicy = current->mempolicy;
2366	switch (mempolicy->mode) {
2367	case MPOL_PREFERRED:
2368	case MPOL_PREFERRED_MANY:
2369	case MPOL_BIND:
2370	case MPOL_INTERLEAVE:
2371	case MPOL_WEIGHTED_INTERLEAVE:
2372		*mask = mempolicy->nodes;
2373		break;
2374
2375	case MPOL_LOCAL:
2376		init_nodemask_of_node(mask, numa_node_id());
2377		break;
2378
2379	default:
2380		BUG();
2381	}
2382	task_unlock(current);
2383
2384	return true;
2385}
2386#endif
2387
2388/*
2389 * mempolicy_in_oom_domain
2390 *
2391 * If tsk's mempolicy is "bind", check for intersection between mask and
2392 * the policy nodemask. Otherwise, return true for all other policies
2393 * including "interleave", as a tsk with "interleave" policy may have
2394 * memory allocated from all nodes in system.
2395 *
2396 * Takes task_lock(tsk) to prevent freeing of its mempolicy.
2397 */
2398bool mempolicy_in_oom_domain(struct task_struct *tsk,
2399					const nodemask_t *mask)
2400{
2401	struct mempolicy *mempolicy;
2402	bool ret = true;
2403
2404	if (!mask)
2405		return ret;
2406
2407	task_lock(tsk);
2408	mempolicy = tsk->mempolicy;
2409	if (mempolicy && mempolicy->mode == MPOL_BIND)
2410		ret = nodes_intersects(mempolicy->nodes, *mask);
2411	task_unlock(tsk);
2412
2413	return ret;
2414}
2415
2416static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order,
2417						int nid, nodemask_t *nodemask)
2418{
2419	struct page *page;
2420	gfp_t preferred_gfp;
2421
2422	/*
2423	 * This is a two pass approach. The first pass will only try the
2424	 * preferred nodes but skip the direct reclaim and allow the
2425	 * allocation to fail, while the second pass will try all the
2426	 * nodes in system.
2427	 */
2428	preferred_gfp = gfp | __GFP_NOWARN;
2429	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2430	page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask);
2431	if (!page)
2432		page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL);
2433
2434	return page;
2435}
2436
2437/**
2438 * alloc_pages_mpol - Allocate pages according to NUMA mempolicy.
2439 * @gfp: GFP flags.
2440 * @order: Order of the page allocation.
2441 * @pol: Pointer to the NUMA mempolicy.
2442 * @ilx: Index for interleave mempolicy (also distinguishes alloc_pages()).
2443 * @nid: Preferred node (usually numa_node_id() but @mpol may override it).
2444 *
2445 * Return: The page on success or NULL if allocation fails.
2446 */
2447static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order,
2448		struct mempolicy *pol, pgoff_t ilx, int nid)
2449{
2450	nodemask_t *nodemask;
2451	struct page *page;
2452
2453	nodemask = policy_nodemask(gfp, pol, ilx, &nid);
2454
2455	if (pol->mode == MPOL_PREFERRED_MANY)
2456		return alloc_pages_preferred_many(gfp, order, nid, nodemask);
2457
2458	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
2459	    /* filter "hugepage" allocation, unless from alloc_pages() */
2460	    order == HPAGE_PMD_ORDER && ilx != NO_INTERLEAVE_INDEX) {
2461		/*
2462		 * For hugepage allocation and non-interleave policy which
2463		 * allows the current node (or other explicitly preferred
2464		 * node) we only try to allocate from the current/preferred
2465		 * node and don't fall back to other nodes, as the cost of
2466		 * remote accesses would likely offset THP benefits.
2467		 *
2468		 * If the policy is interleave or does not allow the current
2469		 * node in its nodemask, we allocate the standard way.
2470		 */
2471		if (pol->mode != MPOL_INTERLEAVE &&
2472		    pol->mode != MPOL_WEIGHTED_INTERLEAVE &&
2473		    (!nodemask || node_isset(nid, *nodemask))) {
2474			/*
2475			 * First, try to allocate THP only on local node, but
2476			 * don't reclaim unnecessarily, just compact.
2477			 */
2478			page = __alloc_frozen_pages_noprof(
2479				gfp | __GFP_THISNODE | __GFP_NORETRY, order,
2480				nid, NULL);
2481			if (page || !(gfp & __GFP_DIRECT_RECLAIM))
2482				return page;
2483			/*
2484			 * If hugepage allocations are configured to always
2485			 * synchronous compact or the vma has been madvised
2486			 * to prefer hugepage backing, retry allowing remote
2487			 * memory with both reclaim and compact as well.
2488			 */
2489		}
2490	}
2491
2492	page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask);
2493
2494	if (unlikely(pol->mode == MPOL_INTERLEAVE ||
2495		     pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) {
2496		/* skip NUMA_INTERLEAVE_HIT update if numa stats is disabled */
2497		if (static_branch_likely(&vm_numa_stat_key) &&
2498		    page_to_nid(page) == nid) {
2499			preempt_disable();
2500			__count_numa_event(page_zone(page), NUMA_INTERLEAVE_HIT);
2501			preempt_enable();
2502		}
2503	}
2504
2505	return page;
2506}
2507
2508struct folio *folio_alloc_mpol_noprof(gfp_t gfp, unsigned int order,
2509		struct mempolicy *pol, pgoff_t ilx, int nid)
2510{
2511	struct page *page = alloc_pages_mpol(gfp | __GFP_COMP, order, pol,
2512			ilx, nid);
2513	if (!page)
2514		return NULL;
2515
2516	set_page_refcounted(page);
2517	return page_rmappable_folio(page);
2518}
2519
2520/**
2521 * vma_alloc_folio - Allocate a folio for a VMA.
2522 * @gfp: GFP flags.
2523 * @order: Order of the folio.
2524 * @vma: Pointer to VMA.
2525 * @addr: Virtual address of the allocation.  Must be inside @vma.
2526 *
2527 * Allocate a folio for a specific address in @vma, using the appropriate
2528 * NUMA policy.  The caller must hold the mmap_lock of the mm_struct of the
2529 * VMA to prevent it from going away.  Should be used for all allocations
2530 * for folios that will be mapped into user space, excepting hugetlbfs, and
2531 * excepting where direct use of folio_alloc_mpol() is more appropriate.
2532 *
2533 * Return: The folio on success or NULL if allocation fails.
2534 */
2535struct folio *vma_alloc_folio_noprof(gfp_t gfp, int order, struct vm_area_struct *vma,
2536		unsigned long addr)
2537{
2538	struct mempolicy *pol;
2539	pgoff_t ilx;
2540	struct folio *folio;
2541
2542	if (vma->vm_flags & VM_DROPPABLE)
2543		gfp |= __GFP_NOWARN;
2544
2545	pol = get_vma_policy(vma, addr, order, &ilx);
2546	folio = folio_alloc_mpol_noprof(gfp, order, pol, ilx, numa_node_id());
2547	mpol_cond_put(pol);
2548	return folio;
2549}
2550EXPORT_SYMBOL(vma_alloc_folio_noprof);
2551
2552struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned order)
2553{
2554	struct mempolicy *pol = &default_policy;
2555
2556	/*
2557	 * No reference counting needed for current->mempolicy
2558	 * nor system default_policy
2559	 */
2560	if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2561		pol = get_task_policy(current);
2562
2563	return alloc_pages_mpol(gfp, order, pol, NO_INTERLEAVE_INDEX,
2564				       numa_node_id());
2565}
2566
2567/**
2568 * alloc_pages - Allocate pages.
2569 * @gfp: GFP flags.
2570 * @order: Power of two of number of pages to allocate.
2571 *
2572 * Allocate 1 << @order contiguous pages.  The physical address of the
2573 * first page is naturally aligned (eg an order-3 allocation will be aligned
2574 * to a multiple of 8 * PAGE_SIZE bytes).  The NUMA policy of the current
2575 * process is honoured when in process context.
2576 *
2577 * Context: Can be called from any context, providing the appropriate GFP
2578 * flags are used.
2579 * Return: The page on success or NULL if allocation fails.
2580 */
2581struct page *alloc_pages_noprof(gfp_t gfp, unsigned int order)
2582{
2583	struct page *page = alloc_frozen_pages_noprof(gfp, order);
2584
2585	if (page)
2586		set_page_refcounted(page);
2587	return page;
2588}
2589EXPORT_SYMBOL(alloc_pages_noprof);
2590
2591struct folio *folio_alloc_noprof(gfp_t gfp, unsigned int order)
2592{
2593	return page_rmappable_folio(alloc_pages_noprof(gfp | __GFP_COMP, order));
2594}
2595EXPORT_SYMBOL(folio_alloc_noprof);
2596
2597static unsigned long alloc_pages_bulk_interleave(gfp_t gfp,
2598		struct mempolicy *pol, unsigned long nr_pages,
2599		struct page **page_array)
2600{
2601	int nodes;
2602	unsigned long nr_pages_per_node;
2603	int delta;
2604	int i;
2605	unsigned long nr_allocated;
2606	unsigned long total_allocated = 0;
2607
2608	nodes = nodes_weight(pol->nodes);
2609	nr_pages_per_node = nr_pages / nodes;
2610	delta = nr_pages - nodes * nr_pages_per_node;
2611
2612	for (i = 0; i < nodes; i++) {
2613		if (delta) {
2614			nr_allocated = alloc_pages_bulk_noprof(gfp,
2615					interleave_nodes(pol), NULL,
2616					nr_pages_per_node + 1,
2617					page_array);
2618			delta--;
2619		} else {
2620			nr_allocated = alloc_pages_bulk_noprof(gfp,
2621					interleave_nodes(pol), NULL,
2622					nr_pages_per_node, page_array);
2623		}
2624
2625		page_array += nr_allocated;
2626		total_allocated += nr_allocated;
2627	}
2628
2629	return total_allocated;
2630}
2631
2632static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp,
2633		struct mempolicy *pol, unsigned long nr_pages,
2634		struct page **page_array)
2635{
2636	struct weighted_interleave_state *state;
2637	struct task_struct *me = current;
2638	unsigned int cpuset_mems_cookie;
2639	unsigned long total_allocated = 0;
2640	unsigned long nr_allocated = 0;
2641	unsigned long rounds;
2642	unsigned long node_pages, delta;
2643	u8 *weights, weight;
2644	unsigned int weight_total = 0;
2645	unsigned long rem_pages = nr_pages;
2646	nodemask_t nodes;
2647	int nnodes, node;
2648	int resume_node = MAX_NUMNODES - 1;
2649	u8 resume_weight = 0;
2650	int prev_node;
2651	int i;
2652
2653	if (!nr_pages)
2654		return 0;
2655
2656	/* read the nodes onto the stack, retry if done during rebind */
2657	do {
2658		cpuset_mems_cookie = read_mems_allowed_begin();
2659		nnodes = read_once_policy_nodemask(pol, &nodes);
2660	} while (read_mems_allowed_retry(cpuset_mems_cookie));
2661
2662	/* if the nodemask has become invalid, we cannot do anything */
2663	if (!nnodes)
2664		return 0;
2665
2666	/* Continue allocating from most recent node and adjust the nr_pages */
2667	node = me->il_prev;
2668	weight = me->il_weight;
2669	if (weight && node_isset(node, nodes)) {
2670		node_pages = min(rem_pages, weight);
2671		nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
2672						  page_array);
2673		page_array += nr_allocated;
2674		total_allocated += nr_allocated;
2675		/* if that's all the pages, no need to interleave */
2676		if (rem_pages <= weight) {
2677			me->il_weight -= rem_pages;
2678			return total_allocated;
2679		}
2680		/* Otherwise we adjust remaining pages, continue from there */
2681		rem_pages -= weight;
2682	}
2683	/* clear active weight in case of an allocation failure */
2684	me->il_weight = 0;
2685	prev_node = node;
2686
2687	/* create a local copy of node weights to operate on outside rcu */
2688	weights = kzalloc(nr_node_ids, GFP_KERNEL);
2689	if (!weights)
2690		return total_allocated;
2691
2692	rcu_read_lock();
2693	state = rcu_dereference(wi_state);
2694	if (state) {
2695		memcpy(weights, state->iw_table, nr_node_ids * sizeof(u8));
2696		rcu_read_unlock();
2697	} else {
2698		rcu_read_unlock();
2699		for (i = 0; i < nr_node_ids; i++)
2700			weights[i] = 1;
2701	}
2702
2703	/* calculate total, detect system default usage */
2704	for_each_node_mask(node, nodes)
2705		weight_total += weights[node];
2706
2707	/*
2708	 * Calculate rounds/partial rounds to minimize __alloc_pages_bulk calls.
2709	 * Track which node weighted interleave should resume from.
2710	 *
2711	 * if (rounds > 0) and (delta == 0), resume_node will always be
2712	 * the node following prev_node and its weight.
2713	 */
2714	rounds = rem_pages / weight_total;
2715	delta = rem_pages % weight_total;
2716	resume_node = next_node_in(prev_node, nodes);
2717	resume_weight = weights[resume_node];
2718	for (i = 0; i < nnodes; i++) {
2719		node = next_node_in(prev_node, nodes);
2720		weight = weights[node];
2721		node_pages = weight * rounds;
2722		/* If a delta exists, add this node's portion of the delta */
2723		if (delta > weight) {
2724			node_pages += weight;
2725			delta -= weight;
2726		} else if (delta) {
2727			/* when delta is depleted, resume from that node */
2728			node_pages += delta;
2729			resume_node = node;
2730			resume_weight = weight - delta;
2731			delta = 0;
2732		}
2733		/* node_pages can be 0 if an allocation fails and rounds == 0 */
2734		if (!node_pages)
2735			break;
2736		nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
2737						  page_array);
2738		page_array += nr_allocated;
2739		total_allocated += nr_allocated;
2740		if (total_allocated == nr_pages)
2741			break;
2742		prev_node = node;
2743	}
2744	me->il_prev = resume_node;
2745	me->il_weight = resume_weight;
2746	kfree(weights);
2747	return total_allocated;
2748}
2749
2750static unsigned long alloc_pages_bulk_preferred_many(gfp_t gfp, int nid,
2751		struct mempolicy *pol, unsigned long nr_pages,
2752		struct page **page_array)
2753{
2754	gfp_t preferred_gfp;
2755	unsigned long nr_allocated = 0;
2756
2757	preferred_gfp = gfp | __GFP_NOWARN;
2758	preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2759
2760	nr_allocated  = alloc_pages_bulk_noprof(preferred_gfp, nid, &pol->nodes,
2761					   nr_pages, page_array);
2762
2763	if (nr_allocated < nr_pages)
2764		nr_allocated += alloc_pages_bulk_noprof(gfp, numa_node_id(), NULL,
2765				nr_pages - nr_allocated,
2766				page_array + nr_allocated);
2767	return nr_allocated;
2768}
2769
2770/* alloc pages bulk and mempolicy should be considered at the
2771 * same time in some situation such as vmalloc.
2772 *
2773 * It can accelerate memory allocation especially interleaving
2774 * allocate memory.
2775 */
2776unsigned long alloc_pages_bulk_mempolicy_noprof(gfp_t gfp,
2777		unsigned long nr_pages, struct page **page_array)
2778{
2779	struct mempolicy *pol = &default_policy;
2780	nodemask_t *nodemask;
2781	int nid;
2782
2783	if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2784		pol = get_task_policy(current);
2785
2786	if (pol->mode == MPOL_INTERLEAVE)
2787		return alloc_pages_bulk_interleave(gfp, pol,
2788							 nr_pages, page_array);
2789
2790	if (pol->mode == MPOL_WEIGHTED_INTERLEAVE)
2791		return alloc_pages_bulk_weighted_interleave(
2792				  gfp, pol, nr_pages, page_array);
2793
2794	if (pol->mode == MPOL_PREFERRED_MANY)
2795		return alloc_pages_bulk_preferred_many(gfp,
2796				numa_node_id(), pol, nr_pages, page_array);
2797
2798	nid = numa_node_id();
2799	nodemask = policy_nodemask(gfp, pol, NO_INTERLEAVE_INDEX, &nid);
2800	return alloc_pages_bulk_noprof(gfp, nid, nodemask,
2801				       nr_pages, page_array);
2802}
2803
2804int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2805{
2806	struct mempolicy *pol = mpol_dup(src->vm_policy);
2807
2808	if (IS_ERR(pol))
2809		return PTR_ERR(pol);
2810	dst->vm_policy = pol;
2811	return 0;
2812}
2813
2814/*
2815 * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it
2816 * rebinds the mempolicy its copying by calling mpol_rebind_policy()
2817 * with the mems_allowed returned by cpuset_mems_allowed().  This
2818 * keeps mempolicies cpuset relative after its cpuset moves.  See
2819 * further kernel/cpuset.c update_nodemask().
2820 *
2821 * current's mempolicy may be rebinded by the other task(the task that changes
2822 * cpuset's mems), so we needn't do rebind work for current task.
2823 */
2824
2825/* Slow path of a mempolicy duplicate */
2826struct mempolicy *__mpol_dup(struct mempolicy *old)
2827{
2828	struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
2829
2830	if (!new)
2831		return ERR_PTR(-ENOMEM);
2832
2833	/* task's mempolicy is protected by alloc_lock */
2834	if (old == current->mempolicy) {
2835		task_lock(current);
2836		*new = *old;
2837		task_unlock(current);
2838	} else
2839		*new = *old;
2840
2841	if (current_cpuset_is_being_rebound()) {
2842		nodemask_t mems = cpuset_mems_allowed(current);
2843		mpol_rebind_policy(new, &mems);
2844	}
2845	atomic_set(&new->refcnt, 1);
2846	return new;
2847}
2848
2849/* Slow path of a mempolicy comparison */
2850bool __mpol_equal(struct mempolicy *a, struct mempolicy *b)
2851{
2852	if (!a || !b)
2853		return false;
2854	if (a->mode != b->mode)
2855		return false;
2856	if (a->flags != b->flags)
2857		return false;
2858	if (a->home_node != b->home_node)
2859		return false;
2860	if (mpol_store_user_nodemask(a))
2861		if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
2862			return false;
2863
2864	switch (a->mode) {
2865	case MPOL_BIND:
2866	case MPOL_INTERLEAVE:
2867	case MPOL_PREFERRED:
2868	case MPOL_PREFERRED_MANY:
2869	case MPOL_WEIGHTED_INTERLEAVE:
2870		return !!nodes_equal(a->nodes, b->nodes);
2871	case MPOL_LOCAL:
2872		return true;
2873	default:
2874		BUG();
2875		return false;
2876	}
2877}
2878
2879/*
2880 * Shared memory backing store policy support.
2881 *
2882 * Remember policies even when nobody has shared memory mapped.
2883 * The policies are kept in Red-Black tree linked from the inode.
2884 * They are protected by the sp->lock rwlock, which should be held
2885 * for any accesses to the tree.
2886 */
2887
2888/*
2889 * lookup first element intersecting start-end.  Caller holds sp->lock for
2890 * reading or for writing
2891 */
2892static struct sp_node *sp_lookup(struct shared_policy *sp,
2893					pgoff_t start, pgoff_t end)
2894{
2895	struct rb_node *n = sp->root.rb_node;
2896
2897	while (n) {
2898		struct sp_node *p = rb_entry(n, struct sp_node, nd);
2899
2900		if (start >= p->end)
2901			n = n->rb_right;
2902		else if (end <= p->start)
2903			n = n->rb_left;
2904		else
2905			break;
2906	}
2907	if (!n)
2908		return NULL;
2909	for (;;) {
2910		struct sp_node *w = NULL;
2911		struct rb_node *prev = rb_prev(n);
2912		if (!prev)
2913			break;
2914		w = rb_entry(prev, struct sp_node, nd);
2915		if (w->end <= start)
2916			break;
2917		n = prev;
2918	}
2919	return rb_entry(n, struct sp_node, nd);
2920}
2921
2922/*
2923 * Insert a new shared policy into the list.  Caller holds sp->lock for
2924 * writing.
2925 */
2926static void sp_insert(struct shared_policy *sp, struct sp_node *new)
2927{
2928	struct rb_node **p = &sp->root.rb_node;
2929	struct rb_node *parent = NULL;
2930	struct sp_node *nd;
2931
2932	while (*p) {
2933		parent = *p;
2934		nd = rb_entry(parent, struct sp_node, nd);
2935		if (new->start < nd->start)
2936			p = &(*p)->rb_left;
2937		else if (new->end > nd->end)
2938			p = &(*p)->rb_right;
2939		else
2940			BUG();
2941	}
2942	rb_link_node(&new->nd, parent, p);
2943	rb_insert_color(&new->nd, &sp->root);
2944}
2945
2946/* Find shared policy intersecting idx */
2947struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
2948						pgoff_t idx)
2949{
2950	struct mempolicy *pol = NULL;
2951	struct sp_node *sn;
2952
2953	if (!sp->root.rb_node)
2954		return NULL;
2955	read_lock(&sp->lock);
2956	sn = sp_lookup(sp, idx, idx+1);
2957	if (sn) {
2958		mpol_get(sn->policy);
2959		pol = sn->policy;
2960	}
2961	read_unlock(&sp->lock);
2962	return pol;
2963}
2964EXPORT_SYMBOL_FOR_MODULES(mpol_shared_policy_lookup, "kvm");
2965
2966static void sp_free(struct sp_node *n)
2967{
2968	mpol_put(n->policy);
2969	kmem_cache_free(sn_cache, n);
2970}
2971
2972/**
2973 * mpol_misplaced - check whether current folio node is valid in policy
2974 *
2975 * @folio: folio to be checked
2976 * @vmf: structure describing the fault
2977 * @addr: virtual address in @vma for shared policy lookup and interleave policy
2978 *
2979 * Lookup current policy node id for vma,addr and "compare to" folio's
2980 * node id.  Policy determination "mimics" alloc_page_vma().
2981 * Called from fault path where we know the vma and faulting address.
2982 *
2983 * Return: NUMA_NO_NODE if the page is in a node that is valid for this
2984 * policy, or a suitable node ID to allocate a replacement folio from.
2985 */
2986int mpol_misplaced(struct folio *folio, struct vm_fault *vmf,
2987		   unsigned long addr)
2988{
2989	struct mempolicy *pol;
2990	pgoff_t ilx;
2991	struct zoneref *z;
2992	int curnid = folio_nid(folio);
2993	struct vm_area_struct *vma = vmf->vma;
2994	int thiscpu = raw_smp_processor_id();
2995	int thisnid = numa_node_id();
2996	int polnid = NUMA_NO_NODE;
2997	int ret = NUMA_NO_NODE;
2998
2999	/*
3000	 * Make sure ptl is held so that we don't preempt and we
3001	 * have a stable smp processor id
3002	 */
3003	lockdep_assert_held(vmf->ptl);
3004	pol = get_vma_policy(vma, addr, folio_order(folio), &ilx);
3005	if (!(pol->flags & MPOL_F_MOF))
3006		goto out;
3007
3008	switch (pol->mode) {
3009	case MPOL_INTERLEAVE:
3010		polnid = interleave_nid(pol, ilx);
3011		break;
3012
3013	case MPOL_WEIGHTED_INTERLEAVE:
3014		polnid = weighted_interleave_nid(pol, ilx);
3015		break;
3016
3017	case MPOL_PREFERRED:
3018		if (node_isset(curnid, pol->nodes))
3019			goto out;
3020		polnid = first_node(pol->nodes);
3021		break;
3022
3023	case MPOL_LOCAL:
3024		polnid = numa_node_id();
3025		break;
3026
3027	case MPOL_BIND:
3028	case MPOL_PREFERRED_MANY:
3029		/*
3030		 * Even though MPOL_PREFERRED_MANY can allocate pages outside
3031		 * policy nodemask we don't allow numa migration to nodes
3032		 * outside policy nodemask for now. This is done so that if we
3033		 * want demotion to slow memory to happen, before allocating
3034		 * from some DRAM node say 'x', we will end up using a
3035		 * MPOL_PREFERRED_MANY mask excluding node 'x'. In such scenario
3036		 * we should not promote to node 'x' from slow memory node.
3037		 */
3038		if (pol->flags & MPOL_F_MORON) {
3039			/*
3040			 * Optimize placement among multiple nodes
3041			 * via NUMA balancing
3042			 */
3043			if (node_isset(thisnid, pol->nodes))
3044				break;
3045			goto out;
3046		}
3047
3048		/*
3049		 * use current page if in policy nodemask,
3050		 * else select nearest allowed node, if any.
3051		 * If no allowed nodes, use current [!misplaced].
3052		 */
3053		if (node_isset(curnid, pol->nodes))
3054			goto out;
3055		z = first_zones_zonelist(
3056				node_zonelist(thisnid, GFP_HIGHUSER),
3057				gfp_zone(GFP_HIGHUSER),
3058				&pol->nodes);
3059		polnid = zonelist_node_idx(z);
3060		break;
3061
3062	default:
3063		BUG();
3064	}
3065
3066	/* Migrate the folio towards the node whose CPU is referencing it */
3067	if (pol->flags & MPOL_F_MORON) {
3068		polnid = thisnid;
3069
3070		if (!should_numa_migrate_memory(current, folio, curnid,
3071						thiscpu))
3072			goto out;
3073	}
3074
3075	if (curnid != polnid)
3076		ret = polnid;
3077out:
3078	mpol_cond_put(pol);
3079
3080	return ret;
3081}
3082
3083/*
3084 * Drop the (possibly final) reference to task->mempolicy.  It needs to be
3085 * dropped after task->mempolicy is set to NULL so that any allocation done as
3086 * part of its kmem_cache_free(), such as by KASAN, doesn't reference a freed
3087 * policy.
3088 */
3089void mpol_put_task_policy(struct task_struct *task)
3090{
3091	struct mempolicy *pol;
3092
3093	task_lock(task);
3094	pol = task->mempolicy;
3095	task->mempolicy = NULL;
3096	task_unlock(task);
3097	mpol_put(pol);
3098}
3099
3100static void sp_delete(struct shared_policy *sp, struct sp_node *n)
3101{
3102	rb_erase(&n->nd, &sp->root);
3103	sp_free(n);
3104}
3105
3106static void sp_node_init(struct sp_node *node, unsigned long start,
3107			unsigned long end, struct mempolicy *pol)
3108{
3109	node->start = start;
3110	node->end = end;
3111	node->policy = pol;
3112}
3113
3114static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
3115				struct mempolicy *pol)
3116{
3117	struct sp_node *n;
3118	struct mempolicy *newpol;
3119
3120	n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
3121	if (!n)
3122		return NULL;
3123
3124	newpol = mpol_dup(pol);
3125	if (IS_ERR(newpol)) {
3126		kmem_cache_free(sn_cache, n);
3127		return NULL;
3128	}
3129	newpol->flags |= MPOL_F_SHARED;
3130	sp_node_init(n, start, end, newpol);
3131
3132	return n;
3133}
3134
3135/* Replace a policy range. */
3136static int shared_policy_replace(struct shared_policy *sp, pgoff_t start,
3137				 pgoff_t end, struct sp_node *new)
3138{
3139	struct sp_node *n;
3140	struct sp_node *n_new = NULL;
3141	struct mempolicy *mpol_new = NULL;
3142	int ret = 0;
3143
3144restart:
3145	write_lock(&sp->lock);
3146	n = sp_lookup(sp, start, end);
3147	/* Take care of old policies in the same range. */
3148	while (n && n->start < end) {
3149		struct rb_node *next = rb_next(&n->nd);
3150		if (n->start >= start) {
3151			if (n->end <= end)
3152				sp_delete(sp, n);
3153			else
3154				n->start = end;
3155		} else {
3156			/* Old policy spanning whole new range. */
3157			if (n->end > end) {
3158				if (!n_new)
3159					goto alloc_new;
3160
3161				*mpol_new = *n->policy;
3162				atomic_set(&mpol_new->refcnt, 1);
3163				sp_node_init(n_new, end, n->end, mpol_new);
3164				n->end = start;
3165				sp_insert(sp, n_new);
3166				n_new = NULL;
3167				mpol_new = NULL;
3168				break;
3169			} else
3170				n->end = start;
3171		}
3172		if (!next)
3173			break;
3174		n = rb_entry(next, struct sp_node, nd);
3175	}
3176	if (new)
3177		sp_insert(sp, new);
3178	write_unlock(&sp->lock);
3179	ret = 0;
3180
3181err_out:
3182	if (mpol_new)
3183		mpol_put(mpol_new);
3184	if (n_new)
3185		kmem_cache_free(sn_cache, n_new);
3186
3187	return ret;
3188
3189alloc_new:
3190	write_unlock(&sp->lock);
3191	ret = -ENOMEM;
3192	n_new = kmem_cache_alloc(sn_cache, GFP_KERNEL);
3193	if (!n_new)
3194		goto err_out;
3195	mpol_new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
3196	if (!mpol_new)
3197		goto err_out;
3198	atomic_set(&mpol_new->refcnt, 1);
3199	goto restart;
3200}
3201
3202/**
3203 * mpol_shared_policy_init - initialize shared policy for inode
3204 * @sp: pointer to inode shared policy
3205 * @mpol:  struct mempolicy to install
3206 *
3207 * Install non-NULL @mpol in inode's shared policy rb-tree.
3208 * On entry, the current task has a reference on a non-NULL @mpol.
3209 * This must be released on exit.
3210 * This is called at get_inode() calls and we can use GFP_KERNEL.
3211 */
3212void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
3213{
3214	int ret;
3215
3216	sp->root = RB_ROOT;		/* empty tree == default mempolicy */
3217	rwlock_init(&sp->lock);
3218
3219	if (mpol) {
3220		struct sp_node *sn;
3221		struct mempolicy *npol;
3222		NODEMASK_SCRATCH(scratch);
3223
3224		if (!scratch)
3225			goto put_mpol;
3226
3227		/* contextualize the tmpfs mount point mempolicy to this file */
3228		npol = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
3229		if (IS_ERR(npol))
3230			goto free_scratch; /* no valid nodemask intersection */
3231
3232		task_lock(current);
3233		ret = mpol_set_nodemask(npol, &mpol->w.user_nodemask, scratch);
3234		task_unlock(current);
3235		if (ret)
3236			goto put_npol;
3237
3238		/* alloc node covering entire file; adds ref to file's npol */
3239		sn = sp_alloc(0, MAX_LFS_FILESIZE >> PAGE_SHIFT, npol);
3240		if (sn)
3241			sp_insert(sp, sn);
3242put_npol:
3243		mpol_put(npol);	/* drop initial ref on file's npol */
3244free_scratch:
3245		NODEMASK_SCRATCH_FREE(scratch);
3246put_mpol:
3247		mpol_put(mpol);	/* drop our incoming ref on sb mpol */
3248	}
3249}
3250EXPORT_SYMBOL_FOR_MODULES(mpol_shared_policy_init, "kvm");
3251
3252int mpol_set_shared_policy(struct shared_policy *sp,
3253			struct vm_area_struct *vma, struct mempolicy *pol)
3254{
3255	int err;
3256	struct sp_node *new = NULL;
3257	unsigned long sz = vma_pages(vma);
3258
3259	if (pol) {
3260		new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol);
3261		if (!new)
3262			return -ENOMEM;
3263	}
3264	err = shared_policy_replace(sp, vma->vm_pgoff, vma->vm_pgoff + sz, new);
3265	if (err && new)
3266		sp_free(new);
3267	return err;
3268}
3269EXPORT_SYMBOL_FOR_MODULES(mpol_set_shared_policy, "kvm");
3270
3271/* Free a backing policy store on inode delete. */
3272void mpol_free_shared_policy(struct shared_policy *sp)
3273{
3274	struct sp_node *n;
3275	struct rb_node *next;
3276
3277	if (!sp->root.rb_node)
3278		return;
3279	write_lock(&sp->lock);
3280	next = rb_first(&sp->root);
3281	while (next) {
3282		n = rb_entry(next, struct sp_node, nd);
3283		next = rb_next(&n->nd);
3284		sp_delete(sp, n);
3285	}
3286	write_unlock(&sp->lock);
3287}
3288EXPORT_SYMBOL_FOR_MODULES(mpol_free_shared_policy, "kvm");
3289
3290#ifdef CONFIG_NUMA_BALANCING
3291static int __initdata numabalancing_override;
3292
3293static void __init check_numabalancing_enable(void)
3294{
3295	bool numabalancing_default = false;
3296
3297	if (IS_ENABLED(CONFIG_NUMA_BALANCING_DEFAULT_ENABLED))
3298		numabalancing_default = true;
3299
3300	/* Parsed by setup_numabalancing. override == 1 enables, -1 disables */
3301	if (numabalancing_override)
3302		set_numabalancing_state(numabalancing_override == 1);
3303
3304	if (num_online_nodes() > 1 && !numabalancing_override) {
3305		pr_info("%s automatic NUMA balancing. Configure with numa_balancing= or the kernel.numa_balancing sysctl\n",
3306			numabalancing_default ? "Enabling" : "Disabling");
3307		set_numabalancing_state(numabalancing_default);
3308	}
3309}
3310
3311static int __init setup_numabalancing(char *str)
3312{
3313	int ret = 0;
3314	if (!str)
3315		goto out;
3316
3317	if (!strcmp(str, "enable")) {
3318		numabalancing_override = 1;
3319		ret = 1;
3320	} else if (!strcmp(str, "disable")) {
3321		numabalancing_override = -1;
3322		ret = 1;
3323	}
3324out:
3325	if (!ret)
3326		pr_warn("Unable to parse numa_balancing=\n");
3327
3328	return ret;
3329}
3330__setup("numa_balancing=", setup_numabalancing);
3331#else
3332static inline void __init check_numabalancing_enable(void)
3333{
3334}
3335#endif /* CONFIG_NUMA_BALANCING */
3336
3337void __init numa_policy_init(void)
3338{
3339	nodemask_t interleave_nodes;
3340	unsigned long largest = 0;
3341	int nid, prefer = 0;
3342
3343	policy_cache = kmem_cache_create("numa_policy",
3344					 sizeof(struct mempolicy),
3345					 0, SLAB_PANIC, NULL);
3346
3347	sn_cache = kmem_cache_create("shared_policy_node",
3348				     sizeof(struct sp_node),
3349				     0, SLAB_PANIC, NULL);
3350
3351	for_each_node(nid) {
3352		preferred_node_policy[nid] = (struct mempolicy) {
3353			.refcnt = ATOMIC_INIT(1),
3354			.mode = MPOL_PREFERRED,
3355			.flags = MPOL_F_MOF | MPOL_F_MORON,
3356			.nodes = nodemask_of_node(nid),
3357		};
3358	}
3359
3360	/*
3361	 * Set interleaving policy for system init. Interleaving is only
3362	 * enabled across suitably sized nodes (default is >= 16MB), or
3363	 * fall back to the largest node if they're all smaller.
3364	 */
3365	nodes_clear(interleave_nodes);
3366	for_each_node_state(nid, N_MEMORY) {
3367		unsigned long total_pages = node_present_pages(nid);
3368
3369		/* Preserve the largest node */
3370		if (largest < total_pages) {
3371			largest = total_pages;
3372			prefer = nid;
3373		}
3374
3375		/* Interleave this node? */
3376		if ((total_pages << PAGE_SHIFT) >= (16 << 20))
3377			node_set(nid, interleave_nodes);
3378	}
3379
3380	/* All too small, use the largest */
3381	if (unlikely(nodes_empty(interleave_nodes)))
3382		node_set(prefer, interleave_nodes);
3383
3384	if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))
3385		pr_err("%s: interleaving failed\n", __func__);
3386
3387	check_numabalancing_enable();
3388}
3389
3390/* Reset policy of current process to default */
3391void numa_default_policy(void)
3392{
3393	do_set_mempolicy(MPOL_DEFAULT, 0, NULL);
3394}
3395
3396/*
3397 * Parse and format mempolicy from/to strings
3398 */
3399static const char * const policy_modes[] =
3400{
3401	[MPOL_DEFAULT]    = "default",
3402	[MPOL_PREFERRED]  = "prefer",
3403	[MPOL_BIND]       = "bind",
3404	[MPOL_INTERLEAVE] = "interleave",
3405	[MPOL_WEIGHTED_INTERLEAVE] = "weighted interleave",
3406	[MPOL_LOCAL]      = "local",
3407	[MPOL_PREFERRED_MANY]  = "prefer (many)",
3408};
3409
3410#ifdef CONFIG_TMPFS
3411/**
3412 * mpol_parse_str - parse string to mempolicy, for tmpfs mpol mount option.
3413 * @str:  string containing mempolicy to parse
3414 * @mpol:  pointer to struct mempolicy pointer, returned on success.
3415 *
3416 * Format of input:
3417 *	<mode>[=<flags>][:<nodelist>]
3418 *
3419 * Return: %0 on success, else %1
3420 */
3421int mpol_parse_str(char *str, struct mempolicy **mpol)
3422{
3423	struct mempolicy *new = NULL;
3424	unsigned short mode_flags;
3425	nodemask_t nodes;
3426	char *nodelist = strchr(str, ':');
3427	char *flags = strchr(str, '=');
3428	int err = 1, mode;
3429
3430	if (flags)
3431		*flags++ = '\0';	/* terminate mode string */
3432
3433	if (nodelist) {
3434		/* NUL-terminate mode or flags string */
3435		*nodelist++ = '\0';
3436		if (nodelist_parse(nodelist, nodes))
3437			goto out;
3438		if (!nodes_subset(nodes, node_states[N_MEMORY]))
3439			goto out;
3440	} else
3441		nodes_clear(nodes);
3442
3443	mode = match_string(policy_modes, MPOL_MAX, str);
3444	if (mode < 0)
3445		goto out;
3446
3447	switch (mode) {
3448	case MPOL_PREFERRED:
3449		/*
3450		 * Insist on a nodelist of one node only, although later
3451		 * we use first_node(nodes) to grab a single node, so here
3452		 * nodelist (or nodes) cannot be empty.
3453		 */
3454		if (nodelist) {
3455			char *rest = nodelist;
3456			while (isdigit(*rest))
3457				rest++;
3458			if (*rest)
3459				goto out;
3460			if (nodes_empty(nodes))
3461				goto out;
3462		}
3463		break;
3464	case MPOL_INTERLEAVE:
3465	case MPOL_WEIGHTED_INTERLEAVE:
3466		/*
3467		 * Default to online nodes with memory if no nodelist
3468		 */
3469		if (!nodelist)
3470			nodes = node_states[N_MEMORY];
3471		break;
3472	case MPOL_LOCAL:
3473		/*
3474		 * Don't allow a nodelist;  mpol_new() checks flags
3475		 */
3476		if (nodelist)
3477			goto out;
3478		break;
3479	case MPOL_DEFAULT:
3480		/*
3481		 * Insist on a empty nodelist
3482		 */
3483		if (!nodelist)
3484			err = 0;
3485		goto out;
3486	case MPOL_PREFERRED_MANY:
3487	case MPOL_BIND:
3488		/*
3489		 * Insist on a nodelist
3490		 */
3491		if (!nodelist)
3492			goto out;
3493	}
3494
3495	mode_flags = 0;
3496	if (flags) {
3497		/*
3498		 * Currently, we only support two mutually exclusive
3499		 * mode flags.
3500		 */
3501		if (!strcmp(flags, "static"))
3502			mode_flags |= MPOL_F_STATIC_NODES;
3503		else if (!strcmp(flags, "relative"))
3504			mode_flags |= MPOL_F_RELATIVE_NODES;
3505		else
3506			goto out;
3507	}
3508
3509	new = mpol_new(mode, mode_flags, &nodes);
3510	if (IS_ERR(new))
3511		goto out;
3512
3513	/*
3514	 * Save nodes for mpol_to_str() to show the tmpfs mount options
3515	 * for /proc/mounts, /proc/pid/mounts and /proc/pid/mountinfo.
3516	 */
3517	if (mode != MPOL_PREFERRED) {
3518		new->nodes = nodes;
3519	} else if (nodelist) {
3520		nodes_clear(new->nodes);
3521		node_set(first_node(nodes), new->nodes);
3522	} else {
3523		new->mode = MPOL_LOCAL;
3524	}
3525
3526	/*
3527	 * Save nodes for contextualization: this will be used to "clone"
3528	 * the mempolicy in a specific context [cpuset] at a later time.
3529	 */
3530	new->w.user_nodemask = nodes;
3531
3532	err = 0;
3533
3534out:
3535	/* Restore string for error message */
3536	if (nodelist)
3537		*--nodelist = ':';
3538	if (flags)
3539		*--flags = '=';
3540	if (!err)
3541		*mpol = new;
3542	return err;
3543}
3544#endif /* CONFIG_TMPFS */
3545
3546/**
3547 * mpol_to_str - format a mempolicy structure for printing
3548 * @buffer:  to contain formatted mempolicy string
3549 * @maxlen:  length of @buffer
3550 * @pol:  pointer to mempolicy to be formatted
3551 *
3552 * Convert @pol into a string.  If @buffer is too short, truncate the string.
3553 * Recommend a @maxlen of at least 51 for the longest mode, "weighted
3554 * interleave", plus the longest flag flags, "relative|balancing", and to
3555 * display at least a few node ids.
3556 */
3557void mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
3558{
3559	char *p = buffer;
3560	nodemask_t nodes = NODE_MASK_NONE;
3561	unsigned short mode = MPOL_DEFAULT;
3562	unsigned short flags = 0;
3563
3564	if (pol &&
3565	    pol != &default_policy &&
3566	    !(pol >= &preferred_node_policy[0] &&
3567	      pol <= &preferred_node_policy[ARRAY_SIZE(preferred_node_policy) - 1])) {
3568		mode = pol->mode;
3569		flags = pol->flags;
3570	}
3571
3572	switch (mode) {
3573	case MPOL_DEFAULT:
3574	case MPOL_LOCAL:
3575		break;
3576	case MPOL_PREFERRED:
3577	case MPOL_PREFERRED_MANY:
3578	case MPOL_BIND:
3579	case MPOL_INTERLEAVE:
3580	case MPOL_WEIGHTED_INTERLEAVE:
3581		nodes = pol->nodes;
3582		break;
3583	default:
3584		WARN_ON_ONCE(1);
3585		snprintf(p, maxlen, "unknown");
3586		return;
3587	}
3588
3589	p += snprintf(p, maxlen, "%s", policy_modes[mode]);
3590
3591	if (flags & MPOL_MODE_FLAGS) {
3592		p += snprintf(p, buffer + maxlen - p, "=");
3593
3594		/*
3595		 * Static and relative are mutually exclusive.
3596		 */
3597		if (flags & MPOL_F_STATIC_NODES)
3598			p += snprintf(p, buffer + maxlen - p, "static");
3599		else if (flags & MPOL_F_RELATIVE_NODES)
3600			p += snprintf(p, buffer + maxlen - p, "relative");
3601
3602		if (flags & MPOL_F_NUMA_BALANCING) {
3603			if (!is_power_of_2(flags & MPOL_MODE_FLAGS))
3604				p += snprintf(p, buffer + maxlen - p, "|");
3605			p += snprintf(p, buffer + maxlen - p, "balancing");
3606		}
3607	}
3608
3609	if (!nodes_empty(nodes))
3610		p += scnprintf(p, buffer + maxlen - p, ":%*pbl",
3611			       nodemask_pr_args(&nodes));
3612}
3613
3614#ifdef CONFIG_SYSFS
3615struct iw_node_attr {
3616	struct kobj_attribute kobj_attr;
3617	int nid;
3618};
3619
3620struct sysfs_wi_group {
3621	struct kobject wi_kobj;
3622	struct mutex kobj_lock;
3623	struct iw_node_attr *nattrs[];
3624};
3625
3626static struct sysfs_wi_group *wi_group;
3627
3628static ssize_t node_show(struct kobject *kobj, struct kobj_attribute *attr,
3629			 char *buf)
3630{
3631	struct iw_node_attr *node_attr;
3632	u8 weight;
3633
3634	node_attr = container_of(attr, struct iw_node_attr, kobj_attr);
3635	weight = get_il_weight(node_attr->nid);
3636	return sysfs_emit(buf, "%d\n", weight);
3637}
3638
3639static ssize_t node_store(struct kobject *kobj, struct kobj_attribute *attr,
3640			  const char *buf, size_t count)
3641{
3642	struct weighted_interleave_state *new_wi_state, *old_wi_state = NULL;
3643	struct iw_node_attr *node_attr;
3644	u8 weight = 0;
3645	int i;
3646
3647	node_attr = container_of(attr, struct iw_node_attr, kobj_attr);
3648	if (count == 0 || sysfs_streq(buf, "") ||
3649	    kstrtou8(buf, 0, &weight) || weight == 0)
3650		return -EINVAL;
3651
3652	new_wi_state = kzalloc(struct_size(new_wi_state, iw_table, nr_node_ids),
3653			       GFP_KERNEL);
3654	if (!new_wi_state)
3655		return -ENOMEM;
3656
3657	mutex_lock(&wi_state_lock);
3658	old_wi_state = rcu_dereference_protected(wi_state,
3659					lockdep_is_held(&wi_state_lock));
3660	if (old_wi_state) {
3661		memcpy(new_wi_state->iw_table, old_wi_state->iw_table,
3662					nr_node_ids * sizeof(u8));
3663	} else {
3664		for (i = 0; i < nr_node_ids; i++)
3665			new_wi_state->iw_table[i] = 1;
3666	}
3667	new_wi_state->iw_table[node_attr->nid] = weight;
3668	new_wi_state->mode_auto = false;
3669
3670	rcu_assign_pointer(wi_state, new_wi_state);
3671	mutex_unlock(&wi_state_lock);
3672	if (old_wi_state) {
3673		synchronize_rcu();
3674		kfree(old_wi_state);
3675	}
3676	return count;
3677}
3678
3679static ssize_t weighted_interleave_auto_show(struct kobject *kobj,
3680		struct kobj_attribute *attr, char *buf)
3681{
3682	struct weighted_interleave_state *state;
3683	bool wi_auto = true;
3684
3685	rcu_read_lock();
3686	state = rcu_dereference(wi_state);
3687	if (state)
3688		wi_auto = state->mode_auto;
3689	rcu_read_unlock();
3690
3691	return sysfs_emit(buf, "%s\n", str_true_false(wi_auto));
3692}
3693
3694static ssize_t weighted_interleave_auto_store(struct kobject *kobj,
3695		struct kobj_attribute *attr, const char *buf, size_t count)
3696{
3697	struct weighted_interleave_state *new_wi_state, *old_wi_state = NULL;
3698	unsigned int *bw;
3699	bool input;
3700	int i;
3701
3702	if (kstrtobool(buf, &input))
3703		return -EINVAL;
3704
3705	new_wi_state = kzalloc(struct_size(new_wi_state, iw_table, nr_node_ids),
3706			       GFP_KERNEL);
3707	if (!new_wi_state)
3708		return -ENOMEM;
3709	for (i = 0; i < nr_node_ids; i++)
3710		new_wi_state->iw_table[i] = 1;
3711
3712	mutex_lock(&wi_state_lock);
3713	if (!input) {
3714		old_wi_state = rcu_dereference_protected(wi_state,
3715					lockdep_is_held(&wi_state_lock));
3716		if (!old_wi_state)
3717			goto update_wi_state;
3718		if (input == old_wi_state->mode_auto) {
3719			mutex_unlock(&wi_state_lock);
3720			return count;
3721		}
3722
3723		memcpy(new_wi_state->iw_table, old_wi_state->iw_table,
3724					       nr_node_ids * sizeof(u8));
3725		goto update_wi_state;
3726	}
3727
3728	bw = node_bw_table;
3729	if (!bw) {
3730		mutex_unlock(&wi_state_lock);
3731		kfree(new_wi_state);
3732		return -ENODEV;
3733	}
3734
3735	new_wi_state->mode_auto = true;
3736	reduce_interleave_weights(bw, new_wi_state->iw_table);
3737
3738update_wi_state:
3739	rcu_assign_pointer(wi_state, new_wi_state);
3740	mutex_unlock(&wi_state_lock);
3741	if (old_wi_state) {
3742		synchronize_rcu();
3743		kfree(old_wi_state);
3744	}
3745	return count;
3746}
3747
3748static void sysfs_wi_node_delete(int nid)
3749{
3750	struct iw_node_attr *attr;
3751
3752	if (nid < 0 || nid >= nr_node_ids)
3753		return;
3754
3755	mutex_lock(&wi_group->kobj_lock);
3756	attr = wi_group->nattrs[nid];
3757	if (!attr) {
3758		mutex_unlock(&wi_group->kobj_lock);
3759		return;
3760	}
3761
3762	wi_group->nattrs[nid] = NULL;
3763	mutex_unlock(&wi_group->kobj_lock);
3764
3765	sysfs_remove_file(&wi_group->wi_kobj, &attr->kobj_attr.attr);
3766	kfree(attr->kobj_attr.attr.name);
3767	kfree(attr);
3768}
3769
3770static void sysfs_wi_node_delete_all(void)
3771{
3772	int nid;
3773
3774	for (nid = 0; nid < nr_node_ids; nid++)
3775		sysfs_wi_node_delete(nid);
3776}
3777
3778static void wi_state_free(void)
3779{
3780	struct weighted_interleave_state *old_wi_state;
3781
3782	mutex_lock(&wi_state_lock);
3783	old_wi_state = rcu_dereference_protected(wi_state,
3784			lockdep_is_held(&wi_state_lock));
3785	rcu_assign_pointer(wi_state, NULL);
3786	mutex_unlock(&wi_state_lock);
3787
3788	if (old_wi_state) {
3789		synchronize_rcu();
3790		kfree(old_wi_state);
3791	}
3792}
3793
3794static struct kobj_attribute wi_auto_attr =
3795	__ATTR(auto, 0664, weighted_interleave_auto_show,
3796			   weighted_interleave_auto_store);
3797
3798static void wi_cleanup(void) {
3799	sysfs_remove_file(&wi_group->wi_kobj, &wi_auto_attr.attr);
3800	sysfs_wi_node_delete_all();
3801	wi_state_free();
3802}
3803
3804static void wi_kobj_release(struct kobject *wi_kobj)
3805{
3806	kfree(wi_group);
3807}
3808
3809static const struct kobj_type wi_ktype = {
3810	.sysfs_ops = &kobj_sysfs_ops,
3811	.release = wi_kobj_release,
3812};
3813
3814static int sysfs_wi_node_add(int nid)
3815{
3816	int ret;
3817	char *name;
3818	struct iw_node_attr *new_attr;
3819
3820	if (nid < 0 || nid >= nr_node_ids) {
3821		pr_err("invalid node id: %d\n", nid);
3822		return -EINVAL;
3823	}
3824
3825	new_attr = kzalloc(sizeof(*new_attr), GFP_KERNEL);
3826	if (!new_attr)
3827		return -ENOMEM;
3828
3829	name = kasprintf(GFP_KERNEL, "node%d", nid);
3830	if (!name) {
3831		kfree(new_attr);
3832		return -ENOMEM;
3833	}
3834
3835	sysfs_attr_init(&new_attr->kobj_attr.attr);
3836	new_attr->kobj_attr.attr.name = name;
3837	new_attr->kobj_attr.attr.mode = 0644;
3838	new_attr->kobj_attr.show = node_show;
3839	new_attr->kobj_attr.store = node_store;
3840	new_attr->nid = nid;
3841
3842	mutex_lock(&wi_group->kobj_lock);
3843	if (wi_group->nattrs[nid]) {
3844		mutex_unlock(&wi_group->kobj_lock);
3845		ret = -EEXIST;
3846		goto out;
3847	}
3848
3849	ret = sysfs_create_file(&wi_group->wi_kobj, &new_attr->kobj_attr.attr);
3850	if (ret) {
3851		mutex_unlock(&wi_group->kobj_lock);
3852		goto out;
3853	}
3854	wi_group->nattrs[nid] = new_attr;
3855	mutex_unlock(&wi_group->kobj_lock);
3856	return 0;
3857
3858out:
3859	kfree(new_attr->kobj_attr.attr.name);
3860	kfree(new_attr);
3861	return ret;
3862}
3863
3864static int wi_node_notifier(struct notifier_block *nb,
3865			       unsigned long action, void *data)
3866{
3867	int err;
3868	struct node_notify *nn = data;
3869	int nid = nn->nid;
3870
3871	switch (action) {
3872	case NODE_ADDED_FIRST_MEMORY:
3873		err = sysfs_wi_node_add(nid);
3874		if (err)
3875			pr_err("failed to add sysfs for node%d during hotplug: %d\n",
3876			       nid, err);
3877		break;
3878	case NODE_REMOVED_LAST_MEMORY:
3879		sysfs_wi_node_delete(nid);
3880		break;
3881	}
3882
3883	return NOTIFY_OK;
3884}
3885
3886static int __init add_weighted_interleave_group(struct kobject *mempolicy_kobj)
3887{
3888	int nid, err;
3889
3890	wi_group = kzalloc(struct_size(wi_group, nattrs, nr_node_ids),
3891			   GFP_KERNEL);
3892	if (!wi_group)
3893		return -ENOMEM;
3894	mutex_init(&wi_group->kobj_lock);
3895
3896	err = kobject_init_and_add(&wi_group->wi_kobj, &wi_ktype, mempolicy_kobj,
3897				   "weighted_interleave");
3898	if (err)
3899		goto err_put_kobj;
3900
3901	err = sysfs_create_file(&wi_group->wi_kobj, &wi_auto_attr.attr);
3902	if (err)
3903		goto err_put_kobj;
3904
3905	for_each_online_node(nid) {
3906		if (!node_state(nid, N_MEMORY))
3907			continue;
3908
3909		err = sysfs_wi_node_add(nid);
3910		if (err) {
3911			pr_err("failed to add sysfs for node%d during init: %d\n",
3912			       nid, err);
3913			goto err_cleanup_kobj;
3914		}
3915	}
3916
3917	hotplug_node_notifier(wi_node_notifier, DEFAULT_CALLBACK_PRI);
3918	return 0;
3919
3920err_cleanup_kobj:
3921	wi_cleanup();
3922	kobject_del(&wi_group->wi_kobj);
3923err_put_kobj:
3924	kobject_put(&wi_group->wi_kobj);
3925	return err;
3926}
3927
3928static int __init mempolicy_sysfs_init(void)
3929{
3930	int err;
3931	static struct kobject *mempolicy_kobj;
3932
3933	mempolicy_kobj = kobject_create_and_add("mempolicy", mm_kobj);
3934	if (!mempolicy_kobj)
3935		return -ENOMEM;
3936
3937	err = add_weighted_interleave_group(mempolicy_kobj);
3938	if (err)
3939		goto err_kobj;
3940
3941	return 0;
3942
3943err_kobj:
3944	kobject_del(mempolicy_kobj);
3945	kobject_put(mempolicy_kobj);
3946	return err;
3947}
3948
3949late_initcall(mempolicy_sysfs_init);
3950#endif /* CONFIG_SYSFS */