huge_memory.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  Copyright (C) 2009  Red Hat, Inc.
   4 */
   5
   6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8#include <linux/mm.h>
   9#include <linux/sched.h>
  10#include <linux/sched/mm.h>
  11#include <linux/sched/numa_balancing.h>
  12#include <linux/highmem.h>
  13#include <linux/hugetlb.h>
  14#include <linux/mmu_notifier.h>
  15#include <linux/rmap.h>
  16#include <linux/swap.h>
  17#include <linux/shrinker.h>
  18#include <linux/mm_inline.h>
  19#include <linux/swapops.h>
  20#include <linux/backing-dev.h>
  21#include <linux/dax.h>
  22#include <linux/mm_types.h>
  23#include <linux/khugepaged.h>
  24#include <linux/freezer.h>
  25#include <linux/mman.h>
  26#include <linux/memremap.h>
  27#include <linux/pagemap.h>
  28#include <linux/debugfs.h>
  29#include <linux/migrate.h>
  30#include <linux/hashtable.h>
  31#include <linux/userfaultfd_k.h>
  32#include <linux/page_idle.h>
  33#include <linux/shmem_fs.h>
  34#include <linux/oom.h>
  35#include <linux/numa.h>
  36#include <linux/page_owner.h>
  37#include <linux/sched/sysctl.h>
  38#include <linux/memory-tiers.h>
  39#include <linux/compat.h>
  40#include <linux/pgalloc.h>
  41#include <linux/pgalloc_tag.h>
  42#include <linux/pagewalk.h>
  43
  44#include <asm/tlb.h>
  45#include "internal.h"
  46#include "swap.h"
  47
  48#define CREATE_TRACE_POINTS
  49#include <trace/events/thp.h>
  50
  51/*
  52 * By default, transparent hugepage support is disabled in order to avoid
  53 * risking an increased memory footprint for applications that are not
  54 * guaranteed to benefit from it. When transparent hugepage support is
  55 * enabled, it is for all mappings, and khugepaged scans all mappings.
  56 * Defrag is invoked by khugepaged hugepage allocations and by page faults
  57 * for all hugepage allocations.
  58 */
  59unsigned long transparent_hugepage_flags __read_mostly =
  60#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
  61	(1<<TRANSPARENT_HUGEPAGE_FLAG)|
  62#endif
  63#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
  64	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
  65#endif
  66	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
  67	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
  68	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
  69
  70static struct shrinker *deferred_split_shrinker;
  71static unsigned long deferred_split_count(struct shrinker *shrink,
  72					  struct shrink_control *sc);
  73static unsigned long deferred_split_scan(struct shrinker *shrink,
  74					 struct shrink_control *sc);
  75static bool split_underused_thp = true;
  76
  77static atomic_t huge_zero_refcount;
  78struct folio *huge_zero_folio __read_mostly;
  79unsigned long huge_zero_pfn __read_mostly = ~0UL;
  80unsigned long huge_anon_orders_always __read_mostly;
  81unsigned long huge_anon_orders_madvise __read_mostly;
  82unsigned long huge_anon_orders_inherit __read_mostly;
  83static bool anon_orders_configured __initdata;
  84
  85static inline bool file_thp_enabled(struct vm_area_struct *vma)
  86{
  87	struct inode *inode;
  88
  89	if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS))
  90		return false;
  91
  92	if (!vma->vm_file)
  93		return false;
  94
  95	inode = file_inode(vma->vm_file);
  96
  97	if (IS_ANON_FILE(inode))
  98		return false;
  99
 100	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
 101}
 102
 103unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 104					 vm_flags_t vm_flags,
 105					 enum tva_type type,
 106					 unsigned long orders)
 107{
 108	const bool smaps = type == TVA_SMAPS;
 109	const bool in_pf = type == TVA_PAGEFAULT;
 110	const bool forced_collapse = type == TVA_FORCED_COLLAPSE;
 111	unsigned long supported_orders;
 112
 113	/* Check the intersection of requested and supported orders. */
 114	if (vma_is_anonymous(vma))
 115		supported_orders = THP_ORDERS_ALL_ANON;
 116	else if (vma_is_special_huge(vma))
 117		supported_orders = THP_ORDERS_ALL_SPECIAL;
 118	else
 119		supported_orders = THP_ORDERS_ALL_FILE_DEFAULT;
 120
 121	orders &= supported_orders;
 122	if (!orders)
 123		return 0;
 124
 125	if (!vma->vm_mm)		/* vdso */
 126		return 0;
 127
 128	if (thp_disabled_by_hw() || vma_thp_disabled(vma, vm_flags, forced_collapse))
 129		return 0;
 130
 131	/* khugepaged doesn't collapse DAX vma, but page fault is fine. */
 132	if (vma_is_dax(vma))
 133		return in_pf ? orders : 0;
 134
 135	/*
 136	 * khugepaged special VMA and hugetlb VMA.
 137	 * Must be checked after dax since some dax mappings may have
 138	 * VM_MIXEDMAP set.
 139	 */
 140	if (!in_pf && !smaps && (vm_flags & VM_NO_KHUGEPAGED))
 141		return 0;
 142
 143	/*
 144	 * Check alignment for file vma and size for both file and anon vma by
 145	 * filtering out the unsuitable orders.
 146	 *
 147	 * Skip the check for page fault. Huge fault does the check in fault
 148	 * handlers.
 149	 */
 150	if (!in_pf) {
 151		int order = highest_order(orders);
 152		unsigned long addr;
 153
 154		while (orders) {
 155			addr = vma->vm_end - (PAGE_SIZE << order);
 156			if (thp_vma_suitable_order(vma, addr, order))
 157				break;
 158			order = next_order(&orders, order);
 159		}
 160
 161		if (!orders)
 162			return 0;
 163	}
 164
 165	/*
 166	 * Enabled via shmem mount options or sysfs settings.
 167	 * Must be done before hugepage flags check since shmem has its
 168	 * own flags.
 169	 */
 170	if (!in_pf && shmem_file(vma->vm_file))
 171		return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file),
 172						   vma, vma->vm_pgoff, 0,
 173						   forced_collapse);
 174
 175	if (!vma_is_anonymous(vma)) {
 176		/*
 177		 * Enforce THP collapse requirements as necessary. Anonymous vmas
 178		 * were already handled in thp_vma_allowable_orders().
 179		 */
 180		if (!forced_collapse &&
 181		    (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) &&
 182						    !hugepage_global_always())))
 183			return 0;
 184
 185		/*
 186		 * Trust that ->huge_fault() handlers know what they are doing
 187		 * in fault path.
 188		 */
 189		if (((in_pf || smaps)) && vma->vm_ops->huge_fault)
 190			return orders;
 191		/* Only regular file is valid in collapse path */
 192		if (((!in_pf || smaps)) && file_thp_enabled(vma))
 193			return orders;
 194		return 0;
 195	}
 196
 197	if (vma_is_temporary_stack(vma))
 198		return 0;
 199
 200	/*
 201	 * THPeligible bit of smaps should show 1 for proper VMAs even
 202	 * though anon_vma is not initialized yet.
 203	 *
 204	 * Allow page fault since anon_vma may be not initialized until
 205	 * the first page fault.
 206	 */
 207	if (!vma->anon_vma)
 208		return (smaps || in_pf) ? orders : 0;
 209
 210	return orders;
 211}
 212
 213static bool get_huge_zero_folio(void)
 214{
 215	struct folio *zero_folio;
 216retry:
 217	if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
 218		return true;
 219
 220	zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) &
 221				 ~__GFP_MOVABLE,
 222			HPAGE_PMD_ORDER);
 223	if (!zero_folio) {
 224		count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
 225		return false;
 226	}
 227	/* Ensure zero folio won't have large_rmappable flag set. */
 228	folio_clear_large_rmappable(zero_folio);
 229	preempt_disable();
 230	if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) {
 231		preempt_enable();
 232		folio_put(zero_folio);
 233		goto retry;
 234	}
 235	WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio));
 236
 237	/* We take additional reference here. It will be put back by shrinker */
 238	atomic_set(&huge_zero_refcount, 2);
 239	preempt_enable();
 240	count_vm_event(THP_ZERO_PAGE_ALLOC);
 241	return true;
 242}
 243
 244static void put_huge_zero_folio(void)
 245{
 246	/*
 247	 * Counter should never go to zero here. Only shrinker can put
 248	 * last reference.
 249	 */
 250	BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
 251}
 252
 253struct folio *mm_get_huge_zero_folio(struct mm_struct *mm)
 254{
 255	if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO))
 256		return huge_zero_folio;
 257
 258	if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm))
 259		return READ_ONCE(huge_zero_folio);
 260
 261	if (!get_huge_zero_folio())
 262		return NULL;
 263
 264	if (mm_flags_test_and_set(MMF_HUGE_ZERO_FOLIO, mm))
 265		put_huge_zero_folio();
 266
 267	return READ_ONCE(huge_zero_folio);
 268}
 269
 270void mm_put_huge_zero_folio(struct mm_struct *mm)
 271{
 272	if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO))
 273		return;
 274
 275	if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm))
 276		put_huge_zero_folio();
 277}
 278
 279static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink,
 280						  struct shrink_control *sc)
 281{
 282	/* we can free zero page only if last reference remains */
 283	return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
 284}
 285
 286static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink,
 287						 struct shrink_control *sc)
 288{
 289	if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
 290		struct folio *zero_folio = xchg(&huge_zero_folio, NULL);
 291		BUG_ON(zero_folio == NULL);
 292		WRITE_ONCE(huge_zero_pfn, ~0UL);
 293		folio_put(zero_folio);
 294		return HPAGE_PMD_NR;
 295	}
 296
 297	return 0;
 298}
 299
 300static struct shrinker *huge_zero_folio_shrinker;
 301
 302#ifdef CONFIG_SYSFS
 303static ssize_t enabled_show(struct kobject *kobj,
 304			    struct kobj_attribute *attr, char *buf)
 305{
 306	const char *output;
 307
 308	if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
 309		output = "[always] madvise never";
 310	else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 311			  &transparent_hugepage_flags))
 312		output = "always [madvise] never";
 313	else
 314		output = "always madvise [never]";
 315
 316	return sysfs_emit(buf, "%s\n", output);
 317}
 318
 319static ssize_t enabled_store(struct kobject *kobj,
 320			     struct kobj_attribute *attr,
 321			     const char *buf, size_t count)
 322{
 323	ssize_t ret = count;
 324
 325	if (sysfs_streq(buf, "always")) {
 326		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
 327		set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
 328	} else if (sysfs_streq(buf, "madvise")) {
 329		clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
 330		set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
 331	} else if (sysfs_streq(buf, "never")) {
 332		clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
 333		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
 334	} else
 335		ret = -EINVAL;
 336
 337	if (ret > 0) {
 338		int err = start_stop_khugepaged();
 339		if (err)
 340			ret = err;
 341	}
 342	return ret;
 343}
 344
 345static struct kobj_attribute enabled_attr = __ATTR_RW(enabled);
 346
 347ssize_t single_hugepage_flag_show(struct kobject *kobj,
 348				  struct kobj_attribute *attr, char *buf,
 349				  enum transparent_hugepage_flag flag)
 350{
 351	return sysfs_emit(buf, "%d\n",
 352			  !!test_bit(flag, &transparent_hugepage_flags));
 353}
 354
 355ssize_t single_hugepage_flag_store(struct kobject *kobj,
 356				 struct kobj_attribute *attr,
 357				 const char *buf, size_t count,
 358				 enum transparent_hugepage_flag flag)
 359{
 360	unsigned long value;
 361	int ret;
 362
 363	ret = kstrtoul(buf, 10, &value);
 364	if (ret < 0)
 365		return ret;
 366	if (value > 1)
 367		return -EINVAL;
 368
 369	if (value)
 370		set_bit(flag, &transparent_hugepage_flags);
 371	else
 372		clear_bit(flag, &transparent_hugepage_flags);
 373
 374	return count;
 375}
 376
 377static ssize_t defrag_show(struct kobject *kobj,
 378			   struct kobj_attribute *attr, char *buf)
 379{
 380	const char *output;
 381
 382	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
 383		     &transparent_hugepage_flags))
 384		output = "[always] defer defer+madvise madvise never";
 385	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
 386			  &transparent_hugepage_flags))
 387		output = "always [defer] defer+madvise madvise never";
 388	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
 389			  &transparent_hugepage_flags))
 390		output = "always defer [defer+madvise] madvise never";
 391	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
 392			  &transparent_hugepage_flags))
 393		output = "always defer defer+madvise [madvise] never";
 394	else
 395		output = "always defer defer+madvise madvise [never]";
 396
 397	return sysfs_emit(buf, "%s\n", output);
 398}
 399
 400static ssize_t defrag_store(struct kobject *kobj,
 401			    struct kobj_attribute *attr,
 402			    const char *buf, size_t count)
 403{
 404	if (sysfs_streq(buf, "always")) {
 405		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
 406		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
 407		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
 408		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
 409	} else if (sysfs_streq(buf, "defer+madvise")) {
 410		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
 411		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
 412		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
 413		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
 414	} else if (sysfs_streq(buf, "defer")) {
 415		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
 416		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
 417		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
 418		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
 419	} else if (sysfs_streq(buf, "madvise")) {
 420		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
 421		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
 422		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
 423		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
 424	} else if (sysfs_streq(buf, "never")) {
 425		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
 426		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
 427		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
 428		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
 429	} else
 430		return -EINVAL;
 431
 432	return count;
 433}
 434static struct kobj_attribute defrag_attr = __ATTR_RW(defrag);
 435
 436static ssize_t use_zero_page_show(struct kobject *kobj,
 437				  struct kobj_attribute *attr, char *buf)
 438{
 439	return single_hugepage_flag_show(kobj, attr, buf,
 440					 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
 441}
 442static ssize_t use_zero_page_store(struct kobject *kobj,
 443		struct kobj_attribute *attr, const char *buf, size_t count)
 444{
 445	return single_hugepage_flag_store(kobj, attr, buf, count,
 446				 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
 447}
 448static struct kobj_attribute use_zero_page_attr = __ATTR_RW(use_zero_page);
 449
 450static ssize_t hpage_pmd_size_show(struct kobject *kobj,
 451				   struct kobj_attribute *attr, char *buf)
 452{
 453	return sysfs_emit(buf, "%lu\n", HPAGE_PMD_SIZE);
 454}
 455static struct kobj_attribute hpage_pmd_size_attr =
 456	__ATTR_RO(hpage_pmd_size);
 457
 458static ssize_t split_underused_thp_show(struct kobject *kobj,
 459			    struct kobj_attribute *attr, char *buf)
 460{
 461	return sysfs_emit(buf, "%d\n", split_underused_thp);
 462}
 463
 464static ssize_t split_underused_thp_store(struct kobject *kobj,
 465			     struct kobj_attribute *attr,
 466			     const char *buf, size_t count)
 467{
 468	int err = kstrtobool(buf, &split_underused_thp);
 469
 470	if (err < 0)
 471		return err;
 472
 473	return count;
 474}
 475
 476static struct kobj_attribute split_underused_thp_attr = __ATTR(
 477	shrink_underused, 0644, split_underused_thp_show, split_underused_thp_store);
 478
 479static struct attribute *hugepage_attr[] = {
 480	&enabled_attr.attr,
 481	&defrag_attr.attr,
 482	&use_zero_page_attr.attr,
 483	&hpage_pmd_size_attr.attr,
 484#ifdef CONFIG_SHMEM
 485	&shmem_enabled_attr.attr,
 486#endif
 487	&split_underused_thp_attr.attr,
 488	NULL,
 489};
 490
 491static const struct attribute_group hugepage_attr_group = {
 492	.attrs = hugepage_attr,
 493};
 494
 495static void hugepage_exit_sysfs(struct kobject *hugepage_kobj);
 496static void thpsize_release(struct kobject *kobj);
 497static DEFINE_SPINLOCK(huge_anon_orders_lock);
 498static LIST_HEAD(thpsize_list);
 499
 500static ssize_t anon_enabled_show(struct kobject *kobj,
 501				 struct kobj_attribute *attr, char *buf)
 502{
 503	int order = to_thpsize(kobj)->order;
 504	const char *output;
 505
 506	if (test_bit(order, &huge_anon_orders_always))
 507		output = "[always] inherit madvise never";
 508	else if (test_bit(order, &huge_anon_orders_inherit))
 509		output = "always [inherit] madvise never";
 510	else if (test_bit(order, &huge_anon_orders_madvise))
 511		output = "always inherit [madvise] never";
 512	else
 513		output = "always inherit madvise [never]";
 514
 515	return sysfs_emit(buf, "%s\n", output);
 516}
 517
 518static ssize_t anon_enabled_store(struct kobject *kobj,
 519				  struct kobj_attribute *attr,
 520				  const char *buf, size_t count)
 521{
 522	int order = to_thpsize(kobj)->order;
 523	ssize_t ret = count;
 524
 525	if (sysfs_streq(buf, "always")) {
 526		spin_lock(&huge_anon_orders_lock);
 527		clear_bit(order, &huge_anon_orders_inherit);
 528		clear_bit(order, &huge_anon_orders_madvise);
 529		set_bit(order, &huge_anon_orders_always);
 530		spin_unlock(&huge_anon_orders_lock);
 531	} else if (sysfs_streq(buf, "inherit")) {
 532		spin_lock(&huge_anon_orders_lock);
 533		clear_bit(order, &huge_anon_orders_always);
 534		clear_bit(order, &huge_anon_orders_madvise);
 535		set_bit(order, &huge_anon_orders_inherit);
 536		spin_unlock(&huge_anon_orders_lock);
 537	} else if (sysfs_streq(buf, "madvise")) {
 538		spin_lock(&huge_anon_orders_lock);
 539		clear_bit(order, &huge_anon_orders_always);
 540		clear_bit(order, &huge_anon_orders_inherit);
 541		set_bit(order, &huge_anon_orders_madvise);
 542		spin_unlock(&huge_anon_orders_lock);
 543	} else if (sysfs_streq(buf, "never")) {
 544		spin_lock(&huge_anon_orders_lock);
 545		clear_bit(order, &huge_anon_orders_always);
 546		clear_bit(order, &huge_anon_orders_inherit);
 547		clear_bit(order, &huge_anon_orders_madvise);
 548		spin_unlock(&huge_anon_orders_lock);
 549	} else
 550		ret = -EINVAL;
 551
 552	if (ret > 0) {
 553		int err;
 554
 555		err = start_stop_khugepaged();
 556		if (err)
 557			ret = err;
 558	}
 559	return ret;
 560}
 561
 562static struct kobj_attribute anon_enabled_attr =
 563	__ATTR(enabled, 0644, anon_enabled_show, anon_enabled_store);
 564
 565static struct attribute *anon_ctrl_attrs[] = {
 566	&anon_enabled_attr.attr,
 567	NULL,
 568};
 569
 570static const struct attribute_group anon_ctrl_attr_grp = {
 571	.attrs = anon_ctrl_attrs,
 572};
 573
 574static struct attribute *file_ctrl_attrs[] = {
 575#ifdef CONFIG_SHMEM
 576	&thpsize_shmem_enabled_attr.attr,
 577#endif
 578	NULL,
 579};
 580
 581static const struct attribute_group file_ctrl_attr_grp = {
 582	.attrs = file_ctrl_attrs,
 583};
 584
 585static struct attribute *any_ctrl_attrs[] = {
 586	NULL,
 587};
 588
 589static const struct attribute_group any_ctrl_attr_grp = {
 590	.attrs = any_ctrl_attrs,
 591};
 592
 593static const struct kobj_type thpsize_ktype = {
 594	.release = &thpsize_release,
 595	.sysfs_ops = &kobj_sysfs_ops,
 596};
 597
 598DEFINE_PER_CPU(struct mthp_stat, mthp_stats) = {{{0}}};
 599
 600static unsigned long sum_mthp_stat(int order, enum mthp_stat_item item)
 601{
 602	unsigned long sum = 0;
 603	int cpu;
 604
 605	for_each_possible_cpu(cpu) {
 606		struct mthp_stat *this = &per_cpu(mthp_stats, cpu);
 607
 608		sum += this->stats[order][item];
 609	}
 610
 611	return sum;
 612}
 613
 614#define DEFINE_MTHP_STAT_ATTR(_name, _index)				\
 615static ssize_t _name##_show(struct kobject *kobj,			\
 616			struct kobj_attribute *attr, char *buf)		\
 617{									\
 618	int order = to_thpsize(kobj)->order;				\
 619									\
 620	return sysfs_emit(buf, "%lu\n", sum_mthp_stat(order, _index));	\
 621}									\
 622static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
 623
 624DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC);
 625DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK);
 626DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
 627DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT);
 628DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN);
 629DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK);
 630DEFINE_MTHP_STAT_ATTR(swpin_fallback_charge, MTHP_STAT_SWPIN_FALLBACK_CHARGE);
 631DEFINE_MTHP_STAT_ATTR(swpout, MTHP_STAT_SWPOUT);
 632DEFINE_MTHP_STAT_ATTR(swpout_fallback, MTHP_STAT_SWPOUT_FALLBACK);
 633#ifdef CONFIG_SHMEM
 634DEFINE_MTHP_STAT_ATTR(shmem_alloc, MTHP_STAT_SHMEM_ALLOC);
 635DEFINE_MTHP_STAT_ATTR(shmem_fallback, MTHP_STAT_SHMEM_FALLBACK);
 636DEFINE_MTHP_STAT_ATTR(shmem_fallback_charge, MTHP_STAT_SHMEM_FALLBACK_CHARGE);
 637#endif
 638DEFINE_MTHP_STAT_ATTR(split, MTHP_STAT_SPLIT);
 639DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED);
 640DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED);
 641DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON);
 642DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED);
 643
 644static struct attribute *anon_stats_attrs[] = {
 645	&anon_fault_alloc_attr.attr,
 646	&anon_fault_fallback_attr.attr,
 647	&anon_fault_fallback_charge_attr.attr,
 648#ifndef CONFIG_SHMEM
 649	&zswpout_attr.attr,
 650	&swpin_attr.attr,
 651	&swpin_fallback_attr.attr,
 652	&swpin_fallback_charge_attr.attr,
 653	&swpout_attr.attr,
 654	&swpout_fallback_attr.attr,
 655#endif
 656	&split_deferred_attr.attr,
 657	&nr_anon_attr.attr,
 658	&nr_anon_partially_mapped_attr.attr,
 659	NULL,
 660};
 661
 662static struct attribute_group anon_stats_attr_grp = {
 663	.name = "stats",
 664	.attrs = anon_stats_attrs,
 665};
 666
 667static struct attribute *file_stats_attrs[] = {
 668#ifdef CONFIG_SHMEM
 669	&shmem_alloc_attr.attr,
 670	&shmem_fallback_attr.attr,
 671	&shmem_fallback_charge_attr.attr,
 672#endif
 673	NULL,
 674};
 675
 676static struct attribute_group file_stats_attr_grp = {
 677	.name = "stats",
 678	.attrs = file_stats_attrs,
 679};
 680
 681static struct attribute *any_stats_attrs[] = {
 682#ifdef CONFIG_SHMEM
 683	&zswpout_attr.attr,
 684	&swpin_attr.attr,
 685	&swpin_fallback_attr.attr,
 686	&swpin_fallback_charge_attr.attr,
 687	&swpout_attr.attr,
 688	&swpout_fallback_attr.attr,
 689#endif
 690	&split_attr.attr,
 691	&split_failed_attr.attr,
 692	NULL,
 693};
 694
 695static struct attribute_group any_stats_attr_grp = {
 696	.name = "stats",
 697	.attrs = any_stats_attrs,
 698};
 699
 700static int sysfs_add_group(struct kobject *kobj,
 701			   const struct attribute_group *grp)
 702{
 703	int ret = -ENOENT;
 704
 705	/*
 706	 * If the group is named, try to merge first, assuming the subdirectory
 707	 * was already created. This avoids the warning emitted by
 708	 * sysfs_create_group() if the directory already exists.
 709	 */
 710	if (grp->name)
 711		ret = sysfs_merge_group(kobj, grp);
 712	if (ret)
 713		ret = sysfs_create_group(kobj, grp);
 714
 715	return ret;
 716}
 717
 718static struct thpsize *thpsize_create(int order, struct kobject *parent)
 719{
 720	unsigned long size = (PAGE_SIZE << order) / SZ_1K;
 721	struct thpsize *thpsize;
 722	int ret = -ENOMEM;
 723
 724	thpsize = kzalloc(sizeof(*thpsize), GFP_KERNEL);
 725	if (!thpsize)
 726		goto err;
 727
 728	thpsize->order = order;
 729
 730	ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent,
 731				   "hugepages-%lukB", size);
 732	if (ret) {
 733		kfree(thpsize);
 734		goto err;
 735	}
 736
 737
 738	ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp);
 739	if (ret)
 740		goto err_put;
 741
 742	ret = sysfs_add_group(&thpsize->kobj, &any_stats_attr_grp);
 743	if (ret)
 744		goto err_put;
 745
 746	if (BIT(order) & THP_ORDERS_ALL_ANON) {
 747		ret = sysfs_add_group(&thpsize->kobj, &anon_ctrl_attr_grp);
 748		if (ret)
 749			goto err_put;
 750
 751		ret = sysfs_add_group(&thpsize->kobj, &anon_stats_attr_grp);
 752		if (ret)
 753			goto err_put;
 754	}
 755
 756	if (BIT(order) & THP_ORDERS_ALL_FILE_DEFAULT) {
 757		ret = sysfs_add_group(&thpsize->kobj, &file_ctrl_attr_grp);
 758		if (ret)
 759			goto err_put;
 760
 761		ret = sysfs_add_group(&thpsize->kobj, &file_stats_attr_grp);
 762		if (ret)
 763			goto err_put;
 764	}
 765
 766	return thpsize;
 767err_put:
 768	kobject_put(&thpsize->kobj);
 769err:
 770	return ERR_PTR(ret);
 771}
 772
 773static void thpsize_release(struct kobject *kobj)
 774{
 775	kfree(to_thpsize(kobj));
 776}
 777
 778static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
 779{
 780	int err;
 781	struct thpsize *thpsize;
 782	unsigned long orders;
 783	int order;
 784
 785	/*
 786	 * Default to setting PMD-sized THP to inherit the global setting and
 787	 * disable all other sizes. powerpc's PMD_ORDER isn't a compile-time
 788	 * constant so we have to do this here.
 789	 */
 790	if (!anon_orders_configured)
 791		huge_anon_orders_inherit = BIT(PMD_ORDER);
 792
 793	*hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
 794	if (unlikely(!*hugepage_kobj)) {
 795		pr_err("failed to create transparent hugepage kobject\n");
 796		return -ENOMEM;
 797	}
 798
 799	err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
 800	if (err) {
 801		pr_err("failed to register transparent hugepage group\n");
 802		goto delete_obj;
 803	}
 804
 805	err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
 806	if (err) {
 807		pr_err("failed to register transparent hugepage group\n");
 808		goto remove_hp_group;
 809	}
 810
 811	orders = THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_FILE_DEFAULT;
 812	order = highest_order(orders);
 813	while (orders) {
 814		thpsize = thpsize_create(order, *hugepage_kobj);
 815		if (IS_ERR(thpsize)) {
 816			pr_err("failed to create thpsize for order %d\n", order);
 817			err = PTR_ERR(thpsize);
 818			goto remove_all;
 819		}
 820		list_add(&thpsize->node, &thpsize_list);
 821		order = next_order(&orders, order);
 822	}
 823
 824	return 0;
 825
 826remove_all:
 827	hugepage_exit_sysfs(*hugepage_kobj);
 828	return err;
 829remove_hp_group:
 830	sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
 831delete_obj:
 832	kobject_put(*hugepage_kobj);
 833	return err;
 834}
 835
 836static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
 837{
 838	struct thpsize *thpsize, *tmp;
 839
 840	list_for_each_entry_safe(thpsize, tmp, &thpsize_list, node) {
 841		list_del(&thpsize->node);
 842		kobject_put(&thpsize->kobj);
 843	}
 844
 845	sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
 846	sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
 847	kobject_put(hugepage_kobj);
 848}
 849#else
 850static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
 851{
 852	return 0;
 853}
 854
 855static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
 856{
 857}
 858#endif /* CONFIG_SYSFS */
 859
 860static int __init thp_shrinker_init(void)
 861{
 862	deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE |
 863						 SHRINKER_MEMCG_AWARE |
 864						 SHRINKER_NONSLAB,
 865						 "thp-deferred_split");
 866	if (!deferred_split_shrinker)
 867		return -ENOMEM;
 868
 869	deferred_split_shrinker->count_objects = deferred_split_count;
 870	deferred_split_shrinker->scan_objects = deferred_split_scan;
 871	shrinker_register(deferred_split_shrinker);
 872
 873	if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) {
 874		/*
 875		 * Bump the reference of the huge_zero_folio and do not
 876		 * initialize the shrinker.
 877		 *
 878		 * huge_zero_folio will always be NULL on failure. We assume
 879		 * that get_huge_zero_folio() will most likely not fail as
 880		 * thp_shrinker_init() is invoked early on during boot.
 881		 */
 882		if (!get_huge_zero_folio())
 883			pr_warn("Allocating persistent huge zero folio failed\n");
 884		return 0;
 885	}
 886
 887	huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero");
 888	if (!huge_zero_folio_shrinker) {
 889		shrinker_free(deferred_split_shrinker);
 890		return -ENOMEM;
 891	}
 892
 893	huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count;
 894	huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan;
 895	shrinker_register(huge_zero_folio_shrinker);
 896
 897	return 0;
 898}
 899
 900static void __init thp_shrinker_exit(void)
 901{
 902	shrinker_free(huge_zero_folio_shrinker);
 903	shrinker_free(deferred_split_shrinker);
 904}
 905
 906static int __init hugepage_init(void)
 907{
 908	int err;
 909	struct kobject *hugepage_kobj;
 910
 911	if (!has_transparent_hugepage()) {
 912		transparent_hugepage_flags = 1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED;
 913		return -EINVAL;
 914	}
 915
 916	/*
 917	 * hugepages can't be allocated by the buddy allocator
 918	 */
 919	MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER > MAX_PAGE_ORDER);
 920
 921	err = hugepage_init_sysfs(&hugepage_kobj);
 922	if (err)
 923		goto err_sysfs;
 924
 925	err = khugepaged_init();
 926	if (err)
 927		goto err_slab;
 928
 929	err = thp_shrinker_init();
 930	if (err)
 931		goto err_shrinker;
 932
 933	/*
 934	 * By default disable transparent hugepages on smaller systems,
 935	 * where the extra memory used could hurt more than TLB overhead
 936	 * is likely to save.  The admin can still enable it through /sys.
 937	 */
 938	if (totalram_pages() < MB_TO_PAGES(512)) {
 939		transparent_hugepage_flags = 0;
 940		return 0;
 941	}
 942
 943	err = start_stop_khugepaged();
 944	if (err)
 945		goto err_khugepaged;
 946
 947	return 0;
 948err_khugepaged:
 949	thp_shrinker_exit();
 950err_shrinker:
 951	khugepaged_destroy();
 952err_slab:
 953	hugepage_exit_sysfs(hugepage_kobj);
 954err_sysfs:
 955	return err;
 956}
 957subsys_initcall(hugepage_init);
 958
 959static int __init setup_transparent_hugepage(char *str)
 960{
 961	int ret = 0;
 962	if (!str)
 963		goto out;
 964	if (!strcmp(str, "always")) {
 965		set_bit(TRANSPARENT_HUGEPAGE_FLAG,
 966			&transparent_hugepage_flags);
 967		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 968			  &transparent_hugepage_flags);
 969		ret = 1;
 970	} else if (!strcmp(str, "madvise")) {
 971		clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
 972			  &transparent_hugepage_flags);
 973		set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 974			&transparent_hugepage_flags);
 975		ret = 1;
 976	} else if (!strcmp(str, "never")) {
 977		clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
 978			  &transparent_hugepage_flags);
 979		clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
 980			  &transparent_hugepage_flags);
 981		ret = 1;
 982	}
 983out:
 984	if (!ret)
 985		pr_warn("transparent_hugepage= cannot parse, ignored\n");
 986	return ret;
 987}
 988__setup("transparent_hugepage=", setup_transparent_hugepage);
 989
 990static char str_dup[PAGE_SIZE] __initdata;
 991static int __init setup_thp_anon(char *str)
 992{
 993	char *token, *range, *policy, *subtoken;
 994	unsigned long always, inherit, madvise;
 995	char *start_size, *end_size;
 996	int start, end, nr;
 997	char *p;
 998
 999	if (!str || strlen(str) + 1 > PAGE_SIZE)
1000		goto err;
1001	strscpy(str_dup, str);
1002
1003	always = huge_anon_orders_always;
1004	madvise = huge_anon_orders_madvise;
1005	inherit = huge_anon_orders_inherit;
1006	p = str_dup;
1007	while ((token = strsep(&p, ";")) != NULL) {
1008		range = strsep(&token, ":");
1009		policy = token;
1010
1011		if (!policy)
1012			goto err;
1013
1014		while ((subtoken = strsep(&range, ",")) != NULL) {
1015			if (strchr(subtoken, '-')) {
1016				start_size = strsep(&subtoken, "-");
1017				end_size = subtoken;
1018
1019				start = get_order_from_str(start_size, THP_ORDERS_ALL_ANON);
1020				end = get_order_from_str(end_size, THP_ORDERS_ALL_ANON);
1021			} else {
1022				start_size = end_size = subtoken;
1023				start = end = get_order_from_str(subtoken,
1024								 THP_ORDERS_ALL_ANON);
1025			}
1026
1027			if (start == -EINVAL) {
1028				pr_err("invalid size %s in thp_anon boot parameter\n", start_size);
1029				goto err;
1030			}
1031
1032			if (end == -EINVAL) {
1033				pr_err("invalid size %s in thp_anon boot parameter\n", end_size);
1034				goto err;
1035			}
1036
1037			if (start < 0 || end < 0 || start > end)
1038				goto err;
1039
1040			nr = end - start + 1;
1041			if (!strcmp(policy, "always")) {
1042				bitmap_set(&always, start, nr);
1043				bitmap_clear(&inherit, start, nr);
1044				bitmap_clear(&madvise, start, nr);
1045			} else if (!strcmp(policy, "madvise")) {
1046				bitmap_set(&madvise, start, nr);
1047				bitmap_clear(&inherit, start, nr);
1048				bitmap_clear(&always, start, nr);
1049			} else if (!strcmp(policy, "inherit")) {
1050				bitmap_set(&inherit, start, nr);
1051				bitmap_clear(&madvise, start, nr);
1052				bitmap_clear(&always, start, nr);
1053			} else if (!strcmp(policy, "never")) {
1054				bitmap_clear(&inherit, start, nr);
1055				bitmap_clear(&madvise, start, nr);
1056				bitmap_clear(&always, start, nr);
1057			} else {
1058				pr_err("invalid policy %s in thp_anon boot parameter\n", policy);
1059				goto err;
1060			}
1061		}
1062	}
1063
1064	huge_anon_orders_always = always;
1065	huge_anon_orders_madvise = madvise;
1066	huge_anon_orders_inherit = inherit;
1067	anon_orders_configured = true;
1068	return 1;
1069
1070err:
1071	pr_warn("thp_anon=%s: error parsing string, ignoring setting\n", str);
1072	return 0;
1073}
1074__setup("thp_anon=", setup_thp_anon);
1075
1076pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
1077{
1078	if (likely(vma->vm_flags & VM_WRITE))
1079		pmd = pmd_mkwrite(pmd, vma);
1080	return pmd;
1081}
1082
1083static struct deferred_split *split_queue_node(int nid)
1084{
1085	struct pglist_data *pgdata = NODE_DATA(nid);
1086
1087	return &pgdata->deferred_split_queue;
1088}
1089
1090#ifdef CONFIG_MEMCG
1091static inline
1092struct mem_cgroup *folio_split_queue_memcg(struct folio *folio,
1093					   struct deferred_split *queue)
1094{
1095	if (mem_cgroup_disabled())
1096		return NULL;
1097	if (split_queue_node(folio_nid(folio)) == queue)
1098		return NULL;
1099	return container_of(queue, struct mem_cgroup, deferred_split_queue);
1100}
1101
1102static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg)
1103{
1104	return memcg ? &memcg->deferred_split_queue : split_queue_node(nid);
1105}
1106#else
1107static inline
1108struct mem_cgroup *folio_split_queue_memcg(struct folio *folio,
1109					   struct deferred_split *queue)
1110{
1111	return NULL;
1112}
1113
1114static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg)
1115{
1116	return split_queue_node(nid);
1117}
1118#endif
1119
1120static struct deferred_split *split_queue_lock(int nid, struct mem_cgroup *memcg)
1121{
1122	struct deferred_split *queue;
1123
1124retry:
1125	queue = memcg_split_queue(nid, memcg);
1126	spin_lock(&queue->split_queue_lock);
1127	/*
1128	 * There is a period between setting memcg to dying and reparenting
1129	 * deferred split queue, and during this period the THPs in the deferred
1130	 * split queue will be hidden from the shrinker side.
1131	 */
1132	if (unlikely(memcg_is_dying(memcg))) {
1133		spin_unlock(&queue->split_queue_lock);
1134		memcg = parent_mem_cgroup(memcg);
1135		goto retry;
1136	}
1137
1138	return queue;
1139}
1140
1141static struct deferred_split *
1142split_queue_lock_irqsave(int nid, struct mem_cgroup *memcg, unsigned long *flags)
1143{
1144	struct deferred_split *queue;
1145
1146retry:
1147	queue = memcg_split_queue(nid, memcg);
1148	spin_lock_irqsave(&queue->split_queue_lock, *flags);
1149	if (unlikely(memcg_is_dying(memcg))) {
1150		spin_unlock_irqrestore(&queue->split_queue_lock, *flags);
1151		memcg = parent_mem_cgroup(memcg);
1152		goto retry;
1153	}
1154
1155	return queue;
1156}
1157
1158static struct deferred_split *folio_split_queue_lock(struct folio *folio)
1159{
1160	return split_queue_lock(folio_nid(folio), folio_memcg(folio));
1161}
1162
1163static struct deferred_split *
1164folio_split_queue_lock_irqsave(struct folio *folio, unsigned long *flags)
1165{
1166	return split_queue_lock_irqsave(folio_nid(folio), folio_memcg(folio), flags);
1167}
1168
1169static inline void split_queue_unlock(struct deferred_split *queue)
1170{
1171	spin_unlock(&queue->split_queue_lock);
1172}
1173
1174static inline void split_queue_unlock_irqrestore(struct deferred_split *queue,
1175						 unsigned long flags)
1176{
1177	spin_unlock_irqrestore(&queue->split_queue_lock, flags);
1178}
1179
1180static inline bool is_transparent_hugepage(const struct folio *folio)
1181{
1182	if (!folio_test_large(folio))
1183		return false;
1184
1185	return is_huge_zero_folio(folio) ||
1186		folio_test_large_rmappable(folio);
1187}
1188
1189static unsigned long __thp_get_unmapped_area(struct file *filp,
1190		unsigned long addr, unsigned long len,
1191		loff_t off, unsigned long flags, unsigned long size,
1192		vm_flags_t vm_flags)
1193{
1194	loff_t off_end = off + len;
1195	loff_t off_align = round_up(off, size);
1196	unsigned long len_pad, ret, off_sub;
1197
1198	if (!IS_ENABLED(CONFIG_64BIT) || in_compat_syscall())
1199		return 0;
1200
1201	if (off_end <= off_align || (off_end - off_align) < size)
1202		return 0;
1203
1204	len_pad = len + size;
1205	if (len_pad < len || (off + len_pad) < off)
1206		return 0;
1207
1208	ret = mm_get_unmapped_area_vmflags(filp, addr, len_pad,
1209					   off >> PAGE_SHIFT, flags, vm_flags);
1210
1211	/*
1212	 * The failure might be due to length padding. The caller will retry
1213	 * without the padding.
1214	 */
1215	if (IS_ERR_VALUE(ret))
1216		return 0;
1217
1218	/*
1219	 * Do not try to align to THP boundary if allocation at the address
1220	 * hint succeeds.
1221	 */
1222	if (ret == addr)
1223		return addr;
1224
1225	off_sub = (off - ret) & (size - 1);
1226
1227	if (mm_flags_test(MMF_TOPDOWN, current->mm) && !off_sub)
1228		return ret + size;
1229
1230	ret += off_sub;
1231	return ret;
1232}
1233
1234unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
1235		unsigned long len, unsigned long pgoff, unsigned long flags,
1236		vm_flags_t vm_flags)
1237{
1238	unsigned long ret;
1239	loff_t off = (loff_t)pgoff << PAGE_SHIFT;
1240
1241	ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags);
1242	if (ret)
1243		return ret;
1244
1245	return mm_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags,
1246					    vm_flags);
1247}
1248
1249unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
1250		unsigned long len, unsigned long pgoff, unsigned long flags)
1251{
1252	return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0);
1253}
1254EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
1255
1256static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma,
1257		unsigned long addr)
1258{
1259	gfp_t gfp = vma_thp_gfp_mask(vma);
1260	const int order = HPAGE_PMD_ORDER;
1261	struct folio *folio;
1262
1263	folio = vma_alloc_folio(gfp, order, vma, addr & HPAGE_PMD_MASK);
1264
1265	if (unlikely(!folio)) {
1266		count_vm_event(THP_FAULT_FALLBACK);
1267		count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
1268		return NULL;
1269	}
1270
1271	VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
1272	if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {
1273		folio_put(folio);
1274		count_vm_event(THP_FAULT_FALLBACK);
1275		count_vm_event(THP_FAULT_FALLBACK_CHARGE);
1276		count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
1277		count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
1278		return NULL;
1279	}
1280	folio_throttle_swaprate(folio, gfp);
1281
1282       /*
1283	* When a folio is not zeroed during allocation (__GFP_ZERO not used)
1284	* or user folios require special handling, folio_zero_user() is used to
1285	* make sure that the page corresponding to the faulting address will be
1286	* hot in the cache after zeroing.
1287	*/
1288	if (user_alloc_needs_zeroing())
1289		folio_zero_user(folio, addr);
1290	/*
1291	 * The memory barrier inside __folio_mark_uptodate makes sure that
1292	 * folio_zero_user writes become visible before the set_pmd_at()
1293	 * write.
1294	 */
1295	__folio_mark_uptodate(folio);
1296	return folio;
1297}
1298
1299void map_anon_folio_pmd_nopf(struct folio *folio, pmd_t *pmd,
1300		struct vm_area_struct *vma, unsigned long haddr)
1301{
1302	pmd_t entry;
1303
1304	entry = folio_mk_pmd(folio, vma->vm_page_prot);
1305	entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1306	folio_add_new_anon_rmap(folio, vma, haddr, RMAP_EXCLUSIVE);
1307	folio_add_lru_vma(folio, vma);
1308	set_pmd_at(vma->vm_mm, haddr, pmd, entry);
1309	update_mmu_cache_pmd(vma, haddr, pmd);
1310	deferred_split_folio(folio, false);
1311}
1312
1313static void map_anon_folio_pmd_pf(struct folio *folio, pmd_t *pmd,
1314		struct vm_area_struct *vma, unsigned long haddr)
1315{
1316	map_anon_folio_pmd_nopf(folio, pmd, vma, haddr);
1317	add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1318	count_vm_event(THP_FAULT_ALLOC);
1319	count_mthp_stat(HPAGE_PMD_ORDER, MTHP_STAT_ANON_FAULT_ALLOC);
1320	count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC);
1321}
1322
1323static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf)
1324{
1325	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1326	struct vm_area_struct *vma = vmf->vma;
1327	struct folio *folio;
1328	pgtable_t pgtable;
1329	vm_fault_t ret = 0;
1330
1331	folio = vma_alloc_anon_folio_pmd(vma, vmf->address);
1332	if (unlikely(!folio))
1333		return VM_FAULT_FALLBACK;
1334
1335	pgtable = pte_alloc_one(vma->vm_mm);
1336	if (unlikely(!pgtable)) {
1337		ret = VM_FAULT_OOM;
1338		goto release;
1339	}
1340
1341	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1342	if (unlikely(!pmd_none(*vmf->pmd))) {
1343		goto unlock_release;
1344	} else {
1345		ret = check_stable_address_space(vma->vm_mm);
1346		if (ret)
1347			goto unlock_release;
1348
1349		/* Deliver the page fault to userland */
1350		if (userfaultfd_missing(vma)) {
1351			spin_unlock(vmf->ptl);
1352			folio_put(folio);
1353			pte_free(vma->vm_mm, pgtable);
1354			ret = handle_userfault(vmf, VM_UFFD_MISSING);
1355			VM_BUG_ON(ret & VM_FAULT_FALLBACK);
1356			return ret;
1357		}
1358		pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
1359		map_anon_folio_pmd_pf(folio, vmf->pmd, vma, haddr);
1360		mm_inc_nr_ptes(vma->vm_mm);
1361		spin_unlock(vmf->ptl);
1362	}
1363
1364	return 0;
1365unlock_release:
1366	spin_unlock(vmf->ptl);
1367release:
1368	if (pgtable)
1369		pte_free(vma->vm_mm, pgtable);
1370	folio_put(folio);
1371	return ret;
1372
1373}
1374
1375vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf)
1376{
1377	struct vm_area_struct *vma = vmf->vma;
1378	vm_fault_t ret = 0;
1379	spinlock_t *ptl;
1380	softleaf_t entry;
1381	struct page *page;
1382	struct folio *folio;
1383
1384	if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
1385		vma_end_read(vma);
1386		return VM_FAULT_RETRY;
1387	}
1388
1389	ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1390	if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) {
1391		spin_unlock(ptl);
1392		return 0;
1393	}
1394
1395	entry = softleaf_from_pmd(vmf->orig_pmd);
1396	page = softleaf_to_page(entry);
1397	folio = page_folio(page);
1398	vmf->page = page;
1399	vmf->pte = NULL;
1400	if (folio_trylock(folio)) {
1401		folio_get(folio);
1402		spin_unlock(ptl);
1403		ret = page_pgmap(page)->ops->migrate_to_ram(vmf);
1404		folio_unlock(folio);
1405		folio_put(folio);
1406	} else {
1407		spin_unlock(ptl);
1408	}
1409
1410	return ret;
1411}
1412
1413/*
1414 * always: directly stall for all thp allocations
1415 * defer: wake kswapd and fail if not immediately available
1416 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
1417 *		  fail if not immediately available
1418 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
1419 *	    available
1420 * never: never stall for any thp allocation
1421 */
1422gfp_t vma_thp_gfp_mask(struct vm_area_struct *vma)
1423{
1424	const bool vma_madvised = vma && (vma->vm_flags & VM_HUGEPAGE);
1425
1426	/* Always do synchronous compaction */
1427	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
1428		return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
1429
1430	/* Kick kcompactd and fail quickly */
1431	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
1432		return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
1433
1434	/* Synchronous compaction if madvised, otherwise kick kcompactd */
1435	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
1436		return GFP_TRANSHUGE_LIGHT |
1437			(vma_madvised ? __GFP_DIRECT_RECLAIM :
1438					__GFP_KSWAPD_RECLAIM);
1439
1440	/* Only do synchronous compaction if madvised */
1441	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
1442		return GFP_TRANSHUGE_LIGHT |
1443		       (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);
1444
1445	return GFP_TRANSHUGE_LIGHT;
1446}
1447
1448/* Caller must hold page table lock. */
1449static void set_huge_zero_folio(pgtable_t pgtable, struct mm_struct *mm,
1450		struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
1451		struct folio *zero_folio)
1452{
1453	pmd_t entry;
1454	entry = folio_mk_pmd(zero_folio, vma->vm_page_prot);
1455	entry = pmd_mkspecial(entry);
1456	pgtable_trans_huge_deposit(mm, pmd, pgtable);
1457	set_pmd_at(mm, haddr, pmd, entry);
1458	mm_inc_nr_ptes(mm);
1459}
1460
1461vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
1462{
1463	struct vm_area_struct *vma = vmf->vma;
1464	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1465	vm_fault_t ret;
1466
1467	if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER))
1468		return VM_FAULT_FALLBACK;
1469	ret = vmf_anon_prepare(vmf);
1470	if (ret)
1471		return ret;
1472	khugepaged_enter_vma(vma, vma->vm_flags);
1473
1474	if (!(vmf->flags & FAULT_FLAG_WRITE) &&
1475			!mm_forbids_zeropage(vma->vm_mm) &&
1476			transparent_hugepage_use_zero_page()) {
1477		pgtable_t pgtable;
1478		struct folio *zero_folio;
1479		vm_fault_t ret;
1480
1481		pgtable = pte_alloc_one(vma->vm_mm);
1482		if (unlikely(!pgtable))
1483			return VM_FAULT_OOM;
1484		zero_folio = mm_get_huge_zero_folio(vma->vm_mm);
1485		if (unlikely(!zero_folio)) {
1486			pte_free(vma->vm_mm, pgtable);
1487			count_vm_event(THP_FAULT_FALLBACK);
1488			return VM_FAULT_FALLBACK;
1489		}
1490		vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1491		ret = 0;
1492		if (pmd_none(*vmf->pmd)) {
1493			ret = check_stable_address_space(vma->vm_mm);
1494			if (ret) {
1495				spin_unlock(vmf->ptl);
1496				pte_free(vma->vm_mm, pgtable);
1497			} else if (userfaultfd_missing(vma)) {
1498				spin_unlock(vmf->ptl);
1499				pte_free(vma->vm_mm, pgtable);
1500				ret = handle_userfault(vmf, VM_UFFD_MISSING);
1501				VM_BUG_ON(ret & VM_FAULT_FALLBACK);
1502			} else {
1503				set_huge_zero_folio(pgtable, vma->vm_mm, vma,
1504						   haddr, vmf->pmd, zero_folio);
1505				update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1506				spin_unlock(vmf->ptl);
1507			}
1508		} else {
1509			spin_unlock(vmf->ptl);
1510			pte_free(vma->vm_mm, pgtable);
1511		}
1512		return ret;
1513	}
1514
1515	return __do_huge_pmd_anonymous_page(vmf);
1516}
1517
1518struct folio_or_pfn {
1519	union {
1520		struct folio *folio;
1521		unsigned long pfn;
1522	};
1523	bool is_folio;
1524};
1525
1526static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr,
1527		pmd_t *pmd, struct folio_or_pfn fop, pgprot_t prot,
1528		bool write)
1529{
1530	struct mm_struct *mm = vma->vm_mm;
1531	pgtable_t pgtable = NULL;
1532	spinlock_t *ptl;
1533	pmd_t entry;
1534
1535	if (addr < vma->vm_start || addr >= vma->vm_end)
1536		return VM_FAULT_SIGBUS;
1537
1538	if (arch_needs_pgtable_deposit()) {
1539		pgtable = pte_alloc_one(vma->vm_mm);
1540		if (!pgtable)
1541			return VM_FAULT_OOM;
1542	}
1543
1544	ptl = pmd_lock(mm, pmd);
1545	if (!pmd_none(*pmd)) {
1546		const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) :
1547					  fop.pfn;
1548
1549		if (write) {
1550			if (pmd_pfn(*pmd) != pfn) {
1551				WARN_ON_ONCE(!is_huge_zero_pmd(*pmd));
1552				goto out_unlock;
1553			}
1554			entry = pmd_mkyoung(*pmd);
1555			entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1556			if (pmdp_set_access_flags(vma, addr, pmd, entry, 1))
1557				update_mmu_cache_pmd(vma, addr, pmd);
1558		}
1559		goto out_unlock;
1560	}
1561
1562	if (fop.is_folio) {
1563		entry = folio_mk_pmd(fop.folio, vma->vm_page_prot);
1564
1565		if (is_huge_zero_folio(fop.folio)) {
1566			entry = pmd_mkspecial(entry);
1567		} else {
1568			folio_get(fop.folio);
1569			folio_add_file_rmap_pmd(fop.folio, &fop.folio->page, vma);
1570			add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PMD_NR);
1571		}
1572	} else {
1573		entry = pmd_mkhuge(pfn_pmd(fop.pfn, prot));
1574		entry = pmd_mkspecial(entry);
1575	}
1576	if (write) {
1577		entry = pmd_mkyoung(pmd_mkdirty(entry));
1578		entry = maybe_pmd_mkwrite(entry, vma);
1579	}
1580
1581	if (pgtable) {
1582		pgtable_trans_huge_deposit(mm, pmd, pgtable);
1583		mm_inc_nr_ptes(mm);
1584		pgtable = NULL;
1585	}
1586
1587	set_pmd_at(mm, addr, pmd, entry);
1588	update_mmu_cache_pmd(vma, addr, pmd);
1589
1590out_unlock:
1591	spin_unlock(ptl);
1592	if (pgtable)
1593		pte_free(mm, pgtable);
1594	return VM_FAULT_NOPAGE;
1595}
1596
1597/**
1598 * vmf_insert_pfn_pmd - insert a pmd size pfn
1599 * @vmf: Structure describing the fault
1600 * @pfn: pfn to insert
1601 * @write: whether it's a write fault
1602 *
1603 * Insert a pmd size pfn. See vmf_insert_pfn() for additional info.
1604 *
1605 * Return: vm_fault_t value.
1606 */
1607vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn,
1608			      bool write)
1609{
1610	unsigned long addr = vmf->address & PMD_MASK;
1611	struct vm_area_struct *vma = vmf->vma;
1612	pgprot_t pgprot = vma->vm_page_prot;
1613	struct folio_or_pfn fop = {
1614		.pfn = pfn,
1615	};
1616
1617	/*
1618	 * If we had pmd_special, we could avoid all these restrictions,
1619	 * but we need to be consistent with PTEs and architectures that
1620	 * can't support a 'special' bit.
1621	 */
1622	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
1623	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
1624						(VM_PFNMAP|VM_MIXEDMAP));
1625	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
1626
1627	pfnmap_setup_cachemode_pfn(pfn, &pgprot);
1628
1629	return insert_pmd(vma, addr, vmf->pmd, fop, pgprot, write);
1630}
1631EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
1632
1633vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio,
1634				bool write)
1635{
1636	struct vm_area_struct *vma = vmf->vma;
1637	unsigned long addr = vmf->address & PMD_MASK;
1638	struct folio_or_pfn fop = {
1639		.folio = folio,
1640		.is_folio = true,
1641	};
1642
1643	if (WARN_ON_ONCE(folio_order(folio) != PMD_ORDER))
1644		return VM_FAULT_SIGBUS;
1645
1646	return insert_pmd(vma, addr, vmf->pmd, fop, vma->vm_page_prot, write);
1647}
1648EXPORT_SYMBOL_GPL(vmf_insert_folio_pmd);
1649
1650#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1651static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
1652{
1653	if (likely(vma->vm_flags & VM_WRITE))
1654		pud = pud_mkwrite(pud);
1655	return pud;
1656}
1657
1658static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr,
1659		pud_t *pud, struct folio_or_pfn fop, pgprot_t prot, bool write)
1660{
1661	struct mm_struct *mm = vma->vm_mm;
1662	spinlock_t *ptl;
1663	pud_t entry;
1664
1665	if (addr < vma->vm_start || addr >= vma->vm_end)
1666		return VM_FAULT_SIGBUS;
1667
1668	ptl = pud_lock(mm, pud);
1669	if (!pud_none(*pud)) {
1670		const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) :
1671					  fop.pfn;
1672
1673		if (write) {
1674			if (WARN_ON_ONCE(pud_pfn(*pud) != pfn))
1675				goto out_unlock;
1676			entry = pud_mkyoung(*pud);
1677			entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma);
1678			if (pudp_set_access_flags(vma, addr, pud, entry, 1))
1679				update_mmu_cache_pud(vma, addr, pud);
1680		}
1681		goto out_unlock;
1682	}
1683
1684	if (fop.is_folio) {
1685		entry = folio_mk_pud(fop.folio, vma->vm_page_prot);
1686
1687		folio_get(fop.folio);
1688		folio_add_file_rmap_pud(fop.folio, &fop.folio->page, vma);
1689		add_mm_counter(mm, mm_counter_file(fop.folio), HPAGE_PUD_NR);
1690	} else {
1691		entry = pud_mkhuge(pfn_pud(fop.pfn, prot));
1692		entry = pud_mkspecial(entry);
1693	}
1694	if (write) {
1695		entry = pud_mkyoung(pud_mkdirty(entry));
1696		entry = maybe_pud_mkwrite(entry, vma);
1697	}
1698	set_pud_at(mm, addr, pud, entry);
1699	update_mmu_cache_pud(vma, addr, pud);
1700out_unlock:
1701	spin_unlock(ptl);
1702	return VM_FAULT_NOPAGE;
1703}
1704
1705/**
1706 * vmf_insert_pfn_pud - insert a pud size pfn
1707 * @vmf: Structure describing the fault
1708 * @pfn: pfn to insert
1709 * @write: whether it's a write fault
1710 *
1711 * Insert a pud size pfn. See vmf_insert_pfn() for additional info.
1712 *
1713 * Return: vm_fault_t value.
1714 */
1715vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn,
1716			      bool write)
1717{
1718	unsigned long addr = vmf->address & PUD_MASK;
1719	struct vm_area_struct *vma = vmf->vma;
1720	pgprot_t pgprot = vma->vm_page_prot;
1721	struct folio_or_pfn fop = {
1722		.pfn = pfn,
1723	};
1724
1725	/*
1726	 * If we had pud_special, we could avoid all these restrictions,
1727	 * but we need to be consistent with PTEs and architectures that
1728	 * can't support a 'special' bit.
1729	 */
1730	BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
1731	BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
1732						(VM_PFNMAP|VM_MIXEDMAP));
1733	BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
1734
1735	pfnmap_setup_cachemode_pfn(pfn, &pgprot);
1736
1737	return insert_pud(vma, addr, vmf->pud, fop, pgprot, write);
1738}
1739EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
1740
1741/**
1742 * vmf_insert_folio_pud - insert a pud size folio mapped by a pud entry
1743 * @vmf: Structure describing the fault
1744 * @folio: folio to insert
1745 * @write: whether it's a write fault
1746 *
1747 * Return: vm_fault_t value.
1748 */
1749vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio,
1750				bool write)
1751{
1752	struct vm_area_struct *vma = vmf->vma;
1753	unsigned long addr = vmf->address & PUD_MASK;
1754	struct folio_or_pfn fop = {
1755		.folio = folio,
1756		.is_folio = true,
1757	};
1758
1759	if (WARN_ON_ONCE(folio_order(folio) != PUD_ORDER))
1760		return VM_FAULT_SIGBUS;
1761
1762	return insert_pud(vma, addr, vmf->pud, fop, vma->vm_page_prot, write);
1763}
1764EXPORT_SYMBOL_GPL(vmf_insert_folio_pud);
1765#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
1766
1767/**
1768 * touch_pmd - Mark page table pmd entry as accessed and dirty (for write)
1769 * @vma: The VMA covering @addr
1770 * @addr: The virtual address
1771 * @pmd: pmd pointer into the page table mapping @addr
1772 * @write: Whether it's a write access
1773 *
1774 * Return: whether the pmd entry is changed
1775 */
1776bool touch_pmd(struct vm_area_struct *vma, unsigned long addr,
1777	       pmd_t *pmd, bool write)
1778{
1779	pmd_t entry;
1780
1781	entry = pmd_mkyoung(*pmd);
1782	if (write)
1783		entry = pmd_mkdirty(entry);
1784	if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
1785				  pmd, entry, write)) {
1786		update_mmu_cache_pmd(vma, addr, pmd);
1787		return true;
1788	}
1789
1790	return false;
1791}
1792
1793static void copy_huge_non_present_pmd(
1794		struct mm_struct *dst_mm, struct mm_struct *src_mm,
1795		pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1796		struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1797		pmd_t pmd, pgtable_t pgtable)
1798{
1799	softleaf_t entry = softleaf_from_pmd(pmd);
1800	struct folio *src_folio;
1801
1802	VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(pmd));
1803
1804	if (softleaf_is_migration_write(entry) ||
1805	    softleaf_is_migration_read_exclusive(entry)) {
1806		entry = make_readable_migration_entry(swp_offset(entry));
1807		pmd = swp_entry_to_pmd(entry);
1808		if (pmd_swp_soft_dirty(*src_pmd))
1809			pmd = pmd_swp_mksoft_dirty(pmd);
1810		if (pmd_swp_uffd_wp(*src_pmd))
1811			pmd = pmd_swp_mkuffd_wp(pmd);
1812		set_pmd_at(src_mm, addr, src_pmd, pmd);
1813	} else if (softleaf_is_device_private(entry)) {
1814		/*
1815		 * For device private entries, since there are no
1816		 * read exclusive entries, writable = !readable
1817		 */
1818		if (softleaf_is_device_private_write(entry)) {
1819			entry = make_readable_device_private_entry(swp_offset(entry));
1820			pmd = swp_entry_to_pmd(entry);
1821
1822			if (pmd_swp_soft_dirty(*src_pmd))
1823				pmd = pmd_swp_mksoft_dirty(pmd);
1824			if (pmd_swp_uffd_wp(*src_pmd))
1825				pmd = pmd_swp_mkuffd_wp(pmd);
1826			set_pmd_at(src_mm, addr, src_pmd, pmd);
1827		}
1828
1829		src_folio = softleaf_to_folio(entry);
1830		VM_WARN_ON(!folio_test_large(src_folio));
1831
1832		folio_get(src_folio);
1833		/*
1834		 * folio_try_dup_anon_rmap_pmd does not fail for
1835		 * device private entries.
1836		 */
1837		folio_try_dup_anon_rmap_pmd(src_folio, &src_folio->page,
1838					    dst_vma, src_vma);
1839	}
1840
1841	add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1842	mm_inc_nr_ptes(dst_mm);
1843	pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
1844	if (!userfaultfd_wp(dst_vma))
1845		pmd = pmd_swp_clear_uffd_wp(pmd);
1846	set_pmd_at(dst_mm, addr, dst_pmd, pmd);
1847}
1848
1849int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1850		  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1851		  struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1852{
1853	spinlock_t *dst_ptl, *src_ptl;
1854	struct page *src_page;
1855	struct folio *src_folio;
1856	pmd_t pmd;
1857	pgtable_t pgtable = NULL;
1858	int ret = -ENOMEM;
1859
1860	pmd = pmdp_get_lockless(src_pmd);
1861	if (unlikely(pmd_present(pmd) && pmd_special(pmd) &&
1862		     !is_huge_zero_pmd(pmd))) {
1863		dst_ptl = pmd_lock(dst_mm, dst_pmd);
1864		src_ptl = pmd_lockptr(src_mm, src_pmd);
1865		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1866		/*
1867		 * No need to recheck the pmd, it can't change with write
1868		 * mmap lock held here.
1869		 *
1870		 * Meanwhile, making sure it's not a CoW VMA with writable
1871		 * mapping, otherwise it means either the anon page wrongly
1872		 * applied special bit, or we made the PRIVATE mapping be
1873		 * able to wrongly write to the backend MMIO.
1874		 */
1875		VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd));
1876		goto set_pmd;
1877	}
1878
1879	/* Skip if can be re-fill on fault */
1880	if (!vma_is_anonymous(dst_vma))
1881		return 0;
1882
1883	pgtable = pte_alloc_one(dst_mm);
1884	if (unlikely(!pgtable))
1885		goto out;
1886
1887	dst_ptl = pmd_lock(dst_mm, dst_pmd);
1888	src_ptl = pmd_lockptr(src_mm, src_pmd);
1889	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1890
1891	ret = -EAGAIN;
1892	pmd = *src_pmd;
1893
1894	if (unlikely(thp_migration_supported() &&
1895		     pmd_is_valid_softleaf(pmd))) {
1896		copy_huge_non_present_pmd(dst_mm, src_mm, dst_pmd, src_pmd, addr,
1897					  dst_vma, src_vma, pmd, pgtable);
1898		ret = 0;
1899		goto out_unlock;
1900	}
1901
1902	if (unlikely(!pmd_trans_huge(pmd))) {
1903		pte_free(dst_mm, pgtable);
1904		goto out_unlock;
1905	}
1906	/*
1907	 * When page table lock is held, the huge zero pmd should not be
1908	 * under splitting since we don't split the page itself, only pmd to
1909	 * a page table.
1910	 */
1911	if (is_huge_zero_pmd(pmd)) {
1912		/*
1913		 * mm_get_huge_zero_folio() will never allocate a new
1914		 * folio here, since we already have a zero page to
1915		 * copy. It just takes a reference.
1916		 */
1917		mm_get_huge_zero_folio(dst_mm);
1918		goto out_zero_page;
1919	}
1920
1921	src_page = pmd_page(pmd);
1922	VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
1923	src_folio = page_folio(src_page);
1924
1925	folio_get(src_folio);
1926	if (unlikely(folio_try_dup_anon_rmap_pmd(src_folio, src_page, dst_vma, src_vma))) {
1927		/* Page maybe pinned: split and retry the fault on PTEs. */
1928		folio_put(src_folio);
1929		pte_free(dst_mm, pgtable);
1930		spin_unlock(src_ptl);
1931		spin_unlock(dst_ptl);
1932		__split_huge_pmd(src_vma, src_pmd, addr, false);
1933		return -EAGAIN;
1934	}
1935	add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1936out_zero_page:
1937	mm_inc_nr_ptes(dst_mm);
1938	pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
1939	pmdp_set_wrprotect(src_mm, addr, src_pmd);
1940	if (!userfaultfd_wp(dst_vma))
1941		pmd = pmd_clear_uffd_wp(pmd);
1942	pmd = pmd_wrprotect(pmd);
1943set_pmd:
1944	pmd = pmd_mkold(pmd);
1945	set_pmd_at(dst_mm, addr, dst_pmd, pmd);
1946
1947	ret = 0;
1948out_unlock:
1949	spin_unlock(src_ptl);
1950	spin_unlock(dst_ptl);
1951out:
1952	return ret;
1953}
1954
1955#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1956void touch_pud(struct vm_area_struct *vma, unsigned long addr,
1957	       pud_t *pud, bool write)
1958{
1959	pud_t _pud;
1960
1961	_pud = pud_mkyoung(*pud);
1962	if (write)
1963		_pud = pud_mkdirty(_pud);
1964	if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
1965				  pud, _pud, write))
1966		update_mmu_cache_pud(vma, addr, pud);
1967}
1968
1969int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1970		  pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1971		  struct vm_area_struct *vma)
1972{
1973	spinlock_t *dst_ptl, *src_ptl;
1974	pud_t pud;
1975	int ret;
1976
1977	dst_ptl = pud_lock(dst_mm, dst_pud);
1978	src_ptl = pud_lockptr(src_mm, src_pud);
1979	spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1980
1981	ret = -EAGAIN;
1982	pud = *src_pud;
1983	if (unlikely(!pud_trans_huge(pud)))
1984		goto out_unlock;
1985
1986	/*
1987	 * TODO: once we support anonymous pages, use
1988	 * folio_try_dup_anon_rmap_*() and split if duplicating fails.
1989	 */
1990	if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) {
1991		pudp_set_wrprotect(src_mm, addr, src_pud);
1992		pud = pud_wrprotect(pud);
1993	}
1994	pud = pud_mkold(pud);
1995	set_pud_at(dst_mm, addr, dst_pud, pud);
1996
1997	ret = 0;
1998out_unlock:
1999	spin_unlock(src_ptl);
2000	spin_unlock(dst_ptl);
2001	return ret;
2002}
2003
2004void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
2005{
2006	bool write = vmf->flags & FAULT_FLAG_WRITE;
2007
2008	vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);
2009	if (unlikely(!pud_same(*vmf->pud, orig_pud)))
2010		goto unlock;
2011
2012	touch_pud(vmf->vma, vmf->address, vmf->pud, write);
2013unlock:
2014	spin_unlock(vmf->ptl);
2015}
2016#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
2017
2018bool huge_pmd_set_accessed(struct vm_fault *vmf)
2019{
2020	bool write = vmf->flags & FAULT_FLAG_WRITE;
2021
2022	if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd)))
2023		return false;
2024
2025	return touch_pmd(vmf->vma, vmf->address, vmf->pmd, write);
2026}
2027
2028static vm_fault_t do_huge_zero_wp_pmd(struct vm_fault *vmf)
2029{
2030	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
2031	struct vm_area_struct *vma = vmf->vma;
2032	struct mmu_notifier_range range;
2033	struct folio *folio;
2034	vm_fault_t ret = 0;
2035
2036	folio = vma_alloc_anon_folio_pmd(vma, vmf->address);
2037	if (unlikely(!folio))
2038		return VM_FAULT_FALLBACK;
2039
2040	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm, haddr,
2041				haddr + HPAGE_PMD_SIZE);
2042	mmu_notifier_invalidate_range_start(&range);
2043	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
2044	if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd)))
2045		goto release;
2046	ret = check_stable_address_space(vma->vm_mm);
2047	if (ret)
2048		goto release;
2049	(void)pmdp_huge_clear_flush(vma, haddr, vmf->pmd);
2050	map_anon_folio_pmd_pf(folio, vmf->pmd, vma, haddr);
2051	goto unlock;
2052release:
2053	folio_put(folio);
2054unlock:
2055	spin_unlock(vmf->ptl);
2056	mmu_notifier_invalidate_range_end(&range);
2057	return ret;
2058}
2059
2060vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf)
2061{
2062	const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
2063	struct vm_area_struct *vma = vmf->vma;
2064	struct folio *folio;
2065	struct page *page;
2066	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
2067	pmd_t orig_pmd = vmf->orig_pmd;
2068
2069	vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
2070	VM_BUG_ON_VMA(!vma->anon_vma, vma);
2071
2072	if (is_huge_zero_pmd(orig_pmd)) {
2073		vm_fault_t ret = do_huge_zero_wp_pmd(vmf);
2074
2075		if (!(ret & VM_FAULT_FALLBACK))
2076			return ret;
2077
2078		/* Fallback to splitting PMD if THP cannot be allocated */
2079		goto fallback;
2080	}
2081
2082	spin_lock(vmf->ptl);
2083
2084	if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
2085		spin_unlock(vmf->ptl);
2086		return 0;
2087	}
2088
2089	page = pmd_page(orig_pmd);
2090	folio = page_folio(page);
2091	VM_BUG_ON_PAGE(!PageHead(page), page);
2092
2093	/* Early check when only holding the PT lock. */
2094	if (PageAnonExclusive(page))
2095		goto reuse;
2096
2097	if (!folio_trylock(folio)) {
2098		folio_get(folio);
2099		spin_unlock(vmf->ptl);
2100		folio_lock(folio);
2101		spin_lock(vmf->ptl);
2102		if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
2103			spin_unlock(vmf->ptl);
2104			folio_unlock(folio);
2105			folio_put(folio);
2106			return 0;
2107		}
2108		folio_put(folio);
2109	}
2110
2111	/* Recheck after temporarily dropping the PT lock. */
2112	if (PageAnonExclusive(page)) {
2113		folio_unlock(folio);
2114		goto reuse;
2115	}
2116
2117	/*
2118	 * See do_wp_page(): we can only reuse the folio exclusively if
2119	 * there are no additional references. Note that we always drain
2120	 * the LRU cache immediately after adding a THP.
2121	 */
2122	if (folio_ref_count(folio) >
2123			1 + folio_test_swapcache(folio) * folio_nr_pages(folio))
2124		goto unlock_fallback;
2125	if (folio_test_swapcache(folio))
2126		folio_free_swap(folio);
2127	if (folio_ref_count(folio) == 1) {
2128		pmd_t entry;
2129
2130		folio_move_anon_rmap(folio, vma);
2131		SetPageAnonExclusive(page);
2132		folio_unlock(folio);
2133reuse:
2134		if (unlikely(unshare)) {
2135			spin_unlock(vmf->ptl);
2136			return 0;
2137		}
2138		entry = pmd_mkyoung(orig_pmd);
2139		entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
2140		if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))
2141			update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
2142		spin_unlock(vmf->ptl);
2143		return 0;
2144	}
2145
2146unlock_fallback:
2147	folio_unlock(folio);
2148	spin_unlock(vmf->ptl);
2149fallback:
2150	__split_huge_pmd(vma, vmf->pmd, vmf->address, false);
2151	return VM_FAULT_FALLBACK;
2152}
2153
2154static inline bool can_change_pmd_writable(struct vm_area_struct *vma,
2155					   unsigned long addr, pmd_t pmd)
2156{
2157	struct page *page;
2158
2159	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
2160		return false;
2161
2162	/* Don't touch entries that are not even readable (NUMA hinting). */
2163	if (pmd_protnone(pmd))
2164		return false;
2165
2166	/* Do we need write faults for softdirty tracking? */
2167	if (pmd_needs_soft_dirty_wp(vma, pmd))
2168		return false;
2169
2170	/* Do we need write faults for uffd-wp tracking? */
2171	if (userfaultfd_huge_pmd_wp(vma, pmd))
2172		return false;
2173
2174	if (!(vma->vm_flags & VM_SHARED)) {
2175		/* See can_change_pte_writable(). */
2176		page = vm_normal_page_pmd(vma, addr, pmd);
2177		return page && PageAnon(page) && PageAnonExclusive(page);
2178	}
2179
2180	/* See can_change_pte_writable(). */
2181	return pmd_dirty(pmd);
2182}
2183
2184/* NUMA hinting page fault entry point for trans huge pmds */
2185vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf)
2186{
2187	struct vm_area_struct *vma = vmf->vma;
2188	struct folio *folio;
2189	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
2190	int nid = NUMA_NO_NODE;
2191	int target_nid, last_cpupid;
2192	pmd_t pmd, old_pmd;
2193	bool writable = false;
2194	int flags = 0;
2195
2196	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
2197	old_pmd = pmdp_get(vmf->pmd);
2198
2199	if (unlikely(!pmd_same(old_pmd, vmf->orig_pmd))) {
2200		spin_unlock(vmf->ptl);
2201		return 0;
2202	}
2203
2204	pmd = pmd_modify(old_pmd, vma->vm_page_prot);
2205
2206	/*
2207	 * Detect now whether the PMD could be writable; this information
2208	 * is only valid while holding the PT lock.
2209	 */
2210	writable = pmd_write(pmd);
2211	if (!writable && vma_wants_manual_pte_write_upgrade(vma) &&
2212	    can_change_pmd_writable(vma, vmf->address, pmd))
2213		writable = true;
2214
2215	folio = vm_normal_folio_pmd(vma, haddr, pmd);
2216	if (!folio)
2217		goto out_map;
2218
2219	nid = folio_nid(folio);
2220
2221	target_nid = numa_migrate_check(folio, vmf, haddr, &flags, writable,
2222					&last_cpupid);
2223	if (target_nid == NUMA_NO_NODE)
2224		goto out_map;
2225	if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) {
2226		flags |= TNF_MIGRATE_FAIL;
2227		goto out_map;
2228	}
2229	/* The folio is isolated and isolation code holds a folio reference. */
2230	spin_unlock(vmf->ptl);
2231	writable = false;
2232
2233	if (!migrate_misplaced_folio(folio, target_nid)) {
2234		flags |= TNF_MIGRATED;
2235		nid = target_nid;
2236		task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags);
2237		return 0;
2238	}
2239
2240	flags |= TNF_MIGRATE_FAIL;
2241	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
2242	if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) {
2243		spin_unlock(vmf->ptl);
2244		return 0;
2245	}
2246out_map:
2247	/* Restore the PMD */
2248	pmd = pmd_modify(pmdp_get(vmf->pmd), vma->vm_page_prot);
2249	pmd = pmd_mkyoung(pmd);
2250	if (writable)
2251		pmd = pmd_mkwrite(pmd, vma);
2252	set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
2253	update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
2254	spin_unlock(vmf->ptl);
2255
2256	if (nid != NUMA_NO_NODE)
2257		task_numa_fault(last_cpupid, nid, HPAGE_PMD_NR, flags);
2258	return 0;
2259}
2260
2261/*
2262 * Return true if we do MADV_FREE successfully on entire pmd page.
2263 * Otherwise, return false.
2264 */
2265bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
2266		pmd_t *pmd, unsigned long addr, unsigned long next)
2267{
2268	spinlock_t *ptl;
2269	pmd_t orig_pmd;
2270	struct folio *folio;
2271	struct mm_struct *mm = tlb->mm;
2272	bool ret = false;
2273
2274	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
2275
2276	ptl = pmd_trans_huge_lock(pmd, vma);
2277	if (!ptl)
2278		goto out_unlocked;
2279
2280	orig_pmd = *pmd;
2281	if (is_huge_zero_pmd(orig_pmd))
2282		goto out;
2283
2284	if (unlikely(!pmd_present(orig_pmd))) {
2285		VM_BUG_ON(thp_migration_supported() &&
2286				  !pmd_is_migration_entry(orig_pmd));
2287		goto out;
2288	}
2289
2290	folio = pmd_folio(orig_pmd);
2291	/*
2292	 * If other processes are mapping this folio, we couldn't discard
2293	 * the folio unless they all do MADV_FREE so let's skip the folio.
2294	 */
2295	if (folio_maybe_mapped_shared(folio))
2296		goto out;
2297
2298	if (!folio_trylock(folio))
2299		goto out;
2300
2301	/*
2302	 * If user want to discard part-pages of THP, split it so MADV_FREE
2303	 * will deactivate only them.
2304	 */
2305	if (next - addr != HPAGE_PMD_SIZE) {
2306		folio_get(folio);
2307		spin_unlock(ptl);
2308		split_folio(folio);
2309		folio_unlock(folio);
2310		folio_put(folio);
2311		goto out_unlocked;
2312	}
2313
2314	if (folio_test_dirty(folio))
2315		folio_clear_dirty(folio);
2316	folio_unlock(folio);
2317
2318	if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
2319		pmdp_invalidate(vma, addr, pmd);
2320		orig_pmd = pmd_mkold(orig_pmd);
2321		orig_pmd = pmd_mkclean(orig_pmd);
2322
2323		set_pmd_at(mm, addr, pmd, orig_pmd);
2324		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
2325	}
2326
2327	folio_mark_lazyfree(folio);
2328	ret = true;
2329out:
2330	spin_unlock(ptl);
2331out_unlocked:
2332	return ret;
2333}
2334
2335static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
2336{
2337	pgtable_t pgtable;
2338
2339	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2340	pte_free(mm, pgtable);
2341	mm_dec_nr_ptes(mm);
2342}
2343
2344int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
2345		 pmd_t *pmd, unsigned long addr)
2346{
2347	pmd_t orig_pmd;
2348	spinlock_t *ptl;
2349
2350	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
2351
2352	ptl = __pmd_trans_huge_lock(pmd, vma);
2353	if (!ptl)
2354		return 0;
2355	/*
2356	 * For architectures like ppc64 we look at deposited pgtable
2357	 * when calling pmdp_huge_get_and_clear. So do the
2358	 * pgtable_trans_huge_withdraw after finishing pmdp related
2359	 * operations.
2360	 */
2361	orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,
2362						tlb->fullmm);
2363	arch_check_zapped_pmd(vma, orig_pmd);
2364	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
2365	if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
2366		if (arch_needs_pgtable_deposit())
2367			zap_deposited_table(tlb->mm, pmd);
2368		spin_unlock(ptl);
2369	} else if (is_huge_zero_pmd(orig_pmd)) {
2370		if (!vma_is_dax(vma) || arch_needs_pgtable_deposit())
2371			zap_deposited_table(tlb->mm, pmd);
2372		spin_unlock(ptl);
2373	} else {
2374		struct folio *folio = NULL;
2375		int flush_needed = 1;
2376
2377		if (pmd_present(orig_pmd)) {
2378			struct page *page = pmd_page(orig_pmd);
2379
2380			folio = page_folio(page);
2381			folio_remove_rmap_pmd(folio, page, vma);
2382			WARN_ON_ONCE(folio_mapcount(folio) < 0);
2383			VM_BUG_ON_PAGE(!PageHead(page), page);
2384		} else if (pmd_is_valid_softleaf(orig_pmd)) {
2385			const softleaf_t entry = softleaf_from_pmd(orig_pmd);
2386
2387			folio = softleaf_to_folio(entry);
2388			flush_needed = 0;
2389
2390			if (!thp_migration_supported())
2391				WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
2392		}
2393
2394		if (folio_test_anon(folio)) {
2395			zap_deposited_table(tlb->mm, pmd);
2396			add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
2397		} else {
2398			if (arch_needs_pgtable_deposit())
2399				zap_deposited_table(tlb->mm, pmd);
2400			add_mm_counter(tlb->mm, mm_counter_file(folio),
2401				       -HPAGE_PMD_NR);
2402
2403			/*
2404			 * Use flush_needed to indicate whether the PMD entry
2405			 * is present, instead of checking pmd_present() again.
2406			 */
2407			if (flush_needed && pmd_young(orig_pmd) &&
2408			    likely(vma_has_recency(vma)))
2409				folio_mark_accessed(folio);
2410		}
2411
2412		if (folio_is_device_private(folio)) {
2413			folio_remove_rmap_pmd(folio, &folio->page, vma);
2414			WARN_ON_ONCE(folio_mapcount(folio) < 0);
2415			folio_put(folio);
2416		}
2417
2418		spin_unlock(ptl);
2419		if (flush_needed)
2420			tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE);
2421	}
2422	return 1;
2423}
2424
2425#ifndef pmd_move_must_withdraw
2426static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
2427					 spinlock_t *old_pmd_ptl,
2428					 struct vm_area_struct *vma)
2429{
2430	/*
2431	 * With split pmd lock we also need to move preallocated
2432	 * PTE page table if new_pmd is on different PMD page table.
2433	 *
2434	 * We also don't deposit and withdraw tables for file pages.
2435	 */
2436	return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);
2437}
2438#endif
2439
2440static pmd_t move_soft_dirty_pmd(pmd_t pmd)
2441{
2442	if (pgtable_supports_soft_dirty()) {
2443		if (unlikely(pmd_is_migration_entry(pmd)))
2444			pmd = pmd_swp_mksoft_dirty(pmd);
2445		else if (pmd_present(pmd))
2446			pmd = pmd_mksoft_dirty(pmd);
2447	}
2448
2449	return pmd;
2450}
2451
2452static pmd_t clear_uffd_wp_pmd(pmd_t pmd)
2453{
2454	if (pmd_none(pmd))
2455		return pmd;
2456	if (pmd_present(pmd))
2457		pmd = pmd_clear_uffd_wp(pmd);
2458	else
2459		pmd = pmd_swp_clear_uffd_wp(pmd);
2460
2461	return pmd;
2462}
2463
2464bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
2465		  unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd)
2466{
2467	spinlock_t *old_ptl, *new_ptl;
2468	pmd_t pmd;
2469	struct mm_struct *mm = vma->vm_mm;
2470	bool force_flush = false;
2471
2472	/*
2473	 * The destination pmd shouldn't be established, free_pgtables()
2474	 * should have released it; but move_page_tables() might have already
2475	 * inserted a page table, if racing against shmem/file collapse.
2476	 */
2477	if (!pmd_none(*new_pmd)) {
2478		VM_BUG_ON(pmd_trans_huge(*new_pmd));
2479		return false;
2480	}
2481
2482	/*
2483	 * We don't have to worry about the ordering of src and dst
2484	 * ptlocks because exclusive mmap_lock prevents deadlock.
2485	 */
2486	old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
2487	if (old_ptl) {
2488		new_ptl = pmd_lockptr(mm, new_pmd);
2489		if (new_ptl != old_ptl)
2490			spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
2491		pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
2492		if (pmd_present(pmd))
2493			force_flush = true;
2494		VM_BUG_ON(!pmd_none(*new_pmd));
2495
2496		if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
2497			pgtable_t pgtable;
2498			pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
2499			pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
2500		}
2501		pmd = move_soft_dirty_pmd(pmd);
2502		if (vma_has_uffd_without_event_remap(vma))
2503			pmd = clear_uffd_wp_pmd(pmd);
2504		set_pmd_at(mm, new_addr, new_pmd, pmd);
2505		if (force_flush)
2506			flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
2507		if (new_ptl != old_ptl)
2508			spin_unlock(new_ptl);
2509		spin_unlock(old_ptl);
2510		return true;
2511	}
2512	return false;
2513}
2514
2515static void change_non_present_huge_pmd(struct mm_struct *mm,
2516		unsigned long addr, pmd_t *pmd, bool uffd_wp,
2517		bool uffd_wp_resolve)
2518{
2519	softleaf_t entry = softleaf_from_pmd(*pmd);
2520	const struct folio *folio = softleaf_to_folio(entry);
2521	pmd_t newpmd;
2522
2523	VM_WARN_ON(!pmd_is_valid_softleaf(*pmd));
2524	if (softleaf_is_migration_write(entry)) {
2525		/*
2526		 * A protection check is difficult so
2527		 * just be safe and disable write
2528		 */
2529		if (folio_test_anon(folio))
2530			entry = make_readable_exclusive_migration_entry(swp_offset(entry));
2531		else
2532			entry = make_readable_migration_entry(swp_offset(entry));
2533		newpmd = swp_entry_to_pmd(entry);
2534		if (pmd_swp_soft_dirty(*pmd))
2535			newpmd = pmd_swp_mksoft_dirty(newpmd);
2536	} else if (softleaf_is_device_private_write(entry)) {
2537		entry = make_readable_device_private_entry(swp_offset(entry));
2538		newpmd = swp_entry_to_pmd(entry);
2539	} else {
2540		newpmd = *pmd;
2541	}
2542
2543	if (uffd_wp)
2544		newpmd = pmd_swp_mkuffd_wp(newpmd);
2545	else if (uffd_wp_resolve)
2546		newpmd = pmd_swp_clear_uffd_wp(newpmd);
2547	if (!pmd_same(*pmd, newpmd))
2548		set_pmd_at(mm, addr, pmd, newpmd);
2549}
2550
2551/*
2552 * Returns
2553 *  - 0 if PMD could not be locked
2554 *  - 1 if PMD was locked but protections unchanged and TLB flush unnecessary
2555 *      or if prot_numa but THP migration is not supported
2556 *  - HPAGE_PMD_NR if protections changed and TLB flush necessary
2557 */
2558int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
2559		    pmd_t *pmd, unsigned long addr, pgprot_t newprot,
2560		    unsigned long cp_flags)
2561{
2562	struct mm_struct *mm = vma->vm_mm;
2563	spinlock_t *ptl;
2564	pmd_t oldpmd, entry;
2565	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
2566	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
2567	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
2568	int ret = 1;
2569
2570	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
2571
2572	if (prot_numa && !thp_migration_supported())
2573		return 1;
2574
2575	ptl = __pmd_trans_huge_lock(pmd, vma);
2576	if (!ptl)
2577		return 0;
2578
2579	if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) {
2580		change_non_present_huge_pmd(mm, addr, pmd, uffd_wp,
2581					    uffd_wp_resolve);
2582		goto unlock;
2583	}
2584
2585	if (prot_numa) {
2586
2587		/*
2588		 * Avoid trapping faults against the zero page. The read-only
2589		 * data is likely to be read-cached on the local CPU and
2590		 * local/remote hits to the zero page are not interesting.
2591		 */
2592		if (is_huge_zero_pmd(*pmd))
2593			goto unlock;
2594
2595		if (pmd_protnone(*pmd))
2596			goto unlock;
2597
2598		if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma,
2599					     vma_is_single_threaded_private(vma)))
2600			goto unlock;
2601	}
2602	/*
2603	 * In case prot_numa, we are under mmap_read_lock(mm). It's critical
2604	 * to not clear pmd intermittently to avoid race with MADV_DONTNEED
2605	 * which is also under mmap_read_lock(mm):
2606	 *
2607	 *	CPU0:				CPU1:
2608	 *				change_huge_pmd(prot_numa=1)
2609	 *				 pmdp_huge_get_and_clear_notify()
2610	 * madvise_dontneed()
2611	 *  zap_pmd_range()
2612	 *   pmd_trans_huge(*pmd) == 0 (without ptl)
2613	 *   // skip the pmd
2614	 *				 set_pmd_at();
2615	 *				 // pmd is re-established
2616	 *
2617	 * The race makes MADV_DONTNEED miss the huge pmd and don't clear it
2618	 * which may break userspace.
2619	 *
2620	 * pmdp_invalidate_ad() is required to make sure we don't miss
2621	 * dirty/young flags set by hardware.
2622	 */
2623	oldpmd = pmdp_invalidate_ad(vma, addr, pmd);
2624
2625	entry = pmd_modify(oldpmd, newprot);
2626	if (uffd_wp)
2627		entry = pmd_mkuffd_wp(entry);
2628	else if (uffd_wp_resolve)
2629		/*
2630		 * Leave the write bit to be handled by PF interrupt
2631		 * handler, then things like COW could be properly
2632		 * handled.
2633		 */
2634		entry = pmd_clear_uffd_wp(entry);
2635
2636	/* See change_pte_range(). */
2637	if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&
2638	    can_change_pmd_writable(vma, addr, entry))
2639		entry = pmd_mkwrite(entry, vma);
2640
2641	ret = HPAGE_PMD_NR;
2642	set_pmd_at(mm, addr, pmd, entry);
2643
2644	if (huge_pmd_needs_flush(oldpmd, entry))
2645		tlb_flush_pmd_range(tlb, addr, HPAGE_PMD_SIZE);
2646unlock:
2647	spin_unlock(ptl);
2648	return ret;
2649}
2650
2651/*
2652 * Returns:
2653 *
2654 * - 0: if pud leaf changed from under us
2655 * - 1: if pud can be skipped
2656 * - HPAGE_PUD_NR: if pud was successfully processed
2657 */
2658#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
2659int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
2660		    pud_t *pudp, unsigned long addr, pgprot_t newprot,
2661		    unsigned long cp_flags)
2662{
2663	struct mm_struct *mm = vma->vm_mm;
2664	pud_t oldpud, entry;
2665	spinlock_t *ptl;
2666
2667	tlb_change_page_size(tlb, HPAGE_PUD_SIZE);
2668
2669	/* NUMA balancing doesn't apply to dax */
2670	if (cp_flags & MM_CP_PROT_NUMA)
2671		return 1;
2672
2673	/*
2674	 * Huge entries on userfault-wp only works with anonymous, while we
2675	 * don't have anonymous PUDs yet.
2676	 */
2677	if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL))
2678		return 1;
2679
2680	ptl = __pud_trans_huge_lock(pudp, vma);
2681	if (!ptl)
2682		return 0;
2683
2684	/*
2685	 * Can't clear PUD or it can race with concurrent zapping.  See
2686	 * change_huge_pmd().
2687	 */
2688	oldpud = pudp_invalidate(vma, addr, pudp);
2689	entry = pud_modify(oldpud, newprot);
2690	set_pud_at(mm, addr, pudp, entry);
2691	tlb_flush_pud_range(tlb, addr, HPAGE_PUD_SIZE);
2692
2693	spin_unlock(ptl);
2694	return HPAGE_PUD_NR;
2695}
2696#endif
2697
2698#ifdef CONFIG_USERFAULTFD
2699/*
2700 * The PT lock for src_pmd and dst_vma/src_vma (for reading) are locked by
2701 * the caller, but it must return after releasing the page_table_lock.
2702 * Just move the page from src_pmd to dst_pmd if possible.
2703 * Return zero if succeeded in moving the page, -EAGAIN if it needs to be
2704 * repeated by the caller, or other errors in case of failure.
2705 */
2706int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval,
2707			struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
2708			unsigned long dst_addr, unsigned long src_addr)
2709{
2710	pmd_t _dst_pmd, src_pmdval;
2711	struct page *src_page;
2712	struct folio *src_folio;
2713	spinlock_t *src_ptl, *dst_ptl;
2714	pgtable_t src_pgtable;
2715	struct mmu_notifier_range range;
2716	int err = 0;
2717
2718	src_pmdval = *src_pmd;
2719	src_ptl = pmd_lockptr(mm, src_pmd);
2720
2721	lockdep_assert_held(src_ptl);
2722	vma_assert_locked(src_vma);
2723	vma_assert_locked(dst_vma);
2724
2725	/* Sanity checks before the operation */
2726	if (WARN_ON_ONCE(!pmd_none(dst_pmdval)) || WARN_ON_ONCE(src_addr & ~HPAGE_PMD_MASK) ||
2727	    WARN_ON_ONCE(dst_addr & ~HPAGE_PMD_MASK)) {
2728		spin_unlock(src_ptl);
2729		return -EINVAL;
2730	}
2731
2732	if (!pmd_trans_huge(src_pmdval)) {
2733		spin_unlock(src_ptl);
2734		if (pmd_is_migration_entry(src_pmdval)) {
2735			pmd_migration_entry_wait(mm, &src_pmdval);
2736			return -EAGAIN;
2737		}
2738		return -ENOENT;
2739	}
2740
2741	src_page = pmd_page(src_pmdval);
2742
2743	if (!is_huge_zero_pmd(src_pmdval)) {
2744		if (unlikely(!PageAnonExclusive(src_page))) {
2745			spin_unlock(src_ptl);
2746			return -EBUSY;
2747		}
2748
2749		src_folio = page_folio(src_page);
2750		folio_get(src_folio);
2751	} else
2752		src_folio = NULL;
2753
2754	spin_unlock(src_ptl);
2755
2756	flush_cache_range(src_vma, src_addr, src_addr + HPAGE_PMD_SIZE);
2757	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, src_addr,
2758				src_addr + HPAGE_PMD_SIZE);
2759	mmu_notifier_invalidate_range_start(&range);
2760
2761	if (src_folio)
2762		folio_lock(src_folio);
2763
2764	dst_ptl = pmd_lockptr(mm, dst_pmd);
2765	double_pt_lock(src_ptl, dst_ptl);
2766	if (unlikely(!pmd_same(*src_pmd, src_pmdval) ||
2767		     !pmd_same(*dst_pmd, dst_pmdval))) {
2768		err = -EAGAIN;
2769		goto unlock_ptls;
2770	}
2771	if (src_folio) {
2772		if (folio_maybe_dma_pinned(src_folio) ||
2773		    !PageAnonExclusive(&src_folio->page)) {
2774			err = -EBUSY;
2775			goto unlock_ptls;
2776		}
2777
2778		if (WARN_ON_ONCE(!folio_test_head(src_folio)) ||
2779		    WARN_ON_ONCE(!folio_test_anon(src_folio))) {
2780			err = -EBUSY;
2781			goto unlock_ptls;
2782		}
2783
2784		src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd);
2785		/* Folio got pinned from under us. Put it back and fail the move. */
2786		if (folio_maybe_dma_pinned(src_folio)) {
2787			set_pmd_at(mm, src_addr, src_pmd, src_pmdval);
2788			err = -EBUSY;
2789			goto unlock_ptls;
2790		}
2791
2792		folio_move_anon_rmap(src_folio, dst_vma);
2793		src_folio->index = linear_page_index(dst_vma, dst_addr);
2794
2795		_dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot);
2796		/* Follow mremap() behavior and treat the entry dirty after the move */
2797		_dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma);
2798	} else {
2799		src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd);
2800		_dst_pmd = move_soft_dirty_pmd(src_pmdval);
2801		_dst_pmd = clear_uffd_wp_pmd(_dst_pmd);
2802	}
2803	set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd);
2804
2805	src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd);
2806	pgtable_trans_huge_deposit(mm, dst_pmd, src_pgtable);
2807unlock_ptls:
2808	double_pt_unlock(src_ptl, dst_ptl);
2809	/* unblock rmap walks */
2810	if (src_folio)
2811		folio_unlock(src_folio);
2812	mmu_notifier_invalidate_range_end(&range);
2813	if (src_folio)
2814		folio_put(src_folio);
2815	return err;
2816}
2817#endif /* CONFIG_USERFAULTFD */
2818
2819/*
2820 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
2821 *
2822 * Note that if it returns page table lock pointer, this routine returns without
2823 * unlocking page table lock. So callers must unlock it.
2824 */
2825spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
2826{
2827	spinlock_t *ptl;
2828
2829	ptl = pmd_lock(vma->vm_mm, pmd);
2830	if (likely(pmd_is_huge(*pmd)))
2831		return ptl;
2832	spin_unlock(ptl);
2833	return NULL;
2834}
2835
2836/*
2837 * Returns page table lock pointer if a given pud maps a thp, NULL otherwise.
2838 *
2839 * Note that if it returns page table lock pointer, this routine returns without
2840 * unlocking page table lock. So callers must unlock it.
2841 */
2842spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
2843{
2844	spinlock_t *ptl;
2845
2846	ptl = pud_lock(vma->vm_mm, pud);
2847	if (likely(pud_trans_huge(*pud)))
2848		return ptl;
2849	spin_unlock(ptl);
2850	return NULL;
2851}
2852
2853#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
2854int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
2855		 pud_t *pud, unsigned long addr)
2856{
2857	spinlock_t *ptl;
2858	pud_t orig_pud;
2859
2860	ptl = __pud_trans_huge_lock(pud, vma);
2861	if (!ptl)
2862		return 0;
2863
2864	orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm);
2865	arch_check_zapped_pud(vma, orig_pud);
2866	tlb_remove_pud_tlb_entry(tlb, pud, addr);
2867	if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
2868		spin_unlock(ptl);
2869		/* No zero page support yet */
2870	} else {
2871		struct page *page = NULL;
2872		struct folio *folio;
2873
2874		/* No support for anonymous PUD pages or migration yet */
2875		VM_WARN_ON_ONCE(vma_is_anonymous(vma) ||
2876				!pud_present(orig_pud));
2877
2878		page = pud_page(orig_pud);
2879		folio = page_folio(page);
2880		folio_remove_rmap_pud(folio, page, vma);
2881		add_mm_counter(tlb->mm, mm_counter_file(folio), -HPAGE_PUD_NR);
2882
2883		spin_unlock(ptl);
2884		tlb_remove_page_size(tlb, page, HPAGE_PUD_SIZE);
2885	}
2886	return 1;
2887}
2888
2889static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
2890		unsigned long haddr)
2891{
2892	struct folio *folio;
2893	struct page *page;
2894	pud_t old_pud;
2895
2896	VM_BUG_ON(haddr & ~HPAGE_PUD_MASK);
2897	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2898	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
2899	VM_BUG_ON(!pud_trans_huge(*pud));
2900
2901	count_vm_event(THP_SPLIT_PUD);
2902
2903	old_pud = pudp_huge_clear_flush(vma, haddr, pud);
2904
2905	if (!vma_is_dax(vma))
2906		return;
2907
2908	page = pud_page(old_pud);
2909	folio = page_folio(page);
2910
2911	if (!folio_test_dirty(folio) && pud_dirty(old_pud))
2912		folio_mark_dirty(folio);
2913	if (!folio_test_referenced(folio) && pud_young(old_pud))
2914		folio_set_referenced(folio);
2915	folio_remove_rmap_pud(folio, page, vma);
2916	folio_put(folio);
2917	add_mm_counter(vma->vm_mm, mm_counter_file(folio),
2918		-HPAGE_PUD_NR);
2919}
2920
2921void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
2922		unsigned long address)
2923{
2924	spinlock_t *ptl;
2925	struct mmu_notifier_range range;
2926
2927	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
2928				address & HPAGE_PUD_MASK,
2929				(address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE);
2930	mmu_notifier_invalidate_range_start(&range);
2931	ptl = pud_lock(vma->vm_mm, pud);
2932	if (unlikely(!pud_trans_huge(*pud)))
2933		goto out;
2934	__split_huge_pud_locked(vma, pud, range.start);
2935
2936out:
2937	spin_unlock(ptl);
2938	mmu_notifier_invalidate_range_end(&range);
2939}
2940#else
2941void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
2942		unsigned long address)
2943{
2944}
2945#endif /* CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
2946
2947static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2948		unsigned long haddr, pmd_t *pmd)
2949{
2950	struct mm_struct *mm = vma->vm_mm;
2951	pgtable_t pgtable;
2952	pmd_t _pmd, old_pmd;
2953	unsigned long addr;
2954	pte_t *pte;
2955	int i;
2956
2957	/*
2958	 * Leave pmd empty until pte is filled note that it is fine to delay
2959	 * notification until mmu_notifier_invalidate_range_end() as we are
2960	 * replacing a zero pmd write protected page with a zero pte write
2961	 * protected page.
2962	 *
2963	 * See Documentation/mm/mmu_notifier.rst
2964	 */
2965	old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);
2966
2967	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2968	pmd_populate(mm, &_pmd, pgtable);
2969
2970	pte = pte_offset_map(&_pmd, haddr);
2971	VM_BUG_ON(!pte);
2972	for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
2973		pte_t entry;
2974
2975		entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot);
2976		entry = pte_mkspecial(entry);
2977		if (pmd_uffd_wp(old_pmd))
2978			entry = pte_mkuffd_wp(entry);
2979		VM_BUG_ON(!pte_none(ptep_get(pte)));
2980		set_pte_at(mm, addr, pte, entry);
2981		pte++;
2982	}
2983	pte_unmap(pte - 1);
2984	smp_wmb(); /* make pte visible before pmd */
2985	pmd_populate(mm, pmd, pgtable);
2986}
2987
2988static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
2989		unsigned long haddr, bool freeze)
2990{
2991	struct mm_struct *mm = vma->vm_mm;
2992	struct folio *folio;
2993	struct page *page;
2994	pgtable_t pgtable;
2995	pmd_t old_pmd, _pmd;
2996	bool soft_dirty, uffd_wp = false, young = false, write = false;
2997	bool anon_exclusive = false, dirty = false;
2998	unsigned long addr;
2999	pte_t *pte;
3000	int i;
3001
3002	VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
3003	VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
3004	VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
3005
3006	VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(*pmd) && !pmd_trans_huge(*pmd));
3007
3008	count_vm_event(THP_SPLIT_PMD);
3009
3010	if (!vma_is_anonymous(vma)) {
3011		old_pmd = pmdp_huge_clear_flush(vma, haddr, pmd);
3012		/*
3013		 * We are going to unmap this huge page. So
3014		 * just go ahead and zap it
3015		 */
3016		if (arch_needs_pgtable_deposit())
3017			zap_deposited_table(mm, pmd);
3018		if (!vma_is_dax(vma) && vma_is_special_huge(vma))
3019			return;
3020		if (unlikely(pmd_is_migration_entry(old_pmd))) {
3021			const softleaf_t old_entry = softleaf_from_pmd(old_pmd);
3022
3023			folio = softleaf_to_folio(old_entry);
3024		} else if (is_huge_zero_pmd(old_pmd)) {
3025			return;
3026		} else {
3027			page = pmd_page(old_pmd);
3028			folio = page_folio(page);
3029			if (!folio_test_dirty(folio) && pmd_dirty(old_pmd))
3030				folio_mark_dirty(folio);
3031			if (!folio_test_referenced(folio) && pmd_young(old_pmd))
3032				folio_set_referenced(folio);
3033			folio_remove_rmap_pmd(folio, page, vma);
3034			folio_put(folio);
3035		}
3036		add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR);
3037		return;
3038	}
3039
3040	if (is_huge_zero_pmd(*pmd)) {
3041		/*
3042		 * FIXME: Do we want to invalidate secondary mmu by calling
3043		 * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below
3044		 * inside __split_huge_pmd() ?
3045		 *
3046		 * We are going from a zero huge page write protected to zero
3047		 * small page also write protected so it does not seems useful
3048		 * to invalidate secondary mmu at this time.
3049		 */
3050		return __split_huge_zero_page_pmd(vma, haddr, pmd);
3051	}
3052
3053	if (pmd_is_migration_entry(*pmd)) {
3054		softleaf_t entry;
3055
3056		old_pmd = *pmd;
3057		entry = softleaf_from_pmd(old_pmd);
3058		page = softleaf_to_page(entry);
3059		folio = page_folio(page);
3060
3061		soft_dirty = pmd_swp_soft_dirty(old_pmd);
3062		uffd_wp = pmd_swp_uffd_wp(old_pmd);
3063
3064		write = softleaf_is_migration_write(entry);
3065		if (PageAnon(page))
3066			anon_exclusive = softleaf_is_migration_read_exclusive(entry);
3067		young = softleaf_is_migration_young(entry);
3068		dirty = softleaf_is_migration_dirty(entry);
3069	} else if (pmd_is_device_private_entry(*pmd)) {
3070		softleaf_t entry;
3071
3072		old_pmd = *pmd;
3073		entry = softleaf_from_pmd(old_pmd);
3074		page = softleaf_to_page(entry);
3075		folio = page_folio(page);
3076
3077		soft_dirty = pmd_swp_soft_dirty(old_pmd);
3078		uffd_wp = pmd_swp_uffd_wp(old_pmd);
3079
3080		write = softleaf_is_device_private_write(entry);
3081		anon_exclusive = PageAnonExclusive(page);
3082
3083		/*
3084		 * Device private THP should be treated the same as regular
3085		 * folios w.r.t anon exclusive handling. See the comments for
3086		 * folio handling and anon_exclusive below.
3087		 */
3088		if (freeze && anon_exclusive &&
3089		    folio_try_share_anon_rmap_pmd(folio, page))
3090			freeze = false;
3091		if (!freeze) {
3092			rmap_t rmap_flags = RMAP_NONE;
3093
3094			folio_ref_add(folio, HPAGE_PMD_NR - 1);
3095			if (anon_exclusive)
3096				rmap_flags |= RMAP_EXCLUSIVE;
3097
3098			folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR,
3099						 vma, haddr, rmap_flags);
3100		}
3101	} else {
3102		/*
3103		 * Up to this point the pmd is present and huge and userland has
3104		 * the whole access to the hugepage during the split (which
3105		 * happens in place). If we overwrite the pmd with the not-huge
3106		 * version pointing to the pte here (which of course we could if
3107		 * all CPUs were bug free), userland could trigger a small page
3108		 * size TLB miss on the small sized TLB while the hugepage TLB
3109		 * entry is still established in the huge TLB. Some CPU doesn't
3110		 * like that. See
3111		 * http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum
3112		 * 383 on page 105. Intel should be safe but is also warns that
3113		 * it's only safe if the permission and cache attributes of the
3114		 * two entries loaded in the two TLB is identical (which should
3115		 * be the case here). But it is generally safer to never allow
3116		 * small and huge TLB entries for the same virtual address to be
3117		 * loaded simultaneously. So instead of doing "pmd_populate();
3118		 * flush_pmd_tlb_range();" we first mark the current pmd
3119		 * notpresent (atomically because here the pmd_trans_huge must
3120		 * remain set at all times on the pmd until the split is
3121		 * complete for this pmd), then we flush the SMP TLB and finally
3122		 * we write the non-huge version of the pmd entry with
3123		 * pmd_populate.
3124		 */
3125		old_pmd = pmdp_invalidate(vma, haddr, pmd);
3126		page = pmd_page(old_pmd);
3127		folio = page_folio(page);
3128		if (pmd_dirty(old_pmd)) {
3129			dirty = true;
3130			folio_set_dirty(folio);
3131		}
3132		write = pmd_write(old_pmd);
3133		young = pmd_young(old_pmd);
3134		soft_dirty = pmd_soft_dirty(old_pmd);
3135		uffd_wp = pmd_uffd_wp(old_pmd);
3136
3137		VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio);
3138		VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
3139
3140		/*
3141		 * Without "freeze", we'll simply split the PMD, propagating the
3142		 * PageAnonExclusive() flag for each PTE by setting it for
3143		 * each subpage -- no need to (temporarily) clear.
3144		 *
3145		 * With "freeze" we want to replace mapped pages by
3146		 * migration entries right away. This is only possible if we
3147		 * managed to clear PageAnonExclusive() -- see
3148		 * set_pmd_migration_entry().
3149		 *
3150		 * In case we cannot clear PageAnonExclusive(), split the PMD
3151		 * only and let try_to_migrate_one() fail later.
3152		 *
3153		 * See folio_try_share_anon_rmap_pmd(): invalidate PMD first.
3154		 */
3155		anon_exclusive = PageAnonExclusive(page);
3156		if (freeze && anon_exclusive &&
3157		    folio_try_share_anon_rmap_pmd(folio, page))
3158			freeze = false;
3159		if (!freeze) {
3160			rmap_t rmap_flags = RMAP_NONE;
3161
3162			folio_ref_add(folio, HPAGE_PMD_NR - 1);
3163			if (anon_exclusive)
3164				rmap_flags |= RMAP_EXCLUSIVE;
3165			folio_add_anon_rmap_ptes(folio, page, HPAGE_PMD_NR,
3166						 vma, haddr, rmap_flags);
3167		}
3168	}
3169
3170	/*
3171	 * Withdraw the table only after we mark the pmd entry invalid.
3172	 * This's critical for some architectures (Power).
3173	 */
3174	pgtable = pgtable_trans_huge_withdraw(mm, pmd);
3175	pmd_populate(mm, &_pmd, pgtable);
3176
3177	pte = pte_offset_map(&_pmd, haddr);
3178	VM_BUG_ON(!pte);
3179
3180	/*
3181	 * Note that NUMA hinting access restrictions are not transferred to
3182	 * avoid any possibility of altering permissions across VMAs.
3183	 */
3184	if (freeze || pmd_is_migration_entry(old_pmd)) {
3185		pte_t entry;
3186		swp_entry_t swp_entry;
3187
3188		for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
3189			if (write)
3190				swp_entry = make_writable_migration_entry(
3191							page_to_pfn(page + i));
3192			else if (anon_exclusive)
3193				swp_entry = make_readable_exclusive_migration_entry(
3194							page_to_pfn(page + i));
3195			else
3196				swp_entry = make_readable_migration_entry(
3197							page_to_pfn(page + i));
3198			if (young)
3199				swp_entry = make_migration_entry_young(swp_entry);
3200			if (dirty)
3201				swp_entry = make_migration_entry_dirty(swp_entry);
3202			entry = swp_entry_to_pte(swp_entry);
3203			if (soft_dirty)
3204				entry = pte_swp_mksoft_dirty(entry);
3205			if (uffd_wp)
3206				entry = pte_swp_mkuffd_wp(entry);
3207			VM_WARN_ON(!pte_none(ptep_get(pte + i)));
3208			set_pte_at(mm, addr, pte + i, entry);
3209		}
3210	} else if (pmd_is_device_private_entry(old_pmd)) {
3211		pte_t entry;
3212		swp_entry_t swp_entry;
3213
3214		for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
3215			/*
3216			 * anon_exclusive was already propagated to the relevant
3217			 * pages corresponding to the pte entries when freeze
3218			 * is false.
3219			 */
3220			if (write)
3221				swp_entry = make_writable_device_private_entry(
3222							page_to_pfn(page + i));
3223			else
3224				swp_entry = make_readable_device_private_entry(
3225							page_to_pfn(page + i));
3226			/*
3227			 * Young and dirty bits are not progated via swp_entry
3228			 */
3229			entry = swp_entry_to_pte(swp_entry);
3230			if (soft_dirty)
3231				entry = pte_swp_mksoft_dirty(entry);
3232			if (uffd_wp)
3233				entry = pte_swp_mkuffd_wp(entry);
3234			VM_WARN_ON(!pte_none(ptep_get(pte + i)));
3235			set_pte_at(mm, addr, pte + i, entry);
3236		}
3237	} else {
3238		pte_t entry;
3239
3240		entry = mk_pte(page, READ_ONCE(vma->vm_page_prot));
3241		if (write)
3242			entry = pte_mkwrite(entry, vma);
3243		if (!young)
3244			entry = pte_mkold(entry);
3245		/* NOTE: this may set soft-dirty too on some archs */
3246		if (dirty)
3247			entry = pte_mkdirty(entry);
3248		if (soft_dirty)
3249			entry = pte_mksoft_dirty(entry);
3250		if (uffd_wp)
3251			entry = pte_mkuffd_wp(entry);
3252
3253		for (i = 0; i < HPAGE_PMD_NR; i++)
3254			VM_WARN_ON(!pte_none(ptep_get(pte + i)));
3255
3256		set_ptes(mm, haddr, pte, entry, HPAGE_PMD_NR);
3257	}
3258	pte_unmap(pte);
3259
3260	if (!pmd_is_migration_entry(*pmd))
3261		folio_remove_rmap_pmd(folio, page, vma);
3262	if (freeze)
3263		put_page(page);
3264
3265	smp_wmb(); /* make pte visible before pmd */
3266	pmd_populate(mm, pmd, pgtable);
3267}
3268
3269void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address,
3270			   pmd_t *pmd, bool freeze)
3271{
3272	VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE));
3273	if (pmd_trans_huge(*pmd) || pmd_is_valid_softleaf(*pmd))
3274		__split_huge_pmd_locked(vma, pmd, address, freeze);
3275}
3276
3277void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
3278		unsigned long address, bool freeze)
3279{
3280	spinlock_t *ptl;
3281	struct mmu_notifier_range range;
3282
3283	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
3284				address & HPAGE_PMD_MASK,
3285				(address & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE);
3286	mmu_notifier_invalidate_range_start(&range);
3287	ptl = pmd_lock(vma->vm_mm, pmd);
3288	split_huge_pmd_locked(vma, range.start, pmd, freeze);
3289	spin_unlock(ptl);
3290	mmu_notifier_invalidate_range_end(&range);
3291}
3292
3293void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
3294		bool freeze)
3295{
3296	pmd_t *pmd = mm_find_pmd(vma->vm_mm, address);
3297
3298	if (!pmd)
3299		return;
3300
3301	__split_huge_pmd(vma, pmd, address, freeze);
3302}
3303
3304static inline void split_huge_pmd_if_needed(struct vm_area_struct *vma, unsigned long address)
3305{
3306	/*
3307	 * If the new address isn't hpage aligned and it could previously
3308	 * contain an hugepage: check if we need to split an huge pmd.
3309	 */
3310	if (!IS_ALIGNED(address, HPAGE_PMD_SIZE) &&
3311	    range_in_vma(vma, ALIGN_DOWN(address, HPAGE_PMD_SIZE),
3312			 ALIGN(address, HPAGE_PMD_SIZE)))
3313		split_huge_pmd_address(vma, address, false);
3314}
3315
3316void vma_adjust_trans_huge(struct vm_area_struct *vma,
3317			   unsigned long start,
3318			   unsigned long end,
3319			   struct vm_area_struct *next)
3320{
3321	/* Check if we need to split start first. */
3322	split_huge_pmd_if_needed(vma, start);
3323
3324	/* Check if we need to split end next. */
3325	split_huge_pmd_if_needed(vma, end);
3326
3327	/* If we're incrementing next->vm_start, we might need to split it. */
3328	if (next)
3329		split_huge_pmd_if_needed(next, end);
3330}
3331
3332static void unmap_folio(struct folio *folio)
3333{
3334	enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_SYNC |
3335		TTU_BATCH_FLUSH;
3336
3337	VM_BUG_ON_FOLIO(!folio_test_large(folio), folio);
3338
3339	if (folio_test_pmd_mappable(folio))
3340		ttu_flags |= TTU_SPLIT_HUGE_PMD;
3341
3342	/*
3343	 * Anon pages need migration entries to preserve them, but file
3344	 * pages can simply be left unmapped, then faulted back on demand.
3345	 * If that is ever changed (perhaps for mlock), update remap_page().
3346	 */
3347	if (folio_test_anon(folio))
3348		try_to_migrate(folio, ttu_flags);
3349	else
3350		try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);
3351
3352	try_to_unmap_flush();
3353}
3354
3355static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma,
3356					    unsigned long addr, pmd_t *pmdp,
3357					    struct folio *folio)
3358{
3359	struct mm_struct *mm = vma->vm_mm;
3360	int ref_count, map_count;
3361	pmd_t orig_pmd = *pmdp;
3362
3363	if (pmd_dirty(orig_pmd))
3364		folio_set_dirty(folio);
3365	if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) {
3366		folio_set_swapbacked(folio);
3367		return false;
3368	}
3369
3370	orig_pmd = pmdp_huge_clear_flush(vma, addr, pmdp);
3371
3372	/*
3373	 * Syncing against concurrent GUP-fast:
3374	 * - clear PMD; barrier; read refcount
3375	 * - inc refcount; barrier; read PMD
3376	 */
3377	smp_mb();
3378
3379	ref_count = folio_ref_count(folio);
3380	map_count = folio_mapcount(folio);
3381
3382	/*
3383	 * Order reads for folio refcount and dirty flag
3384	 * (see comments in __remove_mapping()).
3385	 */
3386	smp_rmb();
3387
3388	/*
3389	 * If the folio or its PMD is redirtied at this point, or if there
3390	 * are unexpected references, we will give up to discard this folio
3391	 * and remap it.
3392	 *
3393	 * The only folio refs must be one from isolation plus the rmap(s).
3394	 */
3395	if (pmd_dirty(orig_pmd))
3396		folio_set_dirty(folio);
3397	if (folio_test_dirty(folio) && !(vma->vm_flags & VM_DROPPABLE)) {
3398		folio_set_swapbacked(folio);
3399		set_pmd_at(mm, addr, pmdp, orig_pmd);
3400		return false;
3401	}
3402
3403	if (ref_count != map_count + 1) {
3404		set_pmd_at(mm, addr, pmdp, orig_pmd);
3405		return false;
3406	}
3407
3408	folio_remove_rmap_pmd(folio, pmd_page(orig_pmd), vma);
3409	zap_deposited_table(mm, pmdp);
3410	add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR);
3411	if (vma->vm_flags & VM_LOCKED)
3412		mlock_drain_local();
3413	folio_put(folio);
3414
3415	return true;
3416}
3417
3418bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr,
3419			   pmd_t *pmdp, struct folio *folio)
3420{
3421	VM_WARN_ON_FOLIO(!folio_test_pmd_mappable(folio), folio);
3422	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
3423	VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
3424	VM_WARN_ON_FOLIO(folio_test_swapbacked(folio), folio);
3425	VM_WARN_ON_ONCE(!IS_ALIGNED(addr, HPAGE_PMD_SIZE));
3426
3427	return __discard_anon_folio_pmd_locked(vma, addr, pmdp, folio);
3428}
3429
3430static void remap_page(struct folio *folio, unsigned long nr, int flags)
3431{
3432	int i = 0;
3433
3434	/* If unmap_folio() uses try_to_migrate() on file, remove this check */
3435	if (!folio_test_anon(folio))
3436		return;
3437	for (;;) {
3438		remove_migration_ptes(folio, folio, RMP_LOCKED | flags);
3439		i += folio_nr_pages(folio);
3440		if (i >= nr)
3441			break;
3442		folio = folio_next(folio);
3443	}
3444}
3445
3446static void lru_add_split_folio(struct folio *folio, struct folio *new_folio,
3447		struct lruvec *lruvec, struct list_head *list)
3448{
3449	VM_BUG_ON_FOLIO(folio_test_lru(new_folio), folio);
3450	lockdep_assert_held(&lruvec->lru_lock);
3451
3452	if (folio_is_device_private(folio))
3453		return;
3454
3455	if (list) {
3456		/* page reclaim is reclaiming a huge page */
3457		VM_WARN_ON(folio_test_lru(folio));
3458		folio_get(new_folio);
3459		list_add_tail(&new_folio->lru, list);
3460	} else {
3461		/* head is still on lru (and we have it frozen) */
3462		VM_WARN_ON(!folio_test_lru(folio));
3463		if (folio_test_unevictable(folio))
3464			new_folio->mlock_count = 0;
3465		else
3466			list_add_tail(&new_folio->lru, &folio->lru);
3467		folio_set_lru(new_folio);
3468	}
3469}
3470
3471static bool page_range_has_hwpoisoned(struct page *page, long nr_pages)
3472{
3473	for (; nr_pages; page++, nr_pages--)
3474		if (PageHWPoison(page))
3475			return true;
3476	return false;
3477}
3478
3479/*
3480 * It splits @folio into @new_order folios and copies the @folio metadata to
3481 * all the resulting folios.
3482 */
3483static void __split_folio_to_order(struct folio *folio, int old_order,
3484		int new_order)
3485{
3486	/* Scan poisoned pages when split a poisoned folio to large folios */
3487	const bool handle_hwpoison = folio_test_has_hwpoisoned(folio) && new_order;
3488	long new_nr_pages = 1 << new_order;
3489	long nr_pages = 1 << old_order;
3490	long i;
3491
3492	folio_clear_has_hwpoisoned(folio);
3493
3494	/* Check first new_nr_pages since the loop below skips them */
3495	if (handle_hwpoison &&
3496	    page_range_has_hwpoisoned(folio_page(folio, 0), new_nr_pages))
3497		folio_set_has_hwpoisoned(folio);
3498	/*
3499	 * Skip the first new_nr_pages, since the new folio from them have all
3500	 * the flags from the original folio.
3501	 */
3502	for (i = new_nr_pages; i < nr_pages; i += new_nr_pages) {
3503		struct page *new_head = &folio->page + i;
3504		/*
3505		 * Careful: new_folio is not a "real" folio before we cleared PageTail.
3506		 * Don't pass it around before clear_compound_head().
3507		 */
3508		struct folio *new_folio = (struct folio *)new_head;
3509
3510		VM_BUG_ON_PAGE(atomic_read(&new_folio->_mapcount) != -1, new_head);
3511
3512		/*
3513		 * Clone page flags before unfreezing refcount.
3514		 *
3515		 * After successful get_page_unless_zero() might follow flags change,
3516		 * for example lock_page() which set PG_waiters.
3517		 *
3518		 * Note that for mapped sub-pages of an anonymous THP,
3519		 * PG_anon_exclusive has been cleared in unmap_folio() and is stored in
3520		 * the migration entry instead from where remap_page() will restore it.
3521		 * We can still have PG_anon_exclusive set on effectively unmapped and
3522		 * unreferenced sub-pages of an anonymous THP: we can simply drop
3523		 * PG_anon_exclusive (-> PG_mappedtodisk) for these here.
3524		 */
3525		new_folio->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
3526		new_folio->flags.f |= (folio->flags.f &
3527				((1L << PG_referenced) |
3528				 (1L << PG_swapbacked) |
3529				 (1L << PG_swapcache) |
3530				 (1L << PG_mlocked) |
3531				 (1L << PG_uptodate) |
3532				 (1L << PG_active) |
3533				 (1L << PG_workingset) |
3534				 (1L << PG_locked) |
3535				 (1L << PG_unevictable) |
3536#ifdef CONFIG_ARCH_USES_PG_ARCH_2
3537				 (1L << PG_arch_2) |
3538#endif
3539#ifdef CONFIG_ARCH_USES_PG_ARCH_3
3540				 (1L << PG_arch_3) |
3541#endif
3542				 (1L << PG_dirty) |
3543				 LRU_GEN_MASK | LRU_REFS_MASK));
3544
3545		if (handle_hwpoison &&
3546		    page_range_has_hwpoisoned(new_head, new_nr_pages))
3547			folio_set_has_hwpoisoned(new_folio);
3548
3549		new_folio->mapping = folio->mapping;
3550		new_folio->index = folio->index + i;
3551
3552		if (folio_test_swapcache(folio))
3553			new_folio->swap.val = folio->swap.val + i;
3554
3555		/* Page flags must be visible before we make the page non-compound. */
3556		smp_wmb();
3557
3558		/*
3559		 * Clear PageTail before unfreezing page refcount.
3560		 *
3561		 * After successful get_page_unless_zero() might follow put_page()
3562		 * which needs correct compound_head().
3563		 */
3564		clear_compound_head(new_head);
3565		if (new_order) {
3566			prep_compound_page(new_head, new_order);
3567			folio_set_large_rmappable(new_folio);
3568		}
3569
3570		if (folio_test_young(folio))
3571			folio_set_young(new_folio);
3572		if (folio_test_idle(folio))
3573			folio_set_idle(new_folio);
3574#ifdef CONFIG_MEMCG
3575		new_folio->memcg_data = folio->memcg_data;
3576#endif
3577
3578		folio_xchg_last_cpupid(new_folio, folio_last_cpupid(folio));
3579	}
3580
3581	if (new_order)
3582		folio_set_order(folio, new_order);
3583	else
3584		ClearPageCompound(&folio->page);
3585}
3586
3587/**
3588 * __split_unmapped_folio() - splits an unmapped @folio to lower order folios in
3589 * two ways: uniform split or non-uniform split.
3590 * @folio: the to-be-split folio
3591 * @new_order: the smallest order of the after split folios (since buddy
3592 *             allocator like split generates folios with orders from @folio's
3593 *             order - 1 to new_order).
3594 * @split_at: in buddy allocator like split, the folio containing @split_at
3595 *            will be split until its order becomes @new_order.
3596 * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller
3597 * @mapping: @folio->mapping
3598 * @split_type: if the split is uniform or not (buddy allocator like split)
3599 *
3600 *
3601 * 1. uniform split: the given @folio into multiple @new_order small folios,
3602 *    where all small folios have the same order. This is done when
3603 *    split_type is SPLIT_TYPE_UNIFORM.
3604 * 2. buddy allocator like (non-uniform) split: the given @folio is split into
3605 *    half and one of the half (containing the given page) is split into half
3606 *    until the given @folio's order becomes @new_order. This is done when
3607 *    split_type is SPLIT_TYPE_NON_UNIFORM.
3608 *
3609 * The high level flow for these two methods are:
3610 *
3611 * 1. uniform split: @xas is split with no expectation of failure and a single
3612 *    __split_folio_to_order() is called to split the @folio into @new_order
3613 *    along with stats update.
3614 * 2. non-uniform split: folio_order - @new_order calls to
3615 *    __split_folio_to_order() are expected to be made in a for loop to split
3616 *    the @folio to one lower order at a time. The folio containing @split_at
3617 *    is split in each iteration. @xas is split into half in each iteration and
3618 *    can fail. A failed @xas split leaves split folios as is without merging
3619 *    them back.
3620 *
3621 * After splitting, the caller's folio reference will be transferred to the
3622 * folio containing @split_at. The caller needs to unlock and/or free
3623 * after-split folios if necessary.
3624 *
3625 * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be
3626 * split but not to @new_order, the caller needs to check)
3627 */
3628static int __split_unmapped_folio(struct folio *folio, int new_order,
3629		struct page *split_at, struct xa_state *xas,
3630		struct address_space *mapping, enum split_type split_type)
3631{
3632	const bool is_anon = folio_test_anon(folio);
3633	int old_order = folio_order(folio);
3634	int start_order = split_type == SPLIT_TYPE_UNIFORM ? new_order : old_order - 1;
3635	struct folio *old_folio = folio;
3636	int split_order;
3637
3638	/*
3639	 * split to new_order one order at a time. For uniform split,
3640	 * folio is split to new_order directly.
3641	 */
3642	for (split_order = start_order;
3643	     split_order >= new_order;
3644	     split_order--) {
3645		int nr_new_folios = 1UL << (old_order - split_order);
3646
3647		/* order-1 anonymous folio is not supported */
3648		if (is_anon && split_order == 1)
3649			continue;
3650
3651		if (mapping) {
3652			/*
3653			 * uniform split has xas_split_alloc() called before
3654			 * irq is disabled to allocate enough memory, whereas
3655			 * non-uniform split can handle ENOMEM.
3656			 * Use the to-be-split folio, so that a parallel
3657			 * folio_try_get() waits on it until xarray is updated
3658			 * with after-split folios and the original one is
3659			 * unfrozen.
3660			 */
3661			if (split_type == SPLIT_TYPE_UNIFORM) {
3662				xas_split(xas, old_folio, old_order);
3663			} else {
3664				xas_set_order(xas, folio->index, split_order);
3665				xas_try_split(xas, old_folio, old_order);
3666				if (xas_error(xas))
3667					return xas_error(xas);
3668			}
3669		}
3670
3671		folio_split_memcg_refs(folio, old_order, split_order);
3672		split_page_owner(&folio->page, old_order, split_order);
3673		pgalloc_tag_split(folio, old_order, split_order);
3674		__split_folio_to_order(folio, old_order, split_order);
3675
3676		if (is_anon) {
3677			mod_mthp_stat(old_order, MTHP_STAT_NR_ANON, -1);
3678			mod_mthp_stat(split_order, MTHP_STAT_NR_ANON, nr_new_folios);
3679		}
3680		/*
3681		 * If uniform split, the process is complete.
3682		 * If non-uniform, continue splitting the folio at @split_at
3683		 * as long as the next @split_order is >= @new_order.
3684		 */
3685		folio = page_folio(split_at);
3686		old_order = split_order;
3687	}
3688
3689	return 0;
3690}
3691
3692/**
3693 * folio_check_splittable() - check if a folio can be split to a given order
3694 * @folio: folio to be split
3695 * @new_order: the smallest order of the after split folios (since buddy
3696 *             allocator like split generates folios with orders from @folio's
3697 *             order - 1 to new_order).
3698 * @split_type: uniform or non-uniform split
3699 *
3700 * folio_check_splittable() checks if @folio can be split to @new_order using
3701 * @split_type method. The truncated folio check must come first.
3702 *
3703 * Context: folio must be locked.
3704 *
3705 * Return: 0 - @folio can be split to @new_order, otherwise an error number is
3706 * returned.
3707 */
3708int folio_check_splittable(struct folio *folio, unsigned int new_order,
3709			   enum split_type split_type)
3710{
3711	VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
3712	/*
3713	 * Folios that just got truncated cannot get split. Signal to the
3714	 * caller that there was a race.
3715	 *
3716	 * TODO: this will also currently refuse folios without a mapping in the
3717	 * swapcache (shmem or to-be-anon folios).
3718	 */
3719	if (!folio->mapping && !folio_test_anon(folio))
3720		return -EBUSY;
3721
3722	if (folio_test_anon(folio)) {
3723		/* order-1 is not supported for anonymous THP. */
3724		if (new_order == 1)
3725			return -EINVAL;
3726	} else if (split_type == SPLIT_TYPE_NON_UNIFORM || new_order) {
3727		if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
3728		    !mapping_large_folio_support(folio->mapping)) {
3729			/*
3730			 * We can always split a folio down to a single page
3731			 * (new_order == 0) uniformly.
3732			 *
3733			 * For any other scenario
3734			 *   a) uniform split targeting a large folio
3735			 *      (new_order > 0)
3736			 *   b) any non-uniform split
3737			 * we must confirm that the file system supports large
3738			 * folios.
3739			 *
3740			 * Note that we might still have THPs in such
3741			 * mappings, which is created from khugepaged when
3742			 * CONFIG_READ_ONLY_THP_FOR_FS is enabled. But in that
3743			 * case, the mapping does not actually support large
3744			 * folios properly.
3745			 */
3746			return -EINVAL;
3747		}
3748	}
3749
3750	/*
3751	 * swapcache folio could only be split to order 0
3752	 *
3753	 * non-uniform split creates after-split folios with orders from
3754	 * folio_order(folio) - 1 to new_order, making it not suitable for any
3755	 * swapcache folio split. Only uniform split to order-0 can be used
3756	 * here.
3757	 */
3758	if ((split_type == SPLIT_TYPE_NON_UNIFORM || new_order) && folio_test_swapcache(folio)) {
3759		return -EINVAL;
3760	}
3761
3762	if (is_huge_zero_folio(folio))
3763		return -EINVAL;
3764
3765	if (folio_test_writeback(folio))
3766		return -EBUSY;
3767
3768	return 0;
3769}
3770
3771/* Number of folio references from the pagecache or the swapcache. */
3772static unsigned int folio_cache_ref_count(const struct folio *folio)
3773{
3774	if (folio_test_anon(folio) && !folio_test_swapcache(folio))
3775		return 0;
3776	return folio_nr_pages(folio);
3777}
3778
3779static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int new_order,
3780					     struct page *split_at, struct xa_state *xas,
3781					     struct address_space *mapping, bool do_lru,
3782					     struct list_head *list, enum split_type split_type,
3783					     pgoff_t end, int *nr_shmem_dropped)
3784{
3785	struct folio *end_folio = folio_next(folio);
3786	struct folio *new_folio, *next;
3787	int old_order = folio_order(folio);
3788	int ret = 0;
3789	struct deferred_split *ds_queue;
3790
3791	VM_WARN_ON_ONCE(!mapping && end);
3792	/* Prevent deferred_split_scan() touching ->_refcount */
3793	ds_queue = folio_split_queue_lock(folio);
3794	if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) {
3795		struct swap_cluster_info *ci = NULL;
3796		struct lruvec *lruvec;
3797
3798		if (old_order > 1) {
3799			if (!list_empty(&folio->_deferred_list)) {
3800				ds_queue->split_queue_len--;
3801				/*
3802				 * Reinitialize page_deferred_list after removing the
3803				 * page from the split_queue, otherwise a subsequent
3804				 * split will see list corruption when checking the
3805				 * page_deferred_list.
3806				 */
3807				list_del_init(&folio->_deferred_list);
3808			}
3809			if (folio_test_partially_mapped(folio)) {
3810				folio_clear_partially_mapped(folio);
3811				mod_mthp_stat(old_order,
3812					MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
3813			}
3814		}
3815		split_queue_unlock(ds_queue);
3816		if (mapping) {
3817			int nr = folio_nr_pages(folio);
3818
3819			if (folio_test_pmd_mappable(folio) &&
3820			    new_order < HPAGE_PMD_ORDER) {
3821				if (folio_test_swapbacked(folio)) {
3822					lruvec_stat_mod_folio(folio,
3823							NR_SHMEM_THPS, -nr);
3824				} else {
3825					lruvec_stat_mod_folio(folio,
3826							NR_FILE_THPS, -nr);
3827					filemap_nr_thps_dec(mapping);
3828				}
3829			}
3830		}
3831
3832		if (folio_test_swapcache(folio)) {
3833			if (mapping) {
3834				VM_WARN_ON_ONCE_FOLIO(mapping, folio);
3835				return -EINVAL;
3836			}
3837
3838			ci = swap_cluster_get_and_lock(folio);
3839		}
3840
3841		/* lock lru list/PageCompound, ref frozen by page_ref_freeze */
3842		if (do_lru)
3843			lruvec = folio_lruvec_lock(folio);
3844
3845		ret = __split_unmapped_folio(folio, new_order, split_at, xas,
3846					     mapping, split_type);
3847
3848		/*
3849		 * Unfreeze after-split folios and put them back to the right
3850		 * list. @folio should be kept frozon until page cache
3851		 * entries are updated with all the other after-split folios
3852		 * to prevent others seeing stale page cache entries.
3853		 * As a result, new_folio starts from the next folio of
3854		 * @folio.
3855		 */
3856		for (new_folio = folio_next(folio); new_folio != end_folio;
3857		     new_folio = next) {
3858			unsigned long nr_pages = folio_nr_pages(new_folio);
3859
3860			next = folio_next(new_folio);
3861
3862			zone_device_private_split_cb(folio, new_folio);
3863
3864			folio_ref_unfreeze(new_folio,
3865					   folio_cache_ref_count(new_folio) + 1);
3866
3867			if (do_lru)
3868				lru_add_split_folio(folio, new_folio, lruvec, list);
3869
3870			/*
3871			 * Anonymous folio with swap cache.
3872			 * NOTE: shmem in swap cache is not supported yet.
3873			 */
3874			if (ci) {
3875				__swap_cache_replace_folio(ci, folio, new_folio);
3876				continue;
3877			}
3878
3879			/* Anonymous folio without swap cache */
3880			if (!mapping)
3881				continue;
3882
3883			/* Add the new folio to the page cache. */
3884			if (new_folio->index < end) {
3885				__xa_store(&mapping->i_pages, new_folio->index,
3886					   new_folio, 0);
3887				continue;
3888			}
3889
3890			VM_WARN_ON_ONCE(!nr_shmem_dropped);
3891			/* Drop folio beyond EOF: ->index >= end */
3892			if (shmem_mapping(mapping) && nr_shmem_dropped)
3893				*nr_shmem_dropped += nr_pages;
3894			else if (folio_test_clear_dirty(new_folio))
3895				folio_account_cleaned(
3896					new_folio, inode_to_wb(mapping->host));
3897			__filemap_remove_folio(new_folio, NULL);
3898			folio_put_refs(new_folio, nr_pages);
3899		}
3900
3901		zone_device_private_split_cb(folio, NULL);
3902		/*
3903		 * Unfreeze @folio only after all page cache entries, which
3904		 * used to point to it, have been updated with new folios.
3905		 * Otherwise, a parallel folio_try_get() can grab @folio
3906		 * and its caller can see stale page cache entries.
3907		 */
3908		folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1);
3909
3910		if (do_lru)
3911			unlock_page_lruvec(lruvec);
3912
3913		if (ci)
3914			swap_cluster_unlock(ci);
3915	} else {
3916		split_queue_unlock(ds_queue);
3917		return -EAGAIN;
3918	}
3919
3920	return ret;
3921}
3922
3923/**
3924 * __folio_split() - split a folio at @split_at to a @new_order folio
3925 * @folio: folio to split
3926 * @new_order: the order of the new folio
3927 * @split_at: a page within the new folio
3928 * @lock_at: a page within @folio to be left locked to caller
3929 * @list: after-split folios will be put on it if non NULL
3930 * @split_type: perform uniform split or not (non-uniform split)
3931 *
3932 * It calls __split_unmapped_folio() to perform uniform and non-uniform split.
3933 * It is in charge of checking whether the split is supported or not and
3934 * preparing @folio for __split_unmapped_folio().
3935 *
3936 * After splitting, the after-split folio containing @lock_at remains locked
3937 * and others are unlocked:
3938 * 1. for uniform split, @lock_at points to one of @folio's subpages;
3939 * 2. for buddy allocator like (non-uniform) split, @lock_at points to @folio.
3940 *
3941 * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be
3942 * split but not to @new_order, the caller needs to check)
3943 */
3944static int __folio_split(struct folio *folio, unsigned int new_order,
3945		struct page *split_at, struct page *lock_at,
3946		struct list_head *list, enum split_type split_type)
3947{
3948	XA_STATE(xas, &folio->mapping->i_pages, folio->index);
3949	struct folio *end_folio = folio_next(folio);
3950	bool is_anon = folio_test_anon(folio);
3951	struct address_space *mapping = NULL;
3952	struct anon_vma *anon_vma = NULL;
3953	int old_order = folio_order(folio);
3954	struct folio *new_folio, *next;
3955	int nr_shmem_dropped = 0;
3956	int remap_flags = 0;
3957	int ret;
3958	pgoff_t end = 0;
3959
3960	VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
3961	VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio);
3962
3963	if (folio != page_folio(split_at) || folio != page_folio(lock_at)) {
3964		ret = -EINVAL;
3965		goto out;
3966	}
3967
3968	if (new_order >= old_order) {
3969		ret = -EINVAL;
3970		goto out;
3971	}
3972
3973	ret = folio_check_splittable(folio, new_order, split_type);
3974	if (ret) {
3975		VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio");
3976		goto out;
3977	}
3978
3979	if (is_anon) {
3980		/*
3981		 * The caller does not necessarily hold an mmap_lock that would
3982		 * prevent the anon_vma disappearing so we first we take a
3983		 * reference to it and then lock the anon_vma for write. This
3984		 * is similar to folio_lock_anon_vma_read except the write lock
3985		 * is taken to serialise against parallel split or collapse
3986		 * operations.
3987		 */
3988		anon_vma = folio_get_anon_vma(folio);
3989		if (!anon_vma) {
3990			ret = -EBUSY;
3991			goto out;
3992		}
3993		anon_vma_lock_write(anon_vma);
3994		mapping = NULL;
3995	} else {
3996		unsigned int min_order;
3997		gfp_t gfp;
3998
3999		mapping = folio->mapping;
4000		min_order = mapping_min_folio_order(folio->mapping);
4001		if (new_order < min_order) {
4002			ret = -EINVAL;
4003			goto out;
4004		}
4005
4006		gfp = current_gfp_context(mapping_gfp_mask(mapping) &
4007							GFP_RECLAIM_MASK);
4008
4009		if (!filemap_release_folio(folio, gfp)) {
4010			ret = -EBUSY;
4011			goto out;
4012		}
4013
4014		if (split_type == SPLIT_TYPE_UNIFORM) {
4015			xas_set_order(&xas, folio->index, new_order);
4016			xas_split_alloc(&xas, folio, old_order, gfp);
4017			if (xas_error(&xas)) {
4018				ret = xas_error(&xas);
4019				goto out;
4020			}
4021		}
4022
4023		anon_vma = NULL;
4024		i_mmap_lock_read(mapping);
4025
4026		/*
4027		 *__split_unmapped_folio() may need to trim off pages beyond
4028		 * EOF: but on 32-bit, i_size_read() takes an irq-unsafe
4029		 * seqlock, which cannot be nested inside the page tree lock.
4030		 * So note end now: i_size itself may be changed at any moment,
4031		 * but folio lock is good enough to serialize the trimming.
4032		 */
4033		end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
4034		if (shmem_mapping(mapping))
4035			end = shmem_fallocend(mapping->host, end);
4036	}
4037
4038	/*
4039	 * Racy check if we can split the page, before unmap_folio() will
4040	 * split PMDs
4041	 */
4042	if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) {
4043		ret = -EAGAIN;
4044		goto out_unlock;
4045	}
4046
4047	unmap_folio(folio);
4048
4049	/* block interrupt reentry in xa_lock and spinlock */
4050	local_irq_disable();
4051	if (mapping) {
4052		/*
4053		 * Check if the folio is present in page cache.
4054		 * We assume all tail are present too, if folio is there.
4055		 */
4056		xas_lock(&xas);
4057		xas_reset(&xas);
4058		if (xas_load(&xas) != folio) {
4059			ret = -EAGAIN;
4060			goto fail;
4061		}
4062	}
4063
4064	ret = __folio_freeze_and_split_unmapped(folio, new_order, split_at, &xas, mapping,
4065						true, list, split_type, end, &nr_shmem_dropped);
4066fail:
4067	if (mapping)
4068		xas_unlock(&xas);
4069
4070	local_irq_enable();
4071
4072	if (nr_shmem_dropped)
4073		shmem_uncharge(mapping->host, nr_shmem_dropped);
4074
4075	if (!ret && is_anon && !folio_is_device_private(folio))
4076		remap_flags = RMP_USE_SHARED_ZEROPAGE;
4077
4078	remap_page(folio, 1 << old_order, remap_flags);
4079
4080	/*
4081	 * Unlock all after-split folios except the one containing
4082	 * @lock_at page. If @folio is not split, it will be kept locked.
4083	 */
4084	for (new_folio = folio; new_folio != end_folio; new_folio = next) {
4085		next = folio_next(new_folio);
4086		if (new_folio == page_folio(lock_at))
4087			continue;
4088
4089		folio_unlock(new_folio);
4090		/*
4091		 * Subpages may be freed if there wasn't any mapping
4092		 * like if add_to_swap() is running on a lru page that
4093		 * had its mapping zapped. And freeing these pages
4094		 * requires taking the lru_lock so we do the put_page
4095		 * of the tail pages after the split is complete.
4096		 */
4097		free_folio_and_swap_cache(new_folio);
4098	}
4099
4100out_unlock:
4101	if (anon_vma) {
4102		anon_vma_unlock_write(anon_vma);
4103		put_anon_vma(anon_vma);
4104	}
4105	if (mapping)
4106		i_mmap_unlock_read(mapping);
4107out:
4108	xas_destroy(&xas);
4109	if (old_order == HPAGE_PMD_ORDER)
4110		count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
4111	count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED);
4112	return ret;
4113}
4114
4115/**
4116 * folio_split_unmapped() - split a large anon folio that is already unmapped
4117 * @folio: folio to split
4118 * @new_order: the order of folios after split
4119 *
4120 * This function is a helper for splitting folios that have already been
4121 * unmapped. The use case is that the device or the CPU can refuse to migrate
4122 * THP pages in the middle of migration, due to allocation issues on either
4123 * side.
4124 *
4125 * anon_vma_lock is not required to be held, mmap_read_lock() or
4126 * mmap_write_lock() should be held. @folio is expected to be locked by the
4127 * caller. device-private and non device-private folios are supported along
4128 * with folios that are in the swapcache. @folio should also be unmapped and
4129 * isolated from LRU (if applicable)
4130 *
4131 * Upon return, the folio is not remapped, split folios are not added to LRU,
4132 * free_folio_and_swap_cache() is not called, and new folios remain locked.
4133 *
4134 * Return: 0 on success, -EAGAIN if the folio cannot be split (e.g., due to
4135 *         insufficient reference count or extra pins).
4136 */
4137int folio_split_unmapped(struct folio *folio, unsigned int new_order)
4138{
4139	int ret = 0;
4140
4141	VM_WARN_ON_ONCE_FOLIO(folio_mapped(folio), folio);
4142	VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio);
4143	VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio);
4144	VM_WARN_ON_ONCE_FOLIO(!folio_test_anon(folio), folio);
4145
4146	if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1)
4147		return -EAGAIN;
4148
4149	local_irq_disable();
4150	ret = __folio_freeze_and_split_unmapped(folio, new_order, &folio->page, NULL,
4151						NULL, false, NULL, SPLIT_TYPE_UNIFORM,
4152						0, NULL);
4153	local_irq_enable();
4154	return ret;
4155}
4156
4157/*
4158 * This function splits a large folio into smaller folios of order @new_order.
4159 * @page can point to any page of the large folio to split. The split operation
4160 * does not change the position of @page.
4161 *
4162 * Prerequisites:
4163 *
4164 * 1) The caller must hold a reference on the @page's owning folio, also known
4165 *    as the large folio.
4166 *
4167 * 2) The large folio must be locked.
4168 *
4169 * 3) The folio must not be pinned. Any unexpected folio references, including
4170 *    GUP pins, will result in the folio not getting split; instead, the caller
4171 *    will receive an -EAGAIN.
4172 *
4173 * 4) @new_order > 1, usually. Splitting to order-1 anonymous folios is not
4174 *    supported for non-file-backed folios, because folio->_deferred_list, which
4175 *    is used by partially mapped folios, is stored in subpage 2, but an order-1
4176 *    folio only has subpages 0 and 1. File-backed order-1 folios are supported,
4177 *    since they do not use _deferred_list.
4178 *
4179 * After splitting, the caller's folio reference will be transferred to @page,
4180 * resulting in a raised refcount of @page after this call. The other pages may
4181 * be freed if they are not mapped.
4182 *
4183 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
4184 *
4185 * Pages in @new_order will inherit the mapping, flags, and so on from the
4186 * huge page.
4187 *
4188 * Returns 0 if the huge page was split successfully.
4189 *
4190 * Returns -EAGAIN if the folio has unexpected reference (e.g., GUP) or if
4191 * the folio was concurrently removed from the page cache.
4192 *
4193 * Returns -EBUSY when trying to split the huge zeropage, if the folio is
4194 * under writeback, if fs-specific folio metadata cannot currently be
4195 * released, or if some unexpected race happened (e.g., anon VMA disappeared,
4196 * truncation).
4197 *
4198 * Callers should ensure that the order respects the address space mapping
4199 * min-order if one is set for non-anonymous folios.
4200 *
4201 * Returns -EINVAL when trying to split to an order that is incompatible
4202 * with the folio. Splitting to order 0 is compatible with all folios.
4203 */
4204int __split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
4205				     unsigned int new_order)
4206{
4207	struct folio *folio = page_folio(page);
4208
4209	return __folio_split(folio, new_order, &folio->page, page, list,
4210			     SPLIT_TYPE_UNIFORM);
4211}
4212
4213/**
4214 * folio_split() - split a folio at @split_at to a @new_order folio
4215 * @folio: folio to split
4216 * @new_order: the order of the new folio
4217 * @split_at: a page within the new folio
4218 * @list: after-split folios are added to @list if not null, otherwise to LRU
4219 *        list
4220 *
4221 * It has the same prerequisites and returns as
4222 * split_huge_page_to_list_to_order().
4223 *
4224 * Split a folio at @split_at to a new_order folio, leave the
4225 * remaining subpages of the original folio as large as possible. For example,
4226 * in the case of splitting an order-9 folio at its third order-3 subpages to
4227 * an order-3 folio, there are 2^(9-3)=64 order-3 subpages in the order-9 folio.
4228 * After the split, there will be a group of folios with different orders and
4229 * the new folio containing @split_at is marked in bracket:
4230 * [order-4, {order-3}, order-3, order-5, order-6, order-7, order-8].
4231 *
4232 * After split, folio is left locked for caller.
4233 *
4234 * Return: 0 - successful, <0 - failed (if -ENOMEM is returned, @folio might be
4235 * split but not to @new_order, the caller needs to check)
4236 */
4237int folio_split(struct folio *folio, unsigned int new_order,
4238		struct page *split_at, struct list_head *list)
4239{
4240	return __folio_split(folio, new_order, split_at, &folio->page, list,
4241			     SPLIT_TYPE_NON_UNIFORM);
4242}
4243
4244/**
4245 * min_order_for_split() - get the minimum order @folio can be split to
4246 * @folio: folio to split
4247 *
4248 * min_order_for_split() tells the minimum order @folio can be split to.
4249 * If a file-backed folio is truncated, 0 will be returned. Any subsequent
4250 * split attempt should get -EBUSY from split checking code.
4251 *
4252 * Return: @folio's minimum order for split
4253 */
4254unsigned int min_order_for_split(struct folio *folio)
4255{
4256	if (folio_test_anon(folio))
4257		return 0;
4258
4259	/*
4260	 * If the folio got truncated, we don't know the previous mapping and
4261	 * consequently the old min order. But it doesn't matter, as any split
4262	 * attempt will immediately fail with -EBUSY as the folio cannot get
4263	 * split until freed.
4264	 */
4265	if (!folio->mapping)
4266		return 0;
4267
4268	return mapping_min_folio_order(folio->mapping);
4269}
4270
4271int split_folio_to_list(struct folio *folio, struct list_head *list)
4272{
4273	return split_huge_page_to_list_to_order(&folio->page, list, 0);
4274}
4275
4276/*
4277 * __folio_unqueue_deferred_split() is not to be called directly:
4278 * the folio_unqueue_deferred_split() inline wrapper in mm/internal.h
4279 * limits its calls to those folios which may have a _deferred_list for
4280 * queueing THP splits, and that list is (racily observed to be) non-empty.
4281 *
4282 * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is
4283 * zero: because even when split_queue_lock is held, a non-empty _deferred_list
4284 * might be in use on deferred_split_scan()'s unlocked on-stack list.
4285 *
4286 * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is
4287 * therefore important to unqueue deferred split before changing folio memcg.
4288 */
4289bool __folio_unqueue_deferred_split(struct folio *folio)
4290{
4291	struct deferred_split *ds_queue;
4292	unsigned long flags;
4293	bool unqueued = false;
4294
4295	WARN_ON_ONCE(folio_ref_count(folio));
4296	WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg_charged(folio));
4297
4298	ds_queue = folio_split_queue_lock_irqsave(folio, &flags);
4299	if (!list_empty(&folio->_deferred_list)) {
4300		ds_queue->split_queue_len--;
4301		if (folio_test_partially_mapped(folio)) {
4302			folio_clear_partially_mapped(folio);
4303			mod_mthp_stat(folio_order(folio),
4304				      MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
4305		}
4306		list_del_init(&folio->_deferred_list);
4307		unqueued = true;
4308	}
4309	split_queue_unlock_irqrestore(ds_queue, flags);
4310
4311	return unqueued;	/* useful for debug warnings */
4312}
4313
4314/* partially_mapped=false won't clear PG_partially_mapped folio flag */
4315void deferred_split_folio(struct folio *folio, bool partially_mapped)
4316{
4317	struct deferred_split *ds_queue;
4318	unsigned long flags;
4319
4320	/*
4321	 * Order 1 folios have no space for a deferred list, but we also
4322	 * won't waste much memory by not adding them to the deferred list.
4323	 */
4324	if (folio_order(folio) <= 1)
4325		return;
4326
4327	if (!partially_mapped && !split_underused_thp)
4328		return;
4329
4330	/*
4331	 * Exclude swapcache: originally to avoid a corrupt deferred split
4332	 * queue. Nowadays that is fully prevented by memcg1_swapout();
4333	 * but if page reclaim is already handling the same folio, it is
4334	 * unnecessary to handle it again in the shrinker, so excluding
4335	 * swapcache here may still be a useful optimization.
4336	 */
4337	if (folio_test_swapcache(folio))
4338		return;
4339
4340	ds_queue = folio_split_queue_lock_irqsave(folio, &flags);
4341	if (partially_mapped) {
4342		if (!folio_test_partially_mapped(folio)) {
4343			folio_set_partially_mapped(folio);
4344			if (folio_test_pmd_mappable(folio))
4345				count_vm_event(THP_DEFERRED_SPLIT_PAGE);
4346			count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED);
4347			mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1);
4348
4349		}
4350	} else {
4351		/* partially mapped folios cannot become non-partially mapped */
4352		VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio);
4353	}
4354	if (list_empty(&folio->_deferred_list)) {
4355		struct mem_cgroup *memcg;
4356
4357		memcg = folio_split_queue_memcg(folio, ds_queue);
4358		list_add_tail(&folio->_deferred_list, &ds_queue->split_queue);
4359		ds_queue->split_queue_len++;
4360		if (memcg)
4361			set_shrinker_bit(memcg, folio_nid(folio),
4362					 shrinker_id(deferred_split_shrinker));
4363	}
4364	split_queue_unlock_irqrestore(ds_queue, flags);
4365}
4366
4367static unsigned long deferred_split_count(struct shrinker *shrink,
4368		struct shrink_control *sc)
4369{
4370	struct pglist_data *pgdata = NODE_DATA(sc->nid);
4371	struct deferred_split *ds_queue = &pgdata->deferred_split_queue;
4372
4373#ifdef CONFIG_MEMCG
4374	if (sc->memcg)
4375		ds_queue = &sc->memcg->deferred_split_queue;
4376#endif
4377	return READ_ONCE(ds_queue->split_queue_len);
4378}
4379
4380static bool thp_underused(struct folio *folio)
4381{
4382	int num_zero_pages = 0, num_filled_pages = 0;
4383	int i;
4384
4385	if (khugepaged_max_ptes_none == HPAGE_PMD_NR - 1)
4386		return false;
4387
4388	if (folio_contain_hwpoisoned_page(folio))
4389		return false;
4390
4391	for (i = 0; i < folio_nr_pages(folio); i++) {
4392		if (pages_identical(folio_page(folio, i), ZERO_PAGE(0))) {
4393			if (++num_zero_pages > khugepaged_max_ptes_none)
4394				return true;
4395		} else {
4396			/*
4397			 * Another path for early exit once the number
4398			 * of non-zero filled pages exceeds threshold.
4399			 */
4400			if (++num_filled_pages >= HPAGE_PMD_NR - khugepaged_max_ptes_none)
4401				return false;
4402		}
4403	}
4404	return false;
4405}
4406
4407static unsigned long deferred_split_scan(struct shrinker *shrink,
4408		struct shrink_control *sc)
4409{
4410	struct deferred_split *ds_queue;
4411	unsigned long flags;
4412	struct folio *folio, *next;
4413	int split = 0, i;
4414	struct folio_batch fbatch;
4415
4416	folio_batch_init(&fbatch);
4417
4418retry:
4419	ds_queue = split_queue_lock_irqsave(sc->nid, sc->memcg, &flags);
4420	/* Take pin on all head pages to avoid freeing them under us */
4421	list_for_each_entry_safe(folio, next, &ds_queue->split_queue,
4422							_deferred_list) {
4423		if (folio_try_get(folio)) {
4424			folio_batch_add(&fbatch, folio);
4425		} else if (folio_test_partially_mapped(folio)) {
4426			/* We lost race with folio_put() */
4427			folio_clear_partially_mapped(folio);
4428			mod_mthp_stat(folio_order(folio),
4429				      MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1);
4430		}
4431		list_del_init(&folio->_deferred_list);
4432		ds_queue->split_queue_len--;
4433		if (!--sc->nr_to_scan)
4434			break;
4435		if (!folio_batch_space(&fbatch))
4436			break;
4437	}
4438	split_queue_unlock_irqrestore(ds_queue, flags);
4439
4440	for (i = 0; i < folio_batch_count(&fbatch); i++) {
4441		bool did_split = false;
4442		bool underused = false;
4443		struct deferred_split *fqueue;
4444
4445		folio = fbatch.folios[i];
4446		if (!folio_test_partially_mapped(folio)) {
4447			/*
4448			 * See try_to_map_unused_to_zeropage(): we cannot
4449			 * optimize zero-filled pages after splitting an
4450			 * mlocked folio.
4451			 */
4452			if (folio_test_mlocked(folio))
4453				goto next;
4454			underused = thp_underused(folio);
4455			if (!underused)
4456				goto next;
4457		}
4458		if (!folio_trylock(folio))
4459			goto next;
4460		if (!split_folio(folio)) {
4461			did_split = true;
4462			if (underused)
4463				count_vm_event(THP_UNDERUSED_SPLIT_PAGE);
4464			split++;
4465		}
4466		folio_unlock(folio);
4467next:
4468		if (did_split || !folio_test_partially_mapped(folio))
4469			continue;
4470		/*
4471		 * Only add back to the queue if folio is partially mapped.
4472		 * If thp_underused returns false, or if split_folio fails
4473		 * in the case it was underused, then consider it used and
4474		 * don't add it back to split_queue.
4475		 */
4476		fqueue = folio_split_queue_lock_irqsave(folio, &flags);
4477		if (list_empty(&folio->_deferred_list)) {
4478			list_add_tail(&folio->_deferred_list, &fqueue->split_queue);
4479			fqueue->split_queue_len++;
4480		}
4481		split_queue_unlock_irqrestore(fqueue, flags);
4482	}
4483	folios_put(&fbatch);
4484
4485	if (sc->nr_to_scan && !list_empty(&ds_queue->split_queue)) {
4486		cond_resched();
4487		goto retry;
4488	}
4489
4490	/*
4491	 * Stop shrinker if we didn't split any page, but the queue is empty.
4492	 * This can happen if pages were freed under us.
4493	 */
4494	if (!split && list_empty(&ds_queue->split_queue))
4495		return SHRINK_STOP;
4496	return split;
4497}
4498
4499#ifdef CONFIG_MEMCG
4500void reparent_deferred_split_queue(struct mem_cgroup *memcg)
4501{
4502	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
4503	struct deferred_split *ds_queue = &memcg->deferred_split_queue;
4504	struct deferred_split *parent_ds_queue = &parent->deferred_split_queue;
4505	int nid;
4506
4507	spin_lock_irq(&ds_queue->split_queue_lock);
4508	spin_lock_nested(&parent_ds_queue->split_queue_lock, SINGLE_DEPTH_NESTING);
4509
4510	if (!ds_queue->split_queue_len)
4511		goto unlock;
4512
4513	list_splice_tail_init(&ds_queue->split_queue, &parent_ds_queue->split_queue);
4514	parent_ds_queue->split_queue_len += ds_queue->split_queue_len;
4515	ds_queue->split_queue_len = 0;
4516
4517	for_each_node(nid)
4518		set_shrinker_bit(parent, nid, shrinker_id(deferred_split_shrinker));
4519
4520unlock:
4521	spin_unlock(&parent_ds_queue->split_queue_lock);
4522	spin_unlock_irq(&ds_queue->split_queue_lock);
4523}
4524#endif
4525
4526#ifdef CONFIG_DEBUG_FS
4527static void split_huge_pages_all(void)
4528{
4529	struct zone *zone;
4530	struct page *page;
4531	struct folio *folio;
4532	unsigned long pfn, max_zone_pfn;
4533	unsigned long total = 0, split = 0;
4534
4535	pr_debug("Split all THPs\n");
4536	for_each_zone(zone) {
4537		if (!managed_zone(zone))
4538			continue;
4539		max_zone_pfn = zone_end_pfn(zone);
4540		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
4541			int nr_pages;
4542
4543			page = pfn_to_online_page(pfn);
4544			if (!page || PageTail(page))
4545				continue;
4546			folio = page_folio(page);
4547			if (!folio_try_get(folio))
4548				continue;
4549
4550			if (unlikely(page_folio(page) != folio))
4551				goto next;
4552
4553			if (zone != folio_zone(folio))
4554				goto next;
4555
4556			if (!folio_test_large(folio)
4557				|| folio_test_hugetlb(folio)
4558				|| !folio_test_lru(folio))
4559				goto next;
4560
4561			total++;
4562			folio_lock(folio);
4563			nr_pages = folio_nr_pages(folio);
4564			if (!split_folio(folio))
4565				split++;
4566			pfn += nr_pages - 1;
4567			folio_unlock(folio);
4568next:
4569			folio_put(folio);
4570			cond_resched();
4571		}
4572	}
4573
4574	pr_debug("%lu of %lu THP split\n", split, total);
4575}
4576
4577static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)
4578{
4579	return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) ||
4580		    is_vm_hugetlb_page(vma);
4581}
4582
4583static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
4584				unsigned long vaddr_end, unsigned int new_order,
4585				long in_folio_offset)
4586{
4587	int ret = 0;
4588	struct task_struct *task;
4589	struct mm_struct *mm;
4590	unsigned long total = 0, split = 0;
4591	unsigned long addr;
4592
4593	vaddr_start &= PAGE_MASK;
4594	vaddr_end &= PAGE_MASK;
4595
4596	task = find_get_task_by_vpid(pid);
4597	if (!task) {
4598		ret = -ESRCH;
4599		goto out;
4600	}
4601
4602	/* Find the mm_struct */
4603	mm = get_task_mm(task);
4604	put_task_struct(task);
4605
4606	if (!mm) {
4607		ret = -EINVAL;
4608		goto out;
4609	}
4610
4611	pr_debug("Split huge pages in pid: %d, vaddr: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n",
4612		 pid, vaddr_start, vaddr_end, new_order, in_folio_offset);
4613
4614	mmap_read_lock(mm);
4615	/*
4616	 * always increase addr by PAGE_SIZE, since we could have a PTE page
4617	 * table filled with PTE-mapped THPs, each of which is distinct.
4618	 */
4619	for (addr = vaddr_start; addr < vaddr_end; addr += PAGE_SIZE) {
4620		struct vm_area_struct *vma = vma_lookup(mm, addr);
4621		struct folio_walk fw;
4622		struct folio *folio;
4623		struct address_space *mapping;
4624		unsigned int target_order = new_order;
4625
4626		if (!vma)
4627			break;
4628
4629		/* skip special VMA and hugetlb VMA */
4630		if (vma_not_suitable_for_thp_split(vma)) {
4631			addr = vma->vm_end;
4632			continue;
4633		}
4634
4635		folio = folio_walk_start(&fw, vma, addr, 0);
4636		if (!folio)
4637			continue;
4638
4639		if (!is_transparent_hugepage(folio))
4640			goto next;
4641
4642		if (!folio_test_anon(folio)) {
4643			mapping = folio->mapping;
4644			target_order = max(new_order,
4645					   mapping_min_folio_order(mapping));
4646		}
4647
4648		if (target_order >= folio_order(folio))
4649			goto next;
4650
4651		total++;
4652		/*
4653		 * For folios with private, split_huge_page_to_list_to_order()
4654		 * will try to drop it before split and then check if the folio
4655		 * can be split or not. So skip the check here.
4656		 */
4657		if (!folio_test_private(folio) &&
4658		    folio_expected_ref_count(folio) != folio_ref_count(folio))
4659			goto next;
4660
4661		if (!folio_trylock(folio))
4662			goto next;
4663		folio_get(folio);
4664		folio_walk_end(&fw, vma);
4665
4666		if (!folio_test_anon(folio) && folio->mapping != mapping)
4667			goto unlock;
4668
4669		if (in_folio_offset < 0 ||
4670		    in_folio_offset >= folio_nr_pages(folio)) {
4671			if (!split_folio_to_order(folio, target_order))
4672				split++;
4673		} else {
4674			struct page *split_at = folio_page(folio,
4675							   in_folio_offset);
4676			if (!folio_split(folio, target_order, split_at, NULL))
4677				split++;
4678		}
4679
4680unlock:
4681
4682		folio_unlock(folio);
4683		folio_put(folio);
4684
4685		cond_resched();
4686		continue;
4687next:
4688		folio_walk_end(&fw, vma);
4689		cond_resched();
4690	}
4691	mmap_read_unlock(mm);
4692	mmput(mm);
4693
4694	pr_debug("%lu of %lu THP split\n", split, total);
4695
4696out:
4697	return ret;
4698}
4699
4700static int split_huge_pages_in_file(const char *file_path, pgoff_t off_start,
4701				pgoff_t off_end, unsigned int new_order,
4702				long in_folio_offset)
4703{
4704	struct filename *file;
4705	struct file *candidate;
4706	struct address_space *mapping;
4707	int ret = -EINVAL;
4708	pgoff_t index;
4709	int nr_pages = 1;
4710	unsigned long total = 0, split = 0;
4711	unsigned int min_order;
4712	unsigned int target_order;
4713
4714	file = getname_kernel(file_path);
4715	if (IS_ERR(file))
4716		return ret;
4717
4718	candidate = file_open_name(file, O_RDONLY, 0);
4719	if (IS_ERR(candidate))
4720		goto out;
4721
4722	pr_debug("split file-backed THPs in file: %s, page offset: [0x%lx - 0x%lx], new_order: %u, in_folio_offset: %ld\n",
4723		 file_path, off_start, off_end, new_order, in_folio_offset);
4724
4725	mapping = candidate->f_mapping;
4726	min_order = mapping_min_folio_order(mapping);
4727	target_order = max(new_order, min_order);
4728
4729	for (index = off_start; index < off_end; index += nr_pages) {
4730		struct folio *folio = filemap_get_folio(mapping, index);
4731
4732		nr_pages = 1;
4733		if (IS_ERR(folio))
4734			continue;
4735
4736		if (!folio_test_large(folio))
4737			goto next;
4738
4739		total++;
4740		nr_pages = folio_nr_pages(folio);
4741
4742		if (target_order >= folio_order(folio))
4743			goto next;
4744
4745		if (!folio_trylock(folio))
4746			goto next;
4747
4748		if (folio->mapping != mapping)
4749			goto unlock;
4750
4751		if (in_folio_offset < 0 || in_folio_offset >= nr_pages) {
4752			if (!split_folio_to_order(folio, target_order))
4753				split++;
4754		} else {
4755			struct page *split_at = folio_page(folio,
4756							   in_folio_offset);
4757			if (!folio_split(folio, target_order, split_at, NULL))
4758				split++;
4759		}
4760
4761unlock:
4762		folio_unlock(folio);
4763next:
4764		folio_put(folio);
4765		cond_resched();
4766	}
4767
4768	filp_close(candidate, NULL);
4769	ret = 0;
4770
4771	pr_debug("%lu of %lu file-backed THP split\n", split, total);
4772out:
4773	putname(file);
4774	return ret;
4775}
4776
4777#define MAX_INPUT_BUF_SZ 255
4778
4779static ssize_t split_huge_pages_write(struct file *file, const char __user *buf,
4780				size_t count, loff_t *ppops)
4781{
4782	static DEFINE_MUTEX(split_debug_mutex);
4783	ssize_t ret;
4784	/*
4785	 * hold pid, start_vaddr, end_vaddr, new_order or
4786	 * file_path, off_start, off_end, new_order
4787	 */
4788	char input_buf[MAX_INPUT_BUF_SZ];
4789	int pid;
4790	unsigned long vaddr_start, vaddr_end;
4791	unsigned int new_order = 0;
4792	long in_folio_offset = -1;
4793
4794	ret = mutex_lock_interruptible(&split_debug_mutex);
4795	if (ret)
4796		return ret;
4797
4798	ret = -EFAULT;
4799
4800	memset(input_buf, 0, MAX_INPUT_BUF_SZ);
4801	if (copy_from_user(input_buf, buf, min_t(size_t, count, MAX_INPUT_BUF_SZ)))
4802		goto out;
4803
4804	input_buf[MAX_INPUT_BUF_SZ - 1] = '\0';
4805
4806	if (input_buf[0] == '/') {
4807		char *tok;
4808		char *tok_buf = input_buf;
4809		char file_path[MAX_INPUT_BUF_SZ];
4810		pgoff_t off_start = 0, off_end = 0;
4811		size_t input_len = strlen(input_buf);
4812
4813		tok = strsep(&tok_buf, ",");
4814		if (tok && tok_buf) {
4815			strscpy(file_path, tok);
4816		} else {
4817			ret = -EINVAL;
4818			goto out;
4819		}
4820
4821		ret = sscanf(tok_buf, "0x%lx,0x%lx,%d,%ld", &off_start, &off_end,
4822				&new_order, &in_folio_offset);
4823		if (ret != 2 && ret != 3 && ret != 4) {
4824			ret = -EINVAL;
4825			goto out;
4826		}
4827		ret = split_huge_pages_in_file(file_path, off_start, off_end,
4828				new_order, in_folio_offset);
4829		if (!ret)
4830			ret = input_len;
4831
4832		goto out;
4833	}
4834
4835	ret = sscanf(input_buf, "%d,0x%lx,0x%lx,%d,%ld", &pid, &vaddr_start,
4836			&vaddr_end, &new_order, &in_folio_offset);
4837	if (ret == 1 && pid == 1) {
4838		split_huge_pages_all();
4839		ret = strlen(input_buf);
4840		goto out;
4841	} else if (ret != 3 && ret != 4 && ret != 5) {
4842		ret = -EINVAL;
4843		goto out;
4844	}
4845
4846	ret = split_huge_pages_pid(pid, vaddr_start, vaddr_end, new_order,
4847			in_folio_offset);
4848	if (!ret)
4849		ret = strlen(input_buf);
4850out:
4851	mutex_unlock(&split_debug_mutex);
4852	return ret;
4853
4854}
4855
4856static const struct file_operations split_huge_pages_fops = {
4857	.owner	 = THIS_MODULE,
4858	.write	 = split_huge_pages_write,
4859};
4860
4861static int __init split_huge_pages_debugfs(void)
4862{
4863	debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
4864			    &split_huge_pages_fops);
4865	return 0;
4866}
4867late_initcall(split_huge_pages_debugfs);
4868#endif
4869
4870#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
4871int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
4872		struct page *page)
4873{
4874	struct folio *folio = page_folio(page);
4875	struct vm_area_struct *vma = pvmw->vma;
4876	struct mm_struct *mm = vma->vm_mm;
4877	unsigned long address = pvmw->address;
4878	bool anon_exclusive;
4879	pmd_t pmdval;
4880	swp_entry_t entry;
4881	pmd_t pmdswp;
4882
4883	if (!(pvmw->pmd && !pvmw->pte))
4884		return 0;
4885
4886	flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
4887	if (unlikely(!pmd_present(*pvmw->pmd)))
4888		pmdval = pmdp_huge_get_and_clear(vma->vm_mm, address, pvmw->pmd);
4889	else
4890		pmdval = pmdp_invalidate(vma, address, pvmw->pmd);
4891
4892	/* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */
4893	anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page);
4894	if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) {
4895		set_pmd_at(mm, address, pvmw->pmd, pmdval);
4896		return -EBUSY;
4897	}
4898
4899	if (pmd_dirty(pmdval))
4900		folio_mark_dirty(folio);
4901	if (pmd_write(pmdval))
4902		entry = make_writable_migration_entry(page_to_pfn(page));
4903	else if (anon_exclusive)
4904		entry = make_readable_exclusive_migration_entry(page_to_pfn(page));
4905	else
4906		entry = make_readable_migration_entry(page_to_pfn(page));
4907	if (pmd_young(pmdval))
4908		entry = make_migration_entry_young(entry);
4909	if (pmd_dirty(pmdval))
4910		entry = make_migration_entry_dirty(entry);
4911	pmdswp = swp_entry_to_pmd(entry);
4912	if (pmd_soft_dirty(pmdval))
4913		pmdswp = pmd_swp_mksoft_dirty(pmdswp);
4914	if (pmd_uffd_wp(pmdval))
4915		pmdswp = pmd_swp_mkuffd_wp(pmdswp);
4916	set_pmd_at(mm, address, pvmw->pmd, pmdswp);
4917	folio_remove_rmap_pmd(folio, page, vma);
4918	folio_put(folio);
4919	trace_set_migration_pmd(address, pmd_val(pmdswp));
4920
4921	return 0;
4922}
4923
4924void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
4925{
4926	struct folio *folio = page_folio(new);
4927	struct vm_area_struct *vma = pvmw->vma;
4928	struct mm_struct *mm = vma->vm_mm;
4929	unsigned long address = pvmw->address;
4930	unsigned long haddr = address & HPAGE_PMD_MASK;
4931	pmd_t pmde;
4932	softleaf_t entry;
4933
4934	if (!(pvmw->pmd && !pvmw->pte))
4935		return;
4936
4937	entry = softleaf_from_pmd(*pvmw->pmd);
4938	folio_get(folio);
4939	pmde = folio_mk_pmd(folio, READ_ONCE(vma->vm_page_prot));
4940
4941	if (pmd_swp_soft_dirty(*pvmw->pmd))
4942		pmde = pmd_mksoft_dirty(pmde);
4943	if (softleaf_is_migration_write(entry))
4944		pmde = pmd_mkwrite(pmde, vma);
4945	if (pmd_swp_uffd_wp(*pvmw->pmd))
4946		pmde = pmd_mkuffd_wp(pmde);
4947	if (!softleaf_is_migration_young(entry))
4948		pmde = pmd_mkold(pmde);
4949	/* NOTE: this may contain setting soft-dirty on some archs */
4950	if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry))
4951		pmde = pmd_mkdirty(pmde);
4952
4953	if (folio_is_device_private(folio)) {
4954		swp_entry_t entry;
4955
4956		if (pmd_write(pmde))
4957			entry = make_writable_device_private_entry(
4958							page_to_pfn(new));
4959		else
4960			entry = make_readable_device_private_entry(
4961							page_to_pfn(new));
4962		pmde = swp_entry_to_pmd(entry);
4963
4964		if (pmd_swp_soft_dirty(*pvmw->pmd))
4965			pmde = pmd_swp_mksoft_dirty(pmde);
4966		if (pmd_swp_uffd_wp(*pvmw->pmd))
4967			pmde = pmd_swp_mkuffd_wp(pmde);
4968	}
4969
4970	if (folio_test_anon(folio)) {
4971		rmap_t rmap_flags = RMAP_NONE;
4972
4973		if (!softleaf_is_migration_read(entry))
4974			rmap_flags |= RMAP_EXCLUSIVE;
4975
4976		folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags);
4977	} else {
4978		folio_add_file_rmap_pmd(folio, new, vma);
4979	}
4980	VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new));
4981	set_pmd_at(mm, haddr, pvmw->pmd, pmde);
4982
4983	/* No need to invalidate - it was non-present before */
4984	update_mmu_cache_pmd(vma, address, pvmw->pmd);
4985	trace_remove_migration_pmd(address, pmd_val(pmde));
4986}
4987#endif