hugetlb.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2#ifndef _LINUX_HUGETLB_H
   3#define _LINUX_HUGETLB_H
   4
   5#include <linux/mm.h>
   6#include <linux/mm_types.h>
   7#include <linux/mmdebug.h>
   8#include <linux/fs.h>
   9#include <linux/hugetlb_inline.h>
  10#include <linux/cgroup.h>
  11#include <linux/page_ref.h>
  12#include <linux/list.h>
  13#include <linux/kref.h>
  14#include <linux/pgtable.h>
  15#include <linux/gfp.h>
  16#include <linux/userfaultfd_k.h>
  17#include <linux/nodemask.h>
  18
  19struct ctl_table;
  20struct user_struct;
  21struct mmu_gather;
  22struct node;
  23
  24void free_huge_folio(struct folio *folio);
  25
  26#ifdef CONFIG_HUGETLB_PAGE
  27
  28#include <linux/pagemap.h>
  29#include <linux/shm.h>
  30#include <asm/tlbflush.h>
  31
  32/*
  33 * For HugeTLB page, there are more metadata to save in the struct page. But
  34 * the head struct page cannot meet our needs, so we have to abuse other tail
  35 * struct page to store the metadata.
  36 */
  37#define __NR_USED_SUBPAGE 3
  38
  39struct hugepage_subpool {
  40	spinlock_t lock;
  41	long count;
  42	long max_hpages;	/* Maximum huge pages or -1 if no maximum. */
  43	long used_hpages;	/* Used count against maximum, includes */
  44				/* both allocated and reserved pages. */
  45	struct hstate *hstate;
  46	long min_hpages;	/* Minimum huge pages or -1 if no minimum. */
  47	long rsv_hpages;	/* Pages reserved against global pool to */
  48				/* satisfy minimum size. */
  49};
  50
  51struct resv_map {
  52	struct kref refs;
  53	spinlock_t lock;
  54	struct list_head regions;
  55	long adds_in_progress;
  56	struct list_head region_cache;
  57	long region_cache_count;
  58	struct rw_semaphore rw_sema;
  59#ifdef CONFIG_CGROUP_HUGETLB
  60	/*
  61	 * On private mappings, the counter to uncharge reservations is stored
  62	 * here. If these fields are 0, then either the mapping is shared, or
  63	 * cgroup accounting is disabled for this resv_map.
  64	 */
  65	struct page_counter *reservation_counter;
  66	unsigned long pages_per_hpage;
  67	struct cgroup_subsys_state *css;
  68#endif
  69};
  70
  71/*
  72 * Region tracking -- allows tracking of reservations and instantiated pages
  73 *                    across the pages in a mapping.
  74 *
  75 * The region data structures are embedded into a resv_map and protected
  76 * by a resv_map's lock.  The set of regions within the resv_map represent
  77 * reservations for huge pages, or huge pages that have already been
  78 * instantiated within the map.  The from and to elements are huge page
  79 * indices into the associated mapping.  from indicates the starting index
  80 * of the region.  to represents the first index past the end of  the region.
  81 *
  82 * For example, a file region structure with from == 0 and to == 4 represents
  83 * four huge pages in a mapping.  It is important to note that the to element
  84 * represents the first element past the end of the region. This is used in
  85 * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
  86 *
  87 * Interval notation of the form [from, to) will be used to indicate that
  88 * the endpoint from is inclusive and to is exclusive.
  89 */
  90struct file_region {
  91	struct list_head link;
  92	long from;
  93	long to;
  94#ifdef CONFIG_CGROUP_HUGETLB
  95	/*
  96	 * On shared mappings, each reserved region appears as a struct
  97	 * file_region in resv_map. These fields hold the info needed to
  98	 * uncharge each reservation.
  99	 */
 100	struct page_counter *reservation_counter;
 101	struct cgroup_subsys_state *css;
 102#endif
 103};
 104
 105struct hugetlb_vma_lock {
 106	struct kref refs;
 107	struct rw_semaphore rw_sema;
 108	struct vm_area_struct *vma;
 109};
 110
 111extern struct resv_map *resv_map_alloc(void);
 112void resv_map_release(struct kref *ref);
 113
 114extern spinlock_t hugetlb_lock;
 115extern int hugetlb_max_hstate __read_mostly;
 116#define for_each_hstate(h) \
 117	for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)
 118
 119struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
 120						long min_hpages);
 121void hugepage_put_subpool(struct hugepage_subpool *spool);
 122
 123void hugetlb_dup_vma_private(struct vm_area_struct *vma);
 124void clear_vma_resv_huge_pages(struct vm_area_struct *vma);
 125int move_hugetlb_page_tables(struct vm_area_struct *vma,
 126			     struct vm_area_struct *new_vma,
 127			     unsigned long old_addr, unsigned long new_addr,
 128			     unsigned long len);
 129int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *,
 130			    struct vm_area_struct *, struct vm_area_struct *);
 131void unmap_hugepage_range(struct vm_area_struct *,
 132			  unsigned long start, unsigned long end,
 133			  struct folio *, zap_flags_t);
 134void __unmap_hugepage_range(struct mmu_gather *tlb,
 135			  struct vm_area_struct *vma,
 136			  unsigned long start, unsigned long end,
 137			  struct folio *, zap_flags_t zap_flags);
 138void hugetlb_report_meminfo(struct seq_file *);
 139int hugetlb_report_node_meminfo(char *buf, int len, int nid);
 140void hugetlb_show_meminfo_node(int nid);
 141unsigned long hugetlb_total_pages(void);
 142vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 143			unsigned long address, unsigned int flags);
 144#ifdef CONFIG_USERFAULTFD
 145int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
 146			     struct vm_area_struct *dst_vma,
 147			     unsigned long dst_addr,
 148			     unsigned long src_addr,
 149			     uffd_flags_t flags,
 150			     struct folio **foliop);
 151#endif /* CONFIG_USERFAULTFD */
 152long hugetlb_reserve_pages(struct inode *inode, long from, long to,
 153			   struct vm_area_desc *desc, vm_flags_t vm_flags);
 154long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
 155						long freed);
 156bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list);
 157int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison);
 158int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
 159				bool *migratable_cleared);
 160void folio_putback_hugetlb(struct folio *folio);
 161void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason);
 162void hugetlb_fix_reserve_counts(struct inode *inode);
 163extern struct mutex *hugetlb_fault_mutex_table;
 164u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx);
 165
 166pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
 167		      unsigned long addr, pud_t *pud);
 168bool hugetlbfs_pagecache_present(struct hstate *h,
 169				 struct vm_area_struct *vma,
 170				 unsigned long address);
 171
 172struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio);
 173
 174extern int sysctl_hugetlb_shm_group __read_mostly;
 175extern struct list_head huge_boot_pages[MAX_NUMNODES];
 176
 177void hugetlb_bootmem_alloc(void);
 178bool hugetlb_bootmem_allocated(void);
 179extern nodemask_t hugetlb_bootmem_nodes;
 180void hugetlb_bootmem_set_nodes(void);
 181
 182/* arch callbacks */
 183
 184#ifndef CONFIG_HIGHPTE
 185/*
 186 * pte_offset_huge() and pte_alloc_huge() are helpers for those architectures
 187 * which may go down to the lowest PTE level in their huge_pte_offset() and
 188 * huge_pte_alloc(): to avoid reliance on pte_offset_map() without pte_unmap().
 189 */
 190static inline pte_t *pte_offset_huge(pmd_t *pmd, unsigned long address)
 191{
 192	return pte_offset_kernel(pmd, address);
 193}
 194static inline pte_t *pte_alloc_huge(struct mm_struct *mm, pmd_t *pmd,
 195				    unsigned long address)
 196{
 197	return pte_alloc(mm, pmd) ? NULL : pte_offset_huge(pmd, address);
 198}
 199#endif
 200
 201pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
 202			unsigned long addr, unsigned long sz);
 203/*
 204 * huge_pte_offset(): Walk the hugetlb pgtable until the last level PTE.
 205 * Returns the pte_t* if found, or NULL if the address is not mapped.
 206 *
 207 * IMPORTANT: we should normally not directly call this function, instead
 208 * this is only a common interface to implement arch-specific
 209 * walker. Please use hugetlb_walk() instead, because that will attempt to
 210 * verify the locking for you.
 211 *
 212 * Since this function will walk all the pgtable pages (including not only
 213 * high-level pgtable page, but also PUD entry that can be unshared
 214 * concurrently for VM_SHARED), the caller of this function should be
 215 * responsible of its thread safety.  One can follow this rule:
 216 *
 217 *  (1) For private mappings: pmd unsharing is not possible, so holding the
 218 *      mmap_lock for either read or write is sufficient. Most callers
 219 *      already hold the mmap_lock, so normally, no special action is
 220 *      required.
 221 *
 222 *  (2) For shared mappings: pmd unsharing is possible (so the PUD-ranged
 223 *      pgtable page can go away from under us!  It can be done by a pmd
 224 *      unshare with a follow up munmap() on the other process), then we
 225 *      need either:
 226 *
 227 *     (2.1) hugetlb vma lock read or write held, to make sure pmd unshare
 228 *           won't happen upon the range (it also makes sure the pte_t we
 229 *           read is the right and stable one), or,
 230 *
 231 *     (2.2) hugetlb mapping i_mmap_rwsem lock held read or write, to make
 232 *           sure even if unshare happened the racy unmap() will wait until
 233 *           i_mmap_rwsem is released.
 234 *
 235 * Option (2.1) is the safest, which guarantees pte stability from pmd
 236 * sharing pov, until the vma lock released.  Option (2.2) doesn't protect
 237 * a concurrent pmd unshare, but it makes sure the pgtable page is safe to
 238 * access.
 239 */
 240pte_t *huge_pte_offset(struct mm_struct *mm,
 241		       unsigned long addr, unsigned long sz);
 242unsigned long hugetlb_mask_last_page(struct hstate *h);
 243int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
 244		unsigned long addr, pte_t *ptep);
 245void huge_pmd_unshare_flush(struct mmu_gather *tlb, struct vm_area_struct *vma);
 246void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
 247				unsigned long *start, unsigned long *end);
 248
 249extern void __hugetlb_zap_begin(struct vm_area_struct *vma,
 250				unsigned long *begin, unsigned long *end);
 251extern void __hugetlb_zap_end(struct vm_area_struct *vma,
 252			      struct zap_details *details);
 253
 254static inline void hugetlb_zap_begin(struct vm_area_struct *vma,
 255				     unsigned long *start, unsigned long *end)
 256{
 257	if (is_vm_hugetlb_page(vma))
 258		__hugetlb_zap_begin(vma, start, end);
 259}
 260
 261static inline void hugetlb_zap_end(struct vm_area_struct *vma,
 262				   struct zap_details *details)
 263{
 264	if (is_vm_hugetlb_page(vma))
 265		__hugetlb_zap_end(vma, details);
 266}
 267
 268void hugetlb_vma_lock_read(struct vm_area_struct *vma);
 269void hugetlb_vma_unlock_read(struct vm_area_struct *vma);
 270void hugetlb_vma_lock_write(struct vm_area_struct *vma);
 271void hugetlb_vma_unlock_write(struct vm_area_struct *vma);
 272int hugetlb_vma_trylock_write(struct vm_area_struct *vma);
 273void hugetlb_vma_assert_locked(struct vm_area_struct *vma);
 274void hugetlb_vma_lock_release(struct kref *kref);
 275long hugetlb_change_protection(struct vm_area_struct *vma,
 276		unsigned long address, unsigned long end, pgprot_t newprot,
 277		unsigned long cp_flags);
 278void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);
 279void fixup_hugetlb_reservations(struct vm_area_struct *vma);
 280void hugetlb_split(struct vm_area_struct *vma, unsigned long addr);
 281int hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
 282
 283#else /* !CONFIG_HUGETLB_PAGE */
 284
 285static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma)
 286{
 287}
 288
 289static inline void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
 290{
 291}
 292
 293static inline unsigned long hugetlb_total_pages(void)
 294{
 295	return 0;
 296}
 297
 298static inline struct address_space *hugetlb_folio_mapping_lock_write(
 299							struct folio *folio)
 300{
 301	return NULL;
 302}
 303
 304static inline int huge_pmd_unshare(struct mmu_gather *tlb,
 305		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
 306{
 307	return 0;
 308}
 309
 310static inline void huge_pmd_unshare_flush(struct mmu_gather *tlb,
 311		struct vm_area_struct *vma)
 312{
 313}
 314
 315static inline void adjust_range_if_pmd_sharing_possible(
 316				struct vm_area_struct *vma,
 317				unsigned long *start, unsigned long *end)
 318{
 319}
 320
 321static inline void hugetlb_zap_begin(
 322				struct vm_area_struct *vma,
 323				unsigned long *start, unsigned long *end)
 324{
 325}
 326
 327static inline void hugetlb_zap_end(
 328				struct vm_area_struct *vma,
 329				struct zap_details *details)
 330{
 331}
 332
 333static inline int copy_hugetlb_page_range(struct mm_struct *dst,
 334					  struct mm_struct *src,
 335					  struct vm_area_struct *dst_vma,
 336					  struct vm_area_struct *src_vma)
 337{
 338	BUG();
 339	return 0;
 340}
 341
 342static inline int move_hugetlb_page_tables(struct vm_area_struct *vma,
 343					   struct vm_area_struct *new_vma,
 344					   unsigned long old_addr,
 345					   unsigned long new_addr,
 346					   unsigned long len)
 347{
 348	BUG();
 349	return 0;
 350}
 351
 352static inline void hugetlb_report_meminfo(struct seq_file *m)
 353{
 354}
 355
 356static inline int hugetlb_report_node_meminfo(char *buf, int len, int nid)
 357{
 358	return 0;
 359}
 360
 361static inline void hugetlb_show_meminfo_node(int nid)
 362{
 363}
 364
 365static inline void hugetlb_vma_lock_read(struct vm_area_struct *vma)
 366{
 367}
 368
 369static inline void hugetlb_vma_unlock_read(struct vm_area_struct *vma)
 370{
 371}
 372
 373static inline void hugetlb_vma_lock_write(struct vm_area_struct *vma)
 374{
 375}
 376
 377static inline void hugetlb_vma_unlock_write(struct vm_area_struct *vma)
 378{
 379}
 380
 381static inline int hugetlb_vma_trylock_write(struct vm_area_struct *vma)
 382{
 383	return 1;
 384}
 385
 386static inline void hugetlb_vma_assert_locked(struct vm_area_struct *vma)
 387{
 388}
 389
 390static inline int is_hugepage_only_range(struct mm_struct *mm,
 391					unsigned long addr, unsigned long len)
 392{
 393	return 0;
 394}
 395
 396#ifdef CONFIG_USERFAULTFD
 397static inline int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
 398					   struct vm_area_struct *dst_vma,
 399					   unsigned long dst_addr,
 400					   unsigned long src_addr,
 401					   uffd_flags_t flags,
 402					   struct folio **foliop)
 403{
 404	BUG();
 405	return 0;
 406}
 407#endif /* CONFIG_USERFAULTFD */
 408
 409static inline pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr,
 410					unsigned long sz)
 411{
 412	return NULL;
 413}
 414
 415static inline bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list)
 416{
 417	return false;
 418}
 419
 420static inline int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison)
 421{
 422	return 0;
 423}
 424
 425static inline int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
 426					bool *migratable_cleared)
 427{
 428	return 0;
 429}
 430
 431static inline void folio_putback_hugetlb(struct folio *folio)
 432{
 433}
 434
 435static inline void move_hugetlb_state(struct folio *old_folio,
 436					struct folio *new_folio, int reason)
 437{
 438}
 439
 440static inline long hugetlb_change_protection(
 441			struct vm_area_struct *vma, unsigned long address,
 442			unsigned long end, pgprot_t newprot,
 443			unsigned long cp_flags)
 444{
 445	return 0;
 446}
 447
 448static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
 449			struct vm_area_struct *vma, unsigned long start,
 450			unsigned long end, struct folio *folio,
 451			zap_flags_t zap_flags)
 452{
 453	BUG();
 454}
 455
 456static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
 457			struct vm_area_struct *vma, unsigned long address,
 458			unsigned int flags)
 459{
 460	BUG();
 461	return 0;
 462}
 463
 464static inline void hugetlb_unshare_all_pmds(struct vm_area_struct *vma) { }
 465
 466static inline void fixup_hugetlb_reservations(struct vm_area_struct *vma)
 467{
 468}
 469
 470static inline void hugetlb_split(struct vm_area_struct *vma, unsigned long addr) {}
 471
 472static inline int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
 473{
 474	return 0;
 475}
 476
 477#endif /* !CONFIG_HUGETLB_PAGE */
 478
 479#ifndef pgd_write
 480static inline int pgd_write(pgd_t pgd)
 481{
 482	BUG();
 483	return 0;
 484}
 485#endif
 486
 487#define HUGETLB_ANON_FILE "anon_hugepage"
 488
 489enum {
 490	/*
 491	 * The file will be used as an shm file so shmfs accounting rules
 492	 * apply
 493	 */
 494	HUGETLB_SHMFS_INODE     = 1,
 495	/*
 496	 * The file is being created on the internal vfs mount and shmfs
 497	 * accounting rules do not apply
 498	 */
 499	HUGETLB_ANONHUGE_INODE  = 2,
 500};
 501
 502#ifdef CONFIG_HUGETLBFS
 503struct hugetlbfs_sb_info {
 504	long	max_inodes;   /* inodes allowed */
 505	long	free_inodes;  /* inodes free */
 506	spinlock_t	stat_lock;
 507	struct hstate *hstate;
 508	struct hugepage_subpool *spool;
 509	kuid_t	uid;
 510	kgid_t	gid;
 511	umode_t mode;
 512};
 513
 514static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
 515{
 516	return sb->s_fs_info;
 517}
 518
 519struct hugetlbfs_inode_info {
 520	struct inode vfs_inode;
 521	unsigned int seals;
 522};
 523
 524static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
 525{
 526	return container_of(inode, struct hugetlbfs_inode_info, vfs_inode);
 527}
 528
 529extern const struct vm_operations_struct hugetlb_vm_ops;
 530struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
 531				int creat_flags, int page_size_log);
 532
 533static inline bool is_file_hugepages(const struct file *file)
 534{
 535	return file->f_op->fop_flags & FOP_HUGE_PAGES;
 536}
 537
 538static inline struct hstate *hstate_inode(struct inode *i)
 539{
 540	return HUGETLBFS_SB(i->i_sb)->hstate;
 541}
 542#else /* !CONFIG_HUGETLBFS */
 543
 544#define is_file_hugepages(file)			false
 545static inline struct file *
 546hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
 547		int creat_flags, int page_size_log)
 548{
 549	return ERR_PTR(-ENOSYS);
 550}
 551
 552static inline struct hstate *hstate_inode(struct inode *i)
 553{
 554	return NULL;
 555}
 556#endif /* !CONFIG_HUGETLBFS */
 557
 558unsigned long
 559hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
 560				    unsigned long len, unsigned long pgoff,
 561				    unsigned long flags);
 562
 563/*
 564 * huegtlb page specific state flags.  These flags are located in page.private
 565 * of the hugetlb head page.  Functions created via the below macros should be
 566 * used to manipulate these flags.
 567 *
 568 * HPG_restore_reserve - Set when a hugetlb page consumes a reservation at
 569 *	allocation time.  Cleared when page is fully instantiated.  Free
 570 *	routine checks flag to restore a reservation on error paths.
 571 *	Synchronization:  Examined or modified by code that knows it has
 572 *	the only reference to page.  i.e. After allocation but before use
 573 *	or when the page is being freed.
 574 * HPG_migratable  - Set after a newly allocated page is added to the page
 575 *	cache and/or page tables.  Indicates the page is a candidate for
 576 *	migration.
 577 *	Synchronization:  Initially set after new page allocation with no
 578 *	locking.  When examined and modified during migration processing
 579 *	(isolate, migrate, putback) the hugetlb_lock is held.
 580 * HPG_temporary - Set on a page that is temporarily allocated from the buddy
 581 *	allocator.  Typically used for migration target pages when no pages
 582 *	are available in the pool.  The hugetlb free page path will
 583 *	immediately free pages with this flag set to the buddy allocator.
 584 *	Synchronization: Can be set after huge page allocation from buddy when
 585 *	code knows it has only reference.  All other examinations and
 586 *	modifications require hugetlb_lock.
 587 * HPG_freed - Set when page is on the free lists.
 588 *	Synchronization: hugetlb_lock held for examination and modification.
 589 * HPG_vmemmap_optimized - Set when the vmemmap pages of the page are freed.
 590 * HPG_raw_hwp_unreliable - Set when the hugetlb page has a hwpoison sub-page
 591 *     that is not tracked by raw_hwp_page list.
 592 */
 593enum hugetlb_page_flags {
 594	HPG_restore_reserve = 0,
 595	HPG_migratable,
 596	HPG_temporary,
 597	HPG_freed,
 598	HPG_vmemmap_optimized,
 599	HPG_raw_hwp_unreliable,
 600	HPG_cma,
 601	__NR_HPAGEFLAGS,
 602};
 603
 604/*
 605 * Macros to create test, set and clear function definitions for
 606 * hugetlb specific page flags.
 607 */
 608#ifdef CONFIG_HUGETLB_PAGE
 609#define TESTHPAGEFLAG(uname, flname)				\
 610static __always_inline						\
 611bool folio_test_hugetlb_##flname(struct folio *folio)		\
 612	{	void *private = &folio->private;		\
 613		return test_bit(HPG_##flname, private);		\
 614	}
 615
 616#define SETHPAGEFLAG(uname, flname)				\
 617static __always_inline						\
 618void folio_set_hugetlb_##flname(struct folio *folio)		\
 619	{	void *private = &folio->private;		\
 620		set_bit(HPG_##flname, private);			\
 621	}
 622
 623#define CLEARHPAGEFLAG(uname, flname)				\
 624static __always_inline						\
 625void folio_clear_hugetlb_##flname(struct folio *folio)		\
 626	{	void *private = &folio->private;		\
 627		clear_bit(HPG_##flname, private);		\
 628	}
 629#else
 630#define TESTHPAGEFLAG(uname, flname)				\
 631static inline bool						\
 632folio_test_hugetlb_##flname(struct folio *folio)		\
 633	{ return 0; }
 634
 635#define SETHPAGEFLAG(uname, flname)				\
 636static inline void						\
 637folio_set_hugetlb_##flname(struct folio *folio) 		\
 638	{ }
 639
 640#define CLEARHPAGEFLAG(uname, flname)				\
 641static inline void						\
 642folio_clear_hugetlb_##flname(struct folio *folio)		\
 643	{ }
 644#endif
 645
 646#define HPAGEFLAG(uname, flname)				\
 647	TESTHPAGEFLAG(uname, flname)				\
 648	SETHPAGEFLAG(uname, flname)				\
 649	CLEARHPAGEFLAG(uname, flname)				\
 650
 651/*
 652 * Create functions associated with hugetlb page flags
 653 */
 654HPAGEFLAG(RestoreReserve, restore_reserve)
 655HPAGEFLAG(Migratable, migratable)
 656HPAGEFLAG(Temporary, temporary)
 657HPAGEFLAG(Freed, freed)
 658HPAGEFLAG(VmemmapOptimized, vmemmap_optimized)
 659HPAGEFLAG(RawHwpUnreliable, raw_hwp_unreliable)
 660HPAGEFLAG(Cma, cma)
 661
 662#ifdef CONFIG_HUGETLB_PAGE
 663
 664#define HSTATE_NAME_LEN 32
 665/* Defines one hugetlb page size */
 666struct hstate {
 667	struct mutex resize_lock;
 668	struct lock_class_key resize_key;
 669	int next_nid_to_alloc;
 670	int next_nid_to_free;
 671	unsigned int order;
 672	unsigned int demote_order;
 673	unsigned long mask;
 674	unsigned long max_huge_pages;
 675	unsigned long nr_huge_pages;
 676	unsigned long free_huge_pages;
 677	unsigned long resv_huge_pages;
 678	unsigned long surplus_huge_pages;
 679	unsigned long nr_overcommit_huge_pages;
 680	struct list_head hugepage_activelist;
 681	struct list_head hugepage_freelists[MAX_NUMNODES];
 682	unsigned int max_huge_pages_node[MAX_NUMNODES];
 683	unsigned int nr_huge_pages_node[MAX_NUMNODES];
 684	unsigned int free_huge_pages_node[MAX_NUMNODES];
 685	unsigned int surplus_huge_pages_node[MAX_NUMNODES];
 686	char name[HSTATE_NAME_LEN];
 687};
 688
 689struct cma;
 690
 691struct huge_bootmem_page {
 692	struct list_head list;
 693	struct hstate *hstate;
 694	unsigned long flags;
 695	struct cma *cma;
 696};
 697
 698#define HUGE_BOOTMEM_HVO		0x0001
 699#define HUGE_BOOTMEM_ZONES_VALID	0x0002
 700#define HUGE_BOOTMEM_CMA		0x0004
 701
 702bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m);
 703
 704int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list);
 705int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn);
 706void wait_for_freed_hugetlb_folios(void);
 707struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 708				unsigned long addr, bool cow_from_owner);
 709struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
 710				nodemask_t *nmask, gfp_t gfp_mask,
 711				bool allow_alloc_fallback);
 712struct folio *alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
 713					  nodemask_t *nmask, gfp_t gfp_mask);
 714
 715int hugetlb_add_to_page_cache(struct folio *folio, struct address_space *mapping,
 716			pgoff_t idx);
 717void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
 718				unsigned long address, struct folio *folio);
 719
 720/* arch callback */
 721int __init __alloc_bootmem_huge_page(struct hstate *h, int nid);
 722int __init alloc_bootmem_huge_page(struct hstate *h, int nid);
 723bool __init hugetlb_node_alloc_supported(void);
 724
 725void __init hugetlb_add_hstate(unsigned order);
 726bool __init arch_hugetlb_valid_size(unsigned long size);
 727struct hstate *size_to_hstate(unsigned long size);
 728
 729#ifndef HUGE_MAX_HSTATE
 730#define HUGE_MAX_HSTATE 1
 731#endif
 732
 733extern struct hstate hstates[HUGE_MAX_HSTATE];
 734extern unsigned int default_hstate_idx;
 735
 736#define default_hstate (hstates[default_hstate_idx])
 737
 738static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
 739{
 740	return HUGETLBFS_SB(inode->i_sb)->spool;
 741}
 742
 743static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio)
 744{
 745	return folio->_hugetlb_subpool;
 746}
 747
 748static inline void hugetlb_set_folio_subpool(struct folio *folio,
 749					struct hugepage_subpool *subpool)
 750{
 751	folio->_hugetlb_subpool = subpool;
 752}
 753
 754static inline struct hstate *hstate_file(struct file *f)
 755{
 756	return hstate_inode(file_inode(f));
 757}
 758
 759static inline struct hstate *hstate_sizelog(int page_size_log)
 760{
 761	if (!page_size_log)
 762		return &default_hstate;
 763
 764	if (page_size_log < BITS_PER_LONG)
 765		return size_to_hstate(1UL << page_size_log);
 766
 767	return NULL;
 768}
 769
 770static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
 771{
 772	return hstate_file(vma->vm_file);
 773}
 774
 775static inline unsigned long huge_page_size(const struct hstate *h)
 776{
 777	return (unsigned long)PAGE_SIZE << h->order;
 778}
 779
 780extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma);
 781
 782extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma);
 783
 784static inline unsigned long huge_page_mask(struct hstate *h)
 785{
 786	return h->mask;
 787}
 788
 789static inline unsigned int huge_page_order(struct hstate *h)
 790{
 791	return h->order;
 792}
 793
 794static inline unsigned huge_page_shift(struct hstate *h)
 795{
 796	return h->order + PAGE_SHIFT;
 797}
 798
 799/**
 800 * hugetlb_linear_page_index() - linear_page_index() but in hugetlb
 801 *				 page size granularity.
 802 * @vma: the hugetlb VMA
 803 * @address: the virtual address within the VMA
 804 *
 805 * Return: the page offset within the mapping in huge page units.
 806 */
 807static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma,
 808		unsigned long address)
 809{
 810	struct hstate *h = hstate_vma(vma);
 811
 812	return ((address - vma->vm_start) >> huge_page_shift(h)) +
 813		(vma->vm_pgoff >> huge_page_order(h));
 814}
 815
 816static inline bool order_is_gigantic(unsigned int order)
 817{
 818	return order > MAX_PAGE_ORDER;
 819}
 820
 821static inline bool hstate_is_gigantic(struct hstate *h)
 822{
 823	return order_is_gigantic(huge_page_order(h));
 824}
 825
 826static inline unsigned int pages_per_huge_page(const struct hstate *h)
 827{
 828	return 1 << h->order;
 829}
 830
 831static inline unsigned int blocks_per_huge_page(struct hstate *h)
 832{
 833	return huge_page_size(h) / 512;
 834}
 835
 836static inline struct folio *filemap_lock_hugetlb_folio(struct hstate *h,
 837				struct address_space *mapping, pgoff_t idx)
 838{
 839	return filemap_lock_folio(mapping, idx << huge_page_order(h));
 840}
 841
 842#include <asm/hugetlb.h>
 843
 844#ifndef is_hugepage_only_range
 845static inline int is_hugepage_only_range(struct mm_struct *mm,
 846					unsigned long addr, unsigned long len)
 847{
 848	return 0;
 849}
 850#define is_hugepage_only_range is_hugepage_only_range
 851#endif
 852
 853#ifndef arch_clear_hugetlb_flags
 854static inline void arch_clear_hugetlb_flags(struct folio *folio) { }
 855#define arch_clear_hugetlb_flags arch_clear_hugetlb_flags
 856#endif
 857
 858#ifndef arch_make_huge_pte
 859static inline pte_t arch_make_huge_pte(pte_t entry, unsigned int shift,
 860				       vm_flags_t flags)
 861{
 862	return pte_mkhuge(entry);
 863}
 864#endif
 865
 866#ifndef arch_has_huge_bootmem_alloc
 867/*
 868 * Some architectures do their own bootmem allocation, so they can't use
 869 * early CMA allocation.
 870 */
 871static inline bool arch_has_huge_bootmem_alloc(void)
 872{
 873	return false;
 874}
 875#endif
 876
 877static inline struct hstate *folio_hstate(struct folio *folio)
 878{
 879	VM_BUG_ON_FOLIO(!folio_test_hugetlb(folio), folio);
 880	return size_to_hstate(folio_size(folio));
 881}
 882
 883static inline unsigned hstate_index_to_shift(unsigned index)
 884{
 885	return hstates[index].order + PAGE_SHIFT;
 886}
 887
 888static inline int hstate_index(struct hstate *h)
 889{
 890	return h - hstates;
 891}
 892
 893int dissolve_free_hugetlb_folio(struct folio *folio);
 894int dissolve_free_hugetlb_folios(unsigned long start_pfn,
 895				    unsigned long end_pfn);
 896
 897#ifdef CONFIG_MEMORY_FAILURE
 898extern void folio_clear_hugetlb_hwpoison(struct folio *folio);
 899#else
 900static inline void folio_clear_hugetlb_hwpoison(struct folio *folio)
 901{
 902}
 903#endif
 904
 905#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
 906#ifndef arch_hugetlb_migration_supported
 907static inline bool arch_hugetlb_migration_supported(struct hstate *h)
 908{
 909	if ((huge_page_shift(h) == PMD_SHIFT) ||
 910		(huge_page_shift(h) == PUD_SHIFT) ||
 911			(huge_page_shift(h) == PGDIR_SHIFT))
 912		return true;
 913	else
 914		return false;
 915}
 916#endif
 917#else
 918static inline bool arch_hugetlb_migration_supported(struct hstate *h)
 919{
 920	return false;
 921}
 922#endif
 923
 924static inline bool hugepage_migration_supported(struct hstate *h)
 925{
 926	return arch_hugetlb_migration_supported(h);
 927}
 928
 929/*
 930 * Movability check is different as compared to migration check.
 931 * It determines whether or not a huge page should be placed on
 932 * movable zone or not. Movability of any huge page should be
 933 * required only if huge page size is supported for migration.
 934 * There won't be any reason for the huge page to be movable if
 935 * it is not migratable to start with. Also the size of the huge
 936 * page should be large enough to be placed under a movable zone
 937 * and still feasible enough to be migratable. Just the presence
 938 * in movable zone does not make the migration feasible.
 939 *
 940 * So even though large huge page sizes like the gigantic ones
 941 * are migratable they should not be movable because its not
 942 * feasible to migrate them from movable zone.
 943 */
 944static inline bool hugepage_movable_supported(struct hstate *h)
 945{
 946	if (!hugepage_migration_supported(h))
 947		return false;
 948
 949	if (hstate_is_gigantic(h))
 950		return false;
 951	return true;
 952}
 953
 954/* Movability of hugepages depends on migration support. */
 955static inline gfp_t htlb_alloc_mask(struct hstate *h)
 956{
 957	gfp_t gfp = __GFP_COMP | __GFP_NOWARN;
 958
 959	gfp |= hugepage_movable_supported(h) ? GFP_HIGHUSER_MOVABLE : GFP_HIGHUSER;
 960
 961	return gfp;
 962}
 963
 964static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
 965{
 966	gfp_t modified_mask = htlb_alloc_mask(h);
 967
 968	/* Some callers might want to enforce node */
 969	modified_mask |= (gfp_mask & __GFP_THISNODE);
 970
 971	modified_mask |= (gfp_mask & __GFP_NOWARN);
 972
 973	return modified_mask;
 974}
 975
 976static inline bool htlb_allow_alloc_fallback(int reason)
 977{
 978	bool allowed_fallback = false;
 979
 980	/*
 981	 * Note: the memory offline, memory failure and migration syscalls will
 982	 * be allowed to fallback to other nodes due to lack of a better chioce,
 983	 * that might break the per-node hugetlb pool. While other cases will
 984	 * set the __GFP_THISNODE to avoid breaking the per-node hugetlb pool.
 985	 */
 986	switch (reason) {
 987	case MR_MEMORY_HOTPLUG:
 988	case MR_MEMORY_FAILURE:
 989	case MR_SYSCALL:
 990	case MR_MEMPOLICY_MBIND:
 991		allowed_fallback = true;
 992		break;
 993	default:
 994		break;
 995	}
 996
 997	return allowed_fallback;
 998}
 999
1000static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
1001					   struct mm_struct *mm, pte_t *pte)
1002{
1003	const unsigned long size = huge_page_size(h);
1004
1005	VM_WARN_ON(size == PAGE_SIZE);
1006
1007	/*
1008	 * hugetlb must use the exact same PT locks as core-mm page table
1009	 * walkers would. When modifying a PTE table, hugetlb must take the
1010	 * PTE PT lock, when modifying a PMD table, hugetlb must take the PMD
1011	 * PT lock etc.
1012	 *
1013	 * The expectation is that any hugetlb folio smaller than a PMD is
1014	 * always mapped into a single PTE table and that any hugetlb folio
1015	 * smaller than a PUD (but at least as big as a PMD) is always mapped
1016	 * into a single PMD table.
1017	 *
1018	 * If that does not hold for an architecture, then that architecture
1019	 * must disable split PT locks such that all *_lockptr() functions
1020	 * will give us the same result: the per-MM PT lock.
1021	 *
1022	 * Note that with e.g., CONFIG_PGTABLE_LEVELS=2 where
1023	 * PGDIR_SIZE==P4D_SIZE==PUD_SIZE==PMD_SIZE, we'd use pud_lockptr()
1024	 * and core-mm would use pmd_lockptr(). However, in such configurations
1025	 * split PMD locks are disabled -- they don't make sense on a single
1026	 * PGDIR page table -- and the end result is the same.
1027	 */
1028	if (size >= PUD_SIZE)
1029		return pud_lockptr(mm, (pud_t *) pte);
1030	else if (size >= PMD_SIZE || IS_ENABLED(CONFIG_HIGHPTE))
1031		return pmd_lockptr(mm, (pmd_t *) pte);
1032	/* pte_alloc_huge() only applies with !CONFIG_HIGHPTE */
1033	return ptep_lockptr(mm, pte);
1034}
1035
1036#ifndef hugepages_supported
1037/*
1038 * Some platform decide whether they support huge pages at boot
1039 * time. Some of them, such as powerpc, set HPAGE_SHIFT to 0
1040 * when there is no such support
1041 */
1042#define hugepages_supported() (HPAGE_SHIFT != 0)
1043#endif
1044
1045void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm);
1046
1047static inline void hugetlb_count_init(struct mm_struct *mm)
1048{
1049	atomic_long_set(&mm->hugetlb_usage, 0);
1050}
1051
1052static inline void hugetlb_count_add(long l, struct mm_struct *mm)
1053{
1054	atomic_long_add(l, &mm->hugetlb_usage);
1055}
1056
1057static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
1058{
1059	atomic_long_sub(l, &mm->hugetlb_usage);
1060}
1061
1062#ifndef huge_ptep_modify_prot_start
1063#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
1064static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
1065						unsigned long addr, pte_t *ptep)
1066{
1067	unsigned long psize = huge_page_size(hstate_vma(vma));
1068
1069	return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, psize);
1070}
1071#endif
1072
1073#ifndef huge_ptep_modify_prot_commit
1074#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
1075static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
1076						unsigned long addr, pte_t *ptep,
1077						pte_t old_pte, pte_t pte)
1078{
1079	unsigned long psize = huge_page_size(hstate_vma(vma));
1080
1081	set_huge_pte_at(vma->vm_mm, addr, ptep, pte, psize);
1082}
1083#endif
1084
1085#ifdef CONFIG_NUMA
1086void hugetlb_register_node(struct node *node);
1087void hugetlb_unregister_node(struct node *node);
1088#endif
1089
1090/*
1091 * Check if a given raw @page in a hugepage is HWPOISON.
1092 */
1093bool is_raw_hwpoison_page_in_hugepage(struct page *page);
1094
1095static inline unsigned long huge_page_mask_align(struct file *file)
1096{
1097	return PAGE_MASK & ~huge_page_mask(hstate_file(file));
1098}
1099
1100#else	/* CONFIG_HUGETLB_PAGE */
1101struct hstate {};
1102
1103static inline unsigned long huge_page_mask_align(struct file *file)
1104{
1105	return 0;
1106}
1107
1108static inline struct hugepage_subpool *hugetlb_folio_subpool(struct folio *folio)
1109{
1110	return NULL;
1111}
1112
1113static inline struct folio *filemap_lock_hugetlb_folio(struct hstate *h,
1114				struct address_space *mapping, pgoff_t idx)
1115{
1116	return NULL;
1117}
1118
1119static inline int isolate_or_dissolve_huge_folio(struct folio *folio,
1120						struct list_head *list)
1121{
1122	return -ENOMEM;
1123}
1124
1125static inline int replace_free_hugepage_folios(unsigned long start_pfn,
1126		unsigned long end_pfn)
1127{
1128	return 0;
1129}
1130
1131static inline void wait_for_freed_hugetlb_folios(void)
1132{
1133}
1134
1135static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
1136					   unsigned long addr,
1137					   bool cow_from_owner)
1138{
1139	return NULL;
1140}
1141
1142static inline struct folio *
1143alloc_hugetlb_folio_reserve(struct hstate *h, int preferred_nid,
1144			    nodemask_t *nmask, gfp_t gfp_mask)
1145{
1146	return NULL;
1147}
1148
1149static inline struct folio *
1150alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid,
1151			nodemask_t *nmask, gfp_t gfp_mask,
1152			bool allow_alloc_fallback)
1153{
1154	return NULL;
1155}
1156
1157static inline int __alloc_bootmem_huge_page(struct hstate *h)
1158{
1159	return 0;
1160}
1161
1162static inline struct hstate *hstate_file(struct file *f)
1163{
1164	return NULL;
1165}
1166
1167static inline struct hstate *hstate_sizelog(int page_size_log)
1168{
1169	return NULL;
1170}
1171
1172static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
1173{
1174	return NULL;
1175}
1176
1177static inline struct hstate *folio_hstate(struct folio *folio)
1178{
1179	return NULL;
1180}
1181
1182static inline struct hstate *size_to_hstate(unsigned long size)
1183{
1184	return NULL;
1185}
1186
1187static inline unsigned long huge_page_size(struct hstate *h)
1188{
1189	return PAGE_SIZE;
1190}
1191
1192static inline unsigned long huge_page_mask(struct hstate *h)
1193{
1194	return PAGE_MASK;
1195}
1196
1197static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
1198{
1199	return PAGE_SIZE;
1200}
1201
1202static inline unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
1203{
1204	return PAGE_SIZE;
1205}
1206
1207static inline unsigned int huge_page_order(struct hstate *h)
1208{
1209	return 0;
1210}
1211
1212static inline unsigned int huge_page_shift(struct hstate *h)
1213{
1214	return PAGE_SHIFT;
1215}
1216
1217static inline bool hstate_is_gigantic(struct hstate *h)
1218{
1219	return false;
1220}
1221
1222static inline unsigned int pages_per_huge_page(struct hstate *h)
1223{
1224	return 1;
1225}
1226
1227static inline unsigned hstate_index_to_shift(unsigned index)
1228{
1229	return 0;
1230}
1231
1232static inline int hstate_index(struct hstate *h)
1233{
1234	return 0;
1235}
1236
1237static inline int dissolve_free_hugetlb_folio(struct folio *folio)
1238{
1239	return 0;
1240}
1241
1242static inline int dissolve_free_hugetlb_folios(unsigned long start_pfn,
1243					   unsigned long end_pfn)
1244{
1245	return 0;
1246}
1247
1248static inline bool hugepage_migration_supported(struct hstate *h)
1249{
1250	return false;
1251}
1252
1253static inline bool hugepage_movable_supported(struct hstate *h)
1254{
1255	return false;
1256}
1257
1258static inline gfp_t htlb_alloc_mask(struct hstate *h)
1259{
1260	return 0;
1261}
1262
1263static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask)
1264{
1265	return 0;
1266}
1267
1268static inline bool htlb_allow_alloc_fallback(int reason)
1269{
1270	return false;
1271}
1272
1273static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
1274					   struct mm_struct *mm, pte_t *pte)
1275{
1276	return &mm->page_table_lock;
1277}
1278
1279static inline void hugetlb_count_init(struct mm_struct *mm)
1280{
1281}
1282
1283static inline void hugetlb_report_usage(struct seq_file *f, struct mm_struct *m)
1284{
1285}
1286
1287static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
1288{
1289}
1290
1291static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
1292					  unsigned long addr, pte_t *ptep)
1293{
1294#ifdef CONFIG_MMU
1295	return ptep_get(ptep);
1296#else
1297	return *ptep;
1298#endif
1299}
1300
1301static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
1302				   pte_t *ptep, pte_t pte, unsigned long sz)
1303{
1304}
1305
1306static inline void hugetlb_register_node(struct node *node)
1307{
1308}
1309
1310static inline void hugetlb_unregister_node(struct node *node)
1311{
1312}
1313
1314static inline bool hugetlbfs_pagecache_present(
1315    struct hstate *h, struct vm_area_struct *vma, unsigned long address)
1316{
1317	return false;
1318}
1319
1320static inline void hugetlb_bootmem_alloc(void)
1321{
1322}
1323
1324static inline bool hugetlb_bootmem_allocated(void)
1325{
1326	return false;
1327}
1328#endif	/* CONFIG_HUGETLB_PAGE */
1329
1330static inline spinlock_t *huge_pte_lock(struct hstate *h,
1331					struct mm_struct *mm, pte_t *pte)
1332{
1333	spinlock_t *ptl;
1334
1335	ptl = huge_pte_lockptr(h, mm, pte);
1336	spin_lock(ptl);
1337	return ptl;
1338}
1339
1340#if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_CMA)
1341extern void __init hugetlb_cma_reserve(int order);
1342#else
1343static inline __init void hugetlb_cma_reserve(int order)
1344{
1345}
1346#endif
1347
1348#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
1349static inline bool hugetlb_pmd_shared(pte_t *pte)
1350{
1351	return ptdesc_pmd_is_shared(virt_to_ptdesc(pte));
1352}
1353#else
1354static inline bool hugetlb_pmd_shared(pte_t *pte)
1355{
1356	return false;
1357}
1358#endif
1359
1360bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr);
1361
1362#ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
1363/*
1364 * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
1365 * implement this.
1366 */
1367#define flush_hugetlb_tlb_range(vma, addr, end)	flush_tlb_range(vma, addr, end)
1368#endif
1369
1370static inline bool __vma_shareable_lock(struct vm_area_struct *vma)
1371{
1372	return (vma->vm_flags & VM_MAYSHARE) && vma->vm_private_data;
1373}
1374
1375bool __vma_private_lock(struct vm_area_struct *vma);
1376
1377/*
1378 * Safe version of huge_pte_offset() to check the locks.  See comments
1379 * above huge_pte_offset().
1380 */
1381static inline pte_t *
1382hugetlb_walk(struct vm_area_struct *vma, unsigned long addr, unsigned long sz)
1383{
1384#if defined(CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING) && defined(CONFIG_LOCKDEP)
1385	struct hugetlb_vma_lock *vma_lock = vma->vm_private_data;
1386
1387	/*
1388	 * If pmd sharing possible, locking needed to safely walk the
1389	 * hugetlb pgtables.  More information can be found at the comment
1390	 * above huge_pte_offset() in the same file.
1391	 *
1392	 * NOTE: lockdep_is_held() is only defined with CONFIG_LOCKDEP.
1393	 */
1394	if (__vma_shareable_lock(vma))
1395		WARN_ON_ONCE(!lockdep_is_held(&vma_lock->rw_sema) &&
1396			     !lockdep_is_held(
1397				 &vma->vm_file->f_mapping->i_mmap_rwsem));
1398#endif
1399	return huge_pte_offset(vma->vm_mm, addr, sz);
1400}
1401
1402#endif /* _LINUX_HUGETLB_H */