pgtable.h

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Copyright (C) 2012 ARM Ltd.
   4 */
   5#ifndef __ASM_PGTABLE_H
   6#define __ASM_PGTABLE_H
   7
   8#include <asm/bug.h>
   9#include <asm/proc-fns.h>
  10
  11#include <asm/memory.h>
  12#include <asm/mte.h>
  13#include <asm/pgtable-hwdef.h>
  14#include <asm/pgtable-prot.h>
  15#include <asm/tlbflush.h>
  16
  17/*
  18 * VMALLOC range.
  19 *
  20 * VMALLOC_START: beginning of the kernel vmalloc space
  21 * VMALLOC_END: extends to the available space below vmemmap
  22 */
  23#define VMALLOC_START		(MODULES_END)
  24#if VA_BITS == VA_BITS_MIN
  25#define VMALLOC_END		(VMEMMAP_START - SZ_8M)
  26#else
  27#define VMEMMAP_UNUSED_NPAGES	((_PAGE_OFFSET(vabits_actual) - PAGE_OFFSET) >> PAGE_SHIFT)
  28#define VMALLOC_END		(VMEMMAP_START + VMEMMAP_UNUSED_NPAGES * sizeof(struct page) - SZ_8M)
  29#endif
  30
  31#define vmemmap			((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
  32
  33#ifndef __ASSEMBLER__
  34
  35#include <asm/cmpxchg.h>
  36#include <asm/fixmap.h>
  37#include <asm/por.h>
  38#include <linux/mmdebug.h>
  39#include <linux/mm_types.h>
  40#include <linux/sched.h>
  41#include <linux/page_table_check.h>
  42
  43static inline void emit_pte_barriers(void)
  44{
  45	/*
  46	 * These barriers are emitted under certain conditions after a pte entry
  47	 * was modified (see e.g. __set_pte_complete()). The dsb makes the store
  48	 * visible to the table walker. The isb ensures that any previous
  49	 * speculative "invalid translation" marker that is in the CPU's
  50	 * pipeline gets cleared, so that any access to that address after
  51	 * setting the pte to valid won't cause a spurious fault. If the thread
  52	 * gets preempted after storing to the pgtable but before emitting these
  53	 * barriers, __switch_to() emits a dsb which ensure the walker gets to
  54	 * see the store. There is no guarantee of an isb being issued though.
  55	 * This is safe because it will still get issued (albeit on a
  56	 * potentially different CPU) when the thread starts running again,
  57	 * before any access to the address.
  58	 */
  59	dsb(ishst);
  60	isb();
  61}
  62
  63static inline void queue_pte_barriers(void)
  64{
  65	unsigned long flags;
  66
  67	if (in_interrupt()) {
  68		emit_pte_barriers();
  69		return;
  70	}
  71
  72	flags = read_thread_flags();
  73
  74	if (flags & BIT(TIF_LAZY_MMU)) {
  75		/* Avoid the atomic op if already set. */
  76		if (!(flags & BIT(TIF_LAZY_MMU_PENDING)))
  77			set_thread_flag(TIF_LAZY_MMU_PENDING);
  78	} else {
  79		emit_pte_barriers();
  80	}
  81}
  82
  83#define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
  84static inline void arch_enter_lazy_mmu_mode(void)
  85{
  86	/*
  87	 * lazy_mmu_mode is not supposed to permit nesting. But in practice this
  88	 * does happen with CONFIG_DEBUG_PAGEALLOC, where a page allocation
  89	 * inside a lazy_mmu_mode section (such as zap_pte_range()) will change
  90	 * permissions on the linear map with apply_to_page_range(), which
  91	 * re-enters lazy_mmu_mode. So we tolerate nesting in our
  92	 * implementation. The first call to arch_leave_lazy_mmu_mode() will
  93	 * flush and clear the flag such that the remainder of the work in the
  94	 * outer nest behaves as if outside of lazy mmu mode. This is safe and
  95	 * keeps tracking simple.
  96	 */
  97
  98	if (in_interrupt())
  99		return;
 100
 101	set_thread_flag(TIF_LAZY_MMU);
 102}
 103
 104static inline void arch_flush_lazy_mmu_mode(void)
 105{
 106	if (in_interrupt())
 107		return;
 108
 109	if (test_and_clear_thread_flag(TIF_LAZY_MMU_PENDING))
 110		emit_pte_barriers();
 111}
 112
 113static inline void arch_leave_lazy_mmu_mode(void)
 114{
 115	if (in_interrupt())
 116		return;
 117
 118	arch_flush_lazy_mmu_mode();
 119	clear_thread_flag(TIF_LAZY_MMU);
 120}
 121
 122#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 123#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
 124
 125/* Set stride and tlb_level in flush_*_tlb_range */
 126#define flush_pmd_tlb_range(vma, addr, end)	\
 127	__flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2)
 128#define flush_pud_tlb_range(vma, addr, end)	\
 129	__flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)
 130#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 131
 132/*
 133 * We use local TLB invalidation instruction when reusing page in
 134 * write protection fault handler to avoid TLBI broadcast in the hot
 135 * path.  This will cause spurious page faults if stale read-only TLB
 136 * entries exist.
 137 */
 138#define flush_tlb_fix_spurious_fault(vma, address, ptep)	\
 139	local_flush_tlb_page_nonotify(vma, address)
 140
 141#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp)	\
 142	local_flush_tlb_page_nonotify(vma, address)
 143
 144/*
 145 * ZERO_PAGE is a global shared page that is always zero: used
 146 * for zero-mapped memory areas etc..
 147 */
 148extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
 149#define ZERO_PAGE(vaddr)	phys_to_page(__pa_symbol(empty_zero_page))
 150
 151#define pte_ERROR(e)	\
 152	pr_err("%s:%d: bad pte %016llx.\n", __FILE__, __LINE__, pte_val(e))
 153
 154#ifdef CONFIG_ARM64_PA_BITS_52
 155static inline phys_addr_t __pte_to_phys(pte_t pte)
 156{
 157	pte_val(pte) &= ~PTE_MAYBE_SHARED;
 158	return (pte_val(pte) & PTE_ADDR_LOW) |
 159		((pte_val(pte) & PTE_ADDR_HIGH) << PTE_ADDR_HIGH_SHIFT);
 160}
 161static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
 162{
 163	return (phys | (phys >> PTE_ADDR_HIGH_SHIFT)) & PHYS_TO_PTE_ADDR_MASK;
 164}
 165#else
 166static inline phys_addr_t __pte_to_phys(pte_t pte)
 167{
 168	return pte_val(pte) & PTE_ADDR_LOW;
 169}
 170
 171static inline pteval_t __phys_to_pte_val(phys_addr_t phys)
 172{
 173	return phys;
 174}
 175#endif
 176
 177#define pte_pfn(pte)		(__pte_to_phys(pte) >> PAGE_SHIFT)
 178#define pfn_pte(pfn,prot)	\
 179	__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 180
 181#define pte_none(pte)		(!pte_val(pte))
 182#define pte_page(pte)		(pfn_to_page(pte_pfn(pte)))
 183
 184/*
 185 * The following only work if pte_present(). Undefined behaviour otherwise.
 186 */
 187#define pte_present(pte)	(pte_valid(pte) || pte_present_invalid(pte))
 188#define pte_young(pte)		(!!(pte_val(pte) & PTE_AF))
 189#define pte_special(pte)	(!!(pte_val(pte) & PTE_SPECIAL))
 190#define pte_write(pte)		(!!(pte_val(pte) & PTE_WRITE))
 191#define pte_rdonly(pte)		(!!(pte_val(pte) & PTE_RDONLY))
 192#define pte_user(pte)		(!!(pte_val(pte) & PTE_USER))
 193#define pte_user_exec(pte)	(!(pte_val(pte) & PTE_UXN))
 194#define pte_cont(pte)		(!!(pte_val(pte) & PTE_CONT))
 195#define pte_tagged(pte)		((pte_val(pte) & PTE_ATTRINDX_MASK) == \
 196				 PTE_ATTRINDX(MT_NORMAL_TAGGED))
 197
 198#define pte_cont_addr_end(addr, end)						\
 199({	unsigned long __boundary = ((addr) + CONT_PTE_SIZE) & CONT_PTE_MASK;	\
 200	(__boundary - 1 < (end) - 1) ? __boundary : (end);			\
 201})
 202
 203#define pmd_cont_addr_end(addr, end)						\
 204({	unsigned long __boundary = ((addr) + CONT_PMD_SIZE) & CONT_PMD_MASK;	\
 205	(__boundary - 1 < (end) - 1) ? __boundary : (end);			\
 206})
 207
 208#define pte_hw_dirty(pte)	(pte_write(pte) && !pte_rdonly(pte))
 209#define pte_sw_dirty(pte)	(!!(pte_val(pte) & PTE_DIRTY))
 210#define pte_dirty(pte)		(pte_sw_dirty(pte) || pte_hw_dirty(pte))
 211
 212#define pte_valid(pte)		(!!(pte_val(pte) & PTE_VALID))
 213#define pte_present_invalid(pte) \
 214	((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID)
 215/*
 216 * Execute-only user mappings do not have the PTE_USER bit set. All valid
 217 * kernel mappings have the PTE_UXN bit set.
 218 */
 219#define pte_valid_not_user(pte) \
 220	((pte_val(pte) & (PTE_VALID | PTE_USER | PTE_UXN)) == (PTE_VALID | PTE_UXN))
 221/*
 222 * Returns true if the pte is valid and has the contiguous bit set.
 223 */
 224#define pte_valid_cont(pte)	(pte_valid(pte) && pte_cont(pte))
 225/*
 226 * Could the pte be present in the TLB? We must check mm_tlb_flush_pending
 227 * so that we don't erroneously return false for pages that have been
 228 * remapped as PROT_NONE but are yet to be flushed from the TLB.
 229 * Note that we can't make any assumptions based on the state of the access
 230 * flag, since __ptep_clear_flush_young() elides a DSB when invalidating the
 231 * TLB.
 232 */
 233#define pte_accessible(mm, pte)	\
 234	(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))
 235
 236static inline bool por_el0_allows_pkey(u8 pkey, bool write, bool execute)
 237{
 238	u64 por;
 239
 240	if (!system_supports_poe())
 241		return true;
 242
 243	por = read_sysreg_s(SYS_POR_EL0);
 244
 245	if (write)
 246		return por_elx_allows_write(por, pkey);
 247
 248	if (execute)
 249		return por_elx_allows_exec(por, pkey);
 250
 251	return por_elx_allows_read(por, pkey);
 252}
 253
 254/*
 255 * p??_access_permitted() is true for valid user mappings (PTE_USER
 256 * bit set, subject to the write permission check). For execute-only
 257 * mappings, like PROT_EXEC with EPAN (both PTE_USER and PTE_UXN bits
 258 * not set) must return false. PROT_NONE mappings do not have the
 259 * PTE_VALID bit set.
 260 */
 261#define pte_access_permitted_no_overlay(pte, write) \
 262	(((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) && (!(write) || pte_write(pte)))
 263#define pte_access_permitted(pte, write) \
 264	(pte_access_permitted_no_overlay(pte, write) && \
 265	por_el0_allows_pkey(FIELD_GET(PTE_PO_IDX_MASK, pte_val(pte)), write, false))
 266#define pmd_access_permitted(pmd, write) \
 267	(pte_access_permitted(pmd_pte(pmd), (write)))
 268#define pud_access_permitted(pud, write) \
 269	(pte_access_permitted(pud_pte(pud), (write)))
 270
 271static inline pte_t clear_pte_bit(pte_t pte, pgprot_t prot)
 272{
 273	pte_val(pte) &= ~pgprot_val(prot);
 274	return pte;
 275}
 276
 277static inline pte_t set_pte_bit(pte_t pte, pgprot_t prot)
 278{
 279	pte_val(pte) |= pgprot_val(prot);
 280	return pte;
 281}
 282
 283static inline pmd_t clear_pmd_bit(pmd_t pmd, pgprot_t prot)
 284{
 285	pmd_val(pmd) &= ~pgprot_val(prot);
 286	return pmd;
 287}
 288
 289static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot)
 290{
 291	pmd_val(pmd) |= pgprot_val(prot);
 292	return pmd;
 293}
 294
 295static inline pte_t pte_mkwrite_novma(pte_t pte)
 296{
 297	pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
 298	if (pte_sw_dirty(pte))
 299		pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
 300	return pte;
 301}
 302
 303static inline pte_t pte_mkclean(pte_t pte)
 304{
 305	pte = clear_pte_bit(pte, __pgprot(PTE_DIRTY));
 306	pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
 307
 308	return pte;
 309}
 310
 311static inline pte_t pte_mkdirty(pte_t pte)
 312{
 313	pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
 314
 315	if (pte_write(pte))
 316		pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
 317
 318	return pte;
 319}
 320
 321static inline pte_t pte_wrprotect(pte_t pte)
 322{
 323	/*
 324	 * If hardware-dirty (PTE_WRITE/DBM bit set and PTE_RDONLY
 325	 * clear), set the PTE_DIRTY bit.
 326	 */
 327	if (pte_hw_dirty(pte))
 328		pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
 329
 330	pte = clear_pte_bit(pte, __pgprot(PTE_WRITE));
 331	pte = set_pte_bit(pte, __pgprot(PTE_RDONLY));
 332	return pte;
 333}
 334
 335static inline pte_t pte_mkold(pte_t pte)
 336{
 337	return clear_pte_bit(pte, __pgprot(PTE_AF));
 338}
 339
 340static inline pte_t pte_mkyoung(pte_t pte)
 341{
 342	return set_pte_bit(pte, __pgprot(PTE_AF));
 343}
 344
 345static inline pte_t pte_mkspecial(pte_t pte)
 346{
 347	return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
 348}
 349
 350static inline pte_t pte_mkcont(pte_t pte)
 351{
 352	return set_pte_bit(pte, __pgprot(PTE_CONT));
 353}
 354
 355static inline pte_t pte_mknoncont(pte_t pte)
 356{
 357	return clear_pte_bit(pte, __pgprot(PTE_CONT));
 358}
 359
 360static inline pte_t pte_mkvalid_k(pte_t pte)
 361{
 362	pte = clear_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID));
 363	pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_VALID_KERNEL));
 364	return pte;
 365}
 366
 367static inline pte_t pte_mkinvalid(pte_t pte)
 368{
 369	pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID));
 370	pte = clear_pte_bit(pte, __pgprot(PTE_VALID));
 371	return pte;
 372}
 373
 374static inline pmd_t pmd_mkcont(pmd_t pmd)
 375{
 376	return __pmd(pmd_val(pmd) | PMD_SECT_CONT);
 377}
 378
 379static inline pmd_t pmd_mknoncont(pmd_t pmd)
 380{
 381	return __pmd(pmd_val(pmd) & ~PMD_SECT_CONT);
 382}
 383
 384#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
 385static inline int pte_uffd_wp(pte_t pte)
 386{
 387	return !!(pte_val(pte) & PTE_UFFD_WP);
 388}
 389
 390static inline pte_t pte_mkuffd_wp(pte_t pte)
 391{
 392	return pte_wrprotect(set_pte_bit(pte, __pgprot(PTE_UFFD_WP)));
 393}
 394
 395static inline pte_t pte_clear_uffd_wp(pte_t pte)
 396{
 397	return clear_pte_bit(pte, __pgprot(PTE_UFFD_WP));
 398}
 399#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 400
 401static inline void __set_pte_nosync(pte_t *ptep, pte_t pte)
 402{
 403	WRITE_ONCE(*ptep, pte);
 404}
 405
 406static inline void __set_pte_complete(pte_t pte)
 407{
 408	/*
 409	 * Only if the new pte is valid and kernel, otherwise TLB maintenance
 410	 * has the necessary barriers.
 411	 */
 412	if (pte_valid_not_user(pte))
 413		queue_pte_barriers();
 414}
 415
 416static inline void __set_pte(pte_t *ptep, pte_t pte)
 417{
 418	__set_pte_nosync(ptep, pte);
 419	__set_pte_complete(pte);
 420}
 421
 422static inline pte_t __ptep_get(pte_t *ptep)
 423{
 424	return READ_ONCE(*ptep);
 425}
 426
 427extern void __sync_icache_dcache(pte_t pteval);
 428bool pgattr_change_is_safe(pteval_t old, pteval_t new);
 429
 430/*
 431 * PTE bits configuration in the presence of hardware Dirty Bit Management
 432 * (PTE_WRITE == PTE_DBM):
 433 *
 434 * Dirty  Writable | PTE_RDONLY  PTE_WRITE  PTE_DIRTY (sw)
 435 *   0      0      |   1           0          0
 436 *   0      1      |   1           1          0
 437 *   1      0      |   1           0          1
 438 *   1      1      |   0           1          x
 439 *
 440 * When hardware DBM is not present, the software PTE_DIRTY bit is updated via
 441 * the page fault mechanism. Checking the dirty status of a pte becomes:
 442 *
 443 *   PTE_DIRTY || (PTE_WRITE && !PTE_RDONLY)
 444 */
 445
 446static inline void __check_safe_pte_update(struct mm_struct *mm, pte_t *ptep,
 447					   pte_t pte)
 448{
 449	pte_t old_pte;
 450
 451	if (!IS_ENABLED(CONFIG_DEBUG_VM))
 452		return;
 453
 454	old_pte = __ptep_get(ptep);
 455
 456	if (!pte_valid(old_pte) || !pte_valid(pte))
 457		return;
 458	if (mm != current->active_mm && atomic_read(&mm->mm_users) <= 1)
 459		return;
 460
 461	/*
 462	 * Check for potential race with hardware updates of the pte
 463	 * (__ptep_set_access_flags safely changes valid ptes without going
 464	 * through an invalid entry).
 465	 */
 466	VM_WARN_ONCE(!pte_young(pte),
 467		     "%s: racy access flag clearing: 0x%016llx -> 0x%016llx",
 468		     __func__, pte_val(old_pte), pte_val(pte));
 469	VM_WARN_ONCE(pte_write(old_pte) && !pte_dirty(pte),
 470		     "%s: racy dirty state clearing: 0x%016llx -> 0x%016llx",
 471		     __func__, pte_val(old_pte), pte_val(pte));
 472	VM_WARN_ONCE(!pgattr_change_is_safe(pte_val(old_pte), pte_val(pte)),
 473		     "%s: unsafe attribute change: 0x%016llx -> 0x%016llx",
 474		     __func__, pte_val(old_pte), pte_val(pte));
 475}
 476
 477static inline void __sync_cache_and_tags(pte_t pte, unsigned int nr_pages)
 478{
 479	if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte))
 480		__sync_icache_dcache(pte);
 481
 482	/*
 483	 * If the PTE would provide user space access to the tags associated
 484	 * with it then ensure that the MTE tags are synchronised.  Although
 485	 * pte_access_permitted_no_overlay() returns false for exec only
 486	 * mappings, they don't expose tags (instruction fetches don't check
 487	 * tags).
 488	 */
 489	if (system_supports_mte() && pte_access_permitted_no_overlay(pte, false) &&
 490	    !pte_special(pte) && pte_tagged(pte))
 491		mte_sync_tags(pte, nr_pages);
 492}
 493
 494/*
 495 * Select all bits except the pfn
 496 */
 497#define pte_pgprot pte_pgprot
 498static inline pgprot_t pte_pgprot(pte_t pte)
 499{
 500	unsigned long pfn = pte_pfn(pte);
 501
 502	return __pgprot(pte_val(pfn_pte(pfn, __pgprot(0))) ^ pte_val(pte));
 503}
 504
 505#define pte_advance_pfn pte_advance_pfn
 506static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr)
 507{
 508	return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte));
 509}
 510
 511/*
 512 * Hugetlb definitions.
 513 */
 514#define HUGE_MAX_HSTATE		4
 515#define HPAGE_SHIFT		PMD_SHIFT
 516#define HPAGE_SIZE		(_AC(1, UL) << HPAGE_SHIFT)
 517#define HPAGE_MASK		(~(HPAGE_SIZE - 1))
 518#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
 519
 520static inline pte_t pgd_pte(pgd_t pgd)
 521{
 522	return __pte(pgd_val(pgd));
 523}
 524
 525static inline pte_t p4d_pte(p4d_t p4d)
 526{
 527	return __pte(p4d_val(p4d));
 528}
 529
 530static inline pte_t pud_pte(pud_t pud)
 531{
 532	return __pte(pud_val(pud));
 533}
 534
 535static inline pud_t pte_pud(pte_t pte)
 536{
 537	return __pud(pte_val(pte));
 538}
 539
 540static inline pmd_t pud_pmd(pud_t pud)
 541{
 542	return __pmd(pud_val(pud));
 543}
 544
 545static inline pte_t pmd_pte(pmd_t pmd)
 546{
 547	return __pte(pmd_val(pmd));
 548}
 549
 550static inline pmd_t pte_pmd(pte_t pte)
 551{
 552	return __pmd(pte_val(pte));
 553}
 554
 555static inline pgprot_t mk_pud_sect_prot(pgprot_t prot)
 556{
 557	return __pgprot((pgprot_val(prot) & ~PUD_TYPE_MASK) | PUD_TYPE_SECT);
 558}
 559
 560static inline pgprot_t mk_pmd_sect_prot(pgprot_t prot)
 561{
 562	return __pgprot((pgprot_val(prot) & ~PMD_TYPE_MASK) | PMD_TYPE_SECT);
 563}
 564
 565static inline pte_t pte_swp_mkexclusive(pte_t pte)
 566{
 567	return set_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));
 568}
 569
 570static inline bool pte_swp_exclusive(pte_t pte)
 571{
 572	return pte_val(pte) & PTE_SWP_EXCLUSIVE;
 573}
 574
 575static inline pte_t pte_swp_clear_exclusive(pte_t pte)
 576{
 577	return clear_pte_bit(pte, __pgprot(PTE_SWP_EXCLUSIVE));
 578}
 579
 580#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
 581static inline pte_t pte_swp_mkuffd_wp(pte_t pte)
 582{
 583	return set_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP));
 584}
 585
 586static inline int pte_swp_uffd_wp(pte_t pte)
 587{
 588	return !!(pte_val(pte) & PTE_SWP_UFFD_WP);
 589}
 590
 591static inline pte_t pte_swp_clear_uffd_wp(pte_t pte)
 592{
 593	return clear_pte_bit(pte, __pgprot(PTE_SWP_UFFD_WP));
 594}
 595#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 596
 597#ifdef CONFIG_NUMA_BALANCING
 598/*
 599 * See the comment in include/linux/pgtable.h
 600 */
 601static inline int pte_protnone(pte_t pte)
 602{
 603	/*
 604	 * pte_present_invalid() tells us that the pte is invalid from HW
 605	 * perspective but present from SW perspective, so the fields are to be
 606	 * interpreted as per the HW layout. The second 2 checks are the unique
 607	 * encoding that we use for PROT_NONE. It is insufficient to only use
 608	 * the first check because we share the same encoding scheme with pmds
 609	 * which support pmd_mkinvalid(), so can be present-invalid without
 610	 * being PROT_NONE.
 611	 */
 612	return pte_present_invalid(pte) && !pte_user(pte) && !pte_user_exec(pte);
 613}
 614
 615static inline int pmd_protnone(pmd_t pmd)
 616{
 617	return pte_protnone(pmd_pte(pmd));
 618}
 619#endif
 620
 621#define pmd_present(pmd)	pte_present(pmd_pte(pmd))
 622#define pmd_dirty(pmd)		pte_dirty(pmd_pte(pmd))
 623#define pmd_young(pmd)		pte_young(pmd_pte(pmd))
 624#define pmd_valid(pmd)		pte_valid(pmd_pte(pmd))
 625#define pmd_user(pmd)		pte_user(pmd_pte(pmd))
 626#define pmd_user_exec(pmd)	pte_user_exec(pmd_pte(pmd))
 627#define pmd_cont(pmd)		pte_cont(pmd_pte(pmd))
 628#define pmd_wrprotect(pmd)	pte_pmd(pte_wrprotect(pmd_pte(pmd)))
 629#define pmd_mkold(pmd)		pte_pmd(pte_mkold(pmd_pte(pmd)))
 630#define pmd_mkwrite_novma(pmd)	pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
 631#define pmd_mkclean(pmd)	pte_pmd(pte_mkclean(pmd_pte(pmd)))
 632#define pmd_mkdirty(pmd)	pte_pmd(pte_mkdirty(pmd_pte(pmd)))
 633#define pmd_mkyoung(pmd)	pte_pmd(pte_mkyoung(pmd_pte(pmd)))
 634#define pmd_mkvalid_k(pmd)	pte_pmd(pte_mkvalid_k(pmd_pte(pmd)))
 635#define pmd_mkinvalid(pmd)	pte_pmd(pte_mkinvalid(pmd_pte(pmd)))
 636#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
 637#define pmd_uffd_wp(pmd)	pte_uffd_wp(pmd_pte(pmd))
 638#define pmd_mkuffd_wp(pmd)	pte_pmd(pte_mkuffd_wp(pmd_pte(pmd)))
 639#define pmd_clear_uffd_wp(pmd)	pte_pmd(pte_clear_uffd_wp(pmd_pte(pmd)))
 640#define pmd_swp_uffd_wp(pmd)	pte_swp_uffd_wp(pmd_pte(pmd))
 641#define pmd_swp_mkuffd_wp(pmd)	pte_pmd(pte_swp_mkuffd_wp(pmd_pte(pmd)))
 642#define pmd_swp_clear_uffd_wp(pmd) \
 643				pte_pmd(pte_swp_clear_uffd_wp(pmd_pte(pmd)))
 644#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_WP */
 645
 646#define pmd_write(pmd)		pte_write(pmd_pte(pmd))
 647
 648static inline pmd_t pmd_mkhuge(pmd_t pmd)
 649{
 650	/*
 651	 * It's possible that the pmd is present-invalid on entry
 652	 * and in that case it needs to remain present-invalid on
 653	 * exit. So ensure the VALID bit does not get modified.
 654	 */
 655	pmdval_t mask = PMD_TYPE_MASK & ~PTE_VALID;
 656	pmdval_t val = PMD_TYPE_SECT & ~PTE_VALID;
 657
 658	return __pmd((pmd_val(pmd) & ~mask) | val);
 659}
 660
 661#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
 662#define pmd_special(pte)	(!!((pmd_val(pte) & PTE_SPECIAL)))
 663static inline pmd_t pmd_mkspecial(pmd_t pmd)
 664{
 665	return set_pmd_bit(pmd, __pgprot(PTE_SPECIAL));
 666}
 667#endif
 668
 669#define __pmd_to_phys(pmd)	__pte_to_phys(pmd_pte(pmd))
 670#define __phys_to_pmd_val(phys)	__phys_to_pte_val(phys)
 671#define pmd_pfn(pmd)		((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
 672#define pfn_pmd(pfn,prot)	__pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 673
 674#define pud_young(pud)		pte_young(pud_pte(pud))
 675#define pud_mkyoung(pud)	pte_pud(pte_mkyoung(pud_pte(pud)))
 676#define pud_mkwrite_novma(pud)	pte_pud(pte_mkwrite_novma(pud_pte(pud)))
 677#define pud_mkvalid_k(pud)	pte_pud(pte_mkvalid_k(pud_pte(pud)))
 678#define pud_write(pud)		pte_write(pud_pte(pud))
 679
 680static inline pud_t pud_mkhuge(pud_t pud)
 681{
 682	/*
 683	 * It's possible that the pud is present-invalid on entry
 684	 * and in that case it needs to remain present-invalid on
 685	 * exit. So ensure the VALID bit does not get modified.
 686	 */
 687	pudval_t mask = PUD_TYPE_MASK & ~PTE_VALID;
 688	pudval_t val = PUD_TYPE_SECT & ~PTE_VALID;
 689
 690	return __pud((pud_val(pud) & ~mask) | val);
 691}
 692
 693#define __pud_to_phys(pud)	__pte_to_phys(pud_pte(pud))
 694#define __phys_to_pud_val(phys)	__phys_to_pte_val(phys)
 695#define pud_pfn(pud)		((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
 696#define pfn_pud(pfn,prot)	__pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
 697
 698#define pmd_pgprot pmd_pgprot
 699static inline pgprot_t pmd_pgprot(pmd_t pmd)
 700{
 701	unsigned long pfn = pmd_pfn(pmd);
 702
 703	return __pgprot(pmd_val(pfn_pmd(pfn, __pgprot(0))) ^ pmd_val(pmd));
 704}
 705
 706#define pud_pgprot pud_pgprot
 707static inline pgprot_t pud_pgprot(pud_t pud)
 708{
 709	unsigned long pfn = pud_pfn(pud);
 710
 711	return __pgprot(pud_val(pfn_pud(pfn, __pgprot(0))) ^ pud_val(pud));
 712}
 713
 714static inline void __set_ptes_anysz(struct mm_struct *mm, pte_t *ptep,
 715				    pte_t pte, unsigned int nr,
 716				    unsigned long pgsize)
 717{
 718	unsigned long stride = pgsize >> PAGE_SHIFT;
 719
 720	switch (pgsize) {
 721	case PAGE_SIZE:
 722		page_table_check_ptes_set(mm, ptep, pte, nr);
 723		break;
 724	case PMD_SIZE:
 725		page_table_check_pmds_set(mm, (pmd_t *)ptep, pte_pmd(pte), nr);
 726		break;
 727#ifndef __PAGETABLE_PMD_FOLDED
 728	case PUD_SIZE:
 729		page_table_check_puds_set(mm, (pud_t *)ptep, pte_pud(pte), nr);
 730		break;
 731#endif
 732	default:
 733		VM_WARN_ON(1);
 734	}
 735
 736	__sync_cache_and_tags(pte, nr * stride);
 737
 738	for (;;) {
 739		__check_safe_pte_update(mm, ptep, pte);
 740		__set_pte_nosync(ptep, pte);
 741		if (--nr == 0)
 742			break;
 743		ptep++;
 744		pte = pte_advance_pfn(pte, stride);
 745	}
 746
 747	__set_pte_complete(pte);
 748}
 749
 750static inline void __set_ptes(struct mm_struct *mm,
 751			      unsigned long __always_unused addr,
 752			      pte_t *ptep, pte_t pte, unsigned int nr)
 753{
 754	__set_ptes_anysz(mm, ptep, pte, nr, PAGE_SIZE);
 755}
 756
 757static inline void __set_pmds(struct mm_struct *mm,
 758			      unsigned long __always_unused addr,
 759			      pmd_t *pmdp, pmd_t pmd, unsigned int nr)
 760{
 761	__set_ptes_anysz(mm, (pte_t *)pmdp, pmd_pte(pmd), nr, PMD_SIZE);
 762}
 763#define set_pmd_at(mm, addr, pmdp, pmd) __set_pmds(mm, addr, pmdp, pmd, 1)
 764
 765static inline void __set_puds(struct mm_struct *mm,
 766			      unsigned long __always_unused addr,
 767			      pud_t *pudp, pud_t pud, unsigned int nr)
 768{
 769	__set_ptes_anysz(mm, (pte_t *)pudp, pud_pte(pud), nr, PUD_SIZE);
 770}
 771#define set_pud_at(mm, addr, pudp, pud) __set_puds(mm, addr, pudp, pud, 1)
 772
 773#define __p4d_to_phys(p4d)	__pte_to_phys(p4d_pte(p4d))
 774#define __phys_to_p4d_val(phys)	__phys_to_pte_val(phys)
 775
 776#define __pgd_to_phys(pgd)	__pte_to_phys(pgd_pte(pgd))
 777#define __phys_to_pgd_val(phys)	__phys_to_pte_val(phys)
 778
 779#define __pgprot_modify(prot,mask,bits) \
 780	__pgprot((pgprot_val(prot) & ~(mask)) | (bits))
 781
 782#define pgprot_nx(prot) \
 783	__pgprot_modify(prot, PTE_MAYBE_GP, PTE_PXN)
 784
 785#define pgprot_decrypted(prot) \
 786	__pgprot_modify(prot, PROT_NS_SHARED, PROT_NS_SHARED)
 787#define pgprot_encrypted(prot) \
 788	__pgprot_modify(prot, PROT_NS_SHARED, 0)
 789
 790/*
 791 * Mark the prot value as uncacheable and unbufferable.
 792 */
 793#define pgprot_noncached(prot) \
 794	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRnE) | PTE_PXN | PTE_UXN)
 795#define pgprot_writecombine(prot) \
 796	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
 797#define pgprot_device(prot) \
 798	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_DEVICE_nGnRE) | PTE_PXN | PTE_UXN)
 799#define pgprot_tagged(prot) \
 800	__pgprot_modify(prot, PTE_ATTRINDX_MASK, PTE_ATTRINDX(MT_NORMAL_TAGGED))
 801#define pgprot_mhp	pgprot_tagged
 802/*
 803 * DMA allocations for non-coherent devices use what the Arm architecture calls
 804 * "Normal non-cacheable" memory, which permits speculation, unaligned accesses
 805 * and merging of writes.  This is different from "Device-nGnR[nE]" memory which
 806 * is intended for MMIO and thus forbids speculation, preserves access size,
 807 * requires strict alignment and can also force write responses to come from the
 808 * endpoint.
 809 */
 810#define pgprot_dmacoherent(prot) \
 811	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
 812			PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
 813
 814#define __HAVE_PHYS_MEM_ACCESS_PROT
 815struct file;
 816extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
 817				     unsigned long size, pgprot_t vma_prot);
 818
 819#define pmd_none(pmd)		(!pmd_val(pmd))
 820
 821#define pmd_table(pmd)		((pmd_val(pmd) & PMD_TYPE_MASK) == \
 822				 PMD_TYPE_TABLE)
 823#define pmd_sect(pmd)		((pmd_val(pmd) & PMD_TYPE_MASK) == \
 824				 PMD_TYPE_SECT)
 825#define pmd_leaf(pmd)		(pmd_present(pmd) && !pmd_table(pmd))
 826#define pmd_bad(pmd)		(!pmd_table(pmd))
 827
 828#define pmd_leaf_size(pmd)	(pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
 829#define pte_leaf_size(pte)	(pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE)
 830
 831#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 832static inline int pmd_trans_huge(pmd_t pmd)
 833{
 834	/*
 835	 * If pmd is present-invalid, pmd_table() won't detect it
 836	 * as a table, so force the valid bit for the comparison.
 837	 */
 838	return pmd_present(pmd) && !pmd_table(__pmd(pmd_val(pmd) | PTE_VALID));
 839}
 840#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 841
 842#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
 843static inline bool pud_sect(pud_t pud) { return false; }
 844static inline bool pud_table(pud_t pud) { return true; }
 845#else
 846#define pud_sect(pud)		((pud_val(pud) & PUD_TYPE_MASK) == \
 847				 PUD_TYPE_SECT)
 848#define pud_table(pud)		((pud_val(pud) & PUD_TYPE_MASK) == \
 849				 PUD_TYPE_TABLE)
 850#endif
 851
 852extern pgd_t swapper_pg_dir[];
 853extern pgd_t idmap_pg_dir[];
 854extern pgd_t tramp_pg_dir[];
 855extern pgd_t reserved_pg_dir[];
 856
 857extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
 858
 859static inline bool in_swapper_pgdir(void *addr)
 860{
 861	return ((unsigned long)addr & PAGE_MASK) ==
 862	        ((unsigned long)swapper_pg_dir & PAGE_MASK);
 863}
 864
 865static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 866{
 867#ifdef __PAGETABLE_PMD_FOLDED
 868	if (in_swapper_pgdir(pmdp)) {
 869		set_swapper_pgd((pgd_t *)pmdp, __pgd(pmd_val(pmd)));
 870		return;
 871	}
 872#endif /* __PAGETABLE_PMD_FOLDED */
 873
 874	WRITE_ONCE(*pmdp, pmd);
 875
 876	if (pmd_valid(pmd))
 877		queue_pte_barriers();
 878}
 879
 880static inline void pmd_clear(pmd_t *pmdp)
 881{
 882	set_pmd(pmdp, __pmd(0));
 883}
 884
 885static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
 886{
 887	return __pmd_to_phys(pmd);
 888}
 889
 890static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 891{
 892	return (unsigned long)__va(pmd_page_paddr(pmd));
 893}
 894
 895/* Find an entry in the third-level page table. */
 896#define pte_offset_phys(dir,addr)	(pmd_page_paddr(READ_ONCE(*(dir))) + pte_index(addr) * sizeof(pte_t))
 897
 898#define pte_set_fixmap(addr)		((pte_t *)set_fixmap_offset(FIX_PTE, addr))
 899#define pte_set_fixmap_offset(pmd, addr)	pte_set_fixmap(pte_offset_phys(pmd, addr))
 900#define pte_clear_fixmap()		clear_fixmap(FIX_PTE)
 901
 902#define pmd_page(pmd)			phys_to_page(__pmd_to_phys(pmd))
 903
 904/* use ONLY for statically allocated translation tables */
 905#define pte_offset_kimg(dir,addr)	((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
 906
 907#if CONFIG_PGTABLE_LEVELS > 2
 908
 909#define pmd_ERROR(e)	\
 910	pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
 911
 912#define pud_none(pud)		(!pud_val(pud))
 913#define pud_bad(pud)		((pud_val(pud) & PUD_TYPE_MASK) != \
 914				 PUD_TYPE_TABLE)
 915#define pud_present(pud)	pte_present(pud_pte(pud))
 916#ifndef __PAGETABLE_PMD_FOLDED
 917#define pud_leaf(pud)		(pud_present(pud) && !pud_table(pud))
 918#else
 919#define pud_leaf(pud)		false
 920#endif
 921#define pud_valid(pud)		pte_valid(pud_pte(pud))
 922#define pud_user(pud)		pte_user(pud_pte(pud))
 923#define pud_user_exec(pud)	pte_user_exec(pud_pte(pud))
 924
 925static inline bool pgtable_l4_enabled(void);
 926
 927static inline void set_pud(pud_t *pudp, pud_t pud)
 928{
 929	if (!pgtable_l4_enabled() && in_swapper_pgdir(pudp)) {
 930		set_swapper_pgd((pgd_t *)pudp, __pgd(pud_val(pud)));
 931		return;
 932	}
 933
 934	WRITE_ONCE(*pudp, pud);
 935
 936	if (pud_valid(pud))
 937		queue_pte_barriers();
 938}
 939
 940static inline void pud_clear(pud_t *pudp)
 941{
 942	set_pud(pudp, __pud(0));
 943}
 944
 945static inline phys_addr_t pud_page_paddr(pud_t pud)
 946{
 947	return __pud_to_phys(pud);
 948}
 949
 950static inline pmd_t *pud_pgtable(pud_t pud)
 951{
 952	return (pmd_t *)__va(pud_page_paddr(pud));
 953}
 954
 955/* Find an entry in the second-level page table. */
 956#define pmd_offset_phys(dir, addr)	(pud_page_paddr(READ_ONCE(*(dir))) + pmd_index(addr) * sizeof(pmd_t))
 957
 958#define pmd_set_fixmap(addr)		((pmd_t *)set_fixmap_offset(FIX_PMD, addr))
 959#define pmd_set_fixmap_offset(pud, addr)	pmd_set_fixmap(pmd_offset_phys(pud, addr))
 960#define pmd_clear_fixmap()		clear_fixmap(FIX_PMD)
 961
 962#define pud_page(pud)			phys_to_page(__pud_to_phys(pud))
 963
 964/* use ONLY for statically allocated translation tables */
 965#define pmd_offset_kimg(dir,addr)	((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
 966
 967#else
 968
 969#define pud_valid(pud)		false
 970#define pud_page_paddr(pud)	({ BUILD_BUG(); 0; })
 971#define pud_user_exec(pud)	pud_user(pud) /* Always 0 with folding */
 972
 973/* Match pmd_offset folding in <asm/generic/pgtable-nopmd.h> */
 974#define pmd_set_fixmap(addr)		NULL
 975#define pmd_set_fixmap_offset(pudp, addr)	((pmd_t *)pudp)
 976#define pmd_clear_fixmap()
 977
 978#define pmd_offset_kimg(dir,addr)	((pmd_t *)dir)
 979
 980#endif	/* CONFIG_PGTABLE_LEVELS > 2 */
 981
 982#if CONFIG_PGTABLE_LEVELS > 3
 983
 984static __always_inline bool pgtable_l4_enabled(void)
 985{
 986	if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2))
 987		return true;
 988	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
 989		return vabits_actual == VA_BITS;
 990	return alternative_has_cap_unlikely(ARM64_HAS_VA52);
 991}
 992
 993static inline bool mm_pud_folded(const struct mm_struct *mm)
 994{
 995	return !pgtable_l4_enabled();
 996}
 997#define mm_pud_folded  mm_pud_folded
 998
 999#define pud_ERROR(e)	\
1000	pr_err("%s:%d: bad pud %016llx.\n", __FILE__, __LINE__, pud_val(e))
1001
1002#define p4d_none(p4d)		(pgtable_l4_enabled() && !p4d_val(p4d))
1003#define p4d_bad(p4d)		(pgtable_l4_enabled() && \
1004				((p4d_val(p4d) & P4D_TYPE_MASK) != \
1005				 P4D_TYPE_TABLE))
1006#define p4d_present(p4d)	(!p4d_none(p4d))
1007
1008static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
1009{
1010	if (in_swapper_pgdir(p4dp)) {
1011		set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
1012		return;
1013	}
1014
1015	WRITE_ONCE(*p4dp, p4d);
1016	queue_pte_barriers();
1017}
1018
1019static inline void p4d_clear(p4d_t *p4dp)
1020{
1021	if (pgtable_l4_enabled())
1022		set_p4d(p4dp, __p4d(0));
1023}
1024
1025static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
1026{
1027	return __p4d_to_phys(p4d);
1028}
1029
1030#define pud_index(addr)		(((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
1031
1032static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr)
1033{
1034	/* Ensure that 'p4dp' indexes a page table according to 'addr' */
1035	VM_BUG_ON(((addr >> P4D_SHIFT) ^ ((u64)p4dp >> 3)) % PTRS_PER_P4D);
1036
1037	return (pud_t *)PTR_ALIGN_DOWN(p4dp, PAGE_SIZE) + pud_index(addr);
1038}
1039
1040static inline pud_t *p4d_pgtable(p4d_t p4d)
1041{
1042	return (pud_t *)__va(p4d_page_paddr(p4d));
1043}
1044
1045static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr)
1046{
1047	BUG_ON(!pgtable_l4_enabled());
1048
1049	return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t);
1050}
1051
1052static inline
1053pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d, unsigned long addr)
1054{
1055	if (!pgtable_l4_enabled())
1056		return p4d_to_folded_pud(p4dp, addr);
1057	return (pud_t *)__va(p4d_page_paddr(p4d)) + pud_index(addr);
1058}
1059#define pud_offset_lockless pud_offset_lockless
1060
1061static inline pud_t *pud_offset(p4d_t *p4dp, unsigned long addr)
1062{
1063	return pud_offset_lockless(p4dp, READ_ONCE(*p4dp), addr);
1064}
1065#define pud_offset	pud_offset
1066
1067static inline pud_t *pud_set_fixmap(unsigned long addr)
1068{
1069	if (!pgtable_l4_enabled())
1070		return NULL;
1071	return (pud_t *)set_fixmap_offset(FIX_PUD, addr);
1072}
1073
1074static inline pud_t *pud_set_fixmap_offset(p4d_t *p4dp, unsigned long addr)
1075{
1076	if (!pgtable_l4_enabled())
1077		return p4d_to_folded_pud(p4dp, addr);
1078	return pud_set_fixmap(pud_offset_phys(p4dp, addr));
1079}
1080
1081static inline void pud_clear_fixmap(void)
1082{
1083	if (pgtable_l4_enabled())
1084		clear_fixmap(FIX_PUD);
1085}
1086
1087/* use ONLY for statically allocated translation tables */
1088static inline pud_t *pud_offset_kimg(p4d_t *p4dp, u64 addr)
1089{
1090	if (!pgtable_l4_enabled())
1091		return p4d_to_folded_pud(p4dp, addr);
1092	return (pud_t *)__phys_to_kimg(pud_offset_phys(p4dp, addr));
1093}
1094
1095#define p4d_page(p4d)		pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))
1096
1097#else
1098
1099static inline bool pgtable_l4_enabled(void) { return false; }
1100
1101#define p4d_page_paddr(p4d)	({ BUILD_BUG(); 0;})
1102
1103/* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */
1104#define pud_set_fixmap(addr)		NULL
1105#define pud_set_fixmap_offset(pgdp, addr)	((pud_t *)pgdp)
1106#define pud_clear_fixmap()
1107
1108#define pud_offset_kimg(dir,addr)	((pud_t *)dir)
1109
1110#endif  /* CONFIG_PGTABLE_LEVELS > 3 */
1111
1112#if CONFIG_PGTABLE_LEVELS > 4
1113
1114static __always_inline bool pgtable_l5_enabled(void)
1115{
1116	if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT))
1117		return vabits_actual == VA_BITS;
1118	return alternative_has_cap_unlikely(ARM64_HAS_VA52);
1119}
1120
1121static inline bool mm_p4d_folded(const struct mm_struct *mm)
1122{
1123	return !pgtable_l5_enabled();
1124}
1125#define mm_p4d_folded  mm_p4d_folded
1126
1127#define p4d_ERROR(e)	\
1128	pr_err("%s:%d: bad p4d %016llx.\n", __FILE__, __LINE__, p4d_val(e))
1129
1130#define pgd_none(pgd)		(pgtable_l5_enabled() && !pgd_val(pgd))
1131#define pgd_bad(pgd)		(pgtable_l5_enabled() && \
1132				((pgd_val(pgd) & PGD_TYPE_MASK) != \
1133				 PGD_TYPE_TABLE))
1134#define pgd_present(pgd)	(!pgd_none(pgd))
1135
1136static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
1137{
1138	if (in_swapper_pgdir(pgdp)) {
1139		set_swapper_pgd(pgdp, __pgd(pgd_val(pgd)));
1140		return;
1141	}
1142
1143	WRITE_ONCE(*pgdp, pgd);
1144	queue_pte_barriers();
1145}
1146
1147static inline void pgd_clear(pgd_t *pgdp)
1148{
1149	if (pgtable_l5_enabled())
1150		set_pgd(pgdp, __pgd(0));
1151}
1152
1153static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
1154{
1155	return __pgd_to_phys(pgd);
1156}
1157
1158#define p4d_index(addr)		(((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
1159
1160static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr)
1161{
1162	/* Ensure that 'pgdp' indexes a page table according to 'addr' */
1163	VM_BUG_ON(((addr >> PGDIR_SHIFT) ^ ((u64)pgdp >> 3)) % PTRS_PER_PGD);
1164
1165	return (p4d_t *)PTR_ALIGN_DOWN(pgdp, PAGE_SIZE) + p4d_index(addr);
1166}
1167
1168static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr)
1169{
1170	BUG_ON(!pgtable_l5_enabled());
1171
1172	return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t);
1173}
1174
1175static inline
1176p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
1177{
1178	if (!pgtable_l5_enabled())
1179		return pgd_to_folded_p4d(pgdp, addr);
1180	return (p4d_t *)__va(pgd_page_paddr(pgd)) + p4d_index(addr);
1181}
1182#define p4d_offset_lockless p4d_offset_lockless
1183
1184static inline p4d_t *p4d_offset(pgd_t *pgdp, unsigned long addr)
1185{
1186	return p4d_offset_lockless(pgdp, READ_ONCE(*pgdp), addr);
1187}
1188
1189static inline p4d_t *p4d_set_fixmap(unsigned long addr)
1190{
1191	if (!pgtable_l5_enabled())
1192		return NULL;
1193	return (p4d_t *)set_fixmap_offset(FIX_P4D, addr);
1194}
1195
1196static inline p4d_t *p4d_set_fixmap_offset(pgd_t *pgdp, unsigned long addr)
1197{
1198	if (!pgtable_l5_enabled())
1199		return pgd_to_folded_p4d(pgdp, addr);
1200	return p4d_set_fixmap(p4d_offset_phys(pgdp, addr));
1201}
1202
1203static inline void p4d_clear_fixmap(void)
1204{
1205	if (pgtable_l5_enabled())
1206		clear_fixmap(FIX_P4D);
1207}
1208
1209/* use ONLY for statically allocated translation tables */
1210static inline p4d_t *p4d_offset_kimg(pgd_t *pgdp, u64 addr)
1211{
1212	if (!pgtable_l5_enabled())
1213		return pgd_to_folded_p4d(pgdp, addr);
1214	return (p4d_t *)__phys_to_kimg(p4d_offset_phys(pgdp, addr));
1215}
1216
1217#define pgd_page(pgd)		pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))
1218
1219#else
1220
1221static inline bool pgtable_l5_enabled(void) { return false; }
1222
1223#define p4d_index(addr)		(((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
1224
1225/* Match p4d_offset folding in <asm/generic/pgtable-nop4d.h> */
1226#define p4d_set_fixmap(addr)		NULL
1227#define p4d_set_fixmap_offset(p4dp, addr)	((p4d_t *)p4dp)
1228#define p4d_clear_fixmap()
1229
1230#define p4d_offset_kimg(dir,addr)	((p4d_t *)dir)
1231
1232static inline
1233p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
1234{
1235	/*
1236	 * With runtime folding of the pud, pud_offset_lockless() passes
1237	 * the 'pgd_t *' we return here to p4d_to_folded_pud(), which
1238	 * will offset the pointer assuming that it points into
1239	 * a page-table page. However, the fast GUP path passes us a
1240	 * pgd_t allocated on the stack and so we must use the original
1241	 * pointer in 'pgdp' to construct the p4d pointer instead of
1242	 * using the generic p4d_offset_lockless() implementation.
1243	 *
1244	 * Note: reusing the original pointer means that we may
1245	 * dereference the same (live) page-table entry multiple times.
1246	 * This is safe because it is still only loaded once in the
1247	 * context of each level and the CPU guarantees same-address
1248	 * read-after-read ordering.
1249	 */
1250	return p4d_offset(pgdp, addr);
1251}
1252#define p4d_offset_lockless p4d_offset_lockless_folded
1253
1254#endif  /* CONFIG_PGTABLE_LEVELS > 4 */
1255
1256#define pgd_ERROR(e)	\
1257	pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e))
1258
1259#define pgd_set_fixmap(addr)	((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
1260#define pgd_clear_fixmap()	clear_fixmap(FIX_PGD)
1261
1262static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
1263{
1264	/*
1265	 * Normal and Normal-Tagged are two different memory types and indices
1266	 * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK.
1267	 */
1268	const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
1269			      PTE_PRESENT_INVALID | PTE_VALID | PTE_WRITE |
1270			      PTE_GP | PTE_ATTRINDX_MASK | PTE_PO_IDX_MASK;
1271
1272	/* preserve the hardware dirty information */
1273	if (pte_hw_dirty(pte))
1274		pte = set_pte_bit(pte, __pgprot(PTE_DIRTY));
1275
1276	pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask);
1277	/*
1278	 * If we end up clearing hw dirtiness for a sw-dirty PTE, set hardware
1279	 * dirtiness again.
1280	 */
1281	if (pte_sw_dirty(pte))
1282		pte = pte_mkdirty(pte);
1283	return pte;
1284}
1285
1286static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
1287{
1288	return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
1289}
1290
1291extern int __ptep_set_access_flags(struct vm_area_struct *vma,
1292				 unsigned long address, pte_t *ptep,
1293				 pte_t entry, int dirty);
1294
1295#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1296#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1297static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
1298					unsigned long address, pmd_t *pmdp,
1299					pmd_t entry, int dirty)
1300{
1301	return __ptep_set_access_flags(vma, address, (pte_t *)pmdp,
1302							pmd_pte(entry), dirty);
1303}
1304#endif
1305
1306#ifdef CONFIG_PAGE_TABLE_CHECK
1307static inline bool pte_user_accessible_page(pte_t pte)
1308{
1309	return pte_valid(pte) && (pte_user(pte) || pte_user_exec(pte));
1310}
1311
1312static inline bool pmd_user_accessible_page(pmd_t pmd)
1313{
1314	return pmd_valid(pmd) && !pmd_table(pmd) && (pmd_user(pmd) || pmd_user_exec(pmd));
1315}
1316
1317static inline bool pud_user_accessible_page(pud_t pud)
1318{
1319	return pud_valid(pud) && !pud_table(pud) && (pud_user(pud) || pud_user_exec(pud));
1320}
1321#endif
1322
1323/*
1324 * Atomic pte/pmd modifications.
1325 */
1326
1327static inline void __pte_clear(struct mm_struct *mm,
1328			       unsigned long addr, pte_t *ptep)
1329{
1330	__set_pte(ptep, __pte(0));
1331}
1332
1333static inline int __ptep_test_and_clear_young(struct vm_area_struct *vma,
1334					      unsigned long address,
1335					      pte_t *ptep)
1336{
1337	pte_t old_pte, pte;
1338
1339	pte = __ptep_get(ptep);
1340	do {
1341		old_pte = pte;
1342		pte = pte_mkold(pte);
1343		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1344					       pte_val(old_pte), pte_val(pte));
1345	} while (pte_val(pte) != pte_val(old_pte));
1346
1347	return pte_young(pte);
1348}
1349
1350static inline int __ptep_clear_flush_young(struct vm_area_struct *vma,
1351					 unsigned long address, pte_t *ptep)
1352{
1353	int young = __ptep_test_and_clear_young(vma, address, ptep);
1354
1355	if (young) {
1356		/*
1357		 * We can elide the trailing DSB here since the worst that can
1358		 * happen is that a CPU continues to use the young entry in its
1359		 * TLB and we mistakenly reclaim the associated page. The
1360		 * window for such an event is bounded by the next
1361		 * context-switch, which provides a DSB to complete the TLB
1362		 * invalidation.
1363		 */
1364		flush_tlb_page_nosync(vma, address);
1365	}
1366
1367	return young;
1368}
1369
1370#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)
1371#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1372static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1373					    unsigned long address,
1374					    pmd_t *pmdp)
1375{
1376	/* Operation applies to PMD table entry only if FEAT_HAFT is enabled */
1377	VM_WARN_ON(pmd_table(READ_ONCE(*pmdp)) && !system_supports_haft());
1378	return __ptep_test_and_clear_young(vma, address, (pte_t *)pmdp);
1379}
1380#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */
1381
1382static inline pte_t __ptep_get_and_clear_anysz(struct mm_struct *mm,
1383					       pte_t *ptep,
1384					       unsigned long pgsize)
1385{
1386	pte_t pte = __pte(xchg_relaxed(&pte_val(*ptep), 0));
1387
1388	switch (pgsize) {
1389	case PAGE_SIZE:
1390		page_table_check_pte_clear(mm, pte);
1391		break;
1392	case PMD_SIZE:
1393		page_table_check_pmd_clear(mm, pte_pmd(pte));
1394		break;
1395#ifndef __PAGETABLE_PMD_FOLDED
1396	case PUD_SIZE:
1397		page_table_check_pud_clear(mm, pte_pud(pte));
1398		break;
1399#endif
1400	default:
1401		VM_WARN_ON(1);
1402	}
1403
1404	return pte;
1405}
1406
1407static inline pte_t __ptep_get_and_clear(struct mm_struct *mm,
1408				       unsigned long address, pte_t *ptep)
1409{
1410	return __ptep_get_and_clear_anysz(mm, ptep, PAGE_SIZE);
1411}
1412
1413static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1414				pte_t *ptep, unsigned int nr, int full)
1415{
1416	for (;;) {
1417		__ptep_get_and_clear(mm, addr, ptep);
1418		if (--nr == 0)
1419			break;
1420		ptep++;
1421		addr += PAGE_SIZE;
1422	}
1423}
1424
1425static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm,
1426				unsigned long addr, pte_t *ptep,
1427				unsigned int nr, int full)
1428{
1429	pte_t pte, tmp_pte;
1430
1431	pte = __ptep_get_and_clear(mm, addr, ptep);
1432	while (--nr) {
1433		ptep++;
1434		addr += PAGE_SIZE;
1435		tmp_pte = __ptep_get_and_clear(mm, addr, ptep);
1436		if (pte_dirty(tmp_pte))
1437			pte = pte_mkdirty(pte);
1438		if (pte_young(tmp_pte))
1439			pte = pte_mkyoung(pte);
1440	}
1441	return pte;
1442}
1443
1444#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1445#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1446static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
1447					    unsigned long address, pmd_t *pmdp)
1448{
1449	return pte_pmd(__ptep_get_and_clear_anysz(mm, (pte_t *)pmdp, PMD_SIZE));
1450}
1451#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1452
1453static inline void ___ptep_set_wrprotect(struct mm_struct *mm,
1454					unsigned long address, pte_t *ptep,
1455					pte_t pte)
1456{
1457	pte_t old_pte;
1458
1459	do {
1460		old_pte = pte;
1461		pte = pte_wrprotect(pte);
1462		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1463					       pte_val(old_pte), pte_val(pte));
1464	} while (pte_val(pte) != pte_val(old_pte));
1465}
1466
1467/*
1468 * __ptep_set_wrprotect - mark read-only while transferring potential hardware
1469 * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit.
1470 */
1471static inline void __ptep_set_wrprotect(struct mm_struct *mm,
1472					unsigned long address, pte_t *ptep)
1473{
1474	___ptep_set_wrprotect(mm, address, ptep, __ptep_get(ptep));
1475}
1476
1477static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address,
1478				pte_t *ptep, unsigned int nr)
1479{
1480	unsigned int i;
1481
1482	for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++)
1483		__ptep_set_wrprotect(mm, address, ptep);
1484}
1485
1486static inline void __clear_young_dirty_pte(struct vm_area_struct *vma,
1487					   unsigned long addr, pte_t *ptep,
1488					   pte_t pte, cydp_t flags)
1489{
1490	pte_t old_pte;
1491
1492	do {
1493		old_pte = pte;
1494
1495		if (flags & CYDP_CLEAR_YOUNG)
1496			pte = pte_mkold(pte);
1497		if (flags & CYDP_CLEAR_DIRTY)
1498			pte = pte_mkclean(pte);
1499
1500		pte_val(pte) = cmpxchg_relaxed(&pte_val(*ptep),
1501					       pte_val(old_pte), pte_val(pte));
1502	} while (pte_val(pte) != pte_val(old_pte));
1503}
1504
1505static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma,
1506					    unsigned long addr, pte_t *ptep,
1507					    unsigned int nr, cydp_t flags)
1508{
1509	pte_t pte;
1510
1511	for (;;) {
1512		pte = __ptep_get(ptep);
1513
1514		if (flags == (CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY))
1515			__set_pte(ptep, pte_mkclean(pte_mkold(pte)));
1516		else
1517			__clear_young_dirty_pte(vma, addr, ptep, pte, flags);
1518
1519		if (--nr == 0)
1520			break;
1521		ptep++;
1522		addr += PAGE_SIZE;
1523	}
1524}
1525
1526#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1527#define __HAVE_ARCH_PMDP_SET_WRPROTECT
1528static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1529				      unsigned long address, pmd_t *pmdp)
1530{
1531	__ptep_set_wrprotect(mm, address, (pte_t *)pmdp);
1532}
1533
1534#define pmdp_establish pmdp_establish
1535static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1536		unsigned long address, pmd_t *pmdp, pmd_t pmd)
1537{
1538	page_table_check_pmd_set(vma->vm_mm, pmdp, pmd);
1539	return __pmd(xchg_relaxed(&pmd_val(*pmdp), pmd_val(pmd)));
1540}
1541#endif
1542
1543/*
1544 * Encode and decode a swap entry:
1545 *	bits 0-1:	present (must be zero)
1546 *	bits 2:		remember PG_anon_exclusive
1547 *	bit  3:		remember uffd-wp state
1548 *	bits 6-10:	swap type
1549 *	bit  11:	PTE_PRESENT_INVALID (must be zero)
1550 *	bits 12-61:	swap offset
1551 */
1552#define __SWP_TYPE_SHIFT	6
1553#define __SWP_TYPE_BITS		5
1554#define __SWP_TYPE_MASK		((1 << __SWP_TYPE_BITS) - 1)
1555#define __SWP_OFFSET_SHIFT	12
1556#define __SWP_OFFSET_BITS	50
1557#define __SWP_OFFSET_MASK	((1UL << __SWP_OFFSET_BITS) - 1)
1558
1559#define __swp_type(x)		(((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK)
1560#define __swp_offset(x)		(((x).val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK)
1561#define __swp_entry(type,offset) ((swp_entry_t) { ((type) << __SWP_TYPE_SHIFT) | ((offset) << __SWP_OFFSET_SHIFT) })
1562
1563#define __pte_to_swp_entry(pte)	((swp_entry_t) { pte_val(pte) })
1564#define __swp_entry_to_pte(swp)	((pte_t) { (swp).val })
1565
1566#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1567#define __pmd_to_swp_entry(pmd)		((swp_entry_t) { pmd_val(pmd) })
1568#define __swp_entry_to_pmd(swp)		__pmd((swp).val)
1569#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
1570
1571/*
1572 * Ensure that there are not more swap files than can be encoded in the kernel
1573 * PTEs.
1574 */
1575#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS)
1576
1577#ifdef CONFIG_ARM64_MTE
1578
1579#define __HAVE_ARCH_PREPARE_TO_SWAP
1580extern int arch_prepare_to_swap(struct folio *folio);
1581
1582#define __HAVE_ARCH_SWAP_INVALIDATE
1583static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
1584{
1585	if (system_supports_mte())
1586		mte_invalidate_tags(type, offset);
1587}
1588
1589static inline void arch_swap_invalidate_area(int type)
1590{
1591	if (system_supports_mte())
1592		mte_invalidate_tags_area(type);
1593}
1594
1595#define __HAVE_ARCH_SWAP_RESTORE
1596extern void arch_swap_restore(swp_entry_t entry, struct folio *folio);
1597
1598#endif /* CONFIG_ARM64_MTE */
1599
1600/*
1601 * On AArch64, the cache coherency is handled via the __set_ptes() function.
1602 */
1603static inline void update_mmu_cache_range(struct vm_fault *vmf,
1604		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
1605		unsigned int nr)
1606{
1607	/*
1608	 * We don't do anything here, so there's a very small chance of
1609	 * us retaking a user fault which we just fixed up. The alternative
1610	 * is doing a dsb(ishst), but that penalises the fastpath.
1611	 */
1612}
1613
1614#define update_mmu_cache(vma, addr, ptep) \
1615	update_mmu_cache_range(NULL, vma, addr, ptep, 1)
1616#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
1617
1618#ifdef CONFIG_ARM64_PA_BITS_52
1619#define phys_to_ttbr(addr)	(((addr) | ((addr) >> 46)) & TTBR_BADDR_MASK_52)
1620#else
1621#define phys_to_ttbr(addr)	(addr)
1622#endif
1623
1624/*
1625 * On arm64 without hardware Access Flag, copying from user will fail because
1626 * the pte is old and cannot be marked young. So we always end up with zeroed
1627 * page after fork() + CoW for pfn mappings. We don't always have a
1628 * hardware-managed access flag on arm64.
1629 */
1630#define arch_has_hw_pte_young		cpu_has_hw_af
1631
1632#ifdef CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG
1633#define arch_has_hw_nonleaf_pmd_young	system_supports_haft
1634#endif
1635
1636/*
1637 * Experimentally, it's cheap to set the access flag in hardware and we
1638 * benefit from prefaulting mappings as 'old' to start with.
1639 */
1640#define arch_wants_old_prefaulted_pte	cpu_has_hw_af
1641
1642/*
1643 * Request exec memory is read into pagecache in at least 64K folios. This size
1644 * can be contpte-mapped when 4K base pages are in use (16 pages into 1 iTLB
1645 * entry), and HPA can coalesce it (4 pages into 1 TLB entry) when 16K base
1646 * pages are in use.
1647 */
1648#define exec_folio_order() ilog2(SZ_64K >> PAGE_SHIFT)
1649
1650static inline bool pud_sect_supported(void)
1651{
1652	return PAGE_SIZE == SZ_4K;
1653}
1654
1655
1656#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
1657#define ptep_modify_prot_start ptep_modify_prot_start
1658extern pte_t ptep_modify_prot_start(struct vm_area_struct *vma,
1659				    unsigned long addr, pte_t *ptep);
1660
1661#define ptep_modify_prot_commit ptep_modify_prot_commit
1662extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
1663				    unsigned long addr, pte_t *ptep,
1664				    pte_t old_pte, pte_t new_pte);
1665
1666#define modify_prot_start_ptes modify_prot_start_ptes
1667extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
1668				    unsigned long addr, pte_t *ptep,
1669				    unsigned int nr);
1670
1671#define modify_prot_commit_ptes modify_prot_commit_ptes
1672extern void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
1673				    pte_t *ptep, pte_t old_pte, pte_t pte,
1674				    unsigned int nr);
1675
1676#ifdef CONFIG_ARM64_CONTPTE
1677
1678/*
1679 * The contpte APIs are used to transparently manage the contiguous bit in ptes
1680 * where it is possible and makes sense to do so. The PTE_CONT bit is considered
1681 * a private implementation detail of the public ptep API (see below).
1682 */
1683extern void __contpte_try_fold(struct mm_struct *mm, unsigned long addr,
1684				pte_t *ptep, pte_t pte);
1685extern void __contpte_try_unfold(struct mm_struct *mm, unsigned long addr,
1686				pte_t *ptep, pte_t pte);
1687extern pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte);
1688extern pte_t contpte_ptep_get_lockless(pte_t *orig_ptep);
1689extern void contpte_set_ptes(struct mm_struct *mm, unsigned long addr,
1690				pte_t *ptep, pte_t pte, unsigned int nr);
1691extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1692				pte_t *ptep, unsigned int nr, int full);
1693extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,
1694				unsigned long addr, pte_t *ptep,
1695				unsigned int nr, int full);
1696extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma,
1697				unsigned long addr, pte_t *ptep);
1698extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma,
1699				unsigned long addr, pte_t *ptep);
1700extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
1701				pte_t *ptep, unsigned int nr);
1702extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma,
1703				unsigned long addr, pte_t *ptep,
1704				pte_t entry, int dirty);
1705extern void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma,
1706				unsigned long addr, pte_t *ptep,
1707				unsigned int nr, cydp_t flags);
1708
1709static __always_inline void contpte_try_fold(struct mm_struct *mm,
1710				unsigned long addr, pte_t *ptep, pte_t pte)
1711{
1712	/*
1713	 * Only bother trying if both the virtual and physical addresses are
1714	 * aligned and correspond to the last entry in a contig range. The core
1715	 * code mostly modifies ranges from low to high, so this is the likely
1716	 * the last modification in the contig range, so a good time to fold.
1717	 * We can't fold special mappings, because there is no associated folio.
1718	 */
1719
1720	const unsigned long contmask = CONT_PTES - 1;
1721	bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask;
1722
1723	if (unlikely(valign)) {
1724		bool palign = (pte_pfn(pte) & contmask) == contmask;
1725
1726		if (unlikely(palign &&
1727		    pte_valid(pte) && !pte_cont(pte) && !pte_special(pte)))
1728			__contpte_try_fold(mm, addr, ptep, pte);
1729	}
1730}
1731
1732static __always_inline void contpte_try_unfold(struct mm_struct *mm,
1733				unsigned long addr, pte_t *ptep, pte_t pte)
1734{
1735	if (unlikely(pte_valid_cont(pte)))
1736		__contpte_try_unfold(mm, addr, ptep, pte);
1737}
1738
1739#define pte_batch_hint pte_batch_hint
1740static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte)
1741{
1742	if (!pte_valid_cont(pte))
1743		return 1;
1744
1745	return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1));
1746}
1747
1748/*
1749 * The below functions constitute the public API that arm64 presents to the
1750 * core-mm to manipulate PTE entries within their page tables (or at least this
1751 * is the subset of the API that arm64 needs to implement). These public
1752 * versions will automatically and transparently apply the contiguous bit where
1753 * it makes sense to do so. Therefore any users that are contig-aware (e.g.
1754 * hugetlb, kernel mapper) should NOT use these APIs, but instead use the
1755 * private versions, which are prefixed with double underscore. All of these
1756 * APIs except for ptep_get_lockless() are expected to be called with the PTL
1757 * held. Although the contiguous bit is considered private to the
1758 * implementation, it is deliberately allowed to leak through the getters (e.g.
1759 * ptep_get()), back to core code. This is required so that pte_leaf_size() can
1760 * provide an accurate size for perf_get_pgtable_size(). But this leakage means
1761 * its possible a pte will be passed to a setter with the contiguous bit set, so
1762 * we explicitly clear the contiguous bit in those cases to prevent accidentally
1763 * setting it in the pgtable.
1764 */
1765
1766#define ptep_get ptep_get
1767static inline pte_t ptep_get(pte_t *ptep)
1768{
1769	pte_t pte = __ptep_get(ptep);
1770
1771	if (likely(!pte_valid_cont(pte)))
1772		return pte;
1773
1774	return contpte_ptep_get(ptep, pte);
1775}
1776
1777#define ptep_get_lockless ptep_get_lockless
1778static inline pte_t ptep_get_lockless(pte_t *ptep)
1779{
1780	pte_t pte = __ptep_get(ptep);
1781
1782	if (likely(!pte_valid_cont(pte)))
1783		return pte;
1784
1785	return contpte_ptep_get_lockless(ptep);
1786}
1787
1788static inline void set_pte(pte_t *ptep, pte_t pte)
1789{
1790	/*
1791	 * We don't have the mm or vaddr so cannot unfold contig entries (since
1792	 * it requires tlb maintenance). set_pte() is not used in core code, so
1793	 * this should never even be called. Regardless do our best to service
1794	 * any call and emit a warning if there is any attempt to set a pte on
1795	 * top of an existing contig range.
1796	 */
1797	pte_t orig_pte = __ptep_get(ptep);
1798
1799	WARN_ON_ONCE(pte_valid_cont(orig_pte));
1800	__set_pte(ptep, pte_mknoncont(pte));
1801}
1802
1803#define set_ptes set_ptes
1804static __always_inline void set_ptes(struct mm_struct *mm, unsigned long addr,
1805				pte_t *ptep, pte_t pte, unsigned int nr)
1806{
1807	pte = pte_mknoncont(pte);
1808
1809	if (likely(nr == 1)) {
1810		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1811		__set_ptes(mm, addr, ptep, pte, 1);
1812		contpte_try_fold(mm, addr, ptep, pte);
1813	} else {
1814		contpte_set_ptes(mm, addr, ptep, pte, nr);
1815	}
1816}
1817
1818static inline void pte_clear(struct mm_struct *mm,
1819				unsigned long addr, pte_t *ptep)
1820{
1821	contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1822	__pte_clear(mm, addr, ptep);
1823}
1824
1825#define clear_full_ptes clear_full_ptes
1826static inline void clear_full_ptes(struct mm_struct *mm, unsigned long addr,
1827				pte_t *ptep, unsigned int nr, int full)
1828{
1829	if (likely(nr == 1)) {
1830		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1831		__clear_full_ptes(mm, addr, ptep, nr, full);
1832	} else {
1833		contpte_clear_full_ptes(mm, addr, ptep, nr, full);
1834	}
1835}
1836
1837#define get_and_clear_full_ptes get_and_clear_full_ptes
1838static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm,
1839				unsigned long addr, pte_t *ptep,
1840				unsigned int nr, int full)
1841{
1842	pte_t pte;
1843
1844	if (likely(nr == 1)) {
1845		contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1846		pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full);
1847	} else {
1848		pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full);
1849	}
1850
1851	return pte;
1852}
1853
1854#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1855static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
1856				unsigned long addr, pte_t *ptep)
1857{
1858	contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
1859	return __ptep_get_and_clear(mm, addr, ptep);
1860}
1861
1862#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1863static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
1864				unsigned long addr, pte_t *ptep)
1865{
1866	pte_t orig_pte = __ptep_get(ptep);
1867
1868	if (likely(!pte_valid_cont(orig_pte)))
1869		return __ptep_test_and_clear_young(vma, addr, ptep);
1870
1871	return contpte_ptep_test_and_clear_young(vma, addr, ptep);
1872}
1873
1874#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1875static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
1876				unsigned long addr, pte_t *ptep)
1877{
1878	pte_t orig_pte = __ptep_get(ptep);
1879
1880	if (likely(!pte_valid_cont(orig_pte)))
1881		return __ptep_clear_flush_young(vma, addr, ptep);
1882
1883	return contpte_ptep_clear_flush_young(vma, addr, ptep);
1884}
1885
1886#define wrprotect_ptes wrprotect_ptes
1887static __always_inline void wrprotect_ptes(struct mm_struct *mm,
1888				unsigned long addr, pte_t *ptep, unsigned int nr)
1889{
1890	if (likely(nr == 1)) {
1891		/*
1892		 * Optimization: wrprotect_ptes() can only be called for present
1893		 * ptes so we only need to check contig bit as condition for
1894		 * unfold, and we can remove the contig bit from the pte we read
1895		 * to avoid re-reading. This speeds up fork() which is sensitive
1896		 * for order-0 folios. Equivalent to contpte_try_unfold().
1897		 */
1898		pte_t orig_pte = __ptep_get(ptep);
1899
1900		if (unlikely(pte_cont(orig_pte))) {
1901			__contpte_try_unfold(mm, addr, ptep, orig_pte);
1902			orig_pte = pte_mknoncont(orig_pte);
1903		}
1904		___ptep_set_wrprotect(mm, addr, ptep, orig_pte);
1905	} else {
1906		contpte_wrprotect_ptes(mm, addr, ptep, nr);
1907	}
1908}
1909
1910#define __HAVE_ARCH_PTEP_SET_WRPROTECT
1911static inline void ptep_set_wrprotect(struct mm_struct *mm,
1912				unsigned long addr, pte_t *ptep)
1913{
1914	wrprotect_ptes(mm, addr, ptep, 1);
1915}
1916
1917#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1918static inline int ptep_set_access_flags(struct vm_area_struct *vma,
1919				unsigned long addr, pte_t *ptep,
1920				pte_t entry, int dirty)
1921{
1922	pte_t orig_pte = __ptep_get(ptep);
1923
1924	entry = pte_mknoncont(entry);
1925
1926	if (likely(!pte_valid_cont(orig_pte)))
1927		return __ptep_set_access_flags(vma, addr, ptep, entry, dirty);
1928
1929	return contpte_ptep_set_access_flags(vma, addr, ptep, entry, dirty);
1930}
1931
1932#define clear_young_dirty_ptes clear_young_dirty_ptes
1933static inline void clear_young_dirty_ptes(struct vm_area_struct *vma,
1934					  unsigned long addr, pte_t *ptep,
1935					  unsigned int nr, cydp_t flags)
1936{
1937	if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
1938		__clear_young_dirty_ptes(vma, addr, ptep, nr, flags);
1939	else
1940		contpte_clear_young_dirty_ptes(vma, addr, ptep, nr, flags);
1941}
1942
1943#else /* CONFIG_ARM64_CONTPTE */
1944
1945#define ptep_get				__ptep_get
1946#define set_pte					__set_pte
1947#define set_ptes				__set_ptes
1948#define pte_clear				__pte_clear
1949#define clear_full_ptes				__clear_full_ptes
1950#define get_and_clear_full_ptes			__get_and_clear_full_ptes
1951#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1952#define ptep_get_and_clear			__ptep_get_and_clear
1953#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1954#define ptep_test_and_clear_young		__ptep_test_and_clear_young
1955#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1956#define ptep_clear_flush_young			__ptep_clear_flush_young
1957#define __HAVE_ARCH_PTEP_SET_WRPROTECT
1958#define ptep_set_wrprotect			__ptep_set_wrprotect
1959#define wrprotect_ptes				__wrprotect_ptes
1960#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1961#define ptep_set_access_flags			__ptep_set_access_flags
1962#define clear_young_dirty_ptes			__clear_young_dirty_ptes
1963
1964#endif /* CONFIG_ARM64_CONTPTE */
1965
1966#endif /* !__ASSEMBLER__ */
1967
1968#endif /* __ASM_PGTABLE_H */