pgtable_64.h

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * pgtable.h: SpitFire page table operations.
   4 *
   5 * Copyright 1996,1997 David S. Miller (davem@caip.rutgers.edu)
   6 * Copyright 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
   7 */
   8
   9#ifndef _SPARC64_PGTABLE_H
  10#define _SPARC64_PGTABLE_H
  11
  12/* This file contains the functions and defines necessary to modify and use
  13 * the SpitFire page tables.
  14 */
  15
  16#include <asm-generic/pgtable-nop4d.h>
  17#include <linux/compiler.h>
  18#include <linux/const.h>
  19#include <asm/types.h>
  20#include <asm/spitfire.h>
  21#include <asm/asi.h>
  22#include <asm/adi.h>
  23#include <asm/page.h>
  24#include <asm/processor.h>
  25
  26/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB).
  27 * The page copy blockops can use 0x6000000 to 0x8000000.
  28 * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range.
  29 * The 4M TSB is mapped in the 0x8400000 to 0x8800000 range.
  30 * The PROM resides in an area spanning 0xf0000000 to 0x100000000.
  31 * The vmalloc area spans 0x100000000 to 0x200000000.
  32 * Since modules need to be in the lowest 32-bits of the address space,
  33 * we place them right before the OBP area from 0x10000000 to 0xf0000000.
  34 * There is a single static kernel PMD which maps from 0x0 to address
  35 * 0x400000000.
  36 */
  37#define	TLBTEMP_BASE		_AC(0x0000000006000000,UL)
  38#define	TSBMAP_8K_BASE		_AC(0x0000000008000000,UL)
  39#define	TSBMAP_4M_BASE		_AC(0x0000000008400000,UL)
  40#define MODULES_VADDR		_AC(0x0000000010000000,UL)
  41#define MODULES_LEN		_AC(0x00000000e0000000,UL)
  42#define MODULES_END		_AC(0x00000000f0000000,UL)
  43#define LOW_OBP_ADDRESS		_AC(0x00000000f0000000,UL)
  44#define HI_OBP_ADDRESS		_AC(0x0000000100000000,UL)
  45#define VMALLOC_START		_AC(0x0000000100000000,UL)
  46#define VMEMMAP_BASE		VMALLOC_END
  47
  48/* PMD_SHIFT determines the size of the area a second-level page
  49 * table can map
  50 */
  51#define PMD_SHIFT	(PAGE_SHIFT + (PAGE_SHIFT-3))
  52#define PMD_SIZE	(_AC(1,UL) << PMD_SHIFT)
  53#define PMD_MASK	(~(PMD_SIZE-1))
  54#define PMD_BITS	(PAGE_SHIFT - 3)
  55
  56/* PUD_SHIFT determines the size of the area a third-level page
  57 * table can map
  58 */
  59#define PUD_SHIFT	(PMD_SHIFT + PMD_BITS)
  60#define PUD_SIZE	(_AC(1,UL) << PUD_SHIFT)
  61#define PUD_MASK	(~(PUD_SIZE-1))
  62#define PUD_BITS	(PAGE_SHIFT - 3)
  63
  64/* PGDIR_SHIFT determines what a fourth-level page table entry can map */
  65#define PGDIR_SHIFT	(PUD_SHIFT + PUD_BITS)
  66#define PGDIR_SIZE	(_AC(1,UL) << PGDIR_SHIFT)
  67#define PGDIR_MASK	(~(PGDIR_SIZE-1))
  68#define PGDIR_BITS	(PAGE_SHIFT - 3)
  69
  70#if (MAX_PHYS_ADDRESS_BITS > PGDIR_SHIFT + PGDIR_BITS)
  71#error MAX_PHYS_ADDRESS_BITS exceeds what kernel page tables can support
  72#endif
  73
  74#if (PGDIR_SHIFT + PGDIR_BITS) != 53
  75#error Page table parameters do not cover virtual address space properly.
  76#endif
  77
  78#if (PMD_SHIFT != HPAGE_SHIFT)
  79#error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages.
  80#endif
  81
  82#ifndef __ASSEMBLER__
  83
  84extern unsigned long VMALLOC_END;
  85
  86#define vmemmap			((struct page *)VMEMMAP_BASE)
  87
  88#include <linux/sched.h>
  89#include <asm/tlbflush.h>
  90
  91bool kern_addr_valid(unsigned long addr);
  92
  93/* Entries per page directory level. */
  94#define PTRS_PER_PTE	(1UL << (PAGE_SHIFT-3))
  95#define PTRS_PER_PMD	(1UL << PMD_BITS)
  96#define PTRS_PER_PUD	(1UL << PUD_BITS)
  97#define PTRS_PER_PGD	(1UL << PGDIR_BITS)
  98
  99#define pmd_ERROR(e)							\
 100	pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n",		\
 101	       __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0))
 102#define pud_ERROR(e)							\
 103	pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n",		\
 104	       __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0))
 105#define pgd_ERROR(e)							\
 106	pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n",		\
 107	       __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0))
 108
 109#endif /* !(__ASSEMBLER__) */
 110
 111/* PTE bits which are the same in SUN4U and SUN4V format.  */
 112#define _PAGE_VALID	  _AC(0x8000000000000000,UL) /* Valid TTE            */
 113#define _PAGE_R	  	  _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
 114#define _PAGE_SPECIAL     _AC(0x0200000000000000,UL) /* Special page         */
 115#define _PAGE_PMD_HUGE    _AC(0x0100000000000000,UL) /* Huge page            */
 116#define _PAGE_PUD_HUGE    _PAGE_PMD_HUGE
 117
 118/* SUN4U pte bits... */
 119#define _PAGE_SZ4MB_4U	  _AC(0x6000000000000000,UL) /* 4MB Page             */
 120#define _PAGE_SZ512K_4U	  _AC(0x4000000000000000,UL) /* 512K Page            */
 121#define _PAGE_SZ64K_4U	  _AC(0x2000000000000000,UL) /* 64K Page             */
 122#define _PAGE_SZ8K_4U	  _AC(0x0000000000000000,UL) /* 8K Page              */
 123#define _PAGE_NFO_4U	  _AC(0x1000000000000000,UL) /* No Fault Only        */
 124#define _PAGE_IE_4U	  _AC(0x0800000000000000,UL) /* Invert Endianness    */
 125#define _PAGE_SOFT2_4U	  _AC(0x07FC000000000000,UL) /* Software bits, set 2 */
 126#define _PAGE_SPECIAL_4U  _AC(0x0200000000000000,UL) /* Special page         */
 127#define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page            */
 128#define _PAGE_RES1_4U	  _AC(0x0002000000000000,UL) /* Reserved             */
 129#define _PAGE_SZ32MB_4U	  _AC(0x0001000000000000,UL) /* (Panther) 32MB page  */
 130#define _PAGE_SZ256MB_4U  _AC(0x2001000000000000,UL) /* (Panther) 256MB page */
 131#define _PAGE_SZALL_4U	  _AC(0x6001000000000000,UL) /* All pgsz bits        */
 132#define _PAGE_SN_4U	  _AC(0x0000800000000000,UL) /* (Cheetah) Snoop      */
 133#define _PAGE_RES2_4U	  _AC(0x0000780000000000,UL) /* Reserved             */
 134#define _PAGE_PADDR_4U	  _AC(0x000007FFFFFFE000,UL) /* (Cheetah) pa[42:13]  */
 135#define _PAGE_SOFT_4U	  _AC(0x0000000000001F80,UL) /* Software bits:       */
 136#define _PAGE_EXEC_4U	  _AC(0x0000000000001000,UL) /* Executable SW bit    */
 137#define _PAGE_MODIFIED_4U _AC(0x0000000000000800,UL) /* Modified (dirty)     */
 138#define _PAGE_ACCESSED_4U _AC(0x0000000000000400,UL) /* Accessed (ref'd)     */
 139#define _PAGE_READ_4U	  _AC(0x0000000000000200,UL) /* Readable SW Bit      */
 140#define _PAGE_WRITE_4U	  _AC(0x0000000000000100,UL) /* Writable SW Bit      */
 141#define _PAGE_PRESENT_4U  _AC(0x0000000000000080,UL) /* Present              */
 142#define _PAGE_L_4U	  _AC(0x0000000000000040,UL) /* Locked TTE           */
 143#define _PAGE_CP_4U	  _AC(0x0000000000000020,UL) /* Cacheable in P-Cache */
 144#define _PAGE_CV_4U	  _AC(0x0000000000000010,UL) /* Cacheable in V-Cache */
 145#define _PAGE_E_4U	  _AC(0x0000000000000008,UL) /* side-Effect          */
 146#define _PAGE_P_4U	  _AC(0x0000000000000004,UL) /* Privileged Page      */
 147#define _PAGE_W_4U	  _AC(0x0000000000000002,UL) /* Writable             */
 148
 149/* SUN4V pte bits... */
 150#define _PAGE_NFO_4V	  _AC(0x4000000000000000,UL) /* No Fault Only        */
 151#define _PAGE_SOFT2_4V	  _AC(0x3F00000000000000,UL) /* Software bits, set 2 */
 152#define _PAGE_MODIFIED_4V _AC(0x2000000000000000,UL) /* Modified (dirty)     */
 153#define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd)     */
 154#define _PAGE_READ_4V	  _AC(0x0800000000000000,UL) /* Readable SW Bit      */
 155#define _PAGE_WRITE_4V	  _AC(0x0400000000000000,UL) /* Writable SW Bit      */
 156#define _PAGE_SPECIAL_4V  _AC(0x0200000000000000,UL) /* Special page         */
 157#define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page            */
 158#define _PAGE_PADDR_4V	  _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13]         */
 159#define _PAGE_IE_4V	  _AC(0x0000000000001000,UL) /* Invert Endianness    */
 160#define _PAGE_E_4V	  _AC(0x0000000000000800,UL) /* side-Effect          */
 161#define _PAGE_CP_4V	  _AC(0x0000000000000400,UL) /* Cacheable in P-Cache */
 162#define _PAGE_CV_4V	  _AC(0x0000000000000200,UL) /* Cacheable in V-Cache */
 163/* Bit 9 is used to enable MCD corruption detection instead on M7 */
 164#define _PAGE_MCD_4V      _AC(0x0000000000000200,UL) /* Memory Corruption    */
 165#define _PAGE_P_4V	  _AC(0x0000000000000100,UL) /* Privileged Page      */
 166#define _PAGE_EXEC_4V	  _AC(0x0000000000000080,UL) /* Executable Page      */
 167#define _PAGE_W_4V	  _AC(0x0000000000000040,UL) /* Writable             */
 168#define _PAGE_SOFT_4V	  _AC(0x0000000000000030,UL) /* Software bits        */
 169#define _PAGE_PRESENT_4V  _AC(0x0000000000000010,UL) /* Present              */
 170#define _PAGE_RESV_4V	  _AC(0x0000000000000008,UL) /* Reserved             */
 171#define _PAGE_SZ16GB_4V	  _AC(0x0000000000000007,UL) /* 16GB Page            */
 172#define _PAGE_SZ2GB_4V	  _AC(0x0000000000000006,UL) /* 2GB Page             */
 173#define _PAGE_SZ256MB_4V  _AC(0x0000000000000005,UL) /* 256MB Page           */
 174#define _PAGE_SZ32MB_4V	  _AC(0x0000000000000004,UL) /* 32MB Page            */
 175#define _PAGE_SZ4MB_4V	  _AC(0x0000000000000003,UL) /* 4MB Page             */
 176#define _PAGE_SZ512K_4V	  _AC(0x0000000000000002,UL) /* 512K Page            */
 177#define _PAGE_SZ64K_4V	  _AC(0x0000000000000001,UL) /* 64K Page             */
 178#define _PAGE_SZ8K_4V	  _AC(0x0000000000000000,UL) /* 8K Page              */
 179#define _PAGE_SZALL_4V	  _AC(0x0000000000000007,UL) /* All pgsz bits        */
 180
 181#define _PAGE_SZBITS_4U	_PAGE_SZ8K_4U
 182#define _PAGE_SZBITS_4V	_PAGE_SZ8K_4V
 183
 184#if REAL_HPAGE_SHIFT != 22
 185#error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up
 186#endif
 187
 188#define _PAGE_SZHUGE_4U	_PAGE_SZ4MB_4U
 189#define _PAGE_SZHUGE_4V	_PAGE_SZ4MB_4V
 190
 191/* We borrow bit 20 to store the exclusive marker in swap PTEs. */
 192#define _PAGE_SWP_EXCLUSIVE	_AC(0x0000000000100000, UL)
 193
 194#ifndef __ASSEMBLER__
 195
 196pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long);
 197
 198unsigned long pte_sz_bits(unsigned long size);
 199
 200extern pgprot_t PAGE_KERNEL;
 201extern pgprot_t PAGE_KERNEL_LOCKED;
 202extern pgprot_t PAGE_COPY;
 203extern pgprot_t PAGE_SHARED;
 204
 205/* XXX This ugliness is for the atyfb driver's sparc mmap() support. XXX */
 206extern unsigned long _PAGE_IE;
 207extern unsigned long _PAGE_E;
 208extern unsigned long _PAGE_CACHE;
 209
 210extern unsigned long pg_iobits;
 211extern unsigned long _PAGE_ALL_SZ_BITS;
 212
 213extern struct page *mem_map_zero;
 214#define ZERO_PAGE(vaddr)	(mem_map_zero)
 215
 216/* PFNs are real physical page numbers.  However, mem_map only begins to record
 217 * per-page information starting at pfn_base.  This is to handle systems where
 218 * the first physical page in the machine is at some huge physical address,
 219 * such as 4GB.   This is common on a partitioned E10000, for example.
 220 */
 221static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
 222{
 223	unsigned long paddr = pfn << PAGE_SHIFT;
 224
 225	BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL);
 226	return __pte(paddr | pgprot_val(prot));
 227}
 228
 229#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 230static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
 231{
 232	pte_t pte = pfn_pte(page_nr, pgprot);
 233
 234	return __pmd(pte_val(pte));
 235}
 236#endif
 237
 238/* This one can be done with two shifts.  */
 239static inline unsigned long pte_pfn(pte_t pte)
 240{
 241	unsigned long ret;
 242
 243	__asm__ __volatile__(
 244	"\n661:	sllx		%1, %2, %0\n"
 245	"	srlx		%0, %3, %0\n"
 246	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 247	"	.word		661b\n"
 248	"	sllx		%1, %4, %0\n"
 249	"	srlx		%0, %5, %0\n"
 250	"	.previous\n"
 251	: "=r" (ret)
 252	: "r" (pte_val(pte)),
 253	  "i" (21), "i" (21 + PAGE_SHIFT),
 254	  "i" (8), "i" (8 + PAGE_SHIFT));
 255
 256	return ret;
 257}
 258#define pte_page(x) pfn_to_page(pte_pfn(x))
 259
 260static inline pte_t pte_modify(pte_t pte, pgprot_t prot)
 261{
 262	unsigned long mask, tmp;
 263
 264	/* SUN4U: 0x630107ffffffec38 (negated == 0x9cfef800000013c7)
 265	 * SUN4V: 0x33ffffffffffee07 (negated == 0xcc000000000011f8)
 266	 *
 267	 * Even if we use negation tricks the result is still a 6
 268	 * instruction sequence, so don't try to play fancy and just
 269	 * do the most straightforward implementation.
 270	 *
 271	 * Note: We encode this into 3 sun4v 2-insn patch sequences.
 272	 */
 273
 274	BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL);
 275	__asm__ __volatile__(
 276	"\n661:	sethi		%%uhi(%2), %1\n"
 277	"	sethi		%%hi(%2), %0\n"
 278	"\n662:	or		%1, %%ulo(%2), %1\n"
 279	"	or		%0, %%lo(%2), %0\n"
 280	"\n663:	sllx		%1, 32, %1\n"
 281	"	or		%0, %1, %0\n"
 282	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 283	"	.word		661b\n"
 284	"	sethi		%%uhi(%3), %1\n"
 285	"	sethi		%%hi(%3), %0\n"
 286	"	.word		662b\n"
 287	"	or		%1, %%ulo(%3), %1\n"
 288	"	or		%0, %%lo(%3), %0\n"
 289	"	.word		663b\n"
 290	"	sllx		%1, 32, %1\n"
 291	"	or		%0, %1, %0\n"
 292	"	.previous\n"
 293	"	.section	.sun_m7_2insn_patch, \"ax\"\n"
 294	"	.word		661b\n"
 295	"	sethi		%%uhi(%4), %1\n"
 296	"	sethi		%%hi(%4), %0\n"
 297	"	.word		662b\n"
 298	"	or		%1, %%ulo(%4), %1\n"
 299	"	or		%0, %%lo(%4), %0\n"
 300	"	.word		663b\n"
 301	"	sllx		%1, 32, %1\n"
 302	"	or		%0, %1, %0\n"
 303	"	.previous\n"
 304	: "=r" (mask), "=r" (tmp)
 305	: "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U |
 306	       _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U |
 307	       _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U),
 308	  "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
 309	       _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V |
 310	       _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V),
 311	  "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
 312	       _PAGE_CP_4V | _PAGE_E_4V |
 313	       _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V));
 314
 315	return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask));
 316}
 317
 318#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 319static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
 320{
 321	pte_t pte = __pte(pmd_val(pmd));
 322
 323	pte = pte_modify(pte, newprot);
 324
 325	return __pmd(pte_val(pte));
 326}
 327#endif
 328
 329static inline pgprot_t pgprot_noncached(pgprot_t prot)
 330{
 331	unsigned long val = pgprot_val(prot);
 332
 333	__asm__ __volatile__(
 334	"\n661:	andn		%0, %2, %0\n"
 335	"	or		%0, %3, %0\n"
 336	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 337	"	.word		661b\n"
 338	"	andn		%0, %4, %0\n"
 339	"	or		%0, %5, %0\n"
 340	"	.previous\n"
 341	"	.section	.sun_m7_2insn_patch, \"ax\"\n"
 342	"	.word		661b\n"
 343	"	andn		%0, %6, %0\n"
 344	"	or		%0, %5, %0\n"
 345	"	.previous\n"
 346	: "=r" (val)
 347	: "0" (val), "i" (_PAGE_CP_4U | _PAGE_CV_4U), "i" (_PAGE_E_4U),
 348	             "i" (_PAGE_CP_4V | _PAGE_CV_4V), "i" (_PAGE_E_4V),
 349	             "i" (_PAGE_CP_4V));
 350
 351	return __pgprot(val);
 352}
 353/* Various pieces of code check for platform support by ifdef testing
 354 * on "pgprot_noncached".  That's broken and should be fixed, but for
 355 * now...
 356 */
 357#define pgprot_noncached pgprot_noncached
 358
 359static inline unsigned long pte_dirty(pte_t pte)
 360{
 361	unsigned long mask;
 362
 363	__asm__ __volatile__(
 364	"\n661:	mov		%1, %0\n"
 365	"	nop\n"
 366	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 367	"	.word		661b\n"
 368	"	sethi		%%uhi(%2), %0\n"
 369	"	sllx		%0, 32, %0\n"
 370	"	.previous\n"
 371	: "=r" (mask)
 372	: "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V));
 373
 374	return (pte_val(pte) & mask);
 375}
 376
 377static inline unsigned long pte_write(pte_t pte)
 378{
 379	unsigned long mask;
 380
 381	__asm__ __volatile__(
 382	"\n661:	mov		%1, %0\n"
 383	"	nop\n"
 384	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 385	"	.word		661b\n"
 386	"	sethi		%%uhi(%2), %0\n"
 387	"	sllx		%0, 32, %0\n"
 388	"	.previous\n"
 389	: "=r" (mask)
 390	: "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V));
 391
 392	return (pte_val(pte) & mask);
 393}
 394
 395#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 396pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags);
 397#define arch_make_huge_pte arch_make_huge_pte
 398static inline unsigned long __pte_default_huge_mask(void)
 399{
 400	unsigned long mask;
 401
 402	__asm__ __volatile__(
 403	"\n661:	sethi		%%uhi(%1), %0\n"
 404	"	sllx		%0, 32, %0\n"
 405	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 406	"	.word		661b\n"
 407	"	mov		%2, %0\n"
 408	"	nop\n"
 409	"	.previous\n"
 410	: "=r" (mask)
 411	: "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V));
 412
 413	return mask;
 414}
 415
 416static inline pte_t pte_mkhuge(pte_t pte)
 417{
 418	return __pte(pte_val(pte) | __pte_default_huge_mask());
 419}
 420
 421static inline bool is_default_hugetlb_pte(pte_t pte)
 422{
 423	unsigned long mask = __pte_default_huge_mask();
 424
 425	return (pte_val(pte) & mask) == mask;
 426}
 427
 428static inline bool is_hugetlb_pmd(pmd_t pmd)
 429{
 430	return !!(pmd_val(pmd) & _PAGE_PMD_HUGE);
 431}
 432
 433static inline bool is_hugetlb_pud(pud_t pud)
 434{
 435	return !!(pud_val(pud) & _PAGE_PUD_HUGE);
 436}
 437
 438#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 439static inline pmd_t pmd_mkhuge(pmd_t pmd)
 440{
 441	pte_t pte = __pte(pmd_val(pmd));
 442
 443	pte = pte_mkhuge(pte);
 444	pte_val(pte) |= _PAGE_PMD_HUGE;
 445
 446	return __pmd(pte_val(pte));
 447}
 448#endif
 449#else
 450static inline bool is_hugetlb_pte(pte_t pte)
 451{
 452	return false;
 453}
 454#endif
 455
 456static inline pte_t __pte_mkhwwrite(pte_t pte)
 457{
 458	unsigned long val = pte_val(pte);
 459
 460	/*
 461	 * Note: we only want to set the HW writable bit if the SW writable bit
 462	 * and the SW dirty bit are set.
 463	 */
 464	__asm__ __volatile__(
 465	"\n661:	or		%0, %2, %0\n"
 466	"	.section	.sun4v_1insn_patch, \"ax\"\n"
 467	"	.word		661b\n"
 468	"	or		%0, %3, %0\n"
 469	"	.previous\n"
 470	: "=r" (val)
 471	: "0" (val), "i" (_PAGE_W_4U), "i" (_PAGE_W_4V));
 472
 473	return __pte(val);
 474}
 475
 476static inline pte_t pte_mkdirty(pte_t pte)
 477{
 478	unsigned long val = pte_val(pte), mask;
 479
 480	__asm__ __volatile__(
 481	"\n661:	mov		%1, %0\n"
 482	"	nop\n"
 483	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 484	"	.word		661b\n"
 485	"	sethi		%%uhi(%2), %0\n"
 486	"	sllx		%0, 32, %0\n"
 487	"	.previous\n"
 488	: "=r" (mask)
 489	: "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V));
 490
 491	pte = __pte(val | mask);
 492	return pte_write(pte) ? __pte_mkhwwrite(pte) : pte;
 493}
 494
 495static inline pte_t pte_mkclean(pte_t pte)
 496{
 497	unsigned long val = pte_val(pte), tmp;
 498
 499	__asm__ __volatile__(
 500	"\n661:	andn		%0, %3, %0\n"
 501	"	nop\n"
 502	"\n662:	nop\n"
 503	"	nop\n"
 504	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 505	"	.word		661b\n"
 506	"	sethi		%%uhi(%4), %1\n"
 507	"	sllx		%1, 32, %1\n"
 508	"	.word		662b\n"
 509	"	or		%1, %%lo(%4), %1\n"
 510	"	andn		%0, %1, %0\n"
 511	"	.previous\n"
 512	: "=r" (val), "=r" (tmp)
 513	: "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U),
 514	  "i" (_PAGE_MODIFIED_4V | _PAGE_W_4V));
 515
 516	return __pte(val);
 517}
 518
 519static inline pte_t pte_mkwrite_novma(pte_t pte)
 520{
 521	unsigned long val = pte_val(pte), mask;
 522
 523	__asm__ __volatile__(
 524	"\n661:	mov		%1, %0\n"
 525	"	nop\n"
 526	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 527	"	.word		661b\n"
 528	"	sethi		%%uhi(%2), %0\n"
 529	"	sllx		%0, 32, %0\n"
 530	"	.previous\n"
 531	: "=r" (mask)
 532	: "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V));
 533
 534	pte = __pte(val | mask);
 535	return pte_dirty(pte) ? __pte_mkhwwrite(pte) : pte;
 536}
 537
 538static inline pte_t pte_wrprotect(pte_t pte)
 539{
 540	unsigned long val = pte_val(pte), tmp;
 541
 542	__asm__ __volatile__(
 543	"\n661:	andn		%0, %3, %0\n"
 544	"	nop\n"
 545	"\n662:	nop\n"
 546	"	nop\n"
 547	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 548	"	.word		661b\n"
 549	"	sethi		%%uhi(%4), %1\n"
 550	"	sllx		%1, 32, %1\n"
 551	"	.word		662b\n"
 552	"	or		%1, %%lo(%4), %1\n"
 553	"	andn		%0, %1, %0\n"
 554	"	.previous\n"
 555	: "=r" (val), "=r" (tmp)
 556	: "0" (val), "i" (_PAGE_WRITE_4U | _PAGE_W_4U),
 557	  "i" (_PAGE_WRITE_4V | _PAGE_W_4V));
 558
 559	return __pte(val);
 560}
 561
 562static inline pte_t pte_mkold(pte_t pte)
 563{
 564	unsigned long mask;
 565
 566	__asm__ __volatile__(
 567	"\n661:	mov		%1, %0\n"
 568	"	nop\n"
 569	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 570	"	.word		661b\n"
 571	"	sethi		%%uhi(%2), %0\n"
 572	"	sllx		%0, 32, %0\n"
 573	"	.previous\n"
 574	: "=r" (mask)
 575	: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
 576
 577	mask |= _PAGE_R;
 578
 579	return __pte(pte_val(pte) & ~mask);
 580}
 581
 582static inline pte_t pte_mkyoung(pte_t pte)
 583{
 584	unsigned long mask;
 585
 586	__asm__ __volatile__(
 587	"\n661:	mov		%1, %0\n"
 588	"	nop\n"
 589	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 590	"	.word		661b\n"
 591	"	sethi		%%uhi(%2), %0\n"
 592	"	sllx		%0, 32, %0\n"
 593	"	.previous\n"
 594	: "=r" (mask)
 595	: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
 596
 597	mask |= _PAGE_R;
 598
 599	return __pte(pte_val(pte) | mask);
 600}
 601
 602static inline pte_t pte_mkspecial(pte_t pte)
 603{
 604	pte_val(pte) |= _PAGE_SPECIAL;
 605	return pte;
 606}
 607
 608static inline pte_t pte_mkmcd(pte_t pte)
 609{
 610	pte_val(pte) |= _PAGE_MCD_4V;
 611	return pte;
 612}
 613
 614static inline pte_t pte_mknotmcd(pte_t pte)
 615{
 616	pte_val(pte) &= ~_PAGE_MCD_4V;
 617	return pte;
 618}
 619
 620static inline unsigned long pte_young(pte_t pte)
 621{
 622	unsigned long mask;
 623
 624	__asm__ __volatile__(
 625	"\n661:	mov		%1, %0\n"
 626	"	nop\n"
 627	"	.section	.sun4v_2insn_patch, \"ax\"\n"
 628	"	.word		661b\n"
 629	"	sethi		%%uhi(%2), %0\n"
 630	"	sllx		%0, 32, %0\n"
 631	"	.previous\n"
 632	: "=r" (mask)
 633	: "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V));
 634
 635	return (pte_val(pte) & mask);
 636}
 637
 638static inline unsigned long pte_exec(pte_t pte)
 639{
 640	unsigned long mask;
 641
 642	__asm__ __volatile__(
 643	"\n661:	sethi		%%hi(%1), %0\n"
 644	"	.section	.sun4v_1insn_patch, \"ax\"\n"
 645	"	.word		661b\n"
 646	"	mov		%2, %0\n"
 647	"	.previous\n"
 648	: "=r" (mask)
 649	: "i" (_PAGE_EXEC_4U), "i" (_PAGE_EXEC_4V));
 650
 651	return (pte_val(pte) & mask);
 652}
 653
 654static inline unsigned long pte_present(pte_t pte)
 655{
 656	unsigned long val = pte_val(pte);
 657
 658	__asm__ __volatile__(
 659	"\n661:	and		%0, %2, %0\n"
 660	"	.section	.sun4v_1insn_patch, \"ax\"\n"
 661	"	.word		661b\n"
 662	"	and		%0, %3, %0\n"
 663	"	.previous\n"
 664	: "=r" (val)
 665	: "0" (val), "i" (_PAGE_PRESENT_4U), "i" (_PAGE_PRESENT_4V));
 666
 667	return val;
 668}
 669
 670#define pte_accessible pte_accessible
 671static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a)
 672{
 673	return pte_val(a) & _PAGE_VALID;
 674}
 675
 676static inline unsigned long pte_special(pte_t pte)
 677{
 678	return pte_val(pte) & _PAGE_SPECIAL;
 679}
 680
 681#define pmd_leaf pmd_leaf
 682static inline bool pmd_leaf(pmd_t pmd)
 683{
 684	pte_t pte = __pte(pmd_val(pmd));
 685
 686	return pte_val(pte) & _PAGE_PMD_HUGE;
 687}
 688
 689static inline unsigned long pmd_pfn(pmd_t pmd)
 690{
 691	pte_t pte = __pte(pmd_val(pmd));
 692
 693	return pte_pfn(pte);
 694}
 695
 696#define pmd_write pmd_write
 697static inline unsigned long pmd_write(pmd_t pmd)
 698{
 699	pte_t pte = __pte(pmd_val(pmd));
 700
 701	return pte_write(pte);
 702}
 703
 704#define pud_write(pud)	pte_write(__pte(pud_val(pud)))
 705
 706#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 707#define pmd_dirty pmd_dirty
 708static inline unsigned long pmd_dirty(pmd_t pmd)
 709{
 710	pte_t pte = __pte(pmd_val(pmd));
 711
 712	return pte_dirty(pte);
 713}
 714
 715#define pmd_young pmd_young
 716static inline unsigned long pmd_young(pmd_t pmd)
 717{
 718	pte_t pte = __pte(pmd_val(pmd));
 719
 720	return pte_young(pte);
 721}
 722
 723static inline unsigned long pmd_trans_huge(pmd_t pmd)
 724{
 725	pte_t pte = __pte(pmd_val(pmd));
 726
 727	return pte_val(pte) & _PAGE_PMD_HUGE;
 728}
 729
 730static inline pmd_t pmd_mkold(pmd_t pmd)
 731{
 732	pte_t pte = __pte(pmd_val(pmd));
 733
 734	pte = pte_mkold(pte);
 735
 736	return __pmd(pte_val(pte));
 737}
 738
 739static inline pmd_t pmd_wrprotect(pmd_t pmd)
 740{
 741	pte_t pte = __pte(pmd_val(pmd));
 742
 743	pte = pte_wrprotect(pte);
 744
 745	return __pmd(pte_val(pte));
 746}
 747
 748static inline pmd_t pmd_mkdirty(pmd_t pmd)
 749{
 750	pte_t pte = __pte(pmd_val(pmd));
 751
 752	pte = pte_mkdirty(pte);
 753
 754	return __pmd(pte_val(pte));
 755}
 756
 757static inline pmd_t pmd_mkclean(pmd_t pmd)
 758{
 759	pte_t pte = __pte(pmd_val(pmd));
 760
 761	pte = pte_mkclean(pte);
 762
 763	return __pmd(pte_val(pte));
 764}
 765
 766static inline pmd_t pmd_mkyoung(pmd_t pmd)
 767{
 768	pte_t pte = __pte(pmd_val(pmd));
 769
 770	pte = pte_mkyoung(pte);
 771
 772	return __pmd(pte_val(pte));
 773}
 774
 775static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
 776{
 777	pte_t pte = __pte(pmd_val(pmd));
 778
 779	pte = pte_mkwrite_novma(pte);
 780
 781	return __pmd(pte_val(pte));
 782}
 783
 784#define pmd_pgprot pmd_pgprot
 785static inline pgprot_t pmd_pgprot(pmd_t entry)
 786{
 787	unsigned long val = pmd_val(entry);
 788
 789	return __pgprot(val);
 790}
 791#endif
 792
 793static inline int pmd_present(pmd_t pmd)
 794{
 795	return pmd_val(pmd) != 0UL;
 796}
 797
 798#define pmd_none(pmd)			(!pmd_val(pmd))
 799
 800/* pmd_bad() is only called on non-trans-huge PMDs.  Our encoding is
 801 * very simple, it's just the physical address.  PTE tables are of
 802 * size PAGE_SIZE so make sure the sub-PAGE_SIZE bits are clear and
 803 * the top bits outside of the range of any physical address size we
 804 * support are clear as well.  We also validate the physical itself.
 805 */
 806#define pmd_bad(pmd)			(pmd_val(pmd) & ~PAGE_MASK)
 807
 808#define pud_none(pud)			(!pud_val(pud))
 809
 810#define pud_bad(pud)			(pud_val(pud) & ~PAGE_MASK)
 811
 812#define p4d_none(p4d)			(!p4d_val(p4d))
 813
 814#define p4d_bad(p4d)			(p4d_val(p4d) & ~PAGE_MASK)
 815
 816#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 817void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 818		pmd_t *pmdp, pmd_t pmd);
 819#else
 820static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
 821			      pmd_t *pmdp, pmd_t pmd)
 822{
 823	*pmdp = pmd;
 824}
 825#endif
 826
 827static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
 828{
 829	unsigned long val = __pa((unsigned long) (ptep));
 830
 831	pmd_val(*pmdp) = val;
 832}
 833
 834#define pud_set(pudp, pmdp)	\
 835	(pud_val(*(pudp)) = (__pa((unsigned long) (pmdp))))
 836static inline unsigned long pmd_page_vaddr(pmd_t pmd)
 837{
 838	pte_t pte = __pte(pmd_val(pmd));
 839	unsigned long pfn;
 840
 841	pfn = pte_pfn(pte);
 842
 843	return ((unsigned long) __va(pfn << PAGE_SHIFT));
 844}
 845
 846static inline pmd_t *pud_pgtable(pud_t pud)
 847{
 848	pte_t pte = __pte(pud_val(pud));
 849	unsigned long pfn;
 850
 851	pfn = pte_pfn(pte);
 852
 853	return ((pmd_t *) __va(pfn << PAGE_SHIFT));
 854}
 855
 856#define pmd_page(pmd) 			virt_to_page((void *)pmd_page_vaddr(pmd))
 857#define pud_page(pud)			virt_to_page((void *)pud_pgtable(pud))
 858#define pmd_clear(pmdp)			(pmd_val(*(pmdp)) = 0UL)
 859#define pud_present(pud)		(pud_val(pud) != 0U)
 860#define pud_clear(pudp)			(pud_val(*(pudp)) = 0UL)
 861#define p4d_pgtable(p4d)		\
 862	((pud_t *) __va(p4d_val(p4d)))
 863#define p4d_present(p4d)		(p4d_val(p4d) != 0U)
 864#define p4d_clear(p4dp)			(p4d_val(*(p4dp)) = 0UL)
 865
 866/* only used by the stubbed out hugetlb gup code, should never be called */
 867#define p4d_page(p4d)			NULL
 868
 869#define pud_leaf pud_leaf
 870static inline bool pud_leaf(pud_t pud)
 871{
 872	pte_t pte = __pte(pud_val(pud));
 873
 874	return pte_val(pte) & _PAGE_PMD_HUGE;
 875}
 876
 877#define pud_pfn pud_pfn
 878static inline unsigned long pud_pfn(pud_t pud)
 879{
 880	pte_t pte = __pte(pud_val(pud));
 881
 882	return pte_pfn(pte);
 883}
 884
 885/* Same in both SUN4V and SUN4U.  */
 886#define pte_none(pte) 			(!pte_val(pte))
 887
 888#define p4d_set(p4dp, pudp)	\
 889	(p4d_val(*(p4dp)) = (__pa((unsigned long) (pudp))))
 890
 891/* We cannot include <linux/mm_types.h> at this point yet: */
 892extern struct mm_struct init_mm;
 893
 894/* Actual page table PTE updates.  */
 895void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
 896		   pte_t *ptep, pte_t orig, int fullmm,
 897		   unsigned int hugepage_shift);
 898
 899static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
 900				pte_t *ptep, pte_t orig, int fullmm,
 901				unsigned int hugepage_shift)
 902{
 903	/* It is more efficient to let flush_tlb_kernel_range()
 904	 * handle init_mm tlb flushes.
 905	 *
 906	 * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
 907	 *             and SUN4V pte layout, so this inline test is fine.
 908	 */
 909	if (likely(mm != &init_mm) && pte_accessible(mm, orig))
 910		tlb_batch_add(mm, vaddr, ptep, orig, fullmm, hugepage_shift);
 911}
 912
 913#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
 914static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
 915					    unsigned long addr,
 916					    pmd_t *pmdp)
 917{
 918	pmd_t pmd = *pmdp;
 919	set_pmd_at(mm, addr, pmdp, __pmd(0UL));
 920	return pmd;
 921}
 922
 923static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
 924			     pte_t *ptep, pte_t pte, int fullmm)
 925{
 926	pte_t orig = *ptep;
 927
 928	*ptep = pte;
 929	maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT);
 930}
 931
 932#define PFN_PTE_SHIFT		PAGE_SHIFT
 933
 934static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
 935		pte_t *ptep, pte_t pte, unsigned int nr)
 936{
 937	for (;;) {
 938		__set_pte_at(mm, addr, ptep, pte, 0);
 939		if (--nr == 0)
 940			break;
 941		ptep++;
 942		pte_val(pte) += PAGE_SIZE;
 943		addr += PAGE_SIZE;
 944	}
 945}
 946#define set_ptes set_ptes
 947
 948#define pte_clear(mm,addr,ptep)		\
 949	set_pte_at((mm), (addr), (ptep), __pte(0UL))
 950
 951#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL
 952#define pte_clear_not_present_full(mm,addr,ptep,fullmm)	\
 953	__set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm))
 954
 955#ifdef DCACHE_ALIASING_POSSIBLE
 956#define __HAVE_ARCH_MOVE_PTE
 957#define move_pte(pte, old_addr, new_addr)				\
 958({									\
 959	pte_t newpte = (pte);						\
 960	if (tlb_type != hypervisor && pte_present(pte)) {		\
 961		unsigned long this_pfn = pte_pfn(pte);			\
 962									\
 963		if (pfn_valid(this_pfn) &&				\
 964		    (((old_addr) ^ (new_addr)) & (1 << 13)))		\
 965			flush_dcache_folio_all(current->mm,		\
 966				page_folio(pfn_to_page(this_pfn)));	\
 967	}								\
 968	newpte;								\
 969})
 970#endif
 971
 972extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 973
 974void paging_init(void);
 975unsigned long find_ecache_flush_span(unsigned long size);
 976
 977struct seq_file;
 978void mmu_info(struct seq_file *);
 979
 980struct vm_area_struct;
 981void update_mmu_cache_range(struct vm_fault *, struct vm_area_struct *,
 982		unsigned long addr, pte_t *ptep, unsigned int nr);
 983#define update_mmu_cache(vma, addr, ptep) \
 984	update_mmu_cache_range(NULL, vma, addr, ptep, 1)
 985#ifdef CONFIG_TRANSPARENT_HUGEPAGE
 986void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
 987			  pmd_t *pmd);
 988
 989#define __HAVE_ARCH_PMDP_INVALIDATE
 990extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
 991			    pmd_t *pmdp);
 992
 993#define __HAVE_ARCH_PGTABLE_DEPOSIT
 994void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
 995				pgtable_t pgtable);
 996
 997#define __HAVE_ARCH_PGTABLE_WITHDRAW
 998pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp);
 999#endif
1000
1001/*
1002 * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that
1003 * are !pte_none() && !pte_present().
1004 *
1005 * Format of swap PTEs:
1006 *
1007 *   6 6 6 6 5 5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3
1008 *   3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2
1009 *   <--------------------------- offset ---------------------------
1010 *
1011 *   3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
1012 *   1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
1013 *   --------------------> E <-- type ---> <------- zeroes -------->
1014 */
1015#define __swp_type(entry)	(((entry).val >> PAGE_SHIFT) & 0x7fUL)
1016#define __swp_offset(entry)	((entry).val >> (PAGE_SHIFT + 8UL))
1017#define __swp_entry(type, offset)	\
1018	( (swp_entry_t) \
1019	  { \
1020		((((long)(type) & 0x7fUL) << PAGE_SHIFT) | \
1021                 ((long)(offset) << (PAGE_SHIFT + 8UL))) \
1022	  } )
1023#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val(pte) })
1024#define __swp_entry_to_pte(x)		((pte_t) { (x).val })
1025
1026static inline bool pte_swp_exclusive(pte_t pte)
1027{
1028	return pte_val(pte) & _PAGE_SWP_EXCLUSIVE;
1029}
1030
1031static inline pte_t pte_swp_mkexclusive(pte_t pte)
1032{
1033	return __pte(pte_val(pte) | _PAGE_SWP_EXCLUSIVE);
1034}
1035
1036static inline pte_t pte_swp_clear_exclusive(pte_t pte)
1037{
1038	return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE);
1039}
1040
1041int page_in_phys_avail(unsigned long paddr);
1042
1043/*
1044 * For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in
1045 * its high 4 bits.  These macros/functions put it there or get it from there.
1046 */
1047#define MK_IOSPACE_PFN(space, pfn)	(pfn | (space << (BITS_PER_LONG - 4)))
1048#define GET_IOSPACE(pfn)		(pfn >> (BITS_PER_LONG - 4))
1049#define GET_PFN(pfn)			(pfn & 0x0fffffffffffffffUL)
1050
1051void adi_restore_tags(struct mm_struct *mm, struct vm_area_struct *vma,
1052		      unsigned long addr, pte_t pte);
1053
1054int adi_save_tags(struct mm_struct *mm, struct vm_area_struct *vma,
1055		  unsigned long addr, pte_t oldpte);
1056
1057#define __HAVE_ARCH_DO_SWAP_PAGE
1058static inline void arch_do_swap_page(struct mm_struct *mm,
1059				     struct vm_area_struct *vma,
1060				     unsigned long addr,
1061				     pte_t pte, pte_t oldpte)
1062{
1063	/* If this is a new page being mapped in, there can be no
1064	 * ADI tags stored away for this page. Skip looking for
1065	 * stored tags
1066	 */
1067	if (pte_none(oldpte))
1068		return;
1069
1070	if (adi_state.enabled && (pte_val(pte) & _PAGE_MCD_4V))
1071		adi_restore_tags(mm, vma, addr, pte);
1072}
1073
1074#define __HAVE_ARCH_UNMAP_ONE
1075static inline int arch_unmap_one(struct mm_struct *mm,
1076				 struct vm_area_struct *vma,
1077				 unsigned long addr, pte_t oldpte)
1078{
1079	if (adi_state.enabled && (pte_val(oldpte) & _PAGE_MCD_4V))
1080		return adi_save_tags(mm, vma, addr, oldpte);
1081	return 0;
1082}
1083
1084static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn,
1085		unsigned long size)
1086{
1087	unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT;
1088	int space = GET_IOSPACE(pfn);
1089	unsigned long phys_base;
1090
1091	phys_base = offset | (((unsigned long) space) << 32UL);
1092
1093	return phys_base >> PAGE_SHIFT;
1094}
1095#define io_remap_pfn_range_pfn io_remap_pfn_range_pfn
1096
1097static inline unsigned long __untagged_addr(unsigned long start)
1098{
1099	if (adi_capable()) {
1100		long addr = start;
1101
1102		/* If userspace has passed a versioned address, kernel
1103		 * will not find it in the VMAs since it does not store
1104		 * the version tags in the list of VMAs. Storing version
1105		 * tags in list of VMAs is impractical since they can be
1106		 * changed any time from userspace without dropping into
1107		 * kernel. Any address search in VMAs will be done with
1108		 * non-versioned addresses. Ensure the ADI version bits
1109		 * are dropped here by sign extending the last bit before
1110		 * ADI bits. IOMMU does not implement version tags.
1111		 */
1112		return (addr << (long)adi_nbits()) >> (long)adi_nbits();
1113	}
1114
1115	return start;
1116}
1117#define untagged_addr(addr) \
1118	((__typeof__(addr))(__untagged_addr((unsigned long)(addr))))
1119
1120static inline bool pte_access_permitted(pte_t pte, bool write)
1121{
1122	u64 prot;
1123
1124	if (tlb_type == hypervisor) {
1125		prot = _PAGE_PRESENT_4V | _PAGE_P_4V;
1126		if (write)
1127			prot |= _PAGE_WRITE_4V;
1128	} else {
1129		prot = _PAGE_PRESENT_4U | _PAGE_P_4U;
1130		if (write)
1131			prot |= _PAGE_WRITE_4U;
1132	}
1133
1134	return (pte_val(pte) & (prot | _PAGE_SPECIAL)) == prot;
1135}
1136#define pte_access_permitted pte_access_permitted
1137
1138/* We provide our own get_unmapped_area to cope with VA holes and
1139 * SHM area cache aliasing for userland.
1140 */
1141#define HAVE_ARCH_UNMAPPED_AREA
1142#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1143
1144/* We provide a special get_unmapped_area for framebuffer mmaps to try and use
1145 * the largest alignment possible such that larget PTEs can be used.
1146 */
1147unsigned long get_fb_unmapped_area(struct file *filp, unsigned long,
1148				   unsigned long, unsigned long,
1149				   unsigned long);
1150#define HAVE_ARCH_FB_UNMAPPED_AREA
1151
1152void sun4v_register_fault_status(void);
1153void sun4v_ktsb_register(void);
1154void __init cheetah_ecache_flush_init(void);
1155void sun4v_patch_tlb_handlers(void);
1156
1157extern unsigned long cmdline_memory_size;
1158
1159asmlinkage void do_sparc64_fault(struct pt_regs *regs);
1160
1161#define pmd_pgtable(PMD)	((pte_t *)pmd_page_vaddr(PMD))
1162
1163#ifdef CONFIG_HUGETLB_PAGE
1164
1165#define pud_leaf_size pud_leaf_size
1166extern unsigned long pud_leaf_size(pud_t pud);
1167
1168#define pmd_leaf_size pmd_leaf_size
1169extern unsigned long pmd_leaf_size(pmd_t pmd);
1170
1171#define pte_leaf_size pte_leaf_size
1172extern unsigned long pte_leaf_size(pte_t pte);
1173
1174#endif /* CONFIG_HUGETLB_PAGE */
1175
1176#endif /* !(__ASSEMBLER__) */
1177
1178#endif /* !(_SPARC64_PGTABLE_H) */