pgtable-64.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (C) 2012 Regents of the University of California
  4 */
  5
  6#ifndef _ASM_RISCV_PGTABLE_64_H
  7#define _ASM_RISCV_PGTABLE_64_H
  8
  9#include <linux/bits.h>
 10#include <linux/const.h>
 11#include <asm/errata_list.h>
 12
 13extern bool pgtable_l4_enabled;
 14extern bool pgtable_l5_enabled;
 15
 16#define PGDIR_SHIFT_L3  30
 17#define PGDIR_SHIFT_L4  39
 18#define PGDIR_SHIFT_L5  48
 19#define PGDIR_SHIFT     (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \
 20		(pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3))
 21/* Size of region mapped by a page global directory */
 22#define PGDIR_SIZE      (_AC(1, UL) << PGDIR_SHIFT)
 23#define PGDIR_MASK      (~(PGDIR_SIZE - 1))
 24
 25/* p4d is folded into pgd in case of 4-level page table */
 26#define P4D_SHIFT_L3   30
 27#define P4D_SHIFT_L4   39
 28#define P4D_SHIFT_L5   39
 29#define P4D_SHIFT      (pgtable_l5_enabled ? P4D_SHIFT_L5 : \
 30		(pgtable_l4_enabled ? P4D_SHIFT_L4 : P4D_SHIFT_L3))
 31#define P4D_SIZE       (_AC(1, UL) << P4D_SHIFT)
 32#define P4D_MASK       (~(P4D_SIZE - 1))
 33
 34/* pud is folded into pgd in case of 3-level page table */
 35#define PUD_SHIFT      30
 36#define PUD_SIZE       (_AC(1, UL) << PUD_SHIFT)
 37#define PUD_MASK       (~(PUD_SIZE - 1))
 38
 39#define PMD_SHIFT       21
 40/* Size of region mapped by a page middle directory */
 41#define PMD_SIZE        (_AC(1, UL) << PMD_SHIFT)
 42#define PMD_MASK        (~(PMD_SIZE - 1))
 43
 44/* Page 4th Directory entry */
 45typedef struct {
 46	unsigned long p4d;
 47} p4d_t;
 48
 49#define p4d_val(x)	((x).p4d)
 50#define __p4d(x)	((p4d_t) { (x) })
 51#define PTRS_PER_P4D	(PAGE_SIZE / sizeof(p4d_t))
 52
 53/* Page Upper Directory entry */
 54typedef struct {
 55	unsigned long pud;
 56} pud_t;
 57
 58#define pud_val(x)      ((x).pud)
 59#define __pud(x)        ((pud_t) { (x) })
 60#define PTRS_PER_PUD    (PAGE_SIZE / sizeof(pud_t))
 61
 62/* Page Middle Directory entry */
 63typedef struct {
 64	unsigned long pmd;
 65} pmd_t;
 66
 67#define pmd_val(x)      ((x).pmd)
 68#define __pmd(x)        ((pmd_t) { (x) })
 69
 70#define PTRS_PER_PMD    (PAGE_SIZE / sizeof(pmd_t))
 71
 72#define MAX_POSSIBLE_PHYSMEM_BITS 56
 73
 74/*
 75 * rv64 PTE format:
 76 * | 63 | 62 61 | 60 54 | 53  10 | 9             8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0
 77 *   N      MT     RSV    PFN      reserved for SW   D   A   G   U   X   W   R   V
 78 */
 79#define _PAGE_PFN_MASK  GENMASK(53, 10)
 80
 81/*
 82 * [63] Svnapot definitions:
 83 * 0 Svnapot disabled
 84 * 1 Svnapot enabled
 85 */
 86#define _PAGE_NAPOT_SHIFT	63
 87#define _PAGE_NAPOT		BIT(_PAGE_NAPOT_SHIFT)
 88/*
 89 * Only 64KB (order 4) napot ptes supported.
 90 */
 91#define NAPOT_CONT_ORDER_BASE 4
 92enum napot_cont_order {
 93	NAPOT_CONT64KB_ORDER = NAPOT_CONT_ORDER_BASE,
 94	NAPOT_ORDER_MAX,
 95};
 96
 97#define for_each_napot_order(order)						\
 98	for (order = NAPOT_CONT_ORDER_BASE; order < NAPOT_ORDER_MAX; order++)
 99#define for_each_napot_order_rev(order)						\
100	for (order = NAPOT_ORDER_MAX - 1;					\
101	     order >= NAPOT_CONT_ORDER_BASE; order--)
102#define napot_cont_order(val)	(__builtin_ctzl((val.pte >> _PAGE_PFN_SHIFT) << 1))
103
104#define napot_cont_shift(order)	((order) + PAGE_SHIFT)
105#define napot_cont_size(order)	BIT(napot_cont_shift(order))
106#define napot_cont_mask(order)	(~(napot_cont_size(order) - 1UL))
107#define napot_pte_num(order)	BIT(order)
108
109#ifdef CONFIG_RISCV_ISA_SVNAPOT
110#define HUGE_MAX_HSTATE		(2 + (NAPOT_ORDER_MAX - NAPOT_CONT_ORDER_BASE))
111#else
112#define HUGE_MAX_HSTATE		2
113#endif
114
115/*
116 * [62:61] Svpbmt Memory Type definitions:
117 *
118 *  00 - PMA    Normal Cacheable, No change to implied PMA memory type
119 *  01 - NC     Non-cacheable, idempotent, weakly-ordered Main Memory
120 *  10 - IO     Non-cacheable, non-idempotent, strongly-ordered I/O memory
121 *  11 - Rsvd   Reserved for future standard use
122 */
123#define _PAGE_NOCACHE_SVPBMT	(1UL << 61)
124#define _PAGE_IO_SVPBMT		(1UL << 62)
125#define _PAGE_MTMASK_SVPBMT	(_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT)
126
127/*
128 * [63:59] T-Head Memory Type definitions:
129 * bit[63] SO - Strong Order
130 * bit[62] C - Cacheable
131 * bit[61] B - Bufferable
132 * bit[60] SH - Shareable
133 * bit[59] Sec - Trustable
134 * 00110 - NC   Weakly-ordered, Non-cacheable, Bufferable, Shareable, Non-trustable
135 * 01110 - PMA  Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable
136 * 10010 - IO   Strongly-ordered, Non-cacheable, Non-bufferable, Shareable, Non-trustable
137 */
138#define _PAGE_PMA_THEAD		((1UL << 62) | (1UL << 61) | (1UL << 60))
139#define _PAGE_NOCACHE_THEAD	((1UL << 61) | (1UL << 60))
140#define _PAGE_IO_THEAD		((1UL << 63) | (1UL << 60))
141#define _PAGE_MTMASK_THEAD	(_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59))
142
143static inline u64 riscv_page_mtmask(void)
144{
145	u64 val;
146
147	ALT_SVPBMT(val, _PAGE_MTMASK);
148	return val;
149}
150
151static inline u64 riscv_page_nocache(void)
152{
153	u64 val;
154
155	ALT_SVPBMT(val, _PAGE_NOCACHE);
156	return val;
157}
158
159static inline u64 riscv_page_io(void)
160{
161	u64 val;
162
163	ALT_SVPBMT(val, _PAGE_IO);
164	return val;
165}
166
167#define _PAGE_NOCACHE		riscv_page_nocache()
168#define _PAGE_IO		riscv_page_io()
169#define _PAGE_MTMASK		riscv_page_mtmask()
170
171/* Set of bits to preserve across pte_modify() */
172#define _PAGE_CHG_MASK  (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ |	\
173					  _PAGE_WRITE | _PAGE_EXEC |	\
174					  _PAGE_USER | _PAGE_GLOBAL |	\
175					  _PAGE_MTMASK))
176
177static inline int pud_present(pud_t pud)
178{
179	return (pud_val(pud) & _PAGE_PRESENT);
180}
181
182static inline int pud_none(pud_t pud)
183{
184	return (pud_val(pud) == 0);
185}
186
187static inline int pud_bad(pud_t pud)
188{
189	return !pud_present(pud) || (pud_val(pud) & _PAGE_LEAF);
190}
191
192#define pud_leaf	pud_leaf
193static inline bool pud_leaf(pud_t pud)
194{
195	return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF);
196}
197
198static inline int pud_user(pud_t pud)
199{
200	return pud_val(pud) & _PAGE_USER;
201}
202
203static inline void set_pud(pud_t *pudp, pud_t pud)
204{
205	WRITE_ONCE(*pudp, pud);
206}
207
208static inline void pud_clear(pud_t *pudp)
209{
210	set_pud(pudp, __pud(0));
211}
212
213static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot)
214{
215	return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
216}
217
218static inline unsigned long _pud_pfn(pud_t pud)
219{
220	return __page_val_to_pfn(pud_val(pud));
221}
222
223static inline pmd_t *pud_pgtable(pud_t pud)
224{
225	return (pmd_t *)pfn_to_virt(__page_val_to_pfn(pud_val(pud)));
226}
227
228static inline struct page *pud_page(pud_t pud)
229{
230	return pfn_to_page(__page_val_to_pfn(pud_val(pud)));
231}
232
233#define mm_p4d_folded  mm_p4d_folded
234static inline bool mm_p4d_folded(struct mm_struct *mm)
235{
236	if (pgtable_l5_enabled)
237		return false;
238
239	return true;
240}
241
242#define mm_pud_folded  mm_pud_folded
243static inline bool mm_pud_folded(struct mm_struct *mm)
244{
245	if (pgtable_l4_enabled)
246		return false;
247
248	return true;
249}
250
251#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
252
253static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot)
254{
255	unsigned long prot_val = pgprot_val(prot);
256
257	ALT_THEAD_PMA(prot_val);
258
259	return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val);
260}
261
262static inline unsigned long _pmd_pfn(pmd_t pmd)
263{
264	return __page_val_to_pfn(pmd_val(pmd));
265}
266
267#define pmd_ERROR(e) \
268	pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e))
269
270#define pud_ERROR(e)   \
271	pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e))
272
273#define p4d_ERROR(e)   \
274	pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e))
275
276static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
277{
278	if (pgtable_l4_enabled)
279		WRITE_ONCE(*p4dp, p4d);
280	else
281		set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) });
282}
283
284static inline int p4d_none(p4d_t p4d)
285{
286	if (pgtable_l4_enabled)
287		return (p4d_val(p4d) == 0);
288
289	return 0;
290}
291
292static inline int p4d_present(p4d_t p4d)
293{
294	if (pgtable_l4_enabled)
295		return (p4d_val(p4d) & _PAGE_PRESENT);
296
297	return 1;
298}
299
300static inline int p4d_bad(p4d_t p4d)
301{
302	if (pgtable_l4_enabled)
303		return !p4d_present(p4d);
304
305	return 0;
306}
307
308static inline void p4d_clear(p4d_t *p4d)
309{
310	if (pgtable_l4_enabled)
311		set_p4d(p4d, __p4d(0));
312}
313
314static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot)
315{
316	return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot));
317}
318
319static inline unsigned long _p4d_pfn(p4d_t p4d)
320{
321	return __page_val_to_pfn(p4d_val(p4d));
322}
323
324static inline pud_t *p4d_pgtable(p4d_t p4d)
325{
326	if (pgtable_l4_enabled)
327		return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d)));
328
329	return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) });
330}
331#define p4d_page_vaddr(p4d)	((unsigned long)p4d_pgtable(p4d))
332
333static inline struct page *p4d_page(p4d_t p4d)
334{
335	return pfn_to_page(__page_val_to_pfn(p4d_val(p4d)));
336}
337
338#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
339
340#define pud_offset pud_offset
341pud_t *pud_offset(p4d_t *p4d, unsigned long address);
342
343static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
344{
345	if (pgtable_l5_enabled)
346		WRITE_ONCE(*pgdp, pgd);
347	else
348		set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) });
349}
350
351static inline int pgd_none(pgd_t pgd)
352{
353	if (pgtable_l5_enabled)
354		return (pgd_val(pgd) == 0);
355
356	return 0;
357}
358
359static inline int pgd_present(pgd_t pgd)
360{
361	if (pgtable_l5_enabled)
362		return (pgd_val(pgd) & _PAGE_PRESENT);
363
364	return 1;
365}
366
367static inline int pgd_bad(pgd_t pgd)
368{
369	if (pgtable_l5_enabled)
370		return !pgd_present(pgd);
371
372	return 0;
373}
374
375static inline void pgd_clear(pgd_t *pgd)
376{
377	if (pgtable_l5_enabled)
378		set_pgd(pgd, __pgd(0));
379}
380
381static inline p4d_t *pgd_pgtable(pgd_t pgd)
382{
383	if (pgtable_l5_enabled)
384		return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd)));
385
386	return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) });
387}
388#define pgd_page_vaddr(pgd)	((unsigned long)pgd_pgtable(pgd))
389
390static inline struct page *pgd_page(pgd_t pgd)
391{
392	return pfn_to_page(__page_val_to_pfn(pgd_val(pgd)));
393}
394#define pgd_page(pgd)	pgd_page(pgd)
395
396#define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1))
397
398#define p4d_offset p4d_offset
399p4d_t *p4d_offset(pgd_t *pgd, unsigned long address);
400
401#ifdef CONFIG_TRANSPARENT_HUGEPAGE
402static inline pte_t pmd_pte(pmd_t pmd);
403static inline pte_t pud_pte(pud_t pud);
404#endif
405
406#endif /* _ASM_RISCV_PGTABLE_64_H */