pgtable.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  4 */
  5#include <linux/init.h>
  6#include <linux/export.h>
  7#include <linux/mm.h>
  8#include <asm/pgalloc.h>
  9#include <asm/pgtable.h>
 10#include <asm/tlbflush.h>
 11
 12struct page *dmw_virt_to_page(unsigned long kaddr)
 13{
 14	return phys_to_page(__pa(kaddr));
 15}
 16EXPORT_SYMBOL(dmw_virt_to_page);
 17
 18struct page *tlb_virt_to_page(unsigned long kaddr)
 19{
 20	return phys_to_page(pfn_to_phys(pte_pfn(*virt_to_kpte(kaddr))));
 21}
 22EXPORT_SYMBOL(tlb_virt_to_page);
 23
 24pgd_t *pgd_alloc(struct mm_struct *mm)
 25{
 26	pgd_t *init, *ret;
 27
 28	ret = __pgd_alloc(mm, 0);
 29	if (ret) {
 30		init = pgd_offset(&init_mm, 0UL);
 31		pgd_init(ret);
 32		memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
 33		       (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
 34	}
 35
 36	return ret;
 37}
 38EXPORT_SYMBOL_GPL(pgd_alloc);
 39
 40void pgd_init(void *addr)
 41{
 42	unsigned long *p, *end;
 43	unsigned long entry;
 44
 45#if !defined(__PAGETABLE_PUD_FOLDED)
 46	entry = (unsigned long)invalid_pud_table;
 47#elif !defined(__PAGETABLE_PMD_FOLDED)
 48	entry = (unsigned long)invalid_pmd_table;
 49#else
 50	entry = (unsigned long)invalid_pte_table;
 51#endif
 52
 53	p = (unsigned long *)addr;
 54	end = p + PTRS_PER_PGD;
 55
 56	do {
 57		p[0] = entry;
 58		p[1] = entry;
 59		p[2] = entry;
 60		p[3] = entry;
 61		p[4] = entry;
 62		p += 8;
 63		p[-3] = entry;
 64		p[-2] = entry;
 65		p[-1] = entry;
 66	} while (p != end);
 67}
 68EXPORT_SYMBOL_GPL(pgd_init);
 69
 70#ifndef __PAGETABLE_PMD_FOLDED
 71void pmd_init(void *addr)
 72{
 73	unsigned long *p, *end;
 74	unsigned long pagetable = (unsigned long)invalid_pte_table;
 75
 76	p = (unsigned long *)addr;
 77	end = p + PTRS_PER_PMD;
 78
 79	do {
 80		p[0] = pagetable;
 81		p[1] = pagetable;
 82		p[2] = pagetable;
 83		p[3] = pagetable;
 84		p[4] = pagetable;
 85		p += 8;
 86		p[-3] = pagetable;
 87		p[-2] = pagetable;
 88		p[-1] = pagetable;
 89	} while (p != end);
 90}
 91EXPORT_SYMBOL_GPL(pmd_init);
 92#endif
 93
 94#ifndef __PAGETABLE_PUD_FOLDED
 95void pud_init(void *addr)
 96{
 97	unsigned long *p, *end;
 98	unsigned long pagetable = (unsigned long)invalid_pmd_table;
 99
100	p = (unsigned long *)addr;
101	end = p + PTRS_PER_PUD;
102
103	do {
104		p[0] = pagetable;
105		p[1] = pagetable;
106		p[2] = pagetable;
107		p[3] = pagetable;
108		p[4] = pagetable;
109		p += 8;
110		p[-3] = pagetable;
111		p[-2] = pagetable;
112		p[-1] = pagetable;
113	} while (p != end);
114}
115EXPORT_SYMBOL_GPL(pud_init);
116#endif
117
118void kernel_pte_init(void *addr)
119{
120	unsigned long *p, *end;
121
122	p = (unsigned long *)addr;
123	end = p + PTRS_PER_PTE;
124
125	do {
126		p[0] = _PAGE_GLOBAL;
127		p[1] = _PAGE_GLOBAL;
128		p[2] = _PAGE_GLOBAL;
129		p[3] = _PAGE_GLOBAL;
130		p[4] = _PAGE_GLOBAL;
131		p += 8;
132		p[-3] = _PAGE_GLOBAL;
133		p[-2] = _PAGE_GLOBAL;
134		p[-1] = _PAGE_GLOBAL;
135	} while (p != end);
136}
137
138void set_pmd_at(struct mm_struct *mm, unsigned long addr,
139		pmd_t *pmdp, pmd_t pmd)
140{
141	WRITE_ONCE(*pmdp, pmd);
142	flush_tlb_all();
143}
144
145void __init pagetable_init(void)
146{
147	/* Initialize the entire pgd.  */
148	pgd_init(swapper_pg_dir);
149	pgd_init(invalid_pg_dir);
150#ifndef __PAGETABLE_PUD_FOLDED
151	pud_init(invalid_pud_table);
152#endif
153#ifndef __PAGETABLE_PMD_FOLDED
154	pmd_init(invalid_pmd_table);
155#endif
156}