hugetlbpage.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * SPARC64 Huge TLB page support.
  4 *
  5 * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
  6 */
  7
  8#include <linux/fs.h>
  9#include <linux/mm.h>
 10#include <linux/sched/mm.h>
 11#include <linux/hugetlb.h>
 12#include <linux/pagemap.h>
 13#include <linux/sysctl.h>
 14
 15#include <asm/mman.h>
 16#include <asm/pgalloc.h>
 17#include <asm/tlb.h>
 18#include <asm/tlbflush.h>
 19#include <asm/cacheflush.h>
 20#include <asm/mmu_context.h>
 21
 22
 23static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
 24{
 25	unsigned long hugepage_size = _PAGE_SZ4MB_4U;
 26
 27	pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4U;
 28
 29	switch (shift) {
 30	case HPAGE_256MB_SHIFT:
 31		hugepage_size = _PAGE_SZ256MB_4U;
 32		pte_val(entry) |= _PAGE_PMD_HUGE;
 33		break;
 34	case HPAGE_SHIFT:
 35		pte_val(entry) |= _PAGE_PMD_HUGE;
 36		break;
 37	case HPAGE_64K_SHIFT:
 38		hugepage_size = _PAGE_SZ64K_4U;
 39		break;
 40	default:
 41		WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
 42	}
 43
 44	pte_val(entry) = pte_val(entry) | hugepage_size;
 45	return entry;
 46}
 47
 48static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift)
 49{
 50	unsigned long hugepage_size = _PAGE_SZ4MB_4V;
 51
 52	pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V;
 53
 54	switch (shift) {
 55	case HPAGE_16GB_SHIFT:
 56		hugepage_size = _PAGE_SZ16GB_4V;
 57		pte_val(entry) |= _PAGE_PUD_HUGE;
 58		break;
 59	case HPAGE_2GB_SHIFT:
 60		hugepage_size = _PAGE_SZ2GB_4V;
 61		pte_val(entry) |= _PAGE_PMD_HUGE;
 62		break;
 63	case HPAGE_256MB_SHIFT:
 64		hugepage_size = _PAGE_SZ256MB_4V;
 65		pte_val(entry) |= _PAGE_PMD_HUGE;
 66		break;
 67	case HPAGE_SHIFT:
 68		pte_val(entry) |= _PAGE_PMD_HUGE;
 69		break;
 70	case HPAGE_64K_SHIFT:
 71		hugepage_size = _PAGE_SZ64K_4V;
 72		break;
 73	default:
 74		WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift);
 75	}
 76
 77	pte_val(entry) = pte_val(entry) | hugepage_size;
 78	return entry;
 79}
 80
 81static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift)
 82{
 83	if (tlb_type == hypervisor)
 84		return sun4v_hugepage_shift_to_tte(entry, shift);
 85	else
 86		return sun4u_hugepage_shift_to_tte(entry, shift);
 87}
 88
 89pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
 90{
 91	pte_t pte;
 92
 93	entry = pte_mkhuge(entry);
 94	pte = hugepage_shift_to_tte(entry, shift);
 95
 96#ifdef CONFIG_SPARC64
 97	/* If this vma has ADI enabled on it, turn on TTE.mcd
 98	 */
 99	if (flags & VM_SPARC_ADI)
100		return pte_mkmcd(pte);
101	else
102		return pte_mknotmcd(pte);
103#else
104	return pte;
105#endif
106}
107
108static unsigned int sun4v_huge_tte_to_shift(pte_t entry)
109{
110	unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V;
111	unsigned int shift;
112
113	switch (tte_szbits) {
114	case _PAGE_SZ16GB_4V:
115		shift = HPAGE_16GB_SHIFT;
116		break;
117	case _PAGE_SZ2GB_4V:
118		shift = HPAGE_2GB_SHIFT;
119		break;
120	case _PAGE_SZ256MB_4V:
121		shift = HPAGE_256MB_SHIFT;
122		break;
123	case _PAGE_SZ4MB_4V:
124		shift = REAL_HPAGE_SHIFT;
125		break;
126	case _PAGE_SZ64K_4V:
127		shift = HPAGE_64K_SHIFT;
128		break;
129	default:
130		shift = PAGE_SHIFT;
131		break;
132	}
133	return shift;
134}
135
136static unsigned int sun4u_huge_tte_to_shift(pte_t entry)
137{
138	unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U;
139	unsigned int shift;
140
141	switch (tte_szbits) {
142	case _PAGE_SZ256MB_4U:
143		shift = HPAGE_256MB_SHIFT;
144		break;
145	case _PAGE_SZ4MB_4U:
146		shift = REAL_HPAGE_SHIFT;
147		break;
148	case _PAGE_SZ64K_4U:
149		shift = HPAGE_64K_SHIFT;
150		break;
151	default:
152		shift = PAGE_SHIFT;
153		break;
154	}
155	return shift;
156}
157
158static unsigned long tte_to_shift(pte_t entry)
159{
160	if (tlb_type == hypervisor)
161		return sun4v_huge_tte_to_shift(entry);
162
163	return sun4u_huge_tte_to_shift(entry);
164}
165
166static unsigned int huge_tte_to_shift(pte_t entry)
167{
168	unsigned long shift = tte_to_shift(entry);
169
170	if (shift == PAGE_SHIFT)
171		WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n",
172			  pte_val(entry));
173
174	return shift;
175}
176
177static unsigned long huge_tte_to_size(pte_t pte)
178{
179	unsigned long size = 1UL << huge_tte_to_shift(pte);
180
181	if (size == REAL_HPAGE_SIZE)
182		size = HPAGE_SIZE;
183	return size;
184}
185
186unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift(*(pte_t *)&pud); }
187unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&pmd); }
188unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); }
189
190pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
191			unsigned long addr, unsigned long sz)
192{
193	pgd_t *pgd;
194	p4d_t *p4d;
195	pud_t *pud;
196	pmd_t *pmd;
197
198	pgd = pgd_offset(mm, addr);
199	p4d = p4d_offset(pgd, addr);
200	pud = pud_alloc(mm, p4d, addr);
201	if (!pud)
202		return NULL;
203	if (sz >= PUD_SIZE)
204		return (pte_t *)pud;
205	pmd = pmd_alloc(mm, pud, addr);
206	if (!pmd)
207		return NULL;
208	if (sz >= PMD_SIZE)
209		return (pte_t *)pmd;
210	return pte_alloc_huge(mm, pmd, addr);
211}
212
213pte_t *huge_pte_offset(struct mm_struct *mm,
214		       unsigned long addr, unsigned long sz)
215{
216	pgd_t *pgd;
217	p4d_t *p4d;
218	pud_t *pud;
219	pmd_t *pmd;
220
221	pgd = pgd_offset(mm, addr);
222	if (pgd_none(*pgd))
223		return NULL;
224	p4d = p4d_offset(pgd, addr);
225	if (p4d_none(*p4d))
226		return NULL;
227	pud = pud_offset(p4d, addr);
228	if (pud_none(*pud))
229		return NULL;
230	if (is_hugetlb_pud(*pud))
231		return (pte_t *)pud;
232	pmd = pmd_offset(pud, addr);
233	if (pmd_none(*pmd))
234		return NULL;
235	if (is_hugetlb_pmd(*pmd))
236		return (pte_t *)pmd;
237	return pte_offset_huge(pmd, addr);
238}
239
240void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
241		     pte_t *ptep, pte_t entry)
242{
243	unsigned int nptes, orig_shift, shift;
244	unsigned long i, size;
245	pte_t orig;
246
247	size = huge_tte_to_size(entry);
248
249	shift = PAGE_SHIFT;
250	if (size >= PUD_SIZE)
251		shift = PUD_SHIFT;
252	else if (size >= PMD_SIZE)
253		shift = PMD_SHIFT;
254	else
255		shift = PAGE_SHIFT;
256
257	nptes = size >> shift;
258
259	if (!pte_present(*ptep) && pte_present(entry))
260		mm->context.hugetlb_pte_count += nptes;
261
262	addr &= ~(size - 1);
263	orig = *ptep;
264	orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig);
265
266	for (i = 0; i < nptes; i++)
267		ptep[i] = __pte(pte_val(entry) + (i << shift));
268
269	maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift);
270	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
271	if (size == HPAGE_SIZE)
272		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0,
273				    orig_shift);
274}
275
276void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
277		     pte_t *ptep, pte_t entry, unsigned long sz)
278{
279	__set_huge_pte_at(mm, addr, ptep, entry);
280}
281
282pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
283			      pte_t *ptep, unsigned long sz)
284{
285	unsigned int i, nptes, orig_shift, shift;
286	unsigned long size;
287	pte_t entry;
288
289	entry = *ptep;
290	size = huge_tte_to_size(entry);
291
292	shift = PAGE_SHIFT;
293	if (size >= PUD_SIZE)
294		shift = PUD_SHIFT;
295	else if (size >= PMD_SIZE)
296		shift = PMD_SHIFT;
297	else
298		shift = PAGE_SHIFT;
299
300	nptes = size >> shift;
301	orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry);
302
303	if (pte_present(entry))
304		mm->context.hugetlb_pte_count -= nptes;
305
306	addr &= ~(size - 1);
307	for (i = 0; i < nptes; i++)
308		ptep[i] = __pte(0UL);
309
310	maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift);
311	/* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */
312	if (size == HPAGE_SIZE)
313		maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0,
314				    orig_shift);
315
316	return entry;
317}