vmem.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 *    Copyright IBM Corp. 2006
  4 */
  5
  6#include <linux/memory_hotplug.h>
  7#include <linux/bootmem_info.h>
  8#include <linux/cpufeature.h>
  9#include <linux/memblock.h>
 10#include <linux/pfn.h>
 11#include <linux/mm.h>
 12#include <linux/init.h>
 13#include <linux/list.h>
 14#include <linux/hugetlb.h>
 15#include <linux/slab.h>
 16#include <linux/sort.h>
 17#include <asm/page-states.h>
 18#include <asm/abs_lowcore.h>
 19#include <asm/cacheflush.h>
 20#include <asm/maccess.h>
 21#include <asm/nospec-branch.h>
 22#include <asm/ctlreg.h>
 23#include <asm/pgalloc.h>
 24#include <asm/setup.h>
 25#include <asm/tlbflush.h>
 26#include <asm/sections.h>
 27#include <asm/set_memory.h>
 28#include <asm/physmem_info.h>
 29
 30static DEFINE_MUTEX(vmem_mutex);
 31
 32static void __ref *vmem_alloc_pages(unsigned int order)
 33{
 34	unsigned long size = PAGE_SIZE << order;
 35
 36	if (slab_is_available())
 37		return (void *)__get_free_pages(GFP_KERNEL, order);
 38	return memblock_alloc(size, size);
 39}
 40
 41static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *altmap)
 42{
 43	unsigned int nr_pages = 1 << order;
 44	struct page *page;
 45
 46	if (altmap) {
 47		vmem_altmap_free(altmap, 1 << order);
 48		return;
 49	}
 50	page = virt_to_page((void *)addr);
 51	if (PageReserved(page)) {
 52		/* allocated from memblock */
 53		while (nr_pages--)
 54			free_bootmem_page(page++);
 55	} else {
 56		free_pages(addr, order);
 57	}
 58}
 59
 60void *vmem_crst_alloc(unsigned long val)
 61{
 62	unsigned long *table;
 63
 64	table = vmem_alloc_pages(CRST_ALLOC_ORDER);
 65	if (!table)
 66		return NULL;
 67	crst_table_init(table, val);
 68	__arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER);
 69	return table;
 70}
 71
 72pte_t __ref *vmem_pte_alloc(void)
 73{
 74	pte_t *pte;
 75
 76	if (slab_is_available())
 77		pte = (pte_t *)page_table_alloc(&init_mm);
 78	else
 79		pte = (pte_t *)memblock_alloc(PAGE_SIZE, PAGE_SIZE);
 80	if (!pte)
 81		return NULL;
 82	memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
 83	__arch_set_page_dat(pte, 1);
 84	return pte;
 85}
 86
 87static void vmem_pte_free(unsigned long *table)
 88{
 89	page_table_free(&init_mm, table);
 90}
 91
 92#define PAGE_UNUSED 0xFD
 93
 94/*
 95 * The unused vmemmap range, which was not yet memset(PAGE_UNUSED) ranges
 96 * from unused_sub_pmd_start to next PMD_SIZE boundary.
 97 */
 98static unsigned long unused_sub_pmd_start;
 99
100static void vmemmap_flush_unused_sub_pmd(void)
101{
102	if (!unused_sub_pmd_start)
103		return;
104	memset((void *)unused_sub_pmd_start, PAGE_UNUSED,
105	       ALIGN(unused_sub_pmd_start, PMD_SIZE) - unused_sub_pmd_start);
106	unused_sub_pmd_start = 0;
107}
108
109static void vmemmap_mark_sub_pmd_used(unsigned long start, unsigned long end)
110{
111	/*
112	 * As we expect to add in the same granularity as we remove, it's
113	 * sufficient to mark only some piece used to block the memmap page from
114	 * getting removed (just in case the memmap never gets initialized,
115	 * e.g., because the memory block never gets onlined).
116	 */
117	memset((void *)start, 0, sizeof(struct page));
118}
119
120static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end)
121{
122	/*
123	 * We only optimize if the new used range directly follows the
124	 * previously unused range (esp., when populating consecutive sections).
125	 */
126	if (unused_sub_pmd_start == start) {
127		unused_sub_pmd_start = end;
128		if (likely(IS_ALIGNED(unused_sub_pmd_start, PMD_SIZE)))
129			unused_sub_pmd_start = 0;
130		return;
131	}
132	vmemmap_flush_unused_sub_pmd();
133	vmemmap_mark_sub_pmd_used(start, end);
134}
135
136static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end)
137{
138	unsigned long page = ALIGN_DOWN(start, PMD_SIZE);
139
140	vmemmap_flush_unused_sub_pmd();
141
142	/* Could be our memmap page is filled with PAGE_UNUSED already ... */
143	vmemmap_mark_sub_pmd_used(start, end);
144
145	/* Mark the unused parts of the new memmap page PAGE_UNUSED. */
146	if (!IS_ALIGNED(start, PMD_SIZE))
147		memset((void *)page, PAGE_UNUSED, start - page);
148	/*
149	 * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of
150	 * consecutive sections. Remember for the last added PMD the last
151	 * unused range in the populated PMD.
152	 */
153	if (!IS_ALIGNED(end, PMD_SIZE))
154		unused_sub_pmd_start = end;
155}
156
157/* Returns true if the PMD is completely unused and can be freed. */
158static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end)
159{
160	unsigned long page = ALIGN_DOWN(start, PMD_SIZE);
161
162	vmemmap_flush_unused_sub_pmd();
163	memset((void *)start, PAGE_UNUSED, end - start);
164	return !memchr_inv((void *)page, PAGE_UNUSED, PMD_SIZE);
165}
166
167/* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */
168static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr,
169				  unsigned long end, bool add, bool direct,
170				  struct vmem_altmap *altmap)
171{
172	unsigned long prot, pages = 0;
173	int ret = -ENOMEM;
174	pte_t *pte;
175
176	prot = pgprot_val(PAGE_KERNEL);
177	pte = pte_offset_kernel(pmd, addr);
178	for (; addr < end; addr += PAGE_SIZE, pte++) {
179		if (!add) {
180			if (pte_none(*pte))
181				continue;
182			if (!direct)
183				vmem_free_pages((unsigned long)pfn_to_virt(pte_pfn(*pte)), get_order(PAGE_SIZE), altmap);
184			pte_clear(&init_mm, addr, pte);
185		} else if (pte_none(*pte)) {
186			if (!direct) {
187				void *new_page = vmemmap_alloc_block_buf(PAGE_SIZE, NUMA_NO_NODE, altmap);
188
189				if (!new_page)
190					goto out;
191				set_pte(pte, __pte(__pa(new_page) | prot));
192			} else {
193				set_pte(pte, __pte(__pa(addr) | prot));
194			}
195		} else {
196			continue;
197		}
198		pages++;
199	}
200	ret = 0;
201out:
202	if (direct)
203		update_page_count(PG_DIRECT_MAP_4K, add ? pages : -pages);
204	return ret;
205}
206
207static void try_free_pte_table(pmd_t *pmd, unsigned long start)
208{
209	pte_t *pte;
210	int i;
211
212	/* We can safely assume this is fully in 1:1 mapping & vmemmap area */
213	pte = pte_offset_kernel(pmd, start);
214	for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
215		if (!pte_none(*pte))
216			return;
217	}
218	vmem_pte_free((unsigned long *) pmd_deref(*pmd));
219	pmd_clear(pmd);
220}
221
222/* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */
223static int __ref modify_pmd_table(pud_t *pud, unsigned long addr,
224				  unsigned long end, bool add, bool direct,
225				  struct vmem_altmap *altmap)
226{
227	unsigned long next, prot, pages = 0;
228	int ret = -ENOMEM;
229	pmd_t *pmd;
230	pte_t *pte;
231
232	prot = pgprot_val(SEGMENT_KERNEL);
233	pmd = pmd_offset(pud, addr);
234	for (; addr < end; addr = next, pmd++) {
235		next = pmd_addr_end(addr, end);
236		if (!add) {
237			if (pmd_none(*pmd))
238				continue;
239			if (pmd_leaf(*pmd)) {
240				if (IS_ALIGNED(addr, PMD_SIZE) &&
241				    IS_ALIGNED(next, PMD_SIZE)) {
242					if (!direct)
243						vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE), altmap);
244					pmd_clear(pmd);
245					pages++;
246				} else if (!direct && vmemmap_unuse_sub_pmd(addr, next)) {
247					vmem_free_pages(pmd_deref(*pmd), get_order(PMD_SIZE), altmap);
248					pmd_clear(pmd);
249				}
250				continue;
251			}
252		} else if (pmd_none(*pmd)) {
253			if (IS_ALIGNED(addr, PMD_SIZE) &&
254			    IS_ALIGNED(next, PMD_SIZE) &&
255			    cpu_has_edat1() && direct &&
256			    !debug_pagealloc_enabled()) {
257				set_pmd(pmd, __pmd(__pa(addr) | prot));
258				pages++;
259				continue;
260			} else if (!direct && cpu_has_edat1()) {
261				void *new_page;
262
263				/*
264				 * Use 1MB frames for vmemmap if available. We
265				 * always use large frames even if they are only
266				 * partially used. Otherwise we would have also
267				 * page tables since vmemmap_populate gets
268				 * called for each section separately.
269				 */
270				new_page = vmemmap_alloc_block_buf(PMD_SIZE, NUMA_NO_NODE, altmap);
271				if (new_page) {
272					set_pmd(pmd, __pmd(__pa(new_page) | prot));
273					if (!IS_ALIGNED(addr, PMD_SIZE) ||
274					    !IS_ALIGNED(next, PMD_SIZE)) {
275						vmemmap_use_new_sub_pmd(addr, next);
276					}
277					continue;
278				}
279			}
280			pte = vmem_pte_alloc();
281			if (!pte)
282				goto out;
283			pmd_populate(&init_mm, pmd, pte);
284		} else if (pmd_leaf(*pmd)) {
285			if (!direct)
286				vmemmap_use_sub_pmd(addr, next);
287			continue;
288		}
289		ret = modify_pte_table(pmd, addr, next, add, direct, altmap);
290		if (ret)
291			goto out;
292		if (!add)
293			try_free_pte_table(pmd, addr & PMD_MASK);
294	}
295	ret = 0;
296out:
297	if (direct)
298		update_page_count(PG_DIRECT_MAP_1M, add ? pages : -pages);
299	return ret;
300}
301
302static void try_free_pmd_table(pud_t *pud, unsigned long start)
303{
304	pmd_t *pmd;
305	int i;
306
307	pmd = pmd_offset(pud, start);
308	for (i = 0; i < PTRS_PER_PMD; i++, pmd++)
309		if (!pmd_none(*pmd))
310			return;
311	vmem_free_pages(pud_deref(*pud), CRST_ALLOC_ORDER, NULL);
312	pud_clear(pud);
313}
314
315static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end,
316			    bool add, bool direct, struct vmem_altmap *altmap)
317{
318	unsigned long next, prot, pages = 0;
319	int ret = -ENOMEM;
320	pud_t *pud;
321	pmd_t *pmd;
322
323	prot = pgprot_val(REGION3_KERNEL);
324	pud = pud_offset(p4d, addr);
325	for (; addr < end; addr = next, pud++) {
326		next = pud_addr_end(addr, end);
327		if (!add) {
328			if (pud_none(*pud))
329				continue;
330			if (pud_leaf(*pud)) {
331				if (IS_ALIGNED(addr, PUD_SIZE) &&
332				    IS_ALIGNED(next, PUD_SIZE)) {
333					if (!direct)
334						vmem_free_pages(pud_deref(*pud), get_order(PUD_SIZE), altmap);
335					pud_clear(pud);
336					pages++;
337					continue;
338				} else {
339					split_pud_page(pud, addr & PUD_MASK);
340				}
341			}
342		} else if (pud_none(*pud)) {
343			if (IS_ALIGNED(addr, PUD_SIZE) &&
344			    IS_ALIGNED(next, PUD_SIZE) &&
345			    cpu_has_edat2() && direct &&
346			    !debug_pagealloc_enabled()) {
347				set_pud(pud, __pud(__pa(addr) | prot));
348				pages++;
349				continue;
350			}
351			pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY);
352			if (!pmd)
353				goto out;
354			pud_populate(&init_mm, pud, pmd);
355		} else if (pud_leaf(*pud)) {
356			continue;
357		}
358		ret = modify_pmd_table(pud, addr, next, add, direct, altmap);
359		if (ret)
360			goto out;
361		if (!add)
362			try_free_pmd_table(pud, addr & PUD_MASK);
363	}
364	ret = 0;
365out:
366	if (direct)
367		update_page_count(PG_DIRECT_MAP_2G, add ? pages : -pages);
368	return ret;
369}
370
371static void try_free_pud_table(p4d_t *p4d, unsigned long start)
372{
373	pud_t *pud;
374	int i;
375
376	pud = pud_offset(p4d, start);
377	for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
378		if (!pud_none(*pud))
379			return;
380	}
381	vmem_free_pages(p4d_deref(*p4d), CRST_ALLOC_ORDER, NULL);
382	p4d_clear(p4d);
383}
384
385static int modify_p4d_table(pgd_t *pgd, unsigned long addr, unsigned long end,
386			    bool add, bool direct, struct vmem_altmap *altmap)
387{
388	unsigned long next;
389	int ret = -ENOMEM;
390	p4d_t *p4d;
391	pud_t *pud;
392
393	p4d = p4d_offset(pgd, addr);
394	for (; addr < end; addr = next, p4d++) {
395		next = p4d_addr_end(addr, end);
396		if (!add) {
397			if (p4d_none(*p4d))
398				continue;
399		} else if (p4d_none(*p4d)) {
400			pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY);
401			if (!pud)
402				goto out;
403			p4d_populate(&init_mm, p4d, pud);
404		}
405		ret = modify_pud_table(p4d, addr, next, add, direct, altmap);
406		if (ret)
407			goto out;
408		if (!add)
409			try_free_pud_table(p4d, addr & P4D_MASK);
410	}
411	ret = 0;
412out:
413	return ret;
414}
415
416static void try_free_p4d_table(pgd_t *pgd, unsigned long start)
417{
418	p4d_t *p4d;
419	int i;
420
421	p4d = p4d_offset(pgd, start);
422	for (i = 0; i < PTRS_PER_P4D; i++, p4d++) {
423		if (!p4d_none(*p4d))
424			return;
425	}
426	vmem_free_pages(pgd_deref(*pgd), CRST_ALLOC_ORDER, NULL);
427	pgd_clear(pgd);
428}
429
430static int modify_pagetable(unsigned long start, unsigned long end, bool add,
431			    bool direct, struct vmem_altmap *altmap)
432{
433	unsigned long addr, next;
434	int ret = -ENOMEM;
435	pgd_t *pgd;
436	p4d_t *p4d;
437
438	if (WARN_ON_ONCE(!PAGE_ALIGNED(start | end)))
439		return -EINVAL;
440	/* Don't mess with any tables not fully in 1:1 mapping, vmemmap & kasan area */
441#ifdef CONFIG_KASAN
442	if (WARN_ON_ONCE(!(start >= KASAN_SHADOW_START && end <= KASAN_SHADOW_END) &&
443			 end > __abs_lowcore))
444		return -EINVAL;
445#else
446	if (WARN_ON_ONCE(end > __abs_lowcore))
447		return -EINVAL;
448#endif
449	for (addr = start; addr < end; addr = next) {
450		next = pgd_addr_end(addr, end);
451		pgd = pgd_offset_k(addr);
452
453		if (!add) {
454			if (pgd_none(*pgd))
455				continue;
456		} else if (pgd_none(*pgd)) {
457			p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY);
458			if (!p4d)
459				goto out;
460			pgd_populate(&init_mm, pgd, p4d);
461		}
462		ret = modify_p4d_table(pgd, addr, next, add, direct, altmap);
463		if (ret)
464			goto out;
465		if (!add)
466			try_free_p4d_table(pgd, addr & PGDIR_MASK);
467	}
468	ret = 0;
469out:
470	if (!add)
471		flush_tlb_kernel_range(start, end);
472	return ret;
473}
474
475static int add_pagetable(unsigned long start, unsigned long end, bool direct,
476			 struct vmem_altmap *altmap)
477{
478	return modify_pagetable(start, end, true, direct, altmap);
479}
480
481static int remove_pagetable(unsigned long start, unsigned long end, bool direct,
482			    struct vmem_altmap *altmap)
483{
484	return modify_pagetable(start, end, false, direct, altmap);
485}
486
487/*
488 * Add a physical memory range to the 1:1 mapping.
489 */
490static int vmem_add_range(unsigned long start, unsigned long size)
491{
492	start = (unsigned long)__va(start);
493	return add_pagetable(start, start + size, true, NULL);
494}
495
496/*
497 * Remove a physical memory range from the 1:1 mapping.
498 */
499static void vmem_remove_range(unsigned long start, unsigned long size)
500{
501	start = (unsigned long)__va(start);
502	remove_pagetable(start, start + size, true, NULL);
503}
504
505/*
506 * Add a backed mem_map array to the virtual mem_map array.
507 */
508int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
509			       struct vmem_altmap *altmap)
510{
511	int ret;
512
513	mutex_lock(&vmem_mutex);
514	/* We don't care about the node, just use NUMA_NO_NODE on allocations */
515	ret = add_pagetable(start, end, false, altmap);
516	if (ret)
517		remove_pagetable(start, end, false, altmap);
518	mutex_unlock(&vmem_mutex);
519	return ret;
520}
521
522#ifdef CONFIG_MEMORY_HOTPLUG
523
524void vmemmap_free(unsigned long start, unsigned long end,
525		  struct vmem_altmap *altmap)
526{
527	mutex_lock(&vmem_mutex);
528	remove_pagetable(start, end, false, altmap);
529	mutex_unlock(&vmem_mutex);
530}
531
532#endif
533
534void vmem_remove_mapping(unsigned long start, unsigned long size)
535{
536	mutex_lock(&vmem_mutex);
537	vmem_remove_range(start, size);
538	mutex_unlock(&vmem_mutex);
539}
540
541struct range arch_get_mappable_range(void)
542{
543	struct range mhp_range;
544
545	mhp_range.start = 0;
546	mhp_range.end = max_mappable - 1;
547	return mhp_range;
548}
549
550int vmem_add_mapping(unsigned long start, unsigned long size)
551{
552	struct range range = arch_get_mappable_range();
553	int ret;
554
555	if (start < range.start ||
556	    start + size > range.end + 1 ||
557	    start + size < start)
558		return -ERANGE;
559
560	mutex_lock(&vmem_mutex);
561	ret = vmem_add_range(start, size);
562	if (ret)
563		vmem_remove_range(start, size);
564	mutex_unlock(&vmem_mutex);
565	return ret;
566}
567
568/*
569 * Allocate new or return existing page-table entry, but do not map it
570 * to any physical address. If missing, allocate segment- and region-
571 * table entries along. Meeting a large segment- or region-table entry
572 * while traversing is an error, since the function is expected to be
573 * called against virtual regions reserved for 4KB mappings only.
574 */
575pte_t *vmem_get_alloc_pte(unsigned long addr, bool alloc)
576{
577	pte_t *ptep = NULL;
578	pgd_t *pgd;
579	p4d_t *p4d;
580	pud_t *pud;
581	pmd_t *pmd;
582	pte_t *pte;
583
584	pgd = pgd_offset_k(addr);
585	if (pgd_none(*pgd)) {
586		if (!alloc)
587			goto out;
588		p4d = vmem_crst_alloc(_REGION2_ENTRY_EMPTY);
589		if (!p4d)
590			goto out;
591		pgd_populate(&init_mm, pgd, p4d);
592	}
593	p4d = p4d_offset(pgd, addr);
594	if (p4d_none(*p4d)) {
595		if (!alloc)
596			goto out;
597		pud = vmem_crst_alloc(_REGION3_ENTRY_EMPTY);
598		if (!pud)
599			goto out;
600		p4d_populate(&init_mm, p4d, pud);
601	}
602	pud = pud_offset(p4d, addr);
603	if (pud_none(*pud)) {
604		if (!alloc)
605			goto out;
606		pmd = vmem_crst_alloc(_SEGMENT_ENTRY_EMPTY);
607		if (!pmd)
608			goto out;
609		pud_populate(&init_mm, pud, pmd);
610	} else if (WARN_ON_ONCE(pud_leaf(*pud))) {
611		goto out;
612	}
613	pmd = pmd_offset(pud, addr);
614	if (pmd_none(*pmd)) {
615		if (!alloc)
616			goto out;
617		pte = vmem_pte_alloc();
618		if (!pte)
619			goto out;
620		pmd_populate(&init_mm, pmd, pte);
621	} else if (WARN_ON_ONCE(pmd_leaf(*pmd))) {
622		goto out;
623	}
624	ptep = pte_offset_kernel(pmd, addr);
625out:
626	return ptep;
627}
628
629int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc)
630{
631	pte_t *ptep, pte;
632
633	if (!IS_ALIGNED(addr, PAGE_SIZE))
634		return -EINVAL;
635	ptep = vmem_get_alloc_pte(addr, alloc);
636	if (!ptep)
637		return -ENOMEM;
638	__ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
639	pte = mk_pte_phys(phys, prot);
640	set_pte(ptep, pte);
641	return 0;
642}
643
644int vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot)
645{
646	int rc;
647
648	mutex_lock(&vmem_mutex);
649	rc = __vmem_map_4k_page(addr, phys, prot, true);
650	mutex_unlock(&vmem_mutex);
651	return rc;
652}
653
654void vmem_unmap_4k_page(unsigned long addr)
655{
656	pte_t *ptep;
657
658	mutex_lock(&vmem_mutex);
659	ptep = virt_to_kpte(addr);
660	__ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
661	pte_clear(&init_mm, addr, ptep);
662	mutex_unlock(&vmem_mutex);
663}
664
665void __init vmem_map_init(void)
666{
667	__set_memory_rox(_stext, _etext);
668	__set_memory_ro(_etext, __end_rodata);
669	__set_memory_rox(__stext_amode31, __etext_amode31);
670	/*
671	 * If the BEAR-enhancement facility is not installed the first
672	 * prefix page is used to return to the previous context with
673	 * an LPSWE instruction and therefore must be executable.
674	 */
675	if (!cpu_has_bear())
676		set_memory_x(0, 1);
677	if (debug_pagealloc_enabled())
678		__set_memory_4k(__va(0), absolute_pointer(__va(0)) + ident_map_size);
679	pr_info("Write protected kernel read-only data: %luk\n",
680		(unsigned long)(__end_rodata - _stext) >> 10);
681}