tlb.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
  4 */
  5
  6#include <linux/mm.h>
  7#include <linux/module.h>
  8#include <linux/sched/signal.h>
  9
 10#include <asm/tlbflush.h>
 11#include <asm/mmu_context.h>
 12#include <as-layout.h>
 13#include <mem_user.h>
 14#include <os.h>
 15#include <skas.h>
 16#include <kern_util.h>
 17
 18struct vm_ops {
 19	struct mm_id *mm_idp;
 20
 21	int (*mmap)(struct mm_id *mm_idp,
 22		    unsigned long virt, unsigned long len, int prot,
 23		    int phys_fd, unsigned long long offset);
 24	int (*unmap)(struct mm_id *mm_idp,
 25		     unsigned long virt, unsigned long len);
 26};
 27
 28static int kern_map(struct mm_id *mm_idp,
 29		    unsigned long virt, unsigned long len, int prot,
 30		    int phys_fd, unsigned long long offset)
 31{
 32	/* TODO: Why is executable needed to be always set in the kernel? */
 33	return os_map_memory((void *)virt, phys_fd, offset, len,
 34			     prot & UM_PROT_READ, prot & UM_PROT_WRITE,
 35			     1);
 36}
 37
 38static int kern_unmap(struct mm_id *mm_idp,
 39		      unsigned long virt, unsigned long len)
 40{
 41	return os_unmap_memory((void *)virt, len);
 42}
 43
 44void report_enomem(void)
 45{
 46	printk(KERN_ERR "UML ran out of memory on the host side! "
 47			"This can happen due to a memory limitation or "
 48			"vm.max_map_count has been reached.\n");
 49}
 50
 51static inline int update_pte_range(pmd_t *pmd, unsigned long addr,
 52				   unsigned long end,
 53				   struct vm_ops *ops)
 54{
 55	pte_t *pte;
 56	int ret = 0;
 57
 58	pte = pte_offset_kernel(pmd, addr);
 59	do {
 60		if (!pte_needsync(*pte))
 61			continue;
 62
 63		if (pte_present(*pte)) {
 64			__u64 offset;
 65			unsigned long phys = pte_val(*pte) & PAGE_MASK;
 66			int fd = phys_mapping(phys, &offset);
 67			int r, w, x, prot;
 68
 69			r = pte_read(*pte);
 70			w = pte_write(*pte);
 71			x = pte_exec(*pte);
 72			if (!pte_young(*pte)) {
 73				r = 0;
 74				w = 0;
 75			} else if (!pte_dirty(*pte))
 76				w = 0;
 77
 78			prot = (r ? UM_PROT_READ : 0) |
 79			       (w ? UM_PROT_WRITE : 0) |
 80			       (x ? UM_PROT_EXEC : 0);
 81
 82			ret = ops->mmap(ops->mm_idp, addr, PAGE_SIZE,
 83					prot, fd, offset);
 84		} else
 85			ret = ops->unmap(ops->mm_idp, addr, PAGE_SIZE);
 86
 87		*pte = pte_mkuptodate(*pte);
 88	} while (pte++, addr += PAGE_SIZE, ((addr < end) && !ret));
 89	return ret;
 90}
 91
 92static inline int update_pmd_range(pud_t *pud, unsigned long addr,
 93				   unsigned long end,
 94				   struct vm_ops *ops)
 95{
 96	pmd_t *pmd;
 97	unsigned long next;
 98	int ret = 0;
 99
100	pmd = pmd_offset(pud, addr);
101	do {
102		next = pmd_addr_end(addr, end);
103		if (!pmd_present(*pmd)) {
104			if (pmd_needsync(*pmd)) {
105				ret = ops->unmap(ops->mm_idp, addr,
106						 next - addr);
107				pmd_mkuptodate(*pmd);
108			}
109		}
110		else ret = update_pte_range(pmd, addr, next, ops);
111	} while (pmd++, addr = next, ((addr < end) && !ret));
112	return ret;
113}
114
115static inline int update_pud_range(p4d_t *p4d, unsigned long addr,
116				   unsigned long end,
117				   struct vm_ops *ops)
118{
119	pud_t *pud;
120	unsigned long next;
121	int ret = 0;
122
123	pud = pud_offset(p4d, addr);
124	do {
125		next = pud_addr_end(addr, end);
126		if (!pud_present(*pud)) {
127			if (pud_needsync(*pud)) {
128				ret = ops->unmap(ops->mm_idp, addr,
129						 next - addr);
130				pud_mkuptodate(*pud);
131			}
132		}
133		else ret = update_pmd_range(pud, addr, next, ops);
134	} while (pud++, addr = next, ((addr < end) && !ret));
135	return ret;
136}
137
138static inline int update_p4d_range(pgd_t *pgd, unsigned long addr,
139				   unsigned long end,
140				   struct vm_ops *ops)
141{
142	p4d_t *p4d;
143	unsigned long next;
144	int ret = 0;
145
146	p4d = p4d_offset(pgd, addr);
147	do {
148		next = p4d_addr_end(addr, end);
149		if (!p4d_present(*p4d)) {
150			if (p4d_needsync(*p4d)) {
151				ret = ops->unmap(ops->mm_idp, addr,
152						 next - addr);
153				p4d_mkuptodate(*p4d);
154			}
155		} else
156			ret = update_pud_range(p4d, addr, next, ops);
157	} while (p4d++, addr = next, ((addr < end) && !ret));
158	return ret;
159}
160
161int um_tlb_sync(struct mm_struct *mm)
162{
163	pgd_t *pgd;
164	struct vm_ops ops;
165	unsigned long addr, next;
166	int ret = 0;
167
168	guard(spinlock_irqsave)(&mm->context.sync_tlb_lock);
169
170	if (mm->context.sync_tlb_range_to == 0)
171		return 0;
172
173	ops.mm_idp = &mm->context.id;
174	if (mm == &init_mm) {
175		ops.mmap = kern_map;
176		ops.unmap = kern_unmap;
177	} else {
178		ops.mmap = map;
179		ops.unmap = unmap;
180	}
181
182	addr = mm->context.sync_tlb_range_from;
183	pgd = pgd_offset(mm, addr);
184	do {
185		next = pgd_addr_end(addr, mm->context.sync_tlb_range_to);
186		if (!pgd_present(*pgd)) {
187			if (pgd_needsync(*pgd)) {
188				ret = ops.unmap(ops.mm_idp, addr,
189						next - addr);
190				pgd_mkuptodate(*pgd);
191			}
192		} else
193			ret = update_p4d_range(pgd, addr, next, &ops);
194	} while (pgd++, addr = next,
195		 ((addr < mm->context.sync_tlb_range_to) && !ret));
196
197	if (ret == -ENOMEM)
198		report_enomem();
199
200	mm->context.sync_tlb_range_from = 0;
201	mm->context.sync_tlb_range_to = 0;
202
203	return ret;
204}
205
206void flush_tlb_all(void)
207{
208	/*
209	 * Don't bother flushing if this address space is about to be
210	 * destroyed.
211	 */
212	if (atomic_read(&current->mm->mm_users) == 0)
213		return;
214
215	flush_tlb_mm(current->mm);
216}
217
218void flush_tlb_mm(struct mm_struct *mm)
219{
220	struct vm_area_struct *vma;
221	VMA_ITERATOR(vmi, mm, 0);
222
223	for_each_vma(vmi, vma)
224		um_tlb_mark_sync(mm, vma->vm_start, vma->vm_end);
225}