mmu.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2019 Western Digital Corporation or its affiliates.
  4 *
  5 * Authors:
  6 *     Anup Patel <anup.patel@wdc.com>
  7 */
  8
  9#include <linux/errno.h>
 10#include <linux/hugetlb.h>
 11#include <linux/module.h>
 12#include <linux/uaccess.h>
 13#include <linux/vmalloc.h>
 14#include <linux/kvm_host.h>
 15#include <linux/sched/signal.h>
 16#include <asm/kvm_mmu.h>
 17#include <asm/kvm_nacl.h>
 18
 19static void mmu_wp_memory_region(struct kvm *kvm, int slot)
 20{
 21	struct kvm_memslots *slots = kvm_memslots(kvm);
 22	struct kvm_memory_slot *memslot = id_to_memslot(slots, slot);
 23	phys_addr_t start = memslot->base_gfn << PAGE_SHIFT;
 24	phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT;
 25	struct kvm_gstage gstage;
 26
 27	gstage.kvm = kvm;
 28	gstage.flags = 0;
 29	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
 30	gstage.pgd = kvm->arch.pgd;
 31
 32	spin_lock(&kvm->mmu_lock);
 33	kvm_riscv_gstage_wp_range(&gstage, start, end);
 34	spin_unlock(&kvm->mmu_lock);
 35	kvm_flush_remote_tlbs_memslot(kvm, memslot);
 36}
 37
 38int kvm_riscv_mmu_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa,
 39			  unsigned long size, bool writable, bool in_atomic)
 40{
 41	int ret = 0;
 42	pgprot_t prot;
 43	unsigned long pfn;
 44	phys_addr_t addr, end;
 45	struct kvm_mmu_memory_cache pcache = {
 46		.gfp_custom = (in_atomic) ? GFP_ATOMIC | __GFP_ACCOUNT : 0,
 47		.gfp_zero = __GFP_ZERO,
 48	};
 49	struct kvm_gstage_mapping map;
 50	struct kvm_gstage gstage;
 51
 52	gstage.kvm = kvm;
 53	gstage.flags = 0;
 54	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
 55	gstage.pgd = kvm->arch.pgd;
 56
 57	end = (gpa + size + PAGE_SIZE - 1) & PAGE_MASK;
 58	pfn = __phys_to_pfn(hpa);
 59	prot = pgprot_noncached(PAGE_WRITE);
 60
 61	for (addr = gpa; addr < end; addr += PAGE_SIZE) {
 62		map.addr = addr;
 63		map.pte = pfn_pte(pfn, prot);
 64		map.pte = pte_mkdirty(map.pte);
 65		map.level = 0;
 66
 67		if (!writable)
 68			map.pte = pte_wrprotect(map.pte);
 69
 70		ret = kvm_mmu_topup_memory_cache(&pcache, kvm_riscv_gstage_pgd_levels);
 71		if (ret)
 72			goto out;
 73
 74		spin_lock(&kvm->mmu_lock);
 75		ret = kvm_riscv_gstage_set_pte(&gstage, &pcache, &map);
 76		spin_unlock(&kvm->mmu_lock);
 77		if (ret)
 78			goto out;
 79
 80		pfn++;
 81	}
 82
 83out:
 84	kvm_mmu_free_memory_cache(&pcache);
 85	return ret;
 86}
 87
 88void kvm_riscv_mmu_iounmap(struct kvm *kvm, gpa_t gpa, unsigned long size)
 89{
 90	struct kvm_gstage gstage;
 91
 92	gstage.kvm = kvm;
 93	gstage.flags = 0;
 94	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
 95	gstage.pgd = kvm->arch.pgd;
 96
 97	spin_lock(&kvm->mmu_lock);
 98	kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false);
 99	spin_unlock(&kvm->mmu_lock);
100}
101
102void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
103					     struct kvm_memory_slot *slot,
104					     gfn_t gfn_offset,
105					     unsigned long mask)
106{
107	phys_addr_t base_gfn = slot->base_gfn + gfn_offset;
108	phys_addr_t start = (base_gfn +  __ffs(mask)) << PAGE_SHIFT;
109	phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT;
110	struct kvm_gstage gstage;
111
112	gstage.kvm = kvm;
113	gstage.flags = 0;
114	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
115	gstage.pgd = kvm->arch.pgd;
116
117	kvm_riscv_gstage_wp_range(&gstage, start, end);
118}
119
120void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
121{
122}
123
124void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free)
125{
126}
127
128void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen)
129{
130}
131
132void kvm_arch_flush_shadow_all(struct kvm *kvm)
133{
134	kvm_riscv_mmu_free_pgd(kvm);
135}
136
137void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
138				   struct kvm_memory_slot *slot)
139{
140	gpa_t gpa = slot->base_gfn << PAGE_SHIFT;
141	phys_addr_t size = slot->npages << PAGE_SHIFT;
142	struct kvm_gstage gstage;
143
144	gstage.kvm = kvm;
145	gstage.flags = 0;
146	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
147	gstage.pgd = kvm->arch.pgd;
148
149	spin_lock(&kvm->mmu_lock);
150	kvm_riscv_gstage_unmap_range(&gstage, gpa, size, false);
151	spin_unlock(&kvm->mmu_lock);
152}
153
154void kvm_arch_commit_memory_region(struct kvm *kvm,
155				struct kvm_memory_slot *old,
156				const struct kvm_memory_slot *new,
157				enum kvm_mr_change change)
158{
159	/*
160	 * At this point memslot has been committed and there is an
161	 * allocated dirty_bitmap[], dirty pages will be tracked while
162	 * the memory slot is write protected.
163	 */
164	if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
165		if (kvm_dirty_log_manual_protect_and_init_set(kvm))
166			return;
167		mmu_wp_memory_region(kvm, new->id);
168	}
169}
170
171int kvm_arch_prepare_memory_region(struct kvm *kvm,
172				const struct kvm_memory_slot *old,
173				struct kvm_memory_slot *new,
174				enum kvm_mr_change change)
175{
176	hva_t hva, reg_end, size;
177	bool writable;
178	int ret = 0;
179
180	if (change != KVM_MR_CREATE && change != KVM_MR_MOVE &&
181			change != KVM_MR_FLAGS_ONLY)
182		return 0;
183
184	/*
185	 * Prevent userspace from creating a memory region outside of the GPA
186	 * space addressable by the KVM guest GPA space.
187	 */
188	if ((new->base_gfn + new->npages) >=
189	    (kvm_riscv_gstage_gpa_size >> PAGE_SHIFT))
190		return -EFAULT;
191
192	hva = new->userspace_addr;
193	size = new->npages << PAGE_SHIFT;
194	reg_end = hva + size;
195	writable = !(new->flags & KVM_MEM_READONLY);
196
197	mmap_read_lock(current->mm);
198
199	/*
200	 * A memory region could potentially cover multiple VMAs, and
201	 * any holes between them, so iterate over all of them.
202	 *
203	 *     +--------------------------------------------+
204	 * +---------------+----------------+   +----------------+
205	 * |   : VMA 1     |      VMA 2     |   |    VMA 3  :    |
206	 * +---------------+----------------+   +----------------+
207	 *     |               memory region                |
208	 *     +--------------------------------------------+
209	 */
210	do {
211		struct vm_area_struct *vma;
212		hva_t vm_end;
213
214		vma = find_vma_intersection(current->mm, hva, reg_end);
215		if (!vma)
216			break;
217
218		/*
219		 * Mapping a read-only VMA is only allowed if the
220		 * memory region is configured as read-only.
221		 */
222		if (writable && !(vma->vm_flags & VM_WRITE)) {
223			ret = -EPERM;
224			break;
225		}
226
227		/* Take the intersection of this VMA with the memory region */
228		vm_end = min(reg_end, vma->vm_end);
229
230		if (vma->vm_flags & VM_PFNMAP) {
231			/* IO region dirty page logging not allowed */
232			if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) {
233				ret = -EINVAL;
234				goto out;
235			}
236		}
237		hva = vm_end;
238	} while (hva < reg_end);
239
240out:
241	mmap_read_unlock(current->mm);
242	return ret;
243}
244
245bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
246{
247	struct kvm_gstage gstage;
248
249	if (!kvm->arch.pgd)
250		return false;
251
252	gstage.kvm = kvm;
253	gstage.flags = 0;
254	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
255	gstage.pgd = kvm->arch.pgd;
256	kvm_riscv_gstage_unmap_range(&gstage, range->start << PAGE_SHIFT,
257				     (range->end - range->start) << PAGE_SHIFT,
258				     range->may_block);
259	return false;
260}
261
262bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
263{
264	pte_t *ptep;
265	u32 ptep_level = 0;
266	u64 size = (range->end - range->start) << PAGE_SHIFT;
267	struct kvm_gstage gstage;
268
269	if (!kvm->arch.pgd)
270		return false;
271
272	WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
273
274	gstage.kvm = kvm;
275	gstage.flags = 0;
276	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
277	gstage.pgd = kvm->arch.pgd;
278	if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT,
279				       &ptep, &ptep_level))
280		return false;
281
282	return ptep_test_and_clear_young(NULL, 0, ptep);
283}
284
285bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
286{
287	pte_t *ptep;
288	u32 ptep_level = 0;
289	u64 size = (range->end - range->start) << PAGE_SHIFT;
290	struct kvm_gstage gstage;
291
292	if (!kvm->arch.pgd)
293		return false;
294
295	WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PUD_SIZE);
296
297	gstage.kvm = kvm;
298	gstage.flags = 0;
299	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
300	gstage.pgd = kvm->arch.pgd;
301	if (!kvm_riscv_gstage_get_leaf(&gstage, range->start << PAGE_SHIFT,
302				       &ptep, &ptep_level))
303		return false;
304
305	return pte_young(ptep_get(ptep));
306}
307
308int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
309		      gpa_t gpa, unsigned long hva, bool is_write,
310		      struct kvm_gstage_mapping *out_map)
311{
312	int ret;
313	kvm_pfn_t hfn;
314	bool writable;
315	short vma_pageshift;
316	gfn_t gfn = gpa >> PAGE_SHIFT;
317	struct vm_area_struct *vma;
318	struct kvm *kvm = vcpu->kvm;
319	struct kvm_mmu_memory_cache *pcache = &vcpu->arch.mmu_page_cache;
320	bool logging = (memslot->dirty_bitmap &&
321			!(memslot->flags & KVM_MEM_READONLY)) ? true : false;
322	unsigned long vma_pagesize, mmu_seq;
323	struct kvm_gstage gstage;
324	struct page *page;
325
326	gstage.kvm = kvm;
327	gstage.flags = 0;
328	gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
329	gstage.pgd = kvm->arch.pgd;
330
331	/* Setup initial state of output mapping */
332	memset(out_map, 0, sizeof(*out_map));
333
334	/* We need minimum second+third level pages */
335	ret = kvm_mmu_topup_memory_cache(pcache, kvm_riscv_gstage_pgd_levels);
336	if (ret) {
337		kvm_err("Failed to topup G-stage cache\n");
338		return ret;
339	}
340
341	mmap_read_lock(current->mm);
342
343	vma = vma_lookup(current->mm, hva);
344	if (unlikely(!vma)) {
345		kvm_err("Failed to find VMA for hva 0x%lx\n", hva);
346		mmap_read_unlock(current->mm);
347		return -EFAULT;
348	}
349
350	if (is_vm_hugetlb_page(vma))
351		vma_pageshift = huge_page_shift(hstate_vma(vma));
352	else
353		vma_pageshift = PAGE_SHIFT;
354	vma_pagesize = 1ULL << vma_pageshift;
355	if (logging || (vma->vm_flags & VM_PFNMAP))
356		vma_pagesize = PAGE_SIZE;
357
358	if (vma_pagesize == PMD_SIZE || vma_pagesize == PUD_SIZE)
359		gfn = (gpa & huge_page_mask(hstate_vma(vma))) >> PAGE_SHIFT;
360
361	/*
362	 * Read mmu_invalidate_seq so that KVM can detect if the results of
363	 * vma_lookup() or __kvm_faultin_pfn() become stale prior to acquiring
364	 * kvm->mmu_lock.
365	 *
366	 * Rely on mmap_read_unlock() for an implicit smp_rmb(), which pairs
367	 * with the smp_wmb() in kvm_mmu_invalidate_end().
368	 */
369	mmu_seq = kvm->mmu_invalidate_seq;
370	mmap_read_unlock(current->mm);
371
372	if (vma_pagesize != PUD_SIZE &&
373	    vma_pagesize != PMD_SIZE &&
374	    vma_pagesize != PAGE_SIZE) {
375		kvm_err("Invalid VMA page size 0x%lx\n", vma_pagesize);
376		return -EFAULT;
377	}
378
379	hfn = __kvm_faultin_pfn(memslot, gfn, is_write ? FOLL_WRITE : 0,
380				&writable, &page);
381	if (hfn == KVM_PFN_ERR_HWPOISON) {
382		send_sig_mceerr(BUS_MCEERR_AR, (void __user *)hva,
383				vma_pageshift, current);
384		return 0;
385	}
386	if (is_error_noslot_pfn(hfn))
387		return -EFAULT;
388
389	/*
390	 * If logging is active then we allow writable pages only
391	 * for write faults.
392	 */
393	if (logging && !is_write)
394		writable = false;
395
396	spin_lock(&kvm->mmu_lock);
397
398	if (mmu_invalidate_retry(kvm, mmu_seq))
399		goto out_unlock;
400
401	if (writable) {
402		mark_page_dirty_in_slot(kvm, memslot, gfn);
403		ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT,
404						vma_pagesize, false, true, out_map);
405	} else {
406		ret = kvm_riscv_gstage_map_page(&gstage, pcache, gpa, hfn << PAGE_SHIFT,
407						vma_pagesize, true, true, out_map);
408	}
409
410	if (ret)
411		kvm_err("Failed to map in G-stage\n");
412
413out_unlock:
414	kvm_release_faultin_page(kvm, page, ret && ret != -EEXIST, writable);
415	spin_unlock(&kvm->mmu_lock);
416	return ret;
417}
418
419int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm)
420{
421	struct page *pgd_page;
422
423	if (kvm->arch.pgd != NULL) {
424		kvm_err("kvm_arch already initialized?\n");
425		return -EINVAL;
426	}
427
428	pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO,
429				get_order(kvm_riscv_gstage_pgd_size));
430	if (!pgd_page)
431		return -ENOMEM;
432	kvm->arch.pgd = page_to_virt(pgd_page);
433	kvm->arch.pgd_phys = page_to_phys(pgd_page);
434
435	return 0;
436}
437
438void kvm_riscv_mmu_free_pgd(struct kvm *kvm)
439{
440	struct kvm_gstage gstage;
441	void *pgd = NULL;
442
443	spin_lock(&kvm->mmu_lock);
444	if (kvm->arch.pgd) {
445		gstage.kvm = kvm;
446		gstage.flags = 0;
447		gstage.vmid = READ_ONCE(kvm->arch.vmid.vmid);
448		gstage.pgd = kvm->arch.pgd;
449		kvm_riscv_gstage_unmap_range(&gstage, 0UL, kvm_riscv_gstage_gpa_size, false);
450		pgd = READ_ONCE(kvm->arch.pgd);
451		kvm->arch.pgd = NULL;
452		kvm->arch.pgd_phys = 0;
453	}
454	spin_unlock(&kvm->mmu_lock);
455
456	if (pgd)
457		free_pages((unsigned long)pgd, get_order(kvm_riscv_gstage_pgd_size));
458}
459
460void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu)
461{
462	unsigned long hgatp = kvm_riscv_gstage_mode << HGATP_MODE_SHIFT;
463	struct kvm_arch *k = &vcpu->kvm->arch;
464
465	hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID;
466	hgatp |= (k->pgd_phys >> PAGE_SHIFT) & HGATP_PPN;
467
468	ncsr_write(CSR_HGATP, hgatp);
469
470	if (!kvm_riscv_gstage_vmid_bits())
471		kvm_riscv_local_hfence_gvma_all();
472}