tlb.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2015 - ARM Ltd
  4 * Author: Marc Zyngier <marc.zyngier@arm.com>
  5 */
  6
  7#include <linux/irqflags.h>
  8
  9#include <asm/kvm_hyp.h>
 10#include <asm/kvm_mmu.h>
 11#include <asm/tlbflush.h>
 12
 13struct tlb_inv_context {
 14	struct kvm_s2_mmu	*mmu;
 15	unsigned long		flags;
 16	u64			tcr;
 17	u64			sctlr;
 18};
 19
 20static void enter_vmid_context(struct kvm_s2_mmu *mmu,
 21			       struct tlb_inv_context *cxt)
 22{
 23	struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
 24	u64 val;
 25
 26	local_irq_save(cxt->flags);
 27
 28	if (vcpu && mmu != vcpu->arch.hw_mmu)
 29		cxt->mmu = vcpu->arch.hw_mmu;
 30	else
 31		cxt->mmu = NULL;
 32
 33	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
 34		/*
 35		 * For CPUs that are affected by ARM errata 1165522 or 1530923,
 36		 * we cannot trust stage-1 to be in a correct state at that
 37		 * point. Since we do not want to force a full load of the
 38		 * vcpu state, we prevent the EL1 page-table walker to
 39		 * allocate new TLBs. This is done by setting the EPD bits
 40		 * in the TCR_EL1 register. We also need to prevent it to
 41		 * allocate IPA->PA walks, so we enable the S1 MMU...
 42		 */
 43		val = cxt->tcr = read_sysreg_el1(SYS_TCR);
 44		val |= TCR_EPD1_MASK | TCR_EPD0_MASK;
 45		write_sysreg_el1(val, SYS_TCR);
 46		val = cxt->sctlr = read_sysreg_el1(SYS_SCTLR);
 47		val |= SCTLR_ELx_M;
 48		write_sysreg_el1(val, SYS_SCTLR);
 49	}
 50
 51	/*
 52	 * With VHE enabled, we have HCR_EL2.{E2H,TGE} = {1,1}, and
 53	 * most TLB operations target EL2/EL0. In order to affect the
 54	 * guest TLBs (EL1/EL0), we need to change one of these two
 55	 * bits. Changing E2H is impossible (goodbye TTBR1_EL2), so
 56	 * let's flip TGE before executing the TLB operation.
 57	 *
 58	 * ARM erratum 1165522 requires some special handling (again),
 59	 * as we need to make sure both stages of translation are in
 60	 * place before clearing TGE. __load_stage2() already
 61	 * has an ISB in order to deal with this.
 62	 */
 63	__load_stage2(mmu, mmu->arch);
 64	val = read_sysreg(hcr_el2);
 65	val &= ~HCR_TGE;
 66	write_sysreg_hcr(val);
 67	isb();
 68}
 69
 70static void exit_vmid_context(struct tlb_inv_context *cxt)
 71{
 72	/*
 73	 * We're done with the TLB operation, let's restore the host's
 74	 * view of HCR_EL2.
 75	 */
 76	write_sysreg_hcr(HCR_HOST_VHE_FLAGS);
 77	isb();
 78
 79	/* ... and the stage-2 MMU context that we switched away from */
 80	if (cxt->mmu)
 81		__load_stage2(cxt->mmu, cxt->mmu->arch);
 82
 83	if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
 84		/* Restore the registers to what they were */
 85		write_sysreg_el1(cxt->tcr, SYS_TCR);
 86		write_sysreg_el1(cxt->sctlr, SYS_SCTLR);
 87	}
 88
 89	local_irq_restore(cxt->flags);
 90}
 91
 92void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu,
 93			      phys_addr_t ipa, int level)
 94{
 95	struct tlb_inv_context cxt;
 96
 97	dsb(ishst);
 98
 99	/* Switch to requested VMID */
100	enter_vmid_context(mmu, &cxt);
101
102	/*
103	 * We could do so much better if we had the VA as well.
104	 * Instead, we invalidate Stage-2 for this IPA, and the
105	 * whole of Stage-1. Weep...
106	 */
107	ipa >>= 12;
108	__tlbi_level(ipas2e1is, ipa, level);
109
110	/*
111	 * We have to ensure completion of the invalidation at Stage-2,
112	 * since a table walk on another CPU could refill a TLB with a
113	 * complete (S1 + S2) walk based on the old Stage-2 mapping if
114	 * the Stage-1 invalidation happened first.
115	 */
116	dsb(ish);
117	__tlbi(vmalle1is);
118	dsb(ish);
119	isb();
120
121	exit_vmid_context(&cxt);
122}
123
124void __kvm_tlb_flush_vmid_ipa_nsh(struct kvm_s2_mmu *mmu,
125				  phys_addr_t ipa, int level)
126{
127	struct tlb_inv_context cxt;
128
129	dsb(nshst);
130
131	/* Switch to requested VMID */
132	enter_vmid_context(mmu, &cxt);
133
134	/*
135	 * We could do so much better if we had the VA as well.
136	 * Instead, we invalidate Stage-2 for this IPA, and the
137	 * whole of Stage-1. Weep...
138	 */
139	ipa >>= 12;
140	__tlbi_level(ipas2e1, ipa, level);
141
142	/*
143	 * We have to ensure completion of the invalidation at Stage-2,
144	 * since a table walk on another CPU could refill a TLB with a
145	 * complete (S1 + S2) walk based on the old Stage-2 mapping if
146	 * the Stage-1 invalidation happened first.
147	 */
148	dsb(nsh);
149	__tlbi(vmalle1);
150	dsb(nsh);
151	isb();
152
153	exit_vmid_context(&cxt);
154}
155
156void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu,
157				phys_addr_t start, unsigned long pages)
158{
159	struct tlb_inv_context cxt;
160	unsigned long stride;
161
162	/*
163	 * Since the range of addresses may not be mapped at
164	 * the same level, assume the worst case as PAGE_SIZE
165	 */
166	stride = PAGE_SIZE;
167	start = round_down(start, stride);
168
169	dsb(ishst);
170
171	/* Switch to requested VMID */
172	enter_vmid_context(mmu, &cxt);
173
174	__flush_s2_tlb_range_op(ipas2e1is, start, pages, stride,
175				TLBI_TTL_UNKNOWN);
176
177	dsb(ish);
178	__tlbi(vmalle1is);
179	dsb(ish);
180	isb();
181
182	exit_vmid_context(&cxt);
183}
184
185void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu)
186{
187	struct tlb_inv_context cxt;
188
189	dsb(ishst);
190
191	/* Switch to requested VMID */
192	enter_vmid_context(mmu, &cxt);
193
194	__tlbi(vmalls12e1is);
195	dsb(ish);
196	isb();
197
198	exit_vmid_context(&cxt);
199}
200
201void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu)
202{
203	struct tlb_inv_context cxt;
204
205	/* Switch to requested VMID */
206	enter_vmid_context(mmu, &cxt);
207
208	__tlbi(vmalle1);
209	asm volatile("ic iallu");
210	dsb(nsh);
211	isb();
212
213	exit_vmid_context(&cxt);
214}
215
216void __kvm_flush_vm_context(void)
217{
218	dsb(ishst);
219	__tlbi(alle1is);
220	dsb(ish);
221}
222
223/*
224 * TLB invalidation emulation for NV. For any given instruction, we
225 * perform the following transformtions:
226 *
227 * - a TLBI targeting EL2 S1 is remapped to EL1 S1
228 * - a non-shareable TLBI is upgraded to being inner-shareable
229 * - an outer-shareable TLBI is also mapped to inner-shareable
230 * - an nXS TLBI is upgraded to XS
231 */
232int __kvm_tlbi_s1e2(struct kvm_s2_mmu *mmu, u64 va, u64 sys_encoding)
233{
234	struct tlb_inv_context cxt;
235	int ret = 0;
236
237	/*
238	 * The guest will have provided its own DSB ISHST before trapping.
239	 * If it hasn't, that's its own problem, and we won't paper over it
240	 * (plus, there is plenty of extra synchronisation before we even
241	 * get here...).
242	 */
243
244	if (mmu)
245		enter_vmid_context(mmu, &cxt);
246
247	switch (sys_encoding) {
248	case OP_TLBI_ALLE2:
249	case OP_TLBI_ALLE2IS:
250	case OP_TLBI_ALLE2OS:
251	case OP_TLBI_VMALLE1:
252	case OP_TLBI_VMALLE1IS:
253	case OP_TLBI_VMALLE1OS:
254	case OP_TLBI_ALLE2NXS:
255	case OP_TLBI_ALLE2ISNXS:
256	case OP_TLBI_ALLE2OSNXS:
257	case OP_TLBI_VMALLE1NXS:
258	case OP_TLBI_VMALLE1ISNXS:
259	case OP_TLBI_VMALLE1OSNXS:
260		__tlbi(vmalle1is);
261		break;
262	case OP_TLBI_VAE2:
263	case OP_TLBI_VAE2IS:
264	case OP_TLBI_VAE2OS:
265	case OP_TLBI_VAE1:
266	case OP_TLBI_VAE1IS:
267	case OP_TLBI_VAE1OS:
268	case OP_TLBI_VAE2NXS:
269	case OP_TLBI_VAE2ISNXS:
270	case OP_TLBI_VAE2OSNXS:
271	case OP_TLBI_VAE1NXS:
272	case OP_TLBI_VAE1ISNXS:
273	case OP_TLBI_VAE1OSNXS:
274		__tlbi(vae1is, va);
275		break;
276	case OP_TLBI_VALE2:
277	case OP_TLBI_VALE2IS:
278	case OP_TLBI_VALE2OS:
279	case OP_TLBI_VALE1:
280	case OP_TLBI_VALE1IS:
281	case OP_TLBI_VALE1OS:
282	case OP_TLBI_VALE2NXS:
283	case OP_TLBI_VALE2ISNXS:
284	case OP_TLBI_VALE2OSNXS:
285	case OP_TLBI_VALE1NXS:
286	case OP_TLBI_VALE1ISNXS:
287	case OP_TLBI_VALE1OSNXS:
288		__tlbi(vale1is, va);
289		break;
290	case OP_TLBI_ASIDE1:
291	case OP_TLBI_ASIDE1IS:
292	case OP_TLBI_ASIDE1OS:
293	case OP_TLBI_ASIDE1NXS:
294	case OP_TLBI_ASIDE1ISNXS:
295	case OP_TLBI_ASIDE1OSNXS:
296		__tlbi(aside1is, va);
297		break;
298	case OP_TLBI_VAAE1:
299	case OP_TLBI_VAAE1IS:
300	case OP_TLBI_VAAE1OS:
301	case OP_TLBI_VAAE1NXS:
302	case OP_TLBI_VAAE1ISNXS:
303	case OP_TLBI_VAAE1OSNXS:
304		__tlbi(vaae1is, va);
305		break;
306	case OP_TLBI_VAALE1:
307	case OP_TLBI_VAALE1IS:
308	case OP_TLBI_VAALE1OS:
309	case OP_TLBI_VAALE1NXS:
310	case OP_TLBI_VAALE1ISNXS:
311	case OP_TLBI_VAALE1OSNXS:
312		__tlbi(vaale1is, va);
313		break;
314	case OP_TLBI_RVAE2:
315	case OP_TLBI_RVAE2IS:
316	case OP_TLBI_RVAE2OS:
317	case OP_TLBI_RVAE1:
318	case OP_TLBI_RVAE1IS:
319	case OP_TLBI_RVAE1OS:
320	case OP_TLBI_RVAE2NXS:
321	case OP_TLBI_RVAE2ISNXS:
322	case OP_TLBI_RVAE2OSNXS:
323	case OP_TLBI_RVAE1NXS:
324	case OP_TLBI_RVAE1ISNXS:
325	case OP_TLBI_RVAE1OSNXS:
326		__tlbi(rvae1is, va);
327		break;
328	case OP_TLBI_RVALE2:
329	case OP_TLBI_RVALE2IS:
330	case OP_TLBI_RVALE2OS:
331	case OP_TLBI_RVALE1:
332	case OP_TLBI_RVALE1IS:
333	case OP_TLBI_RVALE1OS:
334	case OP_TLBI_RVALE2NXS:
335	case OP_TLBI_RVALE2ISNXS:
336	case OP_TLBI_RVALE2OSNXS:
337	case OP_TLBI_RVALE1NXS:
338	case OP_TLBI_RVALE1ISNXS:
339	case OP_TLBI_RVALE1OSNXS:
340		__tlbi(rvale1is, va);
341		break;
342	case OP_TLBI_RVAAE1:
343	case OP_TLBI_RVAAE1IS:
344	case OP_TLBI_RVAAE1OS:
345	case OP_TLBI_RVAAE1NXS:
346	case OP_TLBI_RVAAE1ISNXS:
347	case OP_TLBI_RVAAE1OSNXS:
348		__tlbi(rvaae1is, va);
349		break;
350	case OP_TLBI_RVAALE1:
351	case OP_TLBI_RVAALE1IS:
352	case OP_TLBI_RVAALE1OS:
353	case OP_TLBI_RVAALE1NXS:
354	case OP_TLBI_RVAALE1ISNXS:
355	case OP_TLBI_RVAALE1OSNXS:
356		__tlbi(rvaale1is, va);
357		break;
358	default:
359		ret = -EINVAL;
360	}
361	dsb(ish);
362	isb();
363
364	if (mmu)
365		exit_vmid_context(&cxt);
366
367	return ret;
368}