kvm_nested.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef __ARM64_KVM_NESTED_H
  3#define __ARM64_KVM_NESTED_H
  4
  5#include <linux/bitfield.h>
  6#include <linux/kvm_host.h>
  7#include <asm/kvm_emulate.h>
  8#include <asm/kvm_pgtable.h>
  9
 10static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
 11{
 12	return (!__is_defined(__KVM_NVHE_HYPERVISOR__) &&
 13		cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) &&
 14		vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2));
 15}
 16
 17/* Translation helpers from non-VHE EL2 to EL1 */
 18static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)
 19{
 20	return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT;
 21}
 22
 23static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
 24{
 25	return TCR_EPD1_MASK |				/* disable TTBR1_EL1 */
 26	       ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) |
 27	       tcr_el2_ps_to_tcr_el1_ips(tcr) |
 28	       (tcr & TCR_EL2_TG0_MASK) |
 29	       (tcr & TCR_EL2_ORGN0_MASK) |
 30	       (tcr & TCR_EL2_IRGN0_MASK) |
 31	       (tcr & TCR_EL2_T0SZ_MASK);
 32}
 33
 34static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)
 35{
 36	u64 cpacr_el1 = CPACR_EL1_RES1;
 37
 38	if (cptr_el2 & CPTR_EL2_TTA)
 39		cpacr_el1 |= CPACR_EL1_TTA;
 40	if (!(cptr_el2 & CPTR_EL2_TFP))
 41		cpacr_el1 |= CPACR_EL1_FPEN;
 42	if (!(cptr_el2 & CPTR_EL2_TZ))
 43		cpacr_el1 |= CPACR_EL1_ZEN;
 44
 45	cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM);
 46
 47	return cpacr_el1;
 48}
 49
 50static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val)
 51{
 52	/* Only preserve the minimal set of bits we support */
 53	val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA |
 54		SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE);
 55	val |= SCTLR_EL1_RES1;
 56
 57	return val;
 58}
 59
 60static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)
 61{
 62	/* Clear the ASID field */
 63	return ttbr0 & ~GENMASK_ULL(63, 48);
 64}
 65
 66extern bool forward_smc_trap(struct kvm_vcpu *vcpu);
 67extern bool forward_debug_exception(struct kvm_vcpu *vcpu);
 68extern void kvm_init_nested(struct kvm *kvm);
 69extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu);
 70extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu);
 71extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu);
 72
 73union tlbi_info;
 74
 75extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
 76				       const union tlbi_info *info,
 77				       void (*)(struct kvm_s2_mmu *,
 78						const union tlbi_info *));
 79extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
 80extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
 81
 82extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);
 83extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu);
 84extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu);
 85
 86extern void kvm_nested_setup_mdcr_el2(struct kvm_vcpu *vcpu);
 87
 88struct kvm_s2_trans {
 89	phys_addr_t output;
 90	unsigned long block_size;
 91	bool writable;
 92	bool readable;
 93	int level;
 94	u32 esr;
 95	u64 desc;
 96};
 97
 98static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans)
 99{
100	return trans->output;
101}
102
103static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans)
104{
105	return trans->block_size;
106}
107
108static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans)
109{
110	return trans->esr;
111}
112
113static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans)
114{
115	return trans->readable;
116}
117
118static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans)
119{
120	return trans->writable;
121}
122
123static inline bool kvm_has_xnx(struct kvm *kvm)
124{
125	return cpus_have_final_cap(ARM64_HAS_XNX) &&
126		kvm_has_feat(kvm, ID_AA64MMFR1_EL1, XNX, IMP);
127}
128
129static inline bool kvm_s2_trans_exec_el0(struct kvm *kvm, struct kvm_s2_trans *trans)
130{
131	u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc);
132
133	if (!kvm_has_xnx(kvm))
134		xn &= FIELD_PREP(KVM_PTE_LEAF_ATTR_HI_S2_XN, 0b10);
135
136	switch (xn) {
137	case 0b00:
138	case 0b01:
139		return true;
140	default:
141		return false;
142	}
143}
144
145static inline bool kvm_s2_trans_exec_el1(struct kvm *kvm, struct kvm_s2_trans *trans)
146{
147	u8 xn = FIELD_GET(KVM_PTE_LEAF_ATTR_HI_S2_XN, trans->desc);
148
149	if (!kvm_has_xnx(kvm))
150		xn &= FIELD_PREP(KVM_PTE_LEAF_ATTR_HI_S2_XN, 0b10);
151
152	switch (xn) {
153	case 0b00:
154	case 0b11:
155		return true;
156	default:
157		return false;
158	}
159}
160
161extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
162			      struct kvm_s2_trans *result);
163extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
164				    struct kvm_s2_trans *trans);
165extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2);
166extern void kvm_nested_s2_wp(struct kvm *kvm);
167extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block);
168extern void kvm_nested_s2_flush(struct kvm *kvm);
169
170unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val);
171
172static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr)
173{
174	struct kvm *kvm = vpcu->kvm;
175	u8 CRm = sys_reg_CRm(instr);
176
177	if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
178	      sys_reg_Op1(instr) == TLBI_Op1_EL1))
179		return false;
180
181	if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
182	      (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
183	       kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
184		return false;
185
186	if (CRm == TLBI_CRm_nROS &&
187	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
188		return false;
189
190	if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
191	     CRm == TLBI_CRm_RNS) &&
192	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
193		return false;
194
195	return true;
196}
197
198static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr)
199{
200	struct kvm *kvm = vpcu->kvm;
201	u8 CRm = sys_reg_CRm(instr);
202
203	if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
204	      sys_reg_Op1(instr) == TLBI_Op1_EL2))
205		return false;
206
207	if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
208	      (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
209	       kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
210		return false;
211
212	if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS)
213		return false;
214
215	if (CRm == TLBI_CRm_nROS &&
216	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
217		return false;
218
219	if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
220	     CRm == TLBI_CRm_RNS) &&
221	    !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
222		return false;
223
224	return true;
225}
226
227int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu);
228u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val);
229
230#ifdef CONFIG_ARM64_PTR_AUTH
231bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr);
232#else
233static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr)
234{
235	/* We really should never execute this... */
236	WARN_ON_ONCE(1);
237	*elr = 0xbad9acc0debadbad;
238	return false;
239}
240#endif
241
242#define KVM_NV_GUEST_MAP_SZ	(KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0)
243
244static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans)
245{
246	return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level);
247}
248
249/* Adjust alignment for the contiguous bit as per StageOA() */
250#define contiguous_bit_shift(d, wi, l)					\
251	({								\
252		u8 shift = 0;						\
253									\
254		if ((d) & PTE_CONT) {					\
255			switch (BIT((wi)->pgshift)) {			\
256			case SZ_4K:					\
257				shift = 4;				\
258				break;					\
259			case SZ_16K:					\
260				shift = (l) == 2 ? 5 : 7;		\
261				break;					\
262			case SZ_64K:					\
263				shift = 5;				\
264				break;					\
265			}						\
266		}							\
267									\
268		shift;							\
269	})
270
271static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid)
272{
273	u64 base, tg, num, scale;
274	int shift;
275
276	tg	= FIELD_GET(GENMASK(47, 46), val);
277
278	switch(tg) {
279	case 1:
280		shift = 12;
281		break;
282	case 2:
283		shift = 14;
284		break;
285	case 3:
286	default:		/* IMPDEF: handle tg==0 as 64k */
287		shift = 16;
288		break;
289	}
290
291	base	= (val & GENMASK(36, 0)) << shift;
292
293	if (asid)
294		*asid = FIELD_GET(TLBIR_ASID_MASK, val);
295
296	scale	= FIELD_GET(GENMASK(45, 44), val);
297	num	= FIELD_GET(GENMASK(43, 39), val);
298	*range	= __TLBI_RANGE_PAGES(num, scale) << shift;
299
300	return base;
301}
302
303static inline unsigned int ps_to_output_size(unsigned int ps, bool pa52bit)
304{
305	switch (ps) {
306	case 0: return 32;
307	case 1: return 36;
308	case 2: return 40;
309	case 3: return 42;
310	case 4: return 44;
311	case 5: return 48;
312	case 6: if (pa52bit)
313			return 52;
314		fallthrough;
315	default:
316		return 48;
317	}
318}
319
320enum trans_regime {
321	TR_EL10,
322	TR_EL20,
323	TR_EL2,
324};
325
326struct s1_walk_info;
327
328struct s1_walk_context {
329	struct s1_walk_info	*wi;
330	u64			table_ipa;
331	int			level;
332};
333
334struct s1_walk_filter {
335	int	(*fn)(struct s1_walk_context *, void *);
336	void	*priv;
337};
338
339struct s1_walk_info {
340	struct s1_walk_filter	*filter;
341	u64	     		baddr;
342	enum trans_regime	regime;
343	unsigned int		max_oa_bits;
344	unsigned int		pgshift;
345	unsigned int		txsz;
346	int 	     		sl;
347	u8			sh;
348	bool			as_el0;
349	bool	     		hpd;
350	bool			e0poe;
351	bool			poe;
352	bool			pan;
353	bool	     		be;
354	bool	     		s2;
355	bool			pa52bit;
356	bool			ha;
357};
358
359struct s1_walk_result {
360	union {
361		struct {
362			u64	desc;
363			u64	pa;
364			s8	level;
365			u8	APTable;
366			bool	nG;
367			u16	asid;
368			bool	UXNTable;
369			bool	PXNTable;
370			bool	uwxn;
371			bool	uov;
372			bool	ur;
373			bool	uw;
374			bool	ux;
375			bool	pwxn;
376			bool	pov;
377			bool	pr;
378			bool	pw;
379			bool	px;
380		};
381		struct {
382			u8	fst;
383			bool	ptw;
384			bool	s2;
385		};
386	};
387	bool	failed;
388};
389
390int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
391		       struct s1_walk_result *wr, u64 va);
392int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa,
393			     int *level);
394
395/* VNCR management */
396int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu);
397int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu);
398void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val);
399
400#define vncr_fixmap(c)						\
401	({							\
402		u32 __c = (c);					\
403		BUG_ON(__c >= NR_CPUS);				\
404		(FIX_VNCR - __c);				\
405	})
406
407int __kvm_at_swap_desc(struct kvm *kvm, gpa_t ipa, u64 old, u64 new);
408
409#endif /* __ARM64_KVM_NESTED_H */