kvm_emulate.h

  1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Copyright (C) 2012,2013 - ARM Ltd
  4 * Author: Marc Zyngier <marc.zyngier@arm.com>
  5 *
  6 * Derived from arch/arm/include/kvm_emulate.h
  7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
  8 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
  9 */
 10
 11#ifndef __ARM64_KVM_EMULATE_H__
 12#define __ARM64_KVM_EMULATE_H__
 13
 14#include <linux/bitfield.h>
 15#include <linux/kvm_host.h>
 16
 17#include <asm/debug-monitors.h>
 18#include <asm/esr.h>
 19#include <asm/kvm_arm.h>
 20#include <asm/kvm_hyp.h>
 21#include <asm/kvm_nested.h>
 22#include <asm/ptrace.h>
 23#include <asm/cputype.h>
 24#include <asm/virt.h>
 25
 26#define CURRENT_EL_SP_EL0_VECTOR	0x0
 27#define CURRENT_EL_SP_ELx_VECTOR	0x200
 28#define LOWER_EL_AArch64_VECTOR		0x400
 29#define LOWER_EL_AArch32_VECTOR		0x600
 30
 31enum exception_type {
 32	except_type_sync	= 0,
 33	except_type_irq		= 0x80,
 34	except_type_fiq		= 0x100,
 35	except_type_serror	= 0x180,
 36};
 37
 38#define kvm_exception_type_names		\
 39	{ except_type_sync,	"SYNC"   },	\
 40	{ except_type_irq,	"IRQ"    },	\
 41	{ except_type_fiq,	"FIQ"    },	\
 42	{ except_type_serror,	"SERROR" }
 43
 44bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
 45void kvm_skip_instr32(struct kvm_vcpu *vcpu);
 46
 47void kvm_inject_undefined(struct kvm_vcpu *vcpu);
 48int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr);
 49int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
 50void kvm_inject_size_fault(struct kvm_vcpu *vcpu);
 51
 52static inline int kvm_inject_sea_dabt(struct kvm_vcpu *vcpu, u64 addr)
 53{
 54	return kvm_inject_sea(vcpu, false, addr);
 55}
 56
 57static inline int kvm_inject_sea_iabt(struct kvm_vcpu *vcpu, u64 addr)
 58{
 59	return kvm_inject_sea(vcpu, true, addr);
 60}
 61
 62static inline int kvm_inject_serror(struct kvm_vcpu *vcpu)
 63{
 64	/*
 65	 * ESR_ELx.ISV (later renamed to IDS) indicates whether or not
 66	 * ESR_ELx.ISS contains IMPLEMENTATION DEFINED syndrome information.
 67	 *
 68	 * Set the bit when injecting an SError w/o an ESR to indicate ISS
 69	 * does not follow the architected format.
 70	 */
 71	return kvm_inject_serror_esr(vcpu, ESR_ELx_ISV);
 72}
 73
 74void kvm_vcpu_wfi(struct kvm_vcpu *vcpu);
 75
 76void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);
 77int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);
 78int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);
 79int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
 80int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr);
 81
 82static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)
 83{
 84	u64 esr = FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SVE) |
 85		  ESR_ELx_IL;
 86
 87	kvm_inject_nested_sync(vcpu, esr);
 88}
 89
 90#if defined(__KVM_VHE_HYPERVISOR__) || defined(__KVM_NVHE_HYPERVISOR__)
 91static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
 92{
 93	return !(vcpu->arch.hcr_el2 & HCR_RW);
 94}
 95#else
 96static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
 97{
 98	return vcpu_has_feature(vcpu, KVM_ARM_VCPU_EL1_32BIT);
 99}
100#endif
101
102static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
103{
104	if (!vcpu_has_run_once(vcpu))
105		vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
106
107	/*
108	 * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C
109	 * get set in SCTLR_EL1 such that we can detect when the guest
110	 * MMU gets turned on and do the necessary cache maintenance
111	 * then.
112	 */
113	if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
114		vcpu->arch.hcr_el2 |= HCR_TVM;
115}
116
117static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu)
118{
119	return (unsigned long *)&vcpu->arch.hcr_el2;
120}
121
122static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
123{
124	return vcpu->arch.vsesr_el2;
125}
126
127static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr)
128{
129	vcpu->arch.vsesr_el2 = vsesr;
130}
131
132static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
133{
134	return (unsigned long *)&vcpu_gp_regs(vcpu)->pc;
135}
136
137static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu)
138{
139	return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate;
140}
141
142static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
143{
144	return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT);
145}
146
147static __always_inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu)
148{
149	if (vcpu_mode_is_32bit(vcpu))
150		return kvm_condition_valid32(vcpu);
151
152	return true;
153}
154
155static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
156{
157	*vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT;
158}
159
160/*
161 * vcpu_get_reg and vcpu_set_reg should always be passed a register number
162 * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on
163 * AArch32 with banked registers.
164 */
165static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu,
166					 u8 reg_num)
167{
168	return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num];
169}
170
171static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num,
172				unsigned long val)
173{
174	if (reg_num != 31)
175		vcpu_gp_regs(vcpu)->regs[reg_num] = val;
176}
177
178static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt)
179{
180	switch (ctxt->regs.pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) {
181	case PSR_MODE_EL2h:
182	case PSR_MODE_EL2t:
183		return true;
184	default:
185		return false;
186	}
187}
188
189static inline bool vcpu_is_el2(const struct kvm_vcpu *vcpu)
190{
191	return vcpu_is_el2_ctxt(&vcpu->arch.ctxt);
192}
193
194static inline bool vcpu_el2_e2h_is_set(const struct kvm_vcpu *vcpu)
195{
196	return (!cpus_have_final_cap(ARM64_HAS_HCR_NV1) ||
197		(__vcpu_sys_reg(vcpu, HCR_EL2) & HCR_E2H));
198}
199
200static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu)
201{
202	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE;
203}
204
205static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu)
206{
207	/*
208	 * DDI0487L.b Known Issue D22105
209	 *
210	 * When executing at EL2 and HCR_EL2.{E2H,TGE} = {1, 0} it is
211	 * IMPLEMENTATION DEFINED whether the effective value of HCR_EL2.AMO
212	 * is the value programmed or 1.
213	 *
214	 * Make the implementation choice of treating the effective value as 1 as
215	 * we cannot subsequently catch changes to TGE or AMO that would
216	 * otherwise lead to the SError becoming deliverable.
217	 */
218	if (vcpu_is_el2(vcpu) && vcpu_el2_e2h_is_set(vcpu) && !vcpu_el2_tge_is_set(vcpu))
219		return true;
220
221	return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO;
222}
223
224static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)
225{
226	bool e2h, tge;
227	u64 hcr;
228
229	if (!vcpu_has_nv(vcpu))
230		return false;
231
232	hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
233
234	e2h = (hcr & HCR_E2H);
235	tge = (hcr & HCR_TGE);
236
237	/*
238	 * We are in a hypervisor context if the vcpu mode is EL2 or
239	 * E2H and TGE bits are set. The latter means we are in the user space
240	 * of the VHE kernel. ARMv8.1 ARM describes this as 'InHost'
241	 *
242	 * Note that the HCR_EL2.{E2H,TGE}={0,1} isn't really handled in the
243	 * rest of the KVM code, and will result in a misbehaving guest.
244	 */
245	return vcpu_is_el2(vcpu) || (e2h && tge) || tge;
246}
247
248static inline bool vcpu_is_host_el0(const struct kvm_vcpu *vcpu)
249{
250	return is_hyp_ctxt(vcpu) && !vcpu_is_el2(vcpu);
251}
252
253static inline bool is_nested_ctxt(struct kvm_vcpu *vcpu)
254{
255	return vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu);
256}
257
258static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu)
259{
260	if (!is_nested_ctxt(vcpu))
261		return false;
262
263	return vcpu_el2_amo_is_set(vcpu) ||
264	       (__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
265}
266
267/*
268 * The layout of SPSR for an AArch32 state is different when observed from an
269 * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
270 * view given an AArch64 view.
271 *
272 * In ARM DDI 0487E.a see:
273 *
274 * - The AArch64 view (SPSR_EL2) in section C5.2.18, page C5-426
275 * - The AArch32 view (SPSR_abt) in section G8.2.126, page G8-6256
276 * - The AArch32 view (SPSR_und) in section G8.2.132, page G8-6280
277 *
278 * Which show the following differences:
279 *
280 * | Bit | AA64 | AA32 | Notes                       |
281 * +-----+------+------+-----------------------------|
282 * | 24  | DIT  | J    | J is RES0 in ARMv8          |
283 * | 21  | SS   | DIT  | SS doesn't exist in AArch32 |
284 *
285 * ... and all other bits are (currently) common.
286 */
287static inline unsigned long host_spsr_to_spsr32(unsigned long spsr)
288{
289	const unsigned long overlap = BIT(24) | BIT(21);
290	unsigned long dit = !!(spsr & PSR_AA32_DIT_BIT);
291
292	spsr &= ~overlap;
293
294	spsr |= dit << 21;
295
296	return spsr;
297}
298
299static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu)
300{
301	u32 mode;
302
303	if (vcpu_mode_is_32bit(vcpu)) {
304		mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
305		return mode > PSR_AA32_MODE_USR;
306	}
307
308	mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK;
309
310	return mode != PSR_MODE_EL0t;
311}
312
313static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu)
314{
315	return vcpu->arch.fault.esr_el2;
316}
317
318static inline bool guest_hyp_wfx_traps_enabled(const struct kvm_vcpu *vcpu)
319{
320	u64 esr = kvm_vcpu_get_esr(vcpu);
321	bool is_wfe = !!(esr & ESR_ELx_WFx_ISS_WFE);
322	u64 hcr_el2 = __vcpu_sys_reg(vcpu, HCR_EL2);
323
324	if (!vcpu_has_nv(vcpu) || vcpu_is_el2(vcpu))
325		return false;
326
327	return ((is_wfe && (hcr_el2 & HCR_TWE)) ||
328		(!is_wfe && (hcr_el2 & HCR_TWI)));
329}
330
331static __always_inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
332{
333	u64 esr = kvm_vcpu_get_esr(vcpu);
334
335	if (esr & ESR_ELx_CV)
336		return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
337
338	return -1;
339}
340
341static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu)
342{
343	return vcpu->arch.fault.far_el2;
344}
345
346static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
347{
348	u64 hpfar = vcpu->arch.fault.hpfar_el2;
349
350	if (unlikely(!(hpfar & HPFAR_EL2_NS)))
351		return INVALID_GPA;
352
353	return FIELD_GET(HPFAR_EL2_FIPA, hpfar) << 12;
354}
355
356static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu)
357{
358	return vcpu->arch.fault.disr_el1;
359}
360
361static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
362{
363	return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK;
364}
365
366static __always_inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
367{
368	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_ISV);
369}
370
371static inline unsigned long kvm_vcpu_dabt_iss_nisv_sanitized(const struct kvm_vcpu *vcpu)
372{
373	return kvm_vcpu_get_esr(vcpu) & (ESR_ELx_CM | ESR_ELx_WNR | ESR_ELx_FSC);
374}
375
376static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
377{
378	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SSE);
379}
380
381static inline bool kvm_vcpu_dabt_issf(const struct kvm_vcpu *vcpu)
382{
383	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_SF);
384}
385
386static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
387{
388	return (kvm_vcpu_get_esr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
389}
390
391static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu)
392{
393	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW);
394}
395
396/* Always check for S1PTW *before* using this. */
397static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
398{
399	return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR;
400}
401
402static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
403{
404	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM);
405}
406
407static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
408{
409	return 1 << ((kvm_vcpu_get_esr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
410}
411
412/* This one is not specific to Data Abort */
413static __always_inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
414{
415	return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_IL);
416}
417
418static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
419{
420	return ESR_ELx_EC(kvm_vcpu_get_esr(vcpu));
421}
422
423static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
424{
425	return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
426}
427
428static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu)
429{
430	return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu);
431}
432
433static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
434{
435	return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC;
436}
437
438static inline
439bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu)
440{
441	return esr_fsc_is_permission_fault(kvm_vcpu_get_esr(vcpu));
442}
443
444static inline
445bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu)
446{
447	return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu));
448}
449
450static inline
451u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu)
452{
453	unsigned long esr = kvm_vcpu_get_esr(vcpu);
454
455	BUG_ON(!esr_fsc_is_permission_fault(esr));
456	return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL));
457}
458
459static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu)
460{
461	switch (kvm_vcpu_trap_get_fault(vcpu)) {
462	case ESR_ELx_FSC_EXTABT:
463	case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3):
464	case ESR_ELx_FSC_SECC:
465	case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3):
466		return true;
467	default:
468		return false;
469	}
470}
471
472static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
473{
474	u64 esr = kvm_vcpu_get_esr(vcpu);
475	return ESR_ELx_SYS64_ISS_RT(esr);
476}
477
478static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
479{
480	if (kvm_vcpu_abt_iss1tw(vcpu)) {
481		/*
482		 * Only a permission fault on a S1PTW should be
483		 * considered as a write. Otherwise, page tables baked
484		 * in a read-only memslot will result in an exception
485		 * being delivered in the guest.
486		 *
487		 * The drawback is that we end-up faulting twice if the
488		 * guest is using any of HW AF/DB: a translation fault
489		 * to map the page containing the PT (read only at
490		 * first), then a permission fault to allow the flags
491		 * to be set.
492		 */
493		return kvm_vcpu_trap_is_permission_fault(vcpu);
494	}
495
496	if (kvm_vcpu_trap_is_iabt(vcpu))
497		return false;
498
499	return kvm_vcpu_dabt_iswrite(vcpu);
500}
501
502static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
503{
504	return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
505}
506
507static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
508{
509	if (vcpu_mode_is_32bit(vcpu)) {
510		*vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT;
511	} else {
512		enum vcpu_sysreg r;
513		u64 sctlr;
514
515		r = vcpu_has_nv(vcpu) ? SCTLR_EL2 : SCTLR_EL1;
516
517		sctlr = vcpu_read_sys_reg(vcpu, r);
518		sctlr |= SCTLR_ELx_EE;
519		vcpu_write_sys_reg(vcpu, sctlr, r);
520	}
521}
522
523static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
524{
525	enum vcpu_sysreg r;
526	u64 bit;
527
528	if (vcpu_mode_is_32bit(vcpu))
529		return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT);
530
531	r = is_hyp_ctxt(vcpu) ? SCTLR_EL2 : SCTLR_EL1;
532	bit = vcpu_mode_priv(vcpu) ? SCTLR_ELx_EE : SCTLR_EL1_E0E;
533
534	return vcpu_read_sys_reg(vcpu, r) & bit;
535}
536
537static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,
538						    unsigned long data,
539						    unsigned int len)
540{
541	if (kvm_vcpu_is_be(vcpu)) {
542		switch (len) {
543		case 1:
544			return data & 0xff;
545		case 2:
546			return be16_to_cpu(data & 0xffff);
547		case 4:
548			return be32_to_cpu(data & 0xffffffff);
549		default:
550			return be64_to_cpu(data);
551		}
552	} else {
553		switch (len) {
554		case 1:
555			return data & 0xff;
556		case 2:
557			return le16_to_cpu(data & 0xffff);
558		case 4:
559			return le32_to_cpu(data & 0xffffffff);
560		default:
561			return le64_to_cpu(data);
562		}
563	}
564
565	return data;		/* Leave LE untouched */
566}
567
568static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
569						    unsigned long data,
570						    unsigned int len)
571{
572	if (kvm_vcpu_is_be(vcpu)) {
573		switch (len) {
574		case 1:
575			return data & 0xff;
576		case 2:
577			return cpu_to_be16(data & 0xffff);
578		case 4:
579			return cpu_to_be32(data & 0xffffffff);
580		default:
581			return cpu_to_be64(data);
582		}
583	} else {
584		switch (len) {
585		case 1:
586			return data & 0xff;
587		case 2:
588			return cpu_to_le16(data & 0xffff);
589		case 4:
590			return cpu_to_le32(data & 0xffffffff);
591		default:
592			return cpu_to_le64(data);
593		}
594	}
595
596	return data;		/* Leave LE untouched */
597}
598
599static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
600{
601	WARN_ON(vcpu_get_flag(vcpu, PENDING_EXCEPTION));
602	vcpu_set_flag(vcpu, INCREMENT_PC);
603}
604
605#define kvm_pend_exception(v, e)					\
606	do {								\
607		WARN_ON(vcpu_get_flag((v), INCREMENT_PC));		\
608		vcpu_set_flag((v), PENDING_EXCEPTION);			\
609		vcpu_set_flag((v), e);					\
610	} while (0)
611
612/*
613 * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE
614 * format if E2H isn't set.
615 */
616static inline u64 vcpu_sanitised_cptr_el2(const struct kvm_vcpu *vcpu)
617{
618	u64 cptr = __vcpu_sys_reg(vcpu, CPTR_EL2);
619
620	if (!vcpu_el2_e2h_is_set(vcpu))
621		cptr = translate_cptr_el2_to_cpacr_el1(cptr);
622
623	return cptr;
624}
625
626static inline bool ____cptr_xen_trap_enabled(const struct kvm_vcpu *vcpu,
627					     unsigned int xen)
628{
629	switch (xen) {
630	case 0b00:
631	case 0b10:
632		return true;
633	case 0b01:
634		return vcpu_el2_tge_is_set(vcpu) && !vcpu_is_el2(vcpu);
635	case 0b11:
636	default:
637		return false;
638	}
639}
640
641#define __guest_hyp_cptr_xen_trap_enabled(vcpu, xen)				\
642	(!vcpu_has_nv(vcpu) ? false :						\
643	 ____cptr_xen_trap_enabled(vcpu,					\
644				   SYS_FIELD_GET(CPACR_EL1, xen,		\
645						 vcpu_sanitised_cptr_el2(vcpu))))
646
647static inline bool guest_hyp_fpsimd_traps_enabled(const struct kvm_vcpu *vcpu)
648{
649	return __guest_hyp_cptr_xen_trap_enabled(vcpu, FPEN);
650}
651
652static inline bool guest_hyp_sve_traps_enabled(const struct kvm_vcpu *vcpu)
653{
654	return __guest_hyp_cptr_xen_trap_enabled(vcpu, ZEN);
655}
656
657static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu)
658{
659	struct kvm *kvm = vcpu->kvm;
660
661	if (cpus_have_final_cap(ARM64_HAS_HCX)) {
662		/*
663		 * In general, all HCRX_EL2 bits are gated by a feature.
664		 * The only reason we can set SMPME without checking any
665		 * feature is that its effects are not directly observable
666		 * from the guest.
667		 */
668		vcpu->arch.hcrx_el2 = HCRX_EL2_SMPME;
669
670		if (kvm_has_feat(kvm, ID_AA64ISAR2_EL1, MOPS, IMP))
671			vcpu->arch.hcrx_el2 |= (HCRX_EL2_MSCEn | HCRX_EL2_MCE2);
672
673		if (kvm_has_tcr2(kvm))
674			vcpu->arch.hcrx_el2 |= HCRX_EL2_TCR2En;
675
676		if (kvm_has_fpmr(kvm))
677			vcpu->arch.hcrx_el2 |= HCRX_EL2_EnFPM;
678
679		if (kvm_has_sctlr2(kvm))
680			vcpu->arch.hcrx_el2 |= HCRX_EL2_SCTLR2En;
681	}
682}
683#endif /* __ARM64_KVM_EMULATE_H__ */