arm.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
   4 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
   5 */
   6
   7#include <linux/bug.h>
   8#include <linux/cpu_pm.h>
   9#include <linux/errno.h>
  10#include <linux/err.h>
  11#include <linux/kvm_host.h>
  12#include <linux/list.h>
  13#include <linux/module.h>
  14#include <linux/vmalloc.h>
  15#include <linux/fs.h>
  16#include <linux/mman.h>
  17#include <linux/sched.h>
  18#include <linux/kvm.h>
  19#include <linux/kvm_irqfd.h>
  20#include <linux/irqbypass.h>
  21#include <linux/sched/stat.h>
  22#include <linux/psci.h>
  23#include <trace/events/kvm.h>
  24
  25#define CREATE_TRACE_POINTS
  26#include "trace_arm.h"
  27
  28#include <linux/uaccess.h>
  29#include <asm/ptrace.h>
  30#include <asm/mman.h>
  31#include <asm/tlbflush.h>
  32#include <asm/cacheflush.h>
  33#include <asm/cpufeature.h>
  34#include <asm/virt.h>
  35#include <asm/kvm_arm.h>
  36#include <asm/kvm_asm.h>
  37#include <asm/kvm_emulate.h>
  38#include <asm/kvm_mmu.h>
  39#include <asm/kvm_nested.h>
  40#include <asm/kvm_pkvm.h>
  41#include <asm/kvm_ptrauth.h>
  42#include <asm/sections.h>
  43
  44#include <kvm/arm_hypercalls.h>
  45#include <kvm/arm_pmu.h>
  46#include <kvm/arm_psci.h>
  47
  48#include "sys_regs.h"
  49
  50static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
  51
  52enum kvm_wfx_trap_policy {
  53	KVM_WFX_NOTRAP_SINGLE_TASK, /* Default option */
  54	KVM_WFX_NOTRAP,
  55	KVM_WFX_TRAP,
  56};
  57
  58static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
  59static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
  60
  61DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
  62
  63DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
  64DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
  65
  66DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
  67
  68static bool vgic_present, kvm_arm_initialised;
  69
  70static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized);
  71
  72bool is_kvm_arm_initialised(void)
  73{
  74	return kvm_arm_initialised;
  75}
  76
  77int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
  78{
  79	return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
  80}
  81
  82int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
  83			    struct kvm_enable_cap *cap)
  84{
  85	int r = -EINVAL;
  86
  87	if (cap->flags)
  88		return -EINVAL;
  89
  90	if (kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(cap->cap))
  91		return -EINVAL;
  92
  93	switch (cap->cap) {
  94	case KVM_CAP_ARM_NISV_TO_USER:
  95		r = 0;
  96		set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
  97			&kvm->arch.flags);
  98		break;
  99	case KVM_CAP_ARM_MTE:
 100		mutex_lock(&kvm->lock);
 101		if (system_supports_mte() && !kvm->created_vcpus) {
 102			r = 0;
 103			set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
 104		}
 105		mutex_unlock(&kvm->lock);
 106		break;
 107	case KVM_CAP_ARM_SYSTEM_SUSPEND:
 108		r = 0;
 109		set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
 110		break;
 111	case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
 112		mutex_lock(&kvm->slots_lock);
 113		/*
 114		 * To keep things simple, allow changing the chunk
 115		 * size only when no memory slots have been created.
 116		 */
 117		if (kvm_are_all_memslots_empty(kvm)) {
 118			u64 new_cap = cap->args[0];
 119
 120			if (!new_cap || kvm_is_block_size_supported(new_cap)) {
 121				r = 0;
 122				kvm->arch.mmu.split_page_chunk_size = new_cap;
 123			}
 124		}
 125		mutex_unlock(&kvm->slots_lock);
 126		break;
 127	case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
 128		mutex_lock(&kvm->lock);
 129		if (!kvm->created_vcpus) {
 130			r = 0;
 131			set_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags);
 132		}
 133		mutex_unlock(&kvm->lock);
 134		break;
 135	case KVM_CAP_ARM_SEA_TO_USER:
 136		r = 0;
 137		set_bit(KVM_ARCH_FLAG_EXIT_SEA, &kvm->arch.flags);
 138		break;
 139	default:
 140		break;
 141	}
 142
 143	return r;
 144}
 145
 146static int kvm_arm_default_max_vcpus(void)
 147{
 148	return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
 149}
 150
 151/**
 152 * kvm_arch_init_vm - initializes a VM data structure
 153 * @kvm:	pointer to the KVM struct
 154 * @type:	kvm device type
 155 */
 156int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 157{
 158	int ret;
 159
 160	mutex_init(&kvm->arch.config_lock);
 161
 162#ifdef CONFIG_LOCKDEP
 163	/* Clue in lockdep that the config_lock must be taken inside kvm->lock */
 164	mutex_lock(&kvm->lock);
 165	mutex_lock(&kvm->arch.config_lock);
 166	mutex_unlock(&kvm->arch.config_lock);
 167	mutex_unlock(&kvm->lock);
 168#endif
 169
 170	kvm_init_nested(kvm);
 171
 172	ret = kvm_share_hyp(kvm, kvm + 1);
 173	if (ret)
 174		return ret;
 175
 176	if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) {
 177		ret = -ENOMEM;
 178		goto err_unshare_kvm;
 179	}
 180	cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
 181
 182	ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
 183	if (ret)
 184		goto err_free_cpumask;
 185
 186	if (is_protected_kvm_enabled()) {
 187		/*
 188		 * If any failures occur after this is successful, make sure to
 189		 * call __pkvm_unreserve_vm to unreserve the VM in hyp.
 190		 */
 191		ret = pkvm_init_host_vm(kvm);
 192		if (ret)
 193			goto err_free_cpumask;
 194	}
 195
 196	kvm_vgic_early_init(kvm);
 197
 198	kvm_timer_init_vm(kvm);
 199
 200	/* The maximum number of VCPUs is limited by the host's GIC model */
 201	kvm->max_vcpus = kvm_arm_default_max_vcpus();
 202
 203	kvm_arm_init_hypercalls(kvm);
 204
 205	bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);
 206
 207	return 0;
 208
 209err_free_cpumask:
 210	free_cpumask_var(kvm->arch.supported_cpus);
 211err_unshare_kvm:
 212	kvm_unshare_hyp(kvm, kvm + 1);
 213	return ret;
 214}
 215
 216vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
 217{
 218	return VM_FAULT_SIGBUS;
 219}
 220
 221void kvm_arch_create_vm_debugfs(struct kvm *kvm)
 222{
 223	kvm_sys_regs_create_debugfs(kvm);
 224	kvm_s2_ptdump_create_debugfs(kvm);
 225}
 226
 227static void kvm_destroy_mpidr_data(struct kvm *kvm)
 228{
 229	struct kvm_mpidr_data *data;
 230
 231	mutex_lock(&kvm->arch.config_lock);
 232
 233	data = rcu_dereference_protected(kvm->arch.mpidr_data,
 234					 lockdep_is_held(&kvm->arch.config_lock));
 235	if (data) {
 236		rcu_assign_pointer(kvm->arch.mpidr_data, NULL);
 237		synchronize_rcu();
 238		kfree(data);
 239	}
 240
 241	mutex_unlock(&kvm->arch.config_lock);
 242}
 243
 244/**
 245 * kvm_arch_destroy_vm - destroy the VM data structure
 246 * @kvm:	pointer to the KVM struct
 247 */
 248void kvm_arch_destroy_vm(struct kvm *kvm)
 249{
 250	bitmap_free(kvm->arch.pmu_filter);
 251	free_cpumask_var(kvm->arch.supported_cpus);
 252
 253	kvm_vgic_destroy(kvm);
 254
 255	if (is_protected_kvm_enabled())
 256		pkvm_destroy_hyp_vm(kvm);
 257
 258	kvm_destroy_mpidr_data(kvm);
 259
 260	kfree(kvm->arch.sysreg_masks);
 261	kvm_destroy_vcpus(kvm);
 262
 263	kvm_unshare_hyp(kvm, kvm + 1);
 264
 265	kvm_arm_teardown_hypercalls(kvm);
 266}
 267
 268static bool kvm_has_full_ptr_auth(void)
 269{
 270	bool apa, gpa, api, gpi, apa3, gpa3;
 271	u64 isar1, isar2, val;
 272
 273	/*
 274	 * Check that:
 275	 *
 276	 * - both Address and Generic auth are implemented for a given
 277         *   algorithm (Q5, IMPDEF or Q3)
 278	 * - only a single algorithm is implemented.
 279	 */
 280	if (!system_has_full_ptr_auth())
 281		return false;
 282
 283	isar1 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
 284	isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
 285
 286	apa = !!FIELD_GET(ID_AA64ISAR1_EL1_APA_MASK, isar1);
 287	val = FIELD_GET(ID_AA64ISAR1_EL1_GPA_MASK, isar1);
 288	gpa = (val == ID_AA64ISAR1_EL1_GPA_IMP);
 289
 290	api = !!FIELD_GET(ID_AA64ISAR1_EL1_API_MASK, isar1);
 291	val = FIELD_GET(ID_AA64ISAR1_EL1_GPI_MASK, isar1);
 292	gpi = (val == ID_AA64ISAR1_EL1_GPI_IMP);
 293
 294	apa3 = !!FIELD_GET(ID_AA64ISAR2_EL1_APA3_MASK, isar2);
 295	val  = FIELD_GET(ID_AA64ISAR2_EL1_GPA3_MASK, isar2);
 296	gpa3 = (val == ID_AA64ISAR2_EL1_GPA3_IMP);
 297
 298	return (apa == gpa && api == gpi && apa3 == gpa3 &&
 299		(apa + api + apa3) == 1);
 300}
 301
 302int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 303{
 304	int r;
 305
 306	if (kvm && kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(ext))
 307		return 0;
 308
 309	switch (ext) {
 310	case KVM_CAP_IRQCHIP:
 311		r = vgic_present;
 312		break;
 313	case KVM_CAP_IOEVENTFD:
 314	case KVM_CAP_USER_MEMORY:
 315	case KVM_CAP_SYNC_MMU:
 316	case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
 317	case KVM_CAP_ONE_REG:
 318	case KVM_CAP_ARM_PSCI:
 319	case KVM_CAP_ARM_PSCI_0_2:
 320	case KVM_CAP_READONLY_MEM:
 321	case KVM_CAP_MP_STATE:
 322	case KVM_CAP_IMMEDIATE_EXIT:
 323	case KVM_CAP_VCPU_EVENTS:
 324	case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
 325	case KVM_CAP_ARM_NISV_TO_USER:
 326	case KVM_CAP_ARM_INJECT_EXT_DABT:
 327	case KVM_CAP_SET_GUEST_DEBUG:
 328	case KVM_CAP_VCPU_ATTRIBUTES:
 329	case KVM_CAP_PTP_KVM:
 330	case KVM_CAP_ARM_SYSTEM_SUSPEND:
 331	case KVM_CAP_IRQFD_RESAMPLE:
 332	case KVM_CAP_COUNTER_OFFSET:
 333	case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
 334	case KVM_CAP_ARM_SEA_TO_USER:
 335		r = 1;
 336		break;
 337	case KVM_CAP_SET_GUEST_DEBUG2:
 338		return KVM_GUESTDBG_VALID_MASK;
 339	case KVM_CAP_ARM_SET_DEVICE_ADDR:
 340		r = 1;
 341		break;
 342	case KVM_CAP_NR_VCPUS:
 343		/*
 344		 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
 345		 * architectures, as it does not always bound it to
 346		 * KVM_CAP_MAX_VCPUS. It should not matter much because
 347		 * this is just an advisory value.
 348		 */
 349		r = min_t(unsigned int, num_online_cpus(),
 350			  kvm_arm_default_max_vcpus());
 351		break;
 352	case KVM_CAP_MAX_VCPUS:
 353	case KVM_CAP_MAX_VCPU_ID:
 354		if (kvm)
 355			r = kvm->max_vcpus;
 356		else
 357			r = kvm_arm_default_max_vcpus();
 358		break;
 359	case KVM_CAP_MSI_DEVID:
 360		if (!kvm)
 361			r = -EINVAL;
 362		else
 363			r = kvm->arch.vgic.msis_require_devid;
 364		break;
 365	case KVM_CAP_ARM_USER_IRQ:
 366		/*
 367		 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
 368		 * (bump this number if adding more devices)
 369		 */
 370		r = 1;
 371		break;
 372	case KVM_CAP_ARM_MTE:
 373		r = system_supports_mte();
 374		break;
 375	case KVM_CAP_STEAL_TIME:
 376		r = kvm_arm_pvtime_supported();
 377		break;
 378	case KVM_CAP_ARM_EL1_32BIT:
 379		r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
 380		break;
 381	case KVM_CAP_ARM_EL2:
 382		r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT);
 383		break;
 384	case KVM_CAP_ARM_EL2_E2H0:
 385		r = cpus_have_final_cap(ARM64_HAS_HCR_NV1);
 386		break;
 387	case KVM_CAP_GUEST_DEBUG_HW_BPS:
 388		r = get_num_brps();
 389		break;
 390	case KVM_CAP_GUEST_DEBUG_HW_WPS:
 391		r = get_num_wrps();
 392		break;
 393	case KVM_CAP_ARM_PMU_V3:
 394		r = kvm_supports_guest_pmuv3();
 395		break;
 396	case KVM_CAP_ARM_INJECT_SERROR_ESR:
 397		r = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
 398		break;
 399	case KVM_CAP_ARM_VM_IPA_SIZE:
 400		r = get_kvm_ipa_limit();
 401		break;
 402	case KVM_CAP_ARM_SVE:
 403		r = system_supports_sve();
 404		break;
 405	case KVM_CAP_ARM_PTRAUTH_ADDRESS:
 406	case KVM_CAP_ARM_PTRAUTH_GENERIC:
 407		r = kvm_has_full_ptr_auth();
 408		break;
 409	case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
 410		if (kvm)
 411			r = kvm->arch.mmu.split_page_chunk_size;
 412		else
 413			r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
 414		break;
 415	case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
 416		r = kvm_supported_block_sizes();
 417		break;
 418	case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
 419		r = BIT(0);
 420		break;
 421	case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED:
 422		if (!kvm)
 423			r = -EINVAL;
 424		else
 425			r = kvm_supports_cacheable_pfnmap();
 426		break;
 427
 428	default:
 429		r = 0;
 430	}
 431
 432	return r;
 433}
 434
 435long kvm_arch_dev_ioctl(struct file *filp,
 436			unsigned int ioctl, unsigned long arg)
 437{
 438	return -EINVAL;
 439}
 440
 441struct kvm *kvm_arch_alloc_vm(void)
 442{
 443	size_t sz = sizeof(struct kvm);
 444
 445	if (!has_vhe())
 446		return kzalloc(sz, GFP_KERNEL_ACCOUNT);
 447
 448	return kvzalloc(sz, GFP_KERNEL_ACCOUNT);
 449}
 450
 451int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
 452{
 453	if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
 454		return -EBUSY;
 455
 456	if (id >= kvm->max_vcpus)
 457		return -EINVAL;
 458
 459	return 0;
 460}
 461
 462int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 463{
 464	int err;
 465
 466	spin_lock_init(&vcpu->arch.mp_state_lock);
 467
 468#ifdef CONFIG_LOCKDEP
 469	/* Inform lockdep that the config_lock is acquired after vcpu->mutex */
 470	mutex_lock(&vcpu->mutex);
 471	mutex_lock(&vcpu->kvm->arch.config_lock);
 472	mutex_unlock(&vcpu->kvm->arch.config_lock);
 473	mutex_unlock(&vcpu->mutex);
 474#endif
 475
 476	/* Force users to call KVM_ARM_VCPU_INIT */
 477	vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
 478
 479	vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
 480
 481	/* Set up the timer */
 482	kvm_timer_vcpu_init(vcpu);
 483
 484	kvm_pmu_vcpu_init(vcpu);
 485
 486	kvm_arm_pvtime_vcpu_init(&vcpu->arch);
 487
 488	vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
 489
 490	/*
 491	 * This vCPU may have been created after mpidr_data was initialized.
 492	 * Throw out the pre-computed mappings if that is the case which forces
 493	 * KVM to fall back to iteratively searching the vCPUs.
 494	 */
 495	kvm_destroy_mpidr_data(vcpu->kvm);
 496
 497	err = kvm_vgic_vcpu_init(vcpu);
 498	if (err)
 499		return err;
 500
 501	err = kvm_share_hyp(vcpu, vcpu + 1);
 502	if (err)
 503		kvm_vgic_vcpu_destroy(vcpu);
 504
 505	return err;
 506}
 507
 508void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 509{
 510}
 511
 512void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 513{
 514	if (!is_protected_kvm_enabled())
 515		kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
 516	else
 517		free_hyp_memcache(&vcpu->arch.pkvm_memcache);
 518	kvm_timer_vcpu_terminate(vcpu);
 519	kvm_pmu_vcpu_destroy(vcpu);
 520	kvm_vgic_vcpu_destroy(vcpu);
 521	kvm_arm_vcpu_destroy(vcpu);
 522}
 523
 524void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
 525{
 526
 527}
 528
 529void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
 530{
 531
 532}
 533
 534static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
 535{
 536	if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) {
 537		/*
 538		 * Either we're running an L2 guest, and the API/APK bits come
 539		 * from L1's HCR_EL2, or API/APK are both set.
 540		 */
 541		if (unlikely(is_nested_ctxt(vcpu))) {
 542			u64 val;
 543
 544			val = __vcpu_sys_reg(vcpu, HCR_EL2);
 545			val &= (HCR_API | HCR_APK);
 546			vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
 547			vcpu->arch.hcr_el2 |= val;
 548		} else {
 549			vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
 550		}
 551
 552		/*
 553		 * Save the host keys if there is any chance for the guest
 554		 * to use pauth, as the entry code will reload the guest
 555		 * keys in that case.
 556		 */
 557		if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
 558			struct kvm_cpu_context *ctxt;
 559
 560			ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
 561			ptrauth_save_keys(ctxt);
 562		}
 563	}
 564}
 565
 566static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu)
 567{
 568	if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
 569		return kvm_wfi_trap_policy == KVM_WFX_NOTRAP;
 570
 571	return single_task_running() &&
 572	       vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3 &&
 573	       (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
 574		vcpu->kvm->arch.vgic.nassgireq);
 575}
 576
 577static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu)
 578{
 579	if (unlikely(kvm_wfe_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
 580		return kvm_wfe_trap_policy == KVM_WFX_NOTRAP;
 581
 582	return single_task_running();
 583}
 584
 585void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 586{
 587	struct kvm_s2_mmu *mmu;
 588	int *last_ran;
 589
 590	if (is_protected_kvm_enabled())
 591		goto nommu;
 592
 593	if (vcpu_has_nv(vcpu))
 594		kvm_vcpu_load_hw_mmu(vcpu);
 595
 596	mmu = vcpu->arch.hw_mmu;
 597	last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
 598
 599	/*
 600	 * Ensure a VMID is allocated for the MMU before programming VTTBR_EL2,
 601	 * which happens eagerly in VHE.
 602	 *
 603	 * Also, the VMID allocator only preserves VMIDs that are active at the
 604	 * time of rollover, so KVM might need to grab a new VMID for the MMU if
 605	 * this is called from kvm_sched_in().
 606	 */
 607	kvm_arm_vmid_update(&mmu->vmid);
 608
 609	/*
 610	 * We guarantee that both TLBs and I-cache are private to each
 611	 * vcpu. If detecting that a vcpu from the same VM has
 612	 * previously run on the same physical CPU, call into the
 613	 * hypervisor code to nuke the relevant contexts.
 614	 *
 615	 * We might get preempted before the vCPU actually runs, but
 616	 * over-invalidation doesn't affect correctness.
 617	 */
 618	if (*last_ran != vcpu->vcpu_idx) {
 619		kvm_call_hyp(__kvm_flush_cpu_context, mmu);
 620		*last_ran = vcpu->vcpu_idx;
 621	}
 622
 623nommu:
 624	vcpu->cpu = cpu;
 625
 626	/*
 627	 * The timer must be loaded before the vgic to correctly set up physical
 628	 * interrupt deactivation in nested state (e.g. timer interrupt).
 629	 */
 630	kvm_timer_vcpu_load(vcpu);
 631	kvm_vgic_load(vcpu);
 632	kvm_vcpu_load_debug(vcpu);
 633	kvm_vcpu_load_fgt(vcpu);
 634	if (has_vhe())
 635		kvm_vcpu_load_vhe(vcpu);
 636	kvm_arch_vcpu_load_fp(vcpu);
 637	kvm_vcpu_pmu_restore_guest(vcpu);
 638	if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
 639		kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
 640
 641	if (kvm_vcpu_should_clear_twe(vcpu))
 642		vcpu->arch.hcr_el2 &= ~HCR_TWE;
 643	else
 644		vcpu->arch.hcr_el2 |= HCR_TWE;
 645
 646	if (kvm_vcpu_should_clear_twi(vcpu))
 647		vcpu->arch.hcr_el2 &= ~HCR_TWI;
 648	else
 649		vcpu->arch.hcr_el2 |= HCR_TWI;
 650
 651	vcpu_set_pauth_traps(vcpu);
 652
 653	if (is_protected_kvm_enabled()) {
 654		kvm_call_hyp_nvhe(__pkvm_vcpu_load,
 655				  vcpu->kvm->arch.pkvm.handle,
 656				  vcpu->vcpu_idx, vcpu->arch.hcr_el2);
 657		kvm_call_hyp(__vgic_v3_restore_vmcr_aprs,
 658			     &vcpu->arch.vgic_cpu.vgic_v3);
 659	}
 660
 661	if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
 662		vcpu_set_on_unsupported_cpu(vcpu);
 663}
 664
 665void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 666{
 667	if (is_protected_kvm_enabled()) {
 668		kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3);
 669		kvm_call_hyp_nvhe(__pkvm_vcpu_put);
 670	}
 671
 672	kvm_vcpu_put_debug(vcpu);
 673	kvm_arch_vcpu_put_fp(vcpu);
 674	if (has_vhe())
 675		kvm_vcpu_put_vhe(vcpu);
 676	kvm_timer_vcpu_put(vcpu);
 677	kvm_vgic_put(vcpu);
 678	kvm_vcpu_pmu_restore_host(vcpu);
 679	if (vcpu_has_nv(vcpu))
 680		kvm_vcpu_put_hw_mmu(vcpu);
 681	kvm_arm_vmid_clear_active();
 682
 683	vcpu_clear_on_unsupported_cpu(vcpu);
 684	vcpu->cpu = -1;
 685}
 686
 687static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
 688{
 689	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
 690	kvm_make_request(KVM_REQ_SLEEP, vcpu);
 691	kvm_vcpu_kick(vcpu);
 692}
 693
 694void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
 695{
 696	spin_lock(&vcpu->arch.mp_state_lock);
 697	__kvm_arm_vcpu_power_off(vcpu);
 698	spin_unlock(&vcpu->arch.mp_state_lock);
 699}
 700
 701bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
 702{
 703	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
 704}
 705
 706static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
 707{
 708	WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED);
 709	kvm_make_request(KVM_REQ_SUSPEND, vcpu);
 710	kvm_vcpu_kick(vcpu);
 711}
 712
 713static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
 714{
 715	return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;
 716}
 717
 718int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
 719				    struct kvm_mp_state *mp_state)
 720{
 721	*mp_state = READ_ONCE(vcpu->arch.mp_state);
 722
 723	return 0;
 724}
 725
 726int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
 727				    struct kvm_mp_state *mp_state)
 728{
 729	int ret = 0;
 730
 731	spin_lock(&vcpu->arch.mp_state_lock);
 732
 733	switch (mp_state->mp_state) {
 734	case KVM_MP_STATE_RUNNABLE:
 735		WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
 736		break;
 737	case KVM_MP_STATE_STOPPED:
 738		__kvm_arm_vcpu_power_off(vcpu);
 739		break;
 740	case KVM_MP_STATE_SUSPENDED:
 741		kvm_arm_vcpu_suspend(vcpu);
 742		break;
 743	default:
 744		ret = -EINVAL;
 745	}
 746
 747	spin_unlock(&vcpu->arch.mp_state_lock);
 748
 749	return ret;
 750}
 751
 752/**
 753 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
 754 * @v:		The VCPU pointer
 755 *
 756 * If the guest CPU is not waiting for interrupts or an interrupt line is
 757 * asserted, the CPU is by definition runnable.
 758 */
 759int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
 760{
 761	bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE);
 762
 763	return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
 764		&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
 765}
 766
 767bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
 768{
 769	return vcpu_mode_priv(vcpu);
 770}
 771
 772#ifdef CONFIG_GUEST_PERF_EVENTS
 773unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
 774{
 775	return *vcpu_pc(vcpu);
 776}
 777#endif
 778
 779static void kvm_init_mpidr_data(struct kvm *kvm)
 780{
 781	struct kvm_mpidr_data *data = NULL;
 782	unsigned long c, mask, nr_entries;
 783	u64 aff_set = 0, aff_clr = ~0UL;
 784	struct kvm_vcpu *vcpu;
 785
 786	mutex_lock(&kvm->arch.config_lock);
 787
 788	if (rcu_access_pointer(kvm->arch.mpidr_data) ||
 789	    atomic_read(&kvm->online_vcpus) == 1)
 790		goto out;
 791
 792	kvm_for_each_vcpu(c, vcpu, kvm) {
 793		u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
 794		aff_set |= aff;
 795		aff_clr &= aff;
 796	}
 797
 798	/*
 799	 * A significant bit can be either 0 or 1, and will only appear in
 800	 * aff_set. Use aff_clr to weed out the useless stuff.
 801	 */
 802	mask = aff_set ^ aff_clr;
 803	nr_entries = BIT_ULL(hweight_long(mask));
 804
 805	/*
 806	 * Don't let userspace fool us. If we need more than a single page
 807	 * to describe the compressed MPIDR array, just fall back to the
 808	 * iterative method. Single vcpu VMs do not need this either.
 809	 */
 810	if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE)
 811		data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries),
 812			       GFP_KERNEL_ACCOUNT);
 813
 814	if (!data)
 815		goto out;
 816
 817	data->mpidr_mask = mask;
 818
 819	kvm_for_each_vcpu(c, vcpu, kvm) {
 820		u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
 821		u16 index = kvm_mpidr_index(data, aff);
 822
 823		data->cmpidr_to_idx[index] = c;
 824	}
 825
 826	rcu_assign_pointer(kvm->arch.mpidr_data, data);
 827out:
 828	mutex_unlock(&kvm->arch.config_lock);
 829}
 830
 831/*
 832 * Handle both the initialisation that is being done when the vcpu is
 833 * run for the first time, as well as the updates that must be
 834 * performed each time we get a new thread dealing with this vcpu.
 835 */
 836int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
 837{
 838	struct kvm *kvm = vcpu->kvm;
 839	int ret;
 840
 841	if (!kvm_vcpu_initialized(vcpu))
 842		return -ENOEXEC;
 843
 844	if (!kvm_arm_vcpu_is_finalized(vcpu))
 845		return -EPERM;
 846
 847	if (likely(vcpu_has_run_once(vcpu)))
 848		return 0;
 849
 850	kvm_init_mpidr_data(kvm);
 851
 852	if (likely(irqchip_in_kernel(kvm))) {
 853		/*
 854		 * Map the VGIC hardware resources before running a vcpu the
 855		 * first time on this VM.
 856		 */
 857		ret = kvm_vgic_map_resources(kvm);
 858		if (ret)
 859			return ret;
 860	}
 861
 862	ret = kvm_finalize_sys_regs(vcpu);
 863	if (ret)
 864		return ret;
 865
 866	if (vcpu_has_nv(vcpu)) {
 867		ret = kvm_vcpu_allocate_vncr_tlb(vcpu);
 868		if (ret)
 869			return ret;
 870
 871		ret = kvm_vgic_vcpu_nv_init(vcpu);
 872		if (ret)
 873			return ret;
 874	}
 875
 876	/*
 877	 * This needs to happen after any restriction has been applied
 878	 * to the feature set.
 879	 */
 880	kvm_calculate_traps(vcpu);
 881
 882	ret = kvm_timer_enable(vcpu);
 883	if (ret)
 884		return ret;
 885
 886	if (kvm_vcpu_has_pmu(vcpu)) {
 887		ret = kvm_arm_pmu_v3_enable(vcpu);
 888		if (ret)
 889			return ret;
 890	}
 891
 892	if (is_protected_kvm_enabled()) {
 893		ret = pkvm_create_hyp_vm(kvm);
 894		if (ret)
 895			return ret;
 896
 897		ret = pkvm_create_hyp_vcpu(vcpu);
 898		if (ret)
 899			return ret;
 900	}
 901
 902	mutex_lock(&kvm->arch.config_lock);
 903	set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
 904	mutex_unlock(&kvm->arch.config_lock);
 905
 906	return ret;
 907}
 908
 909bool kvm_arch_intc_initialized(struct kvm *kvm)
 910{
 911	return vgic_initialized(kvm);
 912}
 913
 914void kvm_arm_halt_guest(struct kvm *kvm)
 915{
 916	unsigned long i;
 917	struct kvm_vcpu *vcpu;
 918
 919	kvm_for_each_vcpu(i, vcpu, kvm)
 920		vcpu->arch.pause = true;
 921	kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
 922}
 923
 924void kvm_arm_resume_guest(struct kvm *kvm)
 925{
 926	unsigned long i;
 927	struct kvm_vcpu *vcpu;
 928
 929	kvm_for_each_vcpu(i, vcpu, kvm) {
 930		vcpu->arch.pause = false;
 931		__kvm_vcpu_wake_up(vcpu);
 932	}
 933}
 934
 935static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
 936{
 937	struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
 938
 939	rcuwait_wait_event(wait,
 940			   (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
 941			   TASK_INTERRUPTIBLE);
 942
 943	if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
 944		/* Awaken to handle a signal, request we sleep again later. */
 945		kvm_make_request(KVM_REQ_SLEEP, vcpu);
 946	}
 947
 948	/*
 949	 * Make sure we will observe a potential reset request if we've
 950	 * observed a change to the power state. Pairs with the smp_wmb() in
 951	 * kvm_psci_vcpu_on().
 952	 */
 953	smp_rmb();
 954}
 955
 956/**
 957 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
 958 * @vcpu:	The VCPU pointer
 959 *
 960 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
 961 * the vCPU is runnable.  The vCPU may or may not be scheduled out, depending
 962 * on when a wake event arrives, e.g. there may already be a pending wake event.
 963 */
 964void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
 965{
 966	/*
 967	 * Sync back the state of the GIC CPU interface so that we have
 968	 * the latest PMR and group enables. This ensures that
 969	 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
 970	 * we have pending interrupts, e.g. when determining if the
 971	 * vCPU should block.
 972	 *
 973	 * For the same reason, we want to tell GICv4 that we need
 974	 * doorbells to be signalled, should an interrupt become pending.
 975	 */
 976	preempt_disable();
 977	vcpu_set_flag(vcpu, IN_WFI);
 978	kvm_vgic_put(vcpu);
 979	preempt_enable();
 980
 981	kvm_vcpu_halt(vcpu);
 982	vcpu_clear_flag(vcpu, IN_WFIT);
 983
 984	preempt_disable();
 985	vcpu_clear_flag(vcpu, IN_WFI);
 986	kvm_vgic_load(vcpu);
 987	preempt_enable();
 988}
 989
 990static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
 991{
 992	if (!kvm_arm_vcpu_suspended(vcpu))
 993		return 1;
 994
 995	kvm_vcpu_wfi(vcpu);
 996
 997	/*
 998	 * The suspend state is sticky; we do not leave it until userspace
 999	 * explicitly marks the vCPU as runnable. Request that we suspend again
1000	 * later.
1001	 */
1002	kvm_make_request(KVM_REQ_SUSPEND, vcpu);
1003
1004	/*
1005	 * Check to make sure the vCPU is actually runnable. If so, exit to
1006	 * userspace informing it of the wakeup condition.
1007	 */
1008	if (kvm_arch_vcpu_runnable(vcpu)) {
1009		memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
1010		vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP;
1011		vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
1012		return 0;
1013	}
1014
1015	/*
1016	 * Otherwise, we were unblocked to process a different event, such as a
1017	 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to
1018	 * process the event.
1019	 */
1020	return 1;
1021}
1022
1023/**
1024 * check_vcpu_requests - check and handle pending vCPU requests
1025 * @vcpu:	the VCPU pointer
1026 *
1027 * Return: 1 if we should enter the guest
1028 *	   0 if we should exit to userspace
1029 *	   < 0 if we should exit to userspace, where the return value indicates
1030 *	   an error
1031 */
1032static int check_vcpu_requests(struct kvm_vcpu *vcpu)
1033{
1034	if (kvm_request_pending(vcpu)) {
1035		if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu))
1036			return -EIO;
1037
1038		if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
1039			kvm_vcpu_sleep(vcpu);
1040
1041		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1042			kvm_reset_vcpu(vcpu);
1043
1044		/*
1045		 * Clear IRQ_PENDING requests that were made to guarantee
1046		 * that a VCPU sees new virtual interrupts.
1047		 */
1048		kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
1049
1050		/* Process interrupts deactivated through a trap */
1051		if (kvm_check_request(KVM_REQ_VGIC_PROCESS_UPDATE, vcpu))
1052			kvm_vgic_process_async_update(vcpu);
1053
1054		if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
1055			kvm_update_stolen_time(vcpu);
1056
1057		if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
1058			/* The distributor enable bits were changed */
1059			preempt_disable();
1060			vgic_v4_put(vcpu);
1061			vgic_v4_load(vcpu);
1062			preempt_enable();
1063		}
1064
1065		if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
1066			kvm_vcpu_reload_pmu(vcpu);
1067
1068		if (kvm_check_request(KVM_REQ_RESYNC_PMU_EL0, vcpu))
1069			kvm_vcpu_pmu_restore_guest(vcpu);
1070
1071		if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
1072			return kvm_vcpu_suspend(vcpu);
1073
1074		if (kvm_dirty_ring_check_request(vcpu))
1075			return 0;
1076
1077		check_nested_vcpu_requests(vcpu);
1078	}
1079
1080	return 1;
1081}
1082
1083static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
1084{
1085	if (likely(!vcpu_mode_is_32bit(vcpu)))
1086		return false;
1087
1088	if (vcpu_has_nv(vcpu))
1089		return true;
1090
1091	return !kvm_supports_32bit_el0();
1092}
1093
1094/**
1095 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
1096 * @vcpu:	The VCPU pointer
1097 * @ret:	Pointer to write optional return code
1098 *
1099 * Returns: true if the VCPU needs to return to a preemptible + interruptible
1100 *	    and skip guest entry.
1101 *
1102 * This function disambiguates between two different types of exits: exits to a
1103 * preemptible + interruptible kernel context and exits to userspace. For an
1104 * exit to userspace, this function will write the return code to ret and return
1105 * true. For an exit to preemptible + interruptible kernel context (i.e. check
1106 * for pending work and re-enter), return true without writing to ret.
1107 */
1108static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
1109{
1110	struct kvm_run *run = vcpu->run;
1111
1112	/*
1113	 * If we're using a userspace irqchip, then check if we need
1114	 * to tell a userspace irqchip about timer or PMU level
1115	 * changes and if so, exit to userspace (the actual level
1116	 * state gets updated in kvm_timer_update_run and
1117	 * kvm_pmu_update_run below).
1118	 */
1119	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1120		if (kvm_timer_should_notify_user(vcpu) ||
1121		    kvm_pmu_should_notify_user(vcpu)) {
1122			*ret = -EINTR;
1123			run->exit_reason = KVM_EXIT_INTR;
1124			return true;
1125		}
1126	}
1127
1128	if (unlikely(vcpu_on_unsupported_cpu(vcpu))) {
1129		run->exit_reason = KVM_EXIT_FAIL_ENTRY;
1130		run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED;
1131		run->fail_entry.cpu = smp_processor_id();
1132		*ret = 0;
1133		return true;
1134	}
1135
1136	return kvm_request_pending(vcpu) ||
1137			xfer_to_guest_mode_work_pending();
1138}
1139
1140/*
1141 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
1142 * the vCPU is running.
1143 *
1144 * This must be noinstr as instrumentation may make use of RCU, and this is not
1145 * safe during the EQS.
1146 */
1147static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
1148{
1149	int ret;
1150
1151	guest_state_enter_irqoff();
1152	ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
1153	guest_state_exit_irqoff();
1154
1155	return ret;
1156}
1157
1158/**
1159 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
1160 * @vcpu:	The VCPU pointer
1161 *
1162 * This function is called through the VCPU_RUN ioctl called from user space. It
1163 * will execute VM code in a loop until the time slice for the process is used
1164 * or some emulation is needed from user space in which case the function will
1165 * return with return value 0 and with the kvm_run structure filled in with the
1166 * required data for the requested emulation.
1167 */
1168int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1169{
1170	struct kvm_run *run = vcpu->run;
1171	int ret;
1172
1173	if (run->exit_reason == KVM_EXIT_MMIO) {
1174		ret = kvm_handle_mmio_return(vcpu);
1175		if (ret <= 0)
1176			return ret;
1177	}
1178
1179	vcpu_load(vcpu);
1180
1181	if (!vcpu->wants_to_run) {
1182		ret = -EINTR;
1183		goto out;
1184	}
1185
1186	kvm_sigset_activate(vcpu);
1187
1188	ret = 1;
1189	run->exit_reason = KVM_EXIT_UNKNOWN;
1190	run->flags = 0;
1191	while (ret > 0) {
1192		/*
1193		 * Check conditions before entering the guest
1194		 */
1195		ret = kvm_xfer_to_guest_mode_handle_work(vcpu);
1196		if (!ret)
1197			ret = 1;
1198
1199		if (ret > 0)
1200			ret = check_vcpu_requests(vcpu);
1201
1202		/*
1203		 * Preparing the interrupts to be injected also
1204		 * involves poking the GIC, which must be done in a
1205		 * non-preemptible context.
1206		 */
1207		preempt_disable();
1208
1209		kvm_nested_flush_hwstate(vcpu);
1210
1211		if (kvm_vcpu_has_pmu(vcpu))
1212			kvm_pmu_flush_hwstate(vcpu);
1213
1214		local_irq_disable();
1215
1216		kvm_vgic_flush_hwstate(vcpu);
1217
1218		kvm_pmu_update_vcpu_events(vcpu);
1219
1220		/*
1221		 * Ensure we set mode to IN_GUEST_MODE after we disable
1222		 * interrupts and before the final VCPU requests check.
1223		 * See the comment in kvm_vcpu_exiting_guest_mode() and
1224		 * Documentation/virt/kvm/vcpu-requests.rst
1225		 */
1226		smp_store_mb(vcpu->mode, IN_GUEST_MODE);
1227
1228		if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
1229			vcpu->mode = OUTSIDE_GUEST_MODE;
1230			isb(); /* Ensure work in x_flush_hwstate is committed */
1231			if (kvm_vcpu_has_pmu(vcpu))
1232				kvm_pmu_sync_hwstate(vcpu);
1233			if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1234				kvm_timer_sync_user(vcpu);
1235			kvm_vgic_sync_hwstate(vcpu);
1236			local_irq_enable();
1237			preempt_enable();
1238			continue;
1239		}
1240
1241		kvm_arch_vcpu_ctxflush_fp(vcpu);
1242
1243		/**************************************************************
1244		 * Enter the guest
1245		 */
1246		trace_kvm_entry(*vcpu_pc(vcpu));
1247		guest_timing_enter_irqoff();
1248
1249		ret = kvm_arm_vcpu_enter_exit(vcpu);
1250
1251		vcpu->mode = OUTSIDE_GUEST_MODE;
1252		vcpu->stat.exits++;
1253		/*
1254		 * Back from guest
1255		 *************************************************************/
1256
1257		/*
1258		 * We must sync the PMU state before the vgic state so
1259		 * that the vgic can properly sample the updated state of the
1260		 * interrupt line.
1261		 */
1262		if (kvm_vcpu_has_pmu(vcpu))
1263			kvm_pmu_sync_hwstate(vcpu);
1264
1265		/*
1266		 * Sync the vgic state before syncing the timer state because
1267		 * the timer code needs to know if the virtual timer
1268		 * interrupts are active.
1269		 */
1270		kvm_vgic_sync_hwstate(vcpu);
1271
1272		/*
1273		 * Sync the timer hardware state before enabling interrupts as
1274		 * we don't want vtimer interrupts to race with syncing the
1275		 * timer virtual interrupt state.
1276		 */
1277		if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1278			kvm_timer_sync_user(vcpu);
1279
1280		if (is_hyp_ctxt(vcpu))
1281			kvm_timer_sync_nested(vcpu);
1282
1283		kvm_arch_vcpu_ctxsync_fp(vcpu);
1284
1285		/*
1286		 * We must ensure that any pending interrupts are taken before
1287		 * we exit guest timing so that timer ticks are accounted as
1288		 * guest time. Transiently unmask interrupts so that any
1289		 * pending interrupts are taken.
1290		 *
1291		 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
1292		 * context synchronization event) is necessary to ensure that
1293		 * pending interrupts are taken.
1294		 */
1295		if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) {
1296			local_irq_enable();
1297			isb();
1298			local_irq_disable();
1299		}
1300
1301		guest_timing_exit_irqoff();
1302
1303		local_irq_enable();
1304
1305		trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
1306
1307		/* Exit types that need handling before we can be preempted */
1308		handle_exit_early(vcpu, ret);
1309
1310		kvm_nested_sync_hwstate(vcpu);
1311
1312		preempt_enable();
1313
1314		/*
1315		 * The ARMv8 architecture doesn't give the hypervisor
1316		 * a mechanism to prevent a guest from dropping to AArch32 EL0
1317		 * if implemented by the CPU. If we spot the guest in such
1318		 * state and that we decided it wasn't supposed to do so (like
1319		 * with the asymmetric AArch32 case), return to userspace with
1320		 * a fatal error.
1321		 */
1322		if (vcpu_mode_is_bad_32bit(vcpu)) {
1323			/*
1324			 * As we have caught the guest red-handed, decide that
1325			 * it isn't fit for purpose anymore by making the vcpu
1326			 * invalid. The VMM can try and fix it by issuing  a
1327			 * KVM_ARM_VCPU_INIT if it really wants to.
1328			 */
1329			vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
1330			ret = ARM_EXCEPTION_IL;
1331		}
1332
1333		ret = handle_exit(vcpu, ret);
1334	}
1335
1336	/* Tell userspace about in-kernel device output levels */
1337	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1338		kvm_timer_update_run(vcpu);
1339		kvm_pmu_update_run(vcpu);
1340	}
1341
1342	kvm_sigset_deactivate(vcpu);
1343
1344out:
1345	/*
1346	 * In the unlikely event that we are returning to userspace
1347	 * with pending exceptions or PC adjustment, commit these
1348	 * adjustments in order to give userspace a consistent view of
1349	 * the vcpu state. Note that this relies on __kvm_adjust_pc()
1350	 * being preempt-safe on VHE.
1351	 */
1352	if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||
1353		     vcpu_get_flag(vcpu, INCREMENT_PC)))
1354		kvm_call_hyp(__kvm_adjust_pc, vcpu);
1355
1356	vcpu_put(vcpu);
1357	return ret;
1358}
1359
1360static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
1361{
1362	int bit_index;
1363	bool set;
1364	unsigned long *hcr;
1365
1366	if (number == KVM_ARM_IRQ_CPU_IRQ)
1367		bit_index = __ffs(HCR_VI);
1368	else /* KVM_ARM_IRQ_CPU_FIQ */
1369		bit_index = __ffs(HCR_VF);
1370
1371	hcr = vcpu_hcr(vcpu);
1372	if (level)
1373		set = test_and_set_bit(bit_index, hcr);
1374	else
1375		set = test_and_clear_bit(bit_index, hcr);
1376
1377	/*
1378	 * If we didn't change anything, no need to wake up or kick other CPUs
1379	 */
1380	if (set == level)
1381		return 0;
1382
1383	/*
1384	 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1385	 * trigger a world-switch round on the running physical CPU to set the
1386	 * virtual IRQ/FIQ fields in the HCR appropriately.
1387	 */
1388	kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1389	kvm_vcpu_kick(vcpu);
1390
1391	return 0;
1392}
1393
1394int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1395			  bool line_status)
1396{
1397	u32 irq = irq_level->irq;
1398	unsigned int irq_type, vcpu_id, irq_num;
1399	struct kvm_vcpu *vcpu = NULL;
1400	bool level = irq_level->level;
1401
1402	irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1403	vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1404	vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1405	irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1406
1407	trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level);
1408
1409	switch (irq_type) {
1410	case KVM_ARM_IRQ_TYPE_CPU:
1411		if (irqchip_in_kernel(kvm))
1412			return -ENXIO;
1413
1414		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1415		if (!vcpu)
1416			return -EINVAL;
1417
1418		if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1419			return -EINVAL;
1420
1421		return vcpu_interrupt_line(vcpu, irq_num, level);
1422	case KVM_ARM_IRQ_TYPE_PPI:
1423		if (!irqchip_in_kernel(kvm))
1424			return -ENXIO;
1425
1426		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1427		if (!vcpu)
1428			return -EINVAL;
1429
1430		if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1431			return -EINVAL;
1432
1433		return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
1434	case KVM_ARM_IRQ_TYPE_SPI:
1435		if (!irqchip_in_kernel(kvm))
1436			return -ENXIO;
1437
1438		if (irq_num < VGIC_NR_PRIVATE_IRQS)
1439			return -EINVAL;
1440
1441		return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
1442	}
1443
1444	return -EINVAL;
1445}
1446
1447static unsigned long system_supported_vcpu_features(void)
1448{
1449	unsigned long features = KVM_VCPU_VALID_FEATURES;
1450
1451	if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
1452		clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
1453
1454	if (!kvm_supports_guest_pmuv3())
1455		clear_bit(KVM_ARM_VCPU_PMU_V3, &features);
1456
1457	if (!system_supports_sve())
1458		clear_bit(KVM_ARM_VCPU_SVE, &features);
1459
1460	if (!kvm_has_full_ptr_auth()) {
1461		clear_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features);
1462		clear_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features);
1463	}
1464
1465	if (!cpus_have_final_cap(ARM64_HAS_NESTED_VIRT))
1466		clear_bit(KVM_ARM_VCPU_HAS_EL2, &features);
1467
1468	return features;
1469}
1470
1471static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
1472					const struct kvm_vcpu_init *init)
1473{
1474	unsigned long features = init->features[0];
1475	int i;
1476
1477	if (features & ~KVM_VCPU_VALID_FEATURES)
1478		return -ENOENT;
1479
1480	for (i = 1; i < ARRAY_SIZE(init->features); i++) {
1481		if (init->features[i])
1482			return -ENOENT;
1483	}
1484
1485	if (features & ~system_supported_vcpu_features())
1486		return -EINVAL;
1487
1488	/*
1489	 * For now make sure that both address/generic pointer authentication
1490	 * features are requested by the userspace together.
1491	 */
1492	if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features) !=
1493	    test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
1494		return -EINVAL;
1495
1496	if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
1497		return 0;
1498
1499	/* MTE is incompatible with AArch32 */
1500	if (kvm_has_mte(vcpu->kvm))
1501		return -EINVAL;
1502
1503	/* NV is incompatible with AArch32 */
1504	if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features))
1505		return -EINVAL;
1506
1507	return 0;
1508}
1509
1510static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu,
1511				  const struct kvm_vcpu_init *init)
1512{
1513	unsigned long features = init->features[0];
1514
1515	return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features,
1516			     KVM_VCPU_MAX_FEATURES);
1517}
1518
1519static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
1520{
1521	struct kvm *kvm = vcpu->kvm;
1522	int ret = 0;
1523
1524	/*
1525	 * When the vCPU has a PMU, but no PMU is set for the guest
1526	 * yet, set the default one.
1527	 */
1528	if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
1529		ret = kvm_arm_set_default_pmu(kvm);
1530
1531	/* Prepare for nested if required */
1532	if (!ret && vcpu_has_nv(vcpu))
1533		ret = kvm_vcpu_init_nested(vcpu);
1534
1535	return ret;
1536}
1537
1538static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1539				 const struct kvm_vcpu_init *init)
1540{
1541	unsigned long features = init->features[0];
1542	struct kvm *kvm = vcpu->kvm;
1543	int ret = -EINVAL;
1544
1545	mutex_lock(&kvm->arch.config_lock);
1546
1547	if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) &&
1548	    kvm_vcpu_init_changed(vcpu, init))
1549		goto out_unlock;
1550
1551	bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
1552
1553	ret = kvm_setup_vcpu(vcpu);
1554	if (ret)
1555		goto out_unlock;
1556
1557	/* Now we know what it is, we can reset it. */
1558	kvm_reset_vcpu(vcpu);
1559
1560	set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
1561	vcpu_set_flag(vcpu, VCPU_INITIALIZED);
1562	ret = 0;
1563out_unlock:
1564	mutex_unlock(&kvm->arch.config_lock);
1565	return ret;
1566}
1567
1568static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1569			       const struct kvm_vcpu_init *init)
1570{
1571	int ret;
1572
1573	if (init->target != KVM_ARM_TARGET_GENERIC_V8 &&
1574	    init->target != kvm_target_cpu())
1575		return -EINVAL;
1576
1577	ret = kvm_vcpu_init_check_features(vcpu, init);
1578	if (ret)
1579		return ret;
1580
1581	if (!kvm_vcpu_initialized(vcpu))
1582		return __kvm_vcpu_set_target(vcpu, init);
1583
1584	if (kvm_vcpu_init_changed(vcpu, init))
1585		return -EINVAL;
1586
1587	kvm_reset_vcpu(vcpu);
1588	return 0;
1589}
1590
1591static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1592					 struct kvm_vcpu_init *init)
1593{
1594	bool power_off = false;
1595	int ret;
1596
1597	/*
1598	 * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid
1599	 * reflecting it in the finalized feature set, thus limiting its scope
1600	 * to a single KVM_ARM_VCPU_INIT call.
1601	 */
1602	if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) {
1603		init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF);
1604		power_off = true;
1605	}
1606
1607	ret = kvm_vcpu_set_target(vcpu, init);
1608	if (ret)
1609		return ret;
1610
1611	/*
1612	 * Ensure a rebooted VM will fault in RAM pages and detect if the
1613	 * guest MMU is turned off and flush the caches as needed.
1614	 *
1615	 * S2FWB enforces all memory accesses to RAM being cacheable,
1616	 * ensuring that the data side is always coherent. We still
1617	 * need to invalidate the I-cache though, as FWB does *not*
1618	 * imply CTR_EL0.DIC.
1619	 */
1620	if (vcpu_has_run_once(vcpu)) {
1621		if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1622			stage2_unmap_vm(vcpu->kvm);
1623		else
1624			icache_inval_all_pou();
1625	}
1626
1627	vcpu_reset_hcr(vcpu);
1628
1629	/*
1630	 * Handle the "start in power-off" case.
1631	 */
1632	spin_lock(&vcpu->arch.mp_state_lock);
1633
1634	if (power_off)
1635		__kvm_arm_vcpu_power_off(vcpu);
1636	else
1637		WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
1638
1639	spin_unlock(&vcpu->arch.mp_state_lock);
1640
1641	return 0;
1642}
1643
1644static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1645				 struct kvm_device_attr *attr)
1646{
1647	int ret = -ENXIO;
1648
1649	switch (attr->group) {
1650	default:
1651		ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1652		break;
1653	}
1654
1655	return ret;
1656}
1657
1658static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1659				 struct kvm_device_attr *attr)
1660{
1661	int ret = -ENXIO;
1662
1663	switch (attr->group) {
1664	default:
1665		ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1666		break;
1667	}
1668
1669	return ret;
1670}
1671
1672static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1673				 struct kvm_device_attr *attr)
1674{
1675	int ret = -ENXIO;
1676
1677	switch (attr->group) {
1678	default:
1679		ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1680		break;
1681	}
1682
1683	return ret;
1684}
1685
1686static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1687				   struct kvm_vcpu_events *events)
1688{
1689	memset(events, 0, sizeof(*events));
1690
1691	return __kvm_arm_vcpu_get_events(vcpu, events);
1692}
1693
1694static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1695				   struct kvm_vcpu_events *events)
1696{
1697	int i;
1698
1699	/* check whether the reserved field is zero */
1700	for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1701		if (events->reserved[i])
1702			return -EINVAL;
1703
1704	/* check whether the pad field is zero */
1705	for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1706		if (events->exception.pad[i])
1707			return -EINVAL;
1708
1709	return __kvm_arm_vcpu_set_events(vcpu, events);
1710}
1711
1712long kvm_arch_vcpu_ioctl(struct file *filp,
1713			 unsigned int ioctl, unsigned long arg)
1714{
1715	struct kvm_vcpu *vcpu = filp->private_data;
1716	void __user *argp = (void __user *)arg;
1717	struct kvm_device_attr attr;
1718	long r;
1719
1720	switch (ioctl) {
1721	case KVM_ARM_VCPU_INIT: {
1722		struct kvm_vcpu_init init;
1723
1724		r = -EFAULT;
1725		if (copy_from_user(&init, argp, sizeof(init)))
1726			break;
1727
1728		r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1729		break;
1730	}
1731	case KVM_SET_ONE_REG:
1732	case KVM_GET_ONE_REG: {
1733		struct kvm_one_reg reg;
1734
1735		r = -ENOEXEC;
1736		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1737			break;
1738
1739		r = -EFAULT;
1740		if (copy_from_user(&reg, argp, sizeof(reg)))
1741			break;
1742
1743		/*
1744		 * We could owe a reset due to PSCI. Handle the pending reset
1745		 * here to ensure userspace register accesses are ordered after
1746		 * the reset.
1747		 */
1748		if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1749			kvm_reset_vcpu(vcpu);
1750
1751		if (ioctl == KVM_SET_ONE_REG)
1752			r = kvm_arm_set_reg(vcpu, &reg);
1753		else
1754			r = kvm_arm_get_reg(vcpu, &reg);
1755		break;
1756	}
1757	case KVM_GET_REG_LIST: {
1758		struct kvm_reg_list __user *user_list = argp;
1759		struct kvm_reg_list reg_list;
1760		unsigned n;
1761
1762		r = -ENOEXEC;
1763		if (unlikely(!kvm_vcpu_initialized(vcpu)))
1764			break;
1765
1766		r = -EPERM;
1767		if (!kvm_arm_vcpu_is_finalized(vcpu))
1768			break;
1769
1770		r = -EFAULT;
1771		if (copy_from_user(&reg_list, user_list, sizeof(reg_list)))
1772			break;
1773		n = reg_list.n;
1774		reg_list.n = kvm_arm_num_regs(vcpu);
1775		if (copy_to_user(user_list, &reg_list, sizeof(reg_list)))
1776			break;
1777		r = -E2BIG;
1778		if (n < reg_list.n)
1779			break;
1780		r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1781		break;
1782	}
1783	case KVM_SET_DEVICE_ATTR: {
1784		r = -EFAULT;
1785		if (copy_from_user(&attr, argp, sizeof(attr)))
1786			break;
1787		r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1788		break;
1789	}
1790	case KVM_GET_DEVICE_ATTR: {
1791		r = -EFAULT;
1792		if (copy_from_user(&attr, argp, sizeof(attr)))
1793			break;
1794		r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1795		break;
1796	}
1797	case KVM_HAS_DEVICE_ATTR: {
1798		r = -EFAULT;
1799		if (copy_from_user(&attr, argp, sizeof(attr)))
1800			break;
1801		r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1802		break;
1803	}
1804	case KVM_GET_VCPU_EVENTS: {
1805		struct kvm_vcpu_events events;
1806
1807		if (!kvm_vcpu_initialized(vcpu))
1808			return -ENOEXEC;
1809
1810		if (kvm_arm_vcpu_get_events(vcpu, &events))
1811			return -EINVAL;
1812
1813		if (copy_to_user(argp, &events, sizeof(events)))
1814			return -EFAULT;
1815
1816		return 0;
1817	}
1818	case KVM_SET_VCPU_EVENTS: {
1819		struct kvm_vcpu_events events;
1820
1821		if (!kvm_vcpu_initialized(vcpu))
1822			return -ENOEXEC;
1823
1824		if (copy_from_user(&events, argp, sizeof(events)))
1825			return -EFAULT;
1826
1827		return kvm_arm_vcpu_set_events(vcpu, &events);
1828	}
1829	case KVM_ARM_VCPU_FINALIZE: {
1830		int what;
1831
1832		if (!kvm_vcpu_initialized(vcpu))
1833			return -ENOEXEC;
1834
1835		if (get_user(what, (const int __user *)argp))
1836			return -EFAULT;
1837
1838		return kvm_arm_vcpu_finalize(vcpu, what);
1839	}
1840	default:
1841		r = -EINVAL;
1842	}
1843
1844	return r;
1845}
1846
1847long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
1848				  unsigned long arg)
1849{
1850	return -ENOIOCTLCMD;
1851}
1852
1853void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1854{
1855
1856}
1857
1858static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1859					struct kvm_arm_device_addr *dev_addr)
1860{
1861	switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) {
1862	case KVM_ARM_DEVICE_VGIC_V2:
1863		if (!vgic_present)
1864			return -ENXIO;
1865		return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
1866	default:
1867		return -ENODEV;
1868	}
1869}
1870
1871static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1872{
1873	switch (attr->group) {
1874	case KVM_ARM_VM_SMCCC_CTRL:
1875		return kvm_vm_smccc_has_attr(kvm, attr);
1876	default:
1877		return -ENXIO;
1878	}
1879}
1880
1881static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1882{
1883	switch (attr->group) {
1884	case KVM_ARM_VM_SMCCC_CTRL:
1885		return kvm_vm_smccc_set_attr(kvm, attr);
1886	default:
1887		return -ENXIO;
1888	}
1889}
1890
1891int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1892{
1893	struct kvm *kvm = filp->private_data;
1894	void __user *argp = (void __user *)arg;
1895	struct kvm_device_attr attr;
1896
1897	switch (ioctl) {
1898	case KVM_CREATE_IRQCHIP: {
1899		int ret;
1900		if (!vgic_present)
1901			return -ENXIO;
1902		mutex_lock(&kvm->lock);
1903		ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1904		mutex_unlock(&kvm->lock);
1905		return ret;
1906	}
1907	case KVM_ARM_SET_DEVICE_ADDR: {
1908		struct kvm_arm_device_addr dev_addr;
1909
1910		if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1911			return -EFAULT;
1912		return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1913	}
1914	case KVM_ARM_PREFERRED_TARGET: {
1915		struct kvm_vcpu_init init = {
1916			.target = KVM_ARM_TARGET_GENERIC_V8,
1917		};
1918
1919		if (copy_to_user(argp, &init, sizeof(init)))
1920			return -EFAULT;
1921
1922		return 0;
1923	}
1924	case KVM_ARM_MTE_COPY_TAGS: {
1925		struct kvm_arm_copy_mte_tags copy_tags;
1926
1927		if (copy_from_user(&copy_tags, argp, sizeof(copy_tags)))
1928			return -EFAULT;
1929		return kvm_vm_ioctl_mte_copy_tags(kvm, &copy_tags);
1930	}
1931	case KVM_ARM_SET_COUNTER_OFFSET: {
1932		struct kvm_arm_counter_offset offset;
1933
1934		if (copy_from_user(&offset, argp, sizeof(offset)))
1935			return -EFAULT;
1936		return kvm_vm_ioctl_set_counter_offset(kvm, &offset);
1937	}
1938	case KVM_HAS_DEVICE_ATTR: {
1939		if (copy_from_user(&attr, argp, sizeof(attr)))
1940			return -EFAULT;
1941
1942		return kvm_vm_has_attr(kvm, &attr);
1943	}
1944	case KVM_SET_DEVICE_ATTR: {
1945		if (copy_from_user(&attr, argp, sizeof(attr)))
1946			return -EFAULT;
1947
1948		return kvm_vm_set_attr(kvm, &attr);
1949	}
1950	case KVM_ARM_GET_REG_WRITABLE_MASKS: {
1951		struct reg_mask_range range;
1952
1953		if (copy_from_user(&range, argp, sizeof(range)))
1954			return -EFAULT;
1955		return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range);
1956	}
1957	default:
1958		return -EINVAL;
1959	}
1960}
1961
1962static unsigned long nvhe_percpu_size(void)
1963{
1964	return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1965		(unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1966}
1967
1968static unsigned long nvhe_percpu_order(void)
1969{
1970	unsigned long size = nvhe_percpu_size();
1971
1972	return size ? get_order(size) : 0;
1973}
1974
1975static size_t pkvm_host_sve_state_order(void)
1976{
1977	return get_order(pkvm_host_sve_state_size());
1978}
1979
1980/* A lookup table holding the hypervisor VA for each vector slot */
1981static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1982
1983static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1984{
1985	hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1986}
1987
1988static int kvm_init_vector_slots(void)
1989{
1990	int err;
1991	void *base;
1992
1993	base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1994	kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1995
1996	base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1997	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1998
1999	if (kvm_system_needs_idmapped_vectors() &&
2000	    !is_protected_kvm_enabled()) {
2001		err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
2002					       __BP_HARDEN_HYP_VECS_SZ, &base);
2003		if (err)
2004			return err;
2005	}
2006
2007	kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
2008	kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
2009	return 0;
2010}
2011
2012static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
2013{
2014	struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2015	unsigned long tcr;
2016
2017	/*
2018	 * Calculate the raw per-cpu offset without a translation from the
2019	 * kernel's mapping to the linear mapping, and store it in tpidr_el2
2020	 * so that we can use adr_l to access per-cpu variables in EL2.
2021	 * Also drop the KASAN tag which gets in the way...
2022	 */
2023	params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
2024			    (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
2025
2026	params->mair_el2 = read_sysreg(mair_el1);
2027
2028	tcr = read_sysreg(tcr_el1);
2029	if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
2030		tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK);
2031		tcr |= TCR_EPD1_MASK;
2032	} else {
2033		unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr);
2034
2035		tcr &= TCR_EL2_MASK;
2036		tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips);
2037		if (lpa2_is_enabled())
2038			tcr |= TCR_EL2_DS;
2039	}
2040	tcr |= TCR_T0SZ(hyp_va_bits);
2041	params->tcr_el2 = tcr;
2042
2043	params->pgd_pa = kvm_mmu_get_httbr();
2044	if (is_protected_kvm_enabled())
2045		params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
2046	else
2047		params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
2048	if (cpus_have_final_cap(ARM64_KVM_HVHE))
2049		params->hcr_el2 |= HCR_E2H;
2050	params->vttbr = params->vtcr = 0;
2051
2052	/*
2053	 * Flush the init params from the data cache because the struct will
2054	 * be read while the MMU is off.
2055	 */
2056	kvm_flush_dcache_to_poc(params, sizeof(*params));
2057}
2058
2059static void hyp_install_host_vector(void)
2060{
2061	struct kvm_nvhe_init_params *params;
2062	struct arm_smccc_res res;
2063
2064	/* Switch from the HYP stub to our own HYP init vector */
2065	__hyp_set_vectors(kvm_get_idmap_vector());
2066
2067	/*
2068	 * Call initialization code, and switch to the full blown HYP code.
2069	 * If the cpucaps haven't been finalized yet, something has gone very
2070	 * wrong, and hyp will crash and burn when it uses any
2071	 * cpus_have_*_cap() wrapper.
2072	 */
2073	BUG_ON(!system_capabilities_finalized());
2074	params = this_cpu_ptr_nvhe_sym(kvm_init_params);
2075	arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
2076	WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
2077}
2078
2079static void cpu_init_hyp_mode(void)
2080{
2081	hyp_install_host_vector();
2082
2083	/*
2084	 * Disabling SSBD on a non-VHE system requires us to enable SSBS
2085	 * at EL2.
2086	 */
2087	if (this_cpu_has_cap(ARM64_SSBS) &&
2088	    arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
2089		kvm_call_hyp_nvhe(__kvm_enable_ssbs);
2090	}
2091}
2092
2093static void cpu_hyp_reset(void)
2094{
2095	if (!is_kernel_in_hyp_mode())
2096		__hyp_reset_vectors();
2097}
2098
2099/*
2100 * EL2 vectors can be mapped and rerouted in a number of ways,
2101 * depending on the kernel configuration and CPU present:
2102 *
2103 * - If the CPU is affected by Spectre-v2, the hardening sequence is
2104 *   placed in one of the vector slots, which is executed before jumping
2105 *   to the real vectors.
2106 *
2107 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
2108 *   containing the hardening sequence is mapped next to the idmap page,
2109 *   and executed before jumping to the real vectors.
2110 *
2111 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
2112 *   empty slot is selected, mapped next to the idmap page, and
2113 *   executed before jumping to the real vectors.
2114 *
2115 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
2116 * VHE, as we don't have hypervisor-specific mappings. If the system
2117 * is VHE and yet selects this capability, it will be ignored.
2118 */
2119static void cpu_set_hyp_vector(void)
2120{
2121	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
2122	void *vector = hyp_spectre_vector_selector[data->slot];
2123
2124	if (!is_protected_kvm_enabled())
2125		*this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
2126	else
2127		kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
2128}
2129
2130static void cpu_hyp_init_context(void)
2131{
2132	kvm_init_host_cpu_context(host_data_ptr(host_ctxt));
2133	kvm_init_host_debug_data();
2134
2135	if (!is_kernel_in_hyp_mode())
2136		cpu_init_hyp_mode();
2137}
2138
2139static void cpu_hyp_init_features(void)
2140{
2141	cpu_set_hyp_vector();
2142
2143	if (is_kernel_in_hyp_mode()) {
2144		kvm_timer_init_vhe();
2145		kvm_debug_init_vhe();
2146	}
2147
2148	if (vgic_present)
2149		kvm_vgic_init_cpu_hardware();
2150}
2151
2152static void cpu_hyp_reinit(void)
2153{
2154	cpu_hyp_reset();
2155	cpu_hyp_init_context();
2156	cpu_hyp_init_features();
2157}
2158
2159static void cpu_hyp_init(void *discard)
2160{
2161	if (!__this_cpu_read(kvm_hyp_initialized)) {
2162		cpu_hyp_reinit();
2163		__this_cpu_write(kvm_hyp_initialized, 1);
2164	}
2165}
2166
2167static void cpu_hyp_uninit(void *discard)
2168{
2169	if (!is_protected_kvm_enabled() && __this_cpu_read(kvm_hyp_initialized)) {
2170		cpu_hyp_reset();
2171		__this_cpu_write(kvm_hyp_initialized, 0);
2172	}
2173}
2174
2175int kvm_arch_enable_virtualization_cpu(void)
2176{
2177	/*
2178	 * Most calls to this function are made with migration
2179	 * disabled, but not with preemption disabled. The former is
2180	 * enough to ensure correctness, but most of the helpers
2181	 * expect the later and will throw a tantrum otherwise.
2182	 */
2183	preempt_disable();
2184
2185	cpu_hyp_init(NULL);
2186
2187	kvm_vgic_cpu_up();
2188	kvm_timer_cpu_up();
2189
2190	preempt_enable();
2191
2192	return 0;
2193}
2194
2195void kvm_arch_disable_virtualization_cpu(void)
2196{
2197	kvm_timer_cpu_down();
2198	kvm_vgic_cpu_down();
2199
2200	if (!is_protected_kvm_enabled())
2201		cpu_hyp_uninit(NULL);
2202}
2203
2204#ifdef CONFIG_CPU_PM
2205static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
2206				    unsigned long cmd,
2207				    void *v)
2208{
2209	/*
2210	 * kvm_hyp_initialized is left with its old value over
2211	 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
2212	 * re-enable hyp.
2213	 */
2214	switch (cmd) {
2215	case CPU_PM_ENTER:
2216		if (__this_cpu_read(kvm_hyp_initialized))
2217			/*
2218			 * don't update kvm_hyp_initialized here
2219			 * so that the hyp will be re-enabled
2220			 * when we resume. See below.
2221			 */
2222			cpu_hyp_reset();
2223
2224		return NOTIFY_OK;
2225	case CPU_PM_ENTER_FAILED:
2226	case CPU_PM_EXIT:
2227		if (__this_cpu_read(kvm_hyp_initialized))
2228			/* The hyp was enabled before suspend. */
2229			cpu_hyp_reinit();
2230
2231		return NOTIFY_OK;
2232
2233	default:
2234		return NOTIFY_DONE;
2235	}
2236}
2237
2238static struct notifier_block hyp_init_cpu_pm_nb = {
2239	.notifier_call = hyp_init_cpu_pm_notifier,
2240};
2241
2242static void __init hyp_cpu_pm_init(void)
2243{
2244	if (!is_protected_kvm_enabled())
2245		cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
2246}
2247static void __init hyp_cpu_pm_exit(void)
2248{
2249	if (!is_protected_kvm_enabled())
2250		cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
2251}
2252#else
2253static inline void __init hyp_cpu_pm_init(void)
2254{
2255}
2256static inline void __init hyp_cpu_pm_exit(void)
2257{
2258}
2259#endif
2260
2261static void __init init_cpu_logical_map(void)
2262{
2263	unsigned int cpu;
2264
2265	/*
2266	 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
2267	 * Only copy the set of online CPUs whose features have been checked
2268	 * against the finalized system capabilities. The hypervisor will not
2269	 * allow any other CPUs from the `possible` set to boot.
2270	 */
2271	for_each_online_cpu(cpu)
2272		hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
2273}
2274
2275#define init_psci_0_1_impl_state(config, what)	\
2276	config.psci_0_1_ ## what ## _implemented = psci_ops.what
2277
2278static bool __init init_psci_relay(void)
2279{
2280	/*
2281	 * If PSCI has not been initialized, protected KVM cannot install
2282	 * itself on newly booted CPUs.
2283	 */
2284	if (!psci_ops.get_version) {
2285		kvm_err("Cannot initialize protected mode without PSCI\n");
2286		return false;
2287	}
2288
2289	kvm_host_psci_config.version = psci_ops.get_version();
2290	kvm_host_psci_config.smccc_version = arm_smccc_get_version();
2291
2292	if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
2293		kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
2294		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
2295		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
2296		init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
2297		init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
2298	}
2299	return true;
2300}
2301
2302static int __init init_subsystems(void)
2303{
2304	int err = 0;
2305
2306	/*
2307	 * Enable hardware so that subsystem initialisation can access EL2.
2308	 */
2309	on_each_cpu(cpu_hyp_init, NULL, 1);
2310
2311	/*
2312	 * Register CPU lower-power notifier
2313	 */
2314	hyp_cpu_pm_init();
2315
2316	/*
2317	 * Init HYP view of VGIC
2318	 */
2319	err = kvm_vgic_hyp_init();
2320	switch (err) {
2321	case 0:
2322		vgic_present = true;
2323		break;
2324	case -ENODEV:
2325	case -ENXIO:
2326		/*
2327		 * No VGIC? No pKVM for you.
2328		 *
2329		 * Protected mode assumes that VGICv3 is present, so no point
2330		 * in trying to hobble along if vgic initialization fails.
2331		 */
2332		if (is_protected_kvm_enabled())
2333			goto out;
2334
2335		/*
2336		 * Otherwise, userspace could choose to implement a GIC for its
2337		 * guest on non-cooperative hardware.
2338		 */
2339		vgic_present = false;
2340		err = 0;
2341		break;
2342	default:
2343		goto out;
2344	}
2345
2346	if (kvm_mode == KVM_MODE_NV &&
2347		!(vgic_present && (kvm_vgic_global_state.type == VGIC_V3 ||
2348				   kvm_vgic_global_state.has_gcie_v3_compat))) {
2349		kvm_err("NV support requires GICv3 or GICv5 with legacy support, giving up\n");
2350		err = -EINVAL;
2351		goto out;
2352	}
2353
2354	/*
2355	 * Init HYP architected timer support
2356	 */
2357	err = kvm_timer_hyp_init(vgic_present);
2358	if (err)
2359		goto out;
2360
2361	kvm_register_perf_callbacks(NULL);
2362
2363out:
2364	if (err)
2365		hyp_cpu_pm_exit();
2366
2367	if (err || !is_protected_kvm_enabled())
2368		on_each_cpu(cpu_hyp_uninit, NULL, 1);
2369
2370	return err;
2371}
2372
2373static void __init teardown_subsystems(void)
2374{
2375	kvm_unregister_perf_callbacks();
2376	hyp_cpu_pm_exit();
2377}
2378
2379static void __init teardown_hyp_mode(void)
2380{
2381	bool free_sve = system_supports_sve() && is_protected_kvm_enabled();
2382	int cpu;
2383
2384	free_hyp_pgds();
2385	for_each_possible_cpu(cpu) {
2386		if (per_cpu(kvm_hyp_initialized, cpu))
2387			continue;
2388
2389		free_pages(per_cpu(kvm_arm_hyp_stack_base, cpu), NVHE_STACK_SHIFT - PAGE_SHIFT);
2390
2391		if (!kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu])
2392			continue;
2393
2394		if (free_sve) {
2395			struct cpu_sve_state *sve_state;
2396
2397			sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2398			free_pages((unsigned long) sve_state, pkvm_host_sve_state_order());
2399		}
2400
2401		free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
2402
2403	}
2404}
2405
2406static int __init do_pkvm_init(u32 hyp_va_bits)
2407{
2408	void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
2409	int ret;
2410
2411	preempt_disable();
2412	cpu_hyp_init_context();
2413	ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
2414				num_possible_cpus(), kern_hyp_va(per_cpu_base),
2415				hyp_va_bits);
2416	cpu_hyp_init_features();
2417
2418	/*
2419	 * The stub hypercalls are now disabled, so set our local flag to
2420	 * prevent a later re-init attempt in kvm_arch_enable_virtualization_cpu().
2421	 */
2422	__this_cpu_write(kvm_hyp_initialized, 1);
2423	preempt_enable();
2424
2425	return ret;
2426}
2427
2428static u64 get_hyp_id_aa64pfr0_el1(void)
2429{
2430	/*
2431	 * Track whether the system isn't affected by spectre/meltdown in the
2432	 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs.
2433	 * Although this is per-CPU, we make it global for simplicity, e.g., not
2434	 * to have to worry about vcpu migration.
2435	 *
2436	 * Unlike for non-protected VMs, userspace cannot override this for
2437	 * protected VMs.
2438	 */
2439	u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
2440
2441	val &= ~(ID_AA64PFR0_EL1_CSV2 |
2442		 ID_AA64PFR0_EL1_CSV3);
2443
2444	val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV2,
2445			  arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED);
2446	val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV3,
2447			  arm64_get_meltdown_state() == SPECTRE_UNAFFECTED);
2448
2449	return val;
2450}
2451
2452static void kvm_hyp_init_symbols(void)
2453{
2454	kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1();
2455	kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
2456	kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
2457	kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
2458	kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
2459	kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
2460	kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
2461	kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
2462	kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
2463	kvm_nvhe_sym(__icache_flags) = __icache_flags;
2464	kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
2465
2466	/* Propagate the FGT state to the nVHE side */
2467	kvm_nvhe_sym(hfgrtr_masks)  = hfgrtr_masks;
2468	kvm_nvhe_sym(hfgwtr_masks)  = hfgwtr_masks;
2469	kvm_nvhe_sym(hfgitr_masks)  = hfgitr_masks;
2470	kvm_nvhe_sym(hdfgrtr_masks) = hdfgrtr_masks;
2471	kvm_nvhe_sym(hdfgwtr_masks) = hdfgwtr_masks;
2472	kvm_nvhe_sym(hafgrtr_masks) = hafgrtr_masks;
2473	kvm_nvhe_sym(hfgrtr2_masks) = hfgrtr2_masks;
2474	kvm_nvhe_sym(hfgwtr2_masks) = hfgwtr2_masks;
2475	kvm_nvhe_sym(hfgitr2_masks) = hfgitr2_masks;
2476	kvm_nvhe_sym(hdfgrtr2_masks)= hdfgrtr2_masks;
2477	kvm_nvhe_sym(hdfgwtr2_masks)= hdfgwtr2_masks;
2478
2479	/*
2480	 * Flush entire BSS since part of its data containing init symbols is read
2481	 * while the MMU is off.
2482	 */
2483	kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start),
2484				kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start));
2485}
2486
2487static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
2488{
2489	void *addr = phys_to_virt(hyp_mem_base);
2490	int ret;
2491
2492	ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
2493	if (ret)
2494		return ret;
2495
2496	ret = do_pkvm_init(hyp_va_bits);
2497	if (ret)
2498		return ret;
2499
2500	free_hyp_pgds();
2501
2502	return 0;
2503}
2504
2505static int init_pkvm_host_sve_state(void)
2506{
2507	int cpu;
2508
2509	if (!system_supports_sve())
2510		return 0;
2511
2512	/* Allocate pages for host sve state in protected mode. */
2513	for_each_possible_cpu(cpu) {
2514		struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order());
2515
2516		if (!page)
2517			return -ENOMEM;
2518
2519		per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page);
2520	}
2521
2522	/*
2523	 * Don't map the pages in hyp since these are only used in protected
2524	 * mode, which will (re)create its own mapping when initialized.
2525	 */
2526
2527	return 0;
2528}
2529
2530/*
2531 * Finalizes the initialization of hyp mode, once everything else is initialized
2532 * and the initialziation process cannot fail.
2533 */
2534static void finalize_init_hyp_mode(void)
2535{
2536	int cpu;
2537
2538	if (system_supports_sve() && is_protected_kvm_enabled()) {
2539		for_each_possible_cpu(cpu) {
2540			struct cpu_sve_state *sve_state;
2541
2542			sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2543			per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
2544				kern_hyp_va(sve_state);
2545		}
2546	}
2547}
2548
2549static void pkvm_hyp_init_ptrauth(void)
2550{
2551	struct kvm_cpu_context *hyp_ctxt;
2552	int cpu;
2553
2554	for_each_possible_cpu(cpu) {
2555		hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu);
2556		hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long();
2557		hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long();
2558		hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long();
2559		hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long();
2560		hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long();
2561		hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long();
2562		hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long();
2563		hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long();
2564		hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long();
2565		hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long();
2566	}
2567}
2568
2569/* Inits Hyp-mode on all online CPUs */
2570static int __init init_hyp_mode(void)
2571{
2572	u32 hyp_va_bits;
2573	int cpu;
2574	int err = -ENOMEM;
2575
2576	/*
2577	 * The protected Hyp-mode cannot be initialized if the memory pool
2578	 * allocation has failed.
2579	 */
2580	if (is_protected_kvm_enabled() && !hyp_mem_base)
2581		goto out_err;
2582
2583	/*
2584	 * Allocate Hyp PGD and setup Hyp identity mapping
2585	 */
2586	err = kvm_mmu_init(&hyp_va_bits);
2587	if (err)
2588		goto out_err;
2589
2590	/*
2591	 * Allocate stack pages for Hypervisor-mode
2592	 */
2593	for_each_possible_cpu(cpu) {
2594		unsigned long stack_base;
2595
2596		stack_base = __get_free_pages(GFP_KERNEL, NVHE_STACK_SHIFT - PAGE_SHIFT);
2597		if (!stack_base) {
2598			err = -ENOMEM;
2599			goto out_err;
2600		}
2601
2602		per_cpu(kvm_arm_hyp_stack_base, cpu) = stack_base;
2603	}
2604
2605	/*
2606	 * Allocate and initialize pages for Hypervisor-mode percpu regions.
2607	 */
2608	for_each_possible_cpu(cpu) {
2609		struct page *page;
2610		void *page_addr;
2611
2612		page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
2613		if (!page) {
2614			err = -ENOMEM;
2615			goto out_err;
2616		}
2617
2618		page_addr = page_address(page);
2619		memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
2620		kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
2621	}
2622
2623	/*
2624	 * Map the Hyp-code called directly from the host
2625	 */
2626	err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
2627				  kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
2628	if (err) {
2629		kvm_err("Cannot map world-switch code\n");
2630		goto out_err;
2631	}
2632
2633	err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_start),
2634				  kvm_ksym_ref(__hyp_data_end), PAGE_HYP);
2635	if (err) {
2636		kvm_err("Cannot map .hyp.data section\n");
2637		goto out_err;
2638	}
2639
2640	err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
2641				  kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
2642	if (err) {
2643		kvm_err("Cannot map .hyp.rodata section\n");
2644		goto out_err;
2645	}
2646
2647	err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
2648				  kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
2649	if (err) {
2650		kvm_err("Cannot map rodata section\n");
2651		goto out_err;
2652	}
2653
2654	/*
2655	 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
2656	 * section thanks to an assertion in the linker script. Map it RW and
2657	 * the rest of .bss RO.
2658	 */
2659	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
2660				  kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
2661	if (err) {
2662		kvm_err("Cannot map hyp bss section: %d\n", err);
2663		goto out_err;
2664	}
2665
2666	err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
2667				  kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
2668	if (err) {
2669		kvm_err("Cannot map bss section\n");
2670		goto out_err;
2671	}
2672
2673	/*
2674	 * Map the Hyp stack pages
2675	 */
2676	for_each_possible_cpu(cpu) {
2677		struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2678		char *stack_base = (char *)per_cpu(kvm_arm_hyp_stack_base, cpu);
2679
2680		err = create_hyp_stack(__pa(stack_base), &params->stack_hyp_va);
2681		if (err) {
2682			kvm_err("Cannot map hyp stack\n");
2683			goto out_err;
2684		}
2685
2686		/*
2687		 * Save the stack PA in nvhe_init_params. This will be needed
2688		 * to recreate the stack mapping in protected nVHE mode.
2689		 * __hyp_pa() won't do the right thing there, since the stack
2690		 * has been mapped in the flexible private VA space.
2691		 */
2692		params->stack_pa = __pa(stack_base);
2693	}
2694
2695	for_each_possible_cpu(cpu) {
2696		char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
2697		char *percpu_end = percpu_begin + nvhe_percpu_size();
2698
2699		/* Map Hyp percpu pages */
2700		err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
2701		if (err) {
2702			kvm_err("Cannot map hyp percpu region\n");
2703			goto out_err;
2704		}
2705
2706		/* Prepare the CPU initialization parameters */
2707		cpu_prepare_hyp_mode(cpu, hyp_va_bits);
2708	}
2709
2710	kvm_hyp_init_symbols();
2711
2712	if (is_protected_kvm_enabled()) {
2713		if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) &&
2714		    cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH))
2715			pkvm_hyp_init_ptrauth();
2716
2717		init_cpu_logical_map();
2718
2719		if (!init_psci_relay()) {
2720			err = -ENODEV;
2721			goto out_err;
2722		}
2723
2724		err = init_pkvm_host_sve_state();
2725		if (err)
2726			goto out_err;
2727
2728		err = kvm_hyp_init_protection(hyp_va_bits);
2729		if (err) {
2730			kvm_err("Failed to init hyp memory protection\n");
2731			goto out_err;
2732		}
2733	}
2734
2735	return 0;
2736
2737out_err:
2738	teardown_hyp_mode();
2739	kvm_err("error initializing Hyp mode: %d\n", err);
2740	return err;
2741}
2742
2743struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2744{
2745	struct kvm_vcpu *vcpu = NULL;
2746	struct kvm_mpidr_data *data;
2747	unsigned long i;
2748
2749	mpidr &= MPIDR_HWID_BITMASK;
2750
2751	rcu_read_lock();
2752	data = rcu_dereference(kvm->arch.mpidr_data);
2753
2754	if (data) {
2755		u16 idx = kvm_mpidr_index(data, mpidr);
2756
2757		vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]);
2758		if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu))
2759			vcpu = NULL;
2760	}
2761
2762	rcu_read_unlock();
2763
2764	if (vcpu)
2765		return vcpu;
2766
2767	kvm_for_each_vcpu(i, vcpu, kvm) {
2768		if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2769			return vcpu;
2770	}
2771	return NULL;
2772}
2773
2774bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
2775{
2776	return irqchip_in_kernel(kvm);
2777}
2778
2779int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2780				      struct irq_bypass_producer *prod)
2781{
2782	struct kvm_kernel_irqfd *irqfd =
2783		container_of(cons, struct kvm_kernel_irqfd, consumer);
2784	struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2785
2786	/*
2787	 * The only thing we have a chance of directly-injecting is LPIs. Maybe
2788	 * one day...
2789	 */
2790	if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2791		return 0;
2792
2793	return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2794					  &irqfd->irq_entry);
2795}
2796
2797void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2798				      struct irq_bypass_producer *prod)
2799{
2800	struct kvm_kernel_irqfd *irqfd =
2801		container_of(cons, struct kvm_kernel_irqfd, consumer);
2802	struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2803
2804	if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2805		return;
2806
2807	kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq);
2808}
2809
2810void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd,
2811				   struct kvm_kernel_irq_routing_entry *old,
2812				   struct kvm_kernel_irq_routing_entry *new)
2813{
2814	if (old->type == KVM_IRQ_ROUTING_MSI &&
2815	    new->type == KVM_IRQ_ROUTING_MSI &&
2816	    !memcmp(&old->msi, &new->msi, sizeof(new->msi)))
2817		return;
2818
2819	/*
2820	 * Remapping the vLPI requires taking the its_lock mutex to resolve
2821	 * the new translation. We're in spinlock land at this point, so no
2822	 * chance of resolving the translation.
2823	 *
2824	 * Unmap the vLPI and fall back to software LPI injection.
2825	 */
2826	return kvm_vgic_v4_unset_forwarding(irqfd->kvm, irqfd->producer->irq);
2827}
2828
2829void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2830{
2831	struct kvm_kernel_irqfd *irqfd =
2832		container_of(cons, struct kvm_kernel_irqfd, consumer);
2833
2834	kvm_arm_halt_guest(irqfd->kvm);
2835}
2836
2837void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2838{
2839	struct kvm_kernel_irqfd *irqfd =
2840		container_of(cons, struct kvm_kernel_irqfd, consumer);
2841
2842	kvm_arm_resume_guest(irqfd->kvm);
2843}
2844
2845/* Initialize Hyp-mode and memory mappings on all CPUs */
2846static __init int kvm_arm_init(void)
2847{
2848	int err;
2849	bool in_hyp_mode;
2850
2851	if (!is_hyp_mode_available()) {
2852		kvm_info("HYP mode not available\n");
2853		return -ENODEV;
2854	}
2855
2856	if (kvm_get_mode() == KVM_MODE_NONE) {
2857		kvm_info("KVM disabled from command line\n");
2858		return -ENODEV;
2859	}
2860
2861	err = kvm_sys_reg_table_init();
2862	if (err) {
2863		kvm_info("Error initializing system register tables");
2864		return err;
2865	}
2866
2867	in_hyp_mode = is_kernel_in_hyp_mode();
2868
2869	if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2870	    cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2871		kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2872			 "Only trusted guests should be used on this system.\n");
2873
2874	err = kvm_set_ipa_limit();
2875	if (err)
2876		return err;
2877
2878	err = kvm_arm_init_sve();
2879	if (err)
2880		return err;
2881
2882	err = kvm_arm_vmid_alloc_init();
2883	if (err) {
2884		kvm_err("Failed to initialize VMID allocator.\n");
2885		return err;
2886	}
2887
2888	if (!in_hyp_mode) {
2889		err = init_hyp_mode();
2890		if (err)
2891			goto out_err;
2892	}
2893
2894	err = kvm_init_vector_slots();
2895	if (err) {
2896		kvm_err("Cannot initialise vector slots\n");
2897		goto out_hyp;
2898	}
2899
2900	err = init_subsystems();
2901	if (err)
2902		goto out_hyp;
2903
2904	kvm_info("%s%sVHE%s mode initialized successfully\n",
2905		 in_hyp_mode ? "" : (is_protected_kvm_enabled() ?
2906				     "Protected " : "Hyp "),
2907		 in_hyp_mode ? "" : (cpus_have_final_cap(ARM64_KVM_HVHE) ?
2908				     "h" : "n"),
2909		 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) ? "+NV2": "");
2910
2911	/*
2912	 * FIXME: Do something reasonable if kvm_init() fails after pKVM
2913	 * hypervisor protection is finalized.
2914	 */
2915	err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2916	if (err)
2917		goto out_subs;
2918
2919	/*
2920	 * This should be called after initialization is done and failure isn't
2921	 * possible anymore.
2922	 */
2923	if (!in_hyp_mode)
2924		finalize_init_hyp_mode();
2925
2926	kvm_arm_initialised = true;
2927
2928	return 0;
2929
2930out_subs:
2931	teardown_subsystems();
2932out_hyp:
2933	if (!in_hyp_mode)
2934		teardown_hyp_mode();
2935out_err:
2936	kvm_arm_vmid_alloc_free();
2937	return err;
2938}
2939
2940static int __init early_kvm_mode_cfg(char *arg)
2941{
2942	if (!arg)
2943		return -EINVAL;
2944
2945	if (strcmp(arg, "none") == 0) {
2946		kvm_mode = KVM_MODE_NONE;
2947		return 0;
2948	}
2949
2950	if (!is_hyp_mode_available()) {
2951		pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2952		return 0;
2953	}
2954
2955	if (strcmp(arg, "protected") == 0) {
2956		if (!is_kernel_in_hyp_mode())
2957			kvm_mode = KVM_MODE_PROTECTED;
2958		else
2959			pr_warn_once("Protected KVM not available with VHE\n");
2960
2961		return 0;
2962	}
2963
2964	if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2965		kvm_mode = KVM_MODE_DEFAULT;
2966		return 0;
2967	}
2968
2969	if (strcmp(arg, "nested") == 0 && !WARN_ON(!is_kernel_in_hyp_mode())) {
2970		kvm_mode = KVM_MODE_NV;
2971		return 0;
2972	}
2973
2974	return -EINVAL;
2975}
2976early_param("kvm-arm.mode", early_kvm_mode_cfg);
2977
2978static int __init early_kvm_wfx_trap_policy_cfg(char *arg, enum kvm_wfx_trap_policy *p)
2979{
2980	if (!arg)
2981		return -EINVAL;
2982
2983	if (strcmp(arg, "trap") == 0) {
2984		*p = KVM_WFX_TRAP;
2985		return 0;
2986	}
2987
2988	if (strcmp(arg, "notrap") == 0) {
2989		*p = KVM_WFX_NOTRAP;
2990		return 0;
2991	}
2992
2993	return -EINVAL;
2994}
2995
2996static int __init early_kvm_wfi_trap_policy_cfg(char *arg)
2997{
2998	return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfi_trap_policy);
2999}
3000early_param("kvm-arm.wfi_trap_policy", early_kvm_wfi_trap_policy_cfg);
3001
3002static int __init early_kvm_wfe_trap_policy_cfg(char *arg)
3003{
3004	return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfe_trap_policy);
3005}
3006early_param("kvm-arm.wfe_trap_policy", early_kvm_wfe_trap_policy_cfg);
3007
3008enum kvm_mode kvm_get_mode(void)
3009{
3010	return kvm_mode;
3011}
3012
3013module_init(kvm_arm_init);