svm.c

   1#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   2
   3#include <linux/kvm_host.h>
   4
   5#include "irq.h"
   6#include "mmu.h"
   7#include "kvm_cache_regs.h"
   8#include "x86.h"
   9#include "smm.h"
  10#include "cpuid.h"
  11#include "pmu.h"
  12
  13#include <linux/module.h>
  14#include <linux/mod_devicetable.h>
  15#include <linux/kernel.h>
  16#include <linux/vmalloc.h>
  17#include <linux/highmem.h>
  18#include <linux/amd-iommu.h>
  19#include <linux/sched.h>
  20#include <linux/trace_events.h>
  21#include <linux/slab.h>
  22#include <linux/hashtable.h>
  23#include <linux/objtool.h>
  24#include <linux/psp-sev.h>
  25#include <linux/file.h>
  26#include <linux/pagemap.h>
  27#include <linux/swap.h>
  28#include <linux/rwsem.h>
  29#include <linux/cc_platform.h>
  30#include <linux/smp.h>
  31#include <linux/string_choices.h>
  32#include <linux/mutex.h>
  33
  34#include <asm/apic.h>
  35#include <asm/msr.h>
  36#include <asm/perf_event.h>
  37#include <asm/tlbflush.h>
  38#include <asm/desc.h>
  39#include <asm/debugreg.h>
  40#include <asm/kvm_para.h>
  41#include <asm/irq_remapping.h>
  42#include <asm/spec-ctrl.h>
  43#include <asm/cpu_device_id.h>
  44#include <asm/traps.h>
  45#include <asm/reboot.h>
  46#include <asm/fpu/api.h>
  47
  48#include <trace/events/ipi.h>
  49
  50#include "trace.h"
  51
  52#include "svm.h"
  53#include "svm_ops.h"
  54
  55#include "kvm_onhyperv.h"
  56#include "svm_onhyperv.h"
  57
  58MODULE_AUTHOR("Qumranet");
  59MODULE_DESCRIPTION("KVM support for SVM (AMD-V) extensions");
  60MODULE_LICENSE("GPL");
  61
  62#ifdef MODULE
  63static const struct x86_cpu_id svm_cpu_id[] = {
  64	X86_MATCH_FEATURE(X86_FEATURE_SVM, NULL),
  65	{}
  66};
  67MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  68#endif
  69
  70#define SEG_TYPE_LDT 2
  71#define SEG_TYPE_BUSY_TSS16 3
  72
  73static bool erratum_383_found __read_mostly;
  74
  75/*
  76 * Set osvw_len to higher value when updated Revision Guides
  77 * are published and we know what the new status bits are
  78 */
  79static uint64_t osvw_len = 4, osvw_status;
  80
  81static DEFINE_PER_CPU(u64, current_tsc_ratio);
  82
  83/*
  84 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
  85 * pause_filter_count: On processors that support Pause filtering(indicated
  86 *	by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
  87 *	count value. On VMRUN this value is loaded into an internal counter.
  88 *	Each time a pause instruction is executed, this counter is decremented
  89 *	until it reaches zero at which time a #VMEXIT is generated if pause
  90 *	intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
  91 *	Intercept Filtering for more details.
  92 *	This also indicate if ple logic enabled.
  93 *
  94 * pause_filter_thresh: In addition, some processor families support advanced
  95 *	pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
  96 *	the amount of time a guest is allowed to execute in a pause loop.
  97 *	In this mode, a 16-bit pause filter threshold field is added in the
  98 *	VMCB. The threshold value is a cycle count that is used to reset the
  99 *	pause counter. As with simple pause filtering, VMRUN loads the pause
 100 *	count value from VMCB into an internal counter. Then, on each pause
 101 *	instruction the hardware checks the elapsed number of cycles since
 102 *	the most recent pause instruction against the pause filter threshold.
 103 *	If the elapsed cycle count is greater than the pause filter threshold,
 104 *	then the internal pause count is reloaded from the VMCB and execution
 105 *	continues. If the elapsed cycle count is less than the pause filter
 106 *	threshold, then the internal pause count is decremented. If the count
 107 *	value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
 108 *	triggered. If advanced pause filtering is supported and pause filter
 109 *	threshold field is set to zero, the filter will operate in the simpler,
 110 *	count only mode.
 111 */
 112
 113static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
 114module_param(pause_filter_thresh, ushort, 0444);
 115
 116static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
 117module_param(pause_filter_count, ushort, 0444);
 118
 119/* Default doubles per-vcpu window every exit. */
 120static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 121module_param(pause_filter_count_grow, ushort, 0444);
 122
 123/* Default resets per-vcpu window every exit to pause_filter_count. */
 124static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 125module_param(pause_filter_count_shrink, ushort, 0444);
 126
 127/* Default is to compute the maximum so we can never overflow. */
 128static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
 129module_param(pause_filter_count_max, ushort, 0444);
 130
 131/*
 132 * Use nested page tables by default.  Note, NPT may get forced off by
 133 * svm_hardware_setup() if it's unsupported by hardware or the host kernel.
 134 */
 135bool npt_enabled = true;
 136module_param_named(npt, npt_enabled, bool, 0444);
 137
 138/* allow nested virtualization in KVM/SVM */
 139static int nested = true;
 140module_param(nested, int, 0444);
 141
 142/* enable/disable Next RIP Save */
 143int nrips = true;
 144module_param(nrips, int, 0444);
 145
 146/* enable/disable Virtual VMLOAD VMSAVE */
 147static int vls = true;
 148module_param(vls, int, 0444);
 149
 150/* enable/disable Virtual GIF */
 151int vgif = true;
 152module_param(vgif, int, 0444);
 153
 154/* enable/disable LBR virtualization */
 155int lbrv = true;
 156module_param(lbrv, int, 0444);
 157
 158static int tsc_scaling = true;
 159module_param(tsc_scaling, int, 0444);
 160
 161module_param(enable_device_posted_irqs, bool, 0444);
 162
 163bool __read_mostly dump_invalid_vmcb;
 164module_param(dump_invalid_vmcb, bool, 0644);
 165
 166
 167bool intercept_smi = true;
 168module_param(intercept_smi, bool, 0444);
 169
 170bool vnmi = true;
 171module_param(vnmi, bool, 0444);
 172
 173static bool svm_gp_erratum_intercept = true;
 174
 175static u8 rsm_ins_bytes[] = "\x0f\xaa";
 176
 177static unsigned long iopm_base;
 178
 179DEFINE_PER_CPU(struct svm_cpu_data, svm_data);
 180
 181static DEFINE_MUTEX(vmcb_dump_mutex);
 182
 183/*
 184 * Only MSR_TSC_AUX is switched via the user return hook.  EFER is switched via
 185 * the VMCB, and the SYSCALL/SYSENTER MSRs are handled by VMLOAD/VMSAVE.
 186 *
 187 * RDTSCP and RDPID are not used in the kernel, specifically to allow KVM to
 188 * defer the restoration of TSC_AUX until the CPU returns to userspace.
 189 */
 190int tsc_aux_uret_slot __ro_after_init = -1;
 191
 192static int get_npt_level(void)
 193{
 194#ifdef CONFIG_X86_64
 195	return pgtable_l5_enabled() ? PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL;
 196#else
 197	return PT32E_ROOT_LEVEL;
 198#endif
 199}
 200
 201int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 202{
 203	struct vcpu_svm *svm = to_svm(vcpu);
 204	u64 old_efer = vcpu->arch.efer;
 205	vcpu->arch.efer = efer;
 206
 207	if (!npt_enabled) {
 208		/* Shadow paging assumes NX to be available.  */
 209		efer |= EFER_NX;
 210
 211		if (!(efer & EFER_LMA))
 212			efer &= ~EFER_LME;
 213	}
 214
 215	if ((old_efer & EFER_SVME) != (efer & EFER_SVME)) {
 216		if (!(efer & EFER_SVME)) {
 217			svm_leave_nested(vcpu);
 218			svm_set_gif(svm, true);
 219			/* #GP intercept is still needed for vmware backdoor */
 220			if (!enable_vmware_backdoor)
 221				clr_exception_intercept(svm, GP_VECTOR);
 222
 223			/*
 224			 * Free the nested guest state, unless we are in SMM.
 225			 * In this case we will return to the nested guest
 226			 * as soon as we leave SMM.
 227			 */
 228			if (!is_smm(vcpu))
 229				svm_free_nested(svm);
 230
 231		} else {
 232			int ret = svm_allocate_nested(svm);
 233
 234			if (ret) {
 235				vcpu->arch.efer = old_efer;
 236				return ret;
 237			}
 238
 239			/*
 240			 * Never intercept #GP for SEV guests, KVM can't
 241			 * decrypt guest memory to workaround the erratum.
 242			 */
 243			if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm))
 244				set_exception_intercept(svm, GP_VECTOR);
 245		}
 246	}
 247
 248	svm->vmcb->save.efer = efer | EFER_SVME;
 249	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
 250	return 0;
 251}
 252
 253static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 254{
 255	struct vcpu_svm *svm = to_svm(vcpu);
 256	u32 ret = 0;
 257
 258	if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 259		ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 260	return ret;
 261}
 262
 263static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 264{
 265	struct vcpu_svm *svm = to_svm(vcpu);
 266
 267	if (mask == 0)
 268		svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 269	else
 270		svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 271
 272}
 273
 274static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
 275					   int emul_type,
 276					   bool commit_side_effects)
 277{
 278	struct vcpu_svm *svm = to_svm(vcpu);
 279	unsigned long old_rflags;
 280
 281	/*
 282	 * SEV-ES does not expose the next RIP. The RIP update is controlled by
 283	 * the type of exit and the #VC handler in the guest.
 284	 */
 285	if (sev_es_guest(vcpu->kvm))
 286		goto done;
 287
 288	if (nrips && svm->vmcb->control.next_rip != 0) {
 289		WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 290		svm->next_rip = svm->vmcb->control.next_rip;
 291	}
 292
 293	if (!svm->next_rip) {
 294		if (unlikely(!commit_side_effects))
 295			old_rflags = svm->vmcb->save.rflags;
 296
 297		if (!kvm_emulate_instruction(vcpu, emul_type))
 298			return 0;
 299
 300		if (unlikely(!commit_side_effects))
 301			svm->vmcb->save.rflags = old_rflags;
 302	} else {
 303		kvm_rip_write(vcpu, svm->next_rip);
 304	}
 305
 306done:
 307	if (likely(commit_side_effects))
 308		svm_set_interrupt_shadow(vcpu, 0);
 309
 310	return 1;
 311}
 312
 313static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
 314{
 315	return __svm_skip_emulated_instruction(vcpu, EMULTYPE_SKIP, true);
 316}
 317
 318static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu, u8 vector)
 319{
 320	const int emul_type = EMULTYPE_SKIP | EMULTYPE_SKIP_SOFT_INT |
 321			      EMULTYPE_SET_SOFT_INT_VECTOR(vector);
 322	unsigned long rip, old_rip = kvm_rip_read(vcpu);
 323	struct vcpu_svm *svm = to_svm(vcpu);
 324
 325	/*
 326	 * Due to architectural shortcomings, the CPU doesn't always provide
 327	 * NextRIP, e.g. if KVM intercepted an exception that occurred while
 328	 * the CPU was vectoring an INTO/INT3 in the guest.  Temporarily skip
 329	 * the instruction even if NextRIP is supported to acquire the next
 330	 * RIP so that it can be shoved into the NextRIP field, otherwise
 331	 * hardware will fail to advance guest RIP during event injection.
 332	 * Drop the exception/interrupt if emulation fails and effectively
 333	 * retry the instruction, it's the least awful option.  If NRIPS is
 334	 * in use, the skip must not commit any side effects such as clearing
 335	 * the interrupt shadow or RFLAGS.RF.
 336	 */
 337	if (!__svm_skip_emulated_instruction(vcpu, emul_type, !nrips))
 338		return -EIO;
 339
 340	rip = kvm_rip_read(vcpu);
 341
 342	/*
 343	 * Save the injection information, even when using next_rip, as the
 344	 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection
 345	 * doesn't complete due to a VM-Exit occurring while the CPU is
 346	 * vectoring the event.   Decoding the instruction isn't guaranteed to
 347	 * work as there may be no backing instruction, e.g. if the event is
 348	 * being injected by L1 for L2, or if the guest is patching INT3 into
 349	 * a different instruction.
 350	 */
 351	svm->soft_int_injected = true;
 352	svm->soft_int_csbase = svm->vmcb->save.cs.base;
 353	svm->soft_int_old_rip = old_rip;
 354	svm->soft_int_next_rip = rip;
 355
 356	if (nrips)
 357		kvm_rip_write(vcpu, old_rip);
 358
 359	if (static_cpu_has(X86_FEATURE_NRIPS))
 360		svm->vmcb->control.next_rip = rip;
 361
 362	return 0;
 363}
 364
 365static void svm_inject_exception(struct kvm_vcpu *vcpu)
 366{
 367	struct kvm_queued_exception *ex = &vcpu->arch.exception;
 368	struct vcpu_svm *svm = to_svm(vcpu);
 369
 370	kvm_deliver_exception_payload(vcpu, ex);
 371
 372	if (kvm_exception_is_soft(ex->vector) &&
 373	    svm_update_soft_interrupt_rip(vcpu, ex->vector))
 374		return;
 375
 376	svm->vmcb->control.event_inj = ex->vector
 377		| SVM_EVTINJ_VALID
 378		| (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 379		| SVM_EVTINJ_TYPE_EXEPT;
 380	svm->vmcb->control.event_inj_err = ex->error_code;
 381}
 382
 383static void svm_init_erratum_383(void)
 384{
 385	u64 val;
 386
 387	if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 388		return;
 389
 390	/* Use _safe variants to not break nested virtualization */
 391	if (native_read_msr_safe(MSR_AMD64_DC_CFG, &val))
 392		return;
 393
 394	val |= (1ULL << 47);
 395
 396	native_write_msr_safe(MSR_AMD64_DC_CFG, val);
 397
 398	erratum_383_found = true;
 399}
 400
 401static void svm_init_osvw(struct kvm_vcpu *vcpu)
 402{
 403	/*
 404	 * Guests should see errata 400 and 415 as fixed (assuming that
 405	 * HLT and IO instructions are intercepted).
 406	 */
 407	vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 408	vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 409
 410	/*
 411	 * By increasing VCPU's osvw.length to 3 we are telling the guest that
 412	 * all osvw.status bits inside that length, including bit 0 (which is
 413	 * reserved for erratum 298), are valid. However, if host processor's
 414	 * osvw_len is 0 then osvw_status[0] carries no information. We need to
 415	 * be conservative here and therefore we tell the guest that erratum 298
 416	 * is present (because we really don't know).
 417	 */
 418	if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 419		vcpu->arch.osvw.status |= 1;
 420}
 421
 422static bool __kvm_is_svm_supported(void)
 423{
 424	int cpu = smp_processor_id();
 425	struct cpuinfo_x86 *c = &cpu_data(cpu);
 426
 427	if (c->x86_vendor != X86_VENDOR_AMD &&
 428	    c->x86_vendor != X86_VENDOR_HYGON) {
 429		pr_err("CPU %d isn't AMD or Hygon\n", cpu);
 430		return false;
 431	}
 432
 433	if (!cpu_has(c, X86_FEATURE_SVM)) {
 434		pr_err("SVM not supported by CPU %d\n", cpu);
 435		return false;
 436	}
 437
 438	if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
 439		pr_info("KVM is unsupported when running as an SEV guest\n");
 440		return false;
 441	}
 442
 443	return true;
 444}
 445
 446static bool kvm_is_svm_supported(void)
 447{
 448	bool supported;
 449
 450	migrate_disable();
 451	supported = __kvm_is_svm_supported();
 452	migrate_enable();
 453
 454	return supported;
 455}
 456
 457static int svm_check_processor_compat(void)
 458{
 459	if (!__kvm_is_svm_supported())
 460		return -EIO;
 461
 462	return 0;
 463}
 464
 465static void __svm_write_tsc_multiplier(u64 multiplier)
 466{
 467	if (multiplier == __this_cpu_read(current_tsc_ratio))
 468		return;
 469
 470	wrmsrq(MSR_AMD64_TSC_RATIO, multiplier);
 471	__this_cpu_write(current_tsc_ratio, multiplier);
 472}
 473
 474static __always_inline struct sev_es_save_area *sev_es_host_save_area(struct svm_cpu_data *sd)
 475{
 476	return &sd->save_area->host_sev_es_save;
 477}
 478
 479static inline void kvm_cpu_svm_disable(void)
 480{
 481	uint64_t efer;
 482
 483	wrmsrq(MSR_VM_HSAVE_PA, 0);
 484	rdmsrq(MSR_EFER, efer);
 485	if (efer & EFER_SVME) {
 486		/*
 487		 * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
 488		 * NMI aren't blocked.
 489		 */
 490		stgi();
 491		wrmsrq(MSR_EFER, efer & ~EFER_SVME);
 492	}
 493}
 494
 495static void svm_emergency_disable_virtualization_cpu(void)
 496{
 497	kvm_rebooting = true;
 498
 499	kvm_cpu_svm_disable();
 500}
 501
 502static void svm_disable_virtualization_cpu(void)
 503{
 504	/* Make sure we clean up behind us */
 505	if (tsc_scaling)
 506		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
 507
 508	kvm_cpu_svm_disable();
 509
 510	amd_pmu_disable_virt();
 511}
 512
 513static int svm_enable_virtualization_cpu(void)
 514{
 515
 516	struct svm_cpu_data *sd;
 517	uint64_t efer;
 518	int me = raw_smp_processor_id();
 519
 520	rdmsrq(MSR_EFER, efer);
 521	if (efer & EFER_SVME)
 522		return -EBUSY;
 523
 524	sd = per_cpu_ptr(&svm_data, me);
 525	sd->asid_generation = 1;
 526	sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 527	sd->next_asid = sd->max_asid + 1;
 528	sd->min_asid = max_sev_asid + 1;
 529
 530	wrmsrq(MSR_EFER, efer | EFER_SVME);
 531
 532	wrmsrq(MSR_VM_HSAVE_PA, sd->save_area_pa);
 533
 534	if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 535		/*
 536		 * Set the default value, even if we don't use TSC scaling
 537		 * to avoid having stale value in the msr
 538		 */
 539		__svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT);
 540	}
 541
 542
 543	/*
 544	 * Get OSVW bits.
 545	 *
 546	 * Note that it is possible to have a system with mixed processor
 547	 * revisions and therefore different OSVW bits. If bits are not the same
 548	 * on different processors then choose the worst case (i.e. if erratum
 549	 * is present on one processor and not on another then assume that the
 550	 * erratum is present everywhere).
 551	 */
 552	if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 553		u64 len, status = 0;
 554		int err;
 555
 556		err = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &len);
 557		if (!err)
 558			err = native_read_msr_safe(MSR_AMD64_OSVW_STATUS, &status);
 559
 560		if (err)
 561			osvw_status = osvw_len = 0;
 562		else {
 563			if (len < osvw_len)
 564				osvw_len = len;
 565			osvw_status |= status;
 566			osvw_status &= (1ULL << osvw_len) - 1;
 567		}
 568	} else
 569		osvw_status = osvw_len = 0;
 570
 571	svm_init_erratum_383();
 572
 573	amd_pmu_enable_virt();
 574
 575	return 0;
 576}
 577
 578static void svm_cpu_uninit(int cpu)
 579{
 580	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
 581
 582	if (!sd->save_area)
 583		return;
 584
 585	kfree(sd->sev_vmcbs);
 586	__free_page(__sme_pa_to_page(sd->save_area_pa));
 587	sd->save_area_pa = 0;
 588	sd->save_area = NULL;
 589}
 590
 591static int svm_cpu_init(int cpu)
 592{
 593	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
 594	struct page *save_area_page;
 595	int ret = -ENOMEM;
 596
 597	memset(sd, 0, sizeof(struct svm_cpu_data));
 598	save_area_page = snp_safe_alloc_page_node(cpu_to_node(cpu), GFP_KERNEL);
 599	if (!save_area_page)
 600		return ret;
 601
 602	ret = sev_cpu_init(sd);
 603	if (ret)
 604		goto free_save_area;
 605
 606	sd->save_area = page_address(save_area_page);
 607	sd->save_area_pa = __sme_page_pa(save_area_page);
 608	return 0;
 609
 610free_save_area:
 611	__free_page(save_area_page);
 612	return ret;
 613
 614}
 615
 616static void set_dr_intercepts(struct vcpu_svm *svm)
 617{
 618	struct vmcb *vmcb = svm->vmcb01.ptr;
 619
 620	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ);
 621	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ);
 622	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ);
 623	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ);
 624	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ);
 625	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ);
 626	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ);
 627	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE);
 628	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE);
 629	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE);
 630	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE);
 631	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE);
 632	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE);
 633	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE);
 634	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ);
 635	vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE);
 636
 637	recalc_intercepts(svm);
 638}
 639
 640static void clr_dr_intercepts(struct vcpu_svm *svm)
 641{
 642	struct vmcb *vmcb = svm->vmcb01.ptr;
 643
 644	vmcb->control.intercepts[INTERCEPT_DR] = 0;
 645
 646	recalc_intercepts(svm);
 647}
 648
 649static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
 650{
 651	/*
 652	 * For non-nested case:
 653	 * If the L01 MSR bitmap does not intercept the MSR, then we need to
 654	 * save it.
 655	 *
 656	 * For nested case:
 657	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
 658	 * save it.
 659	 */
 660	void *msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm :
 661					    to_svm(vcpu)->msrpm;
 662
 663	return svm_test_msr_bitmap_write(msrpm, msr);
 664}
 665
 666void svm_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool set)
 667{
 668	struct vcpu_svm *svm = to_svm(vcpu);
 669	void *msrpm = svm->msrpm;
 670
 671	/* Don't disable interception for MSRs userspace wants to handle. */
 672	if (type & MSR_TYPE_R) {
 673		if (!set && kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_READ))
 674			svm_clear_msr_bitmap_read(msrpm, msr);
 675		else
 676			svm_set_msr_bitmap_read(msrpm, msr);
 677	}
 678
 679	if (type & MSR_TYPE_W) {
 680		if (!set && kvm_msr_allowed(vcpu, msr, KVM_MSR_FILTER_WRITE))
 681			svm_clear_msr_bitmap_write(msrpm, msr);
 682		else
 683			svm_set_msr_bitmap_write(msrpm, msr);
 684	}
 685
 686	svm_hv_vmcb_dirty_nested_enlightenments(vcpu);
 687	svm->nested.force_msr_bitmap_recalc = true;
 688}
 689
 690void *svm_alloc_permissions_map(unsigned long size, gfp_t gfp_mask)
 691{
 692	unsigned int order = get_order(size);
 693	struct page *pages = alloc_pages(gfp_mask, order);
 694	void *pm;
 695
 696	if (!pages)
 697		return NULL;
 698
 699	/*
 700	 * Set all bits in the permissions map so that all MSR and I/O accesses
 701	 * are intercepted by default.
 702	 */
 703	pm = page_address(pages);
 704	memset(pm, 0xff, PAGE_SIZE * (1 << order));
 705
 706	return pm;
 707}
 708
 709static void svm_recalc_lbr_msr_intercepts(struct kvm_vcpu *vcpu)
 710{
 711	struct vcpu_svm *svm = to_svm(vcpu);
 712	bool intercept = !(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK);
 713
 714	if (intercept == svm->lbr_msrs_intercepted)
 715		return;
 716
 717	svm_set_intercept_for_msr(vcpu, MSR_IA32_LASTBRANCHFROMIP, MSR_TYPE_RW, intercept);
 718	svm_set_intercept_for_msr(vcpu, MSR_IA32_LASTBRANCHTOIP, MSR_TYPE_RW, intercept);
 719	svm_set_intercept_for_msr(vcpu, MSR_IA32_LASTINTFROMIP, MSR_TYPE_RW, intercept);
 720	svm_set_intercept_for_msr(vcpu, MSR_IA32_LASTINTTOIP, MSR_TYPE_RW, intercept);
 721
 722	if (sev_es_guest(vcpu->kvm))
 723		svm_set_intercept_for_msr(vcpu, MSR_IA32_DEBUGCTLMSR, MSR_TYPE_RW, intercept);
 724
 725	svm->lbr_msrs_intercepted = intercept;
 726}
 727
 728void svm_vcpu_free_msrpm(void *msrpm)
 729{
 730	__free_pages(virt_to_page(msrpm), get_order(MSRPM_SIZE));
 731}
 732
 733static void svm_recalc_msr_intercepts(struct kvm_vcpu *vcpu)
 734{
 735	struct vcpu_svm *svm = to_svm(vcpu);
 736
 737	svm_disable_intercept_for_msr(vcpu, MSR_STAR, MSR_TYPE_RW);
 738	svm_disable_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_CS, MSR_TYPE_RW);
 739
 740#ifdef CONFIG_X86_64
 741	svm_disable_intercept_for_msr(vcpu, MSR_GS_BASE, MSR_TYPE_RW);
 742	svm_disable_intercept_for_msr(vcpu, MSR_FS_BASE, MSR_TYPE_RW);
 743	svm_disable_intercept_for_msr(vcpu, MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
 744	svm_disable_intercept_for_msr(vcpu, MSR_LSTAR, MSR_TYPE_RW);
 745	svm_disable_intercept_for_msr(vcpu, MSR_CSTAR, MSR_TYPE_RW);
 746	svm_disable_intercept_for_msr(vcpu, MSR_SYSCALL_MASK, MSR_TYPE_RW);
 747#endif
 748
 749	if (lbrv)
 750		svm_recalc_lbr_msr_intercepts(vcpu);
 751
 752	if (cpu_feature_enabled(X86_FEATURE_IBPB))
 753		svm_set_intercept_for_msr(vcpu, MSR_IA32_PRED_CMD, MSR_TYPE_W,
 754					  !guest_has_pred_cmd_msr(vcpu));
 755
 756	if (cpu_feature_enabled(X86_FEATURE_FLUSH_L1D))
 757		svm_set_intercept_for_msr(vcpu, MSR_IA32_FLUSH_CMD, MSR_TYPE_W,
 758					  !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D));
 759
 760	/*
 761	 * Disable interception of SPEC_CTRL if KVM doesn't need to manually
 762	 * context switch the MSR (SPEC_CTRL is virtualized by the CPU), or if
 763	 * the guest has a non-zero SPEC_CTRL value, i.e. is likely actively
 764	 * using SPEC_CTRL.
 765	 */
 766	if (cpu_feature_enabled(X86_FEATURE_V_SPEC_CTRL))
 767		svm_set_intercept_for_msr(vcpu, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW,
 768					  !guest_has_spec_ctrl_msr(vcpu));
 769	else
 770		svm_set_intercept_for_msr(vcpu, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW,
 771					  !svm->spec_ctrl);
 772
 773	/*
 774	 * Intercept SYSENTER_EIP and SYSENTER_ESP when emulating an Intel CPU,
 775	 * as AMD hardware only store 32 bits, whereas Intel CPUs track 64 bits.
 776	 */
 777	svm_set_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_EIP, MSR_TYPE_RW,
 778				  guest_cpuid_is_intel_compatible(vcpu));
 779	svm_set_intercept_for_msr(vcpu, MSR_IA32_SYSENTER_ESP, MSR_TYPE_RW,
 780				  guest_cpuid_is_intel_compatible(vcpu));
 781
 782	if (kvm_aperfmperf_in_guest(vcpu->kvm)) {
 783		svm_disable_intercept_for_msr(vcpu, MSR_IA32_APERF, MSR_TYPE_R);
 784		svm_disable_intercept_for_msr(vcpu, MSR_IA32_MPERF, MSR_TYPE_R);
 785	}
 786
 787	if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) {
 788		bool shstk_enabled = guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK);
 789
 790		svm_set_intercept_for_msr(vcpu, MSR_IA32_U_CET, MSR_TYPE_RW, !shstk_enabled);
 791		svm_set_intercept_for_msr(vcpu, MSR_IA32_S_CET, MSR_TYPE_RW, !shstk_enabled);
 792		svm_set_intercept_for_msr(vcpu, MSR_IA32_PL0_SSP, MSR_TYPE_RW, !shstk_enabled);
 793		svm_set_intercept_for_msr(vcpu, MSR_IA32_PL1_SSP, MSR_TYPE_RW, !shstk_enabled);
 794		svm_set_intercept_for_msr(vcpu, MSR_IA32_PL2_SSP, MSR_TYPE_RW, !shstk_enabled);
 795		svm_set_intercept_for_msr(vcpu, MSR_IA32_PL3_SSP, MSR_TYPE_RW, !shstk_enabled);
 796	}
 797
 798	if (sev_es_guest(vcpu->kvm))
 799		sev_es_recalc_msr_intercepts(vcpu);
 800
 801	/*
 802	 * x2APIC intercepts are modified on-demand and cannot be filtered by
 803	 * userspace.
 804	 */
 805}
 806
 807void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
 808{
 809	to_vmcb->save.dbgctl		= from_vmcb->save.dbgctl;
 810	to_vmcb->save.br_from		= from_vmcb->save.br_from;
 811	to_vmcb->save.br_to		= from_vmcb->save.br_to;
 812	to_vmcb->save.last_excp_from	= from_vmcb->save.last_excp_from;
 813	to_vmcb->save.last_excp_to	= from_vmcb->save.last_excp_to;
 814
 815	vmcb_mark_dirty(to_vmcb, VMCB_LBR);
 816}
 817
 818static void __svm_enable_lbrv(struct kvm_vcpu *vcpu)
 819{
 820	to_svm(vcpu)->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
 821}
 822
 823void svm_enable_lbrv(struct kvm_vcpu *vcpu)
 824{
 825	__svm_enable_lbrv(vcpu);
 826	svm_recalc_lbr_msr_intercepts(vcpu);
 827}
 828
 829static void __svm_disable_lbrv(struct kvm_vcpu *vcpu)
 830{
 831	KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm);
 832	to_svm(vcpu)->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
 833}
 834
 835void svm_update_lbrv(struct kvm_vcpu *vcpu)
 836{
 837	struct vcpu_svm *svm = to_svm(vcpu);
 838	bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK;
 839	bool enable_lbrv = (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) ||
 840			    (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
 841			    (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK));
 842
 843	if (enable_lbrv && !current_enable_lbrv)
 844		__svm_enable_lbrv(vcpu);
 845	else if (!enable_lbrv && current_enable_lbrv)
 846		__svm_disable_lbrv(vcpu);
 847
 848	/*
 849	 * During nested transitions, it is possible that the current VMCB has
 850	 * LBR_CTL set, but the previous LBR_CTL had it cleared (or vice versa).
 851	 * In this case, even though LBR_CTL does not need an update, intercepts
 852	 * do, so always recalculate the intercepts here.
 853	 */
 854	svm_recalc_lbr_msr_intercepts(vcpu);
 855}
 856
 857void disable_nmi_singlestep(struct vcpu_svm *svm)
 858{
 859	svm->nmi_singlestep = false;
 860
 861	if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
 862		/* Clear our flags if they were not set by the guest */
 863		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
 864			svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
 865		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
 866			svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
 867	}
 868}
 869
 870static void grow_ple_window(struct kvm_vcpu *vcpu)
 871{
 872	struct vcpu_svm *svm = to_svm(vcpu);
 873	struct vmcb_control_area *control = &svm->vmcb->control;
 874	int old = control->pause_filter_count;
 875
 876	if (kvm_pause_in_guest(vcpu->kvm))
 877		return;
 878
 879	control->pause_filter_count = __grow_ple_window(old,
 880							pause_filter_count,
 881							pause_filter_count_grow,
 882							pause_filter_count_max);
 883
 884	if (control->pause_filter_count != old) {
 885		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 886		trace_kvm_ple_window_update(vcpu->vcpu_id,
 887					    control->pause_filter_count, old);
 888	}
 889}
 890
 891static void shrink_ple_window(struct kvm_vcpu *vcpu)
 892{
 893	struct vcpu_svm *svm = to_svm(vcpu);
 894	struct vmcb_control_area *control = &svm->vmcb->control;
 895	int old = control->pause_filter_count;
 896
 897	if (kvm_pause_in_guest(vcpu->kvm))
 898		return;
 899
 900	control->pause_filter_count =
 901				__shrink_ple_window(old,
 902						    pause_filter_count,
 903						    pause_filter_count_shrink,
 904						    pause_filter_count);
 905	if (control->pause_filter_count != old) {
 906		vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 907		trace_kvm_ple_window_update(vcpu->vcpu_id,
 908					    control->pause_filter_count, old);
 909	}
 910}
 911
 912static void svm_hardware_unsetup(void)
 913{
 914	int cpu;
 915
 916	avic_hardware_unsetup();
 917
 918	sev_hardware_unsetup();
 919
 920	for_each_possible_cpu(cpu)
 921		svm_cpu_uninit(cpu);
 922
 923	__free_pages(__sme_pa_to_page(iopm_base), get_order(IOPM_SIZE));
 924	iopm_base = 0;
 925}
 926
 927static void init_seg(struct vmcb_seg *seg)
 928{
 929	seg->selector = 0;
 930	seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
 931		      SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
 932	seg->limit = 0xffff;
 933	seg->base = 0;
 934}
 935
 936static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
 937{
 938	seg->selector = 0;
 939	seg->attrib = SVM_SELECTOR_P_MASK | type;
 940	seg->limit = 0xffff;
 941	seg->base = 0;
 942}
 943
 944static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
 945{
 946	struct vcpu_svm *svm = to_svm(vcpu);
 947
 948	return svm->nested.ctl.tsc_offset;
 949}
 950
 951static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
 952{
 953	struct vcpu_svm *svm = to_svm(vcpu);
 954
 955	return svm->tsc_ratio_msr;
 956}
 957
 958static void svm_write_tsc_offset(struct kvm_vcpu *vcpu)
 959{
 960	struct vcpu_svm *svm = to_svm(vcpu);
 961
 962	svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset;
 963	svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset;
 964	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 965}
 966
 967void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu)
 968{
 969	preempt_disable();
 970	if (to_svm(vcpu)->guest_state_loaded)
 971		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
 972	preempt_enable();
 973}
 974
 975/* Evaluate instruction intercepts that depend on guest CPUID features. */
 976static void svm_recalc_instruction_intercepts(struct kvm_vcpu *vcpu)
 977{
 978	struct vcpu_svm *svm = to_svm(vcpu);
 979
 980	/*
 981	 * Intercept INVPCID if shadow paging is enabled to sync/free shadow
 982	 * roots, or if INVPCID is disabled in the guest to inject #UD.
 983	 */
 984	if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) {
 985		if (!npt_enabled ||
 986		    !guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_INVPCID))
 987			svm_set_intercept(svm, INTERCEPT_INVPCID);
 988		else
 989			svm_clr_intercept(svm, INTERCEPT_INVPCID);
 990	}
 991
 992	if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) {
 993		if (guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP))
 994			svm_clr_intercept(svm, INTERCEPT_RDTSCP);
 995		else
 996			svm_set_intercept(svm, INTERCEPT_RDTSCP);
 997	}
 998
 999	if (guest_cpuid_is_intel_compatible(vcpu)) {
1000		svm_set_intercept(svm, INTERCEPT_VMLOAD);
1001		svm_set_intercept(svm, INTERCEPT_VMSAVE);
1002		svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1003	} else {
1004		/*
1005		 * If hardware supports Virtual VMLOAD VMSAVE then enable it
1006		 * in VMCB and clear intercepts to avoid #VMEXIT.
1007		 */
1008		if (vls) {
1009			svm_clr_intercept(svm, INTERCEPT_VMLOAD);
1010			svm_clr_intercept(svm, INTERCEPT_VMSAVE);
1011			svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1012		}
1013	}
1014}
1015
1016static void svm_recalc_intercepts(struct kvm_vcpu *vcpu)
1017{
1018	svm_recalc_instruction_intercepts(vcpu);
1019	svm_recalc_msr_intercepts(vcpu);
1020}
1021
1022static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event)
1023{
1024	struct vcpu_svm *svm = to_svm(vcpu);
1025	struct vmcb *vmcb = svm->vmcb01.ptr;
1026	struct vmcb_control_area *control = &vmcb->control;
1027	struct vmcb_save_area *save = &vmcb->save;
1028
1029	svm_set_intercept(svm, INTERCEPT_CR0_READ);
1030	svm_set_intercept(svm, INTERCEPT_CR3_READ);
1031	svm_set_intercept(svm, INTERCEPT_CR4_READ);
1032	svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1033	svm_set_intercept(svm, INTERCEPT_CR3_WRITE);
1034	svm_set_intercept(svm, INTERCEPT_CR4_WRITE);
1035	svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
1036
1037	set_dr_intercepts(svm);
1038
1039	set_exception_intercept(svm, PF_VECTOR);
1040	set_exception_intercept(svm, UD_VECTOR);
1041	set_exception_intercept(svm, MC_VECTOR);
1042	set_exception_intercept(svm, AC_VECTOR);
1043	set_exception_intercept(svm, DB_VECTOR);
1044	/*
1045	 * Guest access to VMware backdoor ports could legitimately
1046	 * trigger #GP because of TSS I/O permission bitmap.
1047	 * We intercept those #GP and allow access to them anyway
1048	 * as VMware does.
1049	 */
1050	if (enable_vmware_backdoor)
1051		set_exception_intercept(svm, GP_VECTOR);
1052
1053	svm_set_intercept(svm, INTERCEPT_INTR);
1054	svm_set_intercept(svm, INTERCEPT_NMI);
1055
1056	if (intercept_smi)
1057		svm_set_intercept(svm, INTERCEPT_SMI);
1058
1059	svm_set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1060	svm_set_intercept(svm, INTERCEPT_RDPMC);
1061	svm_set_intercept(svm, INTERCEPT_CPUID);
1062	svm_set_intercept(svm, INTERCEPT_INVD);
1063	svm_set_intercept(svm, INTERCEPT_INVLPG);
1064	svm_set_intercept(svm, INTERCEPT_INVLPGA);
1065	svm_set_intercept(svm, INTERCEPT_IOIO_PROT);
1066	svm_set_intercept(svm, INTERCEPT_MSR_PROT);
1067	svm_set_intercept(svm, INTERCEPT_TASK_SWITCH);
1068	svm_set_intercept(svm, INTERCEPT_SHUTDOWN);
1069	svm_set_intercept(svm, INTERCEPT_VMRUN);
1070	svm_set_intercept(svm, INTERCEPT_VMMCALL);
1071	svm_set_intercept(svm, INTERCEPT_VMLOAD);
1072	svm_set_intercept(svm, INTERCEPT_VMSAVE);
1073	svm_set_intercept(svm, INTERCEPT_STGI);
1074	svm_set_intercept(svm, INTERCEPT_CLGI);
1075	svm_set_intercept(svm, INTERCEPT_SKINIT);
1076	svm_set_intercept(svm, INTERCEPT_WBINVD);
1077	svm_set_intercept(svm, INTERCEPT_XSETBV);
1078	svm_set_intercept(svm, INTERCEPT_RDPRU);
1079	svm_set_intercept(svm, INTERCEPT_RSM);
1080
1081	if (!kvm_mwait_in_guest(vcpu->kvm)) {
1082		svm_set_intercept(svm, INTERCEPT_MONITOR);
1083		svm_set_intercept(svm, INTERCEPT_MWAIT);
1084	}
1085
1086	if (!kvm_hlt_in_guest(vcpu->kvm)) {
1087		if (cpu_feature_enabled(X86_FEATURE_IDLE_HLT))
1088			svm_set_intercept(svm, INTERCEPT_IDLE_HLT);
1089		else
1090			svm_set_intercept(svm, INTERCEPT_HLT);
1091	}
1092
1093	control->iopm_base_pa = iopm_base;
1094	control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1095	control->int_ctl = V_INTR_MASKING_MASK;
1096
1097	init_seg(&save->es);
1098	init_seg(&save->ss);
1099	init_seg(&save->ds);
1100	init_seg(&save->fs);
1101	init_seg(&save->gs);
1102
1103	save->cs.selector = 0xf000;
1104	save->cs.base = 0xffff0000;
1105	/* Executable/Readable Code Segment */
1106	save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1107		SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1108	save->cs.limit = 0xffff;
1109
1110	save->gdtr.base = 0;
1111	save->gdtr.limit = 0xffff;
1112	save->idtr.base = 0;
1113	save->idtr.limit = 0xffff;
1114
1115	init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1116	init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1117
1118	if (npt_enabled) {
1119		/* Setup VMCB for Nested Paging */
1120		control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1121		svm_clr_intercept(svm, INTERCEPT_INVLPG);
1122		clr_exception_intercept(svm, PF_VECTOR);
1123		svm_clr_intercept(svm, INTERCEPT_CR3_READ);
1124		svm_clr_intercept(svm, INTERCEPT_CR3_WRITE);
1125		save->g_pat = vcpu->arch.pat;
1126		save->cr3 = 0;
1127	}
1128	svm->current_vmcb->asid_generation = 0;
1129	svm->asid = 0;
1130
1131	svm->nested.vmcb12_gpa = INVALID_GPA;
1132	svm->nested.last_vmcb12_gpa = INVALID_GPA;
1133
1134	if (!kvm_pause_in_guest(vcpu->kvm)) {
1135		control->pause_filter_count = pause_filter_count;
1136		if (pause_filter_thresh)
1137			control->pause_filter_thresh = pause_filter_thresh;
1138		svm_set_intercept(svm, INTERCEPT_PAUSE);
1139	} else {
1140		svm_clr_intercept(svm, INTERCEPT_PAUSE);
1141	}
1142
1143	if (enable_apicv && irqchip_in_kernel(vcpu->kvm))
1144		avic_init_vmcb(svm, vmcb);
1145
1146	if (vnmi)
1147		svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK;
1148
1149	if (vgif) {
1150		svm_clr_intercept(svm, INTERCEPT_STGI);
1151		svm_clr_intercept(svm, INTERCEPT_CLGI);
1152		svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1153	}
1154
1155	if (vcpu->kvm->arch.bus_lock_detection_enabled)
1156		svm_set_intercept(svm, INTERCEPT_BUSLOCK);
1157
1158	if (sev_guest(vcpu->kvm))
1159		sev_init_vmcb(svm, init_event);
1160
1161	svm_hv_init_vmcb(vmcb);
1162
1163	kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu);
1164
1165	vmcb_mark_all_dirty(vmcb);
1166
1167	enable_gif(svm);
1168}
1169
1170static void __svm_vcpu_reset(struct kvm_vcpu *vcpu)
1171{
1172	struct vcpu_svm *svm = to_svm(vcpu);
1173
1174	svm_init_osvw(vcpu);
1175
1176	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS))
1177		vcpu->arch.microcode_version = 0x01000065;
1178	svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio;
1179
1180	svm->nmi_masked = false;
1181	svm->awaiting_iret_completion = false;
1182}
1183
1184static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1185{
1186	struct vcpu_svm *svm = to_svm(vcpu);
1187
1188	svm->spec_ctrl = 0;
1189	svm->virt_spec_ctrl = 0;
1190
1191	init_vmcb(vcpu, init_event);
1192
1193	if (!init_event)
1194		__svm_vcpu_reset(vcpu);
1195}
1196
1197void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
1198{
1199	svm->current_vmcb = target_vmcb;
1200	svm->vmcb = target_vmcb->ptr;
1201}
1202
1203static int svm_vcpu_precreate(struct kvm *kvm)
1204{
1205	return avic_alloc_physical_id_table(kvm);
1206}
1207
1208static int svm_vcpu_create(struct kvm_vcpu *vcpu)
1209{
1210	struct vcpu_svm *svm;
1211	struct page *vmcb01_page;
1212	int err;
1213
1214	BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
1215	svm = to_svm(vcpu);
1216
1217	err = -ENOMEM;
1218	vmcb01_page = snp_safe_alloc_page();
1219	if (!vmcb01_page)
1220		goto out;
1221
1222	err = sev_vcpu_create(vcpu);
1223	if (err)
1224		goto error_free_vmcb_page;
1225
1226	err = avic_init_vcpu(svm);
1227	if (err)
1228		goto error_free_sev;
1229
1230	svm->msrpm = svm_vcpu_alloc_msrpm();
1231	if (!svm->msrpm) {
1232		err = -ENOMEM;
1233		goto error_free_sev;
1234	}
1235
1236	svm->x2avic_msrs_intercepted = true;
1237	svm->lbr_msrs_intercepted = true;
1238
1239	svm->vmcb01.ptr = page_address(vmcb01_page);
1240	svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT);
1241	svm_switch_vmcb(svm, &svm->vmcb01);
1242
1243	svm->guest_state_loaded = false;
1244
1245	return 0;
1246
1247error_free_sev:
1248	sev_free_vcpu(vcpu);
1249error_free_vmcb_page:
1250	__free_page(vmcb01_page);
1251out:
1252	return err;
1253}
1254
1255static void svm_vcpu_free(struct kvm_vcpu *vcpu)
1256{
1257	struct vcpu_svm *svm = to_svm(vcpu);
1258
1259	WARN_ON_ONCE(!list_empty(&svm->ir_list));
1260
1261	svm_leave_nested(vcpu);
1262	svm_free_nested(svm);
1263
1264	sev_free_vcpu(vcpu);
1265
1266	__free_page(__sme_pa_to_page(svm->vmcb01.pa));
1267	svm_vcpu_free_msrpm(svm->msrpm);
1268}
1269
1270#ifdef CONFIG_CPU_MITIGATIONS
1271static DEFINE_SPINLOCK(srso_lock);
1272static atomic_t srso_nr_vms;
1273
1274static void svm_srso_clear_bp_spec_reduce(void *ign)
1275{
1276	struct svm_cpu_data *sd = this_cpu_ptr(&svm_data);
1277
1278	if (!sd->bp_spec_reduce_set)
1279		return;
1280
1281	msr_clear_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT);
1282	sd->bp_spec_reduce_set = false;
1283}
1284
1285static void svm_srso_vm_destroy(void)
1286{
1287	if (!cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE))
1288		return;
1289
1290	if (atomic_dec_return(&srso_nr_vms))
1291		return;
1292
1293	guard(spinlock)(&srso_lock);
1294
1295	/*
1296	 * Verify a new VM didn't come along, acquire the lock, and increment
1297	 * the count before this task acquired the lock.
1298	 */
1299	if (atomic_read(&srso_nr_vms))
1300		return;
1301
1302	on_each_cpu(svm_srso_clear_bp_spec_reduce, NULL, 1);
1303}
1304
1305static void svm_srso_vm_init(void)
1306{
1307	if (!cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE))
1308		return;
1309
1310	/*
1311	 * Acquire the lock on 0 => 1 transitions to ensure a potential 1 => 0
1312	 * transition, i.e. destroying the last VM, is fully complete, e.g. so
1313	 * that a delayed IPI doesn't clear BP_SPEC_REDUCE after a vCPU runs.
1314	 */
1315	if (atomic_inc_not_zero(&srso_nr_vms))
1316		return;
1317
1318	guard(spinlock)(&srso_lock);
1319
1320	atomic_inc(&srso_nr_vms);
1321}
1322#else
1323static void svm_srso_vm_init(void) { }
1324static void svm_srso_vm_destroy(void) { }
1325#endif
1326
1327static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
1328{
1329	struct vcpu_svm *svm = to_svm(vcpu);
1330	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
1331
1332	if (sev_es_guest(vcpu->kvm))
1333		sev_es_unmap_ghcb(svm);
1334
1335	if (svm->guest_state_loaded)
1336		return;
1337
1338	/*
1339	 * Save additional host state that will be restored on VMEXIT (sev-es)
1340	 * or subsequent vmload of host save area.
1341	 */
1342	vmsave(sd->save_area_pa);
1343	if (sev_es_guest(vcpu->kvm))
1344		sev_es_prepare_switch_to_guest(svm, sev_es_host_save_area(sd));
1345
1346	if (tsc_scaling)
1347		__svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio);
1348
1349	/*
1350	 * TSC_AUX is always virtualized (context switched by hardware) for
1351	 * SEV-ES guests when the feature is available.  For non-SEV-ES guests,
1352	 * context switch TSC_AUX via the user_return MSR infrastructure (not
1353	 * all CPUs support TSC_AUX virtualization).
1354	 */
1355	if (likely(tsc_aux_uret_slot >= 0) &&
1356	    (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm)))
1357		kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull);
1358
1359	if (cpu_feature_enabled(X86_FEATURE_SRSO_BP_SPEC_REDUCE) &&
1360	    !sd->bp_spec_reduce_set) {
1361		sd->bp_spec_reduce_set = true;
1362		msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT);
1363	}
1364	svm->guest_state_loaded = true;
1365}
1366
1367static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
1368{
1369	to_svm(vcpu)->guest_state_loaded = false;
1370}
1371
1372static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1373{
1374	if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm))
1375		shrink_ple_window(vcpu);
1376
1377	if (kvm_vcpu_apicv_active(vcpu))
1378		avic_vcpu_load(vcpu, cpu);
1379}
1380
1381static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1382{
1383	if (kvm_vcpu_apicv_active(vcpu))
1384		avic_vcpu_put(vcpu);
1385
1386	svm_prepare_host_switch(vcpu);
1387
1388	++vcpu->stat.host_state_reload;
1389}
1390
1391static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1392{
1393	struct vcpu_svm *svm = to_svm(vcpu);
1394	unsigned long rflags = svm->vmcb->save.rflags;
1395
1396	if (svm->nmi_singlestep) {
1397		/* Hide our flags if they were not set by the guest */
1398		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1399			rflags &= ~X86_EFLAGS_TF;
1400		if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1401			rflags &= ~X86_EFLAGS_RF;
1402	}
1403	return rflags;
1404}
1405
1406static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1407{
1408	if (to_svm(vcpu)->nmi_singlestep)
1409		rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
1410
1411       /*
1412        * Any change of EFLAGS.VM is accompanied by a reload of SS
1413        * (caused by either a task switch or an inter-privilege IRET),
1414        * so we do not need to update the CPL here.
1415        */
1416	to_svm(vcpu)->vmcb->save.rflags = rflags;
1417}
1418
1419static bool svm_get_if_flag(struct kvm_vcpu *vcpu)
1420{
1421	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
1422
1423	return sev_es_guest(vcpu->kvm)
1424		? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK
1425		: kvm_get_rflags(vcpu) & X86_EFLAGS_IF;
1426}
1427
1428static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1429{
1430	kvm_register_mark_available(vcpu, reg);
1431
1432	switch (reg) {
1433	case VCPU_EXREG_PDPTR:
1434		/*
1435		 * When !npt_enabled, mmu->pdptrs[] is already available since
1436		 * it is always updated per SDM when moving to CRs.
1437		 */
1438		if (npt_enabled)
1439			load_pdptrs(vcpu, kvm_read_cr3(vcpu));
1440		break;
1441	default:
1442		KVM_BUG_ON(1, vcpu->kvm);
1443	}
1444}
1445
1446static void svm_set_vintr(struct vcpu_svm *svm)
1447{
1448	struct vmcb_control_area *control;
1449
1450	/*
1451	 * The following fields are ignored when AVIC is enabled
1452	 */
1453	WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu));
1454
1455	svm_set_intercept(svm, INTERCEPT_VINTR);
1456
1457	/*
1458	 * Recalculating intercepts may have cleared the VINTR intercept.  If
1459	 * V_INTR_MASKING is enabled in vmcb12, then the effective RFLAGS.IF
1460	 * for L1 physical interrupts is L1's RFLAGS.IF at the time of VMRUN.
1461	 * Requesting an interrupt window if save.RFLAGS.IF=0 is pointless as
1462	 * interrupts will never be unblocked while L2 is running.
1463	 */
1464	if (!svm_is_intercept(svm, INTERCEPT_VINTR))
1465		return;
1466
1467	/*
1468	 * This is just a dummy VINTR to actually cause a vmexit to happen.
1469	 * Actual injection of virtual interrupts happens through EVENTINJ.
1470	 */
1471	control = &svm->vmcb->control;
1472	control->int_vector = 0x0;
1473	control->int_ctl &= ~V_INTR_PRIO_MASK;
1474	control->int_ctl |= V_IRQ_MASK |
1475		((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
1476	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1477}
1478
1479static void svm_clear_vintr(struct vcpu_svm *svm)
1480{
1481	svm_clr_intercept(svm, INTERCEPT_VINTR);
1482
1483	/* Drop int_ctl fields related to VINTR injection.  */
1484	svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1485	if (is_guest_mode(&svm->vcpu)) {
1486		svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK;
1487
1488		WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) !=
1489			(svm->nested.ctl.int_ctl & V_TPR_MASK));
1490
1491		svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl &
1492			V_IRQ_INJECTION_BITS_MASK;
1493
1494		svm->vmcb->control.int_vector = svm->nested.ctl.int_vector;
1495	}
1496
1497	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
1498}
1499
1500static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1501{
1502	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1503	struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save;
1504
1505	switch (seg) {
1506	case VCPU_SREG_CS: return &save->cs;
1507	case VCPU_SREG_DS: return &save->ds;
1508	case VCPU_SREG_ES: return &save->es;
1509	case VCPU_SREG_FS: return &save01->fs;
1510	case VCPU_SREG_GS: return &save01->gs;
1511	case VCPU_SREG_SS: return &save->ss;
1512	case VCPU_SREG_TR: return &save01->tr;
1513	case VCPU_SREG_LDTR: return &save01->ldtr;
1514	}
1515	BUG();
1516	return NULL;
1517}
1518
1519static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1520{
1521	struct vmcb_seg *s = svm_seg(vcpu, seg);
1522
1523	return s->base;
1524}
1525
1526static void svm_get_segment(struct kvm_vcpu *vcpu,
1527			    struct kvm_segment *var, int seg)
1528{
1529	struct vmcb_seg *s = svm_seg(vcpu, seg);
1530
1531	var->base = s->base;
1532	var->limit = s->limit;
1533	var->selector = s->selector;
1534	var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1535	var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1536	var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1537	var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1538	var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1539	var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1540	var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1541
1542	/*
1543	 * AMD CPUs circa 2014 track the G bit for all segments except CS.
1544	 * However, the SVM spec states that the G bit is not observed by the
1545	 * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1546	 * So let's synthesize a legal G bit for all segments, this helps
1547	 * running KVM nested. It also helps cross-vendor migration, because
1548	 * Intel's vmentry has a check on the 'G' bit.
1549	 */
1550	var->g = s->limit > 0xfffff;
1551
1552	/*
1553	 * AMD's VMCB does not have an explicit unusable field, so emulate it
1554	 * for cross vendor migration purposes by "not present"
1555	 */
1556	var->unusable = !var->present;
1557
1558	switch (seg) {
1559	case VCPU_SREG_TR:
1560		/*
1561		 * Work around a bug where the busy flag in the tr selector
1562		 * isn't exposed
1563		 */
1564		var->type |= 0x2;
1565		break;
1566	case VCPU_SREG_DS:
1567	case VCPU_SREG_ES:
1568	case VCPU_SREG_FS:
1569	case VCPU_SREG_GS:
1570		/*
1571		 * The accessed bit must always be set in the segment
1572		 * descriptor cache, although it can be cleared in the
1573		 * descriptor, the cached bit always remains at 1. Since
1574		 * Intel has a check on this, set it here to support
1575		 * cross-vendor migration.
1576		 */
1577		if (!var->unusable)
1578			var->type |= 0x1;
1579		break;
1580	case VCPU_SREG_SS:
1581		/*
1582		 * On AMD CPUs sometimes the DB bit in the segment
1583		 * descriptor is left as 1, although the whole segment has
1584		 * been made unusable. Clear it here to pass an Intel VMX
1585		 * entry check when cross vendor migrating.
1586		 */
1587		if (var->unusable)
1588			var->db = 0;
1589		/* This is symmetric with svm_set_segment() */
1590		var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1591		break;
1592	}
1593}
1594
1595static int svm_get_cpl(struct kvm_vcpu *vcpu)
1596{
1597	struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1598
1599	return save->cpl;
1600}
1601
1602static void svm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1603{
1604	struct kvm_segment cs;
1605
1606	svm_get_segment(vcpu, &cs, VCPU_SREG_CS);
1607	*db = cs.db;
1608	*l = cs.l;
1609}
1610
1611static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1612{
1613	struct vcpu_svm *svm = to_svm(vcpu);
1614
1615	dt->size = svm->vmcb->save.idtr.limit;
1616	dt->address = svm->vmcb->save.idtr.base;
1617}
1618
1619static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1620{
1621	struct vcpu_svm *svm = to_svm(vcpu);
1622
1623	svm->vmcb->save.idtr.limit = dt->size;
1624	svm->vmcb->save.idtr.base = dt->address ;
1625	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1626}
1627
1628static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1629{
1630	struct vcpu_svm *svm = to_svm(vcpu);
1631
1632	dt->size = svm->vmcb->save.gdtr.limit;
1633	dt->address = svm->vmcb->save.gdtr.base;
1634}
1635
1636static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1637{
1638	struct vcpu_svm *svm = to_svm(vcpu);
1639
1640	svm->vmcb->save.gdtr.limit = dt->size;
1641	svm->vmcb->save.gdtr.base = dt->address ;
1642	vmcb_mark_dirty(svm->vmcb, VMCB_DT);
1643}
1644
1645static void sev_post_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1646{
1647	struct vcpu_svm *svm = to_svm(vcpu);
1648
1649	/*
1650	 * For guests that don't set guest_state_protected, the cr3 update is
1651	 * handled via kvm_mmu_load() while entering the guest. For guests
1652	 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to
1653	 * VMCB save area now, since the save area will become the initial
1654	 * contents of the VMSA, and future VMCB save area updates won't be
1655	 * seen.
1656	 */
1657	if (sev_es_guest(vcpu->kvm)) {
1658		svm->vmcb->save.cr3 = cr3;
1659		vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1660	}
1661}
1662
1663static bool svm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1664{
1665	return true;
1666}
1667
1668void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1669{
1670	struct vcpu_svm *svm = to_svm(vcpu);
1671	u64 hcr0 = cr0;
1672	bool old_paging = is_paging(vcpu);
1673
1674#ifdef CONFIG_X86_64
1675	if (vcpu->arch.efer & EFER_LME) {
1676		if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1677			vcpu->arch.efer |= EFER_LMA;
1678			if (!vcpu->arch.guest_state_protected)
1679				svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1680		}
1681
1682		if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1683			vcpu->arch.efer &= ~EFER_LMA;
1684			if (!vcpu->arch.guest_state_protected)
1685				svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1686		}
1687	}
1688#endif
1689	vcpu->arch.cr0 = cr0;
1690
1691	if (!npt_enabled) {
1692		hcr0 |= X86_CR0_PG | X86_CR0_WP;
1693		if (old_paging != is_paging(vcpu))
1694			svm_set_cr4(vcpu, kvm_read_cr4(vcpu));
1695	}
1696
1697	/*
1698	 * re-enable caching here because the QEMU bios
1699	 * does not do it - this results in some delay at
1700	 * reboot
1701	 */
1702	if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1703		hcr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1704
1705	svm->vmcb->save.cr0 = hcr0;
1706	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
1707
1708	/*
1709	 * SEV-ES guests must always keep the CR intercepts cleared. CR
1710	 * tracking is done using the CR write traps.
1711	 */
1712	if (sev_es_guest(vcpu->kvm))
1713		return;
1714
1715	if (hcr0 == cr0) {
1716		/* Selective CR0 write remains on.  */
1717		svm_clr_intercept(svm, INTERCEPT_CR0_READ);
1718		svm_clr_intercept(svm, INTERCEPT_CR0_WRITE);
1719	} else {
1720		svm_set_intercept(svm, INTERCEPT_CR0_READ);
1721		svm_set_intercept(svm, INTERCEPT_CR0_WRITE);
1722	}
1723}
1724
1725static bool svm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1726{
1727	return true;
1728}
1729
1730void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1731{
1732	unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1733	unsigned long old_cr4 = vcpu->arch.cr4;
1734
1735	vcpu->arch.cr4 = cr4;
1736	if (!npt_enabled) {
1737		cr4 |= X86_CR4_PAE;
1738
1739		if (!is_paging(vcpu))
1740			cr4 &= ~(X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE);
1741	}
1742	cr4 |= host_cr4_mce;
1743	to_svm(vcpu)->vmcb->save.cr4 = cr4;
1744	vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1745
1746	if ((cr4 ^ old_cr4) & (X86_CR4_OSXSAVE | X86_CR4_PKE))
1747		vcpu->arch.cpuid_dynamic_bits_dirty = true;
1748}
1749
1750static void svm_set_segment(struct kvm_vcpu *vcpu,
1751			    struct kvm_segment *var, int seg)
1752{
1753	struct vcpu_svm *svm = to_svm(vcpu);
1754	struct vmcb_seg *s = svm_seg(vcpu, seg);
1755
1756	s->base = var->base;
1757	s->limit = var->limit;
1758	s->selector = var->selector;
1759	s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1760	s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1761	s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1762	s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
1763	s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1764	s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1765	s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1766	s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1767
1768	/*
1769	 * This is always accurate, except if SYSRET returned to a segment
1770	 * with SS.DPL != 3.  Intel does not have this quirk, and always
1771	 * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1772	 * would entail passing the CPL to userspace and back.
1773	 */
1774	if (seg == VCPU_SREG_SS)
1775		/* This is symmetric with svm_get_segment() */
1776		svm->vmcb->save.cpl = (var->dpl & 3);
1777
1778	vmcb_mark_dirty(svm->vmcb, VMCB_SEG);
1779}
1780
1781static void svm_update_exception_bitmap(struct kvm_vcpu *vcpu)
1782{
1783	struct vcpu_svm *svm = to_svm(vcpu);
1784
1785	clr_exception_intercept(svm, BP_VECTOR);
1786
1787	if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1788		if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1789			set_exception_intercept(svm, BP_VECTOR);
1790	}
1791}
1792
1793static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1794{
1795	if (sd->next_asid > sd->max_asid) {
1796		++sd->asid_generation;
1797		sd->next_asid = sd->min_asid;
1798		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1799		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
1800	}
1801
1802	svm->current_vmcb->asid_generation = sd->asid_generation;
1803	svm->asid = sd->next_asid++;
1804}
1805
1806static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
1807{
1808	struct vmcb *vmcb = to_svm(vcpu)->vmcb;
1809
1810	if (vcpu->arch.guest_state_protected)
1811		return;
1812
1813	if (unlikely(value != vmcb->save.dr6)) {
1814		vmcb->save.dr6 = value;
1815		vmcb_mark_dirty(vmcb, VMCB_DR);
1816	}
1817}
1818
1819static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1820{
1821	struct vcpu_svm *svm = to_svm(vcpu);
1822
1823	if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm)))
1824		return;
1825
1826	get_debugreg(vcpu->arch.db[0], 0);
1827	get_debugreg(vcpu->arch.db[1], 1);
1828	get_debugreg(vcpu->arch.db[2], 2);
1829	get_debugreg(vcpu->arch.db[3], 3);
1830	/*
1831	 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here,
1832	 * because db_interception might need it.  We can do it before vmentry.
1833	 */
1834	vcpu->arch.dr6 = svm->vmcb->save.dr6;
1835	vcpu->arch.dr7 = svm->vmcb->save.dr7;
1836	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1837	set_dr_intercepts(svm);
1838}
1839
1840static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1841{
1842	struct vcpu_svm *svm = to_svm(vcpu);
1843
1844	if (vcpu->arch.guest_state_protected)
1845		return;
1846
1847	svm->vmcb->save.dr7 = value;
1848	vmcb_mark_dirty(svm->vmcb, VMCB_DR);
1849}
1850
1851static int pf_interception(struct kvm_vcpu *vcpu)
1852{
1853	struct vcpu_svm *svm = to_svm(vcpu);
1854
1855	u64 fault_address = svm->vmcb->control.exit_info_2;
1856	u64 error_code = svm->vmcb->control.exit_info_1;
1857
1858	return kvm_handle_page_fault(vcpu, error_code, fault_address,
1859			static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1860			svm->vmcb->control.insn_bytes : NULL,
1861			svm->vmcb->control.insn_len);
1862}
1863
1864static int npf_interception(struct kvm_vcpu *vcpu)
1865{
1866	struct vcpu_svm *svm = to_svm(vcpu);
1867	int rc;
1868
1869	u64 fault_address = svm->vmcb->control.exit_info_2;
1870	u64 error_code = svm->vmcb->control.exit_info_1;
1871
1872	/*
1873	 * WARN if hardware generates a fault with an error code that collides
1874	 * with KVM-defined sythentic flags.  Clear the flags and continue on,
1875	 * i.e. don't terminate the VM, as KVM can't possibly be relying on a
1876	 * flag that KVM doesn't know about.
1877	 */
1878	if (WARN_ON_ONCE(error_code & PFERR_SYNTHETIC_MASK))
1879		error_code &= ~PFERR_SYNTHETIC_MASK;
1880
1881	if (sev_snp_guest(vcpu->kvm) && (error_code & PFERR_GUEST_ENC_MASK))
1882		error_code |= PFERR_PRIVATE_ACCESS;
1883
1884	trace_kvm_page_fault(vcpu, fault_address, error_code);
1885	rc = kvm_mmu_page_fault(vcpu, fault_address, error_code,
1886				static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
1887				svm->vmcb->control.insn_bytes : NULL,
1888				svm->vmcb->control.insn_len);
1889
1890	if (rc > 0 && error_code & PFERR_GUEST_RMP_MASK)
1891		sev_handle_rmp_fault(vcpu, fault_address, error_code);
1892
1893	return rc;
1894}
1895
1896static int db_interception(struct kvm_vcpu *vcpu)
1897{
1898	struct kvm_run *kvm_run = vcpu->run;
1899	struct vcpu_svm *svm = to_svm(vcpu);
1900
1901	if (!(vcpu->guest_debug &
1902	      (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1903		!svm->nmi_singlestep) {
1904		u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW;
1905		kvm_queue_exception_p(vcpu, DB_VECTOR, payload);
1906		return 1;
1907	}
1908
1909	if (svm->nmi_singlestep) {
1910		disable_nmi_singlestep(svm);
1911		/* Make sure we check for pending NMIs upon entry */
1912		kvm_make_request(KVM_REQ_EVENT, vcpu);
1913	}
1914
1915	if (vcpu->guest_debug &
1916	    (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1917		kvm_run->exit_reason = KVM_EXIT_DEBUG;
1918		kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6;
1919		kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7;
1920		kvm_run->debug.arch.pc =
1921			svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1922		kvm_run->debug.arch.exception = DB_VECTOR;
1923		return 0;
1924	}
1925
1926	return 1;
1927}
1928
1929static int bp_interception(struct kvm_vcpu *vcpu)
1930{
1931	struct vcpu_svm *svm = to_svm(vcpu);
1932	struct kvm_run *kvm_run = vcpu->run;
1933
1934	kvm_run->exit_reason = KVM_EXIT_DEBUG;
1935	kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1936	kvm_run->debug.arch.exception = BP_VECTOR;
1937	return 0;
1938}
1939
1940static int ud_interception(struct kvm_vcpu *vcpu)
1941{
1942	return handle_ud(vcpu);
1943}
1944
1945static int ac_interception(struct kvm_vcpu *vcpu)
1946{
1947	kvm_queue_exception_e(vcpu, AC_VECTOR, 0);
1948	return 1;
1949}
1950
1951static bool is_erratum_383(void)
1952{
1953	int i;
1954	u64 value;
1955
1956	if (!erratum_383_found)
1957		return false;
1958
1959	if (native_read_msr_safe(MSR_IA32_MC0_STATUS, &value))
1960		return false;
1961
1962	/* Bit 62 may or may not be set for this mce */
1963	value &= ~(1ULL << 62);
1964
1965	if (value != 0xb600000000010015ULL)
1966		return false;
1967
1968	/* Clear MCi_STATUS registers */
1969	for (i = 0; i < 6; ++i)
1970		native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0);
1971
1972	if (!native_read_msr_safe(MSR_IA32_MCG_STATUS, &value)) {
1973		value &= ~(1ULL << 2);
1974		native_write_msr_safe(MSR_IA32_MCG_STATUS, value);
1975	}
1976
1977	/* Flush tlb to evict multi-match entries */
1978	__flush_tlb_all();
1979
1980	return true;
1981}
1982
1983static void svm_handle_mce(struct kvm_vcpu *vcpu)
1984{
1985	if (is_erratum_383()) {
1986		/*
1987		 * Erratum 383 triggered. Guest state is corrupt so kill the
1988		 * guest.
1989		 */
1990		pr_err("Guest triggered AMD Erratum 383\n");
1991
1992		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1993
1994		return;
1995	}
1996
1997	/*
1998	 * On an #MC intercept the MCE handler is not called automatically in
1999	 * the host. So do it by hand here.
2000	 */
2001	kvm_machine_check();
2002}
2003
2004static int mc_interception(struct kvm_vcpu *vcpu)
2005{
2006	return 1;
2007}
2008
2009static int shutdown_interception(struct kvm_vcpu *vcpu)
2010{
2011	struct kvm_run *kvm_run = vcpu->run;
2012	struct vcpu_svm *svm = to_svm(vcpu);
2013
2014
2015	/*
2016	 * VMCB is undefined after a SHUTDOWN intercept.  INIT the vCPU to put
2017	 * the VMCB in a known good state.  Unfortuately, KVM doesn't have
2018	 * KVM_MP_STATE_SHUTDOWN and can't add it without potentially breaking
2019	 * userspace.  At a platform view, INIT is acceptable behavior as
2020	 * there exist bare metal platforms that automatically INIT the CPU
2021	 * in response to shutdown.
2022	 *
2023	 * The VM save area for SEV-ES guests has already been encrypted so it
2024	 * cannot be reinitialized, i.e. synthesizing INIT is futile.
2025	 */
2026	if (!sev_es_guest(vcpu->kvm)) {
2027		clear_page(svm->vmcb);
2028#ifdef CONFIG_KVM_SMM
2029		if (is_smm(vcpu))
2030			kvm_smm_changed(vcpu, false);
2031#endif
2032		kvm_vcpu_reset(vcpu, true);
2033	}
2034
2035	kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2036	return 0;
2037}
2038
2039static int io_interception(struct kvm_vcpu *vcpu)
2040{
2041	struct vcpu_svm *svm = to_svm(vcpu);
2042	u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2043	int size, in, string;
2044	unsigned port;
2045
2046	++vcpu->stat.io_exits;
2047	string = (io_info & SVM_IOIO_STR_MASK) != 0;
2048	in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2049	port = io_info >> 16;
2050	size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2051
2052	if (string) {
2053		if (sev_es_guest(vcpu->kvm))
2054			return sev_es_string_io(svm, size, port, in);
2055		else
2056			return kvm_emulate_instruction(vcpu, 0);
2057	}
2058
2059	svm->next_rip = svm->vmcb->control.exit_info_2;
2060
2061	return kvm_fast_pio(vcpu, size, port, in);
2062}
2063
2064static int nmi_interception(struct kvm_vcpu *vcpu)
2065{
2066	return 1;
2067}
2068
2069static int smi_interception(struct kvm_vcpu *vcpu)
2070{
2071	return 1;
2072}
2073
2074static int intr_interception(struct kvm_vcpu *vcpu)
2075{
2076	++vcpu->stat.irq_exits;
2077	return 1;
2078}
2079
2080static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload)
2081{
2082	struct vcpu_svm *svm = to_svm(vcpu);
2083	struct vmcb *vmcb12;
2084	struct kvm_host_map map;
2085	int ret;
2086
2087	if (nested_svm_check_permissions(vcpu))
2088		return 1;
2089
2090	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
2091	if (ret) {
2092		if (ret == -EINVAL)
2093			kvm_inject_gp(vcpu, 0);
2094		return 1;
2095	}
2096
2097	vmcb12 = map.hva;
2098
2099	ret = kvm_skip_emulated_instruction(vcpu);
2100
2101	/* KVM always performs VMLOAD/VMSAVE on VMCB01 (see __svm_vcpu_run()) */
2102	if (vmload) {
2103		svm_copy_vmloadsave_state(svm->vmcb01.ptr, vmcb12);
2104		svm->sysenter_eip_hi = 0;
2105		svm->sysenter_esp_hi = 0;
2106	} else {
2107		svm_copy_vmloadsave_state(vmcb12, svm->vmcb01.ptr);
2108	}
2109
2110	kvm_vcpu_unmap(vcpu, &map);
2111
2112	return ret;
2113}
2114
2115static int vmload_interception(struct kvm_vcpu *vcpu)
2116{
2117	return vmload_vmsave_interception(vcpu, true);
2118}
2119
2120static int vmsave_interception(struct kvm_vcpu *vcpu)
2121{
2122	return vmload_vmsave_interception(vcpu, false);
2123}
2124
2125static int vmrun_interception(struct kvm_vcpu *vcpu)
2126{
2127	if (nested_svm_check_permissions(vcpu))
2128		return 1;
2129
2130	return nested_svm_vmrun(vcpu);
2131}
2132
2133enum {
2134	NONE_SVM_INSTR,
2135	SVM_INSTR_VMRUN,
2136	SVM_INSTR_VMLOAD,
2137	SVM_INSTR_VMSAVE,
2138};
2139
2140/* Return NONE_SVM_INSTR if not SVM instrs, otherwise return decode result */
2141static int svm_instr_opcode(struct kvm_vcpu *vcpu)
2142{
2143	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
2144
2145	if (ctxt->b != 0x1 || ctxt->opcode_len != 2)
2146		return NONE_SVM_INSTR;
2147
2148	switch (ctxt->modrm) {
2149	case 0xd8: /* VMRUN */
2150		return SVM_INSTR_VMRUN;
2151	case 0xda: /* VMLOAD */
2152		return SVM_INSTR_VMLOAD;
2153	case 0xdb: /* VMSAVE */
2154		return SVM_INSTR_VMSAVE;
2155	default:
2156		break;
2157	}
2158
2159	return NONE_SVM_INSTR;
2160}
2161
2162static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode)
2163{
2164	const int guest_mode_exit_codes[] = {
2165		[SVM_INSTR_VMRUN] = SVM_EXIT_VMRUN,
2166		[SVM_INSTR_VMLOAD] = SVM_EXIT_VMLOAD,
2167		[SVM_INSTR_VMSAVE] = SVM_EXIT_VMSAVE,
2168	};
2169	int (*const svm_instr_handlers[])(struct kvm_vcpu *vcpu) = {
2170		[SVM_INSTR_VMRUN] = vmrun_interception,
2171		[SVM_INSTR_VMLOAD] = vmload_interception,
2172		[SVM_INSTR_VMSAVE] = vmsave_interception,
2173	};
2174	struct vcpu_svm *svm = to_svm(vcpu);
2175	int ret;
2176
2177	if (is_guest_mode(vcpu)) {
2178		/* Returns '1' or -errno on failure, '0' on success. */
2179		ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]);
2180		if (ret)
2181			return ret;
2182		return 1;
2183	}
2184	return svm_instr_handlers[opcode](vcpu);
2185}
2186
2187/*
2188 * #GP handling code. Note that #GP can be triggered under the following two
2189 * cases:
2190 *   1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2191 *      some AMD CPUs when EAX of these instructions are in the reserved memory
2192 *      regions (e.g. SMM memory on host).
2193 *   2) VMware backdoor
2194 */
2195static int gp_interception(struct kvm_vcpu *vcpu)
2196{
2197	struct vcpu_svm *svm = to_svm(vcpu);
2198	u32 error_code = svm->vmcb->control.exit_info_1;
2199	int opcode;
2200
2201	/* Both #GP cases have zero error_code */
2202	if (error_code)
2203		goto reinject;
2204
2205	/* Decode the instruction for usage later */
2206	if (x86_decode_emulated_instruction(vcpu, 0, NULL, 0) != EMULATION_OK)
2207		goto reinject;
2208
2209	opcode = svm_instr_opcode(vcpu);
2210
2211	if (opcode == NONE_SVM_INSTR) {
2212		if (!enable_vmware_backdoor)
2213			goto reinject;
2214
2215		/*
2216		 * VMware backdoor emulation on #GP interception only handles
2217		 * IN{S}, OUT{S}, and RDPMC.
2218		 */
2219		if (!is_guest_mode(vcpu))
2220			return kvm_emulate_instruction(vcpu,
2221				EMULTYPE_VMWARE_GP | EMULTYPE_NO_DECODE);
2222	} else {
2223		/* All SVM instructions expect page aligned RAX */
2224		if (svm->vmcb->save.rax & ~PAGE_MASK)
2225			goto reinject;
2226
2227		return emulate_svm_instr(vcpu, opcode);
2228	}
2229
2230reinject:
2231	kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2232	return 1;
2233}
2234
2235void svm_set_gif(struct vcpu_svm *svm, bool value)
2236{
2237	if (value) {
2238		/*
2239		 * If VGIF is enabled, the STGI intercept is only added to
2240		 * detect the opening of the SMI/NMI window; remove it now.
2241		 * Likewise, clear the VINTR intercept, we will set it
2242		 * again while processing KVM_REQ_EVENT if needed.
2243		 */
2244		if (vgif)
2245			svm_clr_intercept(svm, INTERCEPT_STGI);
2246		if (svm_is_intercept(svm, INTERCEPT_VINTR))
2247			svm_clear_vintr(svm);
2248
2249		enable_gif(svm);
2250		if (svm->vcpu.arch.smi_pending ||
2251		    svm->vcpu.arch.nmi_pending ||
2252		    kvm_cpu_has_injectable_intr(&svm->vcpu) ||
2253		    kvm_apic_has_pending_init_or_sipi(&svm->vcpu))
2254			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2255	} else {
2256		disable_gif(svm);
2257
2258		/*
2259		 * After a CLGI no interrupts should come.  But if vGIF is
2260		 * in use, we still rely on the VINTR intercept (rather than
2261		 * STGI) to detect an open interrupt window.
2262		*/
2263		if (!vgif)
2264			svm_clear_vintr(svm);
2265	}
2266}
2267
2268static int stgi_interception(struct kvm_vcpu *vcpu)
2269{
2270	int ret;
2271
2272	if (nested_svm_check_permissions(vcpu))
2273		return 1;
2274
2275	ret = kvm_skip_emulated_instruction(vcpu);
2276	svm_set_gif(to_svm(vcpu), true);
2277	return ret;
2278}
2279
2280static int clgi_interception(struct kvm_vcpu *vcpu)
2281{
2282	int ret;
2283
2284	if (nested_svm_check_permissions(vcpu))
2285		return 1;
2286
2287	ret = kvm_skip_emulated_instruction(vcpu);
2288	svm_set_gif(to_svm(vcpu), false);
2289	return ret;
2290}
2291
2292static int invlpga_interception(struct kvm_vcpu *vcpu)
2293{
2294	gva_t gva = kvm_rax_read(vcpu);
2295	u32 asid = kvm_rcx_read(vcpu);
2296
2297	/* FIXME: Handle an address size prefix. */
2298	if (!is_long_mode(vcpu))
2299		gva = (u32)gva;
2300
2301	trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva);
2302
2303	/* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2304	kvm_mmu_invlpg(vcpu, gva);
2305
2306	return kvm_skip_emulated_instruction(vcpu);
2307}
2308
2309static int skinit_interception(struct kvm_vcpu *vcpu)
2310{
2311	trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu));
2312
2313	kvm_queue_exception(vcpu, UD_VECTOR);
2314	return 1;
2315}
2316
2317static int task_switch_interception(struct kvm_vcpu *vcpu)
2318{
2319	struct vcpu_svm *svm = to_svm(vcpu);
2320	u16 tss_selector;
2321	int reason;
2322	int int_type = svm->vmcb->control.exit_int_info &
2323		SVM_EXITINTINFO_TYPE_MASK;
2324	int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2325	uint32_t type =
2326		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2327	uint32_t idt_v =
2328		svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2329	bool has_error_code = false;
2330	u32 error_code = 0;
2331
2332	tss_selector = (u16)svm->vmcb->control.exit_info_1;
2333
2334	if (svm->vmcb->control.exit_info_2 &
2335	    (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2336		reason = TASK_SWITCH_IRET;
2337	else if (svm->vmcb->control.exit_info_2 &
2338		 (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2339		reason = TASK_SWITCH_JMP;
2340	else if (idt_v)
2341		reason = TASK_SWITCH_GATE;
2342	else
2343		reason = TASK_SWITCH_CALL;
2344
2345	if (reason == TASK_SWITCH_GATE) {
2346		switch (type) {
2347		case SVM_EXITINTINFO_TYPE_NMI:
2348			vcpu->arch.nmi_injected = false;
2349			break;
2350		case SVM_EXITINTINFO_TYPE_EXEPT:
2351			if (svm->vmcb->control.exit_info_2 &
2352			    (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2353				has_error_code = true;
2354				error_code =
2355					(u32)svm->vmcb->control.exit_info_2;
2356			}
2357			kvm_clear_exception_queue(vcpu);
2358			break;
2359		case SVM_EXITINTINFO_TYPE_INTR:
2360		case SVM_EXITINTINFO_TYPE_SOFT:
2361			kvm_clear_interrupt_queue(vcpu);
2362			break;
2363		default:
2364			break;
2365		}
2366	}
2367
2368	if (reason != TASK_SWITCH_GATE ||
2369	    int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2370	    (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2371	     (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
2372		if (!svm_skip_emulated_instruction(vcpu))
2373			return 0;
2374	}
2375
2376	if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2377		int_vec = -1;
2378
2379	return kvm_task_switch(vcpu, tss_selector, int_vec, reason,
2380			       has_error_code, error_code);
2381}
2382
2383static void svm_clr_iret_intercept(struct vcpu_svm *svm)
2384{
2385	if (!sev_es_guest(svm->vcpu.kvm))
2386		svm_clr_intercept(svm, INTERCEPT_IRET);
2387}
2388
2389static void svm_set_iret_intercept(struct vcpu_svm *svm)
2390{
2391	if (!sev_es_guest(svm->vcpu.kvm))
2392		svm_set_intercept(svm, INTERCEPT_IRET);
2393}
2394
2395static int iret_interception(struct kvm_vcpu *vcpu)
2396{
2397	struct vcpu_svm *svm = to_svm(vcpu);
2398
2399	WARN_ON_ONCE(sev_es_guest(vcpu->kvm));
2400
2401	++vcpu->stat.nmi_window_exits;
2402	svm->awaiting_iret_completion = true;
2403
2404	svm_clr_iret_intercept(svm);
2405	svm->nmi_iret_rip = kvm_rip_read(vcpu);
2406
2407	kvm_make_request(KVM_REQ_EVENT, vcpu);
2408	return 1;
2409}
2410
2411static int invlpg_interception(struct kvm_vcpu *vcpu)
2412{
2413	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2414		return kvm_emulate_instruction(vcpu, 0);
2415
2416	kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1);
2417	return kvm_skip_emulated_instruction(vcpu);
2418}
2419
2420static int emulate_on_interception(struct kvm_vcpu *vcpu)
2421{
2422	return kvm_emulate_instruction(vcpu, 0);
2423}
2424
2425static int rsm_interception(struct kvm_vcpu *vcpu)
2426{
2427	return kvm_emulate_instruction_from_buffer(vcpu, rsm_ins_bytes, 2);
2428}
2429
2430static bool check_selective_cr0_intercepted(struct kvm_vcpu *vcpu,
2431					    unsigned long val)
2432{
2433	struct vcpu_svm *svm = to_svm(vcpu);
2434	unsigned long cr0 = vcpu->arch.cr0;
2435	bool ret = false;
2436
2437	if (!is_guest_mode(vcpu) ||
2438	    (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0))))
2439		return false;
2440
2441	cr0 &= ~SVM_CR0_SELECTIVE_MASK;
2442	val &= ~SVM_CR0_SELECTIVE_MASK;
2443
2444	if (cr0 ^ val) {
2445		svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
2446		svm->vmcb->control.exit_code_hi = 0;
2447		ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
2448	}
2449
2450	return ret;
2451}
2452
2453#define CR_VALID (1ULL << 63)
2454
2455static int cr_interception(struct kvm_vcpu *vcpu)
2456{
2457	struct vcpu_svm *svm = to_svm(vcpu);
2458	int reg, cr;
2459	unsigned long val;
2460	int err;
2461
2462	if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
2463		return emulate_on_interception(vcpu);
2464
2465	if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
2466		return emulate_on_interception(vcpu);
2467
2468	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2469	if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
2470		cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
2471	else
2472		cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
2473
2474	err = 0;
2475	if (cr >= 16) { /* mov to cr */
2476		cr -= 16;
2477		val = kvm_register_read(vcpu, reg);
2478		trace_kvm_cr_write(cr, val);
2479		switch (cr) {
2480		case 0:
2481			if (!check_selective_cr0_intercepted(vcpu, val))
2482				err = kvm_set_cr0(vcpu, val);
2483			else
2484				return 1;
2485
2486			break;
2487		case 3:
2488			err = kvm_set_cr3(vcpu, val);
2489			break;
2490		case 4:
2491			err = kvm_set_cr4(vcpu, val);
2492			break;
2493		case 8:
2494			err = kvm_set_cr8(vcpu, val);
2495			break;
2496		default:
2497			WARN(1, "unhandled write to CR%d", cr);
2498			kvm_queue_exception(vcpu, UD_VECTOR);
2499			return 1;
2500		}
2501	} else { /* mov from cr */
2502		switch (cr) {
2503		case 0:
2504			val = kvm_read_cr0(vcpu);
2505			break;
2506		case 2:
2507			val = vcpu->arch.cr2;
2508			break;
2509		case 3:
2510			val = kvm_read_cr3(vcpu);
2511			break;
2512		case 4:
2513			val = kvm_read_cr4(vcpu);
2514			break;
2515		case 8:
2516			val = kvm_get_cr8(vcpu);
2517			break;
2518		default:
2519			WARN(1, "unhandled read from CR%d", cr);
2520			kvm_queue_exception(vcpu, UD_VECTOR);
2521			return 1;
2522		}
2523		kvm_register_write(vcpu, reg, val);
2524		trace_kvm_cr_read(cr, val);
2525	}
2526	return kvm_complete_insn_gp(vcpu, err);
2527}
2528
2529static int cr_trap(struct kvm_vcpu *vcpu)
2530{
2531	struct vcpu_svm *svm = to_svm(vcpu);
2532	unsigned long old_value, new_value;
2533	unsigned int cr;
2534	int ret = 0;
2535
2536	new_value = (unsigned long)svm->vmcb->control.exit_info_1;
2537
2538	cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP;
2539	switch (cr) {
2540	case 0:
2541		old_value = kvm_read_cr0(vcpu);
2542		svm_set_cr0(vcpu, new_value);
2543
2544		kvm_post_set_cr0(vcpu, old_value, new_value);
2545		break;
2546	case 4:
2547		old_value = kvm_read_cr4(vcpu);
2548		svm_set_cr4(vcpu, new_value);
2549
2550		kvm_post_set_cr4(vcpu, old_value, new_value);
2551		break;
2552	case 8:
2553		ret = kvm_set_cr8(vcpu, new_value);
2554		break;
2555	default:
2556		WARN(1, "unhandled CR%d write trap", cr);
2557		kvm_queue_exception(vcpu, UD_VECTOR);
2558		return 1;
2559	}
2560
2561	return kvm_complete_insn_gp(vcpu, ret);
2562}
2563
2564static int dr_interception(struct kvm_vcpu *vcpu)
2565{
2566	struct vcpu_svm *svm = to_svm(vcpu);
2567	int reg, dr;
2568	int err = 0;
2569
2570	/*
2571	 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT
2572	 * for DR7 write only. KVM cannot change DR7 (always swapped as type 'A') so return early.
2573	 */
2574	if (sev_es_guest(vcpu->kvm))
2575		return 1;
2576
2577	if (vcpu->guest_debug == 0) {
2578		/*
2579		 * No more DR vmexits; force a reload of the debug registers
2580		 * and reenter on this instruction.  The next vmexit will
2581		 * retrieve the full state of the debug registers.
2582		 */
2583		clr_dr_intercepts(svm);
2584		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
2585		return 1;
2586	}
2587
2588	if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
2589		return emulate_on_interception(vcpu);
2590
2591	reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
2592	dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
2593	if (dr >= 16) { /* mov to DRn  */
2594		dr -= 16;
2595		err = kvm_set_dr(vcpu, dr, kvm_register_read(vcpu, reg));
2596	} else {
2597		kvm_register_write(vcpu, reg, kvm_get_dr(vcpu, dr));
2598	}
2599
2600	return kvm_complete_insn_gp(vcpu, err);
2601}
2602
2603static int cr8_write_interception(struct kvm_vcpu *vcpu)
2604{
2605	u8 cr8_prev = kvm_get_cr8(vcpu);
2606	int r;
2607
2608	WARN_ON_ONCE(kvm_vcpu_apicv_active(vcpu));
2609
2610	/* instruction emulation calls kvm_set_cr8() */
2611	r = cr_interception(vcpu);
2612	if (lapic_in_kernel(vcpu))
2613		return r;
2614	if (cr8_prev <= kvm_get_cr8(vcpu))
2615		return r;
2616	vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
2617	return 0;
2618}
2619
2620static int efer_trap(struct kvm_vcpu *vcpu)
2621{
2622	struct msr_data msr_info;
2623	int ret;
2624
2625	/*
2626	 * Clear the EFER_SVME bit from EFER. The SVM code always sets this
2627	 * bit in svm_set_efer(), but __kvm_valid_efer() checks it against
2628	 * whether the guest has X86_FEATURE_SVM - this avoids a failure if
2629	 * the guest doesn't have X86_FEATURE_SVM.
2630	 */
2631	msr_info.host_initiated = false;
2632	msr_info.index = MSR_EFER;
2633	msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME;
2634	ret = kvm_set_msr_common(vcpu, &msr_info);
2635
2636	return kvm_complete_insn_gp(vcpu, ret);
2637}
2638
2639static int svm_get_feature_msr(u32 msr, u64 *data)
2640{
2641	*data = 0;
2642
2643	switch (msr) {
2644	case MSR_AMD64_DE_CFG:
2645		if (cpu_feature_enabled(X86_FEATURE_LFENCE_RDTSC))
2646			*data |= MSR_AMD64_DE_CFG_LFENCE_SERIALIZE;
2647		break;
2648	default:
2649		return KVM_MSR_RET_UNSUPPORTED;
2650	}
2651
2652	return 0;
2653}
2654
2655static bool sev_es_prevent_msr_access(struct kvm_vcpu *vcpu,
2656				      struct msr_data *msr_info)
2657{
2658	return sev_es_guest(vcpu->kvm) && vcpu->arch.guest_state_protected &&
2659	       msr_info->index != MSR_IA32_XSS &&
2660	       !msr_write_intercepted(vcpu, msr_info->index);
2661}
2662
2663static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
2664{
2665	struct vcpu_svm *svm = to_svm(vcpu);
2666
2667	if (sev_es_prevent_msr_access(vcpu, msr_info)) {
2668		msr_info->data = 0;
2669		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
2670	}
2671
2672	switch (msr_info->index) {
2673	case MSR_AMD64_TSC_RATIO:
2674		if (!msr_info->host_initiated &&
2675		    !guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR))
2676			return 1;
2677		msr_info->data = svm->tsc_ratio_msr;
2678		break;
2679	case MSR_STAR:
2680		msr_info->data = svm->vmcb01.ptr->save.star;
2681		break;
2682#ifdef CONFIG_X86_64
2683	case MSR_LSTAR:
2684		msr_info->data = svm->vmcb01.ptr->save.lstar;
2685		break;
2686	case MSR_CSTAR:
2687		msr_info->data = svm->vmcb01.ptr->save.cstar;
2688		break;
2689	case MSR_GS_BASE:
2690		msr_info->data = svm->vmcb01.ptr->save.gs.base;
2691		break;
2692	case MSR_FS_BASE:
2693		msr_info->data = svm->vmcb01.ptr->save.fs.base;
2694		break;
2695	case MSR_KERNEL_GS_BASE:
2696		msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base;
2697		break;
2698	case MSR_SYSCALL_MASK:
2699		msr_info->data = svm->vmcb01.ptr->save.sfmask;
2700		break;
2701#endif
2702	case MSR_IA32_SYSENTER_CS:
2703		msr_info->data = svm->vmcb01.ptr->save.sysenter_cs;
2704		break;
2705	case MSR_IA32_SYSENTER_EIP:
2706		msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip;
2707		if (guest_cpuid_is_intel_compatible(vcpu))
2708			msr_info->data |= (u64)svm->sysenter_eip_hi << 32;
2709		break;
2710	case MSR_IA32_SYSENTER_ESP:
2711		msr_info->data = svm->vmcb01.ptr->save.sysenter_esp;
2712		if (guest_cpuid_is_intel_compatible(vcpu))
2713			msr_info->data |= (u64)svm->sysenter_esp_hi << 32;
2714		break;
2715	case MSR_IA32_S_CET:
2716		msr_info->data = svm->vmcb->save.s_cet;
2717		break;
2718	case MSR_IA32_INT_SSP_TAB:
2719		msr_info->data = svm->vmcb->save.isst_addr;
2720		break;
2721	case MSR_KVM_INTERNAL_GUEST_SSP:
2722		msr_info->data = svm->vmcb->save.ssp;
2723		break;
2724	case MSR_TSC_AUX:
2725		msr_info->data = svm->tsc_aux;
2726		break;
2727	case MSR_IA32_DEBUGCTLMSR:
2728		msr_info->data = svm->vmcb->save.dbgctl;
2729		break;
2730	case MSR_IA32_LASTBRANCHFROMIP:
2731		msr_info->data = svm->vmcb->save.br_from;
2732		break;
2733	case MSR_IA32_LASTBRANCHTOIP:
2734		msr_info->data = svm->vmcb->save.br_to;
2735		break;
2736	case MSR_IA32_LASTINTFROMIP:
2737		msr_info->data = svm->vmcb->save.last_excp_from;
2738		break;
2739	case MSR_IA32_LASTINTTOIP:
2740		msr_info->data = svm->vmcb->save.last_excp_to;
2741		break;
2742	case MSR_VM_HSAVE_PA:
2743		msr_info->data = svm->nested.hsave_msr;
2744		break;
2745	case MSR_VM_CR:
2746		msr_info->data = svm->nested.vm_cr_msr;
2747		break;
2748	case MSR_IA32_SPEC_CTRL:
2749		if (!msr_info->host_initiated &&
2750		    !guest_has_spec_ctrl_msr(vcpu))
2751			return 1;
2752
2753		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2754			msr_info->data = svm->vmcb->save.spec_ctrl;
2755		else
2756			msr_info->data = svm->spec_ctrl;
2757		break;
2758	case MSR_AMD64_VIRT_SPEC_CTRL:
2759		if (!msr_info->host_initiated &&
2760		    !guest_cpu_cap_has(vcpu, X86_FEATURE_VIRT_SSBD))
2761			return 1;
2762
2763		msr_info->data = svm->virt_spec_ctrl;
2764		break;
2765	case MSR_F15H_IC_CFG: {
2766
2767		int family, model;
2768
2769		family = guest_cpuid_family(vcpu);
2770		model  = guest_cpuid_model(vcpu);
2771
2772		if (family < 0 || model < 0)
2773			return kvm_get_msr_common(vcpu, msr_info);
2774
2775		msr_info->data = 0;
2776
2777		if (family == 0x15 &&
2778		    (model >= 0x2 && model < 0x20))
2779			msr_info->data = 0x1E;
2780		}
2781		break;
2782	case MSR_AMD64_DE_CFG:
2783		msr_info->data = svm->msr_decfg;
2784		break;
2785	default:
2786		return kvm_get_msr_common(vcpu, msr_info);
2787	}
2788	return 0;
2789}
2790
2791static int svm_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
2792{
2793	struct vcpu_svm *svm = to_svm(vcpu);
2794	if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb))
2795		return kvm_complete_insn_gp(vcpu, err);
2796
2797	svm_vmgexit_inject_exception(svm, X86_TRAP_GP);
2798	return 1;
2799}
2800
2801static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
2802{
2803	struct vcpu_svm *svm = to_svm(vcpu);
2804	int svm_dis, chg_mask;
2805
2806	if (data & ~SVM_VM_CR_VALID_MASK)
2807		return 1;
2808
2809	chg_mask = SVM_VM_CR_VALID_MASK;
2810
2811	if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
2812		chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
2813
2814	svm->nested.vm_cr_msr &= ~chg_mask;
2815	svm->nested.vm_cr_msr |= (data & chg_mask);
2816
2817	svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
2818
2819	/* check for svm_disable while efer.svme is set */
2820	if (svm_dis && (vcpu->arch.efer & EFER_SVME))
2821		return 1;
2822
2823	return 0;
2824}
2825
2826static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
2827{
2828	struct vcpu_svm *svm = to_svm(vcpu);
2829	int ret = 0;
2830
2831	u32 ecx = msr->index;
2832	u64 data = msr->data;
2833
2834	if (sev_es_prevent_msr_access(vcpu, msr))
2835		return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0;
2836
2837	switch (ecx) {
2838	case MSR_AMD64_TSC_RATIO:
2839
2840		if (!guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR)) {
2841
2842			if (!msr->host_initiated)
2843				return 1;
2844			/*
2845			 * In case TSC scaling is not enabled, always
2846			 * leave this MSR at the default value.
2847			 *
2848			 * Due to bug in qemu 6.2.0, it would try to set
2849			 * this msr to 0 if tsc scaling is not enabled.
2850			 * Ignore this value as well.
2851			 */
2852			if (data != 0 && data != svm->tsc_ratio_msr)
2853				return 1;
2854			break;
2855		}
2856
2857		if (data & SVM_TSC_RATIO_RSVD)
2858			return 1;
2859
2860		svm->tsc_ratio_msr = data;
2861
2862		if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR) &&
2863		    is_guest_mode(vcpu))
2864			nested_svm_update_tsc_ratio_msr(vcpu);
2865
2866		break;
2867	case MSR_IA32_CR_PAT:
2868		ret = kvm_set_msr_common(vcpu, msr);
2869		if (ret)
2870			break;
2871
2872		svm->vmcb01.ptr->save.g_pat = data;
2873		if (is_guest_mode(vcpu))
2874			nested_vmcb02_compute_g_pat(svm);
2875		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
2876		break;
2877	case MSR_IA32_SPEC_CTRL:
2878		if (!msr->host_initiated &&
2879		    !guest_has_spec_ctrl_msr(vcpu))
2880			return 1;
2881
2882		if (kvm_spec_ctrl_test_value(data))
2883			return 1;
2884
2885		if (boot_cpu_has(X86_FEATURE_V_SPEC_CTRL))
2886			svm->vmcb->save.spec_ctrl = data;
2887		else
2888			svm->spec_ctrl = data;
2889		if (!data)
2890			break;
2891
2892		/*
2893		 * For non-nested:
2894		 * When it's written (to non-zero) for the first time, pass
2895		 * it through.
2896		 *
2897		 * For nested:
2898		 * The handling of the MSR bitmap for L2 guests is done in
2899		 * nested_svm_merge_msrpm().
2900		 * We update the L1 MSR bit as well since it will end up
2901		 * touching the MSR anyway now.
2902		 */
2903		svm_disable_intercept_for_msr(vcpu, MSR_IA32_SPEC_CTRL, MSR_TYPE_RW);
2904		break;
2905	case MSR_AMD64_VIRT_SPEC_CTRL:
2906		if (!msr->host_initiated &&
2907		    !guest_cpu_cap_has(vcpu, X86_FEATURE_VIRT_SSBD))
2908			return 1;
2909
2910		if (data & ~SPEC_CTRL_SSBD)
2911			return 1;
2912
2913		svm->virt_spec_ctrl = data;
2914		break;
2915	case MSR_STAR:
2916		svm->vmcb01.ptr->save.star = data;
2917		break;
2918#ifdef CONFIG_X86_64
2919	case MSR_LSTAR:
2920		svm->vmcb01.ptr->save.lstar = data;
2921		break;
2922	case MSR_CSTAR:
2923		svm->vmcb01.ptr->save.cstar = data;
2924		break;
2925	case MSR_GS_BASE:
2926		svm->vmcb01.ptr->save.gs.base = data;
2927		break;
2928	case MSR_FS_BASE:
2929		svm->vmcb01.ptr->save.fs.base = data;
2930		break;
2931	case MSR_KERNEL_GS_BASE:
2932		svm->vmcb01.ptr->save.kernel_gs_base = data;
2933		break;
2934	case MSR_SYSCALL_MASK:
2935		svm->vmcb01.ptr->save.sfmask = data;
2936		break;
2937#endif
2938	case MSR_IA32_SYSENTER_CS:
2939		svm->vmcb01.ptr->save.sysenter_cs = data;
2940		break;
2941	case MSR_IA32_SYSENTER_EIP:
2942		svm->vmcb01.ptr->save.sysenter_eip = (u32)data;
2943		/*
2944		 * We only intercept the MSR_IA32_SYSENTER_{EIP|ESP} msrs
2945		 * when we spoof an Intel vendor ID (for cross vendor migration).
2946		 * In this case we use this intercept to track the high
2947		 * 32 bit part of these msrs to support Intel's
2948		 * implementation of SYSENTER/SYSEXIT.
2949		 */
2950		svm->sysenter_eip_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0;
2951		break;
2952	case MSR_IA32_SYSENTER_ESP:
2953		svm->vmcb01.ptr->save.sysenter_esp = (u32)data;
2954		svm->sysenter_esp_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0;
2955		break;
2956	case MSR_IA32_S_CET:
2957		svm->vmcb->save.s_cet = data;
2958		vmcb_mark_dirty(svm->vmcb01.ptr, VMCB_CET);
2959		break;
2960	case MSR_IA32_INT_SSP_TAB:
2961		svm->vmcb->save.isst_addr = data;
2962		vmcb_mark_dirty(svm->vmcb01.ptr, VMCB_CET);
2963		break;
2964	case MSR_KVM_INTERNAL_GUEST_SSP:
2965		svm->vmcb->save.ssp = data;
2966		vmcb_mark_dirty(svm->vmcb01.ptr, VMCB_CET);
2967		break;
2968	case MSR_TSC_AUX:
2969		/*
2970		 * TSC_AUX is always virtualized for SEV-ES guests when the
2971		 * feature is available. The user return MSR support is not
2972		 * required in this case because TSC_AUX is restored on #VMEXIT
2973		 * from the host save area.
2974		 */
2975		if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm))
2976			break;
2977
2978		/*
2979		 * TSC_AUX is usually changed only during boot and never read
2980		 * directly.  Intercept TSC_AUX and switch it via user return.
2981		 */
2982		preempt_disable();
2983		ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull);
2984		preempt_enable();
2985		if (ret)
2986			break;
2987
2988		svm->tsc_aux = data;
2989		break;
2990	case MSR_IA32_DEBUGCTLMSR:
2991		if (!lbrv) {
2992			kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
2993			break;
2994		}
2995
2996		/*
2997		 * Suppress BTF as KVM doesn't virtualize BTF, but there's no
2998		 * way to communicate lack of support to the guest.
2999		 */
3000		if (data & DEBUGCTLMSR_BTF) {
3001			kvm_pr_unimpl_wrmsr(vcpu, MSR_IA32_DEBUGCTLMSR, data);
3002			data &= ~DEBUGCTLMSR_BTF;
3003		}
3004
3005		if (data & DEBUGCTL_RESERVED_BITS)
3006			return 1;
3007
3008		if (svm->vmcb->save.dbgctl == data)
3009			break;
3010
3011		svm->vmcb->save.dbgctl = data;
3012		vmcb_mark_dirty(svm->vmcb, VMCB_LBR);
3013		svm_update_lbrv(vcpu);
3014		break;
3015	case MSR_VM_HSAVE_PA:
3016		/*
3017		 * Old kernels did not validate the value written to
3018		 * MSR_VM_HSAVE_PA.  Allow KVM_SET_MSR to set an invalid
3019		 * value to allow live migrating buggy or malicious guests
3020		 * originating from those kernels.
3021		 */
3022		if (!msr->host_initiated && !page_address_valid(vcpu, data))
3023			return 1;
3024
3025		svm->nested.hsave_msr = data & PAGE_MASK;
3026		break;
3027	case MSR_VM_CR:
3028		return svm_set_vm_cr(vcpu, data);
3029	case MSR_VM_IGNNE:
3030		kvm_pr_unimpl_wrmsr(vcpu, ecx, data);
3031		break;
3032	case MSR_AMD64_DE_CFG: {
3033		u64 supported_de_cfg;
3034
3035		if (svm_get_feature_msr(ecx, &supported_de_cfg))
3036			return 1;
3037
3038		if (data & ~supported_de_cfg)
3039			return 1;
3040
3041		svm->msr_decfg = data;
3042		break;
3043	}
3044	default:
3045		return kvm_set_msr_common(vcpu, msr);
3046	}
3047	return ret;
3048}
3049
3050static int msr_interception(struct kvm_vcpu *vcpu)
3051{
3052	if (to_svm(vcpu)->vmcb->control.exit_info_1)
3053		return kvm_emulate_wrmsr(vcpu);
3054	else
3055		return kvm_emulate_rdmsr(vcpu);
3056}
3057
3058static int interrupt_window_interception(struct kvm_vcpu *vcpu)
3059{
3060	kvm_make_request(KVM_REQ_EVENT, vcpu);
3061	svm_clear_vintr(to_svm(vcpu));
3062
3063	/*
3064	 * If not running nested, for AVIC, the only reason to end up here is ExtINTs.
3065	 * In this case AVIC was temporarily disabled for
3066	 * requesting the IRQ window and we have to re-enable it.
3067	 *
3068	 * If running nested, still remove the VM wide AVIC inhibit to
3069	 * support case in which the interrupt window was requested when the
3070	 * vCPU was not running nested.
3071
3072	 * All vCPUs which run still run nested, will remain to have their
3073	 * AVIC still inhibited due to per-cpu AVIC inhibition.
3074	 */
3075	kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3076
3077	++vcpu->stat.irq_window_exits;
3078	return 1;
3079}
3080
3081static int pause_interception(struct kvm_vcpu *vcpu)
3082{
3083	bool in_kernel;
3084	/*
3085	 * CPL is not made available for an SEV-ES guest, therefore
3086	 * vcpu->arch.preempted_in_kernel can never be true.  Just
3087	 * set in_kernel to false as well.
3088	 */
3089	in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0;
3090
3091	grow_ple_window(vcpu);
3092
3093	kvm_vcpu_on_spin(vcpu, in_kernel);
3094	return kvm_skip_emulated_instruction(vcpu);
3095}
3096
3097static int invpcid_interception(struct kvm_vcpu *vcpu)
3098{
3099	struct vcpu_svm *svm = to_svm(vcpu);
3100	unsigned long type;
3101	gva_t gva;
3102
3103	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_INVPCID)) {
3104		kvm_queue_exception(vcpu, UD_VECTOR);
3105		return 1;
3106	}
3107
3108	/*
3109	 * For an INVPCID intercept:
3110	 * EXITINFO1 provides the linear address of the memory operand.
3111	 * EXITINFO2 provides the contents of the register operand.
3112	 */
3113	type = svm->vmcb->control.exit_info_2;
3114	gva = svm->vmcb->control.exit_info_1;
3115
3116	/*
3117	 * FIXME: Perform segment checks for 32-bit mode, and inject #SS if the
3118	 *        stack segment is used.  The intercept takes priority over all
3119	 *        #GP checks except CPL>0, but somehow still generates a linear
3120	 *        address?  The APM is sorely lacking.
3121	 */
3122	if (is_noncanonical_address(gva, vcpu, 0)) {
3123		kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
3124		return 1;
3125	}
3126
3127	return kvm_handle_invpcid(vcpu, type, gva);
3128}
3129
3130static inline int complete_userspace_buslock(struct kvm_vcpu *vcpu)
3131{
3132	struct vcpu_svm *svm = to_svm(vcpu);
3133
3134	/*
3135	 * If userspace has NOT changed RIP, then KVM's ABI is to let the guest
3136	 * execute the bus-locking instruction.  Set the bus lock counter to '1'
3137	 * to effectively step past the bus lock.
3138	 */
3139	if (kvm_is_linear_rip(vcpu, vcpu->arch.cui_linear_rip))
3140		svm->vmcb->control.bus_lock_counter = 1;
3141
3142	return 1;
3143}
3144
3145static int bus_lock_exit(struct kvm_vcpu *vcpu)
3146{
3147	struct vcpu_svm *svm = to_svm(vcpu);
3148
3149	vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK;
3150	vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK;
3151
3152	vcpu->arch.cui_linear_rip = kvm_get_linear_rip(vcpu);
3153	vcpu->arch.complete_userspace_io = complete_userspace_buslock;
3154
3155	if (is_guest_mode(vcpu))
3156		svm->nested.ctl.bus_lock_rip = vcpu->arch.cui_linear_rip;
3157
3158	return 0;
3159}
3160
3161static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = {
3162	[SVM_EXIT_READ_CR0]			= cr_interception,
3163	[SVM_EXIT_READ_CR3]			= cr_interception,
3164	[SVM_EXIT_READ_CR4]			= cr_interception,
3165	[SVM_EXIT_READ_CR8]			= cr_interception,
3166	[SVM_EXIT_CR0_SEL_WRITE]		= cr_interception,
3167	[SVM_EXIT_WRITE_CR0]			= cr_interception,
3168	[SVM_EXIT_WRITE_CR3]			= cr_interception,
3169	[SVM_EXIT_WRITE_CR4]			= cr_interception,
3170	[SVM_EXIT_WRITE_CR8]			= cr8_write_interception,
3171	[SVM_EXIT_READ_DR0]			= dr_interception,
3172	[SVM_EXIT_READ_DR1]			= dr_interception,
3173	[SVM_EXIT_READ_DR2]			= dr_interception,
3174	[SVM_EXIT_READ_DR3]			= dr_interception,
3175	[SVM_EXIT_READ_DR4]			= dr_interception,
3176	[SVM_EXIT_READ_DR5]			= dr_interception,
3177	[SVM_EXIT_READ_DR6]			= dr_interception,
3178	[SVM_EXIT_READ_DR7]			= dr_interception,
3179	[SVM_EXIT_WRITE_DR0]			= dr_interception,
3180	[SVM_EXIT_WRITE_DR1]			= dr_interception,
3181	[SVM_EXIT_WRITE_DR2]			= dr_interception,
3182	[SVM_EXIT_WRITE_DR3]			= dr_interception,
3183	[SVM_EXIT_WRITE_DR4]			= dr_interception,
3184	[SVM_EXIT_WRITE_DR5]			= dr_interception,
3185	[SVM_EXIT_WRITE_DR6]			= dr_interception,
3186	[SVM_EXIT_WRITE_DR7]			= dr_interception,
3187	[SVM_EXIT_EXCP_BASE + DB_VECTOR]	= db_interception,
3188	[SVM_EXIT_EXCP_BASE + BP_VECTOR]	= bp_interception,
3189	[SVM_EXIT_EXCP_BASE + UD_VECTOR]	= ud_interception,
3190	[SVM_EXIT_EXCP_BASE + PF_VECTOR]	= pf_interception,
3191	[SVM_EXIT_EXCP_BASE + MC_VECTOR]	= mc_interception,
3192	[SVM_EXIT_EXCP_BASE + AC_VECTOR]	= ac_interception,
3193	[SVM_EXIT_EXCP_BASE + GP_VECTOR]	= gp_interception,
3194	[SVM_EXIT_INTR]				= intr_interception,
3195	[SVM_EXIT_NMI]				= nmi_interception,
3196	[SVM_EXIT_SMI]				= smi_interception,
3197	[SVM_EXIT_VINTR]			= interrupt_window_interception,
3198	[SVM_EXIT_RDPMC]			= kvm_emulate_rdpmc,
3199	[SVM_EXIT_CPUID]			= kvm_emulate_cpuid,
3200	[SVM_EXIT_IRET]                         = iret_interception,
3201	[SVM_EXIT_INVD]                         = kvm_emulate_invd,
3202	[SVM_EXIT_PAUSE]			= pause_interception,
3203	[SVM_EXIT_HLT]				= kvm_emulate_halt,
3204	[SVM_EXIT_INVLPG]			= invlpg_interception,
3205	[SVM_EXIT_INVLPGA]			= invlpga_interception,
3206	[SVM_EXIT_IOIO]				= io_interception,
3207	[SVM_EXIT_MSR]				= msr_interception,
3208	[SVM_EXIT_TASK_SWITCH]			= task_switch_interception,
3209	[SVM_EXIT_SHUTDOWN]			= shutdown_interception,
3210	[SVM_EXIT_VMRUN]			= vmrun_interception,
3211	[SVM_EXIT_VMMCALL]			= kvm_emulate_hypercall,
3212	[SVM_EXIT_VMLOAD]			= vmload_interception,
3213	[SVM_EXIT_VMSAVE]			= vmsave_interception,
3214	[SVM_EXIT_STGI]				= stgi_interception,
3215	[SVM_EXIT_CLGI]				= clgi_interception,
3216	[SVM_EXIT_SKINIT]			= skinit_interception,
3217	[SVM_EXIT_RDTSCP]			= kvm_handle_invalid_op,
3218	[SVM_EXIT_WBINVD]                       = kvm_emulate_wbinvd,
3219	[SVM_EXIT_MONITOR]			= kvm_emulate_monitor,
3220	[SVM_EXIT_MWAIT]			= kvm_emulate_mwait,
3221	[SVM_EXIT_XSETBV]			= kvm_emulate_xsetbv,
3222	[SVM_EXIT_RDPRU]			= kvm_handle_invalid_op,
3223	[SVM_EXIT_EFER_WRITE_TRAP]		= efer_trap,
3224	[SVM_EXIT_CR0_WRITE_TRAP]		= cr_trap,
3225	[SVM_EXIT_CR4_WRITE_TRAP]		= cr_trap,
3226	[SVM_EXIT_CR8_WRITE_TRAP]		= cr_trap,
3227	[SVM_EXIT_INVPCID]                      = invpcid_interception,
3228	[SVM_EXIT_IDLE_HLT]			= kvm_emulate_halt,
3229	[SVM_EXIT_NPF]				= npf_interception,
3230	[SVM_EXIT_BUS_LOCK]			= bus_lock_exit,
3231	[SVM_EXIT_RSM]                          = rsm_interception,
3232	[SVM_EXIT_AVIC_INCOMPLETE_IPI]		= avic_incomplete_ipi_interception,
3233	[SVM_EXIT_AVIC_UNACCELERATED_ACCESS]	= avic_unaccelerated_access_interception,
3234#ifdef CONFIG_KVM_AMD_SEV
3235	[SVM_EXIT_VMGEXIT]			= sev_handle_vmgexit,
3236#endif
3237};
3238
3239static void dump_vmcb(struct kvm_vcpu *vcpu)
3240{
3241	struct vcpu_svm *svm = to_svm(vcpu);
3242	struct vmcb_control_area *control = &svm->vmcb->control;
3243	struct vmcb_save_area *save = &svm->vmcb->save;
3244	struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save;
3245	char *vm_type;
3246
3247	if (!dump_invalid_vmcb) {
3248		pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
3249		return;
3250	}
3251
3252	guard(mutex)(&vmcb_dump_mutex);
3253
3254	vm_type = sev_snp_guest(vcpu->kvm) ? "SEV-SNP" :
3255		  sev_es_guest(vcpu->kvm) ? "SEV-ES" :
3256		  sev_guest(vcpu->kvm) ? "SEV" : "SVM";
3257
3258	pr_err("%s vCPU%u VMCB %p, last attempted VMRUN on CPU %d\n",
3259	       vm_type, vcpu->vcpu_id, svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu);
3260	pr_err("VMCB Control Area:\n");
3261	pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff);
3262	pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16);
3263	pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff);
3264	pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16);
3265	pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]);
3266	pr_err("%-20s%08x %08x\n", "intercepts:",
3267              control->intercepts[INTERCEPT_WORD3],
3268	       control->intercepts[INTERCEPT_WORD4]);
3269	pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3270	pr_err("%-20s%d\n", "pause filter threshold:",
3271	       control->pause_filter_thresh);
3272	pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3273	pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3274	pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3275	pr_err("%-20s%d\n", "asid:", control->asid);
3276	pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3277	pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3278	pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3279	pr_err("%-20s%08x\n", "int_state:", control->int_state);
3280	pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3281	pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3282	pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3283	pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3284	pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3285	pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3286	pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3287	pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3288	pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa);
3289	pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3290	pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3291	pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
3292	pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3293	pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3294	pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3295	pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3296	pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa);
3297	pr_err("%-20s%016llx\n", "allowed_sev_features:", control->allowed_sev_features);
3298	pr_err("%-20s%016llx\n", "guest_sev_features:", control->guest_sev_features);
3299
3300	if (sev_es_guest(vcpu->kvm)) {
3301		save = sev_decrypt_vmsa(vcpu);
3302		if (!save)
3303			goto no_vmsa;
3304
3305		save01 = save;
3306	}
3307
3308	pr_err("VMCB State Save Area:\n");
3309	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3310	       "es:",
3311	       save->es.selector, save->es.attrib,
3312	       save->es.limit, save->es.base);
3313	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3314	       "cs:",
3315	       save->cs.selector, save->cs.attrib,
3316	       save->cs.limit, save->cs.base);
3317	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3318	       "ss:",
3319	       save->ss.selector, save->ss.attrib,
3320	       save->ss.limit, save->ss.base);
3321	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3322	       "ds:",
3323	       save->ds.selector, save->ds.attrib,
3324	       save->ds.limit, save->ds.base);
3325	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3326	       "fs:",
3327	       save01->fs.selector, save01->fs.attrib,
3328	       save01->fs.limit, save01->fs.base);
3329	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3330	       "gs:",
3331	       save01->gs.selector, save01->gs.attrib,
3332	       save01->gs.limit, save01->gs.base);
3333	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3334	       "gdtr:",
3335	       save->gdtr.selector, save->gdtr.attrib,
3336	       save->gdtr.limit, save->gdtr.base);
3337	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3338	       "ldtr:",
3339	       save01->ldtr.selector, save01->ldtr.attrib,
3340	       save01->ldtr.limit, save01->ldtr.base);
3341	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3342	       "idtr:",
3343	       save->idtr.selector, save->idtr.attrib,
3344	       save->idtr.limit, save->idtr.base);
3345	pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3346	       "tr:",
3347	       save01->tr.selector, save01->tr.attrib,
3348	       save01->tr.limit, save01->tr.base);
3349	pr_err("vmpl: %d   cpl:  %d               efer:          %016llx\n",
3350	       save->vmpl, save->cpl, save->efer);
3351	pr_err("%-15s %016llx %-13s %016llx\n",
3352	       "cr0:", save->cr0, "cr2:", save->cr2);
3353	pr_err("%-15s %016llx %-13s %016llx\n",
3354	       "cr3:", save->cr3, "cr4:", save->cr4);
3355	pr_err("%-15s %016llx %-13s %016llx\n",
3356	       "dr6:", save->dr6, "dr7:", save->dr7);
3357	pr_err("%-15s %016llx %-13s %016llx\n",
3358	       "rip:", save->rip, "rflags:", save->rflags);
3359	pr_err("%-15s %016llx %-13s %016llx\n",
3360	       "rsp:", save->rsp, "rax:", save->rax);
3361	pr_err("%-15s %016llx %-13s %016llx\n",
3362	       "s_cet:", save->s_cet, "ssp:", save->ssp);
3363	pr_err("%-15s %016llx\n",
3364	       "isst_addr:", save->isst_addr);
3365	pr_err("%-15s %016llx %-13s %016llx\n",
3366	       "star:", save01->star, "lstar:", save01->lstar);
3367	pr_err("%-15s %016llx %-13s %016llx\n",
3368	       "cstar:", save01->cstar, "sfmask:", save01->sfmask);
3369	pr_err("%-15s %016llx %-13s %016llx\n",
3370	       "kernel_gs_base:", save01->kernel_gs_base,
3371	       "sysenter_cs:", save01->sysenter_cs);
3372	pr_err("%-15s %016llx %-13s %016llx\n",
3373	       "sysenter_esp:", save01->sysenter_esp,
3374	       "sysenter_eip:", save01->sysenter_eip);
3375	pr_err("%-15s %016llx %-13s %016llx\n",
3376	       "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3377	pr_err("%-15s %016llx %-13s %016llx\n",
3378	       "br_from:", save->br_from, "br_to:", save->br_to);
3379	pr_err("%-15s %016llx %-13s %016llx\n",
3380	       "excp_from:", save->last_excp_from,
3381	       "excp_to:", save->last_excp_to);
3382
3383	if (sev_es_guest(vcpu->kvm)) {
3384		struct sev_es_save_area *vmsa = (struct sev_es_save_area *)save;
3385
3386		pr_err("%-15s %016llx\n",
3387		       "sev_features", vmsa->sev_features);
3388
3389		pr_err("%-15s %016llx %-13s %016llx\n",
3390		       "pl0_ssp:", vmsa->pl0_ssp, "pl1_ssp:", vmsa->pl1_ssp);
3391		pr_err("%-15s %016llx %-13s %016llx\n",
3392		       "pl2_ssp:", vmsa->pl2_ssp, "pl3_ssp:", vmsa->pl3_ssp);
3393		pr_err("%-15s %016llx\n",
3394		       "u_cet:", vmsa->u_cet);
3395
3396		pr_err("%-15s %016llx %-13s %016llx\n",
3397		       "rax:", vmsa->rax, "rbx:", vmsa->rbx);
3398		pr_err("%-15s %016llx %-13s %016llx\n",
3399		       "rcx:", vmsa->rcx, "rdx:", vmsa->rdx);
3400		pr_err("%-15s %016llx %-13s %016llx\n",
3401		       "rsi:", vmsa->rsi, "rdi:", vmsa->rdi);
3402		pr_err("%-15s %016llx %-13s %016llx\n",
3403		       "rbp:", vmsa->rbp, "rsp:", vmsa->rsp);
3404		pr_err("%-15s %016llx %-13s %016llx\n",
3405		       "r8:", vmsa->r8, "r9:", vmsa->r9);
3406		pr_err("%-15s %016llx %-13s %016llx\n",
3407		       "r10:", vmsa->r10, "r11:", vmsa->r11);
3408		pr_err("%-15s %016llx %-13s %016llx\n",
3409		       "r12:", vmsa->r12, "r13:", vmsa->r13);
3410		pr_err("%-15s %016llx %-13s %016llx\n",
3411		       "r14:", vmsa->r14, "r15:", vmsa->r15);
3412		pr_err("%-15s %016llx %-13s %016llx\n",
3413		       "xcr0:", vmsa->xcr0, "xss:", vmsa->xss);
3414	} else {
3415		pr_err("%-15s %016llx %-13s %016lx\n",
3416		       "rax:", save->rax, "rbx:",
3417		       vcpu->arch.regs[VCPU_REGS_RBX]);
3418		pr_err("%-15s %016lx %-13s %016lx\n",
3419		       "rcx:", vcpu->arch.regs[VCPU_REGS_RCX],
3420		       "rdx:", vcpu->arch.regs[VCPU_REGS_RDX]);
3421		pr_err("%-15s %016lx %-13s %016lx\n",
3422		       "rsi:", vcpu->arch.regs[VCPU_REGS_RSI],
3423		       "rdi:", vcpu->arch.regs[VCPU_REGS_RDI]);
3424		pr_err("%-15s %016lx %-13s %016llx\n",
3425		       "rbp:", vcpu->arch.regs[VCPU_REGS_RBP],
3426		       "rsp:", save->rsp);
3427#ifdef CONFIG_X86_64
3428		pr_err("%-15s %016lx %-13s %016lx\n",
3429		       "r8:", vcpu->arch.regs[VCPU_REGS_R8],
3430		       "r9:", vcpu->arch.regs[VCPU_REGS_R9]);
3431		pr_err("%-15s %016lx %-13s %016lx\n",
3432		       "r10:", vcpu->arch.regs[VCPU_REGS_R10],
3433		       "r11:", vcpu->arch.regs[VCPU_REGS_R11]);
3434		pr_err("%-15s %016lx %-13s %016lx\n",
3435		       "r12:", vcpu->arch.regs[VCPU_REGS_R12],
3436		       "r13:", vcpu->arch.regs[VCPU_REGS_R13]);
3437		pr_err("%-15s %016lx %-13s %016lx\n",
3438		       "r14:", vcpu->arch.regs[VCPU_REGS_R14],
3439		       "r15:", vcpu->arch.regs[VCPU_REGS_R15]);
3440#endif
3441	}
3442
3443no_vmsa:
3444	if (sev_es_guest(vcpu->kvm))
3445		sev_free_decrypted_vmsa(vcpu, save);
3446}
3447
3448static bool svm_check_exit_valid(u64 exit_code)
3449{
3450	return (exit_code < ARRAY_SIZE(svm_exit_handlers) &&
3451		svm_exit_handlers[exit_code]);
3452}
3453
3454static int svm_handle_invalid_exit(struct kvm_vcpu *vcpu, u64 exit_code)
3455{
3456	dump_vmcb(vcpu);
3457	kvm_prepare_unexpected_reason_exit(vcpu, exit_code);
3458	return 0;
3459}
3460
3461int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 exit_code)
3462{
3463	if (!svm_check_exit_valid(exit_code))
3464		return svm_handle_invalid_exit(vcpu, exit_code);
3465
3466#ifdef CONFIG_MITIGATION_RETPOLINE
3467	if (exit_code == SVM_EXIT_MSR)
3468		return msr_interception(vcpu);
3469	else if (exit_code == SVM_EXIT_VINTR)
3470		return interrupt_window_interception(vcpu);
3471	else if (exit_code == SVM_EXIT_INTR)
3472		return intr_interception(vcpu);
3473	else if (exit_code == SVM_EXIT_HLT || exit_code == SVM_EXIT_IDLE_HLT)
3474		return kvm_emulate_halt(vcpu);
3475	else if (exit_code == SVM_EXIT_NPF)
3476		return npf_interception(vcpu);
3477#ifdef CONFIG_KVM_AMD_SEV
3478	else if (exit_code == SVM_EXIT_VMGEXIT)
3479		return sev_handle_vmgexit(vcpu);
3480#endif
3481#endif
3482	return svm_exit_handlers[exit_code](vcpu);
3483}
3484
3485static void svm_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
3486			      u64 *info1, u64 *info2,
3487			      u32 *intr_info, u32 *error_code)
3488{
3489	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3490
3491	*reason = control->exit_code;
3492	*info1 = control->exit_info_1;
3493	*info2 = control->exit_info_2;
3494	*intr_info = control->exit_int_info;
3495	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3496	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3497		*error_code = control->exit_int_info_err;
3498	else
3499		*error_code = 0;
3500}
3501
3502static void svm_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info,
3503			       u32 *error_code)
3504{
3505	struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3506
3507	*intr_info = control->event_inj;
3508
3509	if ((*intr_info & SVM_EXITINTINFO_VALID) &&
3510	    (*intr_info & SVM_EXITINTINFO_VALID_ERR))
3511		*error_code = control->event_inj_err;
3512	else
3513		*error_code = 0;
3514
3515}
3516
3517static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
3518{
3519	struct vcpu_svm *svm = to_svm(vcpu);
3520	struct kvm_run *kvm_run = vcpu->run;
3521	u32 exit_code = svm->vmcb->control.exit_code;
3522
3523	/* SEV-ES guests must use the CR write traps to track CR registers. */
3524	if (!sev_es_guest(vcpu->kvm)) {
3525		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
3526			vcpu->arch.cr0 = svm->vmcb->save.cr0;
3527		if (npt_enabled)
3528			vcpu->arch.cr3 = svm->vmcb->save.cr3;
3529	}
3530
3531	if (is_guest_mode(vcpu)) {
3532		int vmexit;
3533
3534		trace_kvm_nested_vmexit(vcpu, KVM_ISA_SVM);
3535
3536		vmexit = nested_svm_exit_special(svm);
3537
3538		if (vmexit == NESTED_EXIT_CONTINUE)
3539			vmexit = nested_svm_exit_handled(svm);
3540
3541		if (vmexit == NESTED_EXIT_DONE)
3542			return 1;
3543	}
3544
3545	if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3546		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3547		kvm_run->fail_entry.hardware_entry_failure_reason
3548			= svm->vmcb->control.exit_code;
3549		kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu;
3550		dump_vmcb(vcpu);
3551		return 0;
3552	}
3553
3554	if (exit_fastpath != EXIT_FASTPATH_NONE)
3555		return 1;
3556
3557	return svm_invoke_exit_handler(vcpu, exit_code);
3558}
3559
3560static int pre_svm_run(struct kvm_vcpu *vcpu)
3561{
3562	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
3563	struct vcpu_svm *svm = to_svm(vcpu);
3564
3565	/*
3566	 * If the previous vmrun of the vmcb occurred on a different physical
3567	 * cpu, then mark the vmcb dirty and assign a new asid.  Hardware's
3568	 * vmcb clean bits are per logical CPU, as are KVM's asid assignments.
3569	 */
3570	if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) {
3571		svm->current_vmcb->asid_generation = 0;
3572		vmcb_mark_all_dirty(svm->vmcb);
3573		svm->current_vmcb->cpu = vcpu->cpu;
3574        }
3575
3576	if (sev_guest(vcpu->kvm))
3577		return pre_sev_run(svm, vcpu->cpu);
3578
3579	/* FIXME: handle wraparound of asid_generation */
3580	if (svm->current_vmcb->asid_generation != sd->asid_generation)
3581		new_asid(svm, sd);
3582
3583	return 0;
3584}
3585
3586static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3587{
3588	struct vcpu_svm *svm = to_svm(vcpu);
3589
3590	svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3591
3592	if (svm->nmi_l1_to_l2)
3593		return;
3594
3595	/*
3596	 * No need to manually track NMI masking when vNMI is enabled, hardware
3597	 * automatically sets V_NMI_BLOCKING_MASK as appropriate, including the
3598	 * case where software directly injects an NMI.
3599	 */
3600	if (!is_vnmi_enabled(svm)) {
3601		svm->nmi_masked = true;
3602		svm_set_iret_intercept(svm);
3603	}
3604	++vcpu->stat.nmi_injections;
3605}
3606
3607static bool svm_is_vnmi_pending(struct kvm_vcpu *vcpu)
3608{
3609	struct vcpu_svm *svm = to_svm(vcpu);
3610
3611	if (!is_vnmi_enabled(svm))
3612		return false;
3613
3614	return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK);
3615}
3616
3617static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu)
3618{
3619	struct vcpu_svm *svm = to_svm(vcpu);
3620
3621	if (!is_vnmi_enabled(svm))
3622		return false;
3623
3624	if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK)
3625		return false;
3626
3627	svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK;
3628	vmcb_mark_dirty(svm->vmcb, VMCB_INTR);
3629
3630	/*
3631	 * Because the pending NMI is serviced by hardware, KVM can't know when
3632	 * the NMI is "injected", but for all intents and purposes, passing the
3633	 * NMI off to hardware counts as injection.
3634	 */
3635	++vcpu->stat.nmi_injections;
3636
3637	return true;
3638}
3639
3640static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
3641{
3642	struct kvm_queued_interrupt *intr = &vcpu->arch.interrupt;
3643	struct vcpu_svm *svm = to_svm(vcpu);
3644	u32 type;
3645
3646	if (intr->soft) {
3647		if (svm_update_soft_interrupt_rip(vcpu, intr->nr))
3648			return;
3649
3650		type = SVM_EVTINJ_TYPE_SOFT;
3651	} else {
3652		type = SVM_EVTINJ_TYPE_INTR;
3653	}
3654
3655	trace_kvm_inj_virq(intr->nr, intr->soft, reinjected);
3656	++vcpu->stat.irq_injections;
3657
3658	svm->vmcb->control.event_inj = intr->nr | SVM_EVTINJ_VALID | type;
3659}
3660
3661void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
3662				     int trig_mode, int vector)
3663{
3664	/*
3665	 * apic->apicv_active must be read after vcpu->mode.
3666	 * Pairs with smp_store_release in vcpu_enter_guest.
3667	 */
3668	bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE);
3669
3670	/* Note, this is called iff the local APIC is in-kernel. */
3671	if (!READ_ONCE(vcpu->arch.apic->apicv_active)) {
3672		/* Process the interrupt via kvm_check_and_inject_events(). */
3673		kvm_make_request(KVM_REQ_EVENT, vcpu);
3674		kvm_vcpu_kick(vcpu);
3675		return;
3676	}
3677
3678	trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector);
3679	if (in_guest_mode) {
3680		/*
3681		 * Signal the doorbell to tell hardware to inject the IRQ.  If
3682		 * the vCPU exits the guest before the doorbell chimes, hardware
3683		 * will automatically process AVIC interrupts at the next VMRUN.
3684		 */
3685		avic_ring_doorbell(vcpu);
3686	} else {
3687		/*
3688		 * Wake the vCPU if it was blocking.  KVM will then detect the
3689		 * pending IRQ when checking if the vCPU has a wake event.
3690		 */
3691		kvm_vcpu_wake_up(vcpu);
3692	}
3693}
3694
3695static void svm_deliver_interrupt(struct kvm_lapic *apic,  int delivery_mode,
3696				  int trig_mode, int vector)
3697{
3698	kvm_lapic_set_irr(vector, apic);
3699
3700	/*
3701	 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in
3702	 * vcpu_enter_guest() to ensure the write to the vIRR is ordered before
3703	 * the read of guest_mode.  This guarantees that either VMRUN will see
3704	 * and process the new vIRR entry, or that svm_complete_interrupt_delivery
3705	 * will signal the doorbell if the CPU has already entered the guest.
3706	 */
3707	smp_mb__after_atomic();
3708	svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector);
3709}
3710
3711static void svm_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3712{
3713	struct vcpu_svm *svm = to_svm(vcpu);
3714
3715	/*
3716	 * SEV-ES guests must always keep the CR intercepts cleared. CR
3717	 * tracking is done using the CR write traps.
3718	 */
3719	if (sev_es_guest(vcpu->kvm))
3720		return;
3721
3722	if (nested_svm_virtualize_tpr(vcpu))
3723		return;
3724
3725	svm_clr_intercept(svm, INTERCEPT_CR8_WRITE);
3726
3727	if (irr == -1)
3728		return;
3729
3730	if (tpr >= irr)
3731		svm_set_intercept(svm, INTERCEPT_CR8_WRITE);
3732}
3733
3734static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3735{
3736	struct vcpu_svm *svm = to_svm(vcpu);
3737
3738	if (is_vnmi_enabled(svm))
3739		return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK;
3740	else
3741		return svm->nmi_masked;
3742}
3743
3744static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3745{
3746	struct vcpu_svm *svm = to_svm(vcpu);
3747
3748	if (is_vnmi_enabled(svm)) {
3749		if (masked)
3750			svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK;
3751		else
3752			svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
3753
3754	} else {
3755		svm->nmi_masked = masked;
3756		if (masked)
3757			svm_set_iret_intercept(svm);
3758		else
3759			svm_clr_iret_intercept(svm);
3760	}
3761}
3762
3763bool svm_nmi_blocked(struct kvm_vcpu *vcpu)
3764{
3765	struct vcpu_svm *svm = to_svm(vcpu);
3766	struct vmcb *vmcb = svm->vmcb;
3767
3768	if (!gif_set(svm))
3769		return true;
3770
3771	if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3772		return false;
3773
3774	if (svm_get_nmi_mask(vcpu))
3775		return true;
3776
3777	return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK;
3778}
3779
3780static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3781{
3782	struct vcpu_svm *svm = to_svm(vcpu);
3783	if (svm->nested.nested_run_pending)
3784		return -EBUSY;
3785
3786	if (svm_nmi_blocked(vcpu))
3787		return 0;
3788
3789	/* An NMI must not be injected into L2 if it's supposed to VM-Exit.  */
3790	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm))
3791		return -EBUSY;
3792	return 1;
3793}
3794
3795bool svm_interrupt_blocked(struct kvm_vcpu *vcpu)
3796{
3797	struct vcpu_svm *svm = to_svm(vcpu);
3798	struct vmcb *vmcb = svm->vmcb;
3799
3800	if (!gif_set(svm))
3801		return true;
3802
3803	if (is_guest_mode(vcpu)) {
3804		/* As long as interrupts are being delivered...  */
3805		if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK)
3806		    ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF)
3807		    : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF))
3808			return true;
3809
3810		/* ... vmexits aren't blocked by the interrupt shadow  */
3811		if (nested_exit_on_intr(svm))
3812			return false;
3813	} else {
3814		if (!svm_get_if_flag(vcpu))
3815			return true;
3816	}
3817
3818	return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK);
3819}
3820
3821static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
3822{
3823	struct vcpu_svm *svm = to_svm(vcpu);
3824
3825	if (svm->nested.nested_run_pending)
3826		return -EBUSY;
3827
3828	if (svm_interrupt_blocked(vcpu))
3829		return 0;
3830
3831	/*
3832	 * An IRQ must not be injected into L2 if it's supposed to VM-Exit,
3833	 * e.g. if the IRQ arrived asynchronously after checking nested events.
3834	 */
3835	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm))
3836		return -EBUSY;
3837
3838	return 1;
3839}
3840
3841static void svm_enable_irq_window(struct kvm_vcpu *vcpu)
3842{
3843	struct vcpu_svm *svm = to_svm(vcpu);
3844
3845	/*
3846	 * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3847	 * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3848	 * get that intercept, this function will be called again though and
3849	 * we'll get the vintr intercept. However, if the vGIF feature is
3850	 * enabled, the STGI interception will not occur. Enable the irq
3851	 * window under the assumption that the hardware will set the GIF.
3852	 */
3853	if (vgif || gif_set(svm)) {
3854		/*
3855		 * IRQ window is not needed when AVIC is enabled,
3856		 * unless we have pending ExtINT since it cannot be injected
3857		 * via AVIC. In such case, KVM needs to temporarily disable AVIC,
3858		 * and fallback to injecting IRQ via V_IRQ.
3859		 *
3860		 * If running nested, AVIC is already locally inhibited
3861		 * on this vCPU, therefore there is no need to request
3862		 * the VM wide AVIC inhibition.
3863		 */
3864		if (!is_guest_mode(vcpu))
3865			kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN);
3866
3867		svm_set_vintr(svm);
3868	}
3869}
3870
3871static void svm_enable_nmi_window(struct kvm_vcpu *vcpu)
3872{
3873	struct vcpu_svm *svm = to_svm(vcpu);
3874
3875	/*
3876	 * If NMIs are outright masked, i.e. the vCPU is already handling an
3877	 * NMI, and KVM has not yet intercepted an IRET, then there is nothing
3878	 * more to do at this time as KVM has already enabled IRET intercepts.
3879	 * If KVM has already intercepted IRET, then single-step over the IRET,
3880	 * as NMIs aren't architecturally unmasked until the IRET completes.
3881	 *
3882	 * If vNMI is enabled, KVM should never request an NMI window if NMIs
3883	 * are masked, as KVM allows at most one to-be-injected NMI and one
3884	 * pending NMI.  If two NMIs arrive simultaneously, KVM will inject one
3885	 * NMI and set V_NMI_PENDING for the other, but if and only if NMIs are
3886	 * unmasked.  KVM _will_ request an NMI window in some situations, e.g.
3887	 * if the vCPU is in an STI shadow or if GIF=0, KVM can't immediately
3888	 * inject the NMI.  In those situations, KVM needs to single-step over
3889	 * the STI shadow or intercept STGI.
3890	 */
3891	if (svm_get_nmi_mask(vcpu)) {
3892		WARN_ON_ONCE(is_vnmi_enabled(svm));
3893
3894		if (!svm->awaiting_iret_completion)
3895			return; /* IRET will cause a vm exit */
3896	}
3897
3898	/*
3899	 * SEV-ES guests are responsible for signaling when a vCPU is ready to
3900	 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e.
3901	 * KVM can't intercept and single-step IRET to detect when NMIs are
3902	 * unblocked (architecturally speaking).  See SVM_VMGEXIT_NMI_COMPLETE.
3903	 *
3904	 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware
3905	 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not
3906	 * supported NAEs in the GHCB protocol.
3907	 */
3908	if (sev_es_guest(vcpu->kvm))
3909		return;
3910
3911	if (!gif_set(svm)) {
3912		if (vgif)
3913			svm_set_intercept(svm, INTERCEPT_STGI);
3914		return; /* STGI will cause a vm exit */
3915	}
3916
3917	/*
3918	 * Something prevents NMI from been injected. Single step over possible
3919	 * problem (IRET or exception injection or interrupt shadow)
3920	 */
3921	svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
3922	svm->nmi_singlestep = true;
3923	svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3924}
3925
3926static void svm_flush_tlb_asid(struct kvm_vcpu *vcpu)
3927{
3928	struct vcpu_svm *svm = to_svm(vcpu);
3929
3930	/*
3931	 * Unlike VMX, SVM doesn't provide a way to flush only NPT TLB entries.
3932	 * A TLB flush for the current ASID flushes both "host" and "guest" TLB
3933	 * entries, and thus is a superset of Hyper-V's fine grained flushing.
3934	 */
3935	kvm_hv_vcpu_purge_flush_tlb(vcpu);
3936
3937	/*
3938	 * Flush only the current ASID even if the TLB flush was invoked via
3939	 * kvm_flush_remote_tlbs().  Although flushing remote TLBs requires all
3940	 * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and
3941	 * unconditionally does a TLB flush on both nested VM-Enter and nested
3942	 * VM-Exit (via kvm_mmu_reset_context()).
3943	 */
3944	if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3945		svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3946	else
3947		svm->current_vmcb->asid_generation--;
3948}
3949
3950static void svm_flush_tlb_current(struct kvm_vcpu *vcpu)
3951{
3952	hpa_t root_tdp = vcpu->arch.mmu->root.hpa;
3953
3954	/*
3955	 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly
3956	 * flush the NPT mappings via hypercall as flushing the ASID only
3957	 * affects virtual to physical mappings, it does not invalidate guest
3958	 * physical to host physical mappings.
3959	 */
3960	if (svm_hv_is_enlightened_tlb_enabled(vcpu) && VALID_PAGE(root_tdp))
3961		hyperv_flush_guest_mapping(root_tdp);
3962
3963	svm_flush_tlb_asid(vcpu);
3964}
3965
3966static void svm_flush_tlb_all(struct kvm_vcpu *vcpu)
3967{
3968	/*
3969	 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB
3970	 * flushes should be routed to hv_flush_remote_tlbs() without requesting
3971	 * a "regular" remote flush.  Reaching this point means either there's
3972	 * a KVM bug or a prior hv_flush_remote_tlbs() call failed, both of
3973	 * which might be fatal to the guest.  Yell, but try to recover.
3974	 */
3975	if (WARN_ON_ONCE(svm_hv_is_enlightened_tlb_enabled(vcpu)))
3976		hv_flush_remote_tlbs(vcpu->kvm);
3977
3978	svm_flush_tlb_asid(vcpu);
3979}
3980
3981static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
3982{
3983	struct vcpu_svm *svm = to_svm(vcpu);
3984
3985	invlpga(gva, svm->vmcb->control.asid);
3986}
3987
3988static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3989{
3990	struct vcpu_svm *svm = to_svm(vcpu);
3991
3992	if (nested_svm_virtualize_tpr(vcpu))
3993		return;
3994
3995	if (!svm_is_intercept(svm, INTERCEPT_CR8_WRITE)) {
3996		int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3997		kvm_set_cr8(vcpu, cr8);
3998	}
3999}
4000
4001static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
4002{
4003	struct vcpu_svm *svm = to_svm(vcpu);
4004	u64 cr8;
4005
4006	if (nested_svm_virtualize_tpr(vcpu))
4007		return;
4008
4009	cr8 = kvm_get_cr8(vcpu);
4010	svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
4011	svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
4012}
4013
4014static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector,
4015					int type)
4016{
4017	bool is_exception = (type == SVM_EXITINTINFO_TYPE_EXEPT);
4018	bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT);
4019	struct vcpu_svm *svm = to_svm(vcpu);
4020
4021	/*
4022	 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's
4023	 * associated with the original soft exception/interrupt.  next_rip is
4024	 * cleared on all exits that can occur while vectoring an event, so KVM
4025	 * needs to manually set next_rip for re-injection.  Unlike the !nrips
4026	 * case below, this needs to be done if and only if KVM is re-injecting
4027	 * the same event, i.e. if the event is a soft exception/interrupt,
4028	 * otherwise next_rip is unused on VMRUN.
4029	 */
4030	if (nrips && (is_soft || (is_exception && kvm_exception_is_soft(vector))) &&
4031	    kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase))
4032		svm->vmcb->control.next_rip = svm->soft_int_next_rip;
4033	/*
4034	 * If NRIPS isn't enabled, KVM must manually advance RIP prior to
4035	 * injecting the soft exception/interrupt.  That advancement needs to
4036	 * be unwound if vectoring didn't complete.  Note, the new event may
4037	 * not be the injected event, e.g. if KVM injected an INTn, the INTn
4038	 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will
4039	 * be the reported vectored event, but RIP still needs to be unwound.
4040	 */
4041	else if (!nrips && (is_soft || is_exception) &&
4042		 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase))
4043		kvm_rip_write(vcpu, svm->soft_int_old_rip);
4044}
4045
4046static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
4047{
4048	struct vcpu_svm *svm = to_svm(vcpu);
4049	u8 vector;
4050	int type;
4051	u32 exitintinfo = svm->vmcb->control.exit_int_info;
4052	bool nmi_l1_to_l2 = svm->nmi_l1_to_l2;
4053	bool soft_int_injected = svm->soft_int_injected;
4054
4055	svm->nmi_l1_to_l2 = false;
4056	svm->soft_int_injected = false;
4057
4058	/*
4059	 * If we've made progress since setting awaiting_iret_completion, we've
4060	 * executed an IRET and can allow NMI injection.
4061	 */
4062	if (svm->awaiting_iret_completion &&
4063	    kvm_rip_read(vcpu) != svm->nmi_iret_rip) {
4064		svm->awaiting_iret_completion = false;
4065		svm->nmi_masked = false;
4066		kvm_make_request(KVM_REQ_EVENT, vcpu);
4067	}
4068
4069	vcpu->arch.nmi_injected = false;
4070	kvm_clear_exception_queue(vcpu);
4071	kvm_clear_interrupt_queue(vcpu);
4072
4073	if (!(exitintinfo & SVM_EXITINTINFO_VALID))
4074		return;
4075
4076	kvm_make_request(KVM_REQ_EVENT, vcpu);
4077
4078	vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
4079	type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
4080
4081	if (soft_int_injected)
4082		svm_complete_soft_interrupt(vcpu, vector, type);
4083
4084	switch (type) {
4085	case SVM_EXITINTINFO_TYPE_NMI:
4086		vcpu->arch.nmi_injected = true;
4087		svm->nmi_l1_to_l2 = nmi_l1_to_l2;
4088		break;
4089	case SVM_EXITINTINFO_TYPE_EXEPT: {
4090		u32 error_code = 0;
4091
4092		/*
4093		 * Never re-inject a #VC exception.
4094		 */
4095		if (vector == X86_TRAP_VC)
4096			break;
4097
4098		if (exitintinfo & SVM_EXITINTINFO_VALID_ERR)
4099			error_code = svm->vmcb->control.exit_int_info_err;
4100
4101		kvm_requeue_exception(vcpu, vector,
4102				      exitintinfo & SVM_EXITINTINFO_VALID_ERR,
4103				      error_code);
4104		break;
4105	}
4106	case SVM_EXITINTINFO_TYPE_INTR:
4107		kvm_queue_interrupt(vcpu, vector, false);
4108		break;
4109	case SVM_EXITINTINFO_TYPE_SOFT:
4110		kvm_queue_interrupt(vcpu, vector, true);
4111		break;
4112	default:
4113		break;
4114	}
4115
4116}
4117
4118static void svm_cancel_injection(struct kvm_vcpu *vcpu)
4119{
4120	struct vcpu_svm *svm = to_svm(vcpu);
4121	struct vmcb_control_area *control = &svm->vmcb->control;
4122
4123	control->exit_int_info = control->event_inj;
4124	control->exit_int_info_err = control->event_inj_err;
4125	control->event_inj = 0;
4126	svm_complete_interrupts(vcpu);
4127}
4128
4129static int svm_vcpu_pre_run(struct kvm_vcpu *vcpu)
4130{
4131	if (to_kvm_sev_info(vcpu->kvm)->need_init)
4132		return -EINVAL;
4133
4134	return 1;
4135}
4136
4137static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
4138{
4139	struct vcpu_svm *svm = to_svm(vcpu);
4140	struct vmcb_control_area *control = &svm->vmcb->control;
4141
4142	/*
4143	 * Next RIP must be provided as IRQs are disabled, and accessing guest
4144	 * memory to decode the instruction might fault, i.e. might sleep.
4145	 */
4146	if (!nrips || !control->next_rip)
4147		return EXIT_FASTPATH_NONE;
4148
4149	if (is_guest_mode(vcpu))
4150		return EXIT_FASTPATH_NONE;
4151
4152	switch (control->exit_code) {
4153	case SVM_EXIT_MSR:
4154		if (!control->exit_info_1)
4155			break;
4156		return handle_fastpath_wrmsr(vcpu);
4157	case SVM_EXIT_HLT:
4158		return handle_fastpath_hlt(vcpu);
4159	case SVM_EXIT_INVD:
4160		return handle_fastpath_invd(vcpu);
4161	default:
4162		break;
4163	}
4164
4165	return EXIT_FASTPATH_NONE;
4166}
4167
4168static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
4169{
4170	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
4171	struct vcpu_svm *svm = to_svm(vcpu);
4172
4173	guest_state_enter_irqoff();
4174
4175	/*
4176	 * Set RFLAGS.IF prior to VMRUN, as the host's RFLAGS.IF at the time of
4177	 * VMRUN controls whether or not physical IRQs are masked (KVM always
4178	 * runs with V_INTR_MASKING_MASK).  Toggle RFLAGS.IF here to avoid the
4179	 * temptation to do STI+VMRUN+CLI, as AMD CPUs bleed the STI shadow
4180	 * into guest state if delivery of an event during VMRUN triggers a
4181	 * #VMEXIT, and the guest_state transitions already tell lockdep that
4182	 * IRQs are being enabled/disabled.  Note!  GIF=0 for the entirety of
4183	 * this path, so IRQs aren't actually unmasked while running host code.
4184	 */
4185	raw_local_irq_enable();
4186
4187	amd_clear_divider();
4188
4189	if (sev_es_guest(vcpu->kvm))
4190		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted,
4191				      sev_es_host_save_area(sd));
4192	else
4193		__svm_vcpu_run(svm, spec_ctrl_intercepted);
4194
4195	raw_local_irq_disable();
4196
4197	guest_state_exit_irqoff();
4198}
4199
4200static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
4201{
4202	bool force_immediate_exit = run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT;
4203	struct vcpu_svm *svm = to_svm(vcpu);
4204	bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL);
4205
4206	trace_kvm_entry(vcpu, force_immediate_exit);
4207
4208	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4209	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4210	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4211
4212	/*
4213	 * Disable singlestep if we're injecting an interrupt/exception.
4214	 * We don't want our modified rflags to be pushed on the stack where
4215	 * we might not be able to easily reset them if we disabled NMI
4216	 * singlestep later.
4217	 */
4218	if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
4219		/*
4220		 * Event injection happens before external interrupts cause a
4221		 * vmexit and interrupts are disabled here, so smp_send_reschedule
4222		 * is enough to force an immediate vmexit.
4223		 */
4224		disable_nmi_singlestep(svm);
4225		force_immediate_exit = true;
4226	}
4227
4228	if (force_immediate_exit)
4229		smp_send_reschedule(vcpu->cpu);
4230
4231	if (pre_svm_run(vcpu)) {
4232		vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4233		vcpu->run->fail_entry.hardware_entry_failure_reason = SVM_EXIT_ERR;
4234		vcpu->run->fail_entry.cpu = vcpu->cpu;
4235		return EXIT_FASTPATH_EXIT_USERSPACE;
4236	}
4237
4238	sync_lapic_to_cr8(vcpu);
4239
4240	if (unlikely(svm->asid != svm->vmcb->control.asid)) {
4241		svm->vmcb->control.asid = svm->asid;
4242		vmcb_mark_dirty(svm->vmcb, VMCB_ASID);
4243	}
4244	svm->vmcb->save.cr2 = vcpu->arch.cr2;
4245
4246	svm_hv_update_vp_id(svm->vmcb, vcpu);
4247
4248	/*
4249	 * Run with all-zero DR6 unless the guest can write DR6 freely, so that
4250	 * KVM can get the exact cause of a #DB.  Note, loading guest DR6 from
4251	 * KVM's snapshot is only necessary when DR accesses won't exit.
4252	 */
4253	if (unlikely(run_flags & KVM_RUN_LOAD_GUEST_DR6))
4254		svm_set_dr6(vcpu, vcpu->arch.dr6);
4255	else if (likely(!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)))
4256		svm_set_dr6(vcpu, DR6_ACTIVE_LOW);
4257
4258	clgi();
4259
4260	/*
4261	 * Hardware only context switches DEBUGCTL if LBR virtualization is
4262	 * enabled.  Manually load DEBUGCTL if necessary (and restore it after
4263	 * VM-Exit), as running with the host's DEBUGCTL can negatively affect
4264	 * guest state and can even be fatal, e.g. due to Bus Lock Detect.
4265	 */
4266	if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) &&
4267	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
4268		update_debugctlmsr(svm->vmcb->save.dbgctl);
4269
4270	kvm_wait_lapic_expire(vcpu);
4271
4272	/*
4273	 * If this vCPU has touched SPEC_CTRL, restore the guest's value if
4274	 * it's non-zero. Since vmentry is serialising on affected CPUs, there
4275	 * is no need to worry about the conditional branch over the wrmsr
4276	 * being speculatively taken.
4277	 */
4278	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4279		x86_spec_ctrl_set_guest(svm->virt_spec_ctrl);
4280
4281	svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted);
4282
4283	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
4284		x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
4285
4286	if (!sev_es_guest(vcpu->kvm)) {
4287		vcpu->arch.cr2 = svm->vmcb->save.cr2;
4288		vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
4289		vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
4290		vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
4291	}
4292	vcpu->arch.regs_dirty = 0;
4293
4294	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4295		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
4296
4297	if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) &&
4298	    vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl)
4299		update_debugctlmsr(vcpu->arch.host_debugctl);
4300
4301	stgi();
4302
4303	/* Any pending NMI will happen here */
4304
4305	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4306		kvm_after_interrupt(vcpu);
4307
4308	sync_cr8_to_lapic(vcpu);
4309
4310	svm->next_rip = 0;
4311	if (is_guest_mode(vcpu)) {
4312		nested_sync_control_from_vmcb02(svm);
4313
4314		/* Track VMRUNs that have made past consistency checking */
4315		if (svm->nested.nested_run_pending &&
4316		    svm->vmcb->control.exit_code != SVM_EXIT_ERR)
4317                        ++vcpu->stat.nested_run;
4318
4319		svm->nested.nested_run_pending = 0;
4320	}
4321
4322	svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4323	vmcb_mark_all_clean(svm->vmcb);
4324
4325	/* if exit due to PF check for async PF */
4326	if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
4327		vcpu->arch.apf.host_apf_flags =
4328			kvm_read_and_reset_apf_flags();
4329
4330	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
4331
4332	trace_kvm_exit(vcpu, KVM_ISA_SVM);
4333
4334	svm_complete_interrupts(vcpu);
4335
4336	return svm_exit_handlers_fastpath(vcpu);
4337}
4338
4339static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
4340			     int root_level)
4341{
4342	struct vcpu_svm *svm = to_svm(vcpu);
4343	unsigned long cr3;
4344
4345	if (npt_enabled) {
4346		svm->vmcb->control.nested_cr3 = __sme_set(root_hpa);
4347		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
4348
4349		hv_track_root_tdp(vcpu, root_hpa);
4350
4351		cr3 = vcpu->arch.cr3;
4352	} else if (root_level >= PT64_ROOT_4LEVEL) {
4353		cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu);
4354	} else {
4355		/* PCID in the guest should be impossible with a 32-bit MMU. */
4356		WARN_ON_ONCE(kvm_get_active_pcid(vcpu));
4357		cr3 = root_hpa;
4358	}
4359
4360	svm->vmcb->save.cr3 = cr3;
4361	vmcb_mark_dirty(svm->vmcb, VMCB_CR);
4362}
4363
4364static void
4365svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4366{
4367	/*
4368	 * Patch in the VMMCALL instruction:
4369	 */
4370	hypercall[0] = 0x0f;
4371	hypercall[1] = 0x01;
4372	hypercall[2] = 0xd9;
4373}
4374
4375/*
4376 * The kvm parameter can be NULL (module initialization, or invocation before
4377 * VM creation). Be sure to check the kvm parameter before using it.
4378 */
4379static bool svm_has_emulated_msr(struct kvm *kvm, u32 index)
4380{
4381	switch (index) {
4382	case MSR_IA32_MCG_EXT_CTL:
4383	case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR:
4384		return false;
4385	case MSR_IA32_SMBASE:
4386		if (!IS_ENABLED(CONFIG_KVM_SMM))
4387			return false;
4388		/* SEV-ES guests do not support SMM, so report false */
4389		if (kvm && sev_es_guest(kvm))
4390			return false;
4391		break;
4392	default:
4393		break;
4394	}
4395
4396	return true;
4397}
4398
4399static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
4400{
4401	struct vcpu_svm *svm = to_svm(vcpu);
4402
4403	/*
4404	 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM
4405	 * can only disable all variants of by disallowing CR4.OSXSAVE from
4406	 * being set.  As a result, if the host has XSAVE and XSAVES, and the
4407	 * guest has XSAVE enabled, the guest can execute XSAVES without
4408	 * faulting.  Treat XSAVES as enabled in this case regardless of
4409	 * whether it's advertised to the guest so that KVM context switches
4410	 * XSS on VM-Enter/VM-Exit.  Failure to do so would effectively give
4411	 * the guest read/write access to the host's XSS.
4412	 */
4413	guest_cpu_cap_change(vcpu, X86_FEATURE_XSAVES,
4414			     boot_cpu_has(X86_FEATURE_XSAVES) &&
4415			     guest_cpu_cap_has(vcpu, X86_FEATURE_XSAVE));
4416
4417	/*
4418	 * Intercept VMLOAD if the vCPU model is Intel in order to emulate that
4419	 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing
4420	 * SVM on Intel is bonkers and extremely unlikely to work).
4421	 */
4422	if (guest_cpuid_is_intel_compatible(vcpu))
4423		guest_cpu_cap_clear(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD);
4424
4425	if (sev_guest(vcpu->kvm))
4426		sev_vcpu_after_set_cpuid(svm);
4427}
4428
4429static bool svm_has_wbinvd_exit(void)
4430{
4431	return true;
4432}
4433
4434#define PRE_EX(exit)  { .exit_code = (exit), \
4435			.stage = X86_ICPT_PRE_EXCEPT, }
4436#define POST_EX(exit) { .exit_code = (exit), \
4437			.stage = X86_ICPT_POST_EXCEPT, }
4438#define POST_MEM(exit) { .exit_code = (exit), \
4439			.stage = X86_ICPT_POST_MEMACCESS, }
4440
4441static const struct __x86_intercept {
4442	u32 exit_code;
4443	enum x86_intercept_stage stage;
4444} x86_intercept_map[] = {
4445	[x86_intercept_cr_read]		= POST_EX(SVM_EXIT_READ_CR0),
4446	[x86_intercept_cr_write]	= POST_EX(SVM_EXIT_WRITE_CR0),
4447	[x86_intercept_clts]		= POST_EX(SVM_EXIT_WRITE_CR0),
4448	[x86_intercept_lmsw]		= POST_EX(SVM_EXIT_WRITE_CR0),
4449	[x86_intercept_smsw]		= POST_EX(SVM_EXIT_READ_CR0),
4450	[x86_intercept_dr_read]		= POST_EX(SVM_EXIT_READ_DR0),
4451	[x86_intercept_dr_write]	= POST_EX(SVM_EXIT_WRITE_DR0),
4452	[x86_intercept_sldt]		= POST_EX(SVM_EXIT_LDTR_READ),
4453	[x86_intercept_str]		= POST_EX(SVM_EXIT_TR_READ),
4454	[x86_intercept_lldt]		= POST_EX(SVM_EXIT_LDTR_WRITE),
4455	[x86_intercept_ltr]		= POST_EX(SVM_EXIT_TR_WRITE),
4456	[x86_intercept_sgdt]		= POST_EX(SVM_EXIT_GDTR_READ),
4457	[x86_intercept_sidt]		= POST_EX(SVM_EXIT_IDTR_READ),
4458	[x86_intercept_lgdt]		= POST_EX(SVM_EXIT_GDTR_WRITE),
4459	[x86_intercept_lidt]		= POST_EX(SVM_EXIT_IDTR_WRITE),
4460	[x86_intercept_vmrun]		= POST_EX(SVM_EXIT_VMRUN),
4461	[x86_intercept_vmmcall]		= POST_EX(SVM_EXIT_VMMCALL),
4462	[x86_intercept_vmload]		= POST_EX(SVM_EXIT_VMLOAD),
4463	[x86_intercept_vmsave]		= POST_EX(SVM_EXIT_VMSAVE),
4464	[x86_intercept_stgi]		= POST_EX(SVM_EXIT_STGI),
4465	[x86_intercept_clgi]		= POST_EX(SVM_EXIT_CLGI),
4466	[x86_intercept_skinit]		= POST_EX(SVM_EXIT_SKINIT),
4467	[x86_intercept_invlpga]		= POST_EX(SVM_EXIT_INVLPGA),
4468	[x86_intercept_rdtscp]		= POST_EX(SVM_EXIT_RDTSCP),
4469	[x86_intercept_monitor]		= POST_MEM(SVM_EXIT_MONITOR),
4470	[x86_intercept_mwait]		= POST_EX(SVM_EXIT_MWAIT),
4471	[x86_intercept_invlpg]		= POST_EX(SVM_EXIT_INVLPG),
4472	[x86_intercept_invd]		= POST_EX(SVM_EXIT_INVD),
4473	[x86_intercept_wbinvd]		= POST_EX(SVM_EXIT_WBINVD),
4474	[x86_intercept_wrmsr]		= POST_EX(SVM_EXIT_MSR),
4475	[x86_intercept_rdtsc]		= POST_EX(SVM_EXIT_RDTSC),
4476	[x86_intercept_rdmsr]		= POST_EX(SVM_EXIT_MSR),
4477	[x86_intercept_rdpmc]		= POST_EX(SVM_EXIT_RDPMC),
4478	[x86_intercept_cpuid]		= PRE_EX(SVM_EXIT_CPUID),
4479	[x86_intercept_rsm]		= PRE_EX(SVM_EXIT_RSM),
4480	[x86_intercept_pause]		= PRE_EX(SVM_EXIT_PAUSE),
4481	[x86_intercept_pushf]		= PRE_EX(SVM_EXIT_PUSHF),
4482	[x86_intercept_popf]		= PRE_EX(SVM_EXIT_POPF),
4483	[x86_intercept_intn]		= PRE_EX(SVM_EXIT_SWINT),
4484	[x86_intercept_iret]		= PRE_EX(SVM_EXIT_IRET),
4485	[x86_intercept_icebp]		= PRE_EX(SVM_EXIT_ICEBP),
4486	[x86_intercept_hlt]		= POST_EX(SVM_EXIT_HLT),
4487	[x86_intercept_in]		= POST_EX(SVM_EXIT_IOIO),
4488	[x86_intercept_ins]		= POST_EX(SVM_EXIT_IOIO),
4489	[x86_intercept_out]		= POST_EX(SVM_EXIT_IOIO),
4490	[x86_intercept_outs]		= POST_EX(SVM_EXIT_IOIO),
4491	[x86_intercept_xsetbv]		= PRE_EX(SVM_EXIT_XSETBV),
4492};
4493
4494#undef PRE_EX
4495#undef POST_EX
4496#undef POST_MEM
4497
4498static int svm_check_intercept(struct kvm_vcpu *vcpu,
4499			       struct x86_instruction_info *info,
4500			       enum x86_intercept_stage stage,
4501			       struct x86_exception *exception)
4502{
4503	struct vcpu_svm *svm = to_svm(vcpu);
4504	int vmexit, ret = X86EMUL_CONTINUE;
4505	struct __x86_intercept icpt_info;
4506	struct vmcb *vmcb = svm->vmcb;
4507
4508	if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4509		goto out;
4510
4511	icpt_info = x86_intercept_map[info->intercept];
4512
4513	if (stage != icpt_info.stage)
4514		goto out;
4515
4516	switch (icpt_info.exit_code) {
4517	case SVM_EXIT_READ_CR0:
4518		if (info->intercept == x86_intercept_cr_read)
4519			icpt_info.exit_code += info->modrm_reg;
4520		break;
4521	case SVM_EXIT_WRITE_CR0: {
4522		unsigned long cr0, val;
4523
4524		/*
4525		 * Adjust the exit code accordingly if a CR other than CR0 is
4526		 * being written, and skip straight to the common handling as
4527		 * only CR0 has an additional selective intercept.
4528		 */
4529		if (info->intercept == x86_intercept_cr_write && info->modrm_reg) {
4530			icpt_info.exit_code += info->modrm_reg;
4531			break;
4532		}
4533
4534		/*
4535		 * Convert the exit_code to SVM_EXIT_CR0_SEL_WRITE if a
4536		 * selective CR0 intercept is triggered (the common logic will
4537		 * treat the selective intercept as being enabled).  Note, the
4538		 * unconditional intercept has higher priority, i.e. this is
4539		 * only relevant if *only* the selective intercept is enabled.
4540		 */
4541		if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_CR0_WRITE) ||
4542		    !(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))
4543			break;
4544
4545		/* CLTS never triggers INTERCEPT_SELECTIVE_CR0 */
4546		if (info->intercept == x86_intercept_clts)
4547			break;
4548
4549		/* LMSW always triggers INTERCEPT_SELECTIVE_CR0 */
4550		if (info->intercept == x86_intercept_lmsw) {
4551			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4552			break;
4553		}
4554
4555		/*
4556		 * MOV-to-CR0 only triggers INTERCEPT_SELECTIVE_CR0 if any bit
4557		 * other than SVM_CR0_SELECTIVE_MASK is changed.
4558		 */
4559		cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4560		val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4561		if (cr0 ^ val)
4562			icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4563		break;
4564	}
4565	case SVM_EXIT_READ_DR0:
4566	case SVM_EXIT_WRITE_DR0:
4567		icpt_info.exit_code += info->modrm_reg;
4568		break;
4569	case SVM_EXIT_MSR:
4570		if (info->intercept == x86_intercept_wrmsr)
4571			vmcb->control.exit_info_1 = 1;
4572		else
4573			vmcb->control.exit_info_1 = 0;
4574		break;
4575	case SVM_EXIT_PAUSE:
4576		/*
4577		 * We get this for NOP only, but pause
4578		 * is rep not, check this here
4579		 */
4580		if (info->rep_prefix != REPE_PREFIX)
4581			goto out;
4582		break;
4583	case SVM_EXIT_IOIO: {
4584		u64 exit_info;
4585		u32 bytes;
4586
4587		if (info->intercept == x86_intercept_in ||
4588		    info->intercept == x86_intercept_ins) {
4589			exit_info = ((info->src_val & 0xffff) << 16) |
4590				SVM_IOIO_TYPE_MASK;
4591			bytes = info->dst_bytes;
4592		} else {
4593			exit_info = (info->dst_val & 0xffff) << 16;
4594			bytes = info->src_bytes;
4595		}
4596
4597		if (info->intercept == x86_intercept_outs ||
4598		    info->intercept == x86_intercept_ins)
4599			exit_info |= SVM_IOIO_STR_MASK;
4600
4601		if (info->rep_prefix)
4602			exit_info |= SVM_IOIO_REP_MASK;
4603
4604		bytes = min(bytes, 4u);
4605
4606		exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4607
4608		exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4609
4610		vmcb->control.exit_info_1 = exit_info;
4611		vmcb->control.exit_info_2 = info->next_rip;
4612
4613		break;
4614	}
4615	default:
4616		break;
4617	}
4618
4619	/* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4620	if (static_cpu_has(X86_FEATURE_NRIPS))
4621		vmcb->control.next_rip  = info->next_rip;
4622	vmcb->control.exit_code = icpt_info.exit_code;
4623	vmcb->control.exit_code_hi = 0;
4624	vmexit = nested_svm_exit_handled(svm);
4625
4626	ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4627					   : X86EMUL_CONTINUE;
4628
4629out:
4630	return ret;
4631}
4632
4633static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu)
4634{
4635	switch (to_svm(vcpu)->vmcb->control.exit_code) {
4636	case SVM_EXIT_EXCP_BASE + MC_VECTOR:
4637		svm_handle_mce(vcpu);
4638		break;
4639	case SVM_EXIT_INTR:
4640		vcpu->arch.at_instruction_boundary = true;
4641		break;
4642	default:
4643		break;
4644	}
4645}
4646
4647static void svm_setup_mce(struct kvm_vcpu *vcpu)
4648{
4649	/* [63:9] are reserved. */
4650	vcpu->arch.mcg_cap &= 0x1ff;
4651}
4652
4653#ifdef CONFIG_KVM_SMM
4654bool svm_smi_blocked(struct kvm_vcpu *vcpu)
4655{
4656	struct vcpu_svm *svm = to_svm(vcpu);
4657
4658	/* Per APM Vol.2 15.22.2 "Response to SMI" */
4659	if (!gif_set(svm))
4660		return true;
4661
4662	return is_smm(vcpu);
4663}
4664
4665static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
4666{
4667	struct vcpu_svm *svm = to_svm(vcpu);
4668	if (svm->nested.nested_run_pending)
4669		return -EBUSY;
4670
4671	if (svm_smi_blocked(vcpu))
4672		return 0;
4673
4674	/* An SMI must not be injected into L2 if it's supposed to VM-Exit.  */
4675	if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm))
4676		return -EBUSY;
4677
4678	return 1;
4679}
4680
4681static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
4682{
4683	struct vcpu_svm *svm = to_svm(vcpu);
4684	struct kvm_host_map map_save;
4685	int ret;
4686
4687	if (!is_guest_mode(vcpu))
4688		return 0;
4689
4690	/*
4691	 * 32-bit SMRAM format doesn't preserve EFER and SVM state.  Userspace is
4692	 * responsible for ensuring nested SVM and SMIs are mutually exclusive.
4693	 */
4694
4695	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM))
4696		return 1;
4697
4698	smram->smram64.svm_guest_flag = 1;
4699	smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa;
4700
4701	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4702	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4703	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4704
4705	ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
4706	if (ret)
4707		return ret;
4708
4709	/*
4710	 * KVM uses VMCB01 to store L1 host state while L2 runs but
4711	 * VMCB01 is going to be used during SMM and thus the state will
4712	 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save
4713	 * area pointed to by MSR_VM_HSAVE_PA. APM guarantees that the
4714	 * format of the area is identical to guest save area offsetted
4715	 * by 0x400 (matches the offset of 'struct vmcb_save_area'
4716	 * within 'struct vmcb'). Note: HSAVE area may also be used by
4717	 * L1 hypervisor to save additional host context (e.g. KVM does
4718	 * that, see svm_prepare_switch_to_guest()) which must be
4719	 * preserved.
4720	 */
4721	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4722		return 1;
4723
4724	BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400);
4725
4726	svm_copy_vmrun_state(map_save.hva + 0x400,
4727			     &svm->vmcb01.ptr->save);
4728
4729	kvm_vcpu_unmap(vcpu, &map_save);
4730	return 0;
4731}
4732
4733static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
4734{
4735	struct vcpu_svm *svm = to_svm(vcpu);
4736	struct kvm_host_map map, map_save;
4737	struct vmcb *vmcb12;
4738	int ret;
4739
4740	const struct kvm_smram_state_64 *smram64 = &smram->smram64;
4741
4742	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LM))
4743		return 0;
4744
4745	/* Non-zero if SMI arrived while vCPU was in guest mode. */
4746	if (!smram64->svm_guest_flag)
4747		return 0;
4748
4749	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SVM))
4750		return 1;
4751
4752	if (!(smram64->efer & EFER_SVME))
4753		return 1;
4754
4755	if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map))
4756		return 1;
4757
4758	ret = 1;
4759	if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save))
4760		goto unmap_map;
4761
4762	if (svm_allocate_nested(svm))
4763		goto unmap_save;
4764
4765	/*
4766	 * Restore L1 host state from L1 HSAVE area as VMCB01 was
4767	 * used during SMM (see svm_enter_smm())
4768	 */
4769
4770	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400);
4771
4772	/*
4773	 * Enter the nested guest now
4774	 */
4775
4776	vmcb_mark_all_dirty(svm->vmcb01.ptr);
4777
4778	vmcb12 = map.hva;
4779	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
4780	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
4781	ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false);
4782
4783	if (ret)
4784		goto unmap_save;
4785
4786	svm->nested.nested_run_pending = 1;
4787
4788unmap_save:
4789	kvm_vcpu_unmap(vcpu, &map_save);
4790unmap_map:
4791	kvm_vcpu_unmap(vcpu, &map);
4792	return ret;
4793}
4794
4795static void svm_enable_smi_window(struct kvm_vcpu *vcpu)
4796{
4797	struct vcpu_svm *svm = to_svm(vcpu);
4798
4799	if (!gif_set(svm)) {
4800		if (vgif)
4801			svm_set_intercept(svm, INTERCEPT_STGI);
4802		/* STGI will cause a vm exit */
4803	} else {
4804		/* We must be in SMM; RSM will cause a vmexit anyway.  */
4805	}
4806}
4807#endif
4808
4809static int svm_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
4810					 void *insn, int insn_len)
4811{
4812	struct vcpu_svm *svm = to_svm(vcpu);
4813	bool smep, smap, is_user;
4814	u64 error_code;
4815
4816	/* Check that emulation is possible during event vectoring */
4817	if ((svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK) &&
4818	    !kvm_can_emulate_event_vectoring(emul_type))
4819		return X86EMUL_UNHANDLEABLE_VECTORING;
4820
4821	/* Emulation is always possible when KVM has access to all guest state. */
4822	if (!sev_guest(vcpu->kvm))
4823		return X86EMUL_CONTINUE;
4824
4825	/* #UD and #GP should never be intercepted for SEV guests. */
4826	WARN_ON_ONCE(emul_type & (EMULTYPE_TRAP_UD |
4827				  EMULTYPE_TRAP_UD_FORCED |
4828				  EMULTYPE_VMWARE_GP));
4829
4830	/*
4831	 * Emulation is impossible for SEV-ES guests as KVM doesn't have access
4832	 * to guest register state.
4833	 */
4834	if (sev_es_guest(vcpu->kvm))
4835		return X86EMUL_RETRY_INSTR;
4836
4837	/*
4838	 * Emulation is possible if the instruction is already decoded, e.g.
4839	 * when completing I/O after returning from userspace.
4840	 */
4841	if (emul_type & EMULTYPE_NO_DECODE)
4842		return X86EMUL_CONTINUE;
4843
4844	/*
4845	 * Emulation is possible for SEV guests if and only if a prefilled
4846	 * buffer containing the bytes of the intercepted instruction is
4847	 * available. SEV guest memory is encrypted with a guest specific key
4848	 * and cannot be decrypted by KVM, i.e. KVM would read ciphertext and
4849	 * decode garbage.
4850	 *
4851	 * If KVM is NOT trying to simply skip an instruction, inject #UD if
4852	 * KVM reached this point without an instruction buffer.  In practice,
4853	 * this path should never be hit by a well-behaved guest, e.g. KVM
4854	 * doesn't intercept #UD or #GP for SEV guests, but this path is still
4855	 * theoretically reachable, e.g. via unaccelerated fault-like AVIC
4856	 * access, and needs to be handled by KVM to avoid putting the guest
4857	 * into an infinite loop.   Injecting #UD is somewhat arbitrary, but
4858	 * its the least awful option given lack of insight into the guest.
4859	 *
4860	 * If KVM is trying to skip an instruction, simply resume the guest.
4861	 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM
4862	 * will attempt to re-inject the INT3/INTO and skip the instruction.
4863	 * In that scenario, retrying the INT3/INTO and hoping the guest will
4864	 * make forward progress is the only option that has a chance of
4865	 * success (and in practice it will work the vast majority of the time).
4866	 */
4867	if (unlikely(!insn)) {
4868		if (emul_type & EMULTYPE_SKIP)
4869			return X86EMUL_UNHANDLEABLE;
4870
4871		kvm_queue_exception(vcpu, UD_VECTOR);
4872		return X86EMUL_PROPAGATE_FAULT;
4873	}
4874
4875	/*
4876	 * Emulate for SEV guests if the insn buffer is not empty.  The buffer
4877	 * will be empty if the DecodeAssist microcode cannot fetch bytes for
4878	 * the faulting instruction because the code fetch itself faulted, e.g.
4879	 * the guest attempted to fetch from emulated MMIO or a guest page
4880	 * table used to translate CS:RIP resides in emulated MMIO.
4881	 */
4882	if (likely(insn_len))
4883		return X86EMUL_CONTINUE;
4884
4885	/*
4886	 * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
4887	 *
4888	 * Errata:
4889	 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is
4890	 * possible that CPU microcode implementing DecodeAssist will fail to
4891	 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly
4892	 * be '0'.  This happens because microcode reads CS:RIP using a _data_
4893	 * loap uop with CPL=0 privileges.  If the load hits a SMAP #PF, ucode
4894	 * gives up and does not fill the instruction bytes buffer.
4895	 *
4896	 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU
4897	 * supports DecodeAssist, a #NPF was raised, KVM's page fault handler
4898	 * triggered emulation (e.g. for MMIO), and the CPU returned 0 in the
4899	 * GuestIntrBytes field of the VMCB.
4900	 *
4901	 * This does _not_ mean that the erratum has been encountered, as the
4902	 * DecodeAssist will also fail if the load for CS:RIP hits a legitimate
4903	 * #PF, e.g. if the guest attempt to execute from emulated MMIO and
4904	 * encountered a reserved/not-present #PF.
4905	 *
4906	 * To hit the erratum, the following conditions must be true:
4907	 *    1. CR4.SMAP=1 (obviously).
4908	 *    2. CR4.SMEP=0 || CPL=3.  If SMEP=1 and CPL<3, the erratum cannot
4909	 *       have been hit as the guest would have encountered a SMEP
4910	 *       violation #PF, not a #NPF.
4911	 *    3. The #NPF is not due to a code fetch, in which case failure to
4912	 *       retrieve the instruction bytes is legitimate (see abvoe).
4913	 *
4914	 * In addition, don't apply the erratum workaround if the #NPF occurred
4915	 * while translating guest page tables (see below).
4916	 */
4917	error_code = svm->vmcb->control.exit_info_1;
4918	if (error_code & (PFERR_GUEST_PAGE_MASK | PFERR_FETCH_MASK))
4919		goto resume_guest;
4920
4921	smep = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMEP);
4922	smap = kvm_is_cr4_bit_set(vcpu, X86_CR4_SMAP);
4923	is_user = svm_get_cpl(vcpu) == 3;
4924	if (smap && (!smep || is_user)) {
4925		pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n");
4926
4927		/*
4928		 * If the fault occurred in userspace, arbitrarily inject #GP
4929		 * to avoid killing the guest and to hopefully avoid confusing
4930		 * the guest kernel too much, e.g. injecting #PF would not be
4931		 * coherent with respect to the guest's page tables.  Request
4932		 * triple fault if the fault occurred in the kernel as there's
4933		 * no fault that KVM can inject without confusing the guest.
4934		 * In practice, the triple fault is moot as no sane SEV kernel
4935		 * will execute from user memory while also running with SMAP=1.
4936		 */
4937		if (is_user)
4938			kvm_inject_gp(vcpu, 0);
4939		else
4940			kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
4941		return X86EMUL_PROPAGATE_FAULT;
4942	}
4943
4944resume_guest:
4945	/*
4946	 * If the erratum was not hit, simply resume the guest and let it fault
4947	 * again.  While awful, e.g. the vCPU may get stuck in an infinite loop
4948	 * if the fault is at CPL=0, it's the lesser of all evils.  Exiting to
4949	 * userspace will kill the guest, and letting the emulator read garbage
4950	 * will yield random behavior and potentially corrupt the guest.
4951	 *
4952	 * Simply resuming the guest is technically not a violation of the SEV
4953	 * architecture.  AMD's APM states that all code fetches and page table
4954	 * accesses for SEV guest are encrypted, regardless of the C-Bit.  The
4955	 * APM also states that encrypted accesses to MMIO are "ignored", but
4956	 * doesn't explicitly define "ignored", i.e. doing nothing and letting
4957	 * the guest spin is technically "ignoring" the access.
4958	 */
4959	return X86EMUL_RETRY_INSTR;
4960}
4961
4962static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
4963{
4964	struct vcpu_svm *svm = to_svm(vcpu);
4965
4966	return !gif_set(svm);
4967}
4968
4969static void svm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector)
4970{
4971	if (!sev_es_guest(vcpu->kvm))
4972		return kvm_vcpu_deliver_sipi_vector(vcpu, vector);
4973
4974	sev_vcpu_deliver_sipi_vector(vcpu, vector);
4975}
4976
4977static void svm_vm_destroy(struct kvm *kvm)
4978{
4979	avic_vm_destroy(kvm);
4980	sev_vm_destroy(kvm);
4981
4982	svm_srso_vm_destroy();
4983}
4984
4985static int svm_vm_init(struct kvm *kvm)
4986{
4987	int type = kvm->arch.vm_type;
4988
4989	if (type != KVM_X86_DEFAULT_VM &&
4990	    type != KVM_X86_SW_PROTECTED_VM) {
4991		kvm->arch.has_protected_state =
4992			(type == KVM_X86_SEV_ES_VM || type == KVM_X86_SNP_VM);
4993		to_kvm_sev_info(kvm)->need_init = true;
4994
4995		kvm->arch.has_private_mem = (type == KVM_X86_SNP_VM);
4996		kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem;
4997	}
4998
4999	if (!pause_filter_count || !pause_filter_thresh)
5000		kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE);
5001
5002	if (enable_apicv) {
5003		int ret = avic_vm_init(kvm);
5004		if (ret)
5005			return ret;
5006	}
5007
5008	svm_srso_vm_init();
5009	return 0;
5010}
5011
5012static void *svm_alloc_apic_backing_page(struct kvm_vcpu *vcpu)
5013{
5014	struct page *page = snp_safe_alloc_page();
5015
5016	if (!page)
5017		return NULL;
5018
5019	return page_address(page);
5020}
5021
5022struct kvm_x86_ops svm_x86_ops __initdata = {
5023	.name = KBUILD_MODNAME,
5024
5025	.check_processor_compatibility = svm_check_processor_compat,
5026
5027	.hardware_unsetup = svm_hardware_unsetup,
5028	.enable_virtualization_cpu = svm_enable_virtualization_cpu,
5029	.disable_virtualization_cpu = svm_disable_virtualization_cpu,
5030	.emergency_disable_virtualization_cpu = svm_emergency_disable_virtualization_cpu,
5031	.has_emulated_msr = svm_has_emulated_msr,
5032
5033	.vcpu_precreate = svm_vcpu_precreate,
5034	.vcpu_create = svm_vcpu_create,
5035	.vcpu_free = svm_vcpu_free,
5036	.vcpu_reset = svm_vcpu_reset,
5037
5038	.vm_size = sizeof(struct kvm_svm),
5039	.vm_init = svm_vm_init,
5040	.vm_destroy = svm_vm_destroy,
5041
5042	.prepare_switch_to_guest = svm_prepare_switch_to_guest,
5043	.vcpu_load = svm_vcpu_load,
5044	.vcpu_put = svm_vcpu_put,
5045	.vcpu_blocking = avic_vcpu_blocking,
5046	.vcpu_unblocking = avic_vcpu_unblocking,
5047
5048	.update_exception_bitmap = svm_update_exception_bitmap,
5049	.get_feature_msr = svm_get_feature_msr,
5050	.get_msr = svm_get_msr,
5051	.set_msr = svm_set_msr,
5052	.get_segment_base = svm_get_segment_base,
5053	.get_segment = svm_get_segment,
5054	.set_segment = svm_set_segment,
5055	.get_cpl = svm_get_cpl,
5056	.get_cpl_no_cache = svm_get_cpl,
5057	.get_cs_db_l_bits = svm_get_cs_db_l_bits,
5058	.is_valid_cr0 = svm_is_valid_cr0,
5059	.set_cr0 = svm_set_cr0,
5060	.post_set_cr3 = sev_post_set_cr3,
5061	.is_valid_cr4 = svm_is_valid_cr4,
5062	.set_cr4 = svm_set_cr4,
5063	.set_efer = svm_set_efer,
5064	.get_idt = svm_get_idt,
5065	.set_idt = svm_set_idt,
5066	.get_gdt = svm_get_gdt,
5067	.set_gdt = svm_set_gdt,
5068	.set_dr7 = svm_set_dr7,
5069	.sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
5070	.cache_reg = svm_cache_reg,
5071	.get_rflags = svm_get_rflags,
5072	.set_rflags = svm_set_rflags,
5073	.get_if_flag = svm_get_if_flag,
5074
5075	.flush_tlb_all = svm_flush_tlb_all,
5076	.flush_tlb_current = svm_flush_tlb_current,
5077	.flush_tlb_gva = svm_flush_tlb_gva,
5078	.flush_tlb_guest = svm_flush_tlb_asid,
5079
5080	.vcpu_pre_run = svm_vcpu_pre_run,
5081	.vcpu_run = svm_vcpu_run,
5082	.handle_exit = svm_handle_exit,
5083	.skip_emulated_instruction = svm_skip_emulated_instruction,
5084	.update_emulated_instruction = NULL,
5085	.set_interrupt_shadow = svm_set_interrupt_shadow,
5086	.get_interrupt_shadow = svm_get_interrupt_shadow,
5087	.patch_hypercall = svm_patch_hypercall,
5088	.inject_irq = svm_inject_irq,
5089	.inject_nmi = svm_inject_nmi,
5090	.is_vnmi_pending = svm_is_vnmi_pending,
5091	.set_vnmi_pending = svm_set_vnmi_pending,
5092	.inject_exception = svm_inject_exception,
5093	.cancel_injection = svm_cancel_injection,
5094	.interrupt_allowed = svm_interrupt_allowed,
5095	.nmi_allowed = svm_nmi_allowed,
5096	.get_nmi_mask = svm_get_nmi_mask,
5097	.set_nmi_mask = svm_set_nmi_mask,
5098	.enable_nmi_window = svm_enable_nmi_window,
5099	.enable_irq_window = svm_enable_irq_window,
5100	.update_cr8_intercept = svm_update_cr8_intercept,
5101
5102	.x2apic_icr_is_split = true,
5103	.set_virtual_apic_mode = avic_refresh_virtual_apic_mode,
5104	.refresh_apicv_exec_ctrl = avic_refresh_apicv_exec_ctrl,
5105	.apicv_post_state_restore = avic_apicv_post_state_restore,
5106	.required_apicv_inhibits = AVIC_REQUIRED_APICV_INHIBITS,
5107
5108	.get_exit_info = svm_get_exit_info,
5109	.get_entry_info = svm_get_entry_info,
5110
5111	.vcpu_after_set_cpuid = svm_vcpu_after_set_cpuid,
5112
5113	.has_wbinvd_exit = svm_has_wbinvd_exit,
5114
5115	.get_l2_tsc_offset = svm_get_l2_tsc_offset,
5116	.get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier,
5117	.write_tsc_offset = svm_write_tsc_offset,
5118	.write_tsc_multiplier = svm_write_tsc_multiplier,
5119
5120	.load_mmu_pgd = svm_load_mmu_pgd,
5121
5122	.check_intercept = svm_check_intercept,
5123	.handle_exit_irqoff = svm_handle_exit_irqoff,
5124
5125	.nested_ops = &svm_nested_ops,
5126
5127	.deliver_interrupt = svm_deliver_interrupt,
5128	.pi_update_irte = avic_pi_update_irte,
5129	.setup_mce = svm_setup_mce,
5130
5131#ifdef CONFIG_KVM_SMM
5132	.smi_allowed = svm_smi_allowed,
5133	.enter_smm = svm_enter_smm,
5134	.leave_smm = svm_leave_smm,
5135	.enable_smi_window = svm_enable_smi_window,
5136#endif
5137
5138#ifdef CONFIG_KVM_AMD_SEV
5139	.dev_get_attr = sev_dev_get_attr,
5140	.mem_enc_ioctl = sev_mem_enc_ioctl,
5141	.mem_enc_register_region = sev_mem_enc_register_region,
5142	.mem_enc_unregister_region = sev_mem_enc_unregister_region,
5143	.guest_memory_reclaimed = sev_guest_memory_reclaimed,
5144
5145	.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
5146	.vm_move_enc_context_from = sev_vm_move_enc_context_from,
5147#endif
5148	.check_emulate_instruction = svm_check_emulate_instruction,
5149
5150	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
5151
5152	.recalc_intercepts = svm_recalc_intercepts,
5153	.complete_emulated_msr = svm_complete_emulated_msr,
5154
5155	.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
5156	.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
5157	.alloc_apic_backing_page = svm_alloc_apic_backing_page,
5158
5159	.gmem_prepare = sev_gmem_prepare,
5160	.gmem_invalidate = sev_gmem_invalidate,
5161	.gmem_max_mapping_level = sev_gmem_max_mapping_level,
5162};
5163
5164/*
5165 * The default MMIO mask is a single bit (excluding the present bit),
5166 * which could conflict with the memory encryption bit. Check for
5167 * memory encryption support and override the default MMIO mask if
5168 * memory encryption is enabled.
5169 */
5170static __init void svm_adjust_mmio_mask(void)
5171{
5172	unsigned int enc_bit, mask_bit;
5173	u64 msr, mask;
5174
5175	/* If there is no memory encryption support, use existing mask */
5176	if (cpuid_eax(0x80000000) < 0x8000001f)
5177		return;
5178
5179	/* If memory encryption is not enabled, use existing mask */
5180	rdmsrq(MSR_AMD64_SYSCFG, msr);
5181	if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
5182		return;
5183
5184	enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
5185	mask_bit = boot_cpu_data.x86_phys_bits;
5186
5187	/* Increment the mask bit if it is the same as the encryption bit */
5188	if (enc_bit == mask_bit)
5189		mask_bit++;
5190
5191	/*
5192	 * If the mask bit location is below 52, then some bits above the
5193	 * physical addressing limit will always be reserved, so use the
5194	 * rsvd_bits() function to generate the mask. This mask, along with
5195	 * the present bit, will be used to generate a page fault with
5196	 * PFER.RSV = 1.
5197	 *
5198	 * If the mask bit location is 52 (or above), then clear the mask.
5199	 */
5200	mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
5201
5202	kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
5203}
5204
5205static __init void svm_set_cpu_caps(void)
5206{
5207	kvm_set_cpu_caps();
5208
5209	kvm_caps.supported_perf_cap = 0;
5210
5211	kvm_cpu_cap_clear(X86_FEATURE_IBT);
5212
5213	/* CPUID 0x80000001 and 0x8000000A (SVM features) */
5214	if (nested) {
5215		kvm_cpu_cap_set(X86_FEATURE_SVM);
5216		kvm_cpu_cap_set(X86_FEATURE_VMCBCLEAN);
5217
5218		/*
5219		 * KVM currently flushes TLBs on *every* nested SVM transition,
5220		 * and so for all intents and purposes KVM supports flushing by
5221		 * ASID, i.e. KVM is guaranteed to honor every L1 ASID flush.
5222		 */
5223		kvm_cpu_cap_set(X86_FEATURE_FLUSHBYASID);
5224
5225		if (nrips)
5226			kvm_cpu_cap_set(X86_FEATURE_NRIPS);
5227
5228		if (npt_enabled)
5229			kvm_cpu_cap_set(X86_FEATURE_NPT);
5230
5231		if (tsc_scaling)
5232			kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR);
5233
5234		if (vls)
5235			kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD);
5236		if (lbrv)
5237			kvm_cpu_cap_set(X86_FEATURE_LBRV);
5238
5239		if (boot_cpu_has(X86_FEATURE_PAUSEFILTER))
5240			kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER);
5241
5242		if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD))
5243			kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD);
5244
5245		if (vgif)
5246			kvm_cpu_cap_set(X86_FEATURE_VGIF);
5247
5248		if (vnmi)
5249			kvm_cpu_cap_set(X86_FEATURE_VNMI);
5250
5251		/* Nested VM can receive #VMEXIT instead of triggering #GP */
5252		kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK);
5253	}
5254
5255	if (cpu_feature_enabled(X86_FEATURE_BUS_LOCK_THRESHOLD))
5256		kvm_caps.has_bus_lock_exit = true;
5257
5258	/* CPUID 0x80000008 */
5259	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5260	    boot_cpu_has(X86_FEATURE_AMD_SSBD))
5261		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
5262
5263	if (enable_pmu) {
5264		/*
5265		 * Enumerate support for PERFCTR_CORE if and only if KVM has
5266		 * access to enough counters to virtualize "core" support,
5267		 * otherwise limit vPMU support to the legacy number of counters.
5268		 */
5269		if (kvm_pmu_cap.num_counters_gp < AMD64_NUM_COUNTERS_CORE)
5270			kvm_pmu_cap.num_counters_gp = min(AMD64_NUM_COUNTERS,
5271							  kvm_pmu_cap.num_counters_gp);
5272		else
5273			kvm_cpu_cap_check_and_set(X86_FEATURE_PERFCTR_CORE);
5274
5275		if (kvm_pmu_cap.version != 2 ||
5276		    !kvm_cpu_cap_has(X86_FEATURE_PERFCTR_CORE))
5277			kvm_cpu_cap_clear(X86_FEATURE_PERFMON_V2);
5278	}
5279
5280	/* CPUID 0x8000001F (SME/SEV features) */
5281	sev_set_cpu_caps();
5282
5283	/*
5284	 * Clear capabilities that are automatically configured by common code,
5285	 * but that require explicit SVM support (that isn't yet implemented).
5286	 */
5287	kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT);
5288	kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM);
5289
5290	kvm_setup_xss_caps();
5291}
5292
5293static __init int svm_hardware_setup(void)
5294{
5295	void *iopm_va;
5296	int cpu, r;
5297
5298	/*
5299	 * NX is required for shadow paging and for NPT if the NX huge pages
5300	 * mitigation is enabled.
5301	 */
5302	if (!boot_cpu_has(X86_FEATURE_NX)) {
5303		pr_err_ratelimited("NX (Execute Disable) not supported\n");
5304		return -EOPNOTSUPP;
5305	}
5306	kvm_enable_efer_bits(EFER_NX);
5307
5308	kvm_caps.supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS |
5309				     XFEATURE_MASK_BNDCSR);
5310
5311	if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
5312		kvm_enable_efer_bits(EFER_FFXSR);
5313
5314	if (tsc_scaling) {
5315		if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
5316			tsc_scaling = false;
5317		} else {
5318			pr_info("TSC scaling supported\n");
5319			kvm_caps.has_tsc_control = true;
5320		}
5321	}
5322	kvm_caps.max_tsc_scaling_ratio = SVM_TSC_RATIO_MAX;
5323	kvm_caps.tsc_scaling_ratio_frac_bits = 32;
5324
5325	tsc_aux_uret_slot = kvm_add_user_return_msr(MSR_TSC_AUX);
5326
5327	if (boot_cpu_has(X86_FEATURE_AUTOIBRS))
5328		kvm_enable_efer_bits(EFER_AUTOIBRS);
5329
5330	/* Check for pause filtering support */
5331	if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
5332		pause_filter_count = 0;
5333		pause_filter_thresh = 0;
5334	} else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
5335		pause_filter_thresh = 0;
5336	}
5337
5338	if (nested) {
5339		pr_info("Nested Virtualization enabled\n");
5340		kvm_enable_efer_bits(EFER_SVME);
5341		if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ))
5342			kvm_enable_efer_bits(EFER_LMSLE);
5343
5344		r = nested_svm_init_msrpm_merge_offsets();
5345		if (r)
5346			return r;
5347	}
5348
5349	/*
5350	 * KVM's MMU doesn't support using 2-level paging for itself, and thus
5351	 * NPT isn't supported if the host is using 2-level paging since host
5352	 * CR4 is unchanged on VMRUN.
5353	 */
5354	if (!IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_X86_PAE))
5355		npt_enabled = false;
5356
5357	if (!boot_cpu_has(X86_FEATURE_NPT))
5358		npt_enabled = false;
5359
5360	/* Force VM NPT level equal to the host's paging level */
5361	kvm_configure_mmu(npt_enabled, get_npt_level(),
5362			  get_npt_level(), PG_LEVEL_1G);
5363	pr_info("Nested Paging %s\n", str_enabled_disabled(npt_enabled));
5364
5365	/*
5366	 * It seems that on AMD processors PTE's accessed bit is
5367	 * being set by the CPU hardware before the NPF vmexit.
5368	 * This is not expected behaviour and our tests fail because
5369	 * of it.
5370	 * A workaround here is to disable support for
5371	 * GUEST_MAXPHYADDR < HOST_MAXPHYADDR if NPT is enabled.
5372	 * In this case userspace can know if there is support using
5373	 * KVM_CAP_SMALLER_MAXPHYADDR extension and decide how to handle
5374	 * it
5375	 * If future AMD CPU models change the behaviour described above,
5376	 * this variable can be changed accordingly
5377	 */
5378	allow_smaller_maxphyaddr = !npt_enabled;
5379
5380	/* Setup shadow_me_value and shadow_me_mask */
5381	kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask);
5382
5383	svm_adjust_mmio_mask();
5384
5385	nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS);
5386
5387	if (lbrv) {
5388		if (!boot_cpu_has(X86_FEATURE_LBRV))
5389			lbrv = false;
5390		else
5391			pr_info("LBR virtualization supported\n");
5392	}
5393
5394	iopm_va = svm_alloc_permissions_map(IOPM_SIZE, GFP_KERNEL);
5395	if (!iopm_va)
5396		return -ENOMEM;
5397
5398	iopm_base = __sme_set(__pa(iopm_va));
5399
5400	/*
5401	 * Note, SEV setup consumes npt_enabled and enable_mmio_caching (which
5402	 * may be modified by svm_adjust_mmio_mask()), as well as nrips.
5403	 */
5404	sev_hardware_setup();
5405
5406	svm_hv_hardware_setup();
5407
5408	enable_apicv = avic_hardware_setup();
5409	if (!enable_apicv) {
5410		enable_ipiv = false;
5411		svm_x86_ops.vcpu_blocking = NULL;
5412		svm_x86_ops.vcpu_unblocking = NULL;
5413		svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL;
5414	}
5415
5416	if (vls) {
5417		if (!npt_enabled ||
5418		    !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
5419		    !IS_ENABLED(CONFIG_X86_64)) {
5420			vls = false;
5421		} else {
5422			pr_info("Virtual VMLOAD VMSAVE supported\n");
5423		}
5424	}
5425
5426	if (boot_cpu_has(X86_FEATURE_SVME_ADDR_CHK))
5427		svm_gp_erratum_intercept = false;
5428
5429	if (vgif) {
5430		if (!boot_cpu_has(X86_FEATURE_VGIF))
5431			vgif = false;
5432		else
5433			pr_info("Virtual GIF supported\n");
5434	}
5435
5436	vnmi = vgif && vnmi && boot_cpu_has(X86_FEATURE_VNMI);
5437	if (vnmi)
5438		pr_info("Virtual NMI enabled\n");
5439
5440	if (!vnmi) {
5441		svm_x86_ops.is_vnmi_pending = NULL;
5442		svm_x86_ops.set_vnmi_pending = NULL;
5443	}
5444
5445	if (!enable_pmu)
5446		pr_info("PMU virtualization is disabled\n");
5447
5448	svm_set_cpu_caps();
5449
5450	kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_CD_NW_CLEARED;
5451
5452	for_each_possible_cpu(cpu) {
5453		r = svm_cpu_init(cpu);
5454		if (r)
5455			goto err;
5456	}
5457
5458	return 0;
5459
5460err:
5461	svm_hardware_unsetup();
5462	return r;
5463}
5464
5465
5466static struct kvm_x86_init_ops svm_init_ops __initdata = {
5467	.hardware_setup = svm_hardware_setup,
5468
5469	.runtime_ops = &svm_x86_ops,
5470	.pmu_ops = &amd_pmu_ops,
5471};
5472
5473static void __svm_exit(void)
5474{
5475	kvm_x86_vendor_exit();
5476}
5477
5478static int __init svm_init(void)
5479{
5480	int r;
5481
5482	KVM_SANITY_CHECK_VM_STRUCT_SIZE(kvm_svm);
5483
5484	__unused_size_checks();
5485
5486	if (!kvm_is_svm_supported())
5487		return -EOPNOTSUPP;
5488
5489	r = kvm_x86_vendor_init(&svm_init_ops);
5490	if (r)
5491		return r;
5492
5493	/*
5494	 * Common KVM initialization _must_ come last, after this, /dev/kvm is
5495	 * exposed to userspace!
5496	 */
5497	r = kvm_init(sizeof(struct vcpu_svm), __alignof__(struct vcpu_svm),
5498		     THIS_MODULE);
5499	if (r)
5500		goto err_kvm_init;
5501
5502	return 0;
5503
5504err_kvm_init:
5505	__svm_exit();
5506	return r;
5507}
5508
5509static void __exit svm_exit(void)
5510{
5511	kvm_exit();
5512	__svm_exit();
5513}
5514
5515module_init(svm_init)
5516module_exit(svm_exit)