nested.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Kernel-based Virtual Machine driver for Linux
   4 *
   5 * AMD SVM support
   6 *
   7 * Copyright (C) 2006 Qumranet, Inc.
   8 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9 *
  10 * Authors:
  11 *   Yaniv Kamay  <yaniv@qumranet.com>
  12 *   Avi Kivity   <avi@qumranet.com>
  13 */
  14
  15#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  16
  17#include <linux/kvm_types.h>
  18#include <linux/kvm_host.h>
  19#include <linux/kernel.h>
  20
  21#include <asm/msr-index.h>
  22#include <asm/debugreg.h>
  23
  24#include "kvm_emulate.h"
  25#include "trace.h"
  26#include "mmu.h"
  27#include "x86.h"
  28#include "smm.h"
  29#include "cpuid.h"
  30#include "lapic.h"
  31#include "svm.h"
  32#include "hyperv.h"
  33
  34#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
  35
  36static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
  37				       struct x86_exception *fault)
  38{
  39	struct vcpu_svm *svm = to_svm(vcpu);
  40	struct vmcb *vmcb = svm->vmcb;
  41
  42	if (vmcb->control.exit_code != SVM_EXIT_NPF) {
  43		/*
  44		 * TODO: track the cause of the nested page fault, and
  45		 * correctly fill in the high bits of exit_info_1.
  46		 */
  47		vmcb->control.exit_code = SVM_EXIT_NPF;
  48		vmcb->control.exit_code_hi = 0;
  49		vmcb->control.exit_info_1 = (1ULL << 32);
  50		vmcb->control.exit_info_2 = fault->address;
  51	}
  52
  53	vmcb->control.exit_info_1 &= ~0xffffffffULL;
  54	vmcb->control.exit_info_1 |= fault->error_code;
  55
  56	nested_svm_vmexit(svm);
  57}
  58
  59static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
  60{
  61	struct vcpu_svm *svm = to_svm(vcpu);
  62	u64 cr3 = svm->nested.ctl.nested_cr3;
  63	u64 pdpte;
  64	int ret;
  65
  66	/*
  67	 * Note, nCR3 is "assumed" to be 32-byte aligned, i.e. the CPU ignores
  68	 * nCR3[4:0] when loading PDPTEs from memory.
  69	 */
  70	ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
  71				       (cr3 & GENMASK(11, 5)) + index * 8, 8);
  72	if (ret)
  73		return 0;
  74	return pdpte;
  75}
  76
  77static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
  78{
  79	struct vcpu_svm *svm = to_svm(vcpu);
  80
  81	return svm->nested.ctl.nested_cr3;
  82}
  83
  84static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
  85{
  86	struct vcpu_svm *svm = to_svm(vcpu);
  87
  88	WARN_ON(mmu_is_nested(vcpu));
  89
  90	vcpu->arch.mmu = &vcpu->arch.guest_mmu;
  91
  92	/*
  93	 * The NPT format depends on L1's CR4 and EFER, which is in vmcb01.  Note,
  94	 * when called via KVM_SET_NESTED_STATE, that state may _not_ match current
  95	 * vCPU state.  CR0.WP is explicitly ignored, while CR0.PG is required.
  96	 */
  97	kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4,
  98				svm->vmcb01.ptr->save.efer,
  99				svm->nested.ctl.nested_cr3);
 100	vcpu->arch.mmu->get_guest_pgd     = nested_svm_get_tdp_cr3;
 101	vcpu->arch.mmu->get_pdptr         = nested_svm_get_tdp_pdptr;
 102	vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
 103	vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
 104}
 105
 106static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
 107{
 108	vcpu->arch.mmu = &vcpu->arch.root_mmu;
 109	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
 110}
 111
 112static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm)
 113{
 114	if (!guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_V_VMSAVE_VMLOAD))
 115		return true;
 116
 117	if (!nested_npt_enabled(svm))
 118		return true;
 119
 120	if (!(svm->nested.ctl.virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK))
 121		return true;
 122
 123	return false;
 124}
 125
 126void recalc_intercepts(struct vcpu_svm *svm)
 127{
 128	struct vmcb_control_area *c, *h;
 129	struct vmcb_ctrl_area_cached *g;
 130	unsigned int i;
 131
 132	vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 133
 134	if (!is_guest_mode(&svm->vcpu))
 135		return;
 136
 137	c = &svm->vmcb->control;
 138	h = &svm->vmcb01.ptr->control;
 139	g = &svm->nested.ctl;
 140
 141	for (i = 0; i < MAX_INTERCEPT; i++)
 142		c->intercepts[i] = h->intercepts[i];
 143
 144	if (g->int_ctl & V_INTR_MASKING_MASK) {
 145		/*
 146		 * If L2 is active and V_INTR_MASKING is enabled in vmcb12,
 147		 * disable intercept of CR8 writes as L2's CR8 does not affect
 148		 * any interrupt KVM may want to inject.
 149		 *
 150		 * Similarly, disable intercept of virtual interrupts (used to
 151		 * detect interrupt windows) if the saved RFLAGS.IF is '0', as
 152		 * the effective RFLAGS.IF for L1 interrupts will never be set
 153		 * while L2 is running (L2's RFLAGS.IF doesn't affect L1 IRQs).
 154		 */
 155		vmcb_clr_intercept(c, INTERCEPT_CR8_WRITE);
 156		if (!(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF))
 157			vmcb_clr_intercept(c, INTERCEPT_VINTR);
 158	}
 159
 160	/*
 161	 * We want to see VMMCALLs from a nested guest only when Hyper-V L2 TLB
 162	 * flush feature is enabled.
 163	 */
 164	if (!nested_svm_l2_tlb_flush_enabled(&svm->vcpu))
 165		vmcb_clr_intercept(c, INTERCEPT_VMMCALL);
 166
 167	for (i = 0; i < MAX_INTERCEPT; i++)
 168		c->intercepts[i] |= g->intercepts[i];
 169
 170	/* If SMI is not intercepted, ignore guest SMI intercept as well  */
 171	if (!intercept_smi)
 172		vmcb_clr_intercept(c, INTERCEPT_SMI);
 173
 174	if (nested_vmcb_needs_vls_intercept(svm)) {
 175		/*
 176		 * If the virtual VMLOAD/VMSAVE is not enabled for the L2,
 177		 * we must intercept these instructions to correctly
 178		 * emulate them in case L1 doesn't intercept them.
 179		 */
 180		vmcb_set_intercept(c, INTERCEPT_VMLOAD);
 181		vmcb_set_intercept(c, INTERCEPT_VMSAVE);
 182	} else {
 183		WARN_ON(!(c->virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK));
 184	}
 185}
 186
 187/*
 188 * This array (and its actual size) holds the set of offsets (indexing by chunk
 189 * size) to process when merging vmcb12's MSRPM with vmcb01's MSRPM.  Note, the
 190 * set of MSRs for which interception is disabled in vmcb01 is per-vCPU, e.g.
 191 * based on CPUID features.  This array only tracks MSRs that *might* be passed
 192 * through to the guest.
 193 *
 194 * Hardcode the capacity of the array based on the maximum number of _offsets_.
 195 * MSRs are batched together, so there are fewer offsets than MSRs.
 196 */
 197static int nested_svm_msrpm_merge_offsets[7] __ro_after_init;
 198static int nested_svm_nr_msrpm_merge_offsets __ro_after_init;
 199typedef unsigned long nsvm_msrpm_merge_t;
 200
 201int __init nested_svm_init_msrpm_merge_offsets(void)
 202{
 203	static const u32 merge_msrs[] __initconst = {
 204		MSR_STAR,
 205		MSR_IA32_SYSENTER_CS,
 206		MSR_IA32_SYSENTER_EIP,
 207		MSR_IA32_SYSENTER_ESP,
 208	#ifdef CONFIG_X86_64
 209		MSR_GS_BASE,
 210		MSR_FS_BASE,
 211		MSR_KERNEL_GS_BASE,
 212		MSR_LSTAR,
 213		MSR_CSTAR,
 214		MSR_SYSCALL_MASK,
 215	#endif
 216		MSR_IA32_SPEC_CTRL,
 217		MSR_IA32_PRED_CMD,
 218		MSR_IA32_FLUSH_CMD,
 219		MSR_IA32_APERF,
 220		MSR_IA32_MPERF,
 221		MSR_IA32_LASTBRANCHFROMIP,
 222		MSR_IA32_LASTBRANCHTOIP,
 223		MSR_IA32_LASTINTFROMIP,
 224		MSR_IA32_LASTINTTOIP,
 225	};
 226	int i, j;
 227
 228	for (i = 0; i < ARRAY_SIZE(merge_msrs); i++) {
 229		int bit_nr = svm_msrpm_bit_nr(merge_msrs[i]);
 230		u32 offset;
 231
 232		if (WARN_ON(bit_nr < 0))
 233			return -EIO;
 234
 235		/*
 236		 * Merging is done in chunks to reduce the number of accesses
 237		 * to L1's bitmap.
 238		 */
 239		offset = bit_nr / BITS_PER_BYTE / sizeof(nsvm_msrpm_merge_t);
 240
 241		for (j = 0; j < nested_svm_nr_msrpm_merge_offsets; j++) {
 242			if (nested_svm_msrpm_merge_offsets[j] == offset)
 243				break;
 244		}
 245
 246		if (j < nested_svm_nr_msrpm_merge_offsets)
 247			continue;
 248
 249		if (WARN_ON(j >= ARRAY_SIZE(nested_svm_msrpm_merge_offsets)))
 250			return -EIO;
 251
 252		nested_svm_msrpm_merge_offsets[j] = offset;
 253		nested_svm_nr_msrpm_merge_offsets++;
 254	}
 255
 256	return 0;
 257}
 258
 259/*
 260 * Merge L0's (KVM) and L1's (Nested VMCB) MSR permission bitmaps. The function
 261 * is optimized in that it only merges the parts where KVM MSR permission bitmap
 262 * may contain zero bits.
 263 */
 264static bool nested_svm_merge_msrpm(struct kvm_vcpu *vcpu)
 265{
 266	struct vcpu_svm *svm = to_svm(vcpu);
 267	nsvm_msrpm_merge_t *msrpm02 = svm->nested.msrpm;
 268	nsvm_msrpm_merge_t *msrpm01 = svm->msrpm;
 269	int i;
 270
 271	/*
 272	 * MSR bitmap update can be skipped when:
 273	 * - MSR bitmap for L1 hasn't changed.
 274	 * - Nested hypervisor (L1) is attempting to launch the same L2 as
 275	 *   before.
 276	 * - Nested hypervisor (L1) is using Hyper-V emulation interface and
 277	 * tells KVM (L0) there were no changes in MSR bitmap for L2.
 278	 */
 279#ifdef CONFIG_KVM_HYPERV
 280	if (!svm->nested.force_msr_bitmap_recalc) {
 281		struct hv_vmcb_enlightenments *hve = &svm->nested.ctl.hv_enlightenments;
 282
 283		if (kvm_hv_hypercall_enabled(vcpu) &&
 284		    hve->hv_enlightenments_control.msr_bitmap &&
 285		    (svm->nested.ctl.clean & BIT(HV_VMCB_NESTED_ENLIGHTENMENTS)))
 286			goto set_msrpm_base_pa;
 287	}
 288#endif
 289
 290	if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))
 291		return true;
 292
 293	for (i = 0; i < nested_svm_nr_msrpm_merge_offsets; i++) {
 294		const int p = nested_svm_msrpm_merge_offsets[i];
 295		nsvm_msrpm_merge_t l1_val;
 296		gpa_t gpa;
 297
 298		gpa = svm->nested.ctl.msrpm_base_pa + (p * sizeof(l1_val));
 299
 300		if (kvm_vcpu_read_guest(vcpu, gpa, &l1_val, sizeof(l1_val)))
 301			return false;
 302
 303		msrpm02[p] = msrpm01[p] | l1_val;
 304	}
 305
 306	svm->nested.force_msr_bitmap_recalc = false;
 307
 308#ifdef CONFIG_KVM_HYPERV
 309set_msrpm_base_pa:
 310#endif
 311	svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
 312
 313	return true;
 314}
 315
 316/*
 317 * Bits 11:0 of bitmap address are ignored by hardware
 318 */
 319static bool nested_svm_check_bitmap_pa(struct kvm_vcpu *vcpu, u64 pa, u32 size)
 320{
 321	u64 addr = PAGE_ALIGN(pa);
 322
 323	return kvm_vcpu_is_legal_gpa(vcpu, addr) &&
 324	    kvm_vcpu_is_legal_gpa(vcpu, addr + size - 1);
 325}
 326
 327static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
 328					 struct vmcb_ctrl_area_cached *control)
 329{
 330	if (CC(!vmcb12_is_intercept(control, INTERCEPT_VMRUN)))
 331		return false;
 332
 333	if (CC(control->asid == 0))
 334		return false;
 335
 336	if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled))
 337		return false;
 338
 339	if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa,
 340					   MSRPM_SIZE)))
 341		return false;
 342	if (CC(!nested_svm_check_bitmap_pa(vcpu, control->iopm_base_pa,
 343					   IOPM_SIZE)))
 344		return false;
 345
 346	if (CC((control->int_ctl & V_NMI_ENABLE_MASK) &&
 347	       !vmcb12_is_intercept(control, INTERCEPT_NMI))) {
 348		return false;
 349	}
 350
 351	return true;
 352}
 353
 354/* Common checks that apply to both L1 and L2 state.  */
 355static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu,
 356				     struct vmcb_save_area_cached *save)
 357{
 358	if (CC(!(save->efer & EFER_SVME)))
 359		return false;
 360
 361	if (CC((save->cr0 & X86_CR0_CD) == 0 && (save->cr0 & X86_CR0_NW)) ||
 362	    CC(save->cr0 & ~0xffffffffULL))
 363		return false;
 364
 365	if (CC(!kvm_dr6_valid(save->dr6)) || CC(!kvm_dr7_valid(save->dr7)))
 366		return false;
 367
 368	/*
 369	 * These checks are also performed by KVM_SET_SREGS,
 370	 * except that EFER.LMA is not checked by SVM against
 371	 * CR0.PG && EFER.LME.
 372	 */
 373	if ((save->efer & EFER_LME) && (save->cr0 & X86_CR0_PG)) {
 374		if (CC(!(save->cr4 & X86_CR4_PAE)) ||
 375		    CC(!(save->cr0 & X86_CR0_PE)) ||
 376		    CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3)))
 377			return false;
 378	}
 379
 380	/* Note, SVM doesn't have any additional restrictions on CR4. */
 381	if (CC(!__kvm_is_valid_cr4(vcpu, save->cr4)))
 382		return false;
 383
 384	if (CC(!kvm_valid_efer(vcpu, save->efer)))
 385		return false;
 386
 387	return true;
 388}
 389
 390static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu)
 391{
 392	struct vcpu_svm *svm = to_svm(vcpu);
 393	struct vmcb_save_area_cached *save = &svm->nested.save;
 394
 395	return __nested_vmcb_check_save(vcpu, save);
 396}
 397
 398static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu)
 399{
 400	struct vcpu_svm *svm = to_svm(vcpu);
 401	struct vmcb_ctrl_area_cached *ctl = &svm->nested.ctl;
 402
 403	return __nested_vmcb_check_controls(vcpu, ctl);
 404}
 405
 406static
 407void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu,
 408					 struct vmcb_ctrl_area_cached *to,
 409					 struct vmcb_control_area *from)
 410{
 411	unsigned int i;
 412
 413	for (i = 0; i < MAX_INTERCEPT; i++)
 414		to->intercepts[i] = from->intercepts[i];
 415
 416	to->iopm_base_pa        = from->iopm_base_pa;
 417	to->msrpm_base_pa       = from->msrpm_base_pa;
 418	to->tsc_offset          = from->tsc_offset;
 419	to->tlb_ctl             = from->tlb_ctl;
 420	to->int_ctl             = from->int_ctl;
 421	to->int_vector          = from->int_vector;
 422	to->int_state           = from->int_state;
 423	to->exit_code           = from->exit_code;
 424	to->exit_code_hi        = from->exit_code_hi;
 425	to->exit_info_1         = from->exit_info_1;
 426	to->exit_info_2         = from->exit_info_2;
 427	to->exit_int_info       = from->exit_int_info;
 428	to->exit_int_info_err   = from->exit_int_info_err;
 429	to->nested_ctl          = from->nested_ctl;
 430	to->event_inj           = from->event_inj;
 431	to->event_inj_err       = from->event_inj_err;
 432	to->next_rip            = from->next_rip;
 433	to->nested_cr3          = from->nested_cr3;
 434	to->virt_ext            = from->virt_ext;
 435	to->pause_filter_count  = from->pause_filter_count;
 436	to->pause_filter_thresh = from->pause_filter_thresh;
 437
 438	/* Copy asid here because nested_vmcb_check_controls will check it.  */
 439	to->asid           = from->asid;
 440	to->msrpm_base_pa &= ~0x0fffULL;
 441	to->iopm_base_pa  &= ~0x0fffULL;
 442
 443#ifdef CONFIG_KVM_HYPERV
 444	/* Hyper-V extensions (Enlightened VMCB) */
 445	if (kvm_hv_hypercall_enabled(vcpu)) {
 446		to->clean = from->clean;
 447		memcpy(&to->hv_enlightenments, &from->hv_enlightenments,
 448		       sizeof(to->hv_enlightenments));
 449	}
 450#endif
 451}
 452
 453void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm,
 454				       struct vmcb_control_area *control)
 455{
 456	__nested_copy_vmcb_control_to_cache(&svm->vcpu, &svm->nested.ctl, control);
 457}
 458
 459static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to,
 460					     struct vmcb_save_area *from)
 461{
 462	/*
 463	 * Copy only fields that are validated, as we need them
 464	 * to avoid TOC/TOU races.
 465	 */
 466	to->efer = from->efer;
 467	to->cr0 = from->cr0;
 468	to->cr3 = from->cr3;
 469	to->cr4 = from->cr4;
 470
 471	to->dr6 = from->dr6;
 472	to->dr7 = from->dr7;
 473}
 474
 475void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm,
 476				    struct vmcb_save_area *save)
 477{
 478	__nested_copy_vmcb_save_to_cache(&svm->nested.save, save);
 479}
 480
 481/*
 482 * Synchronize fields that are written by the processor, so that
 483 * they can be copied back into the vmcb12.
 484 */
 485void nested_sync_control_from_vmcb02(struct vcpu_svm *svm)
 486{
 487	u32 mask;
 488	svm->nested.ctl.event_inj      = svm->vmcb->control.event_inj;
 489	svm->nested.ctl.event_inj_err  = svm->vmcb->control.event_inj_err;
 490
 491	/* Only a few fields of int_ctl are written by the processor.  */
 492	mask = V_IRQ_MASK | V_TPR_MASK;
 493	/*
 494	 * Don't sync vmcb02 V_IRQ back to vmcb12 if KVM (L0) is intercepting
 495	 * virtual interrupts in order to request an interrupt window, as KVM
 496	 * has usurped vmcb02's int_ctl.  If an interrupt window opens before
 497	 * the next VM-Exit, svm_clear_vintr() will restore vmcb12's int_ctl.
 498	 * If no window opens, V_IRQ will be correctly preserved in vmcb12's
 499	 * int_ctl (because it was never recognized while L2 was running).
 500	 */
 501	if (svm_is_intercept(svm, INTERCEPT_VINTR) &&
 502	    !test_bit(INTERCEPT_VINTR, (unsigned long *)svm->nested.ctl.intercepts))
 503		mask &= ~V_IRQ_MASK;
 504
 505	if (nested_vgif_enabled(svm))
 506		mask |= V_GIF_MASK;
 507
 508	if (nested_vnmi_enabled(svm))
 509		mask |= V_NMI_BLOCKING_MASK | V_NMI_PENDING_MASK;
 510
 511	svm->nested.ctl.int_ctl        &= ~mask;
 512	svm->nested.ctl.int_ctl        |= svm->vmcb->control.int_ctl & mask;
 513}
 514
 515/*
 516 * Transfer any event that L0 or L1 wanted to inject into L2 to
 517 * EXIT_INT_INFO.
 518 */
 519static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm,
 520						struct vmcb *vmcb12)
 521{
 522	struct kvm_vcpu *vcpu = &svm->vcpu;
 523	u32 exit_int_info = 0;
 524	unsigned int nr;
 525
 526	if (vcpu->arch.exception.injected) {
 527		nr = vcpu->arch.exception.vector;
 528		exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT;
 529
 530		if (vcpu->arch.exception.has_error_code) {
 531			exit_int_info |= SVM_EVTINJ_VALID_ERR;
 532			vmcb12->control.exit_int_info_err =
 533				vcpu->arch.exception.error_code;
 534		}
 535
 536	} else if (vcpu->arch.nmi_injected) {
 537		exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
 538
 539	} else if (vcpu->arch.interrupt.injected) {
 540		nr = vcpu->arch.interrupt.nr;
 541		exit_int_info = nr | SVM_EVTINJ_VALID;
 542
 543		if (vcpu->arch.interrupt.soft)
 544			exit_int_info |= SVM_EVTINJ_TYPE_SOFT;
 545		else
 546			exit_int_info |= SVM_EVTINJ_TYPE_INTR;
 547	}
 548
 549	vmcb12->control.exit_int_info = exit_int_info;
 550}
 551
 552static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu)
 553{
 554	/* Handle pending Hyper-V TLB flush requests */
 555	kvm_hv_nested_transtion_tlb_flush(vcpu, npt_enabled);
 556
 557	/*
 558	 * TODO: optimize unconditional TLB flush/MMU sync.  A partial list of
 559	 * things to fix before this can be conditional:
 560	 *
 561	 *  - Flush TLBs for both L1 and L2 remote TLB flush
 562	 *  - Honor L1's request to flush an ASID on nested VMRUN
 563	 *  - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*]
 564	 *  - Don't crush a pending TLB flush in vmcb02 on nested VMRUN
 565	 *  - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST
 566	 *
 567	 * [*] Unlike nested EPT, SVM's ASID management can invalidate nested
 568	 *     NPT guest-physical mappings on VMRUN.
 569	 */
 570	kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
 571	kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
 572}
 573
 574/*
 575 * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true
 576 * if we are emulating VM-Entry into a guest with NPT enabled.
 577 */
 578static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
 579			       bool nested_npt, bool reload_pdptrs)
 580{
 581	if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3)))
 582		return -EINVAL;
 583
 584	if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) &&
 585	    CC(!load_pdptrs(vcpu, cr3)))
 586		return -EINVAL;
 587
 588	vcpu->arch.cr3 = cr3;
 589
 590	/* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */
 591	kvm_init_mmu(vcpu);
 592
 593	if (!nested_npt)
 594		kvm_mmu_new_pgd(vcpu, cr3);
 595
 596	return 0;
 597}
 598
 599void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm)
 600{
 601	if (!svm->nested.vmcb02.ptr)
 602		return;
 603
 604	/* FIXME: merge g_pat from vmcb01 and vmcb12.  */
 605	svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat;
 606}
 607
 608static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12)
 609{
 610	bool new_vmcb12 = false;
 611	struct vmcb *vmcb01 = svm->vmcb01.ptr;
 612	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
 613	struct kvm_vcpu *vcpu = &svm->vcpu;
 614
 615	nested_vmcb02_compute_g_pat(svm);
 616	vmcb_mark_dirty(vmcb02, VMCB_NPT);
 617
 618	/* Load the nested guest state */
 619	if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) {
 620		new_vmcb12 = true;
 621		svm->nested.last_vmcb12_gpa = svm->nested.vmcb12_gpa;
 622		svm->nested.force_msr_bitmap_recalc = true;
 623	}
 624
 625	if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) {
 626		vmcb02->save.es = vmcb12->save.es;
 627		vmcb02->save.cs = vmcb12->save.cs;
 628		vmcb02->save.ss = vmcb12->save.ss;
 629		vmcb02->save.ds = vmcb12->save.ds;
 630		vmcb02->save.cpl = vmcb12->save.cpl;
 631		vmcb_mark_dirty(vmcb02, VMCB_SEG);
 632	}
 633
 634	if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) {
 635		vmcb02->save.gdtr = vmcb12->save.gdtr;
 636		vmcb02->save.idtr = vmcb12->save.idtr;
 637		vmcb_mark_dirty(vmcb02, VMCB_DT);
 638	}
 639
 640	if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
 641	    (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_CET)))) {
 642		vmcb02->save.s_cet  = vmcb12->save.s_cet;
 643		vmcb02->save.isst_addr = vmcb12->save.isst_addr;
 644		vmcb02->save.ssp = vmcb12->save.ssp;
 645		vmcb_mark_dirty(vmcb02, VMCB_CET);
 646	}
 647
 648	kvm_set_rflags(vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED);
 649
 650	svm_set_efer(vcpu, svm->nested.save.efer);
 651
 652	svm_set_cr0(vcpu, svm->nested.save.cr0);
 653	svm_set_cr4(vcpu, svm->nested.save.cr4);
 654
 655	svm->vcpu.arch.cr2 = vmcb12->save.cr2;
 656
 657	kvm_rax_write(vcpu, vmcb12->save.rax);
 658	kvm_rsp_write(vcpu, vmcb12->save.rsp);
 659	kvm_rip_write(vcpu, vmcb12->save.rip);
 660
 661	/* In case we don't even reach vcpu_run, the fields are not updated */
 662	vmcb02->save.rax = vmcb12->save.rax;
 663	vmcb02->save.rsp = vmcb12->save.rsp;
 664	vmcb02->save.rip = vmcb12->save.rip;
 665
 666	/* These bits will be set properly on the first execution when new_vmc12 is true */
 667	if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) {
 668		vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1;
 669		svm->vcpu.arch.dr6  = svm->nested.save.dr6 | DR6_ACTIVE_LOW;
 670		vmcb_mark_dirty(vmcb02, VMCB_DR);
 671	}
 672
 673	if (unlikely(guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
 674		     (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) {
 675		/*
 676		 * Reserved bits of DEBUGCTL are ignored.  Be consistent with
 677		 * svm_set_msr's definition of reserved bits.
 678		 */
 679		svm_copy_lbrs(vmcb02, vmcb12);
 680		vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS;
 681	} else {
 682		svm_copy_lbrs(vmcb02, vmcb01);
 683	}
 684	svm_update_lbrv(&svm->vcpu);
 685}
 686
 687static inline bool is_evtinj_soft(u32 evtinj)
 688{
 689	u32 type = evtinj & SVM_EVTINJ_TYPE_MASK;
 690	u8 vector = evtinj & SVM_EVTINJ_VEC_MASK;
 691
 692	if (!(evtinj & SVM_EVTINJ_VALID))
 693		return false;
 694
 695	if (type == SVM_EVTINJ_TYPE_SOFT)
 696		return true;
 697
 698	return type == SVM_EVTINJ_TYPE_EXEPT && kvm_exception_is_soft(vector);
 699}
 700
 701static bool is_evtinj_nmi(u32 evtinj)
 702{
 703	u32 type = evtinj & SVM_EVTINJ_TYPE_MASK;
 704
 705	if (!(evtinj & SVM_EVTINJ_VALID))
 706		return false;
 707
 708	return type == SVM_EVTINJ_TYPE_NMI;
 709}
 710
 711static void nested_vmcb02_prepare_control(struct vcpu_svm *svm,
 712					  unsigned long vmcb12_rip,
 713					  unsigned long vmcb12_csbase)
 714{
 715	u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK;
 716	u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK;
 717
 718	struct kvm_vcpu *vcpu = &svm->vcpu;
 719	struct vmcb *vmcb01 = svm->vmcb01.ptr;
 720	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
 721	u32 pause_count12;
 722	u32 pause_thresh12;
 723
 724	nested_svm_transition_tlb_flush(vcpu);
 725
 726	/* Enter Guest-Mode */
 727	enter_guest_mode(vcpu);
 728
 729	/*
 730	 * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2,
 731	 * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes.
 732	 */
 733
 734	if (guest_cpu_cap_has(vcpu, X86_FEATURE_VGIF) &&
 735	    (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK))
 736		int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK);
 737	else
 738		int_ctl_vmcb01_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK);
 739
 740	if (vnmi) {
 741		if (vmcb01->control.int_ctl & V_NMI_PENDING_MASK) {
 742			svm->vcpu.arch.nmi_pending++;
 743			kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
 744		}
 745		if (nested_vnmi_enabled(svm))
 746			int_ctl_vmcb12_bits |= (V_NMI_PENDING_MASK |
 747						V_NMI_ENABLE_MASK |
 748						V_NMI_BLOCKING_MASK);
 749	}
 750
 751	/* Copied from vmcb01.  msrpm_base can be overwritten later.  */
 752	vmcb02->control.nested_ctl = vmcb01->control.nested_ctl;
 753	vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa;
 754	vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa;
 755	vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP);
 756
 757	/*
 758	 * Stash vmcb02's counter if the guest hasn't moved past the guilty
 759	 * instruction; otherwise, reset the counter to '0'.
 760	 *
 761	 * In order to detect if L2 has made forward progress or not, track the
 762	 * RIP at which a bus lock has occurred on a per-vmcb12 basis.  If RIP
 763	 * is changed, guest has clearly made forward progress, bus_lock_counter
 764	 * still remained '1', so reset bus_lock_counter to '0'. Eg. In the
 765	 * scenario, where a buslock happened in L1 before VMRUN, the bus lock
 766	 * firmly happened on an instruction in the past. Even if vmcb01's
 767	 * counter is still '1', (because the guilty instruction got patched),
 768	 * the vCPU has clearly made forward progress and so KVM should reset
 769	 * vmcb02's counter to '0'.
 770	 *
 771	 * If the RIP hasn't changed, stash the bus lock counter at nested VMRUN
 772	 * to prevent the same guilty instruction from triggering a VM-Exit. Eg.
 773	 * if userspace rate-limits the vCPU, then it's entirely possible that
 774	 * L1's tick interrupt is pending by the time userspace re-runs the
 775	 * vCPU.  If KVM unconditionally clears the counter on VMRUN, then when
 776	 * L1 re-enters L2, the same instruction will trigger a VM-Exit and the
 777	 * entire cycle start over.
 778	 */
 779	if (vmcb02->save.rip && (svm->nested.ctl.bus_lock_rip == vmcb02->save.rip))
 780		vmcb02->control.bus_lock_counter = 1;
 781	else
 782		vmcb02->control.bus_lock_counter = 0;
 783
 784	/* Done at vmrun: asid.  */
 785
 786	/* Also overwritten later if necessary.  */
 787	vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
 788
 789	/* nested_cr3.  */
 790	if (nested_npt_enabled(svm))
 791		nested_svm_init_mmu_context(vcpu);
 792
 793	vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(
 794			vcpu->arch.l1_tsc_offset,
 795			svm->nested.ctl.tsc_offset,
 796			svm->tsc_ratio_msr);
 797
 798	vmcb02->control.tsc_offset = vcpu->arch.tsc_offset;
 799
 800	if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSCRATEMSR) &&
 801	    svm->tsc_ratio_msr != kvm_caps.default_tsc_scaling_ratio)
 802		nested_svm_update_tsc_ratio_msr(vcpu);
 803
 804	vmcb02->control.int_ctl             =
 805		(svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) |
 806		(vmcb01->control.int_ctl & int_ctl_vmcb01_bits);
 807
 808	vmcb02->control.int_vector          = svm->nested.ctl.int_vector;
 809	vmcb02->control.int_state           = svm->nested.ctl.int_state;
 810	vmcb02->control.event_inj           = svm->nested.ctl.event_inj;
 811	vmcb02->control.event_inj_err       = svm->nested.ctl.event_inj_err;
 812
 813	/*
 814	 * next_rip is consumed on VMRUN as the return address pushed on the
 815	 * stack for injected soft exceptions/interrupts.  If nrips is exposed
 816	 * to L1, take it verbatim from vmcb12.  If nrips is supported in
 817	 * hardware but not exposed to L1, stuff the actual L2 RIP to emulate
 818	 * what a nrips=0 CPU would do (L1 is responsible for advancing RIP
 819	 * prior to injecting the event).
 820	 */
 821	if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
 822		vmcb02->control.next_rip    = svm->nested.ctl.next_rip;
 823	else if (boot_cpu_has(X86_FEATURE_NRIPS))
 824		vmcb02->control.next_rip    = vmcb12_rip;
 825
 826	svm->nmi_l1_to_l2 = is_evtinj_nmi(vmcb02->control.event_inj);
 827	if (is_evtinj_soft(vmcb02->control.event_inj)) {
 828		svm->soft_int_injected = true;
 829		svm->soft_int_csbase = vmcb12_csbase;
 830		svm->soft_int_old_rip = vmcb12_rip;
 831		if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
 832			svm->soft_int_next_rip = svm->nested.ctl.next_rip;
 833		else
 834			svm->soft_int_next_rip = vmcb12_rip;
 835	}
 836
 837	/* LBR_CTL_ENABLE_MASK is controlled by svm_update_lbrv() */
 838
 839	if (!nested_vmcb_needs_vls_intercept(svm))
 840		vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
 841
 842	if (guest_cpu_cap_has(vcpu, X86_FEATURE_PAUSEFILTER))
 843		pause_count12 = svm->nested.ctl.pause_filter_count;
 844	else
 845		pause_count12 = 0;
 846	if (guest_cpu_cap_has(vcpu, X86_FEATURE_PFTHRESHOLD))
 847		pause_thresh12 = svm->nested.ctl.pause_filter_thresh;
 848	else
 849		pause_thresh12 = 0;
 850	if (kvm_pause_in_guest(svm->vcpu.kvm)) {
 851		/* use guest values since host doesn't intercept PAUSE */
 852		vmcb02->control.pause_filter_count = pause_count12;
 853		vmcb02->control.pause_filter_thresh = pause_thresh12;
 854
 855	} else {
 856		/* start from host values otherwise */
 857		vmcb02->control.pause_filter_count = vmcb01->control.pause_filter_count;
 858		vmcb02->control.pause_filter_thresh = vmcb01->control.pause_filter_thresh;
 859
 860		/* ... but ensure filtering is disabled if so requested.  */
 861		if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE)) {
 862			if (!pause_count12)
 863				vmcb02->control.pause_filter_count = 0;
 864			if (!pause_thresh12)
 865				vmcb02->control.pause_filter_thresh = 0;
 866		}
 867	}
 868
 869	/*
 870	 * Merge guest and host intercepts - must be called with vcpu in
 871	 * guest-mode to take effect.
 872	 */
 873	recalc_intercepts(svm);
 874}
 875
 876static void nested_svm_copy_common_state(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
 877{
 878	/*
 879	 * Some VMCB state is shared between L1 and L2 and thus has to be
 880	 * moved at the time of nested vmrun and vmexit.
 881	 *
 882	 * VMLOAD/VMSAVE state would also belong in this category, but KVM
 883	 * always performs VMLOAD and VMSAVE from the VMCB01.
 884	 */
 885	to_vmcb->save.spec_ctrl = from_vmcb->save.spec_ctrl;
 886}
 887
 888int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa,
 889			 struct vmcb *vmcb12, bool from_vmrun)
 890{
 891	struct vcpu_svm *svm = to_svm(vcpu);
 892	int ret;
 893
 894	trace_kvm_nested_vmenter(svm->vmcb->save.rip,
 895				 vmcb12_gpa,
 896				 vmcb12->save.rip,
 897				 vmcb12->control.int_ctl,
 898				 vmcb12->control.event_inj,
 899				 vmcb12->control.nested_ctl,
 900				 vmcb12->control.nested_cr3,
 901				 vmcb12->save.cr3,
 902				 KVM_ISA_SVM);
 903
 904	trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff,
 905				    vmcb12->control.intercepts[INTERCEPT_CR] >> 16,
 906				    vmcb12->control.intercepts[INTERCEPT_EXCEPTION],
 907				    vmcb12->control.intercepts[INTERCEPT_WORD3],
 908				    vmcb12->control.intercepts[INTERCEPT_WORD4],
 909				    vmcb12->control.intercepts[INTERCEPT_WORD5]);
 910
 911
 912	svm->nested.vmcb12_gpa = vmcb12_gpa;
 913
 914	WARN_ON(svm->vmcb == svm->nested.vmcb02.ptr);
 915
 916	nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr);
 917
 918	svm_switch_vmcb(svm, &svm->nested.vmcb02);
 919	nested_vmcb02_prepare_control(svm, vmcb12->save.rip, vmcb12->save.cs.base);
 920	nested_vmcb02_prepare_save(svm, vmcb12);
 921
 922	ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3,
 923				  nested_npt_enabled(svm), from_vmrun);
 924	if (ret)
 925		return ret;
 926
 927	if (!from_vmrun)
 928		kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
 929
 930	svm_set_gif(svm, true);
 931
 932	if (kvm_vcpu_apicv_active(vcpu))
 933		kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
 934
 935	nested_svm_hv_update_vm_vp_ids(vcpu);
 936
 937	return 0;
 938}
 939
 940int nested_svm_vmrun(struct kvm_vcpu *vcpu)
 941{
 942	struct vcpu_svm *svm = to_svm(vcpu);
 943	int ret;
 944	struct vmcb *vmcb12;
 945	struct kvm_host_map map;
 946	u64 vmcb12_gpa;
 947	struct vmcb *vmcb01 = svm->vmcb01.ptr;
 948
 949	if (!svm->nested.hsave_msr) {
 950		kvm_inject_gp(vcpu, 0);
 951		return 1;
 952	}
 953
 954	if (is_smm(vcpu)) {
 955		kvm_queue_exception(vcpu, UD_VECTOR);
 956		return 1;
 957	}
 958
 959	/* This fails when VP assist page is enabled but the supplied GPA is bogus */
 960	ret = kvm_hv_verify_vp_assist(vcpu);
 961	if (ret) {
 962		kvm_inject_gp(vcpu, 0);
 963		return ret;
 964	}
 965
 966	vmcb12_gpa = svm->vmcb->save.rax;
 967	ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map);
 968	if (ret == -EINVAL) {
 969		kvm_inject_gp(vcpu, 0);
 970		return 1;
 971	} else if (ret) {
 972		return kvm_skip_emulated_instruction(vcpu);
 973	}
 974
 975	ret = kvm_skip_emulated_instruction(vcpu);
 976
 977	vmcb12 = map.hva;
 978
 979	if (WARN_ON_ONCE(!svm->nested.initialized))
 980		return -EINVAL;
 981
 982	nested_copy_vmcb_control_to_cache(svm, &vmcb12->control);
 983	nested_copy_vmcb_save_to_cache(svm, &vmcb12->save);
 984
 985	if (!nested_vmcb_check_save(vcpu) ||
 986	    !nested_vmcb_check_controls(vcpu)) {
 987		vmcb12->control.exit_code    = SVM_EXIT_ERR;
 988		vmcb12->control.exit_code_hi = -1u;
 989		vmcb12->control.exit_info_1  = 0;
 990		vmcb12->control.exit_info_2  = 0;
 991		goto out;
 992	}
 993
 994	/*
 995	 * Since vmcb01 is not in use, we can use it to store some of the L1
 996	 * state.
 997	 */
 998	vmcb01->save.efer   = vcpu->arch.efer;
 999	vmcb01->save.cr0    = kvm_read_cr0(vcpu);
1000	vmcb01->save.cr4    = vcpu->arch.cr4;
1001	vmcb01->save.rflags = kvm_get_rflags(vcpu);
1002	vmcb01->save.rip    = kvm_rip_read(vcpu);
1003
1004	if (!npt_enabled)
1005		vmcb01->save.cr3 = kvm_read_cr3(vcpu);
1006
1007	svm->nested.nested_run_pending = 1;
1008
1009	if (enter_svm_guest_mode(vcpu, vmcb12_gpa, vmcb12, true))
1010		goto out_exit_err;
1011
1012	if (nested_svm_merge_msrpm(vcpu))
1013		goto out;
1014
1015out_exit_err:
1016	svm->nested.nested_run_pending = 0;
1017	svm->nmi_l1_to_l2 = false;
1018	svm->soft_int_injected = false;
1019
1020	svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
1021	svm->vmcb->control.exit_code_hi = -1u;
1022	svm->vmcb->control.exit_info_1  = 0;
1023	svm->vmcb->control.exit_info_2  = 0;
1024
1025	nested_svm_vmexit(svm);
1026
1027out:
1028	kvm_vcpu_unmap(vcpu, &map);
1029
1030	return ret;
1031}
1032
1033/* Copy state save area fields which are handled by VMRUN */
1034void svm_copy_vmrun_state(struct vmcb_save_area *to_save,
1035			  struct vmcb_save_area *from_save)
1036{
1037	to_save->es = from_save->es;
1038	to_save->cs = from_save->cs;
1039	to_save->ss = from_save->ss;
1040	to_save->ds = from_save->ds;
1041	to_save->gdtr = from_save->gdtr;
1042	to_save->idtr = from_save->idtr;
1043	to_save->rflags = from_save->rflags | X86_EFLAGS_FIXED;
1044	to_save->efer = from_save->efer;
1045	to_save->cr0 = from_save->cr0;
1046	to_save->cr3 = from_save->cr3;
1047	to_save->cr4 = from_save->cr4;
1048	to_save->rax = from_save->rax;
1049	to_save->rsp = from_save->rsp;
1050	to_save->rip = from_save->rip;
1051	to_save->cpl = 0;
1052
1053	if (kvm_cpu_cap_has(X86_FEATURE_SHSTK)) {
1054		to_save->s_cet  = from_save->s_cet;
1055		to_save->isst_addr = from_save->isst_addr;
1056		to_save->ssp = from_save->ssp;
1057	}
1058}
1059
1060void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb)
1061{
1062	to_vmcb->save.fs = from_vmcb->save.fs;
1063	to_vmcb->save.gs = from_vmcb->save.gs;
1064	to_vmcb->save.tr = from_vmcb->save.tr;
1065	to_vmcb->save.ldtr = from_vmcb->save.ldtr;
1066	to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
1067	to_vmcb->save.star = from_vmcb->save.star;
1068	to_vmcb->save.lstar = from_vmcb->save.lstar;
1069	to_vmcb->save.cstar = from_vmcb->save.cstar;
1070	to_vmcb->save.sfmask = from_vmcb->save.sfmask;
1071	to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
1072	to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
1073	to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
1074}
1075
1076int nested_svm_vmexit(struct vcpu_svm *svm)
1077{
1078	struct kvm_vcpu *vcpu = &svm->vcpu;
1079	struct vmcb *vmcb01 = svm->vmcb01.ptr;
1080	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
1081	struct vmcb *vmcb12;
1082	struct kvm_host_map map;
1083	int rc;
1084
1085	rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map);
1086	if (rc) {
1087		if (rc == -EINVAL)
1088			kvm_inject_gp(vcpu, 0);
1089		return 1;
1090	}
1091
1092	vmcb12 = map.hva;
1093
1094	/* Exit Guest-Mode */
1095	leave_guest_mode(vcpu);
1096	svm->nested.vmcb12_gpa = 0;
1097	WARN_ON_ONCE(svm->nested.nested_run_pending);
1098
1099	kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
1100
1101	/* in case we halted in L2 */
1102	kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
1103
1104	/* Give the current vmcb to the guest */
1105
1106	vmcb12->save.es     = vmcb02->save.es;
1107	vmcb12->save.cs     = vmcb02->save.cs;
1108	vmcb12->save.ss     = vmcb02->save.ss;
1109	vmcb12->save.ds     = vmcb02->save.ds;
1110	vmcb12->save.gdtr   = vmcb02->save.gdtr;
1111	vmcb12->save.idtr   = vmcb02->save.idtr;
1112	vmcb12->save.efer   = svm->vcpu.arch.efer;
1113	vmcb12->save.cr0    = kvm_read_cr0(vcpu);
1114	vmcb12->save.cr3    = kvm_read_cr3(vcpu);
1115	vmcb12->save.cr2    = vmcb02->save.cr2;
1116	vmcb12->save.cr4    = svm->vcpu.arch.cr4;
1117	vmcb12->save.rflags = kvm_get_rflags(vcpu);
1118	vmcb12->save.rip    = kvm_rip_read(vcpu);
1119	vmcb12->save.rsp    = kvm_rsp_read(vcpu);
1120	vmcb12->save.rax    = kvm_rax_read(vcpu);
1121	vmcb12->save.dr7    = vmcb02->save.dr7;
1122	vmcb12->save.dr6    = svm->vcpu.arch.dr6;
1123	vmcb12->save.cpl    = vmcb02->save.cpl;
1124
1125	if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) {
1126		vmcb12->save.s_cet	= vmcb02->save.s_cet;
1127		vmcb12->save.isst_addr	= vmcb02->save.isst_addr;
1128		vmcb12->save.ssp	= vmcb02->save.ssp;
1129	}
1130
1131	vmcb12->control.int_state         = vmcb02->control.int_state;
1132	vmcb12->control.exit_code         = vmcb02->control.exit_code;
1133	vmcb12->control.exit_code_hi      = vmcb02->control.exit_code_hi;
1134	vmcb12->control.exit_info_1       = vmcb02->control.exit_info_1;
1135	vmcb12->control.exit_info_2       = vmcb02->control.exit_info_2;
1136
1137	if (vmcb12->control.exit_code != SVM_EXIT_ERR)
1138		nested_save_pending_event_to_vmcb12(svm, vmcb12);
1139
1140	if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS))
1141		vmcb12->control.next_rip  = vmcb02->control.next_rip;
1142
1143	vmcb12->control.int_ctl           = svm->nested.ctl.int_ctl;
1144	vmcb12->control.event_inj         = svm->nested.ctl.event_inj;
1145	vmcb12->control.event_inj_err     = svm->nested.ctl.event_inj_err;
1146
1147	if (!kvm_pause_in_guest(vcpu->kvm)) {
1148		vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count;
1149		vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS);
1150
1151	}
1152
1153	/*
1154	 * Invalidate bus_lock_rip unless KVM is still waiting for the guest
1155	 * to make forward progress before re-enabling bus lock detection.
1156	 */
1157	if (!vmcb02->control.bus_lock_counter)
1158		svm->nested.ctl.bus_lock_rip = INVALID_GPA;
1159
1160	nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr);
1161
1162	kvm_nested_vmexit_handle_ibrs(vcpu);
1163
1164	svm_switch_vmcb(svm, &svm->vmcb01);
1165
1166	/*
1167	 * Rules for synchronizing int_ctl bits from vmcb02 to vmcb01:
1168	 *
1169	 * V_IRQ, V_IRQ_VECTOR, V_INTR_PRIO_MASK, V_IGN_TPR:  If L1 doesn't
1170	 * intercept interrupts, then KVM will use vmcb02's V_IRQ (and related
1171	 * flags) to detect interrupt windows for L1 IRQs (even if L1 uses
1172	 * virtual interrupt masking).  Raise KVM_REQ_EVENT to ensure that
1173	 * KVM re-requests an interrupt window if necessary, which implicitly
1174	 * copies this bits from vmcb02 to vmcb01.
1175	 *
1176	 * V_TPR: If L1 doesn't use virtual interrupt masking, then L1's vTPR
1177	 * is stored in vmcb02, but its value doesn't need to be copied from/to
1178	 * vmcb01 because it is copied from/to the virtual APIC's TPR register
1179	 * on each VM entry/exit.
1180	 *
1181	 * V_GIF: If nested vGIF is not used, KVM uses vmcb02's V_GIF for L1's
1182	 * V_GIF.  However, GIF is architecturally clear on each VM exit, thus
1183	 * there is no need to copy V_GIF from vmcb02 to vmcb01.
1184	 */
1185	if (!nested_exit_on_intr(svm))
1186		kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
1187
1188	if (unlikely(guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
1189		     (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK)))
1190		svm_copy_lbrs(vmcb12, vmcb02);
1191	else
1192		svm_copy_lbrs(vmcb01, vmcb02);
1193
1194	svm_update_lbrv(vcpu);
1195
1196	if (vnmi) {
1197		if (vmcb02->control.int_ctl & V_NMI_BLOCKING_MASK)
1198			vmcb01->control.int_ctl |= V_NMI_BLOCKING_MASK;
1199		else
1200			vmcb01->control.int_ctl &= ~V_NMI_BLOCKING_MASK;
1201
1202		if (vcpu->arch.nmi_pending) {
1203			vcpu->arch.nmi_pending--;
1204			vmcb01->control.int_ctl |= V_NMI_PENDING_MASK;
1205		} else {
1206			vmcb01->control.int_ctl &= ~V_NMI_PENDING_MASK;
1207		}
1208	}
1209
1210	/*
1211	 * On vmexit the  GIF is set to false and
1212	 * no event can be injected in L1.
1213	 */
1214	svm_set_gif(svm, false);
1215	vmcb01->control.exit_int_info = 0;
1216
1217	svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset;
1218	if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) {
1219		vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset;
1220		vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS);
1221	}
1222
1223	if (kvm_caps.has_tsc_control &&
1224	    vcpu->arch.tsc_scaling_ratio != vcpu->arch.l1_tsc_scaling_ratio) {
1225		vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio;
1226		svm_write_tsc_multiplier(vcpu);
1227	}
1228
1229	svm->nested.ctl.nested_cr3 = 0;
1230
1231	/*
1232	 * Restore processor state that had been saved in vmcb01
1233	 */
1234	kvm_set_rflags(vcpu, vmcb01->save.rflags);
1235	svm_set_efer(vcpu, vmcb01->save.efer);
1236	svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE);
1237	svm_set_cr4(vcpu, vmcb01->save.cr4);
1238	kvm_rax_write(vcpu, vmcb01->save.rax);
1239	kvm_rsp_write(vcpu, vmcb01->save.rsp);
1240	kvm_rip_write(vcpu, vmcb01->save.rip);
1241
1242	svm->vcpu.arch.dr7 = DR7_FIXED_1;
1243	kvm_update_dr7(&svm->vcpu);
1244
1245	trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code,
1246				       vmcb12->control.exit_info_1,
1247				       vmcb12->control.exit_info_2,
1248				       vmcb12->control.exit_int_info,
1249				       vmcb12->control.exit_int_info_err,
1250				       KVM_ISA_SVM);
1251
1252	kvm_vcpu_unmap(vcpu, &map);
1253
1254	nested_svm_transition_tlb_flush(vcpu);
1255
1256	nested_svm_uninit_mmu_context(vcpu);
1257
1258	rc = nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true);
1259	if (rc)
1260		return 1;
1261
1262	/*
1263	 * Drop what we picked up for L2 via svm_complete_interrupts() so it
1264	 * doesn't end up in L1.
1265	 */
1266	svm->vcpu.arch.nmi_injected = false;
1267	kvm_clear_exception_queue(vcpu);
1268	kvm_clear_interrupt_queue(vcpu);
1269
1270	/*
1271	 * If we are here following the completion of a VMRUN that
1272	 * is being single-stepped, queue the pending #DB intercept
1273	 * right now so that it an be accounted for before we execute
1274	 * L1's next instruction.
1275	 */
1276	if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF))
1277		kvm_queue_exception(&(svm->vcpu), DB_VECTOR);
1278
1279	/*
1280	 * Un-inhibit the AVIC right away, so that other vCPUs can start
1281	 * to benefit from it right away.
1282	 */
1283	if (kvm_apicv_activated(vcpu->kvm))
1284		__kvm_vcpu_update_apicv(vcpu);
1285
1286	return 0;
1287}
1288
1289static void nested_svm_triple_fault(struct kvm_vcpu *vcpu)
1290{
1291	struct vcpu_svm *svm = to_svm(vcpu);
1292
1293	if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SHUTDOWN))
1294		return;
1295
1296	kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu);
1297	nested_svm_simple_vmexit(to_svm(vcpu), SVM_EXIT_SHUTDOWN);
1298}
1299
1300int svm_allocate_nested(struct vcpu_svm *svm)
1301{
1302	struct page *vmcb02_page;
1303
1304	if (svm->nested.initialized)
1305		return 0;
1306
1307	vmcb02_page = snp_safe_alloc_page();
1308	if (!vmcb02_page)
1309		return -ENOMEM;
1310	svm->nested.vmcb02.ptr = page_address(vmcb02_page);
1311	svm->nested.vmcb02.pa = __sme_set(page_to_pfn(vmcb02_page) << PAGE_SHIFT);
1312
1313	svm->nested.msrpm = svm_vcpu_alloc_msrpm();
1314	if (!svm->nested.msrpm)
1315		goto err_free_vmcb02;
1316
1317	svm->nested.initialized = true;
1318	return 0;
1319
1320err_free_vmcb02:
1321	__free_page(vmcb02_page);
1322	return -ENOMEM;
1323}
1324
1325void svm_free_nested(struct vcpu_svm *svm)
1326{
1327	if (!svm->nested.initialized)
1328		return;
1329
1330	if (WARN_ON_ONCE(svm->vmcb != svm->vmcb01.ptr))
1331		svm_switch_vmcb(svm, &svm->vmcb01);
1332
1333	svm_vcpu_free_msrpm(svm->nested.msrpm);
1334	svm->nested.msrpm = NULL;
1335
1336	__free_page(virt_to_page(svm->nested.vmcb02.ptr));
1337	svm->nested.vmcb02.ptr = NULL;
1338
1339	/*
1340	 * When last_vmcb12_gpa matches the current vmcb12 gpa,
1341	 * some vmcb12 fields are not loaded if they are marked clean
1342	 * in the vmcb12, since in this case they are up to date already.
1343	 *
1344	 * When the vmcb02 is freed, this optimization becomes invalid.
1345	 */
1346	svm->nested.last_vmcb12_gpa = INVALID_GPA;
1347
1348	svm->nested.initialized = false;
1349}
1350
1351void svm_leave_nested(struct kvm_vcpu *vcpu)
1352{
1353	struct vcpu_svm *svm = to_svm(vcpu);
1354
1355	if (is_guest_mode(vcpu)) {
1356		svm->nested.nested_run_pending = 0;
1357		svm->nested.vmcb12_gpa = INVALID_GPA;
1358
1359		leave_guest_mode(vcpu);
1360
1361		svm_switch_vmcb(svm, &svm->vmcb01);
1362
1363		nested_svm_uninit_mmu_context(vcpu);
1364		vmcb_mark_all_dirty(svm->vmcb);
1365
1366		if (kvm_apicv_activated(vcpu->kvm))
1367			kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu);
1368	}
1369
1370	kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
1371}
1372
1373static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
1374{
1375	gpa_t base = svm->nested.ctl.msrpm_base_pa;
1376	int write, bit_nr;
1377	u8 value, mask;
1378	u32 msr;
1379
1380	if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)))
1381		return NESTED_EXIT_HOST;
1382
1383	msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
1384	bit_nr = svm_msrpm_bit_nr(msr);
1385	write  = svm->vmcb->control.exit_info_1 & 1;
1386
1387	if (bit_nr < 0)
1388		return NESTED_EXIT_DONE;
1389
1390	if (kvm_vcpu_read_guest(&svm->vcpu, base + bit_nr / BITS_PER_BYTE,
1391				&value, sizeof(value)))
1392		return NESTED_EXIT_DONE;
1393
1394	mask = BIT(write) << (bit_nr & (BITS_PER_BYTE - 1));
1395	return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1396}
1397
1398static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
1399{
1400	unsigned port, size, iopm_len;
1401	u16 val, mask;
1402	u8 start_bit;
1403	u64 gpa;
1404
1405	if (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_IOIO_PROT)))
1406		return NESTED_EXIT_HOST;
1407
1408	port = svm->vmcb->control.exit_info_1 >> 16;
1409	size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
1410		SVM_IOIO_SIZE_SHIFT;
1411	gpa  = svm->nested.ctl.iopm_base_pa + (port / 8);
1412	start_bit = port % 8;
1413	iopm_len = (start_bit + size > 8) ? 2 : 1;
1414	mask = (0xf >> (4 - size)) << start_bit;
1415	val = 0;
1416
1417	if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
1418		return NESTED_EXIT_DONE;
1419
1420	return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
1421}
1422
1423static int nested_svm_intercept(struct vcpu_svm *svm)
1424{
1425	u32 exit_code = svm->vmcb->control.exit_code;
1426	int vmexit = NESTED_EXIT_HOST;
1427
1428	switch (exit_code) {
1429	case SVM_EXIT_MSR:
1430		vmexit = nested_svm_exit_handled_msr(svm);
1431		break;
1432	case SVM_EXIT_IOIO:
1433		vmexit = nested_svm_intercept_ioio(svm);
1434		break;
1435	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
1436		/*
1437		 * Host-intercepted exceptions have been checked already in
1438		 * nested_svm_exit_special.  There is nothing to do here,
1439		 * the vmexit is injected by svm_check_nested_events.
1440		 */
1441		vmexit = NESTED_EXIT_DONE;
1442		break;
1443	}
1444	case SVM_EXIT_ERR: {
1445		vmexit = NESTED_EXIT_DONE;
1446		break;
1447	}
1448	default: {
1449		if (vmcb12_is_intercept(&svm->nested.ctl, exit_code))
1450			vmexit = NESTED_EXIT_DONE;
1451	}
1452	}
1453
1454	return vmexit;
1455}
1456
1457int nested_svm_exit_handled(struct vcpu_svm *svm)
1458{
1459	int vmexit;
1460
1461	vmexit = nested_svm_intercept(svm);
1462
1463	if (vmexit == NESTED_EXIT_DONE)
1464		nested_svm_vmexit(svm);
1465
1466	return vmexit;
1467}
1468
1469int nested_svm_check_permissions(struct kvm_vcpu *vcpu)
1470{
1471	if (!(vcpu->arch.efer & EFER_SVME) || !is_paging(vcpu)) {
1472		kvm_queue_exception(vcpu, UD_VECTOR);
1473		return 1;
1474	}
1475
1476	if (to_svm(vcpu)->vmcb->save.cpl) {
1477		kvm_inject_gp(vcpu, 0);
1478		return 1;
1479	}
1480
1481	return 0;
1482}
1483
1484static bool nested_svm_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector,
1485					   u32 error_code)
1486{
1487	struct vcpu_svm *svm = to_svm(vcpu);
1488
1489	return (svm->nested.ctl.intercepts[INTERCEPT_EXCEPTION] & BIT(vector));
1490}
1491
1492static void nested_svm_inject_exception_vmexit(struct kvm_vcpu *vcpu)
1493{
1494	struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit;
1495	struct vcpu_svm *svm = to_svm(vcpu);
1496	struct vmcb *vmcb = svm->vmcb;
1497
1498	vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + ex->vector;
1499	vmcb->control.exit_code_hi = 0;
1500
1501	if (ex->has_error_code)
1502		vmcb->control.exit_info_1 = ex->error_code;
1503
1504	/*
1505	 * EXITINFO2 is undefined for all exception intercepts other
1506	 * than #PF.
1507	 */
1508	if (ex->vector == PF_VECTOR) {
1509		if (ex->has_payload)
1510			vmcb->control.exit_info_2 = ex->payload;
1511		else
1512			vmcb->control.exit_info_2 = vcpu->arch.cr2;
1513	} else if (ex->vector == DB_VECTOR) {
1514		/* See kvm_check_and_inject_events().  */
1515		kvm_deliver_exception_payload(vcpu, ex);
1516
1517		if (vcpu->arch.dr7 & DR7_GD) {
1518			vcpu->arch.dr7 &= ~DR7_GD;
1519			kvm_update_dr7(vcpu);
1520		}
1521	} else {
1522		WARN_ON(ex->has_payload);
1523	}
1524
1525	nested_svm_vmexit(svm);
1526}
1527
1528static inline bool nested_exit_on_init(struct vcpu_svm *svm)
1529{
1530	return vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_INIT);
1531}
1532
1533static int svm_check_nested_events(struct kvm_vcpu *vcpu)
1534{
1535	struct kvm_lapic *apic = vcpu->arch.apic;
1536	struct vcpu_svm *svm = to_svm(vcpu);
1537	/*
1538	 * Only a pending nested run blocks a pending exception.  If there is a
1539	 * previously injected event, the pending exception occurred while said
1540	 * event was being delivered and thus needs to be handled.
1541	 */
1542	bool block_nested_exceptions = svm->nested.nested_run_pending;
1543	/*
1544	 * New events (not exceptions) are only recognized at instruction
1545	 * boundaries.  If an event needs reinjection, then KVM is handling a
1546	 * VM-Exit that occurred _during_ instruction execution; new events are
1547	 * blocked until the instruction completes.
1548	 */
1549	bool block_nested_events = block_nested_exceptions ||
1550				   kvm_event_needs_reinjection(vcpu);
1551
1552	if (lapic_in_kernel(vcpu) &&
1553	    test_bit(KVM_APIC_INIT, &apic->pending_events)) {
1554		if (block_nested_events)
1555			return -EBUSY;
1556		if (!nested_exit_on_init(svm))
1557			return 0;
1558		nested_svm_simple_vmexit(svm, SVM_EXIT_INIT);
1559		return 0;
1560	}
1561
1562	if (vcpu->arch.exception_vmexit.pending) {
1563		if (block_nested_exceptions)
1564                        return -EBUSY;
1565		nested_svm_inject_exception_vmexit(vcpu);
1566		return 0;
1567	}
1568
1569	if (vcpu->arch.exception.pending) {
1570		if (block_nested_exceptions)
1571			return -EBUSY;
1572		return 0;
1573	}
1574
1575#ifdef CONFIG_KVM_SMM
1576	if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) {
1577		if (block_nested_events)
1578			return -EBUSY;
1579		if (!nested_exit_on_smi(svm))
1580			return 0;
1581		nested_svm_simple_vmexit(svm, SVM_EXIT_SMI);
1582		return 0;
1583	}
1584#endif
1585
1586	if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) {
1587		if (block_nested_events)
1588			return -EBUSY;
1589		if (!nested_exit_on_nmi(svm))
1590			return 0;
1591		nested_svm_simple_vmexit(svm, SVM_EXIT_NMI);
1592		return 0;
1593	}
1594
1595	if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) {
1596		if (block_nested_events)
1597			return -EBUSY;
1598		if (!nested_exit_on_intr(svm))
1599			return 0;
1600		trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
1601		nested_svm_simple_vmexit(svm, SVM_EXIT_INTR);
1602		return 0;
1603	}
1604
1605	return 0;
1606}
1607
1608int nested_svm_exit_special(struct vcpu_svm *svm)
1609{
1610	u32 exit_code = svm->vmcb->control.exit_code;
1611	struct kvm_vcpu *vcpu = &svm->vcpu;
1612
1613	switch (exit_code) {
1614	case SVM_EXIT_INTR:
1615	case SVM_EXIT_NMI:
1616	case SVM_EXIT_NPF:
1617		return NESTED_EXIT_HOST;
1618	case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
1619		u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
1620
1621		if (svm->vmcb01.ptr->control.intercepts[INTERCEPT_EXCEPTION] &
1622		    excp_bits)
1623			return NESTED_EXIT_HOST;
1624		else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR &&
1625			 svm->vcpu.arch.apf.host_apf_flags)
1626			/* Trap async PF even if not shadowing */
1627			return NESTED_EXIT_HOST;
1628		break;
1629	}
1630	case SVM_EXIT_VMMCALL:
1631		/* Hyper-V L2 TLB flush hypercall is handled by L0 */
1632		if (guest_hv_cpuid_has_l2_tlb_flush(vcpu) &&
1633		    nested_svm_l2_tlb_flush_enabled(vcpu) &&
1634		    kvm_hv_is_tlb_flush_hcall(vcpu))
1635			return NESTED_EXIT_HOST;
1636		break;
1637	default:
1638		break;
1639	}
1640
1641	return NESTED_EXIT_CONTINUE;
1642}
1643
1644void nested_svm_update_tsc_ratio_msr(struct kvm_vcpu *vcpu)
1645{
1646	struct vcpu_svm *svm = to_svm(vcpu);
1647
1648	vcpu->arch.tsc_scaling_ratio =
1649		kvm_calc_nested_tsc_multiplier(vcpu->arch.l1_tsc_scaling_ratio,
1650					       svm->tsc_ratio_msr);
1651	svm_write_tsc_multiplier(vcpu);
1652}
1653
1654/* Inverse operation of nested_copy_vmcb_control_to_cache(). asid is copied too. */
1655static void nested_copy_vmcb_cache_to_control(struct vmcb_control_area *dst,
1656					      struct vmcb_ctrl_area_cached *from)
1657{
1658	unsigned int i;
1659
1660	memset(dst, 0, sizeof(struct vmcb_control_area));
1661
1662	for (i = 0; i < MAX_INTERCEPT; i++)
1663		dst->intercepts[i] = from->intercepts[i];
1664
1665	dst->iopm_base_pa         = from->iopm_base_pa;
1666	dst->msrpm_base_pa        = from->msrpm_base_pa;
1667	dst->tsc_offset           = from->tsc_offset;
1668	dst->asid                 = from->asid;
1669	dst->tlb_ctl              = from->tlb_ctl;
1670	dst->int_ctl              = from->int_ctl;
1671	dst->int_vector           = from->int_vector;
1672	dst->int_state            = from->int_state;
1673	dst->exit_code            = from->exit_code;
1674	dst->exit_code_hi         = from->exit_code_hi;
1675	dst->exit_info_1          = from->exit_info_1;
1676	dst->exit_info_2          = from->exit_info_2;
1677	dst->exit_int_info        = from->exit_int_info;
1678	dst->exit_int_info_err    = from->exit_int_info_err;
1679	dst->nested_ctl           = from->nested_ctl;
1680	dst->event_inj            = from->event_inj;
1681	dst->event_inj_err        = from->event_inj_err;
1682	dst->next_rip             = from->next_rip;
1683	dst->nested_cr3           = from->nested_cr3;
1684	dst->virt_ext              = from->virt_ext;
1685	dst->pause_filter_count   = from->pause_filter_count;
1686	dst->pause_filter_thresh  = from->pause_filter_thresh;
1687	/* 'clean' and 'hv_enlightenments' are not changed by KVM */
1688}
1689
1690static int svm_get_nested_state(struct kvm_vcpu *vcpu,
1691				struct kvm_nested_state __user *user_kvm_nested_state,
1692				u32 user_data_size)
1693{
1694	struct vcpu_svm *svm;
1695	struct vmcb_control_area *ctl;
1696	unsigned long r;
1697	struct kvm_nested_state kvm_state = {
1698		.flags = 0,
1699		.format = KVM_STATE_NESTED_FORMAT_SVM,
1700		.size = sizeof(kvm_state),
1701	};
1702	struct vmcb __user *user_vmcb = (struct vmcb __user *)
1703		&user_kvm_nested_state->data.svm[0];
1704
1705	if (!vcpu)
1706		return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE;
1707
1708	svm = to_svm(vcpu);
1709
1710	if (user_data_size < kvm_state.size)
1711		goto out;
1712
1713	/* First fill in the header and copy it out.  */
1714	if (is_guest_mode(vcpu)) {
1715		kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa;
1716		kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE;
1717		kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
1718
1719		if (svm->nested.nested_run_pending)
1720			kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
1721	}
1722
1723	if (gif_set(svm))
1724		kvm_state.flags |= KVM_STATE_NESTED_GIF_SET;
1725
1726	if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
1727		return -EFAULT;
1728
1729	if (!is_guest_mode(vcpu))
1730		goto out;
1731
1732	/*
1733	 * Copy over the full size of the VMCB rather than just the size
1734	 * of the structs.
1735	 */
1736	if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE))
1737		return -EFAULT;
1738
1739	ctl = kzalloc(sizeof(*ctl), GFP_KERNEL);
1740	if (!ctl)
1741		return -ENOMEM;
1742
1743	nested_copy_vmcb_cache_to_control(ctl, &svm->nested.ctl);
1744	r = copy_to_user(&user_vmcb->control, ctl,
1745			 sizeof(user_vmcb->control));
1746	kfree(ctl);
1747	if (r)
1748		return -EFAULT;
1749
1750	if (copy_to_user(&user_vmcb->save, &svm->vmcb01.ptr->save,
1751			 sizeof(user_vmcb->save)))
1752		return -EFAULT;
1753out:
1754	return kvm_state.size;
1755}
1756
1757static int svm_set_nested_state(struct kvm_vcpu *vcpu,
1758				struct kvm_nested_state __user *user_kvm_nested_state,
1759				struct kvm_nested_state *kvm_state)
1760{
1761	struct vcpu_svm *svm = to_svm(vcpu);
1762	struct vmcb __user *user_vmcb = (struct vmcb __user *)
1763		&user_kvm_nested_state->data.svm[0];
1764	struct vmcb_control_area *ctl;
1765	struct vmcb_save_area *save;
1766	struct vmcb_save_area_cached save_cached;
1767	struct vmcb_ctrl_area_cached ctl_cached;
1768	unsigned long cr0;
1769	int ret;
1770
1771	BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) >
1772		     KVM_STATE_NESTED_SVM_VMCB_SIZE);
1773
1774	if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM)
1775		return -EINVAL;
1776
1777	if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE |
1778				 KVM_STATE_NESTED_RUN_PENDING |
1779				 KVM_STATE_NESTED_GIF_SET))
1780		return -EINVAL;
1781
1782	/*
1783	 * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's
1784	 * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed.
1785	 */
1786	if (!(vcpu->arch.efer & EFER_SVME)) {
1787		/* GIF=1 and no guest mode are required if SVME=0.  */
1788		if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET)
1789			return -EINVAL;
1790	}
1791
1792	/* SMM temporarily disables SVM, so we cannot be in guest mode.  */
1793	if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
1794		return -EINVAL;
1795
1796	if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) {
1797		svm_leave_nested(vcpu);
1798		svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
1799		return 0;
1800	}
1801
1802	if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa))
1803		return -EINVAL;
1804	if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE)
1805		return -EINVAL;
1806
1807	ctl = memdup_user(&user_vmcb->control, sizeof(*ctl));
1808	if (IS_ERR(ctl))
1809		return PTR_ERR(ctl);
1810
1811	save = memdup_user(&user_vmcb->save, sizeof(*save));
1812	if (IS_ERR(save)) {
1813		kfree(ctl);
1814		return PTR_ERR(save);
1815	}
1816
1817	ret = -EINVAL;
1818	__nested_copy_vmcb_control_to_cache(vcpu, &ctl_cached, ctl);
1819	if (!__nested_vmcb_check_controls(vcpu, &ctl_cached))
1820		goto out_free;
1821
1822	/*
1823	 * Processor state contains L2 state.  Check that it is
1824	 * valid for guest mode (see nested_vmcb_check_save).
1825	 */
1826	cr0 = kvm_read_cr0(vcpu);
1827        if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW))
1828		goto out_free;
1829
1830	/*
1831	 * Validate host state saved from before VMRUN (see
1832	 * nested_svm_check_permissions).
1833	 */
1834	__nested_copy_vmcb_save_to_cache(&save_cached, save);
1835	if (!(save->cr0 & X86_CR0_PG) ||
1836	    !(save->cr0 & X86_CR0_PE) ||
1837	    (save->rflags & X86_EFLAGS_VM) ||
1838	    !__nested_vmcb_check_save(vcpu, &save_cached))
1839		goto out_free;
1840
1841
1842	/*
1843	 * All checks done, we can enter guest mode. Userspace provides
1844	 * vmcb12.control, which will be combined with L1 and stored into
1845	 * vmcb02, and the L1 save state which we store in vmcb01.
1846	 * L2 registers if needed are moved from the current VMCB to VMCB02.
1847	 */
1848
1849	if (is_guest_mode(vcpu))
1850		svm_leave_nested(vcpu);
1851	else
1852		svm->nested.vmcb02.ptr->save = svm->vmcb01.ptr->save;
1853
1854	svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET));
1855
1856	svm->nested.nested_run_pending =
1857		!!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING);
1858
1859	svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa;
1860
1861	svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save);
1862	nested_copy_vmcb_control_to_cache(svm, ctl);
1863
1864	svm_switch_vmcb(svm, &svm->nested.vmcb02);
1865	nested_vmcb02_prepare_control(svm, svm->vmcb->save.rip, svm->vmcb->save.cs.base);
1866
1867	/*
1868	 * While the nested guest CR3 is already checked and set by
1869	 * KVM_SET_SREGS, it was set when nested state was yet loaded,
1870	 * thus MMU might not be initialized correctly.
1871	 * Set it again to fix this.
1872	 */
1873	ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3,
1874				  nested_npt_enabled(svm), false);
1875	if (ret)
1876		goto out_free;
1877
1878	svm->nested.force_msr_bitmap_recalc = true;
1879
1880	kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu);
1881	ret = 0;
1882out_free:
1883	kfree(save);
1884	kfree(ctl);
1885
1886	return ret;
1887}
1888
1889static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu)
1890{
1891	if (WARN_ON(!is_guest_mode(vcpu)))
1892		return true;
1893
1894	if (!vcpu->arch.pdptrs_from_userspace &&
1895	    !nested_npt_enabled(to_svm(vcpu)) && is_pae_paging(vcpu))
1896		/*
1897		 * Reload the guest's PDPTRs since after a migration
1898		 * the guest CR3 might be restored prior to setting the nested
1899		 * state which can lead to a load of wrong PDPTRs.
1900		 */
1901		if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3)))
1902			return false;
1903
1904	if (!nested_svm_merge_msrpm(vcpu)) {
1905		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1906		vcpu->run->internal.suberror =
1907			KVM_INTERNAL_ERROR_EMULATION;
1908		vcpu->run->internal.ndata = 0;
1909		return false;
1910	}
1911
1912	if (kvm_hv_verify_vp_assist(vcpu))
1913		return false;
1914
1915	return true;
1916}
1917
1918struct kvm_x86_nested_ops svm_nested_ops = {
1919	.leave_nested = svm_leave_nested,
1920	.is_exception_vmexit = nested_svm_is_exception_vmexit,
1921	.check_events = svm_check_nested_events,
1922	.triple_fault = nested_svm_triple_fault,
1923	.get_nested_state_pages = svm_get_nested_state_pages,
1924	.get_state = svm_get_nested_state,
1925	.set_state = svm_set_nested_state,
1926	.hv_inject_synthetic_vmexit_post_tlb_flush = svm_hv_inject_synthetic_vmexit_post_tlb_flush,
1927};