FILE
nested.c
File preview with syntax highlighting.
linux-6.19.14/arch/x86/kvm/vmx/nested.c
1// SPDX-License-Identifier: GPL-2.0 2#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 3 4#include <linux/objtool.h> 5#include <linux/percpu.h> 6 7#include <asm/debugreg.h> 8#include <asm/mmu_context.h> 9#include <asm/msr.h> 10 11#include "x86.h" 12#include "cpuid.h" 13#include "hyperv.h" 14#include "mmu.h" 15#include "nested.h" 16#include "pmu.h" 17#include "posted_intr.h" 18#include "sgx.h" 19#include "trace.h" 20#include "vmx.h" 21#include "smm.h" 22#include "x86_ops.h" 23 24static bool __read_mostly enable_shadow_vmcs = 1; 25module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO); 26 27static bool __ro_after_init warn_on_missed_cc; 28module_param(warn_on_missed_cc, bool, 0444); 29 30#define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK 31 32/* 33 * Hyper-V requires all of these, so mark them as supported even though 34 * they are just treated the same as all-context. 35 */ 36#define VMX_VPID_EXTENT_SUPPORTED_MASK \ 37 (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT | \ 38 VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT | \ 39 VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT | \ 40 VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT) 41 42#define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5 43 44enum { 45 VMX_VMREAD_BITMAP, 46 VMX_VMWRITE_BITMAP, 47 VMX_BITMAP_NR 48}; 49static unsigned long *vmx_bitmap[VMX_BITMAP_NR]; 50 51#define vmx_vmread_bitmap (vmx_bitmap[VMX_VMREAD_BITMAP]) 52#define vmx_vmwrite_bitmap (vmx_bitmap[VMX_VMWRITE_BITMAP]) 53 54struct shadow_vmcs_field { 55 u16 encoding; 56 u16 offset; 57}; 58static struct shadow_vmcs_field shadow_read_only_fields[] = { 59#define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) }, 60#include "vmcs_shadow_fields.h" 61}; 62static int max_shadow_read_only_fields = 63 ARRAY_SIZE(shadow_read_only_fields); 64 65static struct shadow_vmcs_field shadow_read_write_fields[] = { 66#define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) }, 67#include "vmcs_shadow_fields.h" 68}; 69static int max_shadow_read_write_fields = 70 ARRAY_SIZE(shadow_read_write_fields); 71 72static void init_vmcs_shadow_fields(void) 73{ 74 int i, j; 75 76 memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE); 77 memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE); 78 79 for (i = j = 0; i < max_shadow_read_only_fields; i++) { 80 struct shadow_vmcs_field entry = shadow_read_only_fields[i]; 81 u16 field = entry.encoding; 82 83 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 84 (i + 1 == max_shadow_read_only_fields || 85 shadow_read_only_fields[i + 1].encoding != field + 1)) 86 pr_err("Missing field from shadow_read_only_field %x\n", 87 field + 1); 88 89 clear_bit(field, vmx_vmread_bitmap); 90 if (field & 1) 91#ifdef CONFIG_X86_64 92 continue; 93#else 94 entry.offset += sizeof(u32); 95#endif 96 shadow_read_only_fields[j++] = entry; 97 } 98 max_shadow_read_only_fields = j; 99 100 for (i = j = 0; i < max_shadow_read_write_fields; i++) { 101 struct shadow_vmcs_field entry = shadow_read_write_fields[i]; 102 u16 field = entry.encoding; 103 104 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 && 105 (i + 1 == max_shadow_read_write_fields || 106 shadow_read_write_fields[i + 1].encoding != field + 1)) 107 pr_err("Missing field from shadow_read_write_field %x\n", 108 field + 1); 109 110 WARN_ONCE(field >= GUEST_ES_AR_BYTES && 111 field <= GUEST_TR_AR_BYTES, 112 "Update vmcs12_write_any() to drop reserved bits from AR_BYTES"); 113 114 /* 115 * PML and the preemption timer can be emulated, but the 116 * processor cannot vmwrite to fields that don't exist 117 * on bare metal. 118 */ 119 switch (field) { 120 case GUEST_PML_INDEX: 121 if (!cpu_has_vmx_pml()) 122 continue; 123 break; 124 case VMX_PREEMPTION_TIMER_VALUE: 125 if (!cpu_has_vmx_preemption_timer()) 126 continue; 127 break; 128 case GUEST_INTR_STATUS: 129 if (!cpu_has_vmx_apicv()) 130 continue; 131 break; 132 default: 133 break; 134 } 135 136 clear_bit(field, vmx_vmwrite_bitmap); 137 clear_bit(field, vmx_vmread_bitmap); 138 if (field & 1) 139#ifdef CONFIG_X86_64 140 continue; 141#else 142 entry.offset += sizeof(u32); 143#endif 144 shadow_read_write_fields[j++] = entry; 145 } 146 max_shadow_read_write_fields = j; 147} 148 149/* 150 * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(), 151 * set the success or error code of an emulated VMX instruction (as specified 152 * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated 153 * instruction. 154 */ 155static int nested_vmx_succeed(struct kvm_vcpu *vcpu) 156{ 157 vmx_set_rflags(vcpu, vmx_get_rflags(vcpu) 158 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 159 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)); 160 return kvm_skip_emulated_instruction(vcpu); 161} 162 163static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu) 164{ 165 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 166 & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | 167 X86_EFLAGS_SF | X86_EFLAGS_OF)) 168 | X86_EFLAGS_CF); 169 return kvm_skip_emulated_instruction(vcpu); 170} 171 172static int nested_vmx_failValid(struct kvm_vcpu *vcpu, 173 u32 vm_instruction_error) 174{ 175 vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu) 176 & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | 177 X86_EFLAGS_SF | X86_EFLAGS_OF)) 178 | X86_EFLAGS_ZF); 179 get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; 180 /* 181 * We don't need to force sync to shadow VMCS because 182 * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all 183 * fields and thus must be synced. 184 */ 185 if (nested_vmx_is_evmptr12_set(to_vmx(vcpu))) 186 to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; 187 188 return kvm_skip_emulated_instruction(vcpu); 189} 190 191static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) 192{ 193 struct vcpu_vmx *vmx = to_vmx(vcpu); 194 195 /* 196 * failValid writes the error number to the current VMCS, which 197 * can't be done if there isn't a current VMCS. 198 */ 199 if (vmx->nested.current_vmptr == INVALID_GPA && 200 !nested_vmx_is_evmptr12_valid(vmx)) 201 return nested_vmx_failInvalid(vcpu); 202 203 return nested_vmx_failValid(vcpu, vm_instruction_error); 204} 205 206static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator) 207{ 208 /* TODO: not to reset guest simply here. */ 209 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); 210 pr_debug_ratelimited("nested vmx abort, indicator %d\n", indicator); 211} 212 213static inline bool vmx_control_verify(u32 control, u32 low, u32 high) 214{ 215 return fixed_bits_valid(control, low, high); 216} 217 218static inline u64 vmx_control_msr(u32 low, u32 high) 219{ 220 return low | ((u64)high << 32); 221} 222 223static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx) 224{ 225 secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 226 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 227 vmx->nested.need_vmcs12_to_shadow_sync = false; 228} 229 230static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) 231{ 232#ifdef CONFIG_KVM_HYPERV 233 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); 234 struct vcpu_vmx *vmx = to_vmx(vcpu); 235 236 kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map); 237 vmx->nested.hv_evmcs = NULL; 238 vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; 239 240 if (hv_vcpu) { 241 hv_vcpu->nested.pa_page_gpa = INVALID_GPA; 242 hv_vcpu->nested.vm_id = 0; 243 hv_vcpu->nested.vp_id = 0; 244 } 245#endif 246} 247 248static bool nested_evmcs_handle_vmclear(struct kvm_vcpu *vcpu, gpa_t vmptr) 249{ 250#ifdef CONFIG_KVM_HYPERV 251 struct vcpu_vmx *vmx = to_vmx(vcpu); 252 /* 253 * When Enlightened VMEntry is enabled on the calling CPU we treat 254 * memory area pointer by vmptr as Enlightened VMCS (as there's no good 255 * way to distinguish it from VMCS12) and we must not corrupt it by 256 * writing to the non-existent 'launch_state' field. The area doesn't 257 * have to be the currently active EVMCS on the calling CPU and there's 258 * nothing KVM has to do to transition it from 'active' to 'non-active' 259 * state. It is possible that the area will stay mapped as 260 * vmx->nested.hv_evmcs but this shouldn't be a problem. 261 */ 262 if (!guest_cpu_cap_has_evmcs(vcpu) || 263 !evmptr_is_valid(nested_get_evmptr(vcpu))) 264 return false; 265 266 if (nested_vmx_evmcs(vmx) && vmptr == vmx->nested.hv_evmcs_vmptr) 267 nested_release_evmcs(vcpu); 268 269 return true; 270#else 271 return false; 272#endif 273} 274 275static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, 276 struct loaded_vmcs *prev) 277{ 278 struct vmcs_host_state *dest, *src; 279 280 if (unlikely(!vmx->vt.guest_state_loaded)) 281 return; 282 283 src = &prev->host_state; 284 dest = &vmx->loaded_vmcs->host_state; 285 286 vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base); 287 dest->ldt_sel = src->ldt_sel; 288#ifdef CONFIG_X86_64 289 dest->ds_sel = src->ds_sel; 290 dest->es_sel = src->es_sel; 291#endif 292} 293 294static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) 295{ 296 struct vcpu_vmx *vmx = to_vmx(vcpu); 297 struct loaded_vmcs *prev; 298 int cpu; 299 300 if (WARN_ON_ONCE(vmx->loaded_vmcs == vmcs)) 301 return; 302 303 cpu = get_cpu(); 304 prev = vmx->loaded_vmcs; 305 vmx->loaded_vmcs = vmcs; 306 vmx_vcpu_load_vmcs(vcpu, cpu); 307 vmx_sync_vmcs_host_state(vmx, prev); 308 put_cpu(); 309 310 vcpu->arch.regs_avail = ~VMX_REGS_LAZY_LOAD_SET; 311 312 /* 313 * All lazily updated registers will be reloaded from VMCS12 on both 314 * vmentry and vmexit. 315 */ 316 vcpu->arch.regs_dirty = 0; 317} 318 319static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu) 320{ 321 struct vcpu_vmx *vmx = to_vmx(vcpu); 322 323 kvm_vcpu_unmap(vcpu, &vmx->nested.apic_access_page_map); 324 kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map); 325 kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map); 326 vmx->nested.pi_desc = NULL; 327} 328 329/* 330 * Free whatever needs to be freed from vmx->nested when L1 goes down, or 331 * just stops using VMX. 332 */ 333static void free_nested(struct kvm_vcpu *vcpu) 334{ 335 struct vcpu_vmx *vmx = to_vmx(vcpu); 336 337 if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) 338 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 339 340 if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon) 341 return; 342 343 kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 344 345 vmx->nested.vmxon = false; 346 vmx->nested.smm.vmxon = false; 347 vmx->nested.vmxon_ptr = INVALID_GPA; 348 free_vpid(vmx->nested.vpid02); 349 vmx->nested.posted_intr_nv = -1; 350 vmx->nested.current_vmptr = INVALID_GPA; 351 if (enable_shadow_vmcs) { 352 vmx_disable_shadow_vmcs(vmx); 353 vmcs_clear(vmx->vmcs01.shadow_vmcs); 354 free_vmcs(vmx->vmcs01.shadow_vmcs); 355 vmx->vmcs01.shadow_vmcs = NULL; 356 } 357 kfree(vmx->nested.cached_vmcs12); 358 vmx->nested.cached_vmcs12 = NULL; 359 kfree(vmx->nested.cached_shadow_vmcs12); 360 vmx->nested.cached_shadow_vmcs12 = NULL; 361 362 nested_put_vmcs12_pages(vcpu); 363 364 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 365 366 nested_release_evmcs(vcpu); 367 368 free_loaded_vmcs(&vmx->nested.vmcs02); 369} 370 371/* 372 * Ensure that the current vmcs of the logical processor is the 373 * vmcs01 of the vcpu before calling free_nested(). 374 */ 375void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu) 376{ 377 vcpu_load(vcpu); 378 vmx_leave_nested(vcpu); 379 vcpu_put(vcpu); 380} 381 382#define EPTP_PA_MASK GENMASK_ULL(51, 12) 383 384static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) 385{ 386 return VALID_PAGE(root_hpa) && 387 ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK)); 388} 389 390static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp, 391 gpa_t addr) 392{ 393 unsigned long roots = 0; 394 uint i; 395 struct kvm_mmu_root_info *cached_root; 396 397 WARN_ON_ONCE(!mmu_is_nested(vcpu)); 398 399 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 400 cached_root = &vcpu->arch.mmu->prev_roots[i]; 401 402 if (nested_ept_root_matches(cached_root->hpa, cached_root->pgd, 403 eptp)) 404 roots |= KVM_MMU_ROOT_PREVIOUS(i); 405 } 406 if (roots) 407 kvm_mmu_invalidate_addr(vcpu, vcpu->arch.mmu, addr, roots); 408} 409 410static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, 411 struct x86_exception *fault) 412{ 413 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 414 struct vcpu_vmx *vmx = to_vmx(vcpu); 415 unsigned long exit_qualification; 416 u32 vm_exit_reason; 417 418 if (vmx->nested.pml_full) { 419 vm_exit_reason = EXIT_REASON_PML_FULL; 420 vmx->nested.pml_full = false; 421 422 /* 423 * It should be impossible to trigger a nested PML Full VM-Exit 424 * for anything other than an EPT Violation from L2. KVM *can* 425 * trigger nEPT page fault injection in response to an EPT 426 * Misconfig, e.g. if the MMIO SPTE was stale and L1's EPT 427 * tables also changed, but KVM should not treat EPT Misconfig 428 * VM-Exits as writes. 429 */ 430 WARN_ON_ONCE(vmx->vt.exit_reason.basic != EXIT_REASON_EPT_VIOLATION); 431 432 /* 433 * PML Full and EPT Violation VM-Exits both use bit 12 to report 434 * "NMI unblocking due to IRET", i.e. the bit can be propagated 435 * as-is from the original EXIT_QUALIFICATION. 436 */ 437 exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; 438 } else { 439 if (fault->error_code & PFERR_RSVD_MASK) { 440 vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; 441 exit_qualification = 0; 442 } else { 443 exit_qualification = fault->exit_qualification; 444 exit_qualification |= vmx_get_exit_qual(vcpu) & 445 (EPT_VIOLATION_GVA_IS_VALID | 446 EPT_VIOLATION_GVA_TRANSLATED); 447 vm_exit_reason = EXIT_REASON_EPT_VIOLATION; 448 } 449 450 /* 451 * Although the caller (kvm_inject_emulated_page_fault) would 452 * have already synced the faulting address in the shadow EPT 453 * tables for the current EPTP12, we also need to sync it for 454 * any other cached EPTP02s based on the same EP4TA, since the 455 * TLB associates mappings to the EP4TA rather than the full EPTP. 456 */ 457 nested_ept_invalidate_addr(vcpu, vmcs12->ept_pointer, 458 fault->address); 459 } 460 461 nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification); 462 vmcs12->guest_physical_address = fault->address; 463} 464 465static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) 466{ 467 struct vcpu_vmx *vmx = to_vmx(vcpu); 468 bool execonly = vmx->nested.msrs.ept_caps & VMX_EPT_EXECUTE_ONLY_BIT; 469 int ept_lpage_level = ept_caps_to_lpage_level(vmx->nested.msrs.ept_caps); 470 471 kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level, 472 nested_ept_ad_enabled(vcpu), 473 nested_ept_get_eptp(vcpu)); 474} 475 476static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) 477{ 478 WARN_ON(mmu_is_nested(vcpu)); 479 480 vcpu->arch.mmu = &vcpu->arch.guest_mmu; 481 nested_ept_new_eptp(vcpu); 482 vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; 483 vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; 484 vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; 485 486 vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; 487} 488 489static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu) 490{ 491 vcpu->arch.mmu = &vcpu->arch.root_mmu; 492 vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; 493} 494 495static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12, 496 u16 error_code) 497{ 498 bool inequality, bit; 499 500 bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0; 501 inequality = 502 (error_code & vmcs12->page_fault_error_code_mask) != 503 vmcs12->page_fault_error_code_match; 504 return inequality ^ bit; 505} 506 507static bool nested_vmx_is_exception_vmexit(struct kvm_vcpu *vcpu, u8 vector, 508 u32 error_code) 509{ 510 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 511 512 /* 513 * Drop bits 31:16 of the error code when performing the #PF mask+match 514 * check. All VMCS fields involved are 32 bits, but Intel CPUs never 515 * set bits 31:16 and VMX disallows setting bits 31:16 in the injected 516 * error code. Including the to-be-dropped bits in the check might 517 * result in an "impossible" or missed exit from L1's perspective. 518 */ 519 if (vector == PF_VECTOR) 520 return nested_vmx_is_page_fault_vmexit(vmcs12, (u16)error_code); 521 522 return (vmcs12->exception_bitmap & (1u << vector)); 523} 524 525static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, 526 struct vmcs12 *vmcs12) 527{ 528 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 529 return 0; 530 531 if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) || 532 CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b))) 533 return -EINVAL; 534 535 return 0; 536} 537 538static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu, 539 struct vmcs12 *vmcs12) 540{ 541 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 542 return 0; 543 544 if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap))) 545 return -EINVAL; 546 547 return 0; 548} 549 550static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu, 551 struct vmcs12 *vmcs12) 552{ 553 if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) 554 return 0; 555 556 if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr))) 557 return -EINVAL; 558 559 if (CC(!nested_cpu_has_vid(vmcs12) && vmcs12->tpr_threshold >> 4)) 560 return -EINVAL; 561 562 return 0; 563} 564 565/* 566 * For x2APIC MSRs, ignore the vmcs01 bitmap. L1 can enable x2APIC without L1 567 * itself utilizing x2APIC. All MSRs were previously set to be intercepted, 568 * only the "disable intercept" case needs to be handled. 569 */ 570static void nested_vmx_disable_intercept_for_x2apic_msr(unsigned long *msr_bitmap_l1, 571 unsigned long *msr_bitmap_l0, 572 u32 msr, int type) 573{ 574 if (type & MSR_TYPE_R && !vmx_test_msr_bitmap_read(msr_bitmap_l1, msr)) 575 vmx_clear_msr_bitmap_read(msr_bitmap_l0, msr); 576 577 if (type & MSR_TYPE_W && !vmx_test_msr_bitmap_write(msr_bitmap_l1, msr)) 578 vmx_clear_msr_bitmap_write(msr_bitmap_l0, msr); 579} 580 581static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) 582{ 583 int msr; 584 585 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 586 unsigned word = msr / BITS_PER_LONG; 587 588 msr_bitmap[word] = ~0; 589 msr_bitmap[word + (0x800 / sizeof(long))] = ~0; 590 } 591} 592 593#define BUILD_NVMX_MSR_INTERCEPT_HELPER(rw) \ 594static inline \ 595void nested_vmx_set_msr_##rw##_intercept(struct vcpu_vmx *vmx, \ 596 unsigned long *msr_bitmap_l1, \ 597 unsigned long *msr_bitmap_l0, u32 msr) \ 598{ \ 599 if (vmx_test_msr_bitmap_##rw(vmx->vmcs01.msr_bitmap, msr) || \ 600 vmx_test_msr_bitmap_##rw(msr_bitmap_l1, msr)) \ 601 vmx_set_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 602 else \ 603 vmx_clear_msr_bitmap_##rw(msr_bitmap_l0, msr); \ 604} 605BUILD_NVMX_MSR_INTERCEPT_HELPER(read) 606BUILD_NVMX_MSR_INTERCEPT_HELPER(write) 607 608static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx, 609 unsigned long *msr_bitmap_l1, 610 unsigned long *msr_bitmap_l0, 611 u32 msr, int types) 612{ 613 if (types & MSR_TYPE_R) 614 nested_vmx_set_msr_read_intercept(vmx, msr_bitmap_l1, 615 msr_bitmap_l0, msr); 616 if (types & MSR_TYPE_W) 617 nested_vmx_set_msr_write_intercept(vmx, msr_bitmap_l1, 618 msr_bitmap_l0, msr); 619} 620 621/* 622 * Merge L0's and L1's MSR bitmap, return false to indicate that 623 * we do not use the hardware. 624 */ 625static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, 626 struct vmcs12 *vmcs12) 627{ 628 struct vcpu_vmx *vmx = to_vmx(vcpu); 629 int msr; 630 unsigned long *msr_bitmap_l1; 631 unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; 632 struct kvm_host_map map; 633 634 /* Nothing to do if the MSR bitmap is not in use. */ 635 if (!cpu_has_vmx_msr_bitmap() || 636 !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 637 return false; 638 639 /* 640 * MSR bitmap update can be skipped when: 641 * - MSR bitmap for L1 hasn't changed. 642 * - Nested hypervisor (L1) is attempting to launch the same L2 as 643 * before. 644 * - Nested hypervisor (L1) has enabled 'Enlightened MSR Bitmap' feature 645 * and tells KVM (L0) there were no changes in MSR bitmap for L2. 646 */ 647 if (!vmx->nested.force_msr_bitmap_recalc) { 648 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 649 650 if (evmcs && evmcs->hv_enlightenments_control.msr_bitmap && 651 evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP) 652 return true; 653 } 654 655 if (kvm_vcpu_map_readonly(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), &map)) 656 return false; 657 658 msr_bitmap_l1 = (unsigned long *)map.hva; 659 660 /* 661 * To keep the control flow simple, pay eight 8-byte writes (sixteen 662 * 4-byte writes on 32-bit systems) up front to enable intercepts for 663 * the x2APIC MSR range and selectively toggle those relevant to L2. 664 */ 665 enable_x2apic_msr_intercepts(msr_bitmap_l0); 666 667 if (nested_cpu_has_virt_x2apic_mode(vmcs12)) { 668 if (nested_cpu_has_apic_reg_virt(vmcs12)) { 669 /* 670 * L0 need not intercept reads for MSRs between 0x800 671 * and 0x8ff, it just lets the processor take the value 672 * from the virtual-APIC page; take those 256 bits 673 * directly from the L1 bitmap. 674 */ 675 for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) { 676 unsigned word = msr / BITS_PER_LONG; 677 678 msr_bitmap_l0[word] = msr_bitmap_l1[word]; 679 } 680 } 681 682 nested_vmx_disable_intercept_for_x2apic_msr( 683 msr_bitmap_l1, msr_bitmap_l0, 684 X2APIC_MSR(APIC_TASKPRI), 685 MSR_TYPE_R | MSR_TYPE_W); 686 687 if (nested_cpu_has_vid(vmcs12)) { 688 nested_vmx_disable_intercept_for_x2apic_msr( 689 msr_bitmap_l1, msr_bitmap_l0, 690 X2APIC_MSR(APIC_EOI), 691 MSR_TYPE_W); 692 nested_vmx_disable_intercept_for_x2apic_msr( 693 msr_bitmap_l1, msr_bitmap_l0, 694 X2APIC_MSR(APIC_SELF_IPI), 695 MSR_TYPE_W); 696 } 697 } 698 699 /* 700 * Always check vmcs01's bitmap to honor userspace MSR filters and any 701 * other runtime changes to vmcs01's bitmap, e.g. dynamic pass-through. 702 */ 703#ifdef CONFIG_X86_64 704 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 705 MSR_FS_BASE, MSR_TYPE_RW); 706 707 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 708 MSR_GS_BASE, MSR_TYPE_RW); 709 710 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 711 MSR_KERNEL_GS_BASE, MSR_TYPE_RW); 712#endif 713 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 714 MSR_IA32_SPEC_CTRL, MSR_TYPE_RW); 715 716 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 717 MSR_IA32_PRED_CMD, MSR_TYPE_W); 718 719 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 720 MSR_IA32_FLUSH_CMD, MSR_TYPE_W); 721 722 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 723 MSR_IA32_APERF, MSR_TYPE_R); 724 725 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 726 MSR_IA32_MPERF, MSR_TYPE_R); 727 728 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 729 MSR_IA32_U_CET, MSR_TYPE_RW); 730 731 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 732 MSR_IA32_S_CET, MSR_TYPE_RW); 733 734 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 735 MSR_IA32_PL0_SSP, MSR_TYPE_RW); 736 737 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 738 MSR_IA32_PL1_SSP, MSR_TYPE_RW); 739 740 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 741 MSR_IA32_PL2_SSP, MSR_TYPE_RW); 742 743 nested_vmx_set_intercept_for_msr(vmx, msr_bitmap_l1, msr_bitmap_l0, 744 MSR_IA32_PL3_SSP, MSR_TYPE_RW); 745 746 kvm_vcpu_unmap(vcpu, &map); 747 748 vmx->nested.force_msr_bitmap_recalc = false; 749 750 return true; 751} 752 753static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu, 754 struct vmcs12 *vmcs12) 755{ 756 struct vcpu_vmx *vmx = to_vmx(vcpu); 757 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 758 759 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 760 vmcs12->vmcs_link_pointer == INVALID_GPA) 761 return; 762 763 if (ghc->gpa != vmcs12->vmcs_link_pointer && 764 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 765 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 766 return; 767 768 kvm_read_guest_cached(vcpu->kvm, ghc, get_shadow_vmcs12(vcpu), 769 VMCS12_SIZE); 770} 771 772static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu, 773 struct vmcs12 *vmcs12) 774{ 775 struct vcpu_vmx *vmx = to_vmx(vcpu); 776 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 777 778 if (!nested_cpu_has_shadow_vmcs(vmcs12) || 779 vmcs12->vmcs_link_pointer == INVALID_GPA) 780 return; 781 782 if (ghc->gpa != vmcs12->vmcs_link_pointer && 783 kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 784 vmcs12->vmcs_link_pointer, VMCS12_SIZE)) 785 return; 786 787 kvm_write_guest_cached(vcpu->kvm, ghc, get_shadow_vmcs12(vcpu), 788 VMCS12_SIZE); 789} 790 791/* 792 * In nested virtualization, check if L1 has set 793 * VM_EXIT_ACK_INTR_ON_EXIT 794 */ 795static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) 796{ 797 return get_vmcs12(vcpu)->vm_exit_controls & 798 VM_EXIT_ACK_INTR_ON_EXIT; 799} 800 801static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, 802 struct vmcs12 *vmcs12) 803{ 804 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 805 CC(!page_address_valid(vcpu, vmcs12->apic_access_addr))) 806 return -EINVAL; 807 else 808 return 0; 809} 810 811static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu, 812 struct vmcs12 *vmcs12) 813{ 814 if (!nested_cpu_has_virt_x2apic_mode(vmcs12) && 815 !nested_cpu_has_apic_reg_virt(vmcs12) && 816 !nested_cpu_has_vid(vmcs12) && 817 !nested_cpu_has_posted_intr(vmcs12)) 818 return 0; 819 820 /* 821 * If virtualize x2apic mode is enabled, 822 * virtualize apic access must be disabled. 823 */ 824 if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) && 825 nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))) 826 return -EINVAL; 827 828 /* 829 * If virtual interrupt delivery is enabled, 830 * we must exit on external interrupts. 831 */ 832 if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu))) 833 return -EINVAL; 834 835 /* 836 * bits 15:8 should be zero in posted_intr_nv, 837 * the descriptor address has been already checked 838 * in nested_get_vmcs12_pages. 839 * 840 * bits 5:0 of posted_intr_desc_addr should be zero. 841 */ 842 if (nested_cpu_has_posted_intr(vmcs12) && 843 (CC(!nested_cpu_has_vid(vmcs12)) || 844 CC(!nested_exit_intr_ack_set(vcpu)) || 845 CC((vmcs12->posted_intr_nv & 0xff00)) || 846 CC(!kvm_vcpu_is_legal_aligned_gpa(vcpu, vmcs12->posted_intr_desc_addr, 64)))) 847 return -EINVAL; 848 849 /* tpr shadow is needed by all apicv features. */ 850 if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))) 851 return -EINVAL; 852 853 return 0; 854} 855 856static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu) 857{ 858 struct vcpu_vmx *vmx = to_vmx(vcpu); 859 u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low, 860 vmx->nested.msrs.misc_high); 861 862 return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER; 863} 864 865static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu, 866 u32 count, u64 addr) 867{ 868 if (count == 0) 869 return 0; 870 871 /* 872 * Exceeding the limit results in architecturally _undefined_ behavior, 873 * i.e. KVM is allowed to do literally anything in response to a bad 874 * limit. Immediately generate a consistency check so that code that 875 * consumes the count doesn't need to worry about extreme edge cases. 876 */ 877 if (count > nested_vmx_max_atomic_switch_msrs(vcpu)) 878 return -EINVAL; 879 880 if (!kvm_vcpu_is_legal_aligned_gpa(vcpu, addr, 16) || 881 !kvm_vcpu_is_legal_gpa(vcpu, (addr + count * sizeof(struct vmx_msr_entry) - 1))) 882 return -EINVAL; 883 884 return 0; 885} 886 887static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu, 888 struct vmcs12 *vmcs12) 889{ 890 if (CC(nested_vmx_check_msr_switch(vcpu, 891 vmcs12->vm_exit_msr_load_count, 892 vmcs12->vm_exit_msr_load_addr)) || 893 CC(nested_vmx_check_msr_switch(vcpu, 894 vmcs12->vm_exit_msr_store_count, 895 vmcs12->vm_exit_msr_store_addr))) 896 return -EINVAL; 897 898 return 0; 899} 900 901static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu, 902 struct vmcs12 *vmcs12) 903{ 904 if (CC(nested_vmx_check_msr_switch(vcpu, 905 vmcs12->vm_entry_msr_load_count, 906 vmcs12->vm_entry_msr_load_addr))) 907 return -EINVAL; 908 909 return 0; 910} 911 912static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu, 913 struct vmcs12 *vmcs12) 914{ 915 if (!nested_cpu_has_pml(vmcs12)) 916 return 0; 917 918 if (CC(!nested_cpu_has_ept(vmcs12)) || 919 CC(!page_address_valid(vcpu, vmcs12->pml_address))) 920 return -EINVAL; 921 922 return 0; 923} 924 925static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu, 926 struct vmcs12 *vmcs12) 927{ 928 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) && 929 !nested_cpu_has_ept(vmcs12))) 930 return -EINVAL; 931 return 0; 932} 933 934static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu, 935 struct vmcs12 *vmcs12) 936{ 937 if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) && 938 !nested_cpu_has_ept(vmcs12))) 939 return -EINVAL; 940 return 0; 941} 942 943static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu, 944 struct vmcs12 *vmcs12) 945{ 946 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 947 return 0; 948 949 if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) || 950 CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap))) 951 return -EINVAL; 952 953 return 0; 954} 955 956static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu, 957 struct vmx_msr_entry *e) 958{ 959 /* x2APIC MSR accesses are not allowed */ 960 if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8)) 961 return -EINVAL; 962 if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */ 963 CC(e->index == MSR_IA32_UCODE_REV)) 964 return -EINVAL; 965 if (CC(e->reserved != 0)) 966 return -EINVAL; 967 return 0; 968} 969 970static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu, 971 struct vmx_msr_entry *e) 972{ 973 if (CC(e->index == MSR_FS_BASE) || 974 CC(e->index == MSR_GS_BASE) || 975 CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */ 976 nested_vmx_msr_check_common(vcpu, e)) 977 return -EINVAL; 978 return 0; 979} 980 981static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu, 982 struct vmx_msr_entry *e) 983{ 984 if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */ 985 nested_vmx_msr_check_common(vcpu, e)) 986 return -EINVAL; 987 return 0; 988} 989 990/* 991 * Load guest's/host's msr at nested entry/exit. 992 * return 0 for success, entry index for failure. 993 * 994 * One of the failure modes for MSR load/store is when a list exceeds the 995 * virtual hardware's capacity. To maintain compatibility with hardware inasmuch 996 * as possible, process all valid entries before failing rather than precheck 997 * for a capacity violation. 998 */ 999static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 1000{ 1001 u32 i; 1002 struct vmx_msr_entry e; 1003 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 1004 1005 for (i = 0; i < count; i++) { 1006 if (WARN_ON_ONCE(i >= max_msr_list_size)) 1007 goto fail; 1008 1009 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e), 1010 &e, sizeof(e))) { 1011 pr_debug_ratelimited( 1012 "%s cannot read MSR entry (%u, 0x%08llx)\n", 1013 __func__, i, gpa + i * sizeof(e)); 1014 goto fail; 1015 } 1016 if (nested_vmx_load_msr_check(vcpu, &e)) { 1017 pr_debug_ratelimited( 1018 "%s check failed (%u, 0x%x, 0x%x)\n", 1019 __func__, i, e.index, e.reserved); 1020 goto fail; 1021 } 1022 if (kvm_emulate_msr_write(vcpu, e.index, e.value)) { 1023 pr_debug_ratelimited( 1024 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1025 __func__, i, e.index, e.value); 1026 goto fail; 1027 } 1028 } 1029 return 0; 1030fail: 1031 /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */ 1032 return i + 1; 1033} 1034 1035static bool nested_vmx_get_vmexit_msr_value(struct kvm_vcpu *vcpu, 1036 u32 msr_index, 1037 u64 *data) 1038{ 1039 struct vcpu_vmx *vmx = to_vmx(vcpu); 1040 1041 /* 1042 * If the L0 hypervisor stored a more accurate value for the TSC that 1043 * does not include the time taken for emulation of the L2->L1 1044 * VM-exit in L0, use the more accurate value. 1045 */ 1046 if (msr_index == MSR_IA32_TSC) { 1047 int i = vmx_find_loadstore_msr_slot(&vmx->msr_autostore.guest, 1048 MSR_IA32_TSC); 1049 1050 if (i >= 0) { 1051 u64 val = vmx->msr_autostore.guest.val[i].value; 1052 1053 *data = kvm_read_l1_tsc(vcpu, val); 1054 return true; 1055 } 1056 } 1057 1058 if (kvm_emulate_msr_read(vcpu, msr_index, data)) { 1059 pr_debug_ratelimited("%s cannot read MSR (0x%x)\n", __func__, 1060 msr_index); 1061 return false; 1062 } 1063 return true; 1064} 1065 1066static bool read_and_check_msr_entry(struct kvm_vcpu *vcpu, u64 gpa, int i, 1067 struct vmx_msr_entry *e) 1068{ 1069 if (kvm_vcpu_read_guest(vcpu, 1070 gpa + i * sizeof(*e), 1071 e, 2 * sizeof(u32))) { 1072 pr_debug_ratelimited( 1073 "%s cannot read MSR entry (%u, 0x%08llx)\n", 1074 __func__, i, gpa + i * sizeof(*e)); 1075 return false; 1076 } 1077 if (nested_vmx_store_msr_check(vcpu, e)) { 1078 pr_debug_ratelimited( 1079 "%s check failed (%u, 0x%x, 0x%x)\n", 1080 __func__, i, e->index, e->reserved); 1081 return false; 1082 } 1083 return true; 1084} 1085 1086static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) 1087{ 1088 u64 data; 1089 u32 i; 1090 struct vmx_msr_entry e; 1091 u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu); 1092 1093 for (i = 0; i < count; i++) { 1094 if (WARN_ON_ONCE(i >= max_msr_list_size)) 1095 return -EINVAL; 1096 1097 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1098 return -EINVAL; 1099 1100 if (!nested_vmx_get_vmexit_msr_value(vcpu, e.index, &data)) 1101 return -EINVAL; 1102 1103 if (kvm_vcpu_write_guest(vcpu, 1104 gpa + i * sizeof(e) + 1105 offsetof(struct vmx_msr_entry, value), 1106 &data, sizeof(data))) { 1107 pr_debug_ratelimited( 1108 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n", 1109 __func__, i, e.index, data); 1110 return -EINVAL; 1111 } 1112 } 1113 return 0; 1114} 1115 1116static bool nested_msr_store_list_has_msr(struct kvm_vcpu *vcpu, u32 msr_index) 1117{ 1118 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1119 u32 count = vmcs12->vm_exit_msr_store_count; 1120 u64 gpa = vmcs12->vm_exit_msr_store_addr; 1121 struct vmx_msr_entry e; 1122 u32 i; 1123 1124 for (i = 0; i < count; i++) { 1125 if (!read_and_check_msr_entry(vcpu, gpa, i, &e)) 1126 return false; 1127 1128 if (e.index == msr_index) 1129 return true; 1130 } 1131 return false; 1132} 1133 1134static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, 1135 u32 msr_index) 1136{ 1137 struct vcpu_vmx *vmx = to_vmx(vcpu); 1138 struct vmx_msrs *autostore = &vmx->msr_autostore.guest; 1139 bool in_vmcs12_store_list; 1140 int msr_autostore_slot; 1141 bool in_autostore_list; 1142 int last; 1143 1144 msr_autostore_slot = vmx_find_loadstore_msr_slot(autostore, msr_index); 1145 in_autostore_list = msr_autostore_slot >= 0; 1146 in_vmcs12_store_list = nested_msr_store_list_has_msr(vcpu, msr_index); 1147 1148 if (in_vmcs12_store_list && !in_autostore_list) { 1149 if (autostore->nr == MAX_NR_LOADSTORE_MSRS) { 1150 /* 1151 * Emulated VMEntry does not fail here. Instead a less 1152 * accurate value will be returned by 1153 * nested_vmx_get_vmexit_msr_value() by reading KVM's 1154 * internal MSR state instead of reading the value from 1155 * the vmcs02 VMExit MSR-store area. 1156 */ 1157 pr_warn_ratelimited( 1158 "Not enough msr entries in msr_autostore. Can't add msr %x\n", 1159 msr_index); 1160 return; 1161 } 1162 last = autostore->nr++; 1163 autostore->val[last].index = msr_index; 1164 } else if (!in_vmcs12_store_list && in_autostore_list) { 1165 last = --autostore->nr; 1166 autostore->val[msr_autostore_slot] = autostore->val[last]; 1167 } 1168} 1169 1170/* 1171 * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are 1172 * emulating VM-Entry into a guest with EPT enabled. On failure, the expected 1173 * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to 1174 * @entry_failure_code. 1175 */ 1176static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, 1177 bool nested_ept, bool reload_pdptrs, 1178 enum vm_entry_failure_code *entry_failure_code) 1179{ 1180 if (CC(!kvm_vcpu_is_legal_cr3(vcpu, cr3))) { 1181 *entry_failure_code = ENTRY_FAIL_DEFAULT; 1182 return -EINVAL; 1183 } 1184 1185 /* 1186 * If PAE paging and EPT are both on, CR3 is not used by the CPU and 1187 * must not be dereferenced. 1188 */ 1189 if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && 1190 CC(!load_pdptrs(vcpu, cr3))) { 1191 *entry_failure_code = ENTRY_FAIL_PDPTE; 1192 return -EINVAL; 1193 } 1194 1195 vcpu->arch.cr3 = cr3; 1196 kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); 1197 1198 /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ 1199 kvm_init_mmu(vcpu); 1200 1201 if (!nested_ept) 1202 kvm_mmu_new_pgd(vcpu, cr3); 1203 1204 return 0; 1205} 1206 1207/* 1208 * Returns if KVM is able to config CPU to tag TLB entries 1209 * populated by L2 differently than TLB entries populated 1210 * by L1. 1211 * 1212 * If L0 uses EPT, L1 and L2 run with different EPTP because 1213 * guest_mode is part of kvm_mmu_page_role. Thus, TLB entries 1214 * are tagged with different EPTP. 1215 * 1216 * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged 1217 * with different VPID (L1 entries are tagged with vmx->vpid 1218 * while L2 entries are tagged with vmx->nested.vpid02). 1219 */ 1220static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) 1221{ 1222 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 1223 1224 return enable_ept || 1225 (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); 1226} 1227 1228static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, 1229 struct vmcs12 *vmcs12, 1230 bool is_vmenter) 1231{ 1232 struct vcpu_vmx *vmx = to_vmx(vcpu); 1233 1234 /* Handle pending Hyper-V TLB flush requests */ 1235 kvm_hv_nested_transtion_tlb_flush(vcpu, enable_ept); 1236 1237 /* 1238 * If VPID is disabled, then guest TLB accesses use VPID=0, i.e. the 1239 * same VPID as the host, and so architecturally, linear and combined 1240 * mappings for VPID=0 must be flushed at VM-Enter and VM-Exit. KVM 1241 * emulates L2 sharing L1's VPID=0 by using vpid01 while running L2, 1242 * and so KVM must also emulate TLB flush of VPID=0, i.e. vpid01. This 1243 * is required if VPID is disabled in KVM, as a TLB flush (there are no 1244 * VPIDs) still occurs from L1's perspective, and KVM may need to 1245 * synchronize the MMU in response to the guest TLB flush. 1246 * 1247 * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. 1248 * EPT is a special snowflake, as guest-physical mappings aren't 1249 * flushed on VPID invalidations, including VM-Enter or VM-Exit with 1250 * VPID disabled. As a result, KVM _never_ needs to sync nEPT 1251 * entries on VM-Enter because L1 can't rely on VM-Enter to flush 1252 * those mappings. 1253 */ 1254 if (!nested_cpu_has_vpid(vmcs12)) { 1255 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1256 return; 1257 } 1258 1259 /* L2 should never have a VPID if VPID is disabled. */ 1260 WARN_ON(!enable_vpid); 1261 1262 /* 1263 * VPID is enabled and in use by vmcs12. If vpid12 is changing, then 1264 * emulate a guest TLB flush as KVM does not track vpid12 history nor 1265 * is the VPID incorporated into the MMU context. I.e. KVM must assume 1266 * that the new vpid12 has never been used and thus represents a new 1267 * guest ASID that cannot have entries in the TLB. 1268 */ 1269 if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { 1270 vmx->nested.last_vpid = vmcs12->virtual_processor_id; 1271 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 1272 return; 1273 } 1274 1275 /* 1276 * If VPID is enabled, used by vmc12, and vpid12 is not changing but 1277 * does not have a unique TLB tag (ASID), i.e. EPT is disabled and 1278 * KVM was unable to allocate a VPID for L2, flush the current context 1279 * as the effective ASID is common to both L1 and L2. 1280 */ 1281 if (!nested_has_guest_tlb_tag(vcpu)) 1282 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); 1283} 1284 1285static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) 1286{ 1287 superset &= mask; 1288 subset &= mask; 1289 1290 return (superset | subset) == superset; 1291} 1292 1293static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data) 1294{ 1295 const u64 feature_bits = VMX_BASIC_DUAL_MONITOR_TREATMENT | 1296 VMX_BASIC_INOUT | 1297 VMX_BASIC_TRUE_CTLS | 1298 VMX_BASIC_NO_HW_ERROR_CODE_CC; 1299 1300 const u64 reserved_bits = GENMASK_ULL(63, 57) | 1301 GENMASK_ULL(47, 45) | 1302 BIT_ULL(31); 1303 1304 u64 vmx_basic = vmcs_config.nested.basic; 1305 1306 BUILD_BUG_ON(feature_bits & reserved_bits); 1307 1308 /* 1309 * Except for 32BIT_PHYS_ADDR_ONLY, which is an anti-feature bit (has 1310 * inverted polarity), the incoming value must not set feature bits or 1311 * reserved bits that aren't allowed/supported by KVM. Fields, i.e. 1312 * multi-bit values, are explicitly checked below. 1313 */ 1314 if (!is_bitwise_subset(vmx_basic, data, feature_bits | reserved_bits)) 1315 return -EINVAL; 1316 1317 /* 1318 * KVM does not emulate a version of VMX that constrains physical 1319 * addresses of VMX structures (e.g. VMCS) to 32-bits. 1320 */ 1321 if (data & VMX_BASIC_32BIT_PHYS_ADDR_ONLY) 1322 return -EINVAL; 1323 1324 if (vmx_basic_vmcs_revision_id(vmx_basic) != 1325 vmx_basic_vmcs_revision_id(data)) 1326 return -EINVAL; 1327 1328 if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data)) 1329 return -EINVAL; 1330 1331 vmx->nested.msrs.basic = data; 1332 return 0; 1333} 1334 1335static void vmx_get_control_msr(struct nested_vmx_msrs *msrs, u32 msr_index, 1336 u32 **low, u32 **high) 1337{ 1338 switch (msr_index) { 1339 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1340 *low = &msrs->pinbased_ctls_low; 1341 *high = &msrs->pinbased_ctls_high; 1342 break; 1343 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1344 *low = &msrs->procbased_ctls_low; 1345 *high = &msrs->procbased_ctls_high; 1346 break; 1347 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1348 *low = &msrs->exit_ctls_low; 1349 *high = &msrs->exit_ctls_high; 1350 break; 1351 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1352 *low = &msrs->entry_ctls_low; 1353 *high = &msrs->entry_ctls_high; 1354 break; 1355 case MSR_IA32_VMX_PROCBASED_CTLS2: 1356 *low = &msrs->secondary_ctls_low; 1357 *high = &msrs->secondary_ctls_high; 1358 break; 1359 default: 1360 BUG(); 1361 } 1362} 1363 1364static int 1365vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1366{ 1367 u32 *lowp, *highp; 1368 u64 supported; 1369 1370 vmx_get_control_msr(&vmcs_config.nested, msr_index, &lowp, &highp); 1371 1372 supported = vmx_control_msr(*lowp, *highp); 1373 1374 /* Check must-be-1 bits are still 1. */ 1375 if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0))) 1376 return -EINVAL; 1377 1378 /* Check must-be-0 bits are still 0. */ 1379 if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32))) 1380 return -EINVAL; 1381 1382 vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp); 1383 *lowp = data; 1384 *highp = data >> 32; 1385 return 0; 1386} 1387 1388static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data) 1389{ 1390 const u64 feature_bits = VMX_MISC_SAVE_EFER_LMA | 1391 VMX_MISC_ACTIVITY_HLT | 1392 VMX_MISC_ACTIVITY_SHUTDOWN | 1393 VMX_MISC_ACTIVITY_WAIT_SIPI | 1394 VMX_MISC_INTEL_PT | 1395 VMX_MISC_RDMSR_IN_SMM | 1396 VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 1397 VMX_MISC_VMXOFF_BLOCK_SMI | 1398 VMX_MISC_ZERO_LEN_INS; 1399 1400 const u64 reserved_bits = BIT_ULL(31) | GENMASK_ULL(13, 9); 1401 1402 u64 vmx_misc = vmx_control_msr(vmcs_config.nested.misc_low, 1403 vmcs_config.nested.misc_high); 1404 1405 BUILD_BUG_ON(feature_bits & reserved_bits); 1406 1407 /* 1408 * The incoming value must not set feature bits or reserved bits that 1409 * aren't allowed/supported by KVM. Fields, i.e. multi-bit values, are 1410 * explicitly checked below. 1411 */ 1412 if (!is_bitwise_subset(vmx_misc, data, feature_bits | reserved_bits)) 1413 return -EINVAL; 1414 1415 if ((vmx->nested.msrs.pinbased_ctls_high & 1416 PIN_BASED_VMX_PREEMPTION_TIMER) && 1417 vmx_misc_preemption_timer_rate(data) != 1418 vmx_misc_preemption_timer_rate(vmx_misc)) 1419 return -EINVAL; 1420 1421 if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc)) 1422 return -EINVAL; 1423 1424 if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc)) 1425 return -EINVAL; 1426 1427 if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc)) 1428 return -EINVAL; 1429 1430 vmx->nested.msrs.misc_low = data; 1431 vmx->nested.msrs.misc_high = data >> 32; 1432 1433 return 0; 1434} 1435 1436static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data) 1437{ 1438 u64 vmx_ept_vpid_cap = vmx_control_msr(vmcs_config.nested.ept_caps, 1439 vmcs_config.nested.vpid_caps); 1440 1441 /* Every bit is either reserved or a feature bit. */ 1442 if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL)) 1443 return -EINVAL; 1444 1445 vmx->nested.msrs.ept_caps = data; 1446 vmx->nested.msrs.vpid_caps = data >> 32; 1447 return 0; 1448} 1449 1450static u64 *vmx_get_fixed0_msr(struct nested_vmx_msrs *msrs, u32 msr_index) 1451{ 1452 switch (msr_index) { 1453 case MSR_IA32_VMX_CR0_FIXED0: 1454 return &msrs->cr0_fixed0; 1455 case MSR_IA32_VMX_CR4_FIXED0: 1456 return &msrs->cr4_fixed0; 1457 default: 1458 BUG(); 1459 } 1460} 1461 1462static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data) 1463{ 1464 const u64 *msr = vmx_get_fixed0_msr(&vmcs_config.nested, msr_index); 1465 1466 /* 1467 * 1 bits (which indicates bits which "must-be-1" during VMX operation) 1468 * must be 1 in the restored value. 1469 */ 1470 if (!is_bitwise_subset(data, *msr, -1ULL)) 1471 return -EINVAL; 1472 1473 *vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data; 1474 return 0; 1475} 1476 1477/* 1478 * Called when userspace is restoring VMX MSRs. 1479 * 1480 * Returns 0 on success, non-0 otherwise. 1481 */ 1482int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) 1483{ 1484 struct vcpu_vmx *vmx = to_vmx(vcpu); 1485 1486 /* 1487 * Don't allow changes to the VMX capability MSRs while the vCPU 1488 * is in VMX operation. 1489 */ 1490 if (vmx->nested.vmxon) 1491 return -EBUSY; 1492 1493 switch (msr_index) { 1494 case MSR_IA32_VMX_BASIC: 1495 return vmx_restore_vmx_basic(vmx, data); 1496 case MSR_IA32_VMX_PINBASED_CTLS: 1497 case MSR_IA32_VMX_PROCBASED_CTLS: 1498 case MSR_IA32_VMX_EXIT_CTLS: 1499 case MSR_IA32_VMX_ENTRY_CTLS: 1500 /* 1501 * The "non-true" VMX capability MSRs are generated from the 1502 * "true" MSRs, so we do not support restoring them directly. 1503 * 1504 * If userspace wants to emulate VMX_BASIC[55]=0, userspace 1505 * should restore the "true" MSRs with the must-be-1 bits 1506 * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND 1507 * DEFAULT SETTINGS". 1508 */ 1509 return -EINVAL; 1510 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1511 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1512 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1513 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1514 case MSR_IA32_VMX_PROCBASED_CTLS2: 1515 return vmx_restore_control_msr(vmx, msr_index, data); 1516 case MSR_IA32_VMX_MISC: 1517 return vmx_restore_vmx_misc(vmx, data); 1518 case MSR_IA32_VMX_CR0_FIXED0: 1519 case MSR_IA32_VMX_CR4_FIXED0: 1520 return vmx_restore_fixed0_msr(vmx, msr_index, data); 1521 case MSR_IA32_VMX_CR0_FIXED1: 1522 case MSR_IA32_VMX_CR4_FIXED1: 1523 /* 1524 * These MSRs are generated based on the vCPU's CPUID, so we 1525 * do not support restoring them directly. 1526 */ 1527 return -EINVAL; 1528 case MSR_IA32_VMX_EPT_VPID_CAP: 1529 return vmx_restore_vmx_ept_vpid_cap(vmx, data); 1530 case MSR_IA32_VMX_VMCS_ENUM: 1531 vmx->nested.msrs.vmcs_enum = data; 1532 return 0; 1533 case MSR_IA32_VMX_VMFUNC: 1534 if (data & ~vmcs_config.nested.vmfunc_controls) 1535 return -EINVAL; 1536 vmx->nested.msrs.vmfunc_controls = data; 1537 return 0; 1538 default: 1539 /* 1540 * The rest of the VMX capability MSRs do not support restore. 1541 */ 1542 return -EINVAL; 1543 } 1544} 1545 1546/* Returns 0 on success, non-0 otherwise. */ 1547int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata) 1548{ 1549 switch (msr_index) { 1550 case MSR_IA32_VMX_BASIC: 1551 *pdata = msrs->basic; 1552 break; 1553 case MSR_IA32_VMX_TRUE_PINBASED_CTLS: 1554 case MSR_IA32_VMX_PINBASED_CTLS: 1555 *pdata = vmx_control_msr( 1556 msrs->pinbased_ctls_low, 1557 msrs->pinbased_ctls_high); 1558 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS) 1559 *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1560 break; 1561 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS: 1562 case MSR_IA32_VMX_PROCBASED_CTLS: 1563 *pdata = vmx_control_msr( 1564 msrs->procbased_ctls_low, 1565 msrs->procbased_ctls_high); 1566 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS) 1567 *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 1568 break; 1569 case MSR_IA32_VMX_TRUE_EXIT_CTLS: 1570 case MSR_IA32_VMX_EXIT_CTLS: 1571 *pdata = vmx_control_msr( 1572 msrs->exit_ctls_low, 1573 msrs->exit_ctls_high); 1574 if (msr_index == MSR_IA32_VMX_EXIT_CTLS) 1575 *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 1576 break; 1577 case MSR_IA32_VMX_TRUE_ENTRY_CTLS: 1578 case MSR_IA32_VMX_ENTRY_CTLS: 1579 *pdata = vmx_control_msr( 1580 msrs->entry_ctls_low, 1581 msrs->entry_ctls_high); 1582 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS) 1583 *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 1584 break; 1585 case MSR_IA32_VMX_MISC: 1586 *pdata = vmx_control_msr( 1587 msrs->misc_low, 1588 msrs->misc_high); 1589 break; 1590 case MSR_IA32_VMX_CR0_FIXED0: 1591 *pdata = msrs->cr0_fixed0; 1592 break; 1593 case MSR_IA32_VMX_CR0_FIXED1: 1594 *pdata = msrs->cr0_fixed1; 1595 break; 1596 case MSR_IA32_VMX_CR4_FIXED0: 1597 *pdata = msrs->cr4_fixed0; 1598 break; 1599 case MSR_IA32_VMX_CR4_FIXED1: 1600 *pdata = msrs->cr4_fixed1; 1601 break; 1602 case MSR_IA32_VMX_VMCS_ENUM: 1603 *pdata = msrs->vmcs_enum; 1604 break; 1605 case MSR_IA32_VMX_PROCBASED_CTLS2: 1606 *pdata = vmx_control_msr( 1607 msrs->secondary_ctls_low, 1608 msrs->secondary_ctls_high); 1609 break; 1610 case MSR_IA32_VMX_EPT_VPID_CAP: 1611 *pdata = msrs->ept_caps | 1612 ((u64)msrs->vpid_caps << 32); 1613 break; 1614 case MSR_IA32_VMX_VMFUNC: 1615 *pdata = msrs->vmfunc_controls; 1616 break; 1617 default: 1618 return 1; 1619 } 1620 1621 return 0; 1622} 1623 1624/* 1625 * Copy the writable VMCS shadow fields back to the VMCS12, in case they have 1626 * been modified by the L1 guest. Note, "writable" in this context means 1627 * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of 1628 * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only" 1629 * VM-exit information fields (which are actually writable if the vCPU is 1630 * configured to support "VMWRITE to any supported field in the VMCS"). 1631 */ 1632static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) 1633{ 1634 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1635 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1636 struct shadow_vmcs_field field; 1637 unsigned long val; 1638 int i; 1639 1640 if (WARN_ON(!shadow_vmcs)) 1641 return; 1642 1643 preempt_disable(); 1644 1645 vmcs_load(shadow_vmcs); 1646 1647 for (i = 0; i < max_shadow_read_write_fields; i++) { 1648 field = shadow_read_write_fields[i]; 1649 val = __vmcs_readl(field.encoding); 1650 vmcs12_write_any(vmcs12, field.encoding, field.offset, val); 1651 } 1652 1653 vmcs_clear(shadow_vmcs); 1654 vmcs_load(vmx->loaded_vmcs->vmcs); 1655 1656 preempt_enable(); 1657} 1658 1659static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) 1660{ 1661 const struct shadow_vmcs_field *fields[] = { 1662 shadow_read_write_fields, 1663 shadow_read_only_fields 1664 }; 1665 const int max_fields[] = { 1666 max_shadow_read_write_fields, 1667 max_shadow_read_only_fields 1668 }; 1669 struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; 1670 struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu); 1671 struct shadow_vmcs_field field; 1672 unsigned long val; 1673 int i, q; 1674 1675 if (WARN_ON(!shadow_vmcs)) 1676 return; 1677 1678 vmcs_load(shadow_vmcs); 1679 1680 for (q = 0; q < ARRAY_SIZE(fields); q++) { 1681 for (i = 0; i < max_fields[q]; i++) { 1682 field = fields[q][i]; 1683 val = vmcs12_read_any(vmcs12, field.encoding, 1684 field.offset); 1685 __vmcs_writel(field.encoding, val); 1686 } 1687 } 1688 1689 vmcs_clear(shadow_vmcs); 1690 vmcs_load(vmx->loaded_vmcs->vmcs); 1691} 1692 1693static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) 1694{ 1695#ifdef CONFIG_KVM_HYPERV 1696 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1697 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 1698 struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(&vmx->vcpu); 1699 1700 /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */ 1701 vmcs12->tpr_threshold = evmcs->tpr_threshold; 1702 vmcs12->guest_rip = evmcs->guest_rip; 1703 1704 if (unlikely(!(hv_clean_fields & 1705 HV_VMX_ENLIGHTENED_CLEAN_FIELD_ENLIGHTENMENTSCONTROL))) { 1706 hv_vcpu->nested.pa_page_gpa = evmcs->partition_assist_page; 1707 hv_vcpu->nested.vm_id = evmcs->hv_vm_id; 1708 hv_vcpu->nested.vp_id = evmcs->hv_vp_id; 1709 } 1710 1711 if (unlikely(!(hv_clean_fields & 1712 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { 1713 vmcs12->guest_rsp = evmcs->guest_rsp; 1714 vmcs12->guest_rflags = evmcs->guest_rflags; 1715 vmcs12->guest_interruptibility_info = 1716 evmcs->guest_interruptibility_info; 1717 /* 1718 * Not present in struct vmcs12: 1719 * vmcs12->guest_ssp = evmcs->guest_ssp; 1720 */ 1721 } 1722 1723 if (unlikely(!(hv_clean_fields & 1724 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { 1725 vmcs12->cpu_based_vm_exec_control = 1726 evmcs->cpu_based_vm_exec_control; 1727 } 1728 1729 if (unlikely(!(hv_clean_fields & 1730 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { 1731 vmcs12->exception_bitmap = evmcs->exception_bitmap; 1732 } 1733 1734 if (unlikely(!(hv_clean_fields & 1735 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { 1736 vmcs12->vm_entry_controls = evmcs->vm_entry_controls; 1737 } 1738 1739 if (unlikely(!(hv_clean_fields & 1740 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { 1741 vmcs12->vm_entry_intr_info_field = 1742 evmcs->vm_entry_intr_info_field; 1743 vmcs12->vm_entry_exception_error_code = 1744 evmcs->vm_entry_exception_error_code; 1745 vmcs12->vm_entry_instruction_len = 1746 evmcs->vm_entry_instruction_len; 1747 } 1748 1749 if (unlikely(!(hv_clean_fields & 1750 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { 1751 vmcs12->host_ia32_pat = evmcs->host_ia32_pat; 1752 vmcs12->host_ia32_efer = evmcs->host_ia32_efer; 1753 vmcs12->host_cr0 = evmcs->host_cr0; 1754 vmcs12->host_cr3 = evmcs->host_cr3; 1755 vmcs12->host_cr4 = evmcs->host_cr4; 1756 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp; 1757 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip; 1758 vmcs12->host_rip = evmcs->host_rip; 1759 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs; 1760 vmcs12->host_es_selector = evmcs->host_es_selector; 1761 vmcs12->host_cs_selector = evmcs->host_cs_selector; 1762 vmcs12->host_ss_selector = evmcs->host_ss_selector; 1763 vmcs12->host_ds_selector = evmcs->host_ds_selector; 1764 vmcs12->host_fs_selector = evmcs->host_fs_selector; 1765 vmcs12->host_gs_selector = evmcs->host_gs_selector; 1766 vmcs12->host_tr_selector = evmcs->host_tr_selector; 1767 vmcs12->host_ia32_perf_global_ctrl = evmcs->host_ia32_perf_global_ctrl; 1768 /* 1769 * Not present in struct vmcs12: 1770 * vmcs12->host_ia32_s_cet = evmcs->host_ia32_s_cet; 1771 * vmcs12->host_ssp = evmcs->host_ssp; 1772 * vmcs12->host_ia32_int_ssp_table_addr = evmcs->host_ia32_int_ssp_table_addr; 1773 */ 1774 } 1775 1776 if (unlikely(!(hv_clean_fields & 1777 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { 1778 vmcs12->pin_based_vm_exec_control = 1779 evmcs->pin_based_vm_exec_control; 1780 vmcs12->vm_exit_controls = evmcs->vm_exit_controls; 1781 vmcs12->secondary_vm_exec_control = 1782 evmcs->secondary_vm_exec_control; 1783 } 1784 1785 if (unlikely(!(hv_clean_fields & 1786 HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { 1787 vmcs12->io_bitmap_a = evmcs->io_bitmap_a; 1788 vmcs12->io_bitmap_b = evmcs->io_bitmap_b; 1789 } 1790 1791 if (unlikely(!(hv_clean_fields & 1792 HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { 1793 vmcs12->msr_bitmap = evmcs->msr_bitmap; 1794 } 1795 1796 if (unlikely(!(hv_clean_fields & 1797 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { 1798 vmcs12->guest_es_base = evmcs->guest_es_base; 1799 vmcs12->guest_cs_base = evmcs->guest_cs_base; 1800 vmcs12->guest_ss_base = evmcs->guest_ss_base; 1801 vmcs12->guest_ds_base = evmcs->guest_ds_base; 1802 vmcs12->guest_fs_base = evmcs->guest_fs_base; 1803 vmcs12->guest_gs_base = evmcs->guest_gs_base; 1804 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base; 1805 vmcs12->guest_tr_base = evmcs->guest_tr_base; 1806 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base; 1807 vmcs12->guest_idtr_base = evmcs->guest_idtr_base; 1808 vmcs12->guest_es_limit = evmcs->guest_es_limit; 1809 vmcs12->guest_cs_limit = evmcs->guest_cs_limit; 1810 vmcs12->guest_ss_limit = evmcs->guest_ss_limit; 1811 vmcs12->guest_ds_limit = evmcs->guest_ds_limit; 1812 vmcs12->guest_fs_limit = evmcs->guest_fs_limit; 1813 vmcs12->guest_gs_limit = evmcs->guest_gs_limit; 1814 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit; 1815 vmcs12->guest_tr_limit = evmcs->guest_tr_limit; 1816 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit; 1817 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit; 1818 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes; 1819 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes; 1820 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes; 1821 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes; 1822 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes; 1823 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes; 1824 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes; 1825 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes; 1826 vmcs12->guest_es_selector = evmcs->guest_es_selector; 1827 vmcs12->guest_cs_selector = evmcs->guest_cs_selector; 1828 vmcs12->guest_ss_selector = evmcs->guest_ss_selector; 1829 vmcs12->guest_ds_selector = evmcs->guest_ds_selector; 1830 vmcs12->guest_fs_selector = evmcs->guest_fs_selector; 1831 vmcs12->guest_gs_selector = evmcs->guest_gs_selector; 1832 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector; 1833 vmcs12->guest_tr_selector = evmcs->guest_tr_selector; 1834 } 1835 1836 if (unlikely(!(hv_clean_fields & 1837 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { 1838 vmcs12->tsc_offset = evmcs->tsc_offset; 1839 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; 1840 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; 1841 vmcs12->encls_exiting_bitmap = evmcs->encls_exiting_bitmap; 1842 vmcs12->tsc_multiplier = evmcs->tsc_multiplier; 1843 } 1844 1845 if (unlikely(!(hv_clean_fields & 1846 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { 1847 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; 1848 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; 1849 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow; 1850 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow; 1851 vmcs12->guest_cr0 = evmcs->guest_cr0; 1852 vmcs12->guest_cr3 = evmcs->guest_cr3; 1853 vmcs12->guest_cr4 = evmcs->guest_cr4; 1854 vmcs12->guest_dr7 = evmcs->guest_dr7; 1855 } 1856 1857 if (unlikely(!(hv_clean_fields & 1858 HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { 1859 vmcs12->host_fs_base = evmcs->host_fs_base; 1860 vmcs12->host_gs_base = evmcs->host_gs_base; 1861 vmcs12->host_tr_base = evmcs->host_tr_base; 1862 vmcs12->host_gdtr_base = evmcs->host_gdtr_base; 1863 vmcs12->host_idtr_base = evmcs->host_idtr_base; 1864 vmcs12->host_rsp = evmcs->host_rsp; 1865 } 1866 1867 if (unlikely(!(hv_clean_fields & 1868 HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { 1869 vmcs12->ept_pointer = evmcs->ept_pointer; 1870 vmcs12->virtual_processor_id = evmcs->virtual_processor_id; 1871 } 1872 1873 if (unlikely(!(hv_clean_fields & 1874 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { 1875 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; 1876 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; 1877 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat; 1878 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer; 1879 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0; 1880 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1; 1881 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2; 1882 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3; 1883 vmcs12->guest_pending_dbg_exceptions = 1884 evmcs->guest_pending_dbg_exceptions; 1885 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp; 1886 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip; 1887 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs; 1888 vmcs12->guest_activity_state = evmcs->guest_activity_state; 1889 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs; 1890 vmcs12->guest_ia32_perf_global_ctrl = evmcs->guest_ia32_perf_global_ctrl; 1891 /* 1892 * Not present in struct vmcs12: 1893 * vmcs12->guest_ia32_s_cet = evmcs->guest_ia32_s_cet; 1894 * vmcs12->guest_ia32_lbr_ctl = evmcs->guest_ia32_lbr_ctl; 1895 * vmcs12->guest_ia32_int_ssp_table_addr = evmcs->guest_ia32_int_ssp_table_addr; 1896 */ 1897 } 1898 1899 /* 1900 * Not used? 1901 * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; 1902 * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; 1903 * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; 1904 * vmcs12->page_fault_error_code_mask = 1905 * evmcs->page_fault_error_code_mask; 1906 * vmcs12->page_fault_error_code_match = 1907 * evmcs->page_fault_error_code_match; 1908 * vmcs12->cr3_target_count = evmcs->cr3_target_count; 1909 * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count; 1910 * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count; 1911 * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count; 1912 */ 1913 1914 /* 1915 * Read only fields: 1916 * vmcs12->guest_physical_address = evmcs->guest_physical_address; 1917 * vmcs12->vm_instruction_error = evmcs->vm_instruction_error; 1918 * vmcs12->vm_exit_reason = evmcs->vm_exit_reason; 1919 * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info; 1920 * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code; 1921 * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field; 1922 * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code; 1923 * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len; 1924 * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info; 1925 * vmcs12->exit_qualification = evmcs->exit_qualification; 1926 * vmcs12->guest_linear_address = evmcs->guest_linear_address; 1927 * 1928 * Not present in struct vmcs12: 1929 * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx; 1930 * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi; 1931 * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi; 1932 * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; 1933 */ 1934 1935 return; 1936#else /* CONFIG_KVM_HYPERV */ 1937 KVM_BUG_ON(1, vmx->vcpu.kvm); 1938#endif /* CONFIG_KVM_HYPERV */ 1939} 1940 1941static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) 1942{ 1943#ifdef CONFIG_KVM_HYPERV 1944 struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; 1945 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 1946 1947 /* 1948 * Should not be changed by KVM: 1949 * 1950 * evmcs->host_es_selector = vmcs12->host_es_selector; 1951 * evmcs->host_cs_selector = vmcs12->host_cs_selector; 1952 * evmcs->host_ss_selector = vmcs12->host_ss_selector; 1953 * evmcs->host_ds_selector = vmcs12->host_ds_selector; 1954 * evmcs->host_fs_selector = vmcs12->host_fs_selector; 1955 * evmcs->host_gs_selector = vmcs12->host_gs_selector; 1956 * evmcs->host_tr_selector = vmcs12->host_tr_selector; 1957 * evmcs->host_ia32_pat = vmcs12->host_ia32_pat; 1958 * evmcs->host_ia32_efer = vmcs12->host_ia32_efer; 1959 * evmcs->host_cr0 = vmcs12->host_cr0; 1960 * evmcs->host_cr3 = vmcs12->host_cr3; 1961 * evmcs->host_cr4 = vmcs12->host_cr4; 1962 * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp; 1963 * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip; 1964 * evmcs->host_rip = vmcs12->host_rip; 1965 * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs; 1966 * evmcs->host_fs_base = vmcs12->host_fs_base; 1967 * evmcs->host_gs_base = vmcs12->host_gs_base; 1968 * evmcs->host_tr_base = vmcs12->host_tr_base; 1969 * evmcs->host_gdtr_base = vmcs12->host_gdtr_base; 1970 * evmcs->host_idtr_base = vmcs12->host_idtr_base; 1971 * evmcs->host_rsp = vmcs12->host_rsp; 1972 * sync_vmcs02_to_vmcs12() doesn't read these: 1973 * evmcs->io_bitmap_a = vmcs12->io_bitmap_a; 1974 * evmcs->io_bitmap_b = vmcs12->io_bitmap_b; 1975 * evmcs->msr_bitmap = vmcs12->msr_bitmap; 1976 * evmcs->ept_pointer = vmcs12->ept_pointer; 1977 * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap; 1978 * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; 1979 * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; 1980 * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; 1981 * evmcs->tpr_threshold = vmcs12->tpr_threshold; 1982 * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; 1983 * evmcs->exception_bitmap = vmcs12->exception_bitmap; 1984 * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer; 1985 * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control; 1986 * evmcs->vm_exit_controls = vmcs12->vm_exit_controls; 1987 * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control; 1988 * evmcs->page_fault_error_code_mask = 1989 * vmcs12->page_fault_error_code_mask; 1990 * evmcs->page_fault_error_code_match = 1991 * vmcs12->page_fault_error_code_match; 1992 * evmcs->cr3_target_count = vmcs12->cr3_target_count; 1993 * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr; 1994 * evmcs->tsc_offset = vmcs12->tsc_offset; 1995 * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl; 1996 * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask; 1997 * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask; 1998 * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow; 1999 * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow; 2000 * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count; 2001 * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count; 2002 * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count; 2003 * evmcs->guest_ia32_perf_global_ctrl = vmcs12->guest_ia32_perf_global_ctrl; 2004 * evmcs->host_ia32_perf_global_ctrl = vmcs12->host_ia32_perf_global_ctrl; 2005 * evmcs->encls_exiting_bitmap = vmcs12->encls_exiting_bitmap; 2006 * evmcs->tsc_multiplier = vmcs12->tsc_multiplier; 2007 * 2008 * Not present in struct vmcs12: 2009 * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx; 2010 * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi; 2011 * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi; 2012 * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip; 2013 * evmcs->host_ia32_s_cet = vmcs12->host_ia32_s_cet; 2014 * evmcs->host_ssp = vmcs12->host_ssp; 2015 * evmcs->host_ia32_int_ssp_table_addr = vmcs12->host_ia32_int_ssp_table_addr; 2016 * evmcs->guest_ia32_s_cet = vmcs12->guest_ia32_s_cet; 2017 * evmcs->guest_ia32_lbr_ctl = vmcs12->guest_ia32_lbr_ctl; 2018 * evmcs->guest_ia32_int_ssp_table_addr = vmcs12->guest_ia32_int_ssp_table_addr; 2019 * evmcs->guest_ssp = vmcs12->guest_ssp; 2020 */ 2021 2022 evmcs->guest_es_selector = vmcs12->guest_es_selector; 2023 evmcs->guest_cs_selector = vmcs12->guest_cs_selector; 2024 evmcs->guest_ss_selector = vmcs12->guest_ss_selector; 2025 evmcs->guest_ds_selector = vmcs12->guest_ds_selector; 2026 evmcs->guest_fs_selector = vmcs12->guest_fs_selector; 2027 evmcs->guest_gs_selector = vmcs12->guest_gs_selector; 2028 evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector; 2029 evmcs->guest_tr_selector = vmcs12->guest_tr_selector; 2030 2031 evmcs->guest_es_limit = vmcs12->guest_es_limit; 2032 evmcs->guest_cs_limit = vmcs12->guest_cs_limit; 2033 evmcs->guest_ss_limit = vmcs12->guest_ss_limit; 2034 evmcs->guest_ds_limit = vmcs12->guest_ds_limit; 2035 evmcs->guest_fs_limit = vmcs12->guest_fs_limit; 2036 evmcs->guest_gs_limit = vmcs12->guest_gs_limit; 2037 evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit; 2038 evmcs->guest_tr_limit = vmcs12->guest_tr_limit; 2039 evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit; 2040 evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit; 2041 2042 evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes; 2043 evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes; 2044 evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes; 2045 evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes; 2046 evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes; 2047 evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes; 2048 evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes; 2049 evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes; 2050 2051 evmcs->guest_es_base = vmcs12->guest_es_base; 2052 evmcs->guest_cs_base = vmcs12->guest_cs_base; 2053 evmcs->guest_ss_base = vmcs12->guest_ss_base; 2054 evmcs->guest_ds_base = vmcs12->guest_ds_base; 2055 evmcs->guest_fs_base = vmcs12->guest_fs_base; 2056 evmcs->guest_gs_base = vmcs12->guest_gs_base; 2057 evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base; 2058 evmcs->guest_tr_base = vmcs12->guest_tr_base; 2059 evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base; 2060 evmcs->guest_idtr_base = vmcs12->guest_idtr_base; 2061 2062 evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat; 2063 evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer; 2064 2065 evmcs->guest_pdptr0 = vmcs12->guest_pdptr0; 2066 evmcs->guest_pdptr1 = vmcs12->guest_pdptr1; 2067 evmcs->guest_pdptr2 = vmcs12->guest_pdptr2; 2068 evmcs->guest_pdptr3 = vmcs12->guest_pdptr3; 2069 2070 evmcs->guest_pending_dbg_exceptions = 2071 vmcs12->guest_pending_dbg_exceptions; 2072 evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp; 2073 evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip; 2074 2075 evmcs->guest_activity_state = vmcs12->guest_activity_state; 2076 evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs; 2077 2078 evmcs->guest_cr0 = vmcs12->guest_cr0; 2079 evmcs->guest_cr3 = vmcs12->guest_cr3; 2080 evmcs->guest_cr4 = vmcs12->guest_cr4; 2081 evmcs->guest_dr7 = vmcs12->guest_dr7; 2082 2083 evmcs->guest_physical_address = vmcs12->guest_physical_address; 2084 2085 evmcs->vm_instruction_error = vmcs12->vm_instruction_error; 2086 evmcs->vm_exit_reason = vmcs12->vm_exit_reason; 2087 evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info; 2088 evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code; 2089 evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field; 2090 evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code; 2091 evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len; 2092 evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info; 2093 2094 evmcs->exit_qualification = vmcs12->exit_qualification; 2095 2096 evmcs->guest_linear_address = vmcs12->guest_linear_address; 2097 evmcs->guest_rsp = vmcs12->guest_rsp; 2098 evmcs->guest_rflags = vmcs12->guest_rflags; 2099 2100 evmcs->guest_interruptibility_info = 2101 vmcs12->guest_interruptibility_info; 2102 evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control; 2103 evmcs->vm_entry_controls = vmcs12->vm_entry_controls; 2104 evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field; 2105 evmcs->vm_entry_exception_error_code = 2106 vmcs12->vm_entry_exception_error_code; 2107 evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len; 2108 2109 evmcs->guest_rip = vmcs12->guest_rip; 2110 2111 evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; 2112 2113 return; 2114#else /* CONFIG_KVM_HYPERV */ 2115 KVM_BUG_ON(1, vmx->vcpu.kvm); 2116#endif /* CONFIG_KVM_HYPERV */ 2117} 2118 2119/* 2120 * This is an equivalent of the nested hypervisor executing the vmptrld 2121 * instruction. 2122 */ 2123static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( 2124 struct kvm_vcpu *vcpu, bool from_launch) 2125{ 2126#ifdef CONFIG_KVM_HYPERV 2127 struct vcpu_vmx *vmx = to_vmx(vcpu); 2128 bool evmcs_gpa_changed = false; 2129 u64 evmcs_gpa; 2130 2131 if (likely(!guest_cpu_cap_has_evmcs(vcpu))) 2132 return EVMPTRLD_DISABLED; 2133 2134 evmcs_gpa = nested_get_evmptr(vcpu); 2135 if (!evmptr_is_valid(evmcs_gpa)) { 2136 nested_release_evmcs(vcpu); 2137 return EVMPTRLD_DISABLED; 2138 } 2139 2140 if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { 2141 vmx->nested.current_vmptr = INVALID_GPA; 2142 2143 nested_release_evmcs(vcpu); 2144 2145 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa), 2146 &vmx->nested.hv_evmcs_map)) 2147 return EVMPTRLD_ERROR; 2148 2149 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva; 2150 2151 /* 2152 * Currently, KVM only supports eVMCS version 1 2153 * (== KVM_EVMCS_VERSION) and thus we expect guest to set this 2154 * value to first u32 field of eVMCS which should specify eVMCS 2155 * VersionNumber. 2156 * 2157 * Guest should be aware of supported eVMCS versions by host by 2158 * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is 2159 * expected to set this CPUID leaf according to the value 2160 * returned in vmcs_version from nested_enable_evmcs(). 2161 * 2162 * However, it turns out that Microsoft Hyper-V fails to comply 2163 * to their own invented interface: When Hyper-V use eVMCS, it 2164 * just sets first u32 field of eVMCS to revision_id specified 2165 * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number 2166 * which is one of the supported versions specified in 2167 * CPUID.0x4000000A.EAX[0:15]. 2168 * 2169 * To overcome Hyper-V bug, we accept here either a supported 2170 * eVMCS version or VMCS12 revision_id as valid values for first 2171 * u32 field of eVMCS. 2172 */ 2173 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) && 2174 (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) { 2175 nested_release_evmcs(vcpu); 2176 return EVMPTRLD_VMFAIL; 2177 } 2178 2179 vmx->nested.hv_evmcs_vmptr = evmcs_gpa; 2180 2181 evmcs_gpa_changed = true; 2182 /* 2183 * Unlike normal vmcs12, enlightened vmcs12 is not fully 2184 * reloaded from guest's memory (read only fields, fields not 2185 * present in struct hv_enlightened_vmcs, ...). Make sure there 2186 * are no leftovers. 2187 */ 2188 if (from_launch) { 2189 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2190 memset(vmcs12, 0, sizeof(*vmcs12)); 2191 vmcs12->hdr.revision_id = VMCS12_REVISION; 2192 } 2193 2194 } 2195 2196 /* 2197 * Clean fields data can't be used on VMLAUNCH and when we switch 2198 * between different L2 guests as KVM keeps a single VMCS12 per L1. 2199 */ 2200 if (from_launch || evmcs_gpa_changed) { 2201 vmx->nested.hv_evmcs->hv_clean_fields &= 2202 ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2203 2204 vmx->nested.force_msr_bitmap_recalc = true; 2205 } 2206 2207 return EVMPTRLD_SUCCEEDED; 2208#else 2209 return EVMPTRLD_DISABLED; 2210#endif 2211} 2212 2213void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) 2214{ 2215 struct vcpu_vmx *vmx = to_vmx(vcpu); 2216 2217 if (nested_vmx_is_evmptr12_valid(vmx)) 2218 copy_vmcs12_to_enlightened(vmx); 2219 else 2220 copy_vmcs12_to_shadow(vmx); 2221 2222 vmx->nested.need_vmcs12_to_shadow_sync = false; 2223} 2224 2225static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) 2226{ 2227 struct vcpu_vmx *vmx = 2228 container_of(timer, struct vcpu_vmx, nested.preemption_timer); 2229 2230 vmx->nested.preemption_timer_expired = true; 2231 kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu); 2232 kvm_vcpu_kick(&vmx->vcpu); 2233 2234 return HRTIMER_NORESTART; 2235} 2236 2237static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) 2238{ 2239 struct vcpu_vmx *vmx = to_vmx(vcpu); 2240 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 2241 2242 u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> 2243 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2244 2245 if (!vmx->nested.has_preemption_timer_deadline) { 2246 vmx->nested.preemption_timer_deadline = 2247 vmcs12->vmx_preemption_timer_value + l1_scaled_tsc; 2248 vmx->nested.has_preemption_timer_deadline = true; 2249 } 2250 return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; 2251} 2252 2253static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, 2254 u64 preemption_timeout) 2255{ 2256 struct vcpu_vmx *vmx = to_vmx(vcpu); 2257 2258 /* 2259 * A timer value of zero is architecturally guaranteed to cause 2260 * a VMExit prior to executing any instructions in the guest. 2261 */ 2262 if (preemption_timeout == 0) { 2263 vmx_preemption_timer_fn(&vmx->nested.preemption_timer); 2264 return; 2265 } 2266 2267 if (vcpu->arch.virtual_tsc_khz == 0) 2268 return; 2269 2270 preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 2271 preemption_timeout *= 1000000; 2272 do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); 2273 hrtimer_start(&vmx->nested.preemption_timer, 2274 ktime_add_ns(ktime_get(), preemption_timeout), 2275 HRTIMER_MODE_ABS_PINNED); 2276} 2277 2278static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2279{ 2280 if (vmx->nested.nested_run_pending && 2281 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) 2282 return vmcs12->guest_ia32_efer; 2283 else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) 2284 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME); 2285 else 2286 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME); 2287} 2288 2289static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) 2290{ 2291 struct kvm *kvm = vmx->vcpu.kvm; 2292 2293 /* 2294 * If vmcs02 hasn't been initialized, set the constant vmcs02 state 2295 * according to L0's settings (vmcs12 is irrelevant here). Host 2296 * fields that come from L0 and are not constant, e.g. HOST_CR3, 2297 * will be set as needed prior to VMLAUNCH/VMRESUME. 2298 */ 2299 if (vmx->nested.vmcs02_initialized) 2300 return; 2301 vmx->nested.vmcs02_initialized = true; 2302 2303 if (vmx->ve_info) 2304 vmcs_write64(VE_INFORMATION_ADDRESS, __pa(vmx->ve_info)); 2305 2306 /* All VMFUNCs are currently emulated through L0 vmexits. */ 2307 if (cpu_has_vmx_vmfunc()) 2308 vmcs_write64(VM_FUNCTION_CONTROL, 0); 2309 2310 if (cpu_has_vmx_posted_intr()) 2311 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR); 2312 2313 if (cpu_has_vmx_msr_bitmap()) 2314 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); 2315 2316 /* 2317 * PML is emulated for L2, but never enabled in hardware as the MMU 2318 * handles A/D emulation. Disabling PML for L2 also avoids having to 2319 * deal with filtering out L2 GPAs from the buffer. 2320 */ 2321 if (enable_pml) { 2322 vmcs_write64(PML_ADDRESS, 0); 2323 vmcs_write16(GUEST_PML_INDEX, -1); 2324 } 2325 2326 if (cpu_has_vmx_encls_vmexit()) 2327 vmcs_write64(ENCLS_EXITING_BITMAP, INVALID_GPA); 2328 2329 if (kvm_notify_vmexit_enabled(kvm)) 2330 vmcs_write32(NOTIFY_WINDOW, kvm->arch.notify_window); 2331 2332 /* 2333 * Set the MSR load/store lists to match L0's settings. Only the 2334 * addresses are constant (for vmcs02), the counts can change based 2335 * on L2's behavior, e.g. switching to/from long mode. 2336 */ 2337 vmcs_write64(VM_EXIT_MSR_STORE_ADDR, __pa(vmx->msr_autostore.guest.val)); 2338 vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val)); 2339 vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); 2340 2341 vmx_set_constant_host_state(vmx); 2342} 2343 2344static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, 2345 struct vmcs12 *vmcs12) 2346{ 2347 prepare_vmcs02_constant_state(vmx); 2348 2349 vmcs_write64(VMCS_LINK_POINTER, INVALID_GPA); 2350 2351 /* 2352 * If VPID is disabled, then guest TLB accesses use VPID=0, i.e. the 2353 * same VPID as the host. Emulate this behavior by using vpid01 for L2 2354 * if VPID is disabled in vmcs12. Note, if VPID is disabled, VM-Enter 2355 * and VM-Exit are architecturally required to flush VPID=0, but *only* 2356 * VPID=0. I.e. using vpid02 would be ok (so long as KVM emulates the 2357 * required flushes), but doing so would cause KVM to over-flush. E.g. 2358 * if L1 runs L2 X with VPID12=1, then runs L2 Y with VPID12 disabled, 2359 * and then runs L2 X again, then KVM can and should retain TLB entries 2360 * for VPID12=1. 2361 */ 2362 if (enable_vpid) { 2363 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02) 2364 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02); 2365 else 2366 vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid); 2367 } 2368} 2369 2370static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs01, 2371 struct vmcs12 *vmcs12) 2372{ 2373 u32 exec_control; 2374 u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); 2375 2376 if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) 2377 prepare_vmcs02_early_rare(vmx, vmcs12); 2378 2379 /* 2380 * PIN CONTROLS 2381 */ 2382 exec_control = __pin_controls_get(vmcs01); 2383 exec_control |= (vmcs12->pin_based_vm_exec_control & 2384 ~PIN_BASED_VMX_PREEMPTION_TIMER); 2385 2386 /* Posted interrupts setting is only taken from vmcs12. */ 2387 vmx->nested.pi_pending = false; 2388 if (nested_cpu_has_posted_intr(vmcs12)) { 2389 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv; 2390 } else { 2391 vmx->nested.posted_intr_nv = -1; 2392 exec_control &= ~PIN_BASED_POSTED_INTR; 2393 } 2394 pin_controls_set(vmx, exec_control); 2395 2396 /* 2397 * EXEC CONTROLS 2398 */ 2399 exec_control = __exec_controls_get(vmcs01); /* L0's desires */ 2400 exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING; 2401 exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING; 2402 exec_control &= ~CPU_BASED_TPR_SHADOW; 2403 exec_control |= vmcs12->cpu_based_vm_exec_control; 2404 2405 vmx->nested.l1_tpr_threshold = -1; 2406 if (exec_control & CPU_BASED_TPR_SHADOW) 2407 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold); 2408#ifdef CONFIG_X86_64 2409 else 2410 exec_control |= CPU_BASED_CR8_LOAD_EXITING | 2411 CPU_BASED_CR8_STORE_EXITING; 2412#endif 2413 2414 /* 2415 * A vmexit (to either L1 hypervisor or L0 userspace) is always needed 2416 * for I/O port accesses. 2417 */ 2418 exec_control |= CPU_BASED_UNCOND_IO_EXITING; 2419 exec_control &= ~CPU_BASED_USE_IO_BITMAPS; 2420 2421 /* 2422 * This bit will be computed in nested_get_vmcs12_pages, because 2423 * we do not have access to L1's MSR bitmap yet. For now, keep 2424 * the same bit as before, hoping to avoid multiple VMWRITEs that 2425 * only set/clear this bit. 2426 */ 2427 exec_control &= ~CPU_BASED_USE_MSR_BITMAPS; 2428 exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS; 2429 2430 exec_controls_set(vmx, exec_control); 2431 2432 /* 2433 * SECONDARY EXEC CONTROLS 2434 */ 2435 if (cpu_has_secondary_exec_ctrls()) { 2436 exec_control = __secondary_exec_controls_get(vmcs01); 2437 2438 /* Take the following fields only from vmcs12 */ 2439 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | 2440 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 2441 SECONDARY_EXEC_ENABLE_INVPCID | 2442 SECONDARY_EXEC_ENABLE_RDTSCP | 2443 SECONDARY_EXEC_ENABLE_XSAVES | 2444 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | 2445 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 2446 SECONDARY_EXEC_APIC_REGISTER_VIRT | 2447 SECONDARY_EXEC_ENABLE_VMFUNC | 2448 SECONDARY_EXEC_DESC); 2449 2450 if (nested_cpu_has(vmcs12, 2451 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) 2452 exec_control |= vmcs12->secondary_vm_exec_control; 2453 2454 /* PML is emulated and never enabled in hardware for L2. */ 2455 exec_control &= ~SECONDARY_EXEC_ENABLE_PML; 2456 2457 /* VMCS shadowing for L2 is emulated for now */ 2458 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; 2459 2460 /* 2461 * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4() 2462 * will not have to rewrite the controls just for this bit. 2463 */ 2464 if (vmx_umip_emulated() && (vmcs12->guest_cr4 & X86_CR4_UMIP)) 2465 exec_control |= SECONDARY_EXEC_DESC; 2466 2467 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY) 2468 vmcs_write16(GUEST_INTR_STATUS, 2469 vmcs12->guest_intr_status); 2470 2471 if (!nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST)) 2472 exec_control &= ~SECONDARY_EXEC_UNRESTRICTED_GUEST; 2473 2474 if (exec_control & SECONDARY_EXEC_ENCLS_EXITING) 2475 vmx_write_encls_bitmap(&vmx->vcpu, vmcs12); 2476 2477 secondary_exec_controls_set(vmx, exec_control); 2478 } 2479 2480 /* 2481 * ENTRY CONTROLS 2482 * 2483 * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE 2484 * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate 2485 * on the related bits (if supported by the CPU) in the hope that 2486 * we can avoid VMWrites during vmx_set_efer(). 2487 * 2488 * Similarly, take vmcs01's PERF_GLOBAL_CTRL in the hope that if KVM is 2489 * loading PERF_GLOBAL_CTRL via the VMCS for L1, then KVM will want to 2490 * do the same for L2. 2491 */ 2492 exec_control = __vm_entry_controls_get(vmcs01); 2493 exec_control |= (vmcs12->vm_entry_controls & 2494 ~VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); 2495 exec_control &= ~(VM_ENTRY_IA32E_MODE | VM_ENTRY_LOAD_IA32_EFER); 2496 if (cpu_has_load_ia32_efer()) { 2497 if (guest_efer & EFER_LMA) 2498 exec_control |= VM_ENTRY_IA32E_MODE; 2499 if (guest_efer != kvm_host.efer) 2500 exec_control |= VM_ENTRY_LOAD_IA32_EFER; 2501 } 2502 vm_entry_controls_set(vmx, exec_control); 2503 2504 /* 2505 * EXIT CONTROLS 2506 * 2507 * L2->L1 exit controls are emulated - the hardware exit is to L0 so 2508 * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER 2509 * bits may be modified by vmx_set_efer() in prepare_vmcs02(). 2510 */ 2511 exec_control = __vm_exit_controls_get(vmcs01); 2512 if (cpu_has_load_ia32_efer() && guest_efer != kvm_host.efer) 2513 exec_control |= VM_EXIT_LOAD_IA32_EFER; 2514 else 2515 exec_control &= ~VM_EXIT_LOAD_IA32_EFER; 2516 vm_exit_controls_set(vmx, exec_control); 2517 2518 /* 2519 * Interrupt/Exception Fields 2520 */ 2521 if (vmx->nested.nested_run_pending) { 2522 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 2523 vmcs12->vm_entry_intr_info_field); 2524 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 2525 vmcs12->vm_entry_exception_error_code); 2526 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 2527 vmcs12->vm_entry_instruction_len); 2528 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 2529 vmcs12->guest_interruptibility_info); 2530 vmx->loaded_vmcs->nmi_known_unmasked = 2531 !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI); 2532 } else { 2533 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); 2534 } 2535} 2536 2537static void vmcs_read_cet_state(struct kvm_vcpu *vcpu, u64 *s_cet, 2538 u64 *ssp, u64 *ssp_tbl) 2539{ 2540 if (guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) || 2541 guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) 2542 *s_cet = vmcs_readl(GUEST_S_CET); 2543 2544 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) { 2545 *ssp = vmcs_readl(GUEST_SSP); 2546 *ssp_tbl = vmcs_readl(GUEST_INTR_SSP_TABLE); 2547 } 2548} 2549 2550static void vmcs_write_cet_state(struct kvm_vcpu *vcpu, u64 s_cet, 2551 u64 ssp, u64 ssp_tbl) 2552{ 2553 if (guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) || 2554 guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) 2555 vmcs_writel(GUEST_S_CET, s_cet); 2556 2557 if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) { 2558 vmcs_writel(GUEST_SSP, ssp); 2559 vmcs_writel(GUEST_INTR_SSP_TABLE, ssp_tbl); 2560 } 2561} 2562 2563static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) 2564{ 2565 struct hv_enlightened_vmcs *hv_evmcs = nested_vmx_evmcs(vmx); 2566 2567 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2568 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) { 2569 2570 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector); 2571 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector); 2572 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector); 2573 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector); 2574 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector); 2575 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector); 2576 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector); 2577 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector); 2578 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit); 2579 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit); 2580 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit); 2581 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit); 2582 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit); 2583 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit); 2584 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit); 2585 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit); 2586 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit); 2587 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit); 2588 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes); 2589 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes); 2590 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes); 2591 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes); 2592 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes); 2593 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes); 2594 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes); 2595 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes); 2596 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base); 2597 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base); 2598 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base); 2599 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base); 2600 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base); 2601 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base); 2602 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base); 2603 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base); 2604 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base); 2605 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base); 2606 2607 vmx_segment_cache_clear(vmx); 2608 } 2609 2610 if (!hv_evmcs || !(hv_evmcs->hv_clean_fields & 2611 HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) { 2612 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs); 2613 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, 2614 vmcs12->guest_pending_dbg_exceptions); 2615 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp); 2616 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip); 2617 2618 /* 2619 * L1 may access the L2's PDPTR, so save them to construct 2620 * vmcs12 2621 */ 2622 if (enable_ept) { 2623 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2624 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2625 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2626 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2627 } 2628 2629 if (kvm_mpx_supported() && vmx->nested.nested_run_pending && 2630 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)) 2631 vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs); 2632 } 2633 2634 if (nested_cpu_has_xsaves(vmcs12)) 2635 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap); 2636 2637 /* 2638 * Whether page-faults are trapped is determined by a combination of 2639 * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF. If L0 2640 * doesn't care about page faults then we should set all of these to 2641 * L1's desires. However, if L0 does care about (some) page faults, it 2642 * is not easy (if at all possible?) to merge L0 and L1's desires, we 2643 * simply ask to exit on each and every L2 page fault. This is done by 2644 * setting MASK=MATCH=0 and (see below) EB.PF=1. 2645 * Note that below we don't need special code to set EB.PF beyond the 2646 * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept, 2647 * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when 2648 * !enable_ept, EB.PF is 1, so the "or" will always be 1. 2649 */ 2650 if (vmx_need_pf_intercept(&vmx->vcpu)) { 2651 /* 2652 * TODO: if both L0 and L1 need the same MASK and MATCH, 2653 * go ahead and use it? 2654 */ 2655 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); 2656 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); 2657 } else { 2658 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, vmcs12->page_fault_error_code_mask); 2659 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, vmcs12->page_fault_error_code_match); 2660 } 2661 2662 if (cpu_has_vmx_apicv()) { 2663 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0); 2664 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1); 2665 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2); 2666 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3); 2667 } 2668 2669 /* 2670 * Make sure the msr_autostore list is up to date before we set the 2671 * count in the vmcs02. 2672 */ 2673 prepare_vmx_msr_autostore_list(&vmx->vcpu, MSR_IA32_TSC); 2674 2675 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vmx->msr_autostore.guest.nr); 2676 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 2677 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 2678 2679 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_CET_STATE) 2680 vmcs_write_cet_state(&vmx->vcpu, vmcs12->guest_s_cet, 2681 vmcs12->guest_ssp, vmcs12->guest_ssp_tbl); 2682 2683 set_cr4_guest_host_mask(vmx); 2684} 2685 2686/* 2687 * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested 2688 * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it 2689 * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2 2690 * guest in a way that will both be appropriate to L1's requests, and our 2691 * needs. In addition to modifying the active vmcs (which is vmcs02), this 2692 * function also has additional necessary side-effects, like setting various 2693 * vcpu->arch fields. 2694 * Returns 0 on success, 1 on failure. Invalid state exit qualification code 2695 * is assigned to entry_failure_code on failure. 2696 */ 2697static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 2698 bool from_vmentry, 2699 enum vm_entry_failure_code *entry_failure_code) 2700{ 2701 struct vcpu_vmx *vmx = to_vmx(vcpu); 2702 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 2703 bool load_guest_pdptrs_vmcs12 = false; 2704 2705 if (vmx->nested.dirty_vmcs12 || nested_vmx_is_evmptr12_valid(vmx)) { 2706 prepare_vmcs02_rare(vmx, vmcs12); 2707 vmx->nested.dirty_vmcs12 = false; 2708 2709 load_guest_pdptrs_vmcs12 = !nested_vmx_is_evmptr12_valid(vmx) || 2710 !(evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); 2711 } 2712 2713 if (vmx->nested.nested_run_pending && 2714 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) { 2715 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7); 2716 vmx_guest_debugctl_write(vcpu, vmcs12->guest_ia32_debugctl & 2717 vmx_get_supported_debugctl(vcpu, false)); 2718 } else { 2719 kvm_set_dr(vcpu, 7, vcpu->arch.dr7); 2720 vmx_guest_debugctl_write(vcpu, vmx->nested.pre_vmenter_debugctl); 2721 } 2722 2723 if (!vmx->nested.nested_run_pending || 2724 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_CET_STATE)) 2725 vmcs_write_cet_state(vcpu, vmx->nested.pre_vmenter_s_cet, 2726 vmx->nested.pre_vmenter_ssp, 2727 vmx->nested.pre_vmenter_ssp_tbl); 2728 2729 if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending || 2730 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 2731 vmcs_write64(GUEST_BNDCFGS, vmx->nested.pre_vmenter_bndcfgs); 2732 vmx_set_rflags(vcpu, vmcs12->guest_rflags); 2733 2734 /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the 2735 * bitwise-or of what L1 wants to trap for L2, and what we want to 2736 * trap. Note that CR0.TS also needs updating - we do this later. 2737 */ 2738 vmx_update_exception_bitmap(vcpu); 2739 vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask; 2740 vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); 2741 2742 if (vmx->nested.nested_run_pending && 2743 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) { 2744 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat); 2745 vcpu->arch.pat = vmcs12->guest_ia32_pat; 2746 } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) { 2747 vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat); 2748 } 2749 2750 vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( 2751 vcpu->arch.l1_tsc_offset, 2752 vmx_get_l2_tsc_offset(vcpu), 2753 vmx_get_l2_tsc_multiplier(vcpu)); 2754 2755 vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( 2756 vcpu->arch.l1_tsc_scaling_ratio, 2757 vmx_get_l2_tsc_multiplier(vcpu)); 2758 2759 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 2760 if (kvm_caps.has_tsc_control) 2761 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 2762 2763 nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); 2764 2765 if (nested_cpu_has_ept(vmcs12)) 2766 nested_ept_init_mmu_context(vcpu); 2767 2768 /* 2769 * Override the CR0/CR4 read shadows after setting the effective guest 2770 * CR0/CR4. The common helpers also set the shadows, but they don't 2771 * account for vmcs12's cr0/4_guest_host_mask. 2772 */ 2773 vmx_set_cr0(vcpu, vmcs12->guest_cr0); 2774 vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12)); 2775 2776 vmx_set_cr4(vcpu, vmcs12->guest_cr4); 2777 vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12)); 2778 2779 vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12); 2780 /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */ 2781 vmx_set_efer(vcpu, vcpu->arch.efer); 2782 2783 /* 2784 * Guest state is invalid and unrestricted guest is disabled, 2785 * which means L1 attempted VMEntry to L2 with invalid state. 2786 * Fail the VMEntry. 2787 * 2788 * However when force loading the guest state (SMM exit or 2789 * loading nested state after migration, it is possible to 2790 * have invalid guest state now, which will be later fixed by 2791 * restoring L2 register state 2792 */ 2793 if (CC(from_vmentry && !vmx_guest_state_valid(vcpu))) { 2794 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2795 return -EINVAL; 2796 } 2797 2798 /* Shadow page tables on either EPT or shadow page tables. */ 2799 if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), 2800 from_vmentry, entry_failure_code)) 2801 return -EINVAL; 2802 2803 /* 2804 * Immediately write vmcs02.GUEST_CR3. It will be propagated to vmcs12 2805 * on nested VM-Exit, which can occur without actually running L2 and 2806 * thus without hitting vmx_load_mmu_pgd(), e.g. if L1 is entering L2 with 2807 * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the 2808 * transition to HLT instead of running L2. 2809 */ 2810 if (enable_ept) 2811 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3); 2812 2813 /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */ 2814 if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) && 2815 is_pae_paging(vcpu)) { 2816 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0); 2817 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1); 2818 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2); 2819 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); 2820 } 2821 2822 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 2823 kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu)) && 2824 WARN_ON_ONCE(__kvm_emulate_msr_write(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 2825 vmcs12->guest_ia32_perf_global_ctrl))) { 2826 *entry_failure_code = ENTRY_FAIL_DEFAULT; 2827 return -EINVAL; 2828 } 2829 2830 kvm_rsp_write(vcpu, vmcs12->guest_rsp); 2831 kvm_rip_write(vcpu, vmcs12->guest_rip); 2832 2833 /* 2834 * It was observed that genuine Hyper-V running in L1 doesn't reset 2835 * 'hv_clean_fields' by itself, it only sets the corresponding dirty 2836 * bits when it changes a field in eVMCS. Mark all fields as clean 2837 * here. 2838 */ 2839 if (nested_vmx_is_evmptr12_valid(vmx)) 2840 evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; 2841 2842 return 0; 2843} 2844 2845static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12) 2846{ 2847 if (CC(!nested_cpu_has_nmi_exiting(vmcs12) && 2848 nested_cpu_has_virtual_nmis(vmcs12))) 2849 return -EINVAL; 2850 2851 if (CC(!nested_cpu_has_virtual_nmis(vmcs12) && 2852 nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING))) 2853 return -EINVAL; 2854 2855 return 0; 2856} 2857 2858static bool nested_vmx_check_eptp(struct kvm_vcpu *vcpu, u64 new_eptp) 2859{ 2860 struct vcpu_vmx *vmx = to_vmx(vcpu); 2861 2862 /* Check for memory type validity */ 2863 switch (new_eptp & VMX_EPTP_MT_MASK) { 2864 case VMX_EPTP_MT_UC: 2865 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT))) 2866 return false; 2867 break; 2868 case VMX_EPTP_MT_WB: 2869 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT))) 2870 return false; 2871 break; 2872 default: 2873 return false; 2874 } 2875 2876 /* Page-walk levels validity. */ 2877 switch (new_eptp & VMX_EPTP_PWL_MASK) { 2878 case VMX_EPTP_PWL_5: 2879 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_5_BIT))) 2880 return false; 2881 break; 2882 case VMX_EPTP_PWL_4: 2883 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_PAGE_WALK_4_BIT))) 2884 return false; 2885 break; 2886 default: 2887 return false; 2888 } 2889 2890 /* Reserved bits should not be set */ 2891 if (CC(!kvm_vcpu_is_legal_gpa(vcpu, new_eptp) || ((new_eptp >> 7) & 0x1f))) 2892 return false; 2893 2894 /* AD, if set, should be supported */ 2895 if (new_eptp & VMX_EPTP_AD_ENABLE_BIT) { 2896 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT))) 2897 return false; 2898 } 2899 2900 return true; 2901} 2902 2903/* 2904 * Checks related to VM-Execution Control Fields 2905 */ 2906static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu, 2907 struct vmcs12 *vmcs12) 2908{ 2909 struct vcpu_vmx *vmx = to_vmx(vcpu); 2910 2911 if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control, 2912 vmx->nested.msrs.pinbased_ctls_low, 2913 vmx->nested.msrs.pinbased_ctls_high)) || 2914 CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control, 2915 vmx->nested.msrs.procbased_ctls_low, 2916 vmx->nested.msrs.procbased_ctls_high))) 2917 return -EINVAL; 2918 2919 if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) && 2920 CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control, 2921 vmx->nested.msrs.secondary_ctls_low, 2922 vmx->nested.msrs.secondary_ctls_high))) 2923 return -EINVAL; 2924 2925 if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) || 2926 nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) || 2927 nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) || 2928 nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) || 2929 nested_vmx_check_apic_access_controls(vcpu, vmcs12) || 2930 nested_vmx_check_apicv_controls(vcpu, vmcs12) || 2931 nested_vmx_check_nmi_controls(vmcs12) || 2932 nested_vmx_check_pml_controls(vcpu, vmcs12) || 2933 nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) || 2934 nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) || 2935 nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) || 2936 CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id)) 2937 return -EINVAL; 2938 2939 if (!nested_cpu_has_preemption_timer(vmcs12) && 2940 nested_cpu_has_save_preemption_timer(vmcs12)) 2941 return -EINVAL; 2942 2943 if (nested_cpu_has_ept(vmcs12) && 2944 CC(!nested_vmx_check_eptp(vcpu, vmcs12->ept_pointer))) 2945 return -EINVAL; 2946 2947 if (nested_cpu_has_vmfunc(vmcs12)) { 2948 if (CC(vmcs12->vm_function_control & 2949 ~vmx->nested.msrs.vmfunc_controls)) 2950 return -EINVAL; 2951 2952 if (nested_cpu_has_eptp_switching(vmcs12)) { 2953 if (CC(!nested_cpu_has_ept(vmcs12)) || 2954 CC(!page_address_valid(vcpu, vmcs12->eptp_list_address))) 2955 return -EINVAL; 2956 } 2957 } 2958 2959 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING) && 2960 CC(!vmcs12->tsc_multiplier)) 2961 return -EINVAL; 2962 2963 return 0; 2964} 2965 2966/* 2967 * Checks related to VM-Exit Control Fields 2968 */ 2969static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu, 2970 struct vmcs12 *vmcs12) 2971{ 2972 struct vcpu_vmx *vmx = to_vmx(vcpu); 2973 2974 if (CC(!vmx_control_verify(vmcs12->vm_exit_controls, 2975 vmx->nested.msrs.exit_ctls_low, 2976 vmx->nested.msrs.exit_ctls_high)) || 2977 CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12))) 2978 return -EINVAL; 2979 2980 return 0; 2981} 2982 2983/* 2984 * Checks related to VM-Entry Control Fields 2985 */ 2986static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu, 2987 struct vmcs12 *vmcs12) 2988{ 2989 struct vcpu_vmx *vmx = to_vmx(vcpu); 2990 2991 if (CC(!vmx_control_verify(vmcs12->vm_entry_controls, 2992 vmx->nested.msrs.entry_ctls_low, 2993 vmx->nested.msrs.entry_ctls_high))) 2994 return -EINVAL; 2995 2996 /* 2997 * From the Intel SDM, volume 3: 2998 * Fields relevant to VM-entry event injection must be set properly. 2999 * These fields are the VM-entry interruption-information field, the 3000 * VM-entry exception error code, and the VM-entry instruction length. 3001 */ 3002 if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) { 3003 u32 intr_info = vmcs12->vm_entry_intr_info_field; 3004 u8 vector = intr_info & INTR_INFO_VECTOR_MASK; 3005 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK; 3006 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK; 3007 bool urg = nested_cpu_has2(vmcs12, 3008 SECONDARY_EXEC_UNRESTRICTED_GUEST); 3009 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE; 3010 3011 /* VM-entry interruption-info field: interruption type */ 3012 if (CC(intr_type == INTR_TYPE_RESERVED) || 3013 CC(intr_type == INTR_TYPE_OTHER_EVENT && 3014 !nested_cpu_supports_monitor_trap_flag(vcpu))) 3015 return -EINVAL; 3016 3017 /* VM-entry interruption-info field: vector */ 3018 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) || 3019 CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) || 3020 CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0)) 3021 return -EINVAL; 3022 3023 /* 3024 * Cannot deliver error code in real mode or if the interrupt 3025 * type is not hardware exception. For other cases, do the 3026 * consistency check only if the vCPU doesn't enumerate 3027 * VMX_BASIC_NO_HW_ERROR_CODE_CC. 3028 */ 3029 if (!prot_mode || intr_type != INTR_TYPE_HARD_EXCEPTION) { 3030 if (CC(has_error_code)) 3031 return -EINVAL; 3032 } else if (!nested_cpu_has_no_hw_errcode_cc(vcpu)) { 3033 if (CC(has_error_code != x86_exception_has_error_code(vector))) 3034 return -EINVAL; 3035 } 3036 3037 /* VM-entry exception error code */ 3038 if (CC(has_error_code && 3039 vmcs12->vm_entry_exception_error_code & GENMASK(31, 16))) 3040 return -EINVAL; 3041 3042 /* VM-entry interruption-info field: reserved bits */ 3043 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK)) 3044 return -EINVAL; 3045 3046 /* VM-entry instruction length */ 3047 switch (intr_type) { 3048 case INTR_TYPE_SOFT_EXCEPTION: 3049 case INTR_TYPE_SOFT_INTR: 3050 case INTR_TYPE_PRIV_SW_EXCEPTION: 3051 if (CC(vmcs12->vm_entry_instruction_len > X86_MAX_INSTRUCTION_LENGTH) || 3052 CC(vmcs12->vm_entry_instruction_len == 0 && 3053 CC(!nested_cpu_has_zero_length_injection(vcpu)))) 3054 return -EINVAL; 3055 } 3056 } 3057 3058 if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12)) 3059 return -EINVAL; 3060 3061 return 0; 3062} 3063 3064static int nested_vmx_check_controls(struct kvm_vcpu *vcpu, 3065 struct vmcs12 *vmcs12) 3066{ 3067 if (nested_check_vm_execution_controls(vcpu, vmcs12) || 3068 nested_check_vm_exit_controls(vcpu, vmcs12) || 3069 nested_check_vm_entry_controls(vcpu, vmcs12)) 3070 return -EINVAL; 3071 3072#ifdef CONFIG_KVM_HYPERV 3073 if (guest_cpu_cap_has_evmcs(vcpu)) 3074 return nested_evmcs_check_controls(vmcs12); 3075#endif 3076 3077 return 0; 3078} 3079 3080static int nested_vmx_check_controls_late(struct kvm_vcpu *vcpu, 3081 struct vmcs12 *vmcs12) 3082{ 3083 void *vapic = to_vmx(vcpu)->nested.virtual_apic_map.hva; 3084 u32 vtpr = vapic ? (*(u32 *)(vapic + APIC_TASKPRI)) >> 4 : 0; 3085 3086 /* 3087 * Don't bother with the consistency checks if KVM isn't configured to 3088 * WARN on missed consistency checks, as KVM needs to rely on hardware 3089 * to fully detect an illegal vTPR vs. TRP Threshold combination due to 3090 * the vTPR being writable by L1 at all times (it's an in-memory value, 3091 * not a VMCS field). I.e. even if the check passes now, it might fail 3092 * at the actual VM-Enter. 3093 * 3094 * Keying off the module param also allows treating an invalid vAPIC 3095 * mapping as a consistency check failure without increasing the risk 3096 * of breaking a "real" VM. 3097 */ 3098 if (!warn_on_missed_cc) 3099 return 0; 3100 3101 if ((exec_controls_get(to_vmx(vcpu)) & CPU_BASED_TPR_SHADOW) && 3102 nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW) && 3103 !nested_cpu_has_vid(vmcs12) && 3104 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) && 3105 (CC(!vapic) || 3106 CC((vmcs12->tpr_threshold & GENMASK(3, 0)) > (vtpr & GENMASK(3, 0))))) 3107 return -EINVAL; 3108 3109 return 0; 3110} 3111 3112static int nested_vmx_check_address_space_size(struct kvm_vcpu *vcpu, 3113 struct vmcs12 *vmcs12) 3114{ 3115#ifdef CONFIG_X86_64 3116 if (CC(!!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) != 3117 !!(vcpu->arch.efer & EFER_LMA))) 3118 return -EINVAL; 3119#endif 3120 return 0; 3121} 3122 3123static bool is_l1_noncanonical_address_on_vmexit(u64 la, struct vmcs12 *vmcs12) 3124{ 3125 /* 3126 * Check that the given linear address is canonical after a VM exit 3127 * from L2, based on HOST_CR4.LA57 value that will be loaded for L1. 3128 */ 3129 u8 l1_address_bits_on_exit = (vmcs12->host_cr4 & X86_CR4_LA57) ? 57 : 48; 3130 3131 return !__is_canonical_address(la, l1_address_bits_on_exit); 3132} 3133 3134static int nested_vmx_check_cet_state_common(struct kvm_vcpu *vcpu, u64 s_cet, 3135 u64 ssp, u64 ssp_tbl) 3136{ 3137 if (CC(!kvm_is_valid_u_s_cet(vcpu, s_cet)) || CC(!IS_ALIGNED(ssp, 4)) || 3138 CC(is_noncanonical_msr_address(ssp_tbl, vcpu))) 3139 return -EINVAL; 3140 3141 return 0; 3142} 3143 3144static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu, 3145 struct vmcs12 *vmcs12) 3146{ 3147 bool ia32e = !!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE); 3148 3149 if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) || 3150 CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) || 3151 CC(!kvm_vcpu_is_legal_cr3(vcpu, vmcs12->host_cr3))) 3152 return -EINVAL; 3153 3154 if (CC(vmcs12->host_cr4 & X86_CR4_CET && !(vmcs12->host_cr0 & X86_CR0_WP))) 3155 return -EINVAL; 3156 3157 if (CC(is_noncanonical_msr_address(vmcs12->host_ia32_sysenter_esp, vcpu)) || 3158 CC(is_noncanonical_msr_address(vmcs12->host_ia32_sysenter_eip, vcpu))) 3159 return -EINVAL; 3160 3161 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) && 3162 CC(!kvm_pat_valid(vmcs12->host_ia32_pat))) 3163 return -EINVAL; 3164 3165 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 3166 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 3167 vmcs12->host_ia32_perf_global_ctrl))) 3168 return -EINVAL; 3169 3170 if (ia32e) { 3171 if (CC(!(vmcs12->host_cr4 & X86_CR4_PAE))) 3172 return -EINVAL; 3173 } else { 3174 if (CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) || 3175 CC(vmcs12->host_cr4 & X86_CR4_PCIDE) || 3176 CC((vmcs12->host_rip) >> 32)) 3177 return -EINVAL; 3178 } 3179 3180 if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3181 CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3182 CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3183 CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3184 CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3185 CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3186 CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) || 3187 CC(vmcs12->host_cs_selector == 0) || 3188 CC(vmcs12->host_tr_selector == 0) || 3189 CC(vmcs12->host_ss_selector == 0 && !ia32e)) 3190 return -EINVAL; 3191 3192 if (CC(is_noncanonical_base_address(vmcs12->host_fs_base, vcpu)) || 3193 CC(is_noncanonical_base_address(vmcs12->host_gs_base, vcpu)) || 3194 CC(is_noncanonical_base_address(vmcs12->host_gdtr_base, vcpu)) || 3195 CC(is_noncanonical_base_address(vmcs12->host_idtr_base, vcpu)) || 3196 CC(is_noncanonical_base_address(vmcs12->host_tr_base, vcpu)) || 3197 CC(is_l1_noncanonical_address_on_vmexit(vmcs12->host_rip, vmcs12))) 3198 return -EINVAL; 3199 3200 /* 3201 * If the load IA32_EFER VM-exit control is 1, bits reserved in the 3202 * IA32_EFER MSR must be 0 in the field for that register. In addition, 3203 * the values of the LMA and LME bits in the field must each be that of 3204 * the host address-space size VM-exit control. 3205 */ 3206 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) { 3207 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) || 3208 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) || 3209 CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME))) 3210 return -EINVAL; 3211 } 3212 3213 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_CET_STATE) { 3214 if (nested_vmx_check_cet_state_common(vcpu, vmcs12->host_s_cet, 3215 vmcs12->host_ssp, 3216 vmcs12->host_ssp_tbl)) 3217 return -EINVAL; 3218 3219 /* 3220 * IA32_S_CET and SSP must be canonical if the host will 3221 * enter 64-bit mode after VM-exit; otherwise, higher 3222 * 32-bits must be all 0s. 3223 */ 3224 if (ia32e) { 3225 if (CC(is_noncanonical_msr_address(vmcs12->host_s_cet, vcpu)) || 3226 CC(is_noncanonical_msr_address(vmcs12->host_ssp, vcpu))) 3227 return -EINVAL; 3228 } else { 3229 if (CC(vmcs12->host_s_cet >> 32) || CC(vmcs12->host_ssp >> 32)) 3230 return -EINVAL; 3231 } 3232 } 3233 3234 return 0; 3235} 3236 3237static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu, 3238 struct vmcs12 *vmcs12) 3239{ 3240 struct vcpu_vmx *vmx = to_vmx(vcpu); 3241 struct gfn_to_hva_cache *ghc = &vmx->nested.shadow_vmcs12_cache; 3242 struct vmcs_hdr hdr; 3243 3244 if (vmcs12->vmcs_link_pointer == INVALID_GPA) 3245 return 0; 3246 3247 if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer))) 3248 return -EINVAL; 3249 3250 if (ghc->gpa != vmcs12->vmcs_link_pointer && 3251 CC(kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, 3252 vmcs12->vmcs_link_pointer, VMCS12_SIZE))) 3253 return -EINVAL; 3254 3255 if (CC(kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 3256 offsetof(struct vmcs12, hdr), 3257 sizeof(hdr)))) 3258 return -EINVAL; 3259 3260 if (CC(hdr.revision_id != VMCS12_REVISION) || 3261 CC(hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12))) 3262 return -EINVAL; 3263 3264 return 0; 3265} 3266 3267/* 3268 * Checks related to Guest Non-register State 3269 */ 3270static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) 3271{ 3272 if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE && 3273 vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT && 3274 vmcs12->guest_activity_state != GUEST_ACTIVITY_WAIT_SIPI)) 3275 return -EINVAL; 3276 3277 return 0; 3278} 3279 3280static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, 3281 struct vmcs12 *vmcs12, 3282 enum vm_entry_failure_code *entry_failure_code) 3283{ 3284 bool ia32e = !!(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE); 3285 3286 *entry_failure_code = ENTRY_FAIL_DEFAULT; 3287 3288 if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || 3289 CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) 3290 return -EINVAL; 3291 3292 if (CC(vmcs12->guest_cr4 & X86_CR4_CET && !(vmcs12->guest_cr0 & X86_CR0_WP))) 3293 return -EINVAL; 3294 3295 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 3296 u64 debugctl = vmcs12->guest_ia32_debugctl; 3297 3298 /* 3299 * FREEZE_IN_SMM is not virtualized, but allow L1 to set it in 3300 * vmcs12's DEBUGCTL under a quirk for backwards compatibility. 3301 * Note that the quirk only relaxes the consistency check. The 3302 * vmcc02 bit is still under the control of the host. In 3303 * particular, if a host administrator decides to clear the bit, 3304 * then L1 has no say in the matter. 3305 */ 3306 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM)) 3307 debugctl &= ~DEBUGCTLMSR_FREEZE_IN_SMM; 3308 3309 if (CC(!kvm_dr7_valid(vmcs12->guest_dr7)) || 3310 CC(!vmx_is_valid_debugctl(vcpu, debugctl, false))) 3311 return -EINVAL; 3312 } 3313 3314 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) && 3315 CC(!kvm_pat_valid(vmcs12->guest_ia32_pat))) 3316 return -EINVAL; 3317 3318 if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { 3319 *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR; 3320 return -EINVAL; 3321 } 3322 3323 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && 3324 CC(!kvm_valid_perf_global_ctrl(vcpu_to_pmu(vcpu), 3325 vmcs12->guest_ia32_perf_global_ctrl))) 3326 return -EINVAL; 3327 3328 if (CC((vmcs12->guest_cr0 & (X86_CR0_PG | X86_CR0_PE)) == X86_CR0_PG)) 3329 return -EINVAL; 3330 3331 if (CC(ia32e && !(vmcs12->guest_cr4 & X86_CR4_PAE)) || 3332 CC(ia32e && !(vmcs12->guest_cr0 & X86_CR0_PG))) 3333 return -EINVAL; 3334 3335 /* 3336 * If the load IA32_EFER VM-entry control is 1, the following checks 3337 * are performed on the field for the IA32_EFER MSR: 3338 * - Bits reserved in the IA32_EFER MSR must be 0. 3339 * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of 3340 * the IA-32e mode guest VM-exit control. It must also be identical 3341 * to bit 8 (LME) if bit 31 in the CR0 field (corresponding to 3342 * CR0.PG) is 1. 3343 */ 3344 if (to_vmx(vcpu)->nested.nested_run_pending && 3345 (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) { 3346 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) || 3347 CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) || 3348 CC(((vmcs12->guest_cr0 & X86_CR0_PG) && 3349 ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME)))) 3350 return -EINVAL; 3351 } 3352 3353 if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) && 3354 (CC(is_noncanonical_msr_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) || 3355 CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD)))) 3356 return -EINVAL; 3357 3358 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_CET_STATE) { 3359 if (nested_vmx_check_cet_state_common(vcpu, vmcs12->guest_s_cet, 3360 vmcs12->guest_ssp, 3361 vmcs12->guest_ssp_tbl)) 3362 return -EINVAL; 3363 3364 /* 3365 * Guest SSP must have 63:N bits identical, rather than 3366 * be canonical (i.e., 63:N-1 bits identical), where N is 3367 * the CPU's maximum linear-address width. Similar to 3368 * is_noncanonical_msr_address(), use the host's 3369 * linear-address width. 3370 */ 3371 if (CC(!__is_canonical_address(vmcs12->guest_ssp, max_host_virt_addr_bits() + 1))) 3372 return -EINVAL; 3373 } 3374 3375 if (nested_check_guest_non_reg_state(vmcs12)) 3376 return -EINVAL; 3377 3378 return 0; 3379} 3380 3381#ifdef CONFIG_KVM_HYPERV 3382static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) 3383{ 3384 struct vcpu_vmx *vmx = to_vmx(vcpu); 3385 3386 /* 3387 * hv_evmcs may end up being not mapped after migration (when 3388 * L2 was running), map it here to make sure vmcs12 changes are 3389 * properly reflected. 3390 */ 3391 if (guest_cpu_cap_has_evmcs(vcpu) && 3392 vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { 3393 enum nested_evmptrld_status evmptrld_status = 3394 nested_vmx_handle_enlightened_vmptrld(vcpu, false); 3395 3396 if (evmptrld_status == EVMPTRLD_VMFAIL || 3397 evmptrld_status == EVMPTRLD_ERROR) 3398 return false; 3399 3400 /* 3401 * Post migration VMCS12 always provides the most actual 3402 * information, copy it to eVMCS upon entry. 3403 */ 3404 vmx->nested.need_vmcs12_to_shadow_sync = true; 3405 } 3406 3407 return true; 3408} 3409#endif 3410 3411static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) 3412{ 3413 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3414 struct vcpu_vmx *vmx = to_vmx(vcpu); 3415 struct kvm_host_map *map; 3416 3417 if (!vcpu->arch.pdptrs_from_userspace && 3418 !nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 3419 /* 3420 * Reload the guest's PDPTRs since after a migration 3421 * the guest CR3 might be restored prior to setting the nested 3422 * state which can lead to a load of wrong PDPTRs. 3423 */ 3424 if (CC(!load_pdptrs(vcpu, vcpu->arch.cr3))) 3425 return false; 3426 } 3427 3428 3429 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3430 map = &vmx->nested.apic_access_page_map; 3431 3432 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->apic_access_addr), map)) { 3433 vmcs_write64(APIC_ACCESS_ADDR, pfn_to_hpa(map->pfn)); 3434 } else { 3435 pr_debug_ratelimited("%s: no backing for APIC-access address in vmcs12\n", 3436 __func__); 3437 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3438 vcpu->run->internal.suberror = 3439 KVM_INTERNAL_ERROR_EMULATION; 3440 vcpu->run->internal.ndata = 0; 3441 return false; 3442 } 3443 } 3444 3445 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 3446 map = &vmx->nested.virtual_apic_map; 3447 3448 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) { 3449 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn)); 3450 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) && 3451 nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) && 3452 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { 3453 /* 3454 * The processor will never use the TPR shadow, simply 3455 * clear the bit from the execution control. Such a 3456 * configuration is useless, but it happens in tests. 3457 * For any other configuration, failing the vm entry is 3458 * _not_ what the processor does but it's basically the 3459 * only possibility we have. 3460 */ 3461 exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW); 3462 } else { 3463 /* 3464 * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to 3465 * force VM-Entry to fail. 3466 */ 3467 vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, INVALID_GPA); 3468 } 3469 } 3470 3471 if (nested_cpu_has_posted_intr(vmcs12)) { 3472 map = &vmx->nested.pi_desc_map; 3473 3474 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) { 3475 vmx->nested.pi_desc = 3476 (struct pi_desc *)(((void *)map->hva) + 3477 offset_in_page(vmcs12->posted_intr_desc_addr)); 3478 vmcs_write64(POSTED_INTR_DESC_ADDR, 3479 pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr)); 3480 } else { 3481 /* 3482 * Defer the KVM_INTERNAL_EXIT until KVM tries to 3483 * access the contents of the VMCS12 posted interrupt 3484 * descriptor. (Note that KVM may do this when it 3485 * should not, per the architectural specification.) 3486 */ 3487 vmx->nested.pi_desc = NULL; 3488 pin_controls_clearbit(vmx, PIN_BASED_POSTED_INTR); 3489 } 3490 } 3491 if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) 3492 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3493 else 3494 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS); 3495 3496 return true; 3497} 3498 3499static bool vmx_get_nested_state_pages(struct kvm_vcpu *vcpu) 3500{ 3501#ifdef CONFIG_KVM_HYPERV 3502 /* 3503 * Note: nested_get_evmcs_page() also updates 'vp_assist_page' copy 3504 * in 'struct kvm_vcpu_hv' in case eVMCS is in use, this is mandatory 3505 * to make nested_evmcs_l2_tlb_flush_enabled() work correctly post 3506 * migration. 3507 */ 3508 if (!nested_get_evmcs_page(vcpu)) { 3509 pr_debug_ratelimited("%s: enlightened vmptrld failed\n", 3510 __func__); 3511 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 3512 vcpu->run->internal.suberror = 3513 KVM_INTERNAL_ERROR_EMULATION; 3514 vcpu->run->internal.ndata = 0; 3515 3516 return false; 3517 } 3518#endif 3519 3520 if (is_guest_mode(vcpu) && !nested_get_vmcs12_pages(vcpu)) 3521 return false; 3522 3523 return true; 3524} 3525 3526static int nested_vmx_write_pml_buffer(struct kvm_vcpu *vcpu, gpa_t gpa) 3527{ 3528 struct vmcs12 *vmcs12; 3529 struct vcpu_vmx *vmx = to_vmx(vcpu); 3530 gpa_t dst; 3531 3532 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) 3533 return 0; 3534 3535 if (WARN_ON_ONCE(vmx->nested.pml_full)) 3536 return 1; 3537 3538 /* 3539 * Check if PML is enabled for the nested guest. Whether eptp bit 6 is 3540 * set is already checked as part of A/D emulation. 3541 */ 3542 vmcs12 = get_vmcs12(vcpu); 3543 if (!nested_cpu_has_pml(vmcs12)) 3544 return 0; 3545 3546 if (vmcs12->guest_pml_index >= PML_LOG_NR_ENTRIES) { 3547 vmx->nested.pml_full = true; 3548 return 1; 3549 } 3550 3551 gpa &= ~0xFFFull; 3552 dst = vmcs12->pml_address + sizeof(u64) * vmcs12->guest_pml_index; 3553 3554 if (kvm_write_guest_page(vcpu->kvm, gpa_to_gfn(dst), &gpa, 3555 offset_in_page(dst), sizeof(gpa))) 3556 return 0; 3557 3558 vmcs12->guest_pml_index--; 3559 3560 return 0; 3561} 3562 3563/* 3564 * Intel's VMX Instruction Reference specifies a common set of prerequisites 3565 * for running VMX instructions (except VMXON, whose prerequisites are 3566 * slightly different). It also specifies what exception to inject otherwise. 3567 * Note that many of these exceptions have priority over VM exits, so they 3568 * don't have to be checked again here. 3569 */ 3570static int nested_vmx_check_permission(struct kvm_vcpu *vcpu) 3571{ 3572 if (!to_vmx(vcpu)->nested.vmxon) { 3573 kvm_queue_exception(vcpu, UD_VECTOR); 3574 return 0; 3575 } 3576 3577 if (vmx_get_cpl(vcpu)) { 3578 kvm_inject_gp(vcpu, 0); 3579 return 0; 3580 } 3581 3582 return 1; 3583} 3584 3585static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 3586 struct vmcs12 *vmcs12); 3587 3588/* 3589 * If from_vmentry is false, this is being called from state restore (either RSM 3590 * or KVM_SET_NESTED_STATE). Otherwise it's called from vmlaunch/vmresume. 3591 * 3592 * Returns: 3593 * NVMX_VMENTRY_SUCCESS: Entered VMX non-root mode 3594 * NVMX_VMENTRY_VMFAIL: Consistency check VMFail 3595 * NVMX_VMENTRY_VMEXIT: Consistency check VMExit 3596 * NVMX_VMENTRY_KVM_INTERNAL_ERROR: KVM internal error 3597 */ 3598enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, 3599 bool from_vmentry) 3600{ 3601 struct vcpu_vmx *vmx = to_vmx(vcpu); 3602 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 3603 enum vm_entry_failure_code entry_failure_code; 3604 union vmx_exit_reason exit_reason = { 3605 .basic = EXIT_REASON_INVALID_STATE, 3606 .failed_vmentry = 1, 3607 }; 3608 u32 failed_index; 3609 3610 trace_kvm_nested_vmenter(kvm_rip_read(vcpu), 3611 vmx->nested.current_vmptr, 3612 vmcs12->guest_rip, 3613 vmcs12->guest_intr_status, 3614 vmcs12->vm_entry_intr_info_field, 3615 vmcs12->secondary_vm_exec_control & SECONDARY_EXEC_ENABLE_EPT, 3616 vmcs12->ept_pointer, 3617 vmcs12->guest_cr3, 3618 KVM_ISA_VMX); 3619 3620 kvm_service_local_tlb_flush_requests(vcpu); 3621 3622 if (!vmx->nested.nested_run_pending || 3623 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) 3624 vmx->nested.pre_vmenter_debugctl = vmx_guest_debugctl_read(); 3625 if (kvm_mpx_supported() && 3626 (!vmx->nested.nested_run_pending || 3627 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))) 3628 vmx->nested.pre_vmenter_bndcfgs = vmcs_read64(GUEST_BNDCFGS); 3629 3630 if (!vmx->nested.nested_run_pending || 3631 !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_CET_STATE)) 3632 vmcs_read_cet_state(vcpu, &vmx->nested.pre_vmenter_s_cet, 3633 &vmx->nested.pre_vmenter_ssp, 3634 &vmx->nested.pre_vmenter_ssp_tbl); 3635 3636 /* 3637 * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled. In the 3638 * event of a "late" VM-Fail, i.e. a VM-Fail detected by hardware but 3639 * not KVM, KVM must unwind its software model to the pre-VM-Entry host 3640 * state. When EPT is disabled, GUEST_CR3 holds KVM's shadow CR3, not 3641 * L1's "real" CR3, which causes nested_vmx_restore_host_state() to 3642 * corrupt vcpu->arch.cr3. Stuffing vmcs01.GUEST_CR3 results in the 3643 * unwind naturally setting arch.cr3 to the correct value. Smashing 3644 * vmcs01.GUEST_CR3 is safe because nested VM-Exits, and the unwind, 3645 * reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is guaranteed to be 3646 * overwritten with a shadow CR3 prior to re-entering L1. 3647 */ 3648 if (!enable_ept) 3649 vmcs_writel(GUEST_CR3, vcpu->arch.cr3); 3650 3651 vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02); 3652 3653 prepare_vmcs02_early(vmx, &vmx->vmcs01, vmcs12); 3654 3655 if (from_vmentry) { 3656 if (unlikely(!nested_get_vmcs12_pages(vcpu))) { 3657 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3658 return NVMX_VMENTRY_KVM_INTERNAL_ERROR; 3659 } 3660 3661 if (nested_vmx_check_controls_late(vcpu, vmcs12)) { 3662 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3663 return NVMX_VMENTRY_VMFAIL; 3664 } 3665 3666 if (nested_vmx_check_guest_state(vcpu, vmcs12, 3667 &entry_failure_code)) { 3668 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3669 vmcs12->exit_qualification = entry_failure_code; 3670 goto vmentry_fail_vmexit; 3671 } 3672 } 3673 3674 enter_guest_mode(vcpu); 3675 3676 if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &entry_failure_code)) { 3677 exit_reason.basic = EXIT_REASON_INVALID_STATE; 3678 vmcs12->exit_qualification = entry_failure_code; 3679 goto vmentry_fail_vmexit_guest_mode; 3680 } 3681 3682 if (from_vmentry) { 3683 failed_index = nested_vmx_load_msr(vcpu, 3684 vmcs12->vm_entry_msr_load_addr, 3685 vmcs12->vm_entry_msr_load_count); 3686 if (failed_index) { 3687 exit_reason.basic = EXIT_REASON_MSR_LOAD_FAIL; 3688 vmcs12->exit_qualification = failed_index; 3689 goto vmentry_fail_vmexit_guest_mode; 3690 } 3691 } else { 3692 /* 3693 * The MMU is not initialized to point at the right entities yet and 3694 * "get pages" would need to read data from the guest (i.e. we will 3695 * need to perform gpa to hpa translation). Request a call 3696 * to nested_get_vmcs12_pages before the next VM-entry. The MSRs 3697 * have already been set at vmentry time and should not be reset. 3698 */ 3699 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 3700 } 3701 3702 /* 3703 * Re-evaluate pending events if L1 had a pending IRQ/NMI/INIT/SIPI 3704 * when it executed VMLAUNCH/VMRESUME, as entering non-root mode can 3705 * effectively unblock various events, e.g. INIT/SIPI cause VM-Exit 3706 * unconditionally. Take care to pull data from vmcs01 as appropriate, 3707 * e.g. when checking for interrupt windows, as vmcs02 is now loaded. 3708 */ 3709 if ((__exec_controls_get(&vmx->vmcs01) & (CPU_BASED_INTR_WINDOW_EXITING | 3710 CPU_BASED_NMI_WINDOW_EXITING)) || 3711 kvm_apic_has_pending_init_or_sipi(vcpu) || 3712 kvm_apic_has_interrupt(vcpu)) 3713 kvm_make_request(KVM_REQ_EVENT, vcpu); 3714 3715 /* 3716 * Do not start the preemption timer hrtimer until after we know 3717 * we are successful, so that only nested_vmx_vmexit needs to cancel 3718 * the timer. 3719 */ 3720 vmx->nested.preemption_timer_expired = false; 3721 if (nested_cpu_has_preemption_timer(vmcs12)) { 3722 u64 timer_value = vmx_calc_preemption_timer_value(vcpu); 3723 vmx_start_preemption_timer(vcpu, timer_value); 3724 } 3725 3726 /* 3727 * Note no nested_vmx_succeed or nested_vmx_fail here. At this point 3728 * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet 3729 * returned as far as L1 is concerned. It will only return (and set 3730 * the success flag) when L2 exits (see nested_vmx_vmexit()). 3731 */ 3732 return NVMX_VMENTRY_SUCCESS; 3733 3734 /* 3735 * A failed consistency check that leads to a VMExit during L1's 3736 * VMEnter to L2 is a variation of a normal VMexit, as explained in 3737 * 26.7 "VM-entry failures during or after loading guest state". 3738 */ 3739vmentry_fail_vmexit_guest_mode: 3740 if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) 3741 vcpu->arch.tsc_offset -= vmcs12->tsc_offset; 3742 leave_guest_mode(vcpu); 3743 3744vmentry_fail_vmexit: 3745 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 3746 3747 if (!from_vmentry) 3748 return NVMX_VMENTRY_VMEXIT; 3749 3750 load_vmcs12_host_state(vcpu, vmcs12); 3751 vmcs12->vm_exit_reason = exit_reason.full; 3752 if (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx)) 3753 vmx->nested.need_vmcs12_to_shadow_sync = true; 3754 return NVMX_VMENTRY_VMEXIT; 3755} 3756 3757/* 3758 * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1 3759 * for running an L2 nested guest. 3760 */ 3761static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) 3762{ 3763 struct vmcs12 *vmcs12; 3764 enum nvmx_vmentry_status status; 3765 struct vcpu_vmx *vmx = to_vmx(vcpu); 3766 u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); 3767 enum nested_evmptrld_status evmptrld_status; 3768 3769 if (!nested_vmx_check_permission(vcpu)) 3770 return 1; 3771 3772 evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, launch); 3773 if (evmptrld_status == EVMPTRLD_ERROR) { 3774 kvm_queue_exception(vcpu, UD_VECTOR); 3775 return 1; 3776 } 3777 3778 kvm_pmu_branch_retired(vcpu); 3779 3780 if (CC(evmptrld_status == EVMPTRLD_VMFAIL)) 3781 return nested_vmx_failInvalid(vcpu); 3782 3783 if (CC(!nested_vmx_is_evmptr12_valid(vmx) && 3784 vmx->nested.current_vmptr == INVALID_GPA)) 3785 return nested_vmx_failInvalid(vcpu); 3786 3787 vmcs12 = get_vmcs12(vcpu); 3788 3789 /* 3790 * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact 3791 * that there *is* a valid VMCS pointer, RFLAGS.CF is set 3792 * rather than RFLAGS.ZF, and no error number is stored to the 3793 * VM-instruction error field. 3794 */ 3795 if (CC(vmcs12->hdr.shadow_vmcs)) 3796 return nested_vmx_failInvalid(vcpu); 3797 3798 if (nested_vmx_is_evmptr12_valid(vmx)) { 3799 struct hv_enlightened_vmcs *evmcs = nested_vmx_evmcs(vmx); 3800 3801 copy_enlightened_to_vmcs12(vmx, evmcs->hv_clean_fields); 3802 /* Enlightened VMCS doesn't have launch state */ 3803 vmcs12->launch_state = !launch; 3804 } else if (enable_shadow_vmcs) { 3805 copy_shadow_to_vmcs12(vmx); 3806 } 3807 3808 /* 3809 * The nested entry process starts with enforcing various prerequisites 3810 * on vmcs12 as required by the Intel SDM, and act appropriately when 3811 * they fail: As the SDM explains, some conditions should cause the 3812 * instruction to fail, while others will cause the instruction to seem 3813 * to succeed, but return an EXIT_REASON_INVALID_STATE. 3814 * To speed up the normal (success) code path, we should avoid checking 3815 * for misconfigurations which will anyway be caught by the processor 3816 * when using the merged vmcs02. 3817 */ 3818 if (CC(interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)) 3819 return nested_vmx_fail(vcpu, VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS); 3820 3821 if (CC(vmcs12->launch_state == launch)) 3822 return nested_vmx_fail(vcpu, 3823 launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS 3824 : VMXERR_VMRESUME_NONLAUNCHED_VMCS); 3825 3826 if (nested_vmx_check_controls(vcpu, vmcs12)) 3827 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3828 3829 if (nested_vmx_check_address_space_size(vcpu, vmcs12)) 3830 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3831 3832 if (nested_vmx_check_host_state(vcpu, vmcs12)) 3833 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD); 3834 3835 /* 3836 * We're finally done with prerequisite checking, and can start with 3837 * the nested entry. 3838 */ 3839 vmx->nested.nested_run_pending = 1; 3840 vmx->nested.has_preemption_timer_deadline = false; 3841 status = nested_vmx_enter_non_root_mode(vcpu, true); 3842 if (unlikely(status != NVMX_VMENTRY_SUCCESS)) 3843 goto vmentry_failed; 3844 3845 /* Hide L1D cache contents from the nested guest. */ 3846 kvm_request_l1tf_flush_l1d(); 3847 3848 /* 3849 * Must happen outside of nested_vmx_enter_non_root_mode() as it will 3850 * also be used as part of restoring nVMX state for 3851 * snapshot restore (migration). 3852 * 3853 * In this flow, it is assumed that vmcs12 cache was 3854 * transferred as part of captured nVMX state and should 3855 * therefore not be read from guest memory (which may not 3856 * exist on destination host yet). 3857 */ 3858 nested_cache_shadow_vmcs12(vcpu, vmcs12); 3859 3860 switch (vmcs12->guest_activity_state) { 3861 case GUEST_ACTIVITY_HLT: 3862 /* 3863 * If we're entering a halted L2 vcpu and the L2 vcpu won't be 3864 * awakened by event injection or by an NMI-window VM-exit or 3865 * by an interrupt-window VM-exit, halt the vcpu. 3866 */ 3867 if (!(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) && 3868 !nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING) && 3869 !(nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING) && 3870 (vmcs12->guest_rflags & X86_EFLAGS_IF))) { 3871 vmx->nested.nested_run_pending = 0; 3872 return kvm_emulate_halt_noskip(vcpu); 3873 } 3874 break; 3875 case GUEST_ACTIVITY_WAIT_SIPI: 3876 vmx->nested.nested_run_pending = 0; 3877 kvm_set_mp_state(vcpu, KVM_MP_STATE_INIT_RECEIVED); 3878 break; 3879 default: 3880 break; 3881 } 3882 3883 return 1; 3884 3885vmentry_failed: 3886 vmx->nested.nested_run_pending = 0; 3887 if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR) 3888 return 0; 3889 if (status == NVMX_VMENTRY_VMEXIT) 3890 return 1; 3891 WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL); 3892 return nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 3893} 3894 3895/* 3896 * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date 3897 * because L2 may have changed some cr0 bits directly (CR0_GUEST_HOST_MASK). 3898 * This function returns the new value we should put in vmcs12.guest_cr0. 3899 * It's not enough to just return the vmcs02 GUEST_CR0. Rather, 3900 * 1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now 3901 * available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0 3902 * didn't trap the bit, because if L1 did, so would L0). 3903 * 2. Bits that L1 asked to trap (and therefore L0 also did) could not have 3904 * been modified by L2, and L1 knows it. So just leave the old value of 3905 * the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0 3906 * isn't relevant, because if L0 traps this bit it can set it to anything. 3907 * 3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have 3908 * changed these bits, and therefore they need to be updated, but L0 3909 * didn't necessarily allow them to be changed in GUEST_CR0 - and rather 3910 * put them in vmcs02 CR0_READ_SHADOW. So take these bits from there. 3911 */ 3912static inline unsigned long 3913vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3914{ 3915 return 3916 /*1*/ (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) | 3917 /*2*/ (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) | 3918 /*3*/ (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask | 3919 vcpu->arch.cr0_guest_owned_bits)); 3920} 3921 3922static inline unsigned long 3923vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 3924{ 3925 return 3926 /*1*/ (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) | 3927 /*2*/ (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) | 3928 /*3*/ (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask | 3929 vcpu->arch.cr4_guest_owned_bits)); 3930} 3931 3932static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, 3933 struct vmcs12 *vmcs12, 3934 u32 vm_exit_reason, u32 exit_intr_info) 3935{ 3936 u32 idt_vectoring; 3937 unsigned int nr; 3938 3939 /* 3940 * Per the SDM, VM-Exits due to double and triple faults are never 3941 * considered to occur during event delivery, even if the double/triple 3942 * fault is the result of an escalating vectoring issue. 3943 * 3944 * Note, the SDM qualifies the double fault behavior with "The original 3945 * event results in a double-fault exception". It's unclear why the 3946 * qualification exists since exits due to double fault can occur only 3947 * while vectoring a different exception (injected events are never 3948 * subject to interception), i.e. there's _always_ an original event. 3949 * 3950 * The SDM also uses NMI as a confusing example for the "original event 3951 * causes the VM exit directly" clause. NMI isn't special in any way, 3952 * the same rule applies to all events that cause an exit directly. 3953 * NMI is an odd choice for the example because NMIs can only occur on 3954 * instruction boundaries, i.e. they _can't_ occur during vectoring. 3955 */ 3956 if ((u16)vm_exit_reason == EXIT_REASON_TRIPLE_FAULT || 3957 ((u16)vm_exit_reason == EXIT_REASON_EXCEPTION_NMI && 3958 is_double_fault(exit_intr_info))) { 3959 vmcs12->idt_vectoring_info_field = 0; 3960 } else if (vcpu->arch.exception.injected) { 3961 nr = vcpu->arch.exception.vector; 3962 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3963 3964 if (kvm_exception_is_soft(nr)) { 3965 vmcs12->vm_exit_instruction_len = 3966 vcpu->arch.event_exit_inst_len; 3967 idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION; 3968 } else 3969 idt_vectoring |= INTR_TYPE_HARD_EXCEPTION; 3970 3971 if (vcpu->arch.exception.has_error_code) { 3972 idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK; 3973 vmcs12->idt_vectoring_error_code = 3974 vcpu->arch.exception.error_code; 3975 } 3976 3977 vmcs12->idt_vectoring_info_field = idt_vectoring; 3978 } else if (vcpu->arch.nmi_injected) { 3979 vmcs12->idt_vectoring_info_field = 3980 INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR; 3981 } else if (vcpu->arch.interrupt.injected) { 3982 nr = vcpu->arch.interrupt.nr; 3983 idt_vectoring = nr | VECTORING_INFO_VALID_MASK; 3984 3985 if (vcpu->arch.interrupt.soft) { 3986 idt_vectoring |= INTR_TYPE_SOFT_INTR; 3987 vmcs12->vm_entry_instruction_len = 3988 vcpu->arch.event_exit_inst_len; 3989 } else 3990 idt_vectoring |= INTR_TYPE_EXT_INTR; 3991 3992 vmcs12->idt_vectoring_info_field = idt_vectoring; 3993 } else { 3994 vmcs12->idt_vectoring_info_field = 0; 3995 } 3996} 3997 3998 3999void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) 4000{ 4001 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4002 gfn_t gfn; 4003 4004 /* 4005 * Don't need to mark the APIC access page dirty; it is never 4006 * written to by the CPU during APIC virtualization. 4007 */ 4008 4009 if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) { 4010 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT; 4011 kvm_vcpu_mark_page_dirty(vcpu, gfn); 4012 } 4013 4014 if (nested_cpu_has_posted_intr(vmcs12)) { 4015 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT; 4016 kvm_vcpu_mark_page_dirty(vcpu, gfn); 4017 } 4018} 4019 4020static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) 4021{ 4022 struct vcpu_vmx *vmx = to_vmx(vcpu); 4023 int max_irr; 4024 void *vapic_page; 4025 u16 status; 4026 4027 if (!vmx->nested.pi_pending) 4028 return 0; 4029 4030 if (!vmx->nested.pi_desc) 4031 goto mmio_needed; 4032 4033 vmx->nested.pi_pending = false; 4034 4035 if (!pi_test_and_clear_on(vmx->nested.pi_desc)) 4036 return 0; 4037 4038 max_irr = pi_find_highest_vector(vmx->nested.pi_desc); 4039 if (max_irr > 0) { 4040 vapic_page = vmx->nested.virtual_apic_map.hva; 4041 if (!vapic_page) 4042 goto mmio_needed; 4043 4044 __kvm_apic_update_irr(vmx->nested.pi_desc->pir, 4045 vapic_page, &max_irr); 4046 status = vmcs_read16(GUEST_INTR_STATUS); 4047 if ((u8)max_irr > ((u8)status & 0xff)) { 4048 status &= ~0xff; 4049 status |= (u8)max_irr; 4050 vmcs_write16(GUEST_INTR_STATUS, status); 4051 } 4052 } 4053 4054 nested_mark_vmcs12_pages_dirty(vcpu); 4055 return 0; 4056 4057mmio_needed: 4058 kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); 4059 return -ENXIO; 4060} 4061 4062static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu) 4063{ 4064 struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; 4065 u32 intr_info = ex->vector | INTR_INFO_VALID_MASK; 4066 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4067 unsigned long exit_qual; 4068 4069 if (ex->has_payload) { 4070 exit_qual = ex->payload; 4071 } else if (ex->vector == PF_VECTOR) { 4072 exit_qual = vcpu->arch.cr2; 4073 } else if (ex->vector == DB_VECTOR) { 4074 exit_qual = vcpu->arch.dr6; 4075 exit_qual &= ~DR6_BT; 4076 exit_qual ^= DR6_ACTIVE_LOW; 4077 } else { 4078 exit_qual = 0; 4079 } 4080 4081 /* 4082 * Unlike AMD's Paged Real Mode, which reports an error code on #PF 4083 * VM-Exits even if the CPU is in Real Mode, Intel VMX never sets the 4084 * "has error code" flags on VM-Exit if the CPU is in Real Mode. 4085 */ 4086 if (ex->has_error_code && is_protmode(vcpu)) { 4087 /* 4088 * Intel CPUs do not generate error codes with bits 31:16 set, 4089 * and more importantly VMX disallows setting bits 31:16 in the 4090 * injected error code for VM-Entry. Drop the bits to mimic 4091 * hardware and avoid inducing failure on nested VM-Entry if L1 4092 * chooses to inject the exception back to L2. AMD CPUs _do_ 4093 * generate "full" 32-bit error codes, so KVM allows userspace 4094 * to inject exception error codes with bits 31:16 set. 4095 */ 4096 vmcs12->vm_exit_intr_error_code = (u16)ex->error_code; 4097 intr_info |= INTR_INFO_DELIVER_CODE_MASK; 4098 } 4099 4100 if (kvm_exception_is_soft(ex->vector)) 4101 intr_info |= INTR_TYPE_SOFT_EXCEPTION; 4102 else 4103 intr_info |= INTR_TYPE_HARD_EXCEPTION; 4104 4105 if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) && 4106 vmx_get_nmi_mask(vcpu)) 4107 intr_info |= INTR_INFO_UNBLOCK_NMI; 4108 4109 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual); 4110} 4111 4112/* 4113 * Returns true if a debug trap is (likely) pending delivery. Infer the class 4114 * of a #DB (trap-like vs. fault-like) from the exception payload (to-be-DR6). 4115 * Using the payload is flawed because code breakpoints (fault-like) and data 4116 * breakpoints (trap-like) set the same bits in DR6 (breakpoint detected), i.e. 4117 * this will return false positives if a to-be-injected code breakpoint #DB is 4118 * pending (from KVM's perspective, but not "pending" across an instruction 4119 * boundary). ICEBP, a.k.a. INT1, is also not reflected here even though it 4120 * too is trap-like. 4121 * 4122 * KVM "works" despite these flaws as ICEBP isn't currently supported by the 4123 * emulator, Monitor Trap Flag is not marked pending on intercepted #DBs (the 4124 * #DB has already happened), and MTF isn't marked pending on code breakpoints 4125 * from the emulator (because such #DBs are fault-like and thus don't trigger 4126 * actions that fire on instruction retire). 4127 */ 4128static unsigned long vmx_get_pending_dbg_trap(struct kvm_queued_exception *ex) 4129{ 4130 if (!ex->pending || ex->vector != DB_VECTOR) 4131 return 0; 4132 4133 /* General Detect #DBs are always fault-like. */ 4134 return ex->payload & ~DR6_BD; 4135} 4136 4137/* 4138 * Returns true if there's a pending #DB exception that is lower priority than 4139 * a pending Monitor Trap Flag VM-Exit. TSS T-flag #DBs are not emulated by 4140 * KVM, but could theoretically be injected by userspace. Note, this code is 4141 * imperfect, see above. 4142 */ 4143static bool vmx_is_low_priority_db_trap(struct kvm_queued_exception *ex) 4144{ 4145 return vmx_get_pending_dbg_trap(ex) & ~DR6_BT; 4146} 4147 4148/* 4149 * Certain VM-exits set the 'pending debug exceptions' field to indicate a 4150 * recognized #DB (data or single-step) that has yet to be delivered. Since KVM 4151 * represents these debug traps with a payload that is said to be compatible 4152 * with the 'pending debug exceptions' field, write the payload to the VMCS 4153 * field if a VM-exit is delivered before the debug trap. 4154 */ 4155static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) 4156{ 4157 unsigned long pending_dbg; 4158 4159 pending_dbg = vmx_get_pending_dbg_trap(&vcpu->arch.exception); 4160 if (pending_dbg) 4161 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS, pending_dbg); 4162} 4163 4164static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) 4165{ 4166 return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && 4167 to_vmx(vcpu)->nested.preemption_timer_expired; 4168} 4169 4170static bool vmx_has_nested_events(struct kvm_vcpu *vcpu, bool for_injection) 4171{ 4172 struct vcpu_vmx *vmx = to_vmx(vcpu); 4173 void *vapic = vmx->nested.virtual_apic_map.hva; 4174 int max_irr, vppr; 4175 4176 if (nested_vmx_preemption_timer_pending(vcpu) || 4177 vmx->nested.mtf_pending) 4178 return true; 4179 4180 /* 4181 * Virtual Interrupt Delivery doesn't require manual injection. Either 4182 * the interrupt is already in GUEST_RVI and will be recognized by CPU 4183 * at VM-Entry, or there is a KVM_REQ_EVENT pending and KVM will move 4184 * the interrupt from the PIR to RVI prior to entering the guest. 4185 */ 4186 if (for_injection) 4187 return false; 4188 4189 if (!nested_cpu_has_vid(get_vmcs12(vcpu)) || 4190 __vmx_interrupt_blocked(vcpu)) 4191 return false; 4192 4193 if (!vapic) 4194 return false; 4195 4196 vppr = *((u32 *)(vapic + APIC_PROCPRI)); 4197 4198 max_irr = vmx_get_rvi(); 4199 if ((max_irr & 0xf0) > (vppr & 0xf0)) 4200 return true; 4201 4202 if (vmx->nested.pi_pending && vmx->nested.pi_desc && 4203 pi_test_on(vmx->nested.pi_desc)) { 4204 max_irr = pi_find_highest_vector(vmx->nested.pi_desc); 4205 if (max_irr > 0 && (max_irr & 0xf0) > (vppr & 0xf0)) 4206 return true; 4207 } 4208 4209 return false; 4210} 4211 4212/* 4213 * Per the Intel SDM's table "Priority Among Concurrent Events", with minor 4214 * edits to fill in missing examples, e.g. #DB due to split-lock accesses, 4215 * and less minor edits to splice in the priority of VMX Non-Root specific 4216 * events, e.g. MTF and NMI/INTR-window exiting. 4217 * 4218 * 1 Hardware Reset and Machine Checks 4219 * - RESET 4220 * - Machine Check 4221 * 4222 * 2 Trap on Task Switch 4223 * - T flag in TSS is set (on task switch) 4224 * 4225 * 3 External Hardware Interventions 4226 * - FLUSH 4227 * - STOPCLK 4228 * - SMI 4229 * - INIT 4230 * 4231 * 3.5 Monitor Trap Flag (MTF) VM-exit[1] 4232 * 4233 * 4 Traps on Previous Instruction 4234 * - Breakpoints 4235 * - Trap-class Debug Exceptions (#DB due to TF flag set, data/I-O 4236 * breakpoint, or #DB due to a split-lock access) 4237 * 4238 * 4.3 VMX-preemption timer expired VM-exit 4239 * 4240 * 4.6 NMI-window exiting VM-exit[2] 4241 * 4242 * 5 Nonmaskable Interrupts (NMI) 4243 * 4244 * 5.5 Interrupt-window exiting VM-exit and Virtual-interrupt delivery 4245 * 4246 * 6 Maskable Hardware Interrupts 4247 * 4248 * 7 Code Breakpoint Fault 4249 * 4250 * 8 Faults from Fetching Next Instruction 4251 * - Code-Segment Limit Violation 4252 * - Code Page Fault 4253 * - Control protection exception (missing ENDBRANCH at target of indirect 4254 * call or jump) 4255 * 4256 * 9 Faults from Decoding Next Instruction 4257 * - Instruction length > 15 bytes 4258 * - Invalid Opcode 4259 * - Coprocessor Not Available 4260 * 4261 *10 Faults on Executing Instruction 4262 * - Overflow 4263 * - Bound error 4264 * - Invalid TSS 4265 * - Segment Not Present 4266 * - Stack fault 4267 * - General Protection 4268 * - Data Page Fault 4269 * - Alignment Check 4270 * - x86 FPU Floating-point exception 4271 * - SIMD floating-point exception 4272 * - Virtualization exception 4273 * - Control protection exception 4274 * 4275 * [1] Per the "Monitor Trap Flag" section: System-management interrupts (SMIs), 4276 * INIT signals, and higher priority events take priority over MTF VM exits. 4277 * MTF VM exits take priority over debug-trap exceptions and lower priority 4278 * events. 4279 * 4280 * [2] Debug-trap exceptions and higher priority events take priority over VM exits 4281 * caused by the VMX-preemption timer. VM exits caused by the VMX-preemption 4282 * timer take priority over VM exits caused by the "NMI-window exiting" 4283 * VM-execution control and lower priority events. 4284 * 4285 * [3] Debug-trap exceptions and higher priority events take priority over VM exits 4286 * caused by "NMI-window exiting". VM exits caused by this control take 4287 * priority over non-maskable interrupts (NMIs) and lower priority events. 4288 * 4289 * [4] Virtual-interrupt delivery has the same priority as that of VM exits due to 4290 * the 1-setting of the "interrupt-window exiting" VM-execution control. Thus, 4291 * non-maskable interrupts (NMIs) and higher priority events take priority over 4292 * delivery of a virtual interrupt; delivery of a virtual interrupt takes 4293 * priority over external interrupts and lower priority events. 4294 */ 4295static int vmx_check_nested_events(struct kvm_vcpu *vcpu) 4296{ 4297 struct kvm_lapic *apic = vcpu->arch.apic; 4298 struct vcpu_vmx *vmx = to_vmx(vcpu); 4299 /* 4300 * Only a pending nested run blocks a pending exception. If there is a 4301 * previously injected event, the pending exception occurred while said 4302 * event was being delivered and thus needs to be handled. 4303 */ 4304 bool block_nested_exceptions = vmx->nested.nested_run_pending; 4305 /* 4306 * Events that don't require injection, i.e. that are virtualized by 4307 * hardware, aren't blocked by a pending VM-Enter as KVM doesn't need 4308 * to regain control in order to deliver the event, and hardware will 4309 * handle event ordering, e.g. with respect to injected exceptions. 4310 * 4311 * But, new events (not exceptions) are only recognized at instruction 4312 * boundaries. If an event needs reinjection, then KVM is handling a 4313 * VM-Exit that occurred _during_ instruction execution; new events, 4314 * irrespective of whether or not they're injected, are blocked until 4315 * the instruction completes. 4316 */ 4317 bool block_non_injected_events = kvm_event_needs_reinjection(vcpu); 4318 /* 4319 * Inject events are blocked by nested VM-Enter, as KVM is responsible 4320 * for managing priority between concurrent events, i.e. KVM needs to 4321 * wait until after VM-Enter completes to deliver injected events. 4322 */ 4323 bool block_nested_events = block_nested_exceptions || 4324 block_non_injected_events; 4325 4326 if (lapic_in_kernel(vcpu) && 4327 test_bit(KVM_APIC_INIT, &apic->pending_events)) { 4328 if (block_nested_events) 4329 return -EBUSY; 4330 nested_vmx_update_pending_dbg(vcpu); 4331 clear_bit(KVM_APIC_INIT, &apic->pending_events); 4332 if (vcpu->arch.mp_state != KVM_MP_STATE_INIT_RECEIVED) 4333 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0); 4334 4335 /* MTF is discarded if the vCPU is in WFS. */ 4336 vmx->nested.mtf_pending = false; 4337 return 0; 4338 } 4339 4340 if (lapic_in_kernel(vcpu) && 4341 test_bit(KVM_APIC_SIPI, &apic->pending_events)) { 4342 if (block_nested_events) 4343 return -EBUSY; 4344 4345 clear_bit(KVM_APIC_SIPI, &apic->pending_events); 4346 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { 4347 nested_vmx_vmexit(vcpu, EXIT_REASON_SIPI_SIGNAL, 0, 4348 apic->sipi_vector & 0xFFUL); 4349 return 0; 4350 } 4351 /* Fallthrough, the SIPI is completely ignored. */ 4352 } 4353 4354 /* 4355 * Process exceptions that are higher priority than Monitor Trap Flag: 4356 * fault-like exceptions, TSS T flag #DB (not emulated by KVM, but 4357 * could theoretically come in from userspace), and ICEBP (INT1). 4358 * 4359 * TODO: SMIs have higher priority than MTF and trap-like #DBs (except 4360 * for TSS T flag #DBs). KVM also doesn't save/restore pending MTF 4361 * across SMI/RSM as it should; that needs to be addressed in order to 4362 * prioritize SMI over MTF and trap-like #DBs. 4363 */ 4364 if (vcpu->arch.exception_vmexit.pending && 4365 !vmx_is_low_priority_db_trap(&vcpu->arch.exception_vmexit)) { 4366 if (block_nested_exceptions) 4367 return -EBUSY; 4368 4369 nested_vmx_inject_exception_vmexit(vcpu); 4370 return 0; 4371 } 4372 4373 if (vcpu->arch.exception.pending && 4374 !vmx_is_low_priority_db_trap(&vcpu->arch.exception)) { 4375 if (block_nested_exceptions) 4376 return -EBUSY; 4377 goto no_vmexit; 4378 } 4379 4380 if (vmx->nested.mtf_pending) { 4381 if (block_nested_events) 4382 return -EBUSY; 4383 nested_vmx_update_pending_dbg(vcpu); 4384 nested_vmx_vmexit(vcpu, EXIT_REASON_MONITOR_TRAP_FLAG, 0, 0); 4385 return 0; 4386 } 4387 4388 if (vcpu->arch.exception_vmexit.pending) { 4389 if (block_nested_exceptions) 4390 return -EBUSY; 4391 4392 nested_vmx_inject_exception_vmexit(vcpu); 4393 return 0; 4394 } 4395 4396 if (vcpu->arch.exception.pending) { 4397 if (block_nested_exceptions) 4398 return -EBUSY; 4399 goto no_vmexit; 4400 } 4401 4402 if (nested_vmx_preemption_timer_pending(vcpu)) { 4403 if (block_nested_events) 4404 return -EBUSY; 4405 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); 4406 return 0; 4407 } 4408 4409 if (vcpu->arch.smi_pending && !is_smm(vcpu)) { 4410 if (block_nested_events) 4411 return -EBUSY; 4412 goto no_vmexit; 4413 } 4414 4415 if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) { 4416 if (block_nested_events) 4417 return -EBUSY; 4418 if (!nested_exit_on_nmi(vcpu)) 4419 goto no_vmexit; 4420 4421 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, 4422 NMI_VECTOR | INTR_TYPE_NMI_INTR | 4423 INTR_INFO_VALID_MASK, 0); 4424 /* 4425 * The NMI-triggered VM exit counts as injection: 4426 * clear this one and block further NMIs. 4427 */ 4428 vcpu->arch.nmi_pending = 0; 4429 vmx_set_nmi_mask(vcpu, true); 4430 return 0; 4431 } 4432 4433 if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) { 4434 int irq; 4435 4436 if (!nested_exit_on_intr(vcpu)) { 4437 if (block_nested_events) 4438 return -EBUSY; 4439 4440 goto no_vmexit; 4441 } 4442 4443 if (!nested_exit_intr_ack_set(vcpu)) { 4444 if (block_nested_events) 4445 return -EBUSY; 4446 4447 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); 4448 return 0; 4449 } 4450 4451 irq = kvm_cpu_get_extint(vcpu); 4452 if (irq != -1) { 4453 if (block_nested_events) 4454 return -EBUSY; 4455 4456 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 4457 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR | irq, 0); 4458 return 0; 4459 } 4460 4461 irq = kvm_apic_has_interrupt(vcpu); 4462 if (WARN_ON_ONCE(irq < 0)) 4463 goto no_vmexit; 4464 4465 /* 4466 * If the IRQ is L2's PI notification vector, process posted 4467 * interrupts for L2 instead of injecting VM-Exit, as the 4468 * detection/morphing architecturally occurs when the IRQ is 4469 * delivered to the CPU. Note, only interrupts that are routed 4470 * through the local APIC trigger posted interrupt processing, 4471 * and enabling posted interrupts requires ACK-on-exit. 4472 */ 4473 if (irq == vmx->nested.posted_intr_nv) { 4474 /* 4475 * Nested posted interrupts are delivered via RVI, i.e. 4476 * aren't injected by KVM, and so can be queued even if 4477 * manual event injection is disallowed. 4478 */ 4479 if (block_non_injected_events) 4480 return -EBUSY; 4481 4482 vmx->nested.pi_pending = true; 4483 kvm_apic_clear_irr(vcpu, irq); 4484 goto no_vmexit; 4485 } 4486 4487 if (block_nested_events) 4488 return -EBUSY; 4489 4490 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 4491 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR | irq, 0); 4492 4493 /* 4494 * ACK the interrupt _after_ emulating VM-Exit, as the IRQ must 4495 * be marked as in-service in vmcs01.GUEST_INTERRUPT_STATUS.SVI 4496 * if APICv is active. 4497 */ 4498 kvm_apic_ack_interrupt(vcpu, irq); 4499 return 0; 4500 } 4501 4502no_vmexit: 4503 return vmx_complete_nested_posted_interrupt(vcpu); 4504} 4505 4506static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) 4507{ 4508 ktime_t remaining = 4509 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer); 4510 u64 value; 4511 4512 if (ktime_to_ns(remaining) <= 0) 4513 return 0; 4514 4515 value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz; 4516 do_div(value, 1000000); 4517 return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; 4518} 4519 4520static bool is_vmcs12_ext_field(unsigned long field) 4521{ 4522 switch (field) { 4523 case GUEST_ES_SELECTOR: 4524 case GUEST_CS_SELECTOR: 4525 case GUEST_SS_SELECTOR: 4526 case GUEST_DS_SELECTOR: 4527 case GUEST_FS_SELECTOR: 4528 case GUEST_GS_SELECTOR: 4529 case GUEST_LDTR_SELECTOR: 4530 case GUEST_TR_SELECTOR: 4531 case GUEST_ES_LIMIT: 4532 case GUEST_CS_LIMIT: 4533 case GUEST_SS_LIMIT: 4534 case GUEST_DS_LIMIT: 4535 case GUEST_FS_LIMIT: 4536 case GUEST_GS_LIMIT: 4537 case GUEST_LDTR_LIMIT: 4538 case GUEST_TR_LIMIT: 4539 case GUEST_GDTR_LIMIT: 4540 case GUEST_IDTR_LIMIT: 4541 case GUEST_ES_AR_BYTES: 4542 case GUEST_DS_AR_BYTES: 4543 case GUEST_FS_AR_BYTES: 4544 case GUEST_GS_AR_BYTES: 4545 case GUEST_LDTR_AR_BYTES: 4546 case GUEST_TR_AR_BYTES: 4547 case GUEST_ES_BASE: 4548 case GUEST_CS_BASE: 4549 case GUEST_SS_BASE: 4550 case GUEST_DS_BASE: 4551 case GUEST_FS_BASE: 4552 case GUEST_GS_BASE: 4553 case GUEST_LDTR_BASE: 4554 case GUEST_TR_BASE: 4555 case GUEST_GDTR_BASE: 4556 case GUEST_IDTR_BASE: 4557 case GUEST_PENDING_DBG_EXCEPTIONS: 4558 case GUEST_BNDCFGS: 4559 return true; 4560 default: 4561 break; 4562 } 4563 4564 return false; 4565} 4566 4567static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4568 struct vmcs12 *vmcs12) 4569{ 4570 struct vcpu_vmx *vmx = to_vmx(vcpu); 4571 4572 vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR); 4573 vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR); 4574 vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR); 4575 vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR); 4576 vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR); 4577 vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR); 4578 vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR); 4579 vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR); 4580 vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT); 4581 vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT); 4582 vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT); 4583 vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT); 4584 vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT); 4585 vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT); 4586 vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT); 4587 vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT); 4588 vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT); 4589 vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT); 4590 vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES); 4591 vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES); 4592 vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES); 4593 vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES); 4594 vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES); 4595 vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES); 4596 vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE); 4597 vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE); 4598 vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE); 4599 vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE); 4600 vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE); 4601 vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE); 4602 vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE); 4603 vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE); 4604 vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE); 4605 vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE); 4606 vmcs12->guest_pending_dbg_exceptions = 4607 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS); 4608 4609 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false; 4610} 4611 4612static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, 4613 struct vmcs12 *vmcs12) 4614{ 4615 struct vcpu_vmx *vmx = to_vmx(vcpu); 4616 int cpu; 4617 4618 if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare) 4619 return; 4620 4621 4622 WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01); 4623 4624 cpu = get_cpu(); 4625 vmx->loaded_vmcs = &vmx->nested.vmcs02; 4626 vmx_vcpu_load_vmcs(vcpu, cpu); 4627 4628 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4629 4630 vmx->loaded_vmcs = &vmx->vmcs01; 4631 vmx_vcpu_load_vmcs(vcpu, cpu); 4632 put_cpu(); 4633} 4634 4635/* 4636 * Update the guest state fields of vmcs12 to reflect changes that 4637 * occurred while L2 was running. (The "IA-32e mode guest" bit of the 4638 * VM-entry controls is also updated, since this is really a guest 4639 * state bit.) 4640 */ 4641static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) 4642{ 4643 struct vcpu_vmx *vmx = to_vmx(vcpu); 4644 4645 if (nested_vmx_is_evmptr12_valid(vmx)) 4646 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 4647 4648 vmx->nested.need_sync_vmcs02_to_vmcs12_rare = 4649 !nested_vmx_is_evmptr12_valid(vmx); 4650 4651 vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); 4652 vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); 4653 4654 vmcs12->guest_rsp = kvm_rsp_read(vcpu); 4655 vmcs12->guest_rip = kvm_rip_read(vcpu); 4656 vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS); 4657 4658 vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); 4659 vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); 4660 4661 vmcs12->guest_interruptibility_info = 4662 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); 4663 4664 if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED) 4665 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT; 4666 else if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) 4667 vmcs12->guest_activity_state = GUEST_ACTIVITY_WAIT_SIPI; 4668 else 4669 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; 4670 4671 if (nested_cpu_has_preemption_timer(vmcs12) && 4672 vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER && 4673 !vmx->nested.nested_run_pending) 4674 vmcs12->vmx_preemption_timer_value = 4675 vmx_get_preemption_timer_value(vcpu); 4676 4677 /* 4678 * In some cases (usually, nested EPT), L2 is allowed to change its 4679 * own CR3 without exiting. If it has changed it, we must keep it. 4680 * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined 4681 * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12. 4682 * 4683 * Additionally, restore L2's PDPTR to vmcs12. 4684 */ 4685 if (enable_ept) { 4686 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3); 4687 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { 4688 vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0); 4689 vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1); 4690 vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2); 4691 vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3); 4692 } 4693 } 4694 4695 vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS); 4696 4697 if (nested_cpu_has_vid(vmcs12)) 4698 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS); 4699 4700 vmcs12->vm_entry_controls = 4701 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) | 4702 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE); 4703 4704 /* 4705 * Note! Save DR7, but intentionally don't grab DEBUGCTL from vmcs02. 4706 * Writes to DEBUGCTL that aren't intercepted by L1 are immediately 4707 * propagated to vmcs12 (see vmx_set_msr()), as the value loaded into 4708 * vmcs02 doesn't strictly track vmcs12. 4709 */ 4710 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS) 4711 vmcs12->guest_dr7 = vcpu->arch.dr7; 4712 4713 if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER) 4714 vmcs12->guest_ia32_efer = vcpu->arch.efer; 4715 4716 vmcs_read_cet_state(&vmx->vcpu, &vmcs12->guest_s_cet, 4717 &vmcs12->guest_ssp, 4718 &vmcs12->guest_ssp_tbl); 4719} 4720 4721/* 4722 * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits 4723 * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12), 4724 * and this function updates it to reflect the changes to the guest state while 4725 * L2 was running (and perhaps made some exits which were handled directly by L0 4726 * without going back to L1), and to reflect the exit reason. 4727 * Note that we do not have to copy here all VMCS fields, just those that 4728 * could have changed by the L2 guest or the exit - i.e., the guest-state and 4729 * exit-information fields only. Other fields are modified by L1 with VMWRITE, 4730 * which already writes to vmcs12 directly. 4731 */ 4732static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, 4733 u32 vm_exit_reason, u32 exit_intr_info, 4734 unsigned long exit_qualification, u32 exit_insn_len) 4735{ 4736 /* update exit information fields: */ 4737 vmcs12->vm_exit_reason = vm_exit_reason; 4738 if (vmx_get_exit_reason(vcpu).enclave_mode) 4739 vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; 4740 vmcs12->exit_qualification = exit_qualification; 4741 4742 /* 4743 * On VM-Exit due to a failed VM-Entry, the VMCS isn't marked launched 4744 * and only EXIT_REASON and EXIT_QUALIFICATION are updated, all other 4745 * exit info fields are unmodified. 4746 */ 4747 if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { 4748 vmcs12->launch_state = 1; 4749 4750 /* vm_entry_intr_info_field is cleared on exit. Emulate this 4751 * instead of reading the real value. */ 4752 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK; 4753 4754 /* 4755 * Transfer the event that L0 or L1 may wanted to inject into 4756 * L2 to IDT_VECTORING_INFO_FIELD. 4757 */ 4758 vmcs12_save_pending_event(vcpu, vmcs12, 4759 vm_exit_reason, exit_intr_info); 4760 4761 vmcs12->vm_exit_intr_info = exit_intr_info; 4762 vmcs12->vm_exit_instruction_len = exit_insn_len; 4763 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 4764 4765 /* 4766 * According to spec, there's no need to store the guest's 4767 * MSRs if the exit is due to a VM-entry failure that occurs 4768 * during or after loading the guest state. Since this exit 4769 * does not fall in that category, we need to save the MSRs. 4770 */ 4771 if (nested_vmx_store_msr(vcpu, 4772 vmcs12->vm_exit_msr_store_addr, 4773 vmcs12->vm_exit_msr_store_count)) 4774 nested_vmx_abort(vcpu, 4775 VMX_ABORT_SAVE_GUEST_MSR_FAIL); 4776 } 4777} 4778 4779/* 4780 * A part of what we need to when the nested L2 guest exits and we want to 4781 * run its L1 parent, is to reset L1's guest state to the host state specified 4782 * in vmcs12. 4783 * This function is to be called not only on normal nested exit, but also on 4784 * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry 4785 * Failures During or After Loading Guest State"). 4786 * This function should be called when the active VMCS is L1's (vmcs01). 4787 */ 4788static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, 4789 struct vmcs12 *vmcs12) 4790{ 4791 enum vm_entry_failure_code ignored; 4792 struct kvm_segment seg; 4793 4794 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) 4795 vcpu->arch.efer = vmcs12->host_ia32_efer; 4796 else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4797 vcpu->arch.efer |= (EFER_LMA | EFER_LME); 4798 else 4799 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME); 4800 vmx_set_efer(vcpu, vcpu->arch.efer); 4801 4802 kvm_rsp_write(vcpu, vmcs12->host_rsp); 4803 kvm_rip_write(vcpu, vmcs12->host_rip); 4804 vmx_set_rflags(vcpu, X86_EFLAGS_FIXED); 4805 vmx_set_interrupt_shadow(vcpu, 0); 4806 4807 /* 4808 * Note that calling vmx_set_cr0 is important, even if cr0 hasn't 4809 * actually changed, because vmx_set_cr0 refers to efer set above. 4810 * 4811 * CR0_GUEST_HOST_MASK is already set in the original vmcs01 4812 * (KVM doesn't change it); 4813 */ 4814 vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); 4815 vmx_set_cr0(vcpu, vmcs12->host_cr0); 4816 4817 /* Same as above - no reason to call set_cr4_guest_host_mask(). */ 4818 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4819 vmx_set_cr4(vcpu, vmcs12->host_cr4); 4820 4821 nested_ept_uninit_mmu_context(vcpu); 4822 4823 /* 4824 * Only PDPTE load can fail as the value of cr3 was checked on entry and 4825 * couldn't have changed. 4826 */ 4827 if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, true, &ignored)) 4828 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); 4829 4830 nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); 4831 4832 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); 4833 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); 4834 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip); 4835 vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base); 4836 vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base); 4837 vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF); 4838 vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF); 4839 4840 /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1. */ 4841 if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS) 4842 vmcs_write64(GUEST_BNDCFGS, 0); 4843 4844 /* 4845 * Load CET state from host state if VM_EXIT_LOAD_CET_STATE is set. 4846 * otherwise CET state should be retained across VM-exit, i.e., 4847 * guest values should be propagated from vmcs12 to vmcs01. 4848 */ 4849 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_CET_STATE) 4850 vmcs_write_cet_state(vcpu, vmcs12->host_s_cet, vmcs12->host_ssp, 4851 vmcs12->host_ssp_tbl); 4852 else 4853 vmcs_write_cet_state(vcpu, vmcs12->guest_s_cet, vmcs12->guest_ssp, 4854 vmcs12->guest_ssp_tbl); 4855 4856 if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) { 4857 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat); 4858 vcpu->arch.pat = vmcs12->host_ia32_pat; 4859 } 4860 if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL) && 4861 kvm_pmu_has_perf_global_ctrl(vcpu_to_pmu(vcpu))) 4862 WARN_ON_ONCE(__kvm_emulate_msr_write(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, 4863 vmcs12->host_ia32_perf_global_ctrl)); 4864 4865 /* Set L1 segment info according to Intel SDM 4866 27.5.2 Loading Host Segment and Descriptor-Table Registers */ 4867 seg = (struct kvm_segment) { 4868 .base = 0, 4869 .limit = 0xFFFFFFFF, 4870 .selector = vmcs12->host_cs_selector, 4871 .type = 11, 4872 .present = 1, 4873 .s = 1, 4874 .g = 1 4875 }; 4876 if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) 4877 seg.l = 1; 4878 else 4879 seg.db = 1; 4880 __vmx_set_segment(vcpu, &seg, VCPU_SREG_CS); 4881 seg = (struct kvm_segment) { 4882 .base = 0, 4883 .limit = 0xFFFFFFFF, 4884 .type = 3, 4885 .present = 1, 4886 .s = 1, 4887 .db = 1, 4888 .g = 1 4889 }; 4890 seg.selector = vmcs12->host_ds_selector; 4891 __vmx_set_segment(vcpu, &seg, VCPU_SREG_DS); 4892 seg.selector = vmcs12->host_es_selector; 4893 __vmx_set_segment(vcpu, &seg, VCPU_SREG_ES); 4894 seg.selector = vmcs12->host_ss_selector; 4895 __vmx_set_segment(vcpu, &seg, VCPU_SREG_SS); 4896 seg.selector = vmcs12->host_fs_selector; 4897 seg.base = vmcs12->host_fs_base; 4898 __vmx_set_segment(vcpu, &seg, VCPU_SREG_FS); 4899 seg.selector = vmcs12->host_gs_selector; 4900 seg.base = vmcs12->host_gs_base; 4901 __vmx_set_segment(vcpu, &seg, VCPU_SREG_GS); 4902 seg = (struct kvm_segment) { 4903 .base = vmcs12->host_tr_base, 4904 .limit = 0x67, 4905 .selector = vmcs12->host_tr_selector, 4906 .type = 11, 4907 .present = 1 4908 }; 4909 __vmx_set_segment(vcpu, &seg, VCPU_SREG_TR); 4910 4911 memset(&seg, 0, sizeof(seg)); 4912 seg.unusable = 1; 4913 __vmx_set_segment(vcpu, &seg, VCPU_SREG_LDTR); 4914 4915 kvm_set_dr(vcpu, 7, 0x400); 4916 vmx_guest_debugctl_write(vcpu, 0); 4917 4918 if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr, 4919 vmcs12->vm_exit_msr_load_count)) 4920 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 4921 4922 to_vt(vcpu)->emulation_required = vmx_emulation_required(vcpu); 4923} 4924 4925static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx) 4926{ 4927 struct vmx_uret_msr *efer_msr; 4928 unsigned int i; 4929 4930 if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER) 4931 return vmcs_read64(GUEST_IA32_EFER); 4932 4933 if (cpu_has_load_ia32_efer()) 4934 return kvm_host.efer; 4935 4936 for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) { 4937 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER) 4938 return vmx->msr_autoload.guest.val[i].value; 4939 } 4940 4941 efer_msr = vmx_find_uret_msr(vmx, MSR_EFER); 4942 if (efer_msr) 4943 return efer_msr->data; 4944 4945 return kvm_host.efer; 4946} 4947 4948static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) 4949{ 4950 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 4951 struct vcpu_vmx *vmx = to_vmx(vcpu); 4952 struct vmx_msr_entry g, h; 4953 gpa_t gpa; 4954 u32 i, j; 4955 4956 vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT); 4957 4958 if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) { 4959 /* 4960 * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set 4961 * as vmcs01.GUEST_DR7 contains a userspace defined value 4962 * and vcpu->arch.dr7 is not squirreled away before the 4963 * nested VMENTER (not worth adding a variable in nested_vmx). 4964 */ 4965 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) 4966 kvm_set_dr(vcpu, 7, DR7_FIXED_1); 4967 else 4968 WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7))); 4969 } 4970 4971 /* Reload DEBUGCTL to ensure vmcs01 has a fresh FREEZE_IN_SMM value. */ 4972 vmx_reload_guest_debugctl(vcpu); 4973 4974 /* 4975 * Note that calling vmx_set_{efer,cr0,cr4} is important as they 4976 * handle a variety of side effects to KVM's software model. 4977 */ 4978 vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx)); 4979 4980 vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); 4981 vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW)); 4982 4983 vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK); 4984 vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW)); 4985 4986 nested_ept_uninit_mmu_context(vcpu); 4987 vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); 4988 kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); 4989 4990 /* 4991 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs 4992 * from vmcs01 (if necessary). The PDPTRs are not loaded on 4993 * VMFail, like everything else we just need to ensure our 4994 * software model is up-to-date. 4995 */ 4996 if (enable_ept && is_pae_paging(vcpu)) 4997 ept_save_pdptrs(vcpu); 4998 4999 kvm_mmu_reset_context(vcpu); 5000 5001 /* 5002 * This nasty bit of open coding is a compromise between blindly 5003 * loading L1's MSRs using the exit load lists (incorrect emulation 5004 * of VMFail), leaving the nested VM's MSRs in the software model 5005 * (incorrect behavior) and snapshotting the modified MSRs (too 5006 * expensive since the lists are unbound by hardware). For each 5007 * MSR that was (prematurely) loaded from the nested VMEntry load 5008 * list, reload it from the exit load list if it exists and differs 5009 * from the guest value. The intent is to stuff host state as 5010 * silently as possible, not to fully process the exit load list. 5011 */ 5012 for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) { 5013 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g)); 5014 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) { 5015 pr_debug_ratelimited( 5016 "%s read MSR index failed (%u, 0x%08llx)\n", 5017 __func__, i, gpa); 5018 goto vmabort; 5019 } 5020 5021 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) { 5022 gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h)); 5023 if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) { 5024 pr_debug_ratelimited( 5025 "%s read MSR failed (%u, 0x%08llx)\n", 5026 __func__, j, gpa); 5027 goto vmabort; 5028 } 5029 if (h.index != g.index) 5030 continue; 5031 if (h.value == g.value) 5032 break; 5033 5034 if (nested_vmx_load_msr_check(vcpu, &h)) { 5035 pr_debug_ratelimited( 5036 "%s check failed (%u, 0x%x, 0x%x)\n", 5037 __func__, j, h.index, h.reserved); 5038 goto vmabort; 5039 } 5040 5041 if (kvm_emulate_msr_write(vcpu, h.index, h.value)) { 5042 pr_debug_ratelimited( 5043 "%s WRMSR failed (%u, 0x%x, 0x%llx)\n", 5044 __func__, j, h.index, h.value); 5045 goto vmabort; 5046 } 5047 } 5048 } 5049 5050 return; 5051 5052vmabort: 5053 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL); 5054} 5055 5056/* 5057 * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1 5058 * and modify vmcs12 to make it see what it would expect to see there if 5059 * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) 5060 */ 5061void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, 5062 u32 exit_intr_info, unsigned long exit_qualification, 5063 u32 exit_insn_len) 5064{ 5065 struct vcpu_vmx *vmx = to_vmx(vcpu); 5066 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 5067 5068 /* Pending MTF traps are discarded on VM-Exit. */ 5069 vmx->nested.mtf_pending = false; 5070 5071 /* trying to cancel vmlaunch/vmresume is a bug */ 5072 WARN_ON_ONCE(vmx->nested.nested_run_pending); 5073 5074#ifdef CONFIG_KVM_HYPERV 5075 if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { 5076 /* 5077 * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map 5078 * Enlightened VMCS after migration and we still need to 5079 * do that when something is forcing L2->L1 exit prior to 5080 * the first L2 run. 5081 */ 5082 (void)nested_get_evmcs_page(vcpu); 5083 } 5084#endif 5085 5086 /* Service pending TLB flush requests for L2 before switching to L1. */ 5087 kvm_service_local_tlb_flush_requests(vcpu); 5088 5089 /* 5090 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between 5091 * now and the new vmentry. Ensure that the VMCS02 PDPTR fields are 5092 * up-to-date before switching to L1. 5093 */ 5094 if (enable_ept && is_pae_paging(vcpu)) 5095 vmx_ept_load_pdptrs(vcpu); 5096 5097 leave_guest_mode(vcpu); 5098 5099 if (nested_cpu_has_preemption_timer(vmcs12)) 5100 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); 5101 5102 if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) { 5103 vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; 5104 if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) 5105 vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; 5106 } 5107 5108 if (likely(!vmx->fail)) { 5109 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 5110 5111 if (vm_exit_reason != -1) 5112 prepare_vmcs12(vcpu, vmcs12, vm_exit_reason, 5113 exit_intr_info, exit_qualification, 5114 exit_insn_len); 5115 5116 /* 5117 * Must happen outside of sync_vmcs02_to_vmcs12() as it will 5118 * also be used to capture vmcs12 cache as part of 5119 * capturing nVMX state for snapshot (migration). 5120 * 5121 * Otherwise, this flush will dirty guest memory at a 5122 * point it is already assumed by user-space to be 5123 * immutable. 5124 */ 5125 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12); 5126 } else { 5127 /* 5128 * The only expected VM-instruction error is "VM entry with 5129 * invalid control field(s)." Anything else indicates a 5130 * problem with L0. 5131 */ 5132 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) != 5133 VMXERR_ENTRY_INVALID_CONTROL_FIELD); 5134 5135 /* VM-Fail at VM-Entry means KVM missed a consistency check. */ 5136 WARN_ON_ONCE(warn_on_missed_cc); 5137 } 5138 5139 /* 5140 * Drop events/exceptions that were queued for re-injection to L2 5141 * (picked up via vmx_complete_interrupts()), as well as exceptions 5142 * that were pending for L2. Note, this must NOT be hoisted above 5143 * prepare_vmcs12(), events/exceptions queued for re-injection need to 5144 * be captured in vmcs12 (see vmcs12_save_pending_event()). 5145 */ 5146 vcpu->arch.nmi_injected = false; 5147 kvm_clear_exception_queue(vcpu); 5148 kvm_clear_interrupt_queue(vcpu); 5149 5150 vmx_switch_vmcs(vcpu, &vmx->vmcs01); 5151 5152 kvm_nested_vmexit_handle_ibrs(vcpu); 5153 5154 /* Update any VMCS fields that might have changed while L2 ran */ 5155 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); 5156 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); 5157 vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); 5158 if (kvm_caps.has_tsc_control) 5159 vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); 5160 5161 if (vmx->nested.l1_tpr_threshold != -1) 5162 vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold); 5163 5164 if (vmx->nested.change_vmcs01_virtual_apic_mode) { 5165 vmx->nested.change_vmcs01_virtual_apic_mode = false; 5166 vmx_set_virtual_apic_mode(vcpu); 5167 } 5168 5169 if (vmx->nested.update_vmcs01_cpu_dirty_logging) { 5170 vmx->nested.update_vmcs01_cpu_dirty_logging = false; 5171 vmx_update_cpu_dirty_logging(vcpu); 5172 } 5173 5174 nested_put_vmcs12_pages(vcpu); 5175 5176 if (vmx->nested.reload_vmcs01_apic_access_page) { 5177 vmx->nested.reload_vmcs01_apic_access_page = false; 5178 kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); 5179 } 5180 5181 if (vmx->nested.update_vmcs01_apicv_status) { 5182 vmx->nested.update_vmcs01_apicv_status = false; 5183 vmx_refresh_apicv_exec_ctrl(vcpu); 5184 } 5185 5186 if (vmx->nested.update_vmcs01_hwapic_isr) { 5187 vmx->nested.update_vmcs01_hwapic_isr = false; 5188 kvm_apic_update_hwapic_isr(vcpu); 5189 } 5190 5191 if ((vm_exit_reason != -1) && 5192 (enable_shadow_vmcs || nested_vmx_is_evmptr12_valid(vmx))) 5193 vmx->nested.need_vmcs12_to_shadow_sync = true; 5194 5195 /* in case we halted in L2 */ 5196 kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); 5197 5198 if (likely(!vmx->fail)) { 5199 if (vm_exit_reason != -1) 5200 trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, 5201 vmcs12->exit_qualification, 5202 vmcs12->idt_vectoring_info_field, 5203 vmcs12->vm_exit_intr_info, 5204 vmcs12->vm_exit_intr_error_code, 5205 KVM_ISA_VMX); 5206 5207 load_vmcs12_host_state(vcpu, vmcs12); 5208 5209 /* 5210 * Process events if an injectable IRQ or NMI is pending, even 5211 * if the event is blocked (RFLAGS.IF is cleared on VM-Exit). 5212 * If an event became pending while L2 was active, KVM needs to 5213 * either inject the event or request an IRQ/NMI window. SMIs 5214 * don't need to be processed as SMM is mutually exclusive with 5215 * non-root mode. INIT/SIPI don't need to be checked as INIT 5216 * is blocked post-VMXON, and SIPIs are ignored. 5217 */ 5218 if (kvm_cpu_has_injectable_intr(vcpu) || vcpu->arch.nmi_pending) 5219 kvm_make_request(KVM_REQ_EVENT, vcpu); 5220 return; 5221 } 5222 5223 /* 5224 * After an early L2 VM-entry failure, we're now back 5225 * in L1 which thinks it just finished a VMLAUNCH or 5226 * VMRESUME instruction, so we need to set the failure 5227 * flag and the VM-instruction error field of the VMCS 5228 * accordingly, and skip the emulated instruction. 5229 */ 5230 (void)nested_vmx_fail(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD); 5231 5232 /* 5233 * Restore L1's host state to KVM's software model. We're here 5234 * because a consistency check was caught by hardware, which 5235 * means some amount of guest state has been propagated to KVM's 5236 * model and needs to be unwound to the host's state. 5237 */ 5238 nested_vmx_restore_host_state(vcpu); 5239 5240 vmx->fail = 0; 5241} 5242 5243static void nested_vmx_triple_fault(struct kvm_vcpu *vcpu) 5244{ 5245 kvm_clear_request(KVM_REQ_TRIPLE_FAULT, vcpu); 5246 nested_vmx_vmexit(vcpu, EXIT_REASON_TRIPLE_FAULT, 0, 0); 5247} 5248 5249/* 5250 * Decode the memory-address operand of a vmx instruction, as recorded on an 5251 * exit caused by such an instruction (run by a guest hypervisor). 5252 * On success, returns 0. When the operand is invalid, returns 1 and throws 5253 * #UD, #GP, or #SS. 5254 */ 5255int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification, 5256 u32 vmx_instruction_info, bool wr, int len, gva_t *ret) 5257{ 5258 gva_t off; 5259 bool exn; 5260 struct kvm_segment s; 5261 5262 /* 5263 * According to Vol. 3B, "Information for VM Exits Due to Instruction 5264 * Execution", on an exit, vmx_instruction_info holds most of the 5265 * addressing components of the operand. Only the displacement part 5266 * is put in exit_qualification (see 3B, "Basic VM-Exit Information"). 5267 * For how an actual address is calculated from all these components, 5268 * refer to Vol. 1, "Operand Addressing". 5269 */ 5270 int scaling = vmx_instruction_info & 3; 5271 int addr_size = (vmx_instruction_info >> 7) & 7; 5272 bool is_reg = vmx_instruction_info & (1u << 10); 5273 int seg_reg = (vmx_instruction_info >> 15) & 7; 5274 int index_reg = (vmx_instruction_info >> 18) & 0xf; 5275 bool index_is_valid = !(vmx_instruction_info & (1u << 22)); 5276 int base_reg = (vmx_instruction_info >> 23) & 0xf; 5277 bool base_is_valid = !(vmx_instruction_info & (1u << 27)); 5278 5279 if (is_reg) { 5280 kvm_queue_exception(vcpu, UD_VECTOR); 5281 return 1; 5282 } 5283 5284 /* Addr = segment_base + offset */ 5285 /* offset = base + [index * scale] + displacement */ 5286 off = exit_qualification; /* holds the displacement */ 5287 if (addr_size == 1) 5288 off = (gva_t)sign_extend64(off, 31); 5289 else if (addr_size == 0) 5290 off = (gva_t)sign_extend64(off, 15); 5291 if (base_is_valid) 5292 off += kvm_register_read(vcpu, base_reg); 5293 if (index_is_valid) 5294 off += kvm_register_read(vcpu, index_reg) << scaling; 5295 vmx_get_segment(vcpu, &s, seg_reg); 5296 5297 /* 5298 * The effective address, i.e. @off, of a memory operand is truncated 5299 * based on the address size of the instruction. Note that this is 5300 * the *effective address*, i.e. the address prior to accounting for 5301 * the segment's base. 5302 */ 5303 if (addr_size == 1) /* 32 bit */ 5304 off &= 0xffffffff; 5305 else if (addr_size == 0) /* 16 bit */ 5306 off &= 0xffff; 5307 5308 /* Checks for #GP/#SS exceptions. */ 5309 exn = false; 5310 if (is_long_mode(vcpu)) { 5311 /* 5312 * The virtual/linear address is never truncated in 64-bit 5313 * mode, e.g. a 32-bit address size can yield a 64-bit virtual 5314 * address when using FS/GS with a non-zero base. 5315 */ 5316 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS) 5317 *ret = s.base + off; 5318 else 5319 *ret = off; 5320 5321 *ret = vmx_get_untagged_addr(vcpu, *ret, 0); 5322 /* Long mode: #GP(0)/#SS(0) if the memory address is in a 5323 * non-canonical form. This is the only check on the memory 5324 * destination for long mode! 5325 */ 5326 exn = is_noncanonical_address(*ret, vcpu, 0); 5327 } else { 5328 /* 5329 * When not in long mode, the virtual/linear address is 5330 * unconditionally truncated to 32 bits regardless of the 5331 * address size. 5332 */ 5333 *ret = (s.base + off) & 0xffffffff; 5334 5335 /* Protected mode: apply checks for segment validity in the 5336 * following order: 5337 * - segment type check (#GP(0) may be thrown) 5338 * - usability check (#GP(0)/#SS(0)) 5339 * - limit check (#GP(0)/#SS(0)) 5340 */ 5341 if (wr) 5342 /* #GP(0) if the destination operand is located in a 5343 * read-only data segment or any code segment. 5344 */ 5345 exn = ((s.type & 0xa) == 0 || (s.type & 8)); 5346 else 5347 /* #GP(0) if the source operand is located in an 5348 * execute-only code segment 5349 */ 5350 exn = ((s.type & 0xa) == 8); 5351 if (exn) { 5352 kvm_queue_exception_e(vcpu, GP_VECTOR, 0); 5353 return 1; 5354 } 5355 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable. 5356 */ 5357 exn = (s.unusable != 0); 5358 5359 /* 5360 * Protected mode: #GP(0)/#SS(0) if the memory operand is 5361 * outside the segment limit. All CPUs that support VMX ignore 5362 * limit checks for flat segments, i.e. segments with base==0, 5363 * limit==0xffffffff and of type expand-up data or code. 5364 */ 5365 if (!(s.base == 0 && s.limit == 0xffffffff && 5366 ((s.type & 8) || !(s.type & 4)))) 5367 exn = exn || ((u64)off + len - 1 > s.limit); 5368 } 5369 if (exn) { 5370 kvm_queue_exception_e(vcpu, 5371 seg_reg == VCPU_SREG_SS ? 5372 SS_VECTOR : GP_VECTOR, 5373 0); 5374 return 1; 5375 } 5376 5377 return 0; 5378} 5379 5380static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, 5381 int *ret) 5382{ 5383 gva_t gva; 5384 struct x86_exception e; 5385 int r; 5386 5387 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 5388 vmcs_read32(VMX_INSTRUCTION_INFO), false, 5389 sizeof(*vmpointer), &gva)) { 5390 *ret = 1; 5391 return -EINVAL; 5392 } 5393 5394 r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e); 5395 if (r != X86EMUL_CONTINUE) { 5396 *ret = kvm_handle_memory_failure(vcpu, r, &e); 5397 return -EINVAL; 5398 } 5399 5400 return 0; 5401} 5402 5403/* 5404 * Allocate a shadow VMCS and associate it with the currently loaded 5405 * VMCS, unless such a shadow VMCS already exists. The newly allocated 5406 * VMCS is also VMCLEARed, so that it is ready for use. 5407 */ 5408static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu) 5409{ 5410 struct vcpu_vmx *vmx = to_vmx(vcpu); 5411 struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs; 5412 5413 /* 5414 * KVM allocates a shadow VMCS only when L1 executes VMXON and frees it 5415 * when L1 executes VMXOFF or the vCPU is forced out of nested 5416 * operation. VMXON faults if the CPU is already post-VMXON, so it 5417 * should be impossible to already have an allocated shadow VMCS. KVM 5418 * doesn't support virtualization of VMCS shadowing, so vmcs01 should 5419 * always be the loaded VMCS. 5420 */ 5421 if (WARN_ON(loaded_vmcs != &vmx->vmcs01 || loaded_vmcs->shadow_vmcs)) 5422 return loaded_vmcs->shadow_vmcs; 5423 5424 loaded_vmcs->shadow_vmcs = alloc_vmcs(true); 5425 if (loaded_vmcs->shadow_vmcs) 5426 vmcs_clear(loaded_vmcs->shadow_vmcs); 5427 5428 return loaded_vmcs->shadow_vmcs; 5429} 5430 5431static int enter_vmx_operation(struct kvm_vcpu *vcpu) 5432{ 5433 struct vcpu_vmx *vmx = to_vmx(vcpu); 5434 int r; 5435 5436 r = alloc_loaded_vmcs(&vmx->nested.vmcs02); 5437 if (r < 0) 5438 goto out_vmcs02; 5439 5440 vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 5441 if (!vmx->nested.cached_vmcs12) 5442 goto out_cached_vmcs12; 5443 5444 vmx->nested.shadow_vmcs12_cache.gpa = INVALID_GPA; 5445 vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT); 5446 if (!vmx->nested.cached_shadow_vmcs12) 5447 goto out_cached_shadow_vmcs12; 5448 5449 if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu)) 5450 goto out_shadow_vmcs; 5451 5452 hrtimer_setup(&vmx->nested.preemption_timer, vmx_preemption_timer_fn, CLOCK_MONOTONIC, 5453 HRTIMER_MODE_ABS_PINNED); 5454 5455 vmx->nested.vpid02 = allocate_vpid(); 5456 5457 vmx->nested.vmcs02_initialized = false; 5458 vmx->nested.vmxon = true; 5459 5460 if (vmx_pt_mode_is_host_guest()) { 5461 vmx->pt_desc.guest.ctl = 0; 5462 pt_update_intercept_for_msr(vcpu); 5463 } 5464 5465 return 0; 5466 5467out_shadow_vmcs: 5468 kfree(vmx->nested.cached_shadow_vmcs12); 5469 5470out_cached_shadow_vmcs12: 5471 kfree(vmx->nested.cached_vmcs12); 5472 5473out_cached_vmcs12: 5474 free_loaded_vmcs(&vmx->nested.vmcs02); 5475 5476out_vmcs02: 5477 return -ENOMEM; 5478} 5479 5480/* Emulate the VMXON instruction. */ 5481static int handle_vmxon(struct kvm_vcpu *vcpu) 5482{ 5483 int ret; 5484 gpa_t vmptr; 5485 uint32_t revision; 5486 struct vcpu_vmx *vmx = to_vmx(vcpu); 5487 const u64 VMXON_NEEDED_FEATURES = FEAT_CTL_LOCKED 5488 | FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX; 5489 5490 /* 5491 * Manually check CR4.VMXE checks, KVM must force CR4.VMXE=1 to enter 5492 * the guest and so cannot rely on hardware to perform the check, 5493 * which has higher priority than VM-Exit (see Intel SDM's pseudocode 5494 * for VMXON). 5495 * 5496 * Rely on hardware for the other pre-VM-Exit checks, CR0.PE=1, !VM86 5497 * and !COMPATIBILITY modes. For an unrestricted guest, KVM doesn't 5498 * force any of the relevant guest state. For a restricted guest, KVM 5499 * does force CR0.PE=1, but only to also force VM86 in order to emulate 5500 * Real Mode, and so there's no need to check CR0.PE manually. 5501 */ 5502 if (!kvm_is_cr4_bit_set(vcpu, X86_CR4_VMXE)) { 5503 kvm_queue_exception(vcpu, UD_VECTOR); 5504 return 1; 5505 } 5506 5507 /* 5508 * The CPL is checked for "not in VMX operation" and for "in VMX root", 5509 * and has higher priority than the VM-Fail due to being post-VMXON, 5510 * i.e. VMXON #GPs outside of VMX non-root if CPL!=0. In VMX non-root, 5511 * VMXON causes VM-Exit and KVM unconditionally forwards VMXON VM-Exits 5512 * from L2 to L1, i.e. there's no need to check for the vCPU being in 5513 * VMX non-root. 5514 * 5515 * Forwarding the VM-Exit unconditionally, i.e. without performing the 5516 * #UD checks (see above), is functionally ok because KVM doesn't allow 5517 * L1 to run L2 without CR4.VMXE=0, and because KVM never modifies L2's 5518 * CR0 or CR4, i.e. it's L2's responsibility to emulate #UDs that are 5519 * missed by hardware due to shadowing CR0 and/or CR4. 5520 */ 5521 if (vmx_get_cpl(vcpu)) { 5522 kvm_inject_gp(vcpu, 0); 5523 return 1; 5524 } 5525 5526 if (vmx->nested.vmxon) 5527 return nested_vmx_fail(vcpu, VMXERR_VMXON_IN_VMX_ROOT_OPERATION); 5528 5529 /* 5530 * Invalid CR0/CR4 generates #GP. These checks are performed if and 5531 * only if the vCPU isn't already in VMX operation, i.e. effectively 5532 * have lower priority than the VM-Fail above. 5533 */ 5534 if (!nested_host_cr0_valid(vcpu, kvm_read_cr0(vcpu)) || 5535 !nested_host_cr4_valid(vcpu, kvm_read_cr4(vcpu))) { 5536 kvm_inject_gp(vcpu, 0); 5537 return 1; 5538 } 5539 5540 if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES) 5541 != VMXON_NEEDED_FEATURES) { 5542 kvm_inject_gp(vcpu, 0); 5543 return 1; 5544 } 5545 5546 if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret)) 5547 return ret; 5548 5549 /* 5550 * SDM 3: 24.11.5 5551 * The first 4 bytes of VMXON region contain the supported 5552 * VMCS revision identifier 5553 * 5554 * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case; 5555 * which replaces physical address width with 32 5556 */ 5557 if (!page_address_valid(vcpu, vmptr)) 5558 return nested_vmx_failInvalid(vcpu); 5559 5560 if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) || 5561 revision != VMCS12_REVISION) 5562 return nested_vmx_failInvalid(vcpu); 5563 5564 vmx->nested.vmxon_ptr = vmptr; 5565 ret = enter_vmx_operation(vcpu); 5566 if (ret) 5567 return ret; 5568 5569 return nested_vmx_succeed(vcpu); 5570} 5571 5572static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu) 5573{ 5574 struct vcpu_vmx *vmx = to_vmx(vcpu); 5575 5576 if (vmx->nested.current_vmptr == INVALID_GPA) 5577 return; 5578 5579 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 5580 5581 if (enable_shadow_vmcs) { 5582 /* copy to memory all shadowed fields in case 5583 they were modified */ 5584 copy_shadow_to_vmcs12(vmx); 5585 vmx_disable_shadow_vmcs(vmx); 5586 } 5587 vmx->nested.posted_intr_nv = -1; 5588 5589 /* Flush VMCS12 to guest memory */ 5590 kvm_vcpu_write_guest_page(vcpu, 5591 vmx->nested.current_vmptr >> PAGE_SHIFT, 5592 vmx->nested.cached_vmcs12, 0, VMCS12_SIZE); 5593 5594 kvm_mmu_free_roots(vcpu->kvm, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL); 5595 5596 vmx->nested.current_vmptr = INVALID_GPA; 5597} 5598 5599/* Emulate the VMXOFF instruction */ 5600static int handle_vmxoff(struct kvm_vcpu *vcpu) 5601{ 5602 if (!nested_vmx_check_permission(vcpu)) 5603 return 1; 5604 5605 free_nested(vcpu); 5606 5607 if (kvm_apic_has_pending_init_or_sipi(vcpu)) 5608 kvm_make_request(KVM_REQ_EVENT, vcpu); 5609 5610 return nested_vmx_succeed(vcpu); 5611} 5612 5613/* Emulate the VMCLEAR instruction */ 5614static int handle_vmclear(struct kvm_vcpu *vcpu) 5615{ 5616 struct vcpu_vmx *vmx = to_vmx(vcpu); 5617 u32 zero = 0; 5618 gpa_t vmptr; 5619 int r; 5620 5621 if (!nested_vmx_check_permission(vcpu)) 5622 return 1; 5623 5624 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5625 return r; 5626 5627 if (!page_address_valid(vcpu, vmptr)) 5628 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_INVALID_ADDRESS); 5629 5630 if (vmptr == vmx->nested.vmxon_ptr) 5631 return nested_vmx_fail(vcpu, VMXERR_VMCLEAR_VMXON_POINTER); 5632 5633 if (likely(!nested_evmcs_handle_vmclear(vcpu, vmptr))) { 5634 if (vmptr == vmx->nested.current_vmptr) 5635 nested_release_vmcs12(vcpu); 5636 5637 /* 5638 * Silently ignore memory errors on VMCLEAR, Intel's pseudocode 5639 * for VMCLEAR includes a "ensure that data for VMCS referenced 5640 * by the operand is in memory" clause that guards writes to 5641 * memory, i.e. doing nothing for I/O is architecturally valid. 5642 * 5643 * FIXME: Suppress failures if and only if no memslot is found, 5644 * i.e. exit to userspace if __copy_to_user() fails. 5645 */ 5646 (void)kvm_vcpu_write_guest(vcpu, 5647 vmptr + offsetof(struct vmcs12, 5648 launch_state), 5649 &zero, sizeof(zero)); 5650 } 5651 5652 return nested_vmx_succeed(vcpu); 5653} 5654 5655/* Emulate the VMLAUNCH instruction */ 5656static int handle_vmlaunch(struct kvm_vcpu *vcpu) 5657{ 5658 return nested_vmx_run(vcpu, true); 5659} 5660 5661/* Emulate the VMRESUME instruction */ 5662static int handle_vmresume(struct kvm_vcpu *vcpu) 5663{ 5664 5665 return nested_vmx_run(vcpu, false); 5666} 5667 5668static int handle_vmread(struct kvm_vcpu *vcpu) 5669{ 5670 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5671 : get_vmcs12(vcpu); 5672 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5673 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5674 struct vcpu_vmx *vmx = to_vmx(vcpu); 5675 struct x86_exception e; 5676 unsigned long field; 5677 u64 value; 5678 gva_t gva = 0; 5679 short offset; 5680 int len, r; 5681 5682 if (!nested_vmx_check_permission(vcpu)) 5683 return 1; 5684 5685 /* Decode instruction info and find the field to read */ 5686 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5687 5688 if (!nested_vmx_is_evmptr12_valid(vmx)) { 5689 /* 5690 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5691 * any VMREAD sets the ALU flags for VMfailInvalid. 5692 */ 5693 if (vmx->nested.current_vmptr == INVALID_GPA || 5694 (is_guest_mode(vcpu) && 5695 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5696 return nested_vmx_failInvalid(vcpu); 5697 5698 offset = get_vmcs12_field_offset(field); 5699 if (offset < 0) 5700 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5701 5702 if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field)) 5703 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5704 5705 /* Read the field, zero-extended to a u64 value */ 5706 value = vmcs12_read_any(vmcs12, field, offset); 5707 } else { 5708 /* 5709 * Hyper-V TLFS (as of 6.0b) explicitly states, that while an 5710 * enlightened VMCS is active VMREAD/VMWRITE instructions are 5711 * unsupported. Unfortunately, certain versions of Windows 11 5712 * don't comply with this requirement which is not enforced in 5713 * genuine Hyper-V. Allow VMREAD from an enlightened VMCS as a 5714 * workaround, as misbehaving guests will panic on VM-Fail. 5715 * Note, enlightened VMCS is incompatible with shadow VMCS so 5716 * all VMREADs from L2 should go to L1. 5717 */ 5718 if (WARN_ON_ONCE(is_guest_mode(vcpu))) 5719 return nested_vmx_failInvalid(vcpu); 5720 5721 offset = evmcs_field_offset(field, NULL); 5722 if (offset < 0) 5723 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5724 5725 /* Read the field, zero-extended to a u64 value */ 5726 value = evmcs_read_any(nested_vmx_evmcs(vmx), field, offset); 5727 } 5728 5729 /* 5730 * Now copy part of this value to register or memory, as requested. 5731 * Note that the number of bits actually copied is 32 or 64 depending 5732 * on the guest's mode (32 or 64 bit), not on the given field's length. 5733 */ 5734 if (instr_info & BIT(10)) { 5735 kvm_register_write(vcpu, (((instr_info) >> 3) & 0xf), value); 5736 } else { 5737 len = is_64_bit_mode(vcpu) ? 8 : 4; 5738 if (get_vmx_mem_address(vcpu, exit_qualification, 5739 instr_info, true, len, &gva)) 5740 return 1; 5741 /* _system ok, nested_vmx_check_permission has verified cpl=0 */ 5742 r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e); 5743 if (r != X86EMUL_CONTINUE) 5744 return kvm_handle_memory_failure(vcpu, r, &e); 5745 } 5746 5747 return nested_vmx_succeed(vcpu); 5748} 5749 5750static bool is_shadow_field_rw(unsigned long field) 5751{ 5752 switch (field) { 5753#define SHADOW_FIELD_RW(x, y) case x: 5754#include "vmcs_shadow_fields.h" 5755 return true; 5756 default: 5757 break; 5758 } 5759 return false; 5760} 5761 5762static bool is_shadow_field_ro(unsigned long field) 5763{ 5764 switch (field) { 5765#define SHADOW_FIELD_RO(x, y) case x: 5766#include "vmcs_shadow_fields.h" 5767 return true; 5768 default: 5769 break; 5770 } 5771 return false; 5772} 5773 5774static int handle_vmwrite(struct kvm_vcpu *vcpu) 5775{ 5776 struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) 5777 : get_vmcs12(vcpu); 5778 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 5779 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5780 struct vcpu_vmx *vmx = to_vmx(vcpu); 5781 struct x86_exception e; 5782 unsigned long field; 5783 short offset; 5784 gva_t gva; 5785 int len, r; 5786 5787 /* 5788 * The value to write might be 32 or 64 bits, depending on L1's long 5789 * mode, and eventually we need to write that into a field of several 5790 * possible lengths. The code below first zero-extends the value to 64 5791 * bit (value), and then copies only the appropriate number of 5792 * bits into the vmcs12 field. 5793 */ 5794 u64 value = 0; 5795 5796 if (!nested_vmx_check_permission(vcpu)) 5797 return 1; 5798 5799 /* 5800 * In VMX non-root operation, when the VMCS-link pointer is INVALID_GPA, 5801 * any VMWRITE sets the ALU flags for VMfailInvalid. 5802 */ 5803 if (vmx->nested.current_vmptr == INVALID_GPA || 5804 (is_guest_mode(vcpu) && 5805 get_vmcs12(vcpu)->vmcs_link_pointer == INVALID_GPA)) 5806 return nested_vmx_failInvalid(vcpu); 5807 5808 if (instr_info & BIT(10)) 5809 value = kvm_register_read(vcpu, (((instr_info) >> 3) & 0xf)); 5810 else { 5811 len = is_64_bit_mode(vcpu) ? 8 : 4; 5812 if (get_vmx_mem_address(vcpu, exit_qualification, 5813 instr_info, false, len, &gva)) 5814 return 1; 5815 r = kvm_read_guest_virt(vcpu, gva, &value, len, &e); 5816 if (r != X86EMUL_CONTINUE) 5817 return kvm_handle_memory_failure(vcpu, r, &e); 5818 } 5819 5820 field = kvm_register_read(vcpu, (((instr_info) >> 28) & 0xf)); 5821 5822 offset = get_vmcs12_field_offset(field); 5823 if (offset < 0) 5824 return nested_vmx_fail(vcpu, VMXERR_UNSUPPORTED_VMCS_COMPONENT); 5825 5826 /* 5827 * If the vCPU supports "VMWRITE to any supported field in the 5828 * VMCS," then the "read-only" fields are actually read/write. 5829 */ 5830 if (vmcs_field_readonly(field) && 5831 !nested_cpu_has_vmwrite_any_field(vcpu)) 5832 return nested_vmx_fail(vcpu, VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT); 5833 5834 /* 5835 * Ensure vmcs12 is up-to-date before any VMWRITE that dirties 5836 * vmcs12, else we may crush a field or consume a stale value. 5837 */ 5838 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) 5839 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 5840 5841 /* 5842 * Some Intel CPUs intentionally drop the reserved bits of the AR byte 5843 * fields on VMWRITE. Emulate this behavior to ensure consistent KVM 5844 * behavior regardless of the underlying hardware, e.g. if an AR_BYTE 5845 * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD 5846 * from L1 will return a different value than VMREAD from L2 (L1 sees 5847 * the stripped down value, L2 sees the full value as stored by KVM). 5848 */ 5849 if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES) 5850 value &= 0x1f0ff; 5851 5852 vmcs12_write_any(vmcs12, field, offset, value); 5853 5854 /* 5855 * Do not track vmcs12 dirty-state if in guest-mode as we actually 5856 * dirty shadow vmcs12 instead of vmcs12. Fields that can be updated 5857 * by L1 without a vmexit are always updated in the vmcs02, i.e. don't 5858 * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path. 5859 */ 5860 if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) { 5861 /* 5862 * L1 can read these fields without exiting, ensure the 5863 * shadow VMCS is up-to-date. 5864 */ 5865 if (enable_shadow_vmcs && is_shadow_field_ro(field)) { 5866 preempt_disable(); 5867 vmcs_load(vmx->vmcs01.shadow_vmcs); 5868 5869 __vmcs_writel(field, value); 5870 5871 vmcs_clear(vmx->vmcs01.shadow_vmcs); 5872 vmcs_load(vmx->loaded_vmcs->vmcs); 5873 preempt_enable(); 5874 } 5875 vmx->nested.dirty_vmcs12 = true; 5876 } 5877 5878 return nested_vmx_succeed(vcpu); 5879} 5880 5881static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr) 5882{ 5883 vmx->nested.current_vmptr = vmptr; 5884 if (enable_shadow_vmcs) { 5885 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS); 5886 vmcs_write64(VMCS_LINK_POINTER, 5887 __pa(vmx->vmcs01.shadow_vmcs)); 5888 vmx->nested.need_vmcs12_to_shadow_sync = true; 5889 } 5890 vmx->nested.dirty_vmcs12 = true; 5891 vmx->nested.force_msr_bitmap_recalc = true; 5892} 5893 5894/* Emulate the VMPTRLD instruction */ 5895static int handle_vmptrld(struct kvm_vcpu *vcpu) 5896{ 5897 struct vcpu_vmx *vmx = to_vmx(vcpu); 5898 gpa_t vmptr; 5899 int r; 5900 5901 if (!nested_vmx_check_permission(vcpu)) 5902 return 1; 5903 5904 if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) 5905 return r; 5906 5907 if (!page_address_valid(vcpu, vmptr)) 5908 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_INVALID_ADDRESS); 5909 5910 if (vmptr == vmx->nested.vmxon_ptr) 5911 return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); 5912 5913 /* Forbid normal VMPTRLD if Enlightened version was used */ 5914 if (nested_vmx_is_evmptr12_valid(vmx)) 5915 return 1; 5916 5917 if (vmx->nested.current_vmptr != vmptr) { 5918 struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; 5919 struct vmcs_hdr hdr; 5920 5921 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, vmptr, VMCS12_SIZE)) { 5922 /* 5923 * Reads from an unbacked page return all 1s, 5924 * which means that the 32 bits located at the 5925 * given physical address won't match the required 5926 * VMCS12_REVISION identifier. 5927 */ 5928 return nested_vmx_fail(vcpu, 5929 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5930 } 5931 5932 if (kvm_read_guest_offset_cached(vcpu->kvm, ghc, &hdr, 5933 offsetof(struct vmcs12, hdr), 5934 sizeof(hdr))) { 5935 return nested_vmx_fail(vcpu, 5936 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5937 } 5938 5939 if (hdr.revision_id != VMCS12_REVISION || 5940 (hdr.shadow_vmcs && 5941 !nested_cpu_has_vmx_shadow_vmcs(vcpu))) { 5942 return nested_vmx_fail(vcpu, 5943 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5944 } 5945 5946 nested_release_vmcs12(vcpu); 5947 5948 /* 5949 * Load VMCS12 from guest memory since it is not already 5950 * cached. 5951 */ 5952 if (kvm_read_guest_cached(vcpu->kvm, ghc, vmx->nested.cached_vmcs12, 5953 VMCS12_SIZE)) { 5954 return nested_vmx_fail(vcpu, 5955 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID); 5956 } 5957 5958 set_current_vmptr(vmx, vmptr); 5959 } 5960 5961 return nested_vmx_succeed(vcpu); 5962} 5963 5964/* Emulate the VMPTRST instruction */ 5965static int handle_vmptrst(struct kvm_vcpu *vcpu) 5966{ 5967 unsigned long exit_qual = vmx_get_exit_qual(vcpu); 5968 u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); 5969 gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; 5970 struct x86_exception e; 5971 gva_t gva; 5972 int r; 5973 5974 if (!nested_vmx_check_permission(vcpu)) 5975 return 1; 5976 5977 if (unlikely(nested_vmx_is_evmptr12_valid(to_vmx(vcpu)))) 5978 return 1; 5979 5980 if (get_vmx_mem_address(vcpu, exit_qual, instr_info, 5981 true, sizeof(gpa_t), &gva)) 5982 return 1; 5983 /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ 5984 r = kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, 5985 sizeof(gpa_t), &e); 5986 if (r != X86EMUL_CONTINUE) 5987 return kvm_handle_memory_failure(vcpu, r, &e); 5988 5989 return nested_vmx_succeed(vcpu); 5990} 5991 5992/* Emulate the INVEPT instruction */ 5993static int handle_invept(struct kvm_vcpu *vcpu) 5994{ 5995 struct vcpu_vmx *vmx = to_vmx(vcpu); 5996 u32 vmx_instruction_info, types; 5997 unsigned long type, roots_to_free; 5998 struct kvm_mmu *mmu; 5999 gva_t gva; 6000 struct x86_exception e; 6001 struct { 6002 u64 eptp, gpa; 6003 } operand; 6004 int i, r, gpr_index; 6005 6006 if (!(vmx->nested.msrs.secondary_ctls_high & 6007 SECONDARY_EXEC_ENABLE_EPT) || 6008 !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) { 6009 kvm_queue_exception(vcpu, UD_VECTOR); 6010 return 1; 6011 } 6012 6013 if (!nested_vmx_check_permission(vcpu)) 6014 return 1; 6015 6016 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6017 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 6018 type = kvm_register_read(vcpu, gpr_index); 6019 6020 types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; 6021 6022 if (type >= 32 || !(types & (1 << type))) 6023 return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6024 6025 /* According to the Intel VMX instruction reference, the memory 6026 * operand is read even if it isn't needed (e.g., for type==global) 6027 */ 6028 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 6029 vmx_instruction_info, false, sizeof(operand), &gva)) 6030 return 1; 6031 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 6032 if (r != X86EMUL_CONTINUE) 6033 return kvm_handle_memory_failure(vcpu, r, &e); 6034 6035 /* 6036 * Nested EPT roots are always held through guest_mmu, 6037 * not root_mmu. 6038 */ 6039 mmu = &vcpu->arch.guest_mmu; 6040 6041 switch (type) { 6042 case VMX_EPT_EXTENT_CONTEXT: 6043 if (!nested_vmx_check_eptp(vcpu, operand.eptp)) 6044 return nested_vmx_fail(vcpu, 6045 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6046 6047 roots_to_free = 0; 6048 if (nested_ept_root_matches(mmu->root.hpa, mmu->root.pgd, 6049 operand.eptp)) 6050 roots_to_free |= KVM_MMU_ROOT_CURRENT; 6051 6052 for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { 6053 if (nested_ept_root_matches(mmu->prev_roots[i].hpa, 6054 mmu->prev_roots[i].pgd, 6055 operand.eptp)) 6056 roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); 6057 } 6058 break; 6059 case VMX_EPT_EXTENT_GLOBAL: 6060 roots_to_free = KVM_MMU_ROOTS_ALL; 6061 break; 6062 default: 6063 BUG(); 6064 break; 6065 } 6066 6067 if (roots_to_free) 6068 kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); 6069 6070 return nested_vmx_succeed(vcpu); 6071} 6072 6073static int handle_invvpid(struct kvm_vcpu *vcpu) 6074{ 6075 struct vcpu_vmx *vmx = to_vmx(vcpu); 6076 u32 vmx_instruction_info; 6077 unsigned long type, types; 6078 gva_t gva; 6079 struct x86_exception e; 6080 struct { 6081 u64 vpid; 6082 u64 gla; 6083 } operand; 6084 u16 vpid02; 6085 int r, gpr_index; 6086 6087 if (!(vmx->nested.msrs.secondary_ctls_high & 6088 SECONDARY_EXEC_ENABLE_VPID) || 6089 !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) { 6090 kvm_queue_exception(vcpu, UD_VECTOR); 6091 return 1; 6092 } 6093 6094 if (!nested_vmx_check_permission(vcpu)) 6095 return 1; 6096 6097 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6098 gpr_index = vmx_get_instr_info_reg2(vmx_instruction_info); 6099 type = kvm_register_read(vcpu, gpr_index); 6100 6101 types = (vmx->nested.msrs.vpid_caps & 6102 VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8; 6103 6104 if (type >= 32 || !(types & (1 << type))) 6105 return nested_vmx_fail(vcpu, 6106 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6107 6108 /* according to the intel vmx instruction reference, the memory 6109 * operand is read even if it isn't needed (e.g., for type==global) 6110 */ 6111 if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), 6112 vmx_instruction_info, false, sizeof(operand), &gva)) 6113 return 1; 6114 r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); 6115 if (r != X86EMUL_CONTINUE) 6116 return kvm_handle_memory_failure(vcpu, r, &e); 6117 6118 if (operand.vpid >> 16) 6119 return nested_vmx_fail(vcpu, 6120 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6121 6122 /* 6123 * Always flush the effective vpid02, i.e. never flush the current VPID 6124 * and never explicitly flush vpid01. INVVPID targets a VPID, not a 6125 * VMCS, and so whether or not the current vmcs12 has VPID enabled is 6126 * irrelevant (and there may not be a loaded vmcs12). 6127 */ 6128 vpid02 = nested_get_vpid02(vcpu); 6129 switch (type) { 6130 case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: 6131 /* 6132 * LAM doesn't apply to addresses that are inputs to TLB 6133 * invalidation. 6134 */ 6135 if (!operand.vpid || 6136 is_noncanonical_invlpg_address(operand.gla, vcpu)) 6137 return nested_vmx_fail(vcpu, 6138 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6139 vpid_sync_vcpu_addr(vpid02, operand.gla); 6140 break; 6141 case VMX_VPID_EXTENT_SINGLE_CONTEXT: 6142 case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: 6143 if (!operand.vpid) 6144 return nested_vmx_fail(vcpu, 6145 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); 6146 vpid_sync_context(vpid02); 6147 break; 6148 case VMX_VPID_EXTENT_ALL_CONTEXT: 6149 vpid_sync_context(vpid02); 6150 break; 6151 default: 6152 WARN_ON_ONCE(1); 6153 return kvm_skip_emulated_instruction(vcpu); 6154 } 6155 6156 /* 6157 * Sync the shadow page tables if EPT is disabled, L1 is invalidating 6158 * linear mappings for L2 (tagged with L2's VPID). Free all guest 6159 * roots as VPIDs are not tracked in the MMU role. 6160 * 6161 * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share 6162 * an MMU when EPT is disabled. 6163 * 6164 * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR. 6165 */ 6166 if (!enable_ept) 6167 kvm_mmu_free_guest_mode_roots(vcpu->kvm, &vcpu->arch.root_mmu); 6168 6169 return nested_vmx_succeed(vcpu); 6170} 6171 6172static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, 6173 struct vmcs12 *vmcs12) 6174{ 6175 u32 index = kvm_rcx_read(vcpu); 6176 u64 new_eptp; 6177 6178 if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12))) 6179 return 1; 6180 if (index >= VMFUNC_EPTP_ENTRIES) 6181 return 1; 6182 6183 if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, 6184 &new_eptp, index * 8, 8)) 6185 return 1; 6186 6187 /* 6188 * If the (L2) guest does a vmfunc to the currently 6189 * active ept pointer, we don't have to do anything else 6190 */ 6191 if (vmcs12->ept_pointer != new_eptp) { 6192 if (!nested_vmx_check_eptp(vcpu, new_eptp)) 6193 return 1; 6194 6195 vmcs12->ept_pointer = new_eptp; 6196 nested_ept_new_eptp(vcpu); 6197 6198 if (!nested_cpu_has_vpid(vmcs12)) 6199 kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); 6200 } 6201 6202 return 0; 6203} 6204 6205static int handle_vmfunc(struct kvm_vcpu *vcpu) 6206{ 6207 struct vcpu_vmx *vmx = to_vmx(vcpu); 6208 struct vmcs12 *vmcs12; 6209 u32 function = kvm_rax_read(vcpu); 6210 6211 /* 6212 * VMFUNC should never execute cleanly while L1 is active; KVM supports 6213 * VMFUNC for nested VMs, but not for L1. 6214 */ 6215 if (WARN_ON_ONCE(!is_guest_mode(vcpu))) { 6216 kvm_queue_exception(vcpu, UD_VECTOR); 6217 return 1; 6218 } 6219 6220 vmcs12 = get_vmcs12(vcpu); 6221 6222 /* 6223 * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC 6224 * is enabled in vmcs02 if and only if it's enabled in vmcs12. 6225 */ 6226 if (WARN_ON_ONCE((function > 63) || !nested_cpu_has_vmfunc(vmcs12))) { 6227 kvm_queue_exception(vcpu, UD_VECTOR); 6228 return 1; 6229 } 6230 6231 if (!(vmcs12->vm_function_control & BIT_ULL(function))) 6232 goto fail; 6233 6234 switch (function) { 6235 case 0: 6236 if (nested_vmx_eptp_switching(vcpu, vmcs12)) 6237 goto fail; 6238 break; 6239 default: 6240 goto fail; 6241 } 6242 return kvm_skip_emulated_instruction(vcpu); 6243 6244fail: 6245 /* 6246 * This is effectively a reflected VM-Exit, as opposed to a synthesized 6247 * nested VM-Exit. Pass the original exit reason, i.e. don't hardcode 6248 * EXIT_REASON_VMFUNC as the exit reason. 6249 */ 6250 nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full, 6251 vmx_get_intr_info(vcpu), 6252 vmx_get_exit_qual(vcpu)); 6253 return 1; 6254} 6255 6256/* 6257 * Return true if an IO instruction with the specified port and size should cause 6258 * a VM-exit into L1. 6259 */ 6260bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, 6261 int size) 6262{ 6263 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6264 gpa_t bitmap, last_bitmap; 6265 u8 b; 6266 6267 last_bitmap = INVALID_GPA; 6268 b = -1; 6269 6270 while (size > 0) { 6271 if (port < 0x8000) 6272 bitmap = vmcs12->io_bitmap_a; 6273 else if (port < 0x10000) 6274 bitmap = vmcs12->io_bitmap_b; 6275 else 6276 return true; 6277 bitmap += (port & 0x7fff) / 8; 6278 6279 if (last_bitmap != bitmap) 6280 if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1)) 6281 return true; 6282 if (b & (1 << (port & 7))) 6283 return true; 6284 6285 port++; 6286 size--; 6287 last_bitmap = bitmap; 6288 } 6289 6290 return false; 6291} 6292 6293static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, 6294 struct vmcs12 *vmcs12) 6295{ 6296 unsigned long exit_qualification; 6297 unsigned short port; 6298 int size; 6299 6300 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) 6301 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); 6302 6303 exit_qualification = vmx_get_exit_qual(vcpu); 6304 6305 port = exit_qualification >> 16; 6306 size = (exit_qualification & 7) + 1; 6307 6308 return nested_vmx_check_io_bitmaps(vcpu, port, size); 6309} 6310 6311/* 6312 * Return 1 if we should exit from L2 to L1 to handle an MSR access, 6313 * rather than handle it ourselves in L0. I.e., check whether L1 expressed 6314 * disinterest in the current event (read or write a specific MSR) by using an 6315 * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps. 6316 */ 6317static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, 6318 struct vmcs12 *vmcs12, 6319 union vmx_exit_reason exit_reason) 6320{ 6321 u32 msr_index; 6322 gpa_t bitmap; 6323 6324 if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS)) 6325 return true; 6326 6327 if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM || 6328 exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) 6329 msr_index = vmx_get_exit_qual(vcpu); 6330 else 6331 msr_index = kvm_rcx_read(vcpu); 6332 6333 /* 6334 * The MSR_BITMAP page is divided into four 1024-byte bitmaps, 6335 * for the four combinations of read/write and low/high MSR numbers. 6336 * First we need to figure out which of the four to use: 6337 */ 6338 bitmap = vmcs12->msr_bitmap; 6339 if (exit_reason.basic == EXIT_REASON_MSR_WRITE || 6340 exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) 6341 bitmap += 2048; 6342 if (msr_index >= 0xc0000000) { 6343 msr_index -= 0xc0000000; 6344 bitmap += 1024; 6345 } 6346 6347 /* Then read the msr_index'th bit from this bitmap: */ 6348 if (msr_index < 1024*8) { 6349 unsigned char b; 6350 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1)) 6351 return true; 6352 return 1 & (b >> (msr_index & 7)); 6353 } else 6354 return true; /* let L1 handle the wrong parameter */ 6355} 6356 6357/* 6358 * Return 1 if we should exit from L2 to L1 to handle a CR access exit, 6359 * rather than handle it ourselves in L0. I.e., check if L1 wanted to 6360 * intercept (via guest_host_mask etc.) the current event. 6361 */ 6362static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, 6363 struct vmcs12 *vmcs12) 6364{ 6365 unsigned long exit_qualification = vmx_get_exit_qual(vcpu); 6366 int cr = exit_qualification & 15; 6367 int reg; 6368 unsigned long val; 6369 6370 switch ((exit_qualification >> 4) & 3) { 6371 case 0: /* mov to cr */ 6372 reg = (exit_qualification >> 8) & 15; 6373 val = kvm_register_read(vcpu, reg); 6374 switch (cr) { 6375 case 0: 6376 if (vmcs12->cr0_guest_host_mask & 6377 (val ^ vmcs12->cr0_read_shadow)) 6378 return true; 6379 break; 6380 case 3: 6381 if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) 6382 return true; 6383 break; 6384 case 4: 6385 if (vmcs12->cr4_guest_host_mask & 6386 (vmcs12->cr4_read_shadow ^ val)) 6387 return true; 6388 break; 6389 case 8: 6390 if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING)) 6391 return true; 6392 break; 6393 } 6394 break; 6395 case 2: /* clts */ 6396 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) && 6397 (vmcs12->cr0_read_shadow & X86_CR0_TS)) 6398 return true; 6399 break; 6400 case 1: /* mov from cr */ 6401 switch (cr) { 6402 case 3: 6403 if (vmcs12->cpu_based_vm_exec_control & 6404 CPU_BASED_CR3_STORE_EXITING) 6405 return true; 6406 break; 6407 case 8: 6408 if (vmcs12->cpu_based_vm_exec_control & 6409 CPU_BASED_CR8_STORE_EXITING) 6410 return true; 6411 break; 6412 } 6413 break; 6414 case 3: /* lmsw */ 6415 /* 6416 * lmsw can change bits 1..3 of cr0, and only set bit 0 of 6417 * cr0. Other attempted changes are ignored, with no exit. 6418 */ 6419 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f; 6420 if (vmcs12->cr0_guest_host_mask & 0xe & 6421 (val ^ vmcs12->cr0_read_shadow)) 6422 return true; 6423 if ((vmcs12->cr0_guest_host_mask & 0x1) && 6424 !(vmcs12->cr0_read_shadow & 0x1) && 6425 (val & 0x1)) 6426 return true; 6427 break; 6428 } 6429 return false; 6430} 6431 6432static bool nested_vmx_exit_handled_encls(struct kvm_vcpu *vcpu, 6433 struct vmcs12 *vmcs12) 6434{ 6435 u32 encls_leaf; 6436 6437 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SGX) || 6438 !nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENCLS_EXITING)) 6439 return false; 6440 6441 encls_leaf = kvm_rax_read(vcpu); 6442 if (encls_leaf > 62) 6443 encls_leaf = 63; 6444 return vmcs12->encls_exiting_bitmap & BIT_ULL(encls_leaf); 6445} 6446 6447static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu, 6448 struct vmcs12 *vmcs12, gpa_t bitmap) 6449{ 6450 u32 vmx_instruction_info; 6451 unsigned long field; 6452 u8 b; 6453 6454 if (!nested_cpu_has_shadow_vmcs(vmcs12)) 6455 return true; 6456 6457 /* Decode instruction info and find the field to access */ 6458 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); 6459 field = kvm_register_read(vcpu, (((vmx_instruction_info) >> 28) & 0xf)); 6460 6461 /* Out-of-range fields always cause a VM exit from L2 to L1 */ 6462 if (field >> 15) 6463 return true; 6464 6465 if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1)) 6466 return true; 6467 6468 return 1 & (b >> (field & 7)); 6469} 6470 6471static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) 6472{ 6473 u32 entry_intr_info = vmcs12->vm_entry_intr_info_field; 6474 6475 if (nested_cpu_has_mtf(vmcs12)) 6476 return true; 6477 6478 /* 6479 * An MTF VM-exit may be injected into the guest by setting the 6480 * interruption-type to 7 (other event) and the vector field to 0. Such 6481 * is the case regardless of the 'monitor trap flag' VM-execution 6482 * control. 6483 */ 6484 return entry_intr_info == (INTR_INFO_VALID_MASK 6485 | INTR_TYPE_OTHER_EVENT); 6486} 6487 6488/* 6489 * Return true if L0 wants to handle an exit from L2 regardless of whether or not 6490 * L1 wants the exit. Only call this when in is_guest_mode (L2). 6491 */ 6492static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, 6493 union vmx_exit_reason exit_reason) 6494{ 6495 u32 intr_info; 6496 6497 switch ((u16)exit_reason.basic) { 6498 case EXIT_REASON_EXCEPTION_NMI: 6499 intr_info = vmx_get_intr_info(vcpu); 6500 if (is_nmi(intr_info)) 6501 return true; 6502 else if (is_page_fault(intr_info)) 6503 return vcpu->arch.apf.host_apf_flags || 6504 vmx_need_pf_intercept(vcpu); 6505 else if (is_debug(intr_info) && 6506 vcpu->guest_debug & 6507 (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) 6508 return true; 6509 else if (is_breakpoint(intr_info) && 6510 vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) 6511 return true; 6512 else if (is_alignment_check(intr_info) && 6513 !vmx_guest_inject_ac(vcpu)) 6514 return true; 6515 else if (is_ve_fault(intr_info)) 6516 return true; 6517 return false; 6518 case EXIT_REASON_EXTERNAL_INTERRUPT: 6519 return true; 6520 case EXIT_REASON_MCE_DURING_VMENTRY: 6521 return true; 6522 case EXIT_REASON_EPT_VIOLATION: 6523 /* 6524 * L0 always deals with the EPT violation. If nested EPT is 6525 * used, and the nested mmu code discovers that the address is 6526 * missing in the guest EPT table (EPT12), the EPT violation 6527 * will be injected with nested_ept_inject_page_fault() 6528 */ 6529 return true; 6530 case EXIT_REASON_EPT_MISCONFIG: 6531 /* 6532 * L2 never uses directly L1's EPT, but rather L0's own EPT 6533 * table (shadow on EPT) or a merged EPT table that L0 built 6534 * (EPT on EPT). So any problems with the structure of the 6535 * table is L0's fault. 6536 */ 6537 return true; 6538 case EXIT_REASON_PREEMPTION_TIMER: 6539 return true; 6540 case EXIT_REASON_PML_FULL: 6541 /* 6542 * PML is emulated for an L1 VMM and should never be enabled in 6543 * vmcs02, always "handle" PML_FULL by exiting to userspace. 6544 */ 6545 return true; 6546 case EXIT_REASON_VMFUNC: 6547 /* VM functions are emulated through L2->L0 vmexits. */ 6548 return true; 6549 case EXIT_REASON_BUS_LOCK: 6550 /* 6551 * At present, bus lock VM exit is never exposed to L1. 6552 * Handle L2's bus locks in L0 directly. 6553 */ 6554 return true; 6555#ifdef CONFIG_KVM_HYPERV 6556 case EXIT_REASON_VMCALL: 6557 /* Hyper-V L2 TLB flush hypercall is handled by L0 */ 6558 return guest_hv_cpuid_has_l2_tlb_flush(vcpu) && 6559 nested_evmcs_l2_tlb_flush_enabled(vcpu) && 6560 kvm_hv_is_tlb_flush_hcall(vcpu); 6561#endif 6562 default: 6563 break; 6564 } 6565 return false; 6566} 6567 6568/* 6569 * Return 1 if L1 wants to intercept an exit from L2. Only call this when in 6570 * is_guest_mode (L2). 6571 */ 6572static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, 6573 union vmx_exit_reason exit_reason) 6574{ 6575 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6576 u32 intr_info; 6577 6578 switch ((u16)exit_reason.basic) { 6579 case EXIT_REASON_EXCEPTION_NMI: 6580 intr_info = vmx_get_intr_info(vcpu); 6581 if (is_nmi(intr_info)) 6582 return true; 6583 else if (is_page_fault(intr_info)) 6584 return true; 6585 return vmcs12->exception_bitmap & 6586 (1u << (intr_info & INTR_INFO_VECTOR_MASK)); 6587 case EXIT_REASON_EXTERNAL_INTERRUPT: 6588 return nested_exit_on_intr(vcpu); 6589 case EXIT_REASON_TRIPLE_FAULT: 6590 return true; 6591 case EXIT_REASON_INTERRUPT_WINDOW: 6592 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING); 6593 case EXIT_REASON_NMI_WINDOW: 6594 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING); 6595 case EXIT_REASON_TASK_SWITCH: 6596 return true; 6597 case EXIT_REASON_CPUID: 6598 return true; 6599 case EXIT_REASON_HLT: 6600 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING); 6601 case EXIT_REASON_INVD: 6602 return true; 6603 case EXIT_REASON_INVLPG: 6604 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6605 case EXIT_REASON_RDPMC: 6606 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING); 6607 case EXIT_REASON_RDRAND: 6608 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING); 6609 case EXIT_REASON_RDSEED: 6610 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING); 6611 case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP: 6612 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING); 6613 case EXIT_REASON_VMREAD: 6614 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6615 vmcs12->vmread_bitmap); 6616 case EXIT_REASON_VMWRITE: 6617 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12, 6618 vmcs12->vmwrite_bitmap); 6619 case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR: 6620 case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD: 6621 case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME: 6622 case EXIT_REASON_VMOFF: case EXIT_REASON_VMON: 6623 case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID: 6624 /* 6625 * VMX instructions trap unconditionally. This allows L1 to 6626 * emulate them for its L2 guest, i.e., allows 3-level nesting! 6627 */ 6628 return true; 6629 case EXIT_REASON_CR_ACCESS: 6630 return nested_vmx_exit_handled_cr(vcpu, vmcs12); 6631 case EXIT_REASON_DR_ACCESS: 6632 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING); 6633 case EXIT_REASON_IO_INSTRUCTION: 6634 return nested_vmx_exit_handled_io(vcpu, vmcs12); 6635 case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR: 6636 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC); 6637 case EXIT_REASON_MSR_READ: 6638 case EXIT_REASON_MSR_WRITE: 6639 case EXIT_REASON_MSR_READ_IMM: 6640 case EXIT_REASON_MSR_WRITE_IMM: 6641 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason); 6642 case EXIT_REASON_INVALID_STATE: 6643 return true; 6644 case EXIT_REASON_MWAIT_INSTRUCTION: 6645 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING); 6646 case EXIT_REASON_MONITOR_TRAP_FLAG: 6647 return nested_vmx_exit_handled_mtf(vmcs12); 6648 case EXIT_REASON_MONITOR_INSTRUCTION: 6649 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING); 6650 case EXIT_REASON_PAUSE_INSTRUCTION: 6651 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) || 6652 nested_cpu_has2(vmcs12, 6653 SECONDARY_EXEC_PAUSE_LOOP_EXITING); 6654 case EXIT_REASON_MCE_DURING_VMENTRY: 6655 return true; 6656 case EXIT_REASON_TPR_BELOW_THRESHOLD: 6657 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); 6658 case EXIT_REASON_APIC_ACCESS: 6659 case EXIT_REASON_APIC_WRITE: 6660 case EXIT_REASON_EOI_INDUCED: 6661 /* 6662 * The controls for "virtualize APIC accesses," "APIC- 6663 * register virtualization," and "virtual-interrupt 6664 * delivery" only come from vmcs12. 6665 */ 6666 return true; 6667 case EXIT_REASON_INVPCID: 6668 return 6669 nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && 6670 nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING); 6671 case EXIT_REASON_WBINVD: 6672 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING); 6673 case EXIT_REASON_XSETBV: 6674 return true; 6675 case EXIT_REASON_XSAVES: 6676 case EXIT_REASON_XRSTORS: 6677 /* 6678 * Always forward XSAVES/XRSTORS to L1 as KVM doesn't utilize 6679 * XSS-bitmap, and always loads vmcs02 with vmcs12's XSS-bitmap 6680 * verbatim, i.e. any exit is due to L1's bitmap. WARN if 6681 * XSAVES isn't enabled, as the CPU is supposed to inject #UD 6682 * in that case, before consulting the XSS-bitmap. 6683 */ 6684 WARN_ON_ONCE(!nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_XSAVES)); 6685 return true; 6686 case EXIT_REASON_UMWAIT: 6687 case EXIT_REASON_TPAUSE: 6688 return nested_cpu_has2(vmcs12, 6689 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE); 6690 case EXIT_REASON_ENCLS: 6691 return nested_vmx_exit_handled_encls(vcpu, vmcs12); 6692 case EXIT_REASON_NOTIFY: 6693 /* Notify VM exit is not exposed to L1 */ 6694 return false; 6695 case EXIT_REASON_SEAMCALL: 6696 case EXIT_REASON_TDCALL: 6697 /* 6698 * SEAMCALL and TDCALL unconditionally VM-Exit, but aren't 6699 * virtualized by KVM for L1 hypervisors, i.e. L1 should 6700 * never want or expect such an exit. 6701 */ 6702 return false; 6703 default: 6704 return true; 6705 } 6706} 6707 6708/* 6709 * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was 6710 * reflected into L1. 6711 */ 6712bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) 6713{ 6714 struct vcpu_vmx *vmx = to_vmx(vcpu); 6715 union vmx_exit_reason exit_reason = vmx->vt.exit_reason; 6716 unsigned long exit_qual; 6717 u32 exit_intr_info; 6718 6719 WARN_ON_ONCE(vmx->nested.nested_run_pending); 6720 6721 /* 6722 * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM 6723 * has already loaded L2's state. 6724 */ 6725 if (unlikely(vmx->fail)) { 6726 trace_kvm_nested_vmenter_failed( 6727 "hardware VM-instruction error: ", 6728 vmcs_read32(VM_INSTRUCTION_ERROR)); 6729 exit_intr_info = 0; 6730 exit_qual = 0; 6731 goto reflect_vmexit; 6732 } 6733 6734 trace_kvm_nested_vmexit(vcpu, KVM_ISA_VMX); 6735 6736 /* If L0 (KVM) wants the exit, it trumps L1's desires. */ 6737 if (nested_vmx_l0_wants_exit(vcpu, exit_reason)) 6738 return false; 6739 6740 /* If L1 doesn't want the exit, handle it in L0. */ 6741 if (!nested_vmx_l1_wants_exit(vcpu, exit_reason)) 6742 return false; 6743 6744 /* 6745 * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For 6746 * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would 6747 * need to be synthesized by querying the in-kernel LAPIC, but external 6748 * interrupts are never reflected to L1 so it's a non-issue. 6749 */ 6750 exit_intr_info = vmx_get_intr_info(vcpu); 6751 if (is_exception_with_error_code(exit_intr_info)) { 6752 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6753 6754 vmcs12->vm_exit_intr_error_code = 6755 vmcs_read32(VM_EXIT_INTR_ERROR_CODE); 6756 } 6757 exit_qual = vmx_get_exit_qual(vcpu); 6758 6759reflect_vmexit: 6760 nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); 6761 return true; 6762} 6763 6764static int vmx_get_nested_state(struct kvm_vcpu *vcpu, 6765 struct kvm_nested_state __user *user_kvm_nested_state, 6766 u32 user_data_size) 6767{ 6768 struct vcpu_vmx *vmx; 6769 struct vmcs12 *vmcs12; 6770 struct kvm_nested_state kvm_state = { 6771 .flags = 0, 6772 .format = KVM_STATE_NESTED_FORMAT_VMX, 6773 .size = sizeof(kvm_state), 6774 .hdr.vmx.flags = 0, 6775 .hdr.vmx.vmxon_pa = INVALID_GPA, 6776 .hdr.vmx.vmcs12_pa = INVALID_GPA, 6777 .hdr.vmx.preemption_timer_deadline = 0, 6778 }; 6779 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6780 &user_kvm_nested_state->data.vmx[0]; 6781 6782 if (!vcpu) 6783 return kvm_state.size + sizeof(*user_vmx_nested_state); 6784 6785 vmx = to_vmx(vcpu); 6786 vmcs12 = get_vmcs12(vcpu); 6787 6788 if (guest_cpu_cap_has(vcpu, X86_FEATURE_VMX) && 6789 (vmx->nested.vmxon || vmx->nested.smm.vmxon)) { 6790 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr; 6791 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr; 6792 6793 if (vmx_has_valid_vmcs12(vcpu)) { 6794 kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); 6795 6796 /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ 6797 if (nested_vmx_is_evmptr12_set(vmx)) 6798 kvm_state.flags |= KVM_STATE_NESTED_EVMCS; 6799 6800 if (is_guest_mode(vcpu) && 6801 nested_cpu_has_shadow_vmcs(vmcs12) && 6802 vmcs12->vmcs_link_pointer != INVALID_GPA) 6803 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12); 6804 } 6805 6806 if (vmx->nested.smm.vmxon) 6807 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON; 6808 6809 if (vmx->nested.smm.guest_mode) 6810 kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE; 6811 6812 if (is_guest_mode(vcpu)) { 6813 kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; 6814 6815 if (vmx->nested.nested_run_pending) 6816 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; 6817 6818 if (vmx->nested.mtf_pending) 6819 kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING; 6820 6821 if (nested_cpu_has_preemption_timer(vmcs12) && 6822 vmx->nested.has_preemption_timer_deadline) { 6823 kvm_state.hdr.vmx.flags |= 6824 KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE; 6825 kvm_state.hdr.vmx.preemption_timer_deadline = 6826 vmx->nested.preemption_timer_deadline; 6827 } 6828 } 6829 } 6830 6831 if (user_data_size < kvm_state.size) 6832 goto out; 6833 6834 if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) 6835 return -EFAULT; 6836 6837 if (!vmx_has_valid_vmcs12(vcpu)) 6838 goto out; 6839 6840 /* 6841 * When running L2, the authoritative vmcs12 state is in the 6842 * vmcs02. When running L1, the authoritative vmcs12 state is 6843 * in the shadow or enlightened vmcs linked to vmcs01, unless 6844 * need_vmcs12_to_shadow_sync is set, in which case, the authoritative 6845 * vmcs12 state is in the vmcs12 already. 6846 */ 6847 if (is_guest_mode(vcpu)) { 6848 sync_vmcs02_to_vmcs12(vcpu, vmcs12); 6849 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); 6850 } else { 6851 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); 6852 if (!vmx->nested.need_vmcs12_to_shadow_sync) { 6853 if (nested_vmx_is_evmptr12_valid(vmx)) 6854 /* 6855 * L1 hypervisor is not obliged to keep eVMCS 6856 * clean fields data always up-to-date while 6857 * not in guest mode, 'hv_clean_fields' is only 6858 * supposed to be actual upon vmentry so we need 6859 * to ignore it here and do full copy. 6860 */ 6861 copy_enlightened_to_vmcs12(vmx, 0); 6862 else if (enable_shadow_vmcs) 6863 copy_shadow_to_vmcs12(vmx); 6864 } 6865 } 6866 6867 BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE); 6868 BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE); 6869 6870 /* 6871 * Copy over the full allocated size of vmcs12 rather than just the size 6872 * of the struct. 6873 */ 6874 if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE)) 6875 return -EFAULT; 6876 6877 if (nested_cpu_has_shadow_vmcs(vmcs12) && 6878 vmcs12->vmcs_link_pointer != INVALID_GPA) { 6879 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12, 6880 get_shadow_vmcs12(vcpu), VMCS12_SIZE)) 6881 return -EFAULT; 6882 } 6883out: 6884 return kvm_state.size; 6885} 6886 6887void vmx_leave_nested(struct kvm_vcpu *vcpu) 6888{ 6889 if (is_guest_mode(vcpu)) { 6890 to_vmx(vcpu)->nested.nested_run_pending = 0; 6891 nested_vmx_vmexit(vcpu, -1, 0, 0); 6892 } 6893 free_nested(vcpu); 6894} 6895 6896static int vmx_set_nested_state(struct kvm_vcpu *vcpu, 6897 struct kvm_nested_state __user *user_kvm_nested_state, 6898 struct kvm_nested_state *kvm_state) 6899{ 6900 struct vcpu_vmx *vmx = to_vmx(vcpu); 6901 struct vmcs12 *vmcs12; 6902 enum vm_entry_failure_code ignored; 6903 struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = 6904 &user_kvm_nested_state->data.vmx[0]; 6905 int ret; 6906 6907 if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX) 6908 return -EINVAL; 6909 6910 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) { 6911 if (kvm_state->hdr.vmx.smm.flags) 6912 return -EINVAL; 6913 6914 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) 6915 return -EINVAL; 6916 6917 /* 6918 * KVM_STATE_NESTED_EVMCS used to signal that KVM should 6919 * enable eVMCS capability on vCPU. However, since then 6920 * code was changed such that flag signals vmcs12 should 6921 * be copied into eVMCS in guest memory. 6922 * 6923 * To preserve backwards compatibility, allow user 6924 * to set this flag even when there is no VMXON region. 6925 */ 6926 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS) 6927 return -EINVAL; 6928 } else { 6929 if (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX)) 6930 return -EINVAL; 6931 6932 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa)) 6933 return -EINVAL; 6934 } 6935 6936 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6937 (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 6938 return -EINVAL; 6939 6940 if (kvm_state->hdr.vmx.smm.flags & 6941 ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON)) 6942 return -EINVAL; 6943 6944 if (kvm_state->hdr.vmx.flags & ~KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) 6945 return -EINVAL; 6946 6947 /* 6948 * SMM temporarily disables VMX, so we cannot be in guest mode, 6949 * nor can VMLAUNCH/VMRESUME be pending. Outside SMM, SMM flags 6950 * must be zero. 6951 */ 6952 if (is_smm(vcpu) ? 6953 (kvm_state->flags & 6954 (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING)) 6955 : kvm_state->hdr.vmx.smm.flags) 6956 return -EINVAL; 6957 6958 if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) && 6959 !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON)) 6960 return -EINVAL; 6961 6962 if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) && 6963 (!guest_cpu_cap_has(vcpu, X86_FEATURE_VMX) || 6964 !vmx->nested.enlightened_vmcs_enabled)) 6965 return -EINVAL; 6966 6967 vmx_leave_nested(vcpu); 6968 6969 if (kvm_state->hdr.vmx.vmxon_pa == INVALID_GPA) 6970 return 0; 6971 6972 vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa; 6973 ret = enter_vmx_operation(vcpu); 6974 if (ret) 6975 return ret; 6976 6977 /* Empty 'VMXON' state is permitted if no VMCS loaded */ 6978 if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12)) { 6979 /* See vmx_has_valid_vmcs12. */ 6980 if ((kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE) || 6981 (kvm_state->flags & KVM_STATE_NESTED_EVMCS) || 6982 (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA)) 6983 return -EINVAL; 6984 else 6985 return 0; 6986 } 6987 6988 if (kvm_state->hdr.vmx.vmcs12_pa != INVALID_GPA) { 6989 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa || 6990 !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa)) 6991 return -EINVAL; 6992 6993 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa); 6994#ifdef CONFIG_KVM_HYPERV 6995 } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) { 6996 /* 6997 * nested_vmx_handle_enlightened_vmptrld() cannot be called 6998 * directly from here as HV_X64_MSR_VP_ASSIST_PAGE may not be 6999 * restored yet. EVMCS will be mapped from 7000 * nested_get_vmcs12_pages(). 7001 */ 7002 vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; 7003 kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); 7004#endif 7005 } else { 7006 return -EINVAL; 7007 } 7008 7009 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) { 7010 vmx->nested.smm.vmxon = true; 7011 vmx->nested.vmxon = false; 7012 7013 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) 7014 vmx->nested.smm.guest_mode = true; 7015 } 7016 7017 vmcs12 = get_vmcs12(vcpu); 7018 if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12))) 7019 return -EFAULT; 7020 7021 if (vmcs12->hdr.revision_id != VMCS12_REVISION) 7022 return -EINVAL; 7023 7024 if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) 7025 return 0; 7026 7027 vmx->nested.nested_run_pending = 7028 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING); 7029 7030 vmx->nested.mtf_pending = 7031 !!(kvm_state->flags & KVM_STATE_NESTED_MTF_PENDING); 7032 7033 ret = -EINVAL; 7034 if (nested_cpu_has_shadow_vmcs(vmcs12) && 7035 vmcs12->vmcs_link_pointer != INVALID_GPA) { 7036 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu); 7037 7038 if (kvm_state->size < 7039 sizeof(*kvm_state) + 7040 sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12)) 7041 goto error_guest_mode; 7042 7043 if (copy_from_user(shadow_vmcs12, 7044 user_vmx_nested_state->shadow_vmcs12, 7045 sizeof(*shadow_vmcs12))) { 7046 ret = -EFAULT; 7047 goto error_guest_mode; 7048 } 7049 7050 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION || 7051 !shadow_vmcs12->hdr.shadow_vmcs) 7052 goto error_guest_mode; 7053 } 7054 7055 vmx->nested.has_preemption_timer_deadline = false; 7056 if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) { 7057 vmx->nested.has_preemption_timer_deadline = true; 7058 vmx->nested.preemption_timer_deadline = 7059 kvm_state->hdr.vmx.preemption_timer_deadline; 7060 } 7061 7062 if (nested_vmx_check_controls(vcpu, vmcs12) || 7063 nested_vmx_check_host_state(vcpu, vmcs12) || 7064 nested_vmx_check_guest_state(vcpu, vmcs12, &ignored)) 7065 goto error_guest_mode; 7066 7067 vmx->nested.dirty_vmcs12 = true; 7068 vmx->nested.force_msr_bitmap_recalc = true; 7069 ret = nested_vmx_enter_non_root_mode(vcpu, false); 7070 if (ret) 7071 goto error_guest_mode; 7072 7073 if (vmx->nested.mtf_pending) 7074 kvm_make_request(KVM_REQ_EVENT, vcpu); 7075 7076 return 0; 7077 7078error_guest_mode: 7079 vmx->nested.nested_run_pending = 0; 7080 return ret; 7081} 7082 7083void nested_vmx_set_vmcs_shadowing_bitmap(void) 7084{ 7085 if (enable_shadow_vmcs) { 7086 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap)); 7087 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap)); 7088 } 7089} 7090 7091/* 7092 * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6. Undo 7093 * that madness to get the encoding for comparison. 7094 */ 7095#define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10))) 7096 7097static u64 nested_vmx_calc_vmcs_enum_msr(void) 7098{ 7099 /* 7100 * Note these are the so called "index" of the VMCS field encoding, not 7101 * the index into vmcs12. 7102 */ 7103 unsigned int max_idx, idx; 7104 int i; 7105 7106 /* 7107 * For better or worse, KVM allows VMREAD/VMWRITE to all fields in 7108 * vmcs12, regardless of whether or not the associated feature is 7109 * exposed to L1. Simply find the field with the highest index. 7110 */ 7111 max_idx = 0; 7112 for (i = 0; i < nr_vmcs12_fields; i++) { 7113 /* The vmcs12 table is very, very sparsely populated. */ 7114 if (!vmcs12_field_offsets[i]) 7115 continue; 7116 7117 idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i)); 7118 if (idx > max_idx) 7119 max_idx = idx; 7120 } 7121 7122 return (u64)max_idx << VMCS_FIELD_INDEX_SHIFT; 7123} 7124 7125static void nested_vmx_setup_pinbased_ctls(struct vmcs_config *vmcs_conf, 7126 struct nested_vmx_msrs *msrs) 7127{ 7128 msrs->pinbased_ctls_low = 7129 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 7130 7131 msrs->pinbased_ctls_high = vmcs_conf->pin_based_exec_ctrl; 7132 msrs->pinbased_ctls_high &= 7133 PIN_BASED_EXT_INTR_MASK | 7134 PIN_BASED_NMI_EXITING | 7135 PIN_BASED_VIRTUAL_NMIS | 7136 (enable_apicv ? PIN_BASED_POSTED_INTR : 0); 7137 msrs->pinbased_ctls_high |= 7138 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 7139 PIN_BASED_VMX_PREEMPTION_TIMER; 7140} 7141 7142static void nested_vmx_setup_exit_ctls(struct vmcs_config *vmcs_conf, 7143 struct nested_vmx_msrs *msrs) 7144{ 7145 msrs->exit_ctls_low = 7146 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR; 7147 7148 msrs->exit_ctls_high = vmcs_conf->vmexit_ctrl; 7149 msrs->exit_ctls_high &= 7150#ifdef CONFIG_X86_64 7151 VM_EXIT_HOST_ADDR_SPACE_SIZE | 7152#endif 7153 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT | 7154 VM_EXIT_CLEAR_BNDCFGS | VM_EXIT_LOAD_CET_STATE; 7155 msrs->exit_ctls_high |= 7156 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR | 7157 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER | 7158 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT | 7159 VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL; 7160 7161 if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && 7162 !kvm_cpu_cap_has(X86_FEATURE_IBT)) 7163 msrs->exit_ctls_high &= ~VM_EXIT_LOAD_CET_STATE; 7164 7165 /* We support free control of debug control saving. */ 7166 msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS; 7167} 7168 7169static void nested_vmx_setup_entry_ctls(struct vmcs_config *vmcs_conf, 7170 struct nested_vmx_msrs *msrs) 7171{ 7172 msrs->entry_ctls_low = 7173 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR; 7174 7175 msrs->entry_ctls_high = vmcs_conf->vmentry_ctrl; 7176 msrs->entry_ctls_high &= 7177#ifdef CONFIG_X86_64 7178 VM_ENTRY_IA32E_MODE | 7179#endif 7180 VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS | 7181 VM_ENTRY_LOAD_CET_STATE; 7182 msrs->entry_ctls_high |= 7183 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER | 7184 VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL); 7185 7186 if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && 7187 !kvm_cpu_cap_has(X86_FEATURE_IBT)) 7188 msrs->entry_ctls_high &= ~VM_ENTRY_LOAD_CET_STATE; 7189 7190 /* We support free control of debug control loading. */ 7191 msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS; 7192} 7193 7194static void nested_vmx_setup_cpubased_ctls(struct vmcs_config *vmcs_conf, 7195 struct nested_vmx_msrs *msrs) 7196{ 7197 msrs->procbased_ctls_low = 7198 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR; 7199 7200 msrs->procbased_ctls_high = vmcs_conf->cpu_based_exec_ctrl; 7201 msrs->procbased_ctls_high &= 7202 CPU_BASED_INTR_WINDOW_EXITING | 7203 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING | 7204 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING | 7205 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING | 7206 CPU_BASED_CR3_STORE_EXITING | 7207#ifdef CONFIG_X86_64 7208 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING | 7209#endif 7210 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 7211 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG | 7212 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING | 7213 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING | 7214 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 7215 /* 7216 * We can allow some features even when not supported by the 7217 * hardware. For example, L1 can specify an MSR bitmap - and we 7218 * can use it to avoid exits to L1 - even when L0 runs L2 7219 * without MSR bitmaps. 7220 */ 7221 msrs->procbased_ctls_high |= 7222 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR | 7223 CPU_BASED_USE_MSR_BITMAPS; 7224 7225 /* We support free control of CR3 access interception. */ 7226 msrs->procbased_ctls_low &= 7227 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING); 7228} 7229 7230static void nested_vmx_setup_secondary_ctls(u32 ept_caps, 7231 struct vmcs_config *vmcs_conf, 7232 struct nested_vmx_msrs *msrs) 7233{ 7234 msrs->secondary_ctls_low = 0; 7235 7236 msrs->secondary_ctls_high = vmcs_conf->cpu_based_2nd_exec_ctrl; 7237 msrs->secondary_ctls_high &= 7238 SECONDARY_EXEC_DESC | 7239 SECONDARY_EXEC_ENABLE_RDTSCP | 7240 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | 7241 SECONDARY_EXEC_WBINVD_EXITING | 7242 SECONDARY_EXEC_APIC_REGISTER_VIRT | 7243 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | 7244 SECONDARY_EXEC_RDRAND_EXITING | 7245 SECONDARY_EXEC_ENABLE_INVPCID | 7246 SECONDARY_EXEC_ENABLE_VMFUNC | 7247 SECONDARY_EXEC_RDSEED_EXITING | 7248 SECONDARY_EXEC_ENABLE_XSAVES | 7249 SECONDARY_EXEC_TSC_SCALING | 7250 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE; 7251 7252 /* 7253 * We can emulate "VMCS shadowing," even if the hardware 7254 * doesn't support it. 7255 */ 7256 msrs->secondary_ctls_high |= 7257 SECONDARY_EXEC_SHADOW_VMCS; 7258 7259 if (enable_ept) { 7260 /* nested EPT: emulate EPT also to L1 */ 7261 msrs->secondary_ctls_high |= 7262 SECONDARY_EXEC_ENABLE_EPT; 7263 msrs->ept_caps = 7264 VMX_EPT_PAGE_WALK_4_BIT | 7265 VMX_EPT_PAGE_WALK_5_BIT | 7266 VMX_EPTP_WB_BIT | 7267 VMX_EPT_INVEPT_BIT | 7268 VMX_EPT_EXECUTE_ONLY_BIT; 7269 7270 msrs->ept_caps &= ept_caps; 7271 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT | 7272 VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT | 7273 VMX_EPT_1GB_PAGE_BIT; 7274 if (enable_ept_ad_bits) { 7275 msrs->secondary_ctls_high |= 7276 SECONDARY_EXEC_ENABLE_PML; 7277 msrs->ept_caps |= VMX_EPT_AD_BIT; 7278 } 7279 7280 /* 7281 * Advertise EPTP switching irrespective of hardware support, 7282 * KVM emulates it in software so long as VMFUNC is supported. 7283 */ 7284 if (cpu_has_vmx_vmfunc()) 7285 msrs->vmfunc_controls = VMX_VMFUNC_EPTP_SWITCHING; 7286 } 7287 7288 /* 7289 * Old versions of KVM use the single-context version without 7290 * checking for support, so declare that it is supported even 7291 * though it is treated as global context. The alternative is 7292 * not failing the single-context invvpid, and it is worse. 7293 */ 7294 if (enable_vpid) { 7295 msrs->secondary_ctls_high |= 7296 SECONDARY_EXEC_ENABLE_VPID; 7297 msrs->vpid_caps = VMX_VPID_INVVPID_BIT | 7298 VMX_VPID_EXTENT_SUPPORTED_MASK; 7299 } 7300 7301 if (enable_unrestricted_guest) 7302 msrs->secondary_ctls_high |= 7303 SECONDARY_EXEC_UNRESTRICTED_GUEST; 7304 7305 if (flexpriority_enabled) 7306 msrs->secondary_ctls_high |= 7307 SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; 7308 7309 if (enable_sgx) 7310 msrs->secondary_ctls_high |= SECONDARY_EXEC_ENCLS_EXITING; 7311} 7312 7313static void nested_vmx_setup_misc_data(struct vmcs_config *vmcs_conf, 7314 struct nested_vmx_msrs *msrs) 7315{ 7316 msrs->misc_low = (u32)vmcs_conf->misc & VMX_MISC_SAVE_EFER_LMA; 7317 msrs->misc_low |= 7318 VMX_MISC_VMWRITE_SHADOW_RO_FIELDS | 7319 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE | 7320 VMX_MISC_ACTIVITY_HLT | 7321 VMX_MISC_ACTIVITY_WAIT_SIPI; 7322 msrs->misc_high = 0; 7323} 7324 7325static void nested_vmx_setup_basic(struct nested_vmx_msrs *msrs) 7326{ 7327 /* 7328 * This MSR reports some information about VMX support. We 7329 * should return information about the VMX we emulate for the 7330 * guest, and the VMCS structure we give it - not about the 7331 * VMX support of the underlying hardware. 7332 */ 7333 msrs->basic = vmx_basic_encode_vmcs_info(VMCS12_REVISION, VMCS12_SIZE, 7334 X86_MEMTYPE_WB); 7335 7336 msrs->basic |= VMX_BASIC_TRUE_CTLS; 7337 if (cpu_has_vmx_basic_inout()) 7338 msrs->basic |= VMX_BASIC_INOUT; 7339 if (cpu_has_vmx_basic_no_hw_errcode_cc()) 7340 msrs->basic |= VMX_BASIC_NO_HW_ERROR_CODE_CC; 7341} 7342 7343static void nested_vmx_setup_cr_fixed(struct nested_vmx_msrs *msrs) 7344{ 7345 /* 7346 * These MSRs specify bits which the guest must keep fixed on 7347 * while L1 is in VMXON mode (in L1's root mode, or running an L2). 7348 * We picked the standard core2 setting. 7349 */ 7350#define VMXON_CR0_ALWAYSON (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE) 7351#define VMXON_CR4_ALWAYSON X86_CR4_VMXE 7352 msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON; 7353 msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON; 7354 7355 /* These MSRs specify bits which the guest must keep fixed off. */ 7356 rdmsrq(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); 7357 rdmsrq(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); 7358 7359 if (vmx_umip_emulated()) 7360 msrs->cr4_fixed1 |= X86_CR4_UMIP; 7361} 7362 7363/* 7364 * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be 7365 * returned for the various VMX controls MSRs when nested VMX is enabled. 7366 * The same values should also be used to verify that vmcs12 control fields are 7367 * valid during nested entry from L1 to L2. 7368 * Each of these control msrs has a low and high 32-bit half: A low bit is on 7369 * if the corresponding bit in the (32-bit) control field *must* be on, and a 7370 * bit in the high half is on if the corresponding bit in the control field 7371 * may be on. See also vmx_control_verify(). 7372 */ 7373void nested_vmx_setup_ctls_msrs(struct vmcs_config *vmcs_conf, u32 ept_caps) 7374{ 7375 struct nested_vmx_msrs *msrs = &vmcs_conf->nested; 7376 7377 /* 7378 * Note that as a general rule, the high half of the MSRs (bits in 7379 * the control fields which may be 1) should be initialized by the 7380 * intersection of the underlying hardware's MSR (i.e., features which 7381 * can be supported) and the list of features we want to expose - 7382 * because they are known to be properly supported in our code. 7383 * Also, usually, the low half of the MSRs (bits which must be 1) can 7384 * be set to 0, meaning that L1 may turn off any of these bits. The 7385 * reason is that if one of these bits is necessary, it will appear 7386 * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control 7387 * fields of vmcs01 and vmcs02, will turn these bits off - and 7388 * nested_vmx_l1_wants_exit() will not pass related exits to L1. 7389 * These rules have exceptions below. 7390 */ 7391 nested_vmx_setup_pinbased_ctls(vmcs_conf, msrs); 7392 7393 nested_vmx_setup_exit_ctls(vmcs_conf, msrs); 7394 7395 nested_vmx_setup_entry_ctls(vmcs_conf, msrs); 7396 7397 nested_vmx_setup_cpubased_ctls(vmcs_conf, msrs); 7398 7399 nested_vmx_setup_secondary_ctls(ept_caps, vmcs_conf, msrs); 7400 7401 nested_vmx_setup_misc_data(vmcs_conf, msrs); 7402 7403 nested_vmx_setup_basic(msrs); 7404 7405 nested_vmx_setup_cr_fixed(msrs); 7406 7407 msrs->vmcs_enum = nested_vmx_calc_vmcs_enum_msr(); 7408} 7409 7410void nested_vmx_hardware_unsetup(void) 7411{ 7412 int i; 7413 7414 if (enable_shadow_vmcs) { 7415 for (i = 0; i < VMX_BITMAP_NR; i++) 7416 free_page((unsigned long)vmx_bitmap[i]); 7417 } 7418} 7419 7420__init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) 7421{ 7422 int i; 7423 7424 if (!cpu_has_vmx_shadow_vmcs()) 7425 enable_shadow_vmcs = 0; 7426 if (enable_shadow_vmcs) { 7427 for (i = 0; i < VMX_BITMAP_NR; i++) { 7428 /* 7429 * The vmx_bitmap is not tied to a VM and so should 7430 * not be charged to a memcg. 7431 */ 7432 vmx_bitmap[i] = (unsigned long *) 7433 __get_free_page(GFP_KERNEL); 7434 if (!vmx_bitmap[i]) { 7435 nested_vmx_hardware_unsetup(); 7436 return -ENOMEM; 7437 } 7438 } 7439 7440 init_vmcs_shadow_fields(); 7441 } 7442 7443 exit_handlers[EXIT_REASON_VMCLEAR] = handle_vmclear; 7444 exit_handlers[EXIT_REASON_VMLAUNCH] = handle_vmlaunch; 7445 exit_handlers[EXIT_REASON_VMPTRLD] = handle_vmptrld; 7446 exit_handlers[EXIT_REASON_VMPTRST] = handle_vmptrst; 7447 exit_handlers[EXIT_REASON_VMREAD] = handle_vmread; 7448 exit_handlers[EXIT_REASON_VMRESUME] = handle_vmresume; 7449 exit_handlers[EXIT_REASON_VMWRITE] = handle_vmwrite; 7450 exit_handlers[EXIT_REASON_VMOFF] = handle_vmxoff; 7451 exit_handlers[EXIT_REASON_VMON] = handle_vmxon; 7452 exit_handlers[EXIT_REASON_INVEPT] = handle_invept; 7453 exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; 7454 exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; 7455 7456 return 0; 7457} 7458 7459struct kvm_x86_nested_ops vmx_nested_ops = { 7460 .leave_nested = vmx_leave_nested, 7461 .is_exception_vmexit = nested_vmx_is_exception_vmexit, 7462 .check_events = vmx_check_nested_events, 7463 .has_events = vmx_has_nested_events, 7464 .triple_fault = nested_vmx_triple_fault, 7465 .get_state = vmx_get_nested_state, 7466 .set_state = vmx_set_nested_state, 7467 .get_nested_state_pages = vmx_get_nested_state_pages, 7468 .write_log_dirty = nested_vmx_write_pml_buffer, 7469#ifdef CONFIG_KVM_HYPERV 7470 .enable_evmcs = nested_enable_evmcs, 7471 .get_evmcs_version = nested_get_evmcs_version, 7472 .hv_inject_synthetic_vmexit_post_tlb_flush = vmx_hv_inject_synthetic_vmexit_post_tlb_flush, 7473#endif 7474};
~/
Read-only file preview.