main.c

   1// SPDX-License-Identifier: GPL-2.0
   2#include <linux/moduleparam.h>
   3
   4#include "x86_ops.h"
   5#include "vmx.h"
   6#include "mmu.h"
   7#include "nested.h"
   8#include "pmu.h"
   9#include "posted_intr.h"
  10#include "tdx.h"
  11#include "tdx_arch.h"
  12
  13#ifdef CONFIG_KVM_INTEL_TDX
  14static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt));
  15
  16static void vt_disable_virtualization_cpu(void)
  17{
  18	/* Note, TDX *and* VMX need to be disabled if TDX is enabled. */
  19	if (enable_tdx)
  20		tdx_disable_virtualization_cpu();
  21	vmx_disable_virtualization_cpu();
  22}
  23
  24static __init int vt_hardware_setup(void)
  25{
  26	int ret;
  27
  28	ret = vmx_hardware_setup();
  29	if (ret)
  30		return ret;
  31
  32	if (enable_tdx)
  33		tdx_hardware_setup();
  34
  35	return 0;
  36}
  37
  38static int vt_vm_init(struct kvm *kvm)
  39{
  40	if (is_td(kvm))
  41		return tdx_vm_init(kvm);
  42
  43	return vmx_vm_init(kvm);
  44}
  45
  46static void vt_vm_pre_destroy(struct kvm *kvm)
  47{
  48	if (is_td(kvm))
  49		return tdx_mmu_release_hkid(kvm);
  50}
  51
  52static void vt_vm_destroy(struct kvm *kvm)
  53{
  54	if (is_td(kvm))
  55               return tdx_vm_destroy(kvm);
  56
  57       vmx_vm_destroy(kvm);
  58}
  59
  60static int vt_vcpu_precreate(struct kvm *kvm)
  61{
  62	if (is_td(kvm))
  63		return 0;
  64
  65	return vmx_vcpu_precreate(kvm);
  66}
  67
  68static int vt_vcpu_create(struct kvm_vcpu *vcpu)
  69{
  70	if (is_td_vcpu(vcpu))
  71		return tdx_vcpu_create(vcpu);
  72
  73	return vmx_vcpu_create(vcpu);
  74}
  75
  76static void vt_vcpu_free(struct kvm_vcpu *vcpu)
  77{
  78	if (is_td_vcpu(vcpu)) {
  79		tdx_vcpu_free(vcpu);
  80		return;
  81	}
  82
  83	vmx_vcpu_free(vcpu);
  84}
  85
  86static void vt_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
  87{
  88	if (is_td_vcpu(vcpu)) {
  89		tdx_vcpu_reset(vcpu, init_event);
  90		return;
  91	}
  92
  93	vmx_vcpu_reset(vcpu, init_event);
  94}
  95
  96static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  97{
  98	if (is_td_vcpu(vcpu)) {
  99		tdx_vcpu_load(vcpu, cpu);
 100		return;
 101	}
 102
 103	vmx_vcpu_load(vcpu, cpu);
 104}
 105
 106static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu)
 107{
 108	/*
 109	 * Basic TDX does not support feature PML. KVM does not enable PML in
 110	 * TD's VMCS, nor does it allocate or flush PML buffer for TDX.
 111	 */
 112	if (WARN_ON_ONCE(is_td_vcpu(vcpu)))
 113		return;
 114
 115	vmx_update_cpu_dirty_logging(vcpu);
 116}
 117
 118static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu)
 119{
 120	if (is_td_vcpu(vcpu)) {
 121		tdx_prepare_switch_to_guest(vcpu);
 122		return;
 123	}
 124
 125	vmx_prepare_switch_to_guest(vcpu);
 126}
 127
 128static void vt_vcpu_put(struct kvm_vcpu *vcpu)
 129{
 130	if (is_td_vcpu(vcpu)) {
 131		tdx_vcpu_put(vcpu);
 132		return;
 133	}
 134
 135	vmx_vcpu_put(vcpu);
 136}
 137
 138static int vt_vcpu_pre_run(struct kvm_vcpu *vcpu)
 139{
 140	if (is_td_vcpu(vcpu))
 141		return tdx_vcpu_pre_run(vcpu);
 142
 143	return vmx_vcpu_pre_run(vcpu);
 144}
 145
 146static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 147{
 148	if (is_td_vcpu(vcpu))
 149		return tdx_vcpu_run(vcpu, run_flags);
 150
 151	return vmx_vcpu_run(vcpu, run_flags);
 152}
 153
 154static int vt_handle_exit(struct kvm_vcpu *vcpu,
 155			  enum exit_fastpath_completion fastpath)
 156{
 157	if (is_td_vcpu(vcpu))
 158		return tdx_handle_exit(vcpu, fastpath);
 159
 160	return vmx_handle_exit(vcpu, fastpath);
 161}
 162
 163static int vt_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 164{
 165	if (unlikely(is_td_vcpu(vcpu)))
 166		return tdx_set_msr(vcpu, msr_info);
 167
 168	return vmx_set_msr(vcpu, msr_info);
 169}
 170
 171/*
 172 * The kvm parameter can be NULL (module initialization, or invocation before
 173 * VM creation). Be sure to check the kvm parameter before using it.
 174 */
 175static bool vt_has_emulated_msr(struct kvm *kvm, u32 index)
 176{
 177	if (kvm && is_td(kvm))
 178		return tdx_has_emulated_msr(index);
 179
 180	return vmx_has_emulated_msr(kvm, index);
 181}
 182
 183static int vt_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 184{
 185	if (unlikely(is_td_vcpu(vcpu)))
 186		return tdx_get_msr(vcpu, msr_info);
 187
 188	return vmx_get_msr(vcpu, msr_info);
 189}
 190
 191static void vt_recalc_intercepts(struct kvm_vcpu *vcpu)
 192{
 193	/*
 194	 * TDX doesn't allow VMM to configure interception of instructions or
 195	 * MSR accesses.  TDX guest requests MSR accesses by calling TDVMCALL.
 196	 * The MSR filters will be applied when handling the TDVMCALL for
 197	 * RDMSR/WRMSR if the userspace has set any.
 198	 */
 199	if (is_td_vcpu(vcpu))
 200		return;
 201
 202	vmx_recalc_intercepts(vcpu);
 203}
 204
 205static int vt_complete_emulated_msr(struct kvm_vcpu *vcpu, int err)
 206{
 207	if (is_td_vcpu(vcpu))
 208		return tdx_complete_emulated_msr(vcpu, err);
 209
 210	return vmx_complete_emulated_msr(vcpu, err);
 211}
 212
 213#ifdef CONFIG_KVM_SMM
 214static int vt_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
 215{
 216	if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
 217		return 0;
 218
 219	return vmx_smi_allowed(vcpu, for_injection);
 220}
 221
 222static int vt_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
 223{
 224	if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
 225		return 0;
 226
 227	return vmx_enter_smm(vcpu, smram);
 228}
 229
 230static int vt_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram)
 231{
 232	if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
 233		return 0;
 234
 235	return vmx_leave_smm(vcpu, smram);
 236}
 237
 238static void vt_enable_smi_window(struct kvm_vcpu *vcpu)
 239{
 240	if (KVM_BUG_ON(is_td_vcpu(vcpu), vcpu->kvm))
 241		return;
 242
 243	/* RSM will cause a vmexit anyway.  */
 244	vmx_enable_smi_window(vcpu);
 245}
 246#endif
 247
 248static int vt_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
 249					void *insn, int insn_len)
 250{
 251	/*
 252	 * For TDX, this can only be triggered for MMIO emulation.  Let the
 253	 * guest retry after installing the SPTE with suppress #VE bit cleared,
 254	 * so that the guest will receive #VE when retry.  The guest is expected
 255	 * to call TDG.VP.VMCALL<MMIO> to request VMM to do MMIO emulation on
 256	 * #VE.
 257	 */
 258	if (is_td_vcpu(vcpu))
 259		return X86EMUL_RETRY_INSTR;
 260
 261	return vmx_check_emulate_instruction(vcpu, emul_type, insn, insn_len);
 262}
 263
 264static bool vt_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
 265{
 266	/*
 267	 * INIT and SIPI are always blocked for TDX, i.e., INIT handling and
 268	 * the OP vcpu_deliver_sipi_vector() won't be called.
 269	 */
 270	if (is_td_vcpu(vcpu))
 271		return true;
 272
 273	return vmx_apic_init_signal_blocked(vcpu);
 274}
 275
 276static void vt_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
 277{
 278	/* Only x2APIC mode is supported for TD. */
 279	if (is_td_vcpu(vcpu))
 280		return;
 281
 282	return vmx_set_virtual_apic_mode(vcpu);
 283}
 284
 285static void vt_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
 286{
 287	if (is_td_vcpu(vcpu))
 288		return;
 289
 290	return vmx_hwapic_isr_update(vcpu, max_isr);
 291}
 292
 293static int vt_sync_pir_to_irr(struct kvm_vcpu *vcpu)
 294{
 295	if (is_td_vcpu(vcpu))
 296		return -1;
 297
 298	return vmx_sync_pir_to_irr(vcpu);
 299}
 300
 301static void vt_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode,
 302			   int trig_mode, int vector)
 303{
 304	if (is_td_vcpu(apic->vcpu)) {
 305		tdx_deliver_interrupt(apic, delivery_mode, trig_mode,
 306					     vector);
 307		return;
 308	}
 309
 310	vmx_deliver_interrupt(apic, delivery_mode, trig_mode, vector);
 311}
 312
 313static void vt_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
 314{
 315	if (is_td_vcpu(vcpu))
 316		return;
 317
 318	vmx_vcpu_after_set_cpuid(vcpu);
 319}
 320
 321static void vt_update_exception_bitmap(struct kvm_vcpu *vcpu)
 322{
 323	if (is_td_vcpu(vcpu))
 324		return;
 325
 326	vmx_update_exception_bitmap(vcpu);
 327}
 328
 329static u64 vt_get_segment_base(struct kvm_vcpu *vcpu, int seg)
 330{
 331	if (is_td_vcpu(vcpu))
 332		return 0;
 333
 334	return vmx_get_segment_base(vcpu, seg);
 335}
 336
 337static void vt_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var,
 338			      int seg)
 339{
 340	if (is_td_vcpu(vcpu)) {
 341		memset(var, 0, sizeof(*var));
 342		return;
 343	}
 344
 345	vmx_get_segment(vcpu, var, seg);
 346}
 347
 348static void vt_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var,
 349			      int seg)
 350{
 351	if (is_td_vcpu(vcpu))
 352		return;
 353
 354	vmx_set_segment(vcpu, var, seg);
 355}
 356
 357static int vt_get_cpl(struct kvm_vcpu *vcpu)
 358{
 359	if (is_td_vcpu(vcpu))
 360		return 0;
 361
 362	return vmx_get_cpl(vcpu);
 363}
 364
 365static int vt_get_cpl_no_cache(struct kvm_vcpu *vcpu)
 366{
 367	if (is_td_vcpu(vcpu))
 368		return 0;
 369
 370	return vmx_get_cpl_no_cache(vcpu);
 371}
 372
 373static void vt_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
 374{
 375	if (is_td_vcpu(vcpu)) {
 376		*db = 0;
 377		*l = 0;
 378		return;
 379	}
 380
 381	vmx_get_cs_db_l_bits(vcpu, db, l);
 382}
 383
 384static bool vt_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 385{
 386	if (is_td_vcpu(vcpu))
 387		return true;
 388
 389	return vmx_is_valid_cr0(vcpu, cr0);
 390}
 391
 392static void vt_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 393{
 394	if (is_td_vcpu(vcpu))
 395		return;
 396
 397	vmx_set_cr0(vcpu, cr0);
 398}
 399
 400static bool vt_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 401{
 402	if (is_td_vcpu(vcpu))
 403		return true;
 404
 405	return vmx_is_valid_cr4(vcpu, cr4);
 406}
 407
 408static void vt_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 409{
 410	if (is_td_vcpu(vcpu))
 411		return;
 412
 413	vmx_set_cr4(vcpu, cr4);
 414}
 415
 416static int vt_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 417{
 418	if (is_td_vcpu(vcpu))
 419		return 0;
 420
 421	return vmx_set_efer(vcpu, efer);
 422}
 423
 424static void vt_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
 425{
 426	if (is_td_vcpu(vcpu)) {
 427		memset(dt, 0, sizeof(*dt));
 428		return;
 429	}
 430
 431	vmx_get_idt(vcpu, dt);
 432}
 433
 434static void vt_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
 435{
 436	if (is_td_vcpu(vcpu))
 437		return;
 438
 439	vmx_set_idt(vcpu, dt);
 440}
 441
 442static void vt_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
 443{
 444	if (is_td_vcpu(vcpu)) {
 445		memset(dt, 0, sizeof(*dt));
 446		return;
 447	}
 448
 449	vmx_get_gdt(vcpu, dt);
 450}
 451
 452static void vt_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
 453{
 454	if (is_td_vcpu(vcpu))
 455		return;
 456
 457	vmx_set_gdt(vcpu, dt);
 458}
 459
 460static void vt_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
 461{
 462	if (is_td_vcpu(vcpu))
 463		return;
 464
 465	vmx_set_dr7(vcpu, val);
 466}
 467
 468static void vt_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
 469{
 470	/*
 471	 * MOV-DR exiting is always cleared for TD guest, even in debug mode.
 472	 * Thus KVM_DEBUGREG_WONT_EXIT can never be set and it should never
 473	 * reach here for TD vcpu.
 474	 */
 475	if (is_td_vcpu(vcpu))
 476		return;
 477
 478	vmx_sync_dirty_debug_regs(vcpu);
 479}
 480
 481static void vt_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
 482{
 483	if (WARN_ON_ONCE(is_td_vcpu(vcpu)))
 484		return;
 485
 486	vmx_cache_reg(vcpu, reg);
 487}
 488
 489static unsigned long vt_get_rflags(struct kvm_vcpu *vcpu)
 490{
 491	if (is_td_vcpu(vcpu))
 492		return 0;
 493
 494	return vmx_get_rflags(vcpu);
 495}
 496
 497static void vt_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 498{
 499	if (is_td_vcpu(vcpu))
 500		return;
 501
 502	vmx_set_rflags(vcpu, rflags);
 503}
 504
 505static bool vt_get_if_flag(struct kvm_vcpu *vcpu)
 506{
 507	if (is_td_vcpu(vcpu))
 508		return false;
 509
 510	return vmx_get_if_flag(vcpu);
 511}
 512
 513static void vt_flush_tlb_all(struct kvm_vcpu *vcpu)
 514{
 515	if (is_td_vcpu(vcpu)) {
 516		tdx_flush_tlb_all(vcpu);
 517		return;
 518	}
 519
 520	vmx_flush_tlb_all(vcpu);
 521}
 522
 523static void vt_flush_tlb_current(struct kvm_vcpu *vcpu)
 524{
 525	if (is_td_vcpu(vcpu)) {
 526		tdx_flush_tlb_current(vcpu);
 527		return;
 528	}
 529
 530	vmx_flush_tlb_current(vcpu);
 531}
 532
 533static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
 534{
 535	if (is_td_vcpu(vcpu))
 536		return;
 537
 538	vmx_flush_tlb_gva(vcpu, addr);
 539}
 540
 541static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu)
 542{
 543	if (is_td_vcpu(vcpu))
 544		return;
 545
 546	vmx_flush_tlb_guest(vcpu);
 547}
 548
 549static void vt_inject_nmi(struct kvm_vcpu *vcpu)
 550{
 551	if (is_td_vcpu(vcpu)) {
 552		tdx_inject_nmi(vcpu);
 553		return;
 554	}
 555
 556	vmx_inject_nmi(vcpu);
 557}
 558
 559static int vt_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection)
 560{
 561	/*
 562	 * The TDX module manages NMI windows and NMI reinjection, and hides NMI
 563	 * blocking, all KVM can do is throw an NMI over the wall.
 564	 */
 565	if (is_td_vcpu(vcpu))
 566		return true;
 567
 568	return vmx_nmi_allowed(vcpu, for_injection);
 569}
 570
 571static bool vt_get_nmi_mask(struct kvm_vcpu *vcpu)
 572{
 573	/*
 574	 * KVM can't get NMI blocking status for TDX guest, assume NMIs are
 575	 * always unmasked.
 576	 */
 577	if (is_td_vcpu(vcpu))
 578		return false;
 579
 580	return vmx_get_nmi_mask(vcpu);
 581}
 582
 583static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
 584{
 585	if (is_td_vcpu(vcpu))
 586		return;
 587
 588	vmx_set_nmi_mask(vcpu, masked);
 589}
 590
 591static void vt_enable_nmi_window(struct kvm_vcpu *vcpu)
 592{
 593	/* Refer to the comments in tdx_inject_nmi(). */
 594	if (is_td_vcpu(vcpu))
 595		return;
 596
 597	vmx_enable_nmi_window(vcpu);
 598}
 599
 600static void vt_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa,
 601			    int pgd_level)
 602{
 603	if (is_td_vcpu(vcpu)) {
 604		tdx_load_mmu_pgd(vcpu, root_hpa, pgd_level);
 605		return;
 606	}
 607
 608	vmx_load_mmu_pgd(vcpu, root_hpa, pgd_level);
 609}
 610
 611static void vt_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 612{
 613	if (is_td_vcpu(vcpu))
 614		return;
 615
 616	vmx_set_interrupt_shadow(vcpu, mask);
 617}
 618
 619static u32 vt_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 620{
 621	if (is_td_vcpu(vcpu))
 622		return 0;
 623
 624	return vmx_get_interrupt_shadow(vcpu);
 625}
 626
 627static void vt_patch_hypercall(struct kvm_vcpu *vcpu,
 628				  unsigned char *hypercall)
 629{
 630	/*
 631	 * Because guest memory is protected, guest can't be patched. TD kernel
 632	 * is modified to use TDG.VP.VMCALL for hypercall.
 633	 */
 634	if (is_td_vcpu(vcpu))
 635		return;
 636
 637	vmx_patch_hypercall(vcpu, hypercall);
 638}
 639
 640static void vt_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
 641{
 642	if (is_td_vcpu(vcpu))
 643		return;
 644
 645	vmx_inject_irq(vcpu, reinjected);
 646}
 647
 648static void vt_inject_exception(struct kvm_vcpu *vcpu)
 649{
 650	if (is_td_vcpu(vcpu))
 651		return;
 652
 653	vmx_inject_exception(vcpu);
 654}
 655
 656static void vt_cancel_injection(struct kvm_vcpu *vcpu)
 657{
 658	if (is_td_vcpu(vcpu))
 659		return;
 660
 661	vmx_cancel_injection(vcpu);
 662}
 663
 664static int vt_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection)
 665{
 666	if (is_td_vcpu(vcpu))
 667		return tdx_interrupt_allowed(vcpu);
 668
 669	return vmx_interrupt_allowed(vcpu, for_injection);
 670}
 671
 672static void vt_enable_irq_window(struct kvm_vcpu *vcpu)
 673{
 674	if (is_td_vcpu(vcpu))
 675		return;
 676
 677	vmx_enable_irq_window(vcpu);
 678}
 679
 680static void vt_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code)
 681{
 682	*intr_info = 0;
 683	*error_code = 0;
 684
 685	if (is_td_vcpu(vcpu))
 686		return;
 687
 688	vmx_get_entry_info(vcpu, intr_info, error_code);
 689}
 690
 691static void vt_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
 692			u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code)
 693{
 694	if (is_td_vcpu(vcpu)) {
 695		tdx_get_exit_info(vcpu, reason, info1, info2, intr_info,
 696				  error_code);
 697		return;
 698	}
 699
 700	vmx_get_exit_info(vcpu, reason, info1, info2, intr_info, error_code);
 701}
 702
 703static void vt_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
 704{
 705	if (is_td_vcpu(vcpu))
 706		return;
 707
 708	vmx_update_cr8_intercept(vcpu, tpr, irr);
 709}
 710
 711static void vt_set_apic_access_page_addr(struct kvm_vcpu *vcpu)
 712{
 713	if (is_td_vcpu(vcpu))
 714		return;
 715
 716	vmx_set_apic_access_page_addr(vcpu);
 717}
 718
 719static void vt_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
 720{
 721	if (is_td_vcpu(vcpu)) {
 722		KVM_BUG_ON(!kvm_vcpu_apicv_active(vcpu), vcpu->kvm);
 723		return;
 724	}
 725
 726	vmx_refresh_apicv_exec_ctrl(vcpu);
 727}
 728
 729static void vt_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
 730{
 731	if (is_td_vcpu(vcpu))
 732		return;
 733
 734	vmx_load_eoi_exitmap(vcpu, eoi_exit_bitmap);
 735}
 736
 737static int vt_set_tss_addr(struct kvm *kvm, unsigned int addr)
 738{
 739	if (is_td(kvm))
 740		return 0;
 741
 742	return vmx_set_tss_addr(kvm, addr);
 743}
 744
 745static int vt_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
 746{
 747	if (is_td(kvm))
 748		return 0;
 749
 750	return vmx_set_identity_map_addr(kvm, ident_addr);
 751}
 752
 753static u64 vt_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
 754{
 755	/* TDX doesn't support L2 guest at the moment. */
 756	if (is_td_vcpu(vcpu))
 757		return 0;
 758
 759	return vmx_get_l2_tsc_offset(vcpu);
 760}
 761
 762static u64 vt_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu)
 763{
 764	/* TDX doesn't support L2 guest at the moment. */
 765	if (is_td_vcpu(vcpu))
 766		return 0;
 767
 768	return vmx_get_l2_tsc_multiplier(vcpu);
 769}
 770
 771static void vt_write_tsc_offset(struct kvm_vcpu *vcpu)
 772{
 773	/* In TDX, tsc offset can't be changed. */
 774	if (is_td_vcpu(vcpu))
 775		return;
 776
 777	vmx_write_tsc_offset(vcpu);
 778}
 779
 780static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu)
 781{
 782	/* In TDX, tsc multiplier can't be changed. */
 783	if (is_td_vcpu(vcpu))
 784		return;
 785
 786	vmx_write_tsc_multiplier(vcpu);
 787}
 788
 789#ifdef CONFIG_X86_64
 790static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc,
 791			      bool *expired)
 792{
 793	/* VMX-preemption timer isn't available for TDX. */
 794	if (is_td_vcpu(vcpu))
 795		return -EINVAL;
 796
 797	return vmx_set_hv_timer(vcpu, guest_deadline_tsc, expired);
 798}
 799
 800static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu)
 801{
 802	/* VMX-preemption timer can't be set.  See vt_set_hv_timer(). */
 803	if (is_td_vcpu(vcpu))
 804		return;
 805
 806	vmx_cancel_hv_timer(vcpu);
 807}
 808#endif
 809
 810static void vt_setup_mce(struct kvm_vcpu *vcpu)
 811{
 812	if (is_td_vcpu(vcpu))
 813		return;
 814
 815	vmx_setup_mce(vcpu);
 816}
 817
 818static int vt_mem_enc_ioctl(struct kvm *kvm, void __user *argp)
 819{
 820	if (!is_td(kvm))
 821		return -ENOTTY;
 822
 823	return tdx_vm_ioctl(kvm, argp);
 824}
 825
 826static int vt_vcpu_mem_enc_ioctl(struct kvm_vcpu *vcpu, void __user *argp)
 827{
 828	if (!is_td_vcpu(vcpu))
 829		return -EINVAL;
 830
 831	return tdx_vcpu_ioctl(vcpu, argp);
 832}
 833
 834static int vt_vcpu_mem_enc_unlocked_ioctl(struct kvm_vcpu *vcpu, void __user *argp)
 835{
 836	if (!is_td_vcpu(vcpu))
 837		return -EINVAL;
 838
 839	return tdx_vcpu_unlocked_ioctl(vcpu, argp);
 840}
 841
 842static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn,
 843				     bool is_private)
 844{
 845	if (is_td(kvm))
 846		return tdx_gmem_max_mapping_level(kvm, pfn, is_private);
 847
 848	return 0;
 849}
 850
 851#define vt_op(name) vt_##name
 852#define vt_op_tdx_only(name) vt_##name
 853#else /* CONFIG_KVM_INTEL_TDX */
 854#define vt_op(name) vmx_##name
 855#define vt_op_tdx_only(name) NULL
 856#endif /* CONFIG_KVM_INTEL_TDX */
 857
 858#define VMX_REQUIRED_APICV_INHIBITS				\
 859	(BIT(APICV_INHIBIT_REASON_DISABLED) |			\
 860	 BIT(APICV_INHIBIT_REASON_ABSENT) |			\
 861	 BIT(APICV_INHIBIT_REASON_HYPERV) |			\
 862	 BIT(APICV_INHIBIT_REASON_BLOCKIRQ) |			\
 863	 BIT(APICV_INHIBIT_REASON_PHYSICAL_ID_ALIASED) |	\
 864	 BIT(APICV_INHIBIT_REASON_APIC_ID_MODIFIED) |		\
 865	 BIT(APICV_INHIBIT_REASON_APIC_BASE_MODIFIED))
 866
 867struct kvm_x86_ops vt_x86_ops __initdata = {
 868	.name = KBUILD_MODNAME,
 869
 870	.check_processor_compatibility = vmx_check_processor_compat,
 871
 872	.hardware_unsetup = vmx_hardware_unsetup,
 873
 874	.enable_virtualization_cpu = vmx_enable_virtualization_cpu,
 875	.disable_virtualization_cpu = vt_op(disable_virtualization_cpu),
 876	.emergency_disable_virtualization_cpu = vmx_emergency_disable_virtualization_cpu,
 877
 878	.has_emulated_msr = vt_op(has_emulated_msr),
 879
 880	.vm_size = sizeof(struct kvm_vmx),
 881
 882	.vm_init = vt_op(vm_init),
 883	.vm_destroy = vt_op(vm_destroy),
 884	.vm_pre_destroy = vt_op_tdx_only(vm_pre_destroy),
 885
 886	.vcpu_precreate = vt_op(vcpu_precreate),
 887	.vcpu_create = vt_op(vcpu_create),
 888	.vcpu_free = vt_op(vcpu_free),
 889	.vcpu_reset = vt_op(vcpu_reset),
 890
 891	.prepare_switch_to_guest = vt_op(prepare_switch_to_guest),
 892	.vcpu_load = vt_op(vcpu_load),
 893	.vcpu_put = vt_op(vcpu_put),
 894
 895	.HOST_OWNED_DEBUGCTL = VMX_HOST_OWNED_DEBUGCTL_BITS,
 896
 897	.update_exception_bitmap = vt_op(update_exception_bitmap),
 898	.get_feature_msr = vmx_get_feature_msr,
 899	.get_msr = vt_op(get_msr),
 900	.set_msr = vt_op(set_msr),
 901
 902	.get_segment_base = vt_op(get_segment_base),
 903	.get_segment = vt_op(get_segment),
 904	.set_segment = vt_op(set_segment),
 905	.get_cpl = vt_op(get_cpl),
 906	.get_cpl_no_cache = vt_op(get_cpl_no_cache),
 907	.get_cs_db_l_bits = vt_op(get_cs_db_l_bits),
 908	.is_valid_cr0 = vt_op(is_valid_cr0),
 909	.set_cr0 = vt_op(set_cr0),
 910	.is_valid_cr4 = vt_op(is_valid_cr4),
 911	.set_cr4 = vt_op(set_cr4),
 912	.set_efer = vt_op(set_efer),
 913	.get_idt = vt_op(get_idt),
 914	.set_idt = vt_op(set_idt),
 915	.get_gdt = vt_op(get_gdt),
 916	.set_gdt = vt_op(set_gdt),
 917	.set_dr7 = vt_op(set_dr7),
 918	.sync_dirty_debug_regs = vt_op(sync_dirty_debug_regs),
 919	.cache_reg = vt_op(cache_reg),
 920	.get_rflags = vt_op(get_rflags),
 921	.set_rflags = vt_op(set_rflags),
 922	.get_if_flag = vt_op(get_if_flag),
 923
 924	.flush_tlb_all = vt_op(flush_tlb_all),
 925	.flush_tlb_current = vt_op(flush_tlb_current),
 926	.flush_tlb_gva = vt_op(flush_tlb_gva),
 927	.flush_tlb_guest = vt_op(flush_tlb_guest),
 928
 929	.vcpu_pre_run = vt_op(vcpu_pre_run),
 930	.vcpu_run = vt_op(vcpu_run),
 931	.handle_exit = vt_op(handle_exit),
 932	.skip_emulated_instruction = vmx_skip_emulated_instruction,
 933	.update_emulated_instruction = vmx_update_emulated_instruction,
 934	.set_interrupt_shadow = vt_op(set_interrupt_shadow),
 935	.get_interrupt_shadow = vt_op(get_interrupt_shadow),
 936	.patch_hypercall = vt_op(patch_hypercall),
 937	.inject_irq = vt_op(inject_irq),
 938	.inject_nmi = vt_op(inject_nmi),
 939	.inject_exception = vt_op(inject_exception),
 940	.cancel_injection = vt_op(cancel_injection),
 941	.interrupt_allowed = vt_op(interrupt_allowed),
 942	.nmi_allowed = vt_op(nmi_allowed),
 943	.get_nmi_mask = vt_op(get_nmi_mask),
 944	.set_nmi_mask = vt_op(set_nmi_mask),
 945	.enable_nmi_window = vt_op(enable_nmi_window),
 946	.enable_irq_window = vt_op(enable_irq_window),
 947	.update_cr8_intercept = vt_op(update_cr8_intercept),
 948
 949	.x2apic_icr_is_split = false,
 950	.set_virtual_apic_mode = vt_op(set_virtual_apic_mode),
 951	.set_apic_access_page_addr = vt_op(set_apic_access_page_addr),
 952	.refresh_apicv_exec_ctrl = vt_op(refresh_apicv_exec_ctrl),
 953	.load_eoi_exitmap = vt_op(load_eoi_exitmap),
 954	.apicv_pre_state_restore = pi_apicv_pre_state_restore,
 955	.required_apicv_inhibits = VMX_REQUIRED_APICV_INHIBITS,
 956	.hwapic_isr_update = vt_op(hwapic_isr_update),
 957	.sync_pir_to_irr = vt_op(sync_pir_to_irr),
 958	.deliver_interrupt = vt_op(deliver_interrupt),
 959	.dy_apicv_has_pending_interrupt = pi_has_pending_interrupt,
 960
 961	.set_tss_addr = vt_op(set_tss_addr),
 962	.set_identity_map_addr = vt_op(set_identity_map_addr),
 963	.get_mt_mask = vmx_get_mt_mask,
 964
 965	.get_exit_info = vt_op(get_exit_info),
 966	.get_entry_info = vt_op(get_entry_info),
 967
 968	.vcpu_after_set_cpuid = vt_op(vcpu_after_set_cpuid),
 969
 970	.has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
 971
 972	.get_l2_tsc_offset = vt_op(get_l2_tsc_offset),
 973	.get_l2_tsc_multiplier = vt_op(get_l2_tsc_multiplier),
 974	.write_tsc_offset = vt_op(write_tsc_offset),
 975	.write_tsc_multiplier = vt_op(write_tsc_multiplier),
 976
 977	.load_mmu_pgd = vt_op(load_mmu_pgd),
 978
 979	.check_intercept = vmx_check_intercept,
 980	.handle_exit_irqoff = vmx_handle_exit_irqoff,
 981
 982	.update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging),
 983
 984	.nested_ops = &vmx_nested_ops,
 985
 986	.pi_update_irte = vmx_pi_update_irte,
 987	.pi_start_bypass = vmx_pi_start_bypass,
 988
 989#ifdef CONFIG_X86_64
 990	.set_hv_timer = vt_op(set_hv_timer),
 991	.cancel_hv_timer = vt_op(cancel_hv_timer),
 992#endif
 993
 994	.setup_mce = vt_op(setup_mce),
 995
 996#ifdef CONFIG_KVM_SMM
 997	.smi_allowed = vt_op(smi_allowed),
 998	.enter_smm = vt_op(enter_smm),
 999	.leave_smm = vt_op(leave_smm),
1000	.enable_smi_window = vt_op(enable_smi_window),
1001#endif
1002
1003	.check_emulate_instruction = vt_op(check_emulate_instruction),
1004	.apic_init_signal_blocked = vt_op(apic_init_signal_blocked),
1005	.migrate_timers = vmx_migrate_timers,
1006
1007	.recalc_intercepts = vt_op(recalc_intercepts),
1008	.complete_emulated_msr = vt_op(complete_emulated_msr),
1009
1010	.vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector,
1011
1012	.get_untagged_addr = vmx_get_untagged_addr,
1013
1014	.mem_enc_ioctl = vt_op_tdx_only(mem_enc_ioctl),
1015	.vcpu_mem_enc_ioctl = vt_op_tdx_only(vcpu_mem_enc_ioctl),
1016	.vcpu_mem_enc_unlocked_ioctl = vt_op_tdx_only(vcpu_mem_enc_unlocked_ioctl),
1017
1018	.gmem_max_mapping_level = vt_op_tdx_only(gmem_max_mapping_level)
1019};
1020
1021struct kvm_x86_init_ops vt_init_ops __initdata = {
1022	.hardware_setup = vt_op(hardware_setup),
1023	.handle_intel_pt_intr = NULL,
1024
1025	.runtime_ops = &vt_x86_ops,
1026	.pmu_ops = &intel_pmu_ops,
1027};
1028
1029static void __exit vt_exit(void)
1030{
1031	kvm_exit();
1032	tdx_cleanup();
1033	vmx_exit();
1034}
1035module_exit(vt_exit);
1036
1037static int __init vt_init(void)
1038{
1039	unsigned vcpu_size, vcpu_align;
1040	int r;
1041
1042	r = vmx_init();
1043	if (r)
1044		return r;
1045
1046	/* tdx_init() has been taken */
1047	r = tdx_bringup();
1048	if (r)
1049		goto err_tdx_bringup;
1050
1051	/*
1052	 * TDX and VMX have different vCPU structures.  Calculate the
1053	 * maximum size/align so that kvm_init() can use the larger
1054	 * values to create the kmem_vcpu_cache.
1055	 */
1056	vcpu_size = sizeof(struct vcpu_vmx);
1057	vcpu_align = __alignof__(struct vcpu_vmx);
1058	if (enable_tdx) {
1059		vcpu_size = max_t(unsigned, vcpu_size,
1060				sizeof(struct vcpu_tdx));
1061		vcpu_align = max_t(unsigned, vcpu_align,
1062				__alignof__(struct vcpu_tdx));
1063		kvm_caps.supported_vm_types |= BIT(KVM_X86_TDX_VM);
1064	}
1065
1066	/*
1067	 * Common KVM initialization _must_ come last, after this, /dev/kvm is
1068	 * exposed to userspace!
1069	 */
1070	r = kvm_init(vcpu_size, vcpu_align, THIS_MODULE);
1071	if (r)
1072		goto err_kvm_init;
1073
1074	return 0;
1075
1076err_kvm_init:
1077	tdx_cleanup();
1078err_tdx_bringup:
1079	vmx_exit();
1080	return r;
1081}
1082module_init(vt_init);