cpuid.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Kernel-based Virtual Machine driver for Linux
   4 * cpuid support routines
   5 *
   6 * derived from arch/x86/kvm/x86.c
   7 *
   8 * Copyright 2011 Red Hat, Inc. and/or its affiliates.
   9 * Copyright IBM Corporation, 2008
  10 */
  11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  12
  13#include <linux/kvm_host.h>
  14#include "linux/lockdep.h"
  15#include <linux/export.h>
  16#include <linux/vmalloc.h>
  17#include <linux/uaccess.h>
  18#include <linux/sched/stat.h>
  19
  20#include <asm/processor.h>
  21#include <asm/user.h>
  22#include <asm/fpu/xstate.h>
  23#include <asm/sgx.h>
  24#include <asm/cpuid/api.h>
  25#include "cpuid.h"
  26#include "lapic.h"
  27#include "mmu.h"
  28#include "trace.h"
  29#include "pmu.h"
  30#include "xen.h"
  31
  32/*
  33 * Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be
  34 * aligned to sizeof(unsigned long) because it's not accessed via bitops.
  35 */
  36u32 kvm_cpu_caps[NR_KVM_CPU_CAPS] __read_mostly;
  37EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_cpu_caps);
  38
  39struct cpuid_xstate_sizes {
  40	u32 eax;
  41	u32 ebx;
  42	u32 ecx;
  43};
  44
  45static struct cpuid_xstate_sizes xstate_sizes[XFEATURE_MAX] __ro_after_init;
  46
  47void __init kvm_init_xstate_sizes(void)
  48{
  49	u32 ign;
  50	int i;
  51
  52	for (i = XFEATURE_YMM; i < ARRAY_SIZE(xstate_sizes); i++) {
  53		struct cpuid_xstate_sizes *xs = &xstate_sizes[i];
  54
  55		cpuid_count(0xD, i, &xs->eax, &xs->ebx, &xs->ecx, &ign);
  56	}
  57}
  58
  59u32 xstate_required_size(u64 xstate_bv, bool compacted)
  60{
  61	u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
  62	int i;
  63
  64	xstate_bv &= XFEATURE_MASK_EXTEND;
  65	for (i = XFEATURE_YMM; i < ARRAY_SIZE(xstate_sizes) && xstate_bv; i++) {
  66		struct cpuid_xstate_sizes *xs = &xstate_sizes[i];
  67		u32 offset;
  68
  69		if (!(xstate_bv & BIT_ULL(i)))
  70			continue;
  71
  72		/* ECX[1]: 64B alignment in compacted form */
  73		if (compacted)
  74			offset = (xs->ecx & 0x2) ? ALIGN(ret, 64) : ret;
  75		else
  76			offset = xs->ebx;
  77		ret = max(ret, offset + xs->eax);
  78		xstate_bv &= ~BIT_ULL(i);
  79	}
  80
  81	return ret;
  82}
  83
  84struct kvm_cpuid_entry2 *kvm_find_cpuid_entry2(
  85	struct kvm_cpuid_entry2 *entries, int nent, u32 function, u64 index)
  86{
  87	struct kvm_cpuid_entry2 *e;
  88	int i;
  89
  90	/*
  91	 * KVM has a semi-arbitrary rule that querying the guest's CPUID model
  92	 * with IRQs disabled is disallowed.  The CPUID model can legitimately
  93	 * have over one hundred entries, i.e. the lookup is slow, and IRQs are
  94	 * typically disabled in KVM only when KVM is in a performance critical
  95	 * path, e.g. the core VM-Enter/VM-Exit run loop.  Nothing will break
  96	 * if this rule is violated, this assertion is purely to flag potential
  97	 * performance issues.  If this fires, consider moving the lookup out
  98	 * of the hotpath, e.g. by caching information during CPUID updates.
  99	 */
 100	lockdep_assert_irqs_enabled();
 101
 102	for (i = 0; i < nent; i++) {
 103		e = &entries[i];
 104
 105		if (e->function != function)
 106			continue;
 107
 108		/*
 109		 * If the index isn't significant, use the first entry with a
 110		 * matching function.  It's userspace's responsibility to not
 111		 * provide "duplicate" entries in all cases.
 112		 */
 113		if (!(e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) || e->index == index)
 114			return e;
 115
 116
 117		/*
 118		 * Similarly, use the first matching entry if KVM is doing a
 119		 * lookup (as opposed to emulating CPUID) for a function that's
 120		 * architecturally defined as not having a significant index.
 121		 */
 122		if (index == KVM_CPUID_INDEX_NOT_SIGNIFICANT) {
 123			/*
 124			 * Direct lookups from KVM should not diverge from what
 125			 * KVM defines internally (the architectural behavior).
 126			 */
 127			WARN_ON_ONCE(cpuid_function_is_indexed(function));
 128			return e;
 129		}
 130	}
 131
 132	return NULL;
 133}
 134EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_find_cpuid_entry2);
 135
 136static int kvm_check_cpuid(struct kvm_vcpu *vcpu)
 137{
 138	struct kvm_cpuid_entry2 *best;
 139	u64 xfeatures;
 140
 141	/*
 142	 * The existing code assumes virtual address is 48-bit or 57-bit in the
 143	 * canonical address checks; exit if it is ever changed.
 144	 */
 145	best = kvm_find_cpuid_entry(vcpu, 0x80000008);
 146	if (best) {
 147		int vaddr_bits = (best->eax & 0xff00) >> 8;
 148
 149		if (vaddr_bits != 48 && vaddr_bits != 57 && vaddr_bits != 0)
 150			return -EINVAL;
 151	}
 152
 153	/*
 154	 * Exposing dynamic xfeatures to the guest requires additional
 155	 * enabling in the FPU, e.g. to expand the guest XSAVE state size.
 156	 */
 157	best = kvm_find_cpuid_entry_index(vcpu, 0xd, 0);
 158	if (!best)
 159		return 0;
 160
 161	xfeatures = best->eax | ((u64)best->edx << 32);
 162	xfeatures &= XFEATURE_MASK_USER_DYNAMIC;
 163	if (!xfeatures)
 164		return 0;
 165
 166	return fpu_enable_guest_xfd_features(&vcpu->arch.guest_fpu, xfeatures);
 167}
 168
 169static u32 kvm_apply_cpuid_pv_features_quirk(struct kvm_vcpu *vcpu);
 170static void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu);
 171
 172/* Check whether the supplied CPUID data is equal to what is already set for the vCPU. */
 173static int kvm_cpuid_check_equal(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2,
 174				 int nent)
 175{
 176	struct kvm_cpuid_entry2 *orig;
 177	int i;
 178
 179	/*
 180	 * Apply runtime CPUID updates to the incoming CPUID entries to avoid
 181	 * false positives due mismatches on KVM-owned feature flags.
 182	 *
 183	 * Note!  @e2 and @nent track the _old_ CPUID entries!
 184	 */
 185	kvm_update_cpuid_runtime(vcpu);
 186	kvm_apply_cpuid_pv_features_quirk(vcpu);
 187
 188	if (nent != vcpu->arch.cpuid_nent)
 189		return -EINVAL;
 190
 191	for (i = 0; i < nent; i++) {
 192		orig = &vcpu->arch.cpuid_entries[i];
 193		if (e2[i].function != orig->function ||
 194		    e2[i].index != orig->index ||
 195		    e2[i].flags != orig->flags ||
 196		    e2[i].eax != orig->eax || e2[i].ebx != orig->ebx ||
 197		    e2[i].ecx != orig->ecx || e2[i].edx != orig->edx)
 198			return -EINVAL;
 199	}
 200
 201	return 0;
 202}
 203
 204static struct kvm_hypervisor_cpuid kvm_get_hypervisor_cpuid(struct kvm_vcpu *vcpu,
 205							    const char *sig)
 206{
 207	struct kvm_hypervisor_cpuid cpuid = {};
 208	struct kvm_cpuid_entry2 *entry;
 209	u32 base;
 210
 211	for_each_possible_cpuid_base_hypervisor(base) {
 212		entry = kvm_find_cpuid_entry(vcpu, base);
 213
 214		if (entry) {
 215			u32 signature[3];
 216
 217			signature[0] = entry->ebx;
 218			signature[1] = entry->ecx;
 219			signature[2] = entry->edx;
 220
 221			if (!memcmp(signature, sig, sizeof(signature))) {
 222				cpuid.base = base;
 223				cpuid.limit = entry->eax;
 224				break;
 225			}
 226		}
 227	}
 228
 229	return cpuid;
 230}
 231
 232static u32 kvm_apply_cpuid_pv_features_quirk(struct kvm_vcpu *vcpu)
 233{
 234	struct kvm_hypervisor_cpuid kvm_cpuid;
 235	struct kvm_cpuid_entry2 *best;
 236
 237	kvm_cpuid = kvm_get_hypervisor_cpuid(vcpu, KVM_SIGNATURE);
 238	if (!kvm_cpuid.base)
 239		return 0;
 240
 241	best = kvm_find_cpuid_entry(vcpu, kvm_cpuid.base | KVM_CPUID_FEATURES);
 242	if (!best)
 243		return 0;
 244
 245	if (kvm_hlt_in_guest(vcpu->kvm))
 246		best->eax &= ~(1 << KVM_FEATURE_PV_UNHALT);
 247
 248	return best->eax;
 249}
 250
 251/*
 252 * Calculate guest's supported XCR0 taking into account guest CPUID data and
 253 * KVM's supported XCR0 (comprised of host's XCR0 and KVM_SUPPORTED_XCR0).
 254 */
 255static u64 cpuid_get_supported_xcr0(struct kvm_vcpu *vcpu)
 256{
 257	struct kvm_cpuid_entry2 *best;
 258
 259	best = kvm_find_cpuid_entry_index(vcpu, 0xd, 0);
 260	if (!best)
 261		return 0;
 262
 263	return (best->eax | ((u64)best->edx << 32)) & kvm_caps.supported_xcr0;
 264}
 265
 266static u64 cpuid_get_supported_xss(struct kvm_vcpu *vcpu)
 267{
 268	struct kvm_cpuid_entry2 *best;
 269
 270	best = kvm_find_cpuid_entry_index(vcpu, 0xd, 1);
 271	if (!best)
 272		return 0;
 273
 274	return (best->ecx | ((u64)best->edx << 32)) & kvm_caps.supported_xss;
 275}
 276
 277static __always_inline void kvm_update_feature_runtime(struct kvm_vcpu *vcpu,
 278						       struct kvm_cpuid_entry2 *entry,
 279						       unsigned int x86_feature,
 280						       bool has_feature)
 281{
 282	cpuid_entry_change(entry, x86_feature, has_feature);
 283	guest_cpu_cap_change(vcpu, x86_feature, has_feature);
 284}
 285
 286static void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu)
 287{
 288	struct kvm_cpuid_entry2 *best;
 289
 290	vcpu->arch.cpuid_dynamic_bits_dirty = false;
 291
 292	best = kvm_find_cpuid_entry(vcpu, 1);
 293	if (best) {
 294		kvm_update_feature_runtime(vcpu, best, X86_FEATURE_OSXSAVE,
 295					   kvm_is_cr4_bit_set(vcpu, X86_CR4_OSXSAVE));
 296
 297		kvm_update_feature_runtime(vcpu, best, X86_FEATURE_APIC,
 298					   vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE);
 299
 300		if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT))
 301			kvm_update_feature_runtime(vcpu, best, X86_FEATURE_MWAIT,
 302						   vcpu->arch.ia32_misc_enable_msr &
 303						   MSR_IA32_MISC_ENABLE_MWAIT);
 304	}
 305
 306	best = kvm_find_cpuid_entry_index(vcpu, 7, 0);
 307	if (best)
 308		kvm_update_feature_runtime(vcpu, best, X86_FEATURE_OSPKE,
 309					   kvm_is_cr4_bit_set(vcpu, X86_CR4_PKE));
 310
 311
 312	best = kvm_find_cpuid_entry_index(vcpu, 0xD, 0);
 313	if (best)
 314		best->ebx = xstate_required_size(vcpu->arch.xcr0, false);
 315
 316	best = kvm_find_cpuid_entry_index(vcpu, 0xD, 1);
 317	if (best && (cpuid_entry_has(best, X86_FEATURE_XSAVES) ||
 318		     cpuid_entry_has(best, X86_FEATURE_XSAVEC)))
 319		best->ebx = xstate_required_size(vcpu->arch.xcr0 |
 320						 vcpu->arch.ia32_xss, true);
 321}
 322
 323static bool kvm_cpuid_has_hyperv(struct kvm_vcpu *vcpu)
 324{
 325#ifdef CONFIG_KVM_HYPERV
 326	struct kvm_cpuid_entry2 *entry;
 327
 328	entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_INTERFACE);
 329	return entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX;
 330#else
 331	return false;
 332#endif
 333}
 334
 335static bool guest_cpuid_is_amd_or_hygon(struct kvm_vcpu *vcpu)
 336{
 337	struct kvm_cpuid_entry2 *entry;
 338
 339	entry = kvm_find_cpuid_entry(vcpu, 0);
 340	if (!entry)
 341		return false;
 342
 343	return is_guest_vendor_amd(entry->ebx, entry->ecx, entry->edx) ||
 344	       is_guest_vendor_hygon(entry->ebx, entry->ecx, entry->edx);
 345}
 346
 347/*
 348 * This isn't truly "unsafe", but except for the cpu_caps initialization code,
 349 * all register lookups should use __cpuid_entry_get_reg(), which provides
 350 * compile-time validation of the input.
 351 */
 352static u32 cpuid_get_reg_unsafe(struct kvm_cpuid_entry2 *entry, u32 reg)
 353{
 354	switch (reg) {
 355	case CPUID_EAX:
 356		return entry->eax;
 357	case CPUID_EBX:
 358		return entry->ebx;
 359	case CPUID_ECX:
 360		return entry->ecx;
 361	case CPUID_EDX:
 362		return entry->edx;
 363	default:
 364		WARN_ON_ONCE(1);
 365		return 0;
 366	}
 367}
 368
 369static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func,
 370			       bool include_partially_emulated);
 371
 372void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
 373{
 374	struct kvm_lapic *apic = vcpu->arch.apic;
 375	struct kvm_cpuid_entry2 *best;
 376	struct kvm_cpuid_entry2 *entry;
 377	bool allow_gbpages;
 378	int i;
 379
 380	memset(vcpu->arch.cpu_caps, 0, sizeof(vcpu->arch.cpu_caps));
 381	BUILD_BUG_ON(ARRAY_SIZE(reverse_cpuid) != NR_KVM_CPU_CAPS);
 382
 383	/*
 384	 * Reset guest capabilities to userspace's guest CPUID definition, i.e.
 385	 * honor userspace's definition for features that don't require KVM or
 386	 * hardware management/support (or that KVM simply doesn't care about).
 387	 */
 388	for (i = 0; i < NR_KVM_CPU_CAPS; i++) {
 389		const struct cpuid_reg cpuid = reverse_cpuid[i];
 390		struct kvm_cpuid_entry2 emulated;
 391
 392		if (!cpuid.function)
 393			continue;
 394
 395		entry = kvm_find_cpuid_entry_index(vcpu, cpuid.function, cpuid.index);
 396		if (!entry)
 397			continue;
 398
 399		cpuid_func_emulated(&emulated, cpuid.function, true);
 400
 401		/*
 402		 * A vCPU has a feature if it's supported by KVM and is enabled
 403		 * in guest CPUID.  Note, this includes features that are
 404		 * supported by KVM but aren't advertised to userspace!
 405		 */
 406		vcpu->arch.cpu_caps[i] = kvm_cpu_caps[i] |
 407					 cpuid_get_reg_unsafe(&emulated, cpuid.reg);
 408		vcpu->arch.cpu_caps[i] &= cpuid_get_reg_unsafe(entry, cpuid.reg);
 409	}
 410
 411	kvm_update_cpuid_runtime(vcpu);
 412
 413	/*
 414	 * If TDP is enabled, let the guest use GBPAGES if they're supported in
 415	 * hardware.  The hardware page walker doesn't let KVM disable GBPAGES,
 416	 * i.e. won't treat them as reserved, and KVM doesn't redo the GVA->GPA
 417	 * walk for performance and complexity reasons.  Not to mention KVM
 418	 * _can't_ solve the problem because GVA->GPA walks aren't visible to
 419	 * KVM once a TDP translation is installed.  Mimic hardware behavior so
 420	 * that KVM's is at least consistent, i.e. doesn't randomly inject #PF.
 421	 * If TDP is disabled, honor *only* guest CPUID as KVM has full control
 422	 * and can install smaller shadow pages if the host lacks 1GiB support.
 423	 */
 424	allow_gbpages = tdp_enabled ? boot_cpu_has(X86_FEATURE_GBPAGES) :
 425				      guest_cpu_cap_has(vcpu, X86_FEATURE_GBPAGES);
 426	guest_cpu_cap_change(vcpu, X86_FEATURE_GBPAGES, allow_gbpages);
 427
 428	best = kvm_find_cpuid_entry(vcpu, 1);
 429	if (best && apic) {
 430		if (cpuid_entry_has(best, X86_FEATURE_TSC_DEADLINE_TIMER))
 431			apic->lapic_timer.timer_mode_mask = 3 << 17;
 432		else
 433			apic->lapic_timer.timer_mode_mask = 1 << 17;
 434
 435		kvm_apic_set_version(vcpu);
 436	}
 437
 438	vcpu->arch.guest_supported_xcr0 = cpuid_get_supported_xcr0(vcpu);
 439	vcpu->arch.guest_supported_xss = cpuid_get_supported_xss(vcpu);
 440
 441	vcpu->arch.pv_cpuid.features = kvm_apply_cpuid_pv_features_quirk(vcpu);
 442
 443	vcpu->arch.is_amd_compatible = guest_cpuid_is_amd_or_hygon(vcpu);
 444	vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
 445	vcpu->arch.reserved_gpa_bits = kvm_vcpu_reserved_gpa_bits_raw(vcpu);
 446
 447	kvm_pmu_refresh(vcpu);
 448
 449#define __kvm_cpu_cap_has(UNUSED_, f) kvm_cpu_cap_has(f)
 450	vcpu->arch.cr4_guest_rsvd_bits = __cr4_reserved_bits(__kvm_cpu_cap_has, UNUSED_) |
 451					 __cr4_reserved_bits(guest_cpu_cap_has, vcpu);
 452#undef __kvm_cpu_cap_has
 453
 454	kvm_hv_set_cpuid(vcpu, kvm_cpuid_has_hyperv(vcpu));
 455
 456	/* Invoke the vendor callback only after the above state is updated. */
 457	kvm_x86_call(vcpu_after_set_cpuid)(vcpu);
 458
 459	/*
 460	 * Except for the MMU, which needs to do its thing any vendor specific
 461	 * adjustments to the reserved GPA bits.
 462	 */
 463	kvm_mmu_after_set_cpuid(vcpu);
 464
 465	kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu);
 466}
 467
 468int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu)
 469{
 470	struct kvm_cpuid_entry2 *best;
 471
 472	best = kvm_find_cpuid_entry(vcpu, 0x80000000);
 473	if (!best || best->eax < 0x80000008)
 474		goto not_found;
 475	best = kvm_find_cpuid_entry(vcpu, 0x80000008);
 476	if (best)
 477		return best->eax & 0xff;
 478not_found:
 479	return 36;
 480}
 481
 482int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu)
 483{
 484	struct kvm_cpuid_entry2 *best;
 485
 486	best = kvm_find_cpuid_entry(vcpu, 0x80000000);
 487	if (!best || best->eax < 0x80000008)
 488		goto not_found;
 489	best = kvm_find_cpuid_entry(vcpu, 0x80000008);
 490	if (best)
 491		return (best->eax >> 16) & 0xff;
 492not_found:
 493	return 0;
 494}
 495
 496/*
 497 * This "raw" version returns the reserved GPA bits without any adjustments for
 498 * encryption technologies that usurp bits.  The raw mask should be used if and
 499 * only if hardware does _not_ strip the usurped bits, e.g. in virtual MTRRs.
 500 */
 501u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu)
 502{
 503	return rsvd_bits(cpuid_maxphyaddr(vcpu), 63);
 504}
 505
 506static int kvm_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid_entry2 *e2,
 507                        int nent)
 508{
 509	u32 vcpu_caps[NR_KVM_CPU_CAPS];
 510	int r;
 511
 512	/*
 513	 * Apply pending runtime CPUID updates to the current CPUID entries to
 514	 * avoid false positives due to mismatches on KVM-owned feature flags.
 515	 */
 516	if (vcpu->arch.cpuid_dynamic_bits_dirty)
 517		kvm_update_cpuid_runtime(vcpu);
 518
 519	/*
 520	 * Swap the existing (old) entries with the incoming (new) entries in
 521	 * order to massage the new entries, e.g. to account for dynamic bits
 522	 * that KVM controls, without losing the current guest CPUID, which KVM
 523	 * needs to preserve in order to unwind on failure.
 524	 *
 525	 * Similarly, save the vCPU's current cpu_caps so that the capabilities
 526	 * can be updated alongside the CPUID entries when performing runtime
 527	 * updates.  Full initialization is done if and only if the vCPU hasn't
 528	 * run, i.e. only if userspace is potentially changing CPUID features.
 529	 */
 530	swap(vcpu->arch.cpuid_entries, e2);
 531	swap(vcpu->arch.cpuid_nent, nent);
 532
 533	memcpy(vcpu_caps, vcpu->arch.cpu_caps, sizeof(vcpu_caps));
 534	BUILD_BUG_ON(sizeof(vcpu_caps) != sizeof(vcpu->arch.cpu_caps));
 535
 536	/*
 537	 * KVM does not correctly handle changing guest CPUID after KVM_RUN, as
 538	 * MAXPHYADDR, GBPAGES support, AMD reserved bit behavior, etc.. aren't
 539	 * tracked in kvm_mmu_page_role.  As a result, KVM may miss guest page
 540	 * faults due to reusing SPs/SPTEs. In practice no sane VMM mucks with
 541	 * the core vCPU model on the fly. It would've been better to forbid any
 542	 * KVM_SET_CPUID{,2} calls after KVM_RUN altogether but unfortunately
 543	 * some VMMs (e.g. QEMU) reuse vCPU fds for CPU hotplug/unplug and do
 544	 * KVM_SET_CPUID{,2} again. To support this legacy behavior, check
 545	 * whether the supplied CPUID data is equal to what's already set.
 546	 */
 547	if (kvm_vcpu_has_run(vcpu)) {
 548		r = kvm_cpuid_check_equal(vcpu, e2, nent);
 549		if (r)
 550			goto err;
 551		goto success;
 552	}
 553
 554#ifdef CONFIG_KVM_HYPERV
 555	if (kvm_cpuid_has_hyperv(vcpu)) {
 556		r = kvm_hv_vcpu_init(vcpu);
 557		if (r)
 558			goto err;
 559	}
 560#endif
 561
 562	r = kvm_check_cpuid(vcpu);
 563	if (r)
 564		goto err;
 565
 566#ifdef CONFIG_KVM_XEN
 567	vcpu->arch.xen.cpuid = kvm_get_hypervisor_cpuid(vcpu, XEN_SIGNATURE);
 568#endif
 569	kvm_vcpu_after_set_cpuid(vcpu);
 570
 571success:
 572	kvfree(e2);
 573	return 0;
 574
 575err:
 576	memcpy(vcpu->arch.cpu_caps, vcpu_caps, sizeof(vcpu_caps));
 577	swap(vcpu->arch.cpuid_entries, e2);
 578	swap(vcpu->arch.cpuid_nent, nent);
 579	return r;
 580}
 581
 582/* when an old userspace process fills a new kernel module */
 583int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
 584			     struct kvm_cpuid *cpuid,
 585			     struct kvm_cpuid_entry __user *entries)
 586{
 587	int r, i;
 588	struct kvm_cpuid_entry *e = NULL;
 589	struct kvm_cpuid_entry2 *e2 = NULL;
 590
 591	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 592		return -E2BIG;
 593
 594	if (cpuid->nent) {
 595		e = vmemdup_array_user(entries, cpuid->nent, sizeof(*e));
 596		if (IS_ERR(e))
 597			return PTR_ERR(e);
 598
 599		e2 = kvmalloc_array(cpuid->nent, sizeof(*e2), GFP_KERNEL_ACCOUNT);
 600		if (!e2) {
 601			r = -ENOMEM;
 602			goto out_free_cpuid;
 603		}
 604	}
 605	for (i = 0; i < cpuid->nent; i++) {
 606		e2[i].function = e[i].function;
 607		e2[i].eax = e[i].eax;
 608		e2[i].ebx = e[i].ebx;
 609		e2[i].ecx = e[i].ecx;
 610		e2[i].edx = e[i].edx;
 611		e2[i].index = 0;
 612		e2[i].flags = 0;
 613		e2[i].padding[0] = 0;
 614		e2[i].padding[1] = 0;
 615		e2[i].padding[2] = 0;
 616	}
 617
 618	r = kvm_set_cpuid(vcpu, e2, cpuid->nent);
 619	if (r)
 620		kvfree(e2);
 621
 622out_free_cpuid:
 623	kvfree(e);
 624
 625	return r;
 626}
 627
 628int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
 629			      struct kvm_cpuid2 *cpuid,
 630			      struct kvm_cpuid_entry2 __user *entries)
 631{
 632	struct kvm_cpuid_entry2 *e2 = NULL;
 633	int r;
 634
 635	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
 636		return -E2BIG;
 637
 638	if (cpuid->nent) {
 639		e2 = vmemdup_array_user(entries, cpuid->nent, sizeof(*e2));
 640		if (IS_ERR(e2))
 641			return PTR_ERR(e2);
 642	}
 643
 644	r = kvm_set_cpuid(vcpu, e2, cpuid->nent);
 645	if (r)
 646		kvfree(e2);
 647
 648	return r;
 649}
 650
 651int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
 652			      struct kvm_cpuid2 *cpuid,
 653			      struct kvm_cpuid_entry2 __user *entries)
 654{
 655	if (cpuid->nent < vcpu->arch.cpuid_nent)
 656		return -E2BIG;
 657
 658	if (vcpu->arch.cpuid_dynamic_bits_dirty)
 659		kvm_update_cpuid_runtime(vcpu);
 660
 661	if (copy_to_user(entries, vcpu->arch.cpuid_entries,
 662			 vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
 663		return -EFAULT;
 664
 665	cpuid->nent = vcpu->arch.cpuid_nent;
 666	return 0;
 667}
 668
 669static __always_inline u32 raw_cpuid_get(struct cpuid_reg cpuid)
 670{
 671	struct kvm_cpuid_entry2 entry;
 672	u32 base;
 673
 674	/*
 675	 * KVM only supports features defined by Intel (0x0), AMD (0x80000000),
 676	 * and Centaur (0xc0000000).  WARN if a feature for new vendor base is
 677	 * defined, as this and other code would need to be updated.
 678	 */
 679	base = cpuid.function & 0xffff0000;
 680	if (WARN_ON_ONCE(base && base != 0x80000000 && base != 0xc0000000))
 681		return 0;
 682
 683	if (cpuid_eax(base) < cpuid.function)
 684		return 0;
 685
 686	cpuid_count(cpuid.function, cpuid.index,
 687		    &entry.eax, &entry.ebx, &entry.ecx, &entry.edx);
 688
 689	return *__cpuid_entry_get_reg(&entry, cpuid.reg);
 690}
 691
 692/*
 693 * For kernel-defined leafs, mask KVM's supported feature set with the kernel's
 694 * capabilities as well as raw CPUID.  For KVM-defined leafs, consult only raw
 695 * CPUID, as KVM is the one and only authority (in the kernel).
 696 */
 697#define kvm_cpu_cap_init(leaf, feature_initializers...)			\
 698do {									\
 699	const struct cpuid_reg cpuid = x86_feature_cpuid(leaf * 32);	\
 700	const u32 __maybe_unused kvm_cpu_cap_init_in_progress = leaf;	\
 701	const u32 *kernel_cpu_caps = boot_cpu_data.x86_capability;	\
 702	u32 kvm_cpu_cap_passthrough = 0;				\
 703	u32 kvm_cpu_cap_synthesized = 0;				\
 704	u32 kvm_cpu_cap_emulated = 0;					\
 705	u32 kvm_cpu_cap_features = 0;					\
 706									\
 707	feature_initializers						\
 708									\
 709	kvm_cpu_caps[leaf] = kvm_cpu_cap_features;			\
 710									\
 711	if (leaf < NCAPINTS)						\
 712		kvm_cpu_caps[leaf] &= kernel_cpu_caps[leaf];		\
 713									\
 714	kvm_cpu_caps[leaf] |= kvm_cpu_cap_passthrough;			\
 715	kvm_cpu_caps[leaf] &= (raw_cpuid_get(cpuid) |			\
 716			       kvm_cpu_cap_synthesized);		\
 717	kvm_cpu_caps[leaf] |= kvm_cpu_cap_emulated;			\
 718} while (0)
 719
 720/*
 721 * Assert that the feature bit being declared, e.g. via F(), is in the CPUID
 722 * word that's being initialized.  Exempt 0x8000_0001.EDX usage of 0x1.EDX
 723 * features, as AMD duplicated many 0x1.EDX features into 0x8000_0001.EDX.
 724 */
 725#define KVM_VALIDATE_CPU_CAP_USAGE(name)				\
 726do {									\
 727	u32 __leaf = __feature_leaf(X86_FEATURE_##name);		\
 728									\
 729	BUILD_BUG_ON(__leaf != kvm_cpu_cap_init_in_progress);		\
 730} while (0)
 731
 732#define F(name)							\
 733({								\
 734	KVM_VALIDATE_CPU_CAP_USAGE(name);			\
 735	kvm_cpu_cap_features |= feature_bit(name);		\
 736})
 737
 738/* Scattered Flag - For features that are scattered by cpufeatures.h. */
 739#define SCATTERED_F(name)					\
 740({								\
 741	BUILD_BUG_ON(X86_FEATURE_##name >= MAX_CPU_FEATURES);	\
 742	KVM_VALIDATE_CPU_CAP_USAGE(name);			\
 743	if (boot_cpu_has(X86_FEATURE_##name))			\
 744		F(name);					\
 745})
 746
 747/* Features that KVM supports only on 64-bit kernels. */
 748#define X86_64_F(name)						\
 749({								\
 750	KVM_VALIDATE_CPU_CAP_USAGE(name);			\
 751	if (IS_ENABLED(CONFIG_X86_64))				\
 752		F(name);					\
 753})
 754
 755/*
 756 * Emulated Feature - For features that KVM emulates in software irrespective
 757 * of host CPU/kernel support.
 758 */
 759#define EMULATED_F(name)					\
 760({								\
 761	kvm_cpu_cap_emulated |= feature_bit(name);		\
 762	F(name);						\
 763})
 764
 765/*
 766 * Synthesized Feature - For features that are synthesized into boot_cpu_data,
 767 * i.e. may not be present in the raw CPUID, but can still be advertised to
 768 * userspace.  Primarily used for mitigation related feature flags.
 769 */
 770#define SYNTHESIZED_F(name)					\
 771({								\
 772	kvm_cpu_cap_synthesized |= feature_bit(name);		\
 773	F(name);						\
 774})
 775
 776/*
 777 * Passthrough Feature - For features that KVM supports based purely on raw
 778 * hardware CPUID, i.e. that KVM virtualizes even if the host kernel doesn't
 779 * use the feature.  Simply force set the feature in KVM's capabilities, raw
 780 * CPUID support will be factored in by kvm_cpu_cap_mask().
 781 */
 782#define PASSTHROUGH_F(name)					\
 783({								\
 784	kvm_cpu_cap_passthrough |= feature_bit(name);		\
 785	F(name);						\
 786})
 787
 788/*
 789 * Aliased Features - For features in 0x8000_0001.EDX that are duplicates of
 790 * identical 0x1.EDX features, and thus are aliased from 0x1 to 0x8000_0001.
 791 */
 792#define ALIASED_1_EDX_F(name)							\
 793({										\
 794	BUILD_BUG_ON(__feature_leaf(X86_FEATURE_##name) != CPUID_1_EDX);	\
 795	BUILD_BUG_ON(kvm_cpu_cap_init_in_progress != CPUID_8000_0001_EDX);	\
 796	kvm_cpu_cap_features |= feature_bit(name);				\
 797})
 798
 799/*
 800 * Vendor Features - For features that KVM supports, but are added in later
 801 * because they require additional vendor enabling.
 802 */
 803#define VENDOR_F(name)						\
 804({								\
 805	KVM_VALIDATE_CPU_CAP_USAGE(name);			\
 806})
 807
 808/*
 809 * Runtime Features - For features that KVM dynamically sets/clears at runtime,
 810 * e.g. when CR4 changes, but which are never advertised to userspace.
 811 */
 812#define RUNTIME_F(name)						\
 813({								\
 814	KVM_VALIDATE_CPU_CAP_USAGE(name);			\
 815})
 816
 817/*
 818 * Undefine the MSR bit macro to avoid token concatenation issues when
 819 * processing X86_FEATURE_SPEC_CTRL_SSBD.
 820 */
 821#undef SPEC_CTRL_SSBD
 822
 823/* DS is defined by ptrace-abi.h on 32-bit builds. */
 824#undef DS
 825
 826void kvm_set_cpu_caps(void)
 827{
 828	memset(kvm_cpu_caps, 0, sizeof(kvm_cpu_caps));
 829
 830	BUILD_BUG_ON(sizeof(kvm_cpu_caps) - (NKVMCAPINTS * sizeof(*kvm_cpu_caps)) >
 831		     sizeof(boot_cpu_data.x86_capability));
 832
 833	kvm_cpu_cap_init(CPUID_1_ECX,
 834		F(XMM3),
 835		F(PCLMULQDQ),
 836		VENDOR_F(DTES64),
 837		/*
 838		 * NOTE: MONITOR (and MWAIT) are emulated as NOP, but *not*
 839		 * advertised to guests via CPUID!  MWAIT is also technically a
 840		 * runtime flag thanks to IA32_MISC_ENABLES; mark it as such so
 841		 * that KVM is aware that it's a known, unadvertised flag.
 842		 */
 843		RUNTIME_F(MWAIT),
 844		/* DS-CPL */
 845		VENDOR_F(VMX),
 846		/* SMX, EST */
 847		/* TM2 */
 848		F(SSSE3),
 849		/* CNXT-ID */
 850		/* Reserved */
 851		F(FMA),
 852		F(CX16),
 853		/* xTPR Update */
 854		F(PDCM),
 855		F(PCID),
 856		/* Reserved, DCA */
 857		F(XMM4_1),
 858		F(XMM4_2),
 859		EMULATED_F(X2APIC),
 860		F(MOVBE),
 861		F(POPCNT),
 862		EMULATED_F(TSC_DEADLINE_TIMER),
 863		F(AES),
 864		F(XSAVE),
 865		RUNTIME_F(OSXSAVE),
 866		F(AVX),
 867		F(F16C),
 868		F(RDRAND),
 869		EMULATED_F(HYPERVISOR),
 870	);
 871
 872	kvm_cpu_cap_init(CPUID_1_EDX,
 873		F(FPU),
 874		F(VME),
 875		F(DE),
 876		F(PSE),
 877		F(TSC),
 878		F(MSR),
 879		F(PAE),
 880		F(MCE),
 881		F(CX8),
 882		F(APIC),
 883		/* Reserved */
 884		F(SEP),
 885		F(MTRR),
 886		F(PGE),
 887		F(MCA),
 888		F(CMOV),
 889		F(PAT),
 890		F(PSE36),
 891		/* PSN */
 892		F(CLFLUSH),
 893		/* Reserved */
 894		VENDOR_F(DS),
 895		/* ACPI */
 896		F(MMX),
 897		F(FXSR),
 898		F(XMM),
 899		F(XMM2),
 900		F(SELFSNOOP),
 901		/* HTT, TM, Reserved, PBE */
 902	);
 903
 904	kvm_cpu_cap_init(CPUID_7_0_EBX,
 905		F(FSGSBASE),
 906		EMULATED_F(TSC_ADJUST),
 907		F(SGX),
 908		F(BMI1),
 909		F(HLE),
 910		F(AVX2),
 911		F(FDP_EXCPTN_ONLY),
 912		F(SMEP),
 913		F(BMI2),
 914		F(ERMS),
 915		F(INVPCID),
 916		F(RTM),
 917		F(ZERO_FCS_FDS),
 918		VENDOR_F(MPX),
 919		F(AVX512F),
 920		F(AVX512DQ),
 921		F(RDSEED),
 922		F(ADX),
 923		F(SMAP),
 924		F(AVX512IFMA),
 925		F(CLFLUSHOPT),
 926		F(CLWB),
 927		VENDOR_F(INTEL_PT),
 928		F(AVX512PF),
 929		F(AVX512ER),
 930		F(AVX512CD),
 931		F(SHA_NI),
 932		F(AVX512BW),
 933		F(AVX512VL),
 934	);
 935
 936	kvm_cpu_cap_init(CPUID_7_ECX,
 937		F(AVX512VBMI),
 938		PASSTHROUGH_F(LA57),
 939		F(PKU),
 940		RUNTIME_F(OSPKE),
 941		F(RDPID),
 942		F(AVX512_VPOPCNTDQ),
 943		F(UMIP),
 944		F(AVX512_VBMI2),
 945		F(GFNI),
 946		F(VAES),
 947		F(VPCLMULQDQ),
 948		F(AVX512_VNNI),
 949		F(AVX512_BITALG),
 950		F(CLDEMOTE),
 951		F(MOVDIRI),
 952		F(MOVDIR64B),
 953		VENDOR_F(WAITPKG),
 954		F(SGX_LC),
 955		F(BUS_LOCK_DETECT),
 956		X86_64_F(SHSTK),
 957	);
 958
 959	/*
 960	 * PKU not yet implemented for shadow paging and requires OSPKE
 961	 * to be set on the host. Clear it if that is not the case
 962	 */
 963	if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE))
 964		kvm_cpu_cap_clear(X86_FEATURE_PKU);
 965
 966	/*
 967	 * Shadow Stacks aren't implemented in the Shadow MMU.  Shadow Stack
 968	 * accesses require "magic" Writable=0,Dirty=1 protection, which KVM
 969	 * doesn't know how to emulate or map.
 970	 */
 971	if (!tdp_enabled)
 972		kvm_cpu_cap_clear(X86_FEATURE_SHSTK);
 973
 974	kvm_cpu_cap_init(CPUID_7_EDX,
 975		F(AVX512_4VNNIW),
 976		F(AVX512_4FMAPS),
 977		F(SPEC_CTRL),
 978		F(SPEC_CTRL_SSBD),
 979		EMULATED_F(ARCH_CAPABILITIES),
 980		F(INTEL_STIBP),
 981		F(MD_CLEAR),
 982		F(AVX512_VP2INTERSECT),
 983		F(FSRM),
 984		F(SERIALIZE),
 985		F(TSXLDTRK),
 986		F(AVX512_FP16),
 987		F(AMX_TILE),
 988		F(AMX_INT8),
 989		F(AMX_BF16),
 990		F(FLUSH_L1D),
 991		F(IBT),
 992	);
 993
 994	/*
 995	 * Disable support for IBT and SHSTK if KVM is configured to emulate
 996	 * accesses to reserved GPAs, as KVM's emulator doesn't support IBT or
 997	 * SHSTK, nor does KVM handle Shadow Stack #PFs (see above).
 998	 */
 999	if (allow_smaller_maxphyaddr) {
1000		kvm_cpu_cap_clear(X86_FEATURE_SHSTK);
1001		kvm_cpu_cap_clear(X86_FEATURE_IBT);
1002	}
1003
1004	if (boot_cpu_has(X86_FEATURE_AMD_IBPB_RET) &&
1005	    boot_cpu_has(X86_FEATURE_AMD_IBPB) &&
1006	    boot_cpu_has(X86_FEATURE_AMD_IBRS))
1007		kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL);
1008	if (boot_cpu_has(X86_FEATURE_STIBP))
1009		kvm_cpu_cap_set(X86_FEATURE_INTEL_STIBP);
1010	if (boot_cpu_has(X86_FEATURE_AMD_SSBD))
1011		kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD);
1012
1013	kvm_cpu_cap_init(CPUID_7_1_EAX,
1014		F(SHA512),
1015		F(SM3),
1016		F(SM4),
1017		F(AVX_VNNI),
1018		F(AVX512_BF16),
1019		F(CMPCCXADD),
1020		F(FZRM),
1021		F(FSRS),
1022		F(FSRC),
1023		F(WRMSRNS),
1024		X86_64_F(LKGS),
1025		F(AMX_FP16),
1026		F(AVX_IFMA),
1027		F(LAM),
1028	);
1029
1030	kvm_cpu_cap_init(CPUID_7_1_ECX,
1031		SCATTERED_F(MSR_IMM),
1032	);
1033
1034	kvm_cpu_cap_init(CPUID_7_1_EDX,
1035		F(AVX_VNNI_INT8),
1036		F(AVX_NE_CONVERT),
1037		F(AMX_COMPLEX),
1038		F(AVX_VNNI_INT16),
1039		F(PREFETCHITI),
1040		F(AVX10),
1041	);
1042
1043	kvm_cpu_cap_init(CPUID_7_2_EDX,
1044		F(INTEL_PSFD),
1045		F(IPRED_CTRL),
1046		F(RRSBA_CTRL),
1047		F(DDPD_U),
1048		F(BHI_CTRL),
1049		F(MCDT_NO),
1050	);
1051
1052	kvm_cpu_cap_init(CPUID_D_1_EAX,
1053		F(XSAVEOPT),
1054		F(XSAVEC),
1055		F(XGETBV1),
1056		F(XSAVES),
1057		X86_64_F(XFD),
1058	);
1059
1060	kvm_cpu_cap_init(CPUID_12_EAX,
1061		SCATTERED_F(SGX1),
1062		SCATTERED_F(SGX2),
1063		SCATTERED_F(SGX_EDECCSSA),
1064	);
1065
1066	kvm_cpu_cap_init(CPUID_24_0_EBX,
1067		F(AVX10_128),
1068		F(AVX10_256),
1069		F(AVX10_512),
1070	);
1071
1072	kvm_cpu_cap_init(CPUID_8000_0001_ECX,
1073		F(LAHF_LM),
1074		F(CMP_LEGACY),
1075		VENDOR_F(SVM),
1076		/* ExtApicSpace */
1077		F(CR8_LEGACY),
1078		F(ABM),
1079		F(SSE4A),
1080		F(MISALIGNSSE),
1081		F(3DNOWPREFETCH),
1082		F(OSVW),
1083		/* IBS */
1084		F(XOP),
1085		/* SKINIT, WDT, LWP */
1086		F(FMA4),
1087		F(TBM),
1088		F(TOPOEXT),
1089		VENDOR_F(PERFCTR_CORE),
1090	);
1091
1092	kvm_cpu_cap_init(CPUID_8000_0001_EDX,
1093		ALIASED_1_EDX_F(FPU),
1094		ALIASED_1_EDX_F(VME),
1095		ALIASED_1_EDX_F(DE),
1096		ALIASED_1_EDX_F(PSE),
1097		ALIASED_1_EDX_F(TSC),
1098		ALIASED_1_EDX_F(MSR),
1099		ALIASED_1_EDX_F(PAE),
1100		ALIASED_1_EDX_F(MCE),
1101		ALIASED_1_EDX_F(CX8),
1102		ALIASED_1_EDX_F(APIC),
1103		/* Reserved */
1104		F(SYSCALL),
1105		ALIASED_1_EDX_F(MTRR),
1106		ALIASED_1_EDX_F(PGE),
1107		ALIASED_1_EDX_F(MCA),
1108		ALIASED_1_EDX_F(CMOV),
1109		ALIASED_1_EDX_F(PAT),
1110		ALIASED_1_EDX_F(PSE36),
1111		/* Reserved */
1112		F(NX),
1113		/* Reserved */
1114		F(MMXEXT),
1115		ALIASED_1_EDX_F(MMX),
1116		ALIASED_1_EDX_F(FXSR),
1117		F(FXSR_OPT),
1118		X86_64_F(GBPAGES),
1119		F(RDTSCP),
1120		/* Reserved */
1121		X86_64_F(LM),
1122		F(3DNOWEXT),
1123		F(3DNOW),
1124	);
1125
1126	if (!tdp_enabled && IS_ENABLED(CONFIG_X86_64))
1127		kvm_cpu_cap_set(X86_FEATURE_GBPAGES);
1128
1129	kvm_cpu_cap_init(CPUID_8000_0007_EDX,
1130		SCATTERED_F(CONSTANT_TSC),
1131	);
1132
1133	kvm_cpu_cap_init(CPUID_8000_0008_EBX,
1134		F(CLZERO),
1135		F(XSAVEERPTR),
1136		F(WBNOINVD),
1137		F(AMD_IBPB),
1138		F(AMD_IBRS),
1139		F(AMD_SSBD),
1140		F(VIRT_SSBD),
1141		F(AMD_SSB_NO),
1142		F(AMD_STIBP),
1143		F(AMD_STIBP_ALWAYS_ON),
1144		F(AMD_IBRS_SAME_MODE),
1145		PASSTHROUGH_F(EFER_LMSLE_MBZ),
1146		F(AMD_PSFD),
1147		F(AMD_IBPB_RET),
1148	);
1149
1150	/*
1151	 * AMD has separate bits for each SPEC_CTRL bit.
1152	 * arch/x86/kernel/cpu/bugs.c is kind enough to
1153	 * record that in cpufeatures so use them.
1154	 */
1155	if (boot_cpu_has(X86_FEATURE_IBPB)) {
1156		kvm_cpu_cap_set(X86_FEATURE_AMD_IBPB);
1157		if (boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
1158		    !boot_cpu_has_bug(X86_BUG_EIBRS_PBRSB))
1159			kvm_cpu_cap_set(X86_FEATURE_AMD_IBPB_RET);
1160	}
1161	if (boot_cpu_has(X86_FEATURE_IBRS))
1162		kvm_cpu_cap_set(X86_FEATURE_AMD_IBRS);
1163	if (boot_cpu_has(X86_FEATURE_STIBP))
1164		kvm_cpu_cap_set(X86_FEATURE_AMD_STIBP);
1165	if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
1166		kvm_cpu_cap_set(X86_FEATURE_AMD_SSBD);
1167	if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1168		kvm_cpu_cap_set(X86_FEATURE_AMD_SSB_NO);
1169	/*
1170	 * The preference is to use SPEC CTRL MSR instead of the
1171	 * VIRT_SPEC MSR.
1172	 */
1173	if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
1174	    !boot_cpu_has(X86_FEATURE_AMD_SSBD))
1175		kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
1176
1177	/* All SVM features required additional vendor module enabling. */
1178	kvm_cpu_cap_init(CPUID_8000_000A_EDX,
1179		VENDOR_F(NPT),
1180		VENDOR_F(VMCBCLEAN),
1181		VENDOR_F(FLUSHBYASID),
1182		VENDOR_F(NRIPS),
1183		VENDOR_F(TSCRATEMSR),
1184		VENDOR_F(V_VMSAVE_VMLOAD),
1185		VENDOR_F(LBRV),
1186		VENDOR_F(PAUSEFILTER),
1187		VENDOR_F(PFTHRESHOLD),
1188		VENDOR_F(VGIF),
1189		VENDOR_F(VNMI),
1190		VENDOR_F(SVME_ADDR_CHK),
1191	);
1192
1193	kvm_cpu_cap_init(CPUID_8000_001F_EAX,
1194		VENDOR_F(SME),
1195		VENDOR_F(SEV),
1196		/* VM_PAGE_FLUSH */
1197		VENDOR_F(SEV_ES),
1198		F(SME_COHERENT),
1199	);
1200
1201	kvm_cpu_cap_init(CPUID_8000_0021_EAX,
1202		F(NO_NESTED_DATA_BP),
1203		F(WRMSR_XX_BASE_NS),
1204		/*
1205		 * Synthesize "LFENCE is serializing" into the AMD-defined entry
1206		 * in KVM's supported CPUID, i.e. if the feature is reported as
1207		 * supported by the kernel.  LFENCE_RDTSC was a Linux-defined
1208		 * synthetic feature long before AMD joined the bandwagon, e.g.
1209		 * LFENCE is serializing on most CPUs that support SSE2.  On
1210		 * CPUs that don't support AMD's leaf, ANDing with the raw host
1211		 * CPUID will drop the flags, and reporting support in AMD's
1212		 * leaf can make it easier for userspace to detect the feature.
1213		 */
1214		SYNTHESIZED_F(LFENCE_RDTSC),
1215		/* SmmPgCfgLock */
1216		/* 4: Resv */
1217		SYNTHESIZED_F(VERW_CLEAR),
1218		F(NULL_SEL_CLR_BASE),
1219		/* UpperAddressIgnore */
1220		F(AUTOIBRS),
1221		F(PREFETCHI),
1222		EMULATED_F(NO_SMM_CTL_MSR),
1223		/* PrefetchCtlMsr */
1224		/* GpOnUserCpuid */
1225		/* EPSF */
1226		SYNTHESIZED_F(SBPB),
1227		SYNTHESIZED_F(IBPB_BRTYPE),
1228		SYNTHESIZED_F(SRSO_NO),
1229		F(SRSO_USER_KERNEL_NO),
1230	);
1231
1232	kvm_cpu_cap_init(CPUID_8000_0021_ECX,
1233		SYNTHESIZED_F(TSA_SQ_NO),
1234		SYNTHESIZED_F(TSA_L1_NO),
1235	);
1236
1237	kvm_cpu_cap_init(CPUID_8000_0022_EAX,
1238		F(PERFMON_V2),
1239	);
1240
1241	if (!static_cpu_has_bug(X86_BUG_NULL_SEG))
1242		kvm_cpu_cap_set(X86_FEATURE_NULL_SEL_CLR_BASE);
1243
1244	kvm_cpu_cap_init(CPUID_C000_0001_EDX,
1245		F(XSTORE),
1246		F(XSTORE_EN),
1247		F(XCRYPT),
1248		F(XCRYPT_EN),
1249		F(ACE2),
1250		F(ACE2_EN),
1251		F(PHE),
1252		F(PHE_EN),
1253		F(PMM),
1254		F(PMM_EN),
1255	);
1256
1257	/*
1258	 * Hide RDTSCP and RDPID if either feature is reported as supported but
1259	 * probing MSR_TSC_AUX failed.  This is purely a sanity check and
1260	 * should never happen, but the guest will likely crash if RDTSCP or
1261	 * RDPID is misreported, and KVM has botched MSR_TSC_AUX emulation in
1262	 * the past.  For example, the sanity check may fire if this instance of
1263	 * KVM is running as L1 on top of an older, broken KVM.
1264	 */
1265	if (WARN_ON((kvm_cpu_cap_has(X86_FEATURE_RDTSCP) ||
1266		     kvm_cpu_cap_has(X86_FEATURE_RDPID)) &&
1267		     !kvm_is_supported_user_return_msr(MSR_TSC_AUX))) {
1268		kvm_cpu_cap_clear(X86_FEATURE_RDTSCP);
1269		kvm_cpu_cap_clear(X86_FEATURE_RDPID);
1270	}
1271}
1272EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cpu_caps);
1273
1274#undef F
1275#undef SCATTERED_F
1276#undef X86_64_F
1277#undef EMULATED_F
1278#undef SYNTHESIZED_F
1279#undef PASSTHROUGH_F
1280#undef ALIASED_1_EDX_F
1281#undef VENDOR_F
1282#undef RUNTIME_F
1283
1284struct kvm_cpuid_array {
1285	struct kvm_cpuid_entry2 *entries;
1286	int maxnent;
1287	int nent;
1288};
1289
1290static struct kvm_cpuid_entry2 *get_next_cpuid(struct kvm_cpuid_array *array)
1291{
1292	if (array->nent >= array->maxnent)
1293		return NULL;
1294
1295	return &array->entries[array->nent++];
1296}
1297
1298static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array,
1299					      u32 function, u32 index)
1300{
1301	struct kvm_cpuid_entry2 *entry = get_next_cpuid(array);
1302
1303	if (!entry)
1304		return NULL;
1305
1306	memset(entry, 0, sizeof(*entry));
1307	entry->function = function;
1308	entry->index = index;
1309	switch (function & 0xC0000000) {
1310	case 0x40000000:
1311		/* Hypervisor leaves are always synthesized by __do_cpuid_func.  */
1312		return entry;
1313
1314	case 0x80000000:
1315		/*
1316		 * 0x80000021 is sometimes synthesized by __do_cpuid_func, which
1317		 * would result in out-of-bounds calls to do_host_cpuid.
1318		 */
1319		{
1320			static int max_cpuid_80000000;
1321			if (!READ_ONCE(max_cpuid_80000000))
1322				WRITE_ONCE(max_cpuid_80000000, cpuid_eax(0x80000000));
1323			if (function > READ_ONCE(max_cpuid_80000000))
1324				return entry;
1325		}
1326		break;
1327
1328	default:
1329		break;
1330	}
1331
1332	cpuid_count(entry->function, entry->index,
1333		    &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
1334
1335	if (cpuid_function_is_indexed(function))
1336		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1337
1338	return entry;
1339}
1340
1341static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func,
1342			       bool include_partially_emulated)
1343{
1344	memset(entry, 0, sizeof(*entry));
1345
1346	entry->function = func;
1347	entry->index = 0;
1348	entry->flags = 0;
1349
1350	switch (func) {
1351	case 0:
1352		entry->eax = 7;
1353		return 1;
1354	case 1:
1355		entry->ecx = feature_bit(MOVBE);
1356		/*
1357		 * KVM allows userspace to enumerate MONITOR+MWAIT support to
1358		 * the guest, but the MWAIT feature flag is never advertised
1359		 * to userspace because MONITOR+MWAIT aren't virtualized by
1360		 * hardware, can't be faithfully emulated in software (KVM
1361		 * emulates them as NOPs), and allowing the guest to execute
1362		 * them natively requires enabling a per-VM capability.
1363		 */
1364		if (include_partially_emulated)
1365			entry->ecx |= feature_bit(MWAIT);
1366		return 1;
1367	case 7:
1368		entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1369		entry->eax = 0;
1370		if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP))
1371			entry->ecx = feature_bit(RDPID);
1372		return 1;
1373	default:
1374		return 0;
1375	}
1376}
1377
1378static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func)
1379{
1380	if (array->nent >= array->maxnent)
1381		return -E2BIG;
1382
1383	array->nent += cpuid_func_emulated(&array->entries[array->nent], func, false);
1384	return 0;
1385}
1386
1387static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
1388{
1389	struct kvm_cpuid_entry2 *entry;
1390	int r, i, max_idx;
1391
1392	/* all calls to cpuid_count() should be made on the same cpu */
1393	get_cpu();
1394
1395	r = -E2BIG;
1396
1397	entry = do_host_cpuid(array, function, 0);
1398	if (!entry)
1399		goto out;
1400
1401	switch (function) {
1402	case 0:
1403		/* Limited to the highest leaf implemented in KVM. */
1404		entry->eax = min(entry->eax, 0x24U);
1405		break;
1406	case 1:
1407		cpuid_entry_override(entry, CPUID_1_EDX);
1408		cpuid_entry_override(entry, CPUID_1_ECX);
1409		break;
1410	case 2:
1411		/*
1412		 * On ancient CPUs, function 2 entries are STATEFUL.  That is,
1413		 * CPUID(function=2, index=0) may return different results each
1414		 * time, with the least-significant byte in EAX enumerating the
1415		 * number of times software should do CPUID(2, 0).
1416		 *
1417		 * Modern CPUs, i.e. every CPU KVM has *ever* run on are less
1418		 * idiotic.  Intel's SDM states that EAX & 0xff "will always
1419		 * return 01H. Software should ignore this value and not
1420		 * interpret it as an informational descriptor", while AMD's
1421		 * APM states that CPUID(2) is reserved.
1422		 *
1423		 * WARN if a frankenstein CPU that supports virtualization and
1424		 * a stateful CPUID.0x2 is encountered.
1425		 */
1426		WARN_ON_ONCE((entry->eax & 0xff) > 1);
1427		break;
1428	/* functions 4 and 0x8000001d have additional index. */
1429	case 4:
1430	case 0x8000001d:
1431		/*
1432		 * Read entries until the cache type in the previous entry is
1433		 * zero, i.e. indicates an invalid entry.
1434		 */
1435		for (i = 1; entry->eax & 0x1f; ++i) {
1436			entry = do_host_cpuid(array, function, i);
1437			if (!entry)
1438				goto out;
1439		}
1440		break;
1441	case 6: /* Thermal management */
1442		entry->eax = 0x4; /* allow ARAT */
1443		entry->ebx = 0;
1444		entry->ecx = 0;
1445		entry->edx = 0;
1446		break;
1447	/* function 7 has additional index. */
1448	case 7:
1449		max_idx = entry->eax = min(entry->eax, 2u);
1450		cpuid_entry_override(entry, CPUID_7_0_EBX);
1451		cpuid_entry_override(entry, CPUID_7_ECX);
1452		cpuid_entry_override(entry, CPUID_7_EDX);
1453
1454		/* KVM only supports up to 0x7.2, capped above via min(). */
1455		if (max_idx >= 1) {
1456			entry = do_host_cpuid(array, function, 1);
1457			if (!entry)
1458				goto out;
1459
1460			cpuid_entry_override(entry, CPUID_7_1_EAX);
1461			cpuid_entry_override(entry, CPUID_7_1_ECX);
1462			cpuid_entry_override(entry, CPUID_7_1_EDX);
1463			entry->ebx = 0;
1464		}
1465		if (max_idx >= 2) {
1466			entry = do_host_cpuid(array, function, 2);
1467			if (!entry)
1468				goto out;
1469
1470			cpuid_entry_override(entry, CPUID_7_2_EDX);
1471			entry->ecx = 0;
1472			entry->ebx = 0;
1473			entry->eax = 0;
1474		}
1475		break;
1476	case 0xa: { /* Architectural Performance Monitoring */
1477		union cpuid10_eax eax = { };
1478		union cpuid10_edx edx = { };
1479
1480		if (!enable_pmu || !static_cpu_has(X86_FEATURE_ARCH_PERFMON)) {
1481			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1482			break;
1483		}
1484
1485		eax.split.version_id = kvm_pmu_cap.version;
1486		eax.split.num_counters = kvm_pmu_cap.num_counters_gp;
1487		eax.split.bit_width = kvm_pmu_cap.bit_width_gp;
1488		eax.split.mask_length = kvm_pmu_cap.events_mask_len;
1489		edx.split.num_counters_fixed = kvm_pmu_cap.num_counters_fixed;
1490		edx.split.bit_width_fixed = kvm_pmu_cap.bit_width_fixed;
1491
1492		if (kvm_pmu_cap.version)
1493			edx.split.anythread_deprecated = 1;
1494
1495		entry->eax = eax.full;
1496		entry->ebx = kvm_pmu_cap.events_mask;
1497		entry->ecx = 0;
1498		entry->edx = edx.full;
1499		break;
1500	}
1501	case 0x1f:
1502	case 0xb:
1503		/*
1504		 * No topology; a valid topology is indicated by the presence
1505		 * of subleaf 1.
1506		 */
1507		entry->eax = entry->ebx = entry->ecx = 0;
1508		break;
1509	case 0xd: {
1510		u64 permitted_xcr0 = kvm_get_filtered_xcr0();
1511		u64 permitted_xss = kvm_caps.supported_xss;
1512
1513		entry->eax &= permitted_xcr0;
1514		entry->ebx = xstate_required_size(permitted_xcr0, false);
1515		entry->ecx = entry->ebx;
1516		entry->edx &= permitted_xcr0 >> 32;
1517		if (!permitted_xcr0)
1518			break;
1519
1520		entry = do_host_cpuid(array, function, 1);
1521		if (!entry)
1522			goto out;
1523
1524		cpuid_entry_override(entry, CPUID_D_1_EAX);
1525		if (entry->eax & (feature_bit(XSAVES) | feature_bit(XSAVEC)))
1526			entry->ebx = xstate_required_size(permitted_xcr0 | permitted_xss,
1527							  true);
1528		else {
1529			WARN_ON_ONCE(permitted_xss != 0);
1530			entry->ebx = 0;
1531		}
1532		entry->ecx &= permitted_xss;
1533		entry->edx &= permitted_xss >> 32;
1534
1535		for (i = 2; i < 64; ++i) {
1536			bool s_state;
1537			if (permitted_xcr0 & BIT_ULL(i))
1538				s_state = false;
1539			else if (permitted_xss & BIT_ULL(i))
1540				s_state = true;
1541			else
1542				continue;
1543
1544			entry = do_host_cpuid(array, function, i);
1545			if (!entry)
1546				goto out;
1547
1548			/*
1549			 * The supported check above should have filtered out
1550			 * invalid sub-leafs.  Only valid sub-leafs should
1551			 * reach this point, and they should have a non-zero
1552			 * save state size.  Furthermore, check whether the
1553			 * processor agrees with permitted_xcr0/permitted_xss
1554			 * on whether this is an XCR0- or IA32_XSS-managed area.
1555			 */
1556			if (WARN_ON_ONCE(!entry->eax || (entry->ecx & 0x1) != s_state)) {
1557				--array->nent;
1558				continue;
1559			}
1560
1561			if (!kvm_cpu_cap_has(X86_FEATURE_XFD))
1562				entry->ecx &= ~BIT_ULL(2);
1563			entry->edx = 0;
1564		}
1565		break;
1566	}
1567	case 0x12:
1568		/* Intel SGX */
1569		if (!kvm_cpu_cap_has(X86_FEATURE_SGX)) {
1570			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1571			break;
1572		}
1573
1574		/*
1575		 * Index 0: Sub-features, MISCSELECT (a.k.a extended features)
1576		 * and max enclave sizes.   The SGX sub-features and MISCSELECT
1577		 * are restricted by kernel and KVM capabilities (like most
1578		 * feature flags), while enclave size is unrestricted.
1579		 */
1580		cpuid_entry_override(entry, CPUID_12_EAX);
1581		entry->ebx &= SGX_MISC_EXINFO;
1582
1583		entry = do_host_cpuid(array, function, 1);
1584		if (!entry)
1585			goto out;
1586
1587		/*
1588		 * Index 1: SECS.ATTRIBUTES.  ATTRIBUTES are restricted a la
1589		 * feature flags.  Advertise all supported flags, including
1590		 * privileged attributes that require explicit opt-in from
1591		 * userspace.  ATTRIBUTES.XFRM is not adjusted as userspace is
1592		 * expected to derive it from supported XCR0.
1593		 */
1594		entry->eax &= SGX_ATTR_PRIV_MASK | SGX_ATTR_UNPRIV_MASK;
1595		entry->ebx &= 0;
1596		break;
1597	/* Intel PT */
1598	case 0x14:
1599		if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) {
1600			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1601			break;
1602		}
1603
1604		for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
1605			if (!do_host_cpuid(array, function, i))
1606				goto out;
1607		}
1608		break;
1609	/* Intel AMX TILE */
1610	case 0x1d:
1611		if (!kvm_cpu_cap_has(X86_FEATURE_AMX_TILE)) {
1612			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1613			break;
1614		}
1615
1616		for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
1617			if (!do_host_cpuid(array, function, i))
1618				goto out;
1619		}
1620		break;
1621	case 0x1e: /* TMUL information */
1622		if (!kvm_cpu_cap_has(X86_FEATURE_AMX_TILE)) {
1623			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1624			break;
1625		}
1626		break;
1627	case 0x24: {
1628		u8 avx10_version;
1629
1630		if (!kvm_cpu_cap_has(X86_FEATURE_AVX10)) {
1631			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1632			break;
1633		}
1634
1635		/*
1636		 * The AVX10 version is encoded in EBX[7:0].  Note, the version
1637		 * is guaranteed to be >=1 if AVX10 is supported.  Note #2, the
1638		 * version needs to be captured before overriding EBX features!
1639		 */
1640		avx10_version = min_t(u8, entry->ebx & 0xff, 1);
1641		cpuid_entry_override(entry, CPUID_24_0_EBX);
1642		entry->ebx |= avx10_version;
1643
1644		entry->eax = 0;
1645		entry->ecx = 0;
1646		entry->edx = 0;
1647		break;
1648	}
1649	case KVM_CPUID_SIGNATURE: {
1650		const u32 *sigptr = (const u32 *)KVM_SIGNATURE;
1651		entry->eax = KVM_CPUID_FEATURES;
1652		entry->ebx = sigptr[0];
1653		entry->ecx = sigptr[1];
1654		entry->edx = sigptr[2];
1655		break;
1656	}
1657	case KVM_CPUID_FEATURES:
1658		entry->eax = (1 << KVM_FEATURE_CLOCKSOURCE) |
1659			     (1 << KVM_FEATURE_NOP_IO_DELAY) |
1660			     (1 << KVM_FEATURE_CLOCKSOURCE2) |
1661			     (1 << KVM_FEATURE_ASYNC_PF) |
1662			     (1 << KVM_FEATURE_PV_EOI) |
1663			     (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
1664			     (1 << KVM_FEATURE_PV_UNHALT) |
1665			     (1 << KVM_FEATURE_PV_TLB_FLUSH) |
1666			     (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
1667			     (1 << KVM_FEATURE_PV_SEND_IPI) |
1668			     (1 << KVM_FEATURE_POLL_CONTROL) |
1669			     (1 << KVM_FEATURE_PV_SCHED_YIELD) |
1670			     (1 << KVM_FEATURE_ASYNC_PF_INT);
1671
1672		if (sched_info_on())
1673			entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
1674
1675		entry->ebx = 0;
1676		entry->ecx = 0;
1677		entry->edx = 0;
1678		break;
1679	case 0x80000000:
1680		entry->eax = min(entry->eax, 0x80000022);
1681		/*
1682		 * Serializing LFENCE is reported in a multitude of ways, and
1683		 * NullSegClearsBase is not reported in CPUID on Zen2; help
1684		 * userspace by providing the CPUID leaf ourselves.
1685		 *
1686		 * However, only do it if the host has CPUID leaf 0x8000001d.
1687		 * QEMU thinks that it can query the host blindly for that
1688		 * CPUID leaf if KVM reports that it supports 0x8000001d or
1689		 * above.  The processor merrily returns values from the
1690		 * highest Intel leaf which QEMU tries to use as the guest's
1691		 * 0x8000001d.  Even worse, this can result in an infinite
1692		 * loop if said highest leaf has no subleaves indexed by ECX.
1693		 */
1694		if (entry->eax >= 0x8000001d &&
1695		    (static_cpu_has(X86_FEATURE_LFENCE_RDTSC)
1696		     || !static_cpu_has_bug(X86_BUG_NULL_SEG)))
1697			entry->eax = max(entry->eax, 0x80000021);
1698		break;
1699	case 0x80000001:
1700		entry->ebx &= ~GENMASK(27, 16);
1701		cpuid_entry_override(entry, CPUID_8000_0001_EDX);
1702		cpuid_entry_override(entry, CPUID_8000_0001_ECX);
1703		break;
1704	case 0x80000005:
1705		/*  Pass host L1 cache and TLB info. */
1706		break;
1707	case 0x80000006:
1708		/* Drop reserved bits, pass host L2 cache and TLB info. */
1709		entry->edx &= ~GENMASK(17, 16);
1710		break;
1711	case 0x80000007: /* Advanced power management */
1712		cpuid_entry_override(entry, CPUID_8000_0007_EDX);
1713
1714		/* mask against host */
1715		entry->edx &= boot_cpu_data.x86_power;
1716		entry->eax = entry->ebx = entry->ecx = 0;
1717		break;
1718	case 0x80000008: {
1719		/*
1720		 * GuestPhysAddrSize (EAX[23:16]) is intended for software
1721		 * use.
1722		 *
1723		 * KVM's ABI is to report the effective MAXPHYADDR for the
1724		 * guest in PhysAddrSize (phys_as), and the maximum
1725		 * *addressable* GPA in GuestPhysAddrSize (g_phys_as).
1726		 *
1727		 * GuestPhysAddrSize is valid if and only if TDP is enabled,
1728		 * in which case the max GPA that can be addressed by KVM may
1729		 * be less than the max GPA that can be legally generated by
1730		 * the guest, e.g. if MAXPHYADDR>48 but the CPU doesn't
1731		 * support 5-level TDP.
1732		 */
1733		unsigned int virt_as = max((entry->eax >> 8) & 0xff, 48U);
1734		unsigned int phys_as, g_phys_as;
1735
1736		/*
1737		 * If TDP (NPT) is disabled use the adjusted host MAXPHYADDR as
1738		 * the guest operates in the same PA space as the host, i.e.
1739		 * reductions in MAXPHYADDR for memory encryption affect shadow
1740		 * paging, too.
1741		 *
1742		 * If TDP is enabled, use the raw bare metal MAXPHYADDR as
1743		 * reductions to the HPAs do not affect GPAs.  The max
1744		 * addressable GPA is the same as the max effective GPA, except
1745		 * that it's capped at 48 bits if 5-level TDP isn't supported
1746		 * (hardware processes bits 51:48 only when walking the fifth
1747		 * level page table).
1748		 */
1749		if (!tdp_enabled) {
1750			phys_as = boot_cpu_data.x86_phys_bits;
1751			g_phys_as = 0;
1752		} else {
1753			phys_as = entry->eax & 0xff;
1754			g_phys_as = phys_as;
1755			if (kvm_mmu_get_max_tdp_level() < 5)
1756				g_phys_as = min(g_phys_as, 48U);
1757		}
1758
1759		entry->eax = phys_as | (virt_as << 8) | (g_phys_as << 16);
1760		entry->ecx &= ~(GENMASK(31, 16) | GENMASK(11, 8));
1761		entry->edx = 0;
1762		cpuid_entry_override(entry, CPUID_8000_0008_EBX);
1763		break;
1764	}
1765	case 0x8000000A:
1766		if (!kvm_cpu_cap_has(X86_FEATURE_SVM)) {
1767			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1768			break;
1769		}
1770		entry->eax = 1; /* SVM revision 1 */
1771		entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
1772				   ASID emulation to nested SVM */
1773		entry->ecx = 0; /* Reserved */
1774		cpuid_entry_override(entry, CPUID_8000_000A_EDX);
1775		break;
1776	case 0x80000019:
1777		entry->ecx = entry->edx = 0;
1778		break;
1779	case 0x8000001a:
1780		entry->eax &= GENMASK(2, 0);
1781		entry->ebx = entry->ecx = entry->edx = 0;
1782		break;
1783	case 0x8000001e:
1784		/* Do not return host topology information.  */
1785		entry->eax = entry->ebx = entry->ecx = 0;
1786		entry->edx = 0; /* reserved */
1787		break;
1788	case 0x8000001F:
1789		if (!kvm_cpu_cap_has(X86_FEATURE_SEV)) {
1790			entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1791		} else {
1792			cpuid_entry_override(entry, CPUID_8000_001F_EAX);
1793			/* Clear NumVMPL since KVM does not support VMPL.  */
1794			entry->ebx &= ~GENMASK(31, 12);
1795			/*
1796			 * Enumerate '0' for "PA bits reduction", the adjusted
1797			 * MAXPHYADDR is enumerated directly (see 0x80000008).
1798			 */
1799			entry->ebx &= ~GENMASK(11, 6);
1800		}
1801		break;
1802	case 0x80000020:
1803		entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1804		break;
1805	case 0x80000021:
1806		entry->ebx = entry->edx = 0;
1807		cpuid_entry_override(entry, CPUID_8000_0021_EAX);
1808		cpuid_entry_override(entry, CPUID_8000_0021_ECX);
1809		break;
1810	/* AMD Extended Performance Monitoring and Debug */
1811	case 0x80000022: {
1812		union cpuid_0x80000022_ebx ebx = { };
1813
1814		entry->ecx = entry->edx = 0;
1815		if (!enable_pmu || !kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) {
1816			entry->eax = entry->ebx = 0;
1817			break;
1818		}
1819
1820		cpuid_entry_override(entry, CPUID_8000_0022_EAX);
1821
1822		ebx.split.num_core_pmc = kvm_pmu_cap.num_counters_gp;
1823		entry->ebx = ebx.full;
1824		break;
1825	}
1826	/*Add support for Centaur's CPUID instruction*/
1827	case 0xC0000000:
1828		/*Just support up to 0xC0000004 now*/
1829		entry->eax = min(entry->eax, 0xC0000004);
1830		break;
1831	case 0xC0000001:
1832		cpuid_entry_override(entry, CPUID_C000_0001_EDX);
1833		break;
1834	case 3: /* Processor serial number */
1835	case 5: /* MONITOR/MWAIT */
1836	case 0xC0000002:
1837	case 0xC0000003:
1838	case 0xC0000004:
1839	default:
1840		entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
1841		break;
1842	}
1843
1844	r = 0;
1845
1846out:
1847	put_cpu();
1848
1849	return r;
1850}
1851
1852static int do_cpuid_func(struct kvm_cpuid_array *array, u32 func,
1853			 unsigned int type)
1854{
1855	if (type == KVM_GET_EMULATED_CPUID)
1856		return __do_cpuid_func_emulated(array, func);
1857
1858	return __do_cpuid_func(array, func);
1859}
1860
1861#define CENTAUR_CPUID_SIGNATURE 0xC0000000
1862
1863static int get_cpuid_func(struct kvm_cpuid_array *array, u32 func,
1864			  unsigned int type)
1865{
1866	u32 limit;
1867	int r;
1868
1869	if (func == CENTAUR_CPUID_SIGNATURE &&
1870	    boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR &&
1871	    boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
1872		return 0;
1873
1874	r = do_cpuid_func(array, func, type);
1875	if (r)
1876		return r;
1877
1878	limit = array->entries[array->nent - 1].eax;
1879	for (func = func + 1; func <= limit; ++func) {
1880		r = do_cpuid_func(array, func, type);
1881		if (r)
1882			break;
1883	}
1884
1885	return r;
1886}
1887
1888static bool sanity_check_entries(struct kvm_cpuid_entry2 __user *entries,
1889				 __u32 num_entries, unsigned int ioctl_type)
1890{
1891	int i;
1892	__u32 pad[3];
1893
1894	if (ioctl_type != KVM_GET_EMULATED_CPUID)
1895		return false;
1896
1897	/*
1898	 * We want to make sure that ->padding is being passed clean from
1899	 * userspace in case we want to use it for something in the future.
1900	 *
1901	 * Sadly, this wasn't enforced for KVM_GET_SUPPORTED_CPUID and so we
1902	 * have to give ourselves satisfied only with the emulated side. /me
1903	 * sheds a tear.
1904	 */
1905	for (i = 0; i < num_entries; i++) {
1906		if (copy_from_user(pad, entries[i].padding, sizeof(pad)))
1907			return true;
1908
1909		if (pad[0] || pad[1] || pad[2])
1910			return true;
1911	}
1912	return false;
1913}
1914
1915int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid,
1916			    struct kvm_cpuid_entry2 __user *entries,
1917			    unsigned int type)
1918{
1919	static const u32 funcs[] = {
1920		0, 0x80000000, CENTAUR_CPUID_SIGNATURE, KVM_CPUID_SIGNATURE,
1921	};
1922
1923	struct kvm_cpuid_array array = {
1924		.nent = 0,
1925	};
1926	int r, i;
1927
1928	if (cpuid->nent < 1)
1929		return -E2BIG;
1930	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
1931		cpuid->nent = KVM_MAX_CPUID_ENTRIES;
1932
1933	if (sanity_check_entries(entries, cpuid->nent, type))
1934		return -EINVAL;
1935
1936	array.entries = kvcalloc(cpuid->nent, sizeof(struct kvm_cpuid_entry2), GFP_KERNEL);
1937	if (!array.entries)
1938		return -ENOMEM;
1939
1940	array.maxnent = cpuid->nent;
1941
1942	for (i = 0; i < ARRAY_SIZE(funcs); i++) {
1943		r = get_cpuid_func(&array, funcs[i], type);
1944		if (r)
1945			goto out_free;
1946	}
1947	cpuid->nent = array.nent;
1948
1949	if (copy_to_user(entries, array.entries,
1950			 array.nent * sizeof(struct kvm_cpuid_entry2)))
1951		r = -EFAULT;
1952
1953out_free:
1954	kvfree(array.entries);
1955	return r;
1956}
1957
1958/*
1959 * Intel CPUID semantics treats any query for an out-of-range leaf as if the
1960 * highest basic leaf (i.e. CPUID.0H:EAX) were requested.  AMD CPUID semantics
1961 * returns all zeroes for any undefined leaf, whether or not the leaf is in
1962 * range.  Centaur/VIA follows Intel semantics.
1963 *
1964 * A leaf is considered out-of-range if its function is higher than the maximum
1965 * supported leaf of its associated class or if its associated class does not
1966 * exist.
1967 *
1968 * There are three primary classes to be considered, with their respective
1969 * ranges described as "<base> - <top>[,<base2> - <top2>] inclusive.  A primary
1970 * class exists if a guest CPUID entry for its <base> leaf exists.  For a given
1971 * class, CPUID.<base>.EAX contains the max supported leaf for the class.
1972 *
1973 *  - Basic:      0x00000000 - 0x3fffffff, 0x50000000 - 0x7fffffff
1974 *  - Hypervisor: 0x40000000 - 0x4fffffff
1975 *  - Extended:   0x80000000 - 0xbfffffff
1976 *  - Centaur:    0xc0000000 - 0xcfffffff
1977 *
1978 * The Hypervisor class is further subdivided into sub-classes that each act as
1979 * their own independent class associated with a 0x100 byte range.  E.g. if Qemu
1980 * is advertising support for both HyperV and KVM, the resulting Hypervisor
1981 * CPUID sub-classes are:
1982 *
1983 *  - HyperV:     0x40000000 - 0x400000ff
1984 *  - KVM:        0x40000100 - 0x400001ff
1985 */
1986static struct kvm_cpuid_entry2 *
1987get_out_of_range_cpuid_entry(struct kvm_vcpu *vcpu, u32 *fn_ptr, u32 index)
1988{
1989	struct kvm_cpuid_entry2 *basic, *class;
1990	u32 function = *fn_ptr;
1991
1992	basic = kvm_find_cpuid_entry(vcpu, 0);
1993	if (!basic)
1994		return NULL;
1995
1996	if (is_guest_vendor_amd(basic->ebx, basic->ecx, basic->edx) ||
1997	    is_guest_vendor_hygon(basic->ebx, basic->ecx, basic->edx))
1998		return NULL;
1999
2000	if (function >= 0x40000000 && function <= 0x4fffffff)
2001		class = kvm_find_cpuid_entry(vcpu, function & 0xffffff00);
2002	else if (function >= 0xc0000000)
2003		class = kvm_find_cpuid_entry(vcpu, 0xc0000000);
2004	else
2005		class = kvm_find_cpuid_entry(vcpu, function & 0x80000000);
2006
2007	if (class && function <= class->eax)
2008		return NULL;
2009
2010	/*
2011	 * Leaf specific adjustments are also applied when redirecting to the
2012	 * max basic entry, e.g. if the max basic leaf is 0xb but there is no
2013	 * entry for CPUID.0xb.index (see below), then the output value for EDX
2014	 * needs to be pulled from CPUID.0xb.1.
2015	 */
2016	*fn_ptr = basic->eax;
2017
2018	/*
2019	 * The class does not exist or the requested function is out of range;
2020	 * the effective CPUID entry is the max basic leaf.  Note, the index of
2021	 * the original requested leaf is observed!
2022	 */
2023	return kvm_find_cpuid_entry_index(vcpu, basic->eax, index);
2024}
2025
2026bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx,
2027	       u32 *ecx, u32 *edx, bool exact_only)
2028{
2029	u32 orig_function = *eax, function = *eax, index = *ecx;
2030	struct kvm_cpuid_entry2 *entry;
2031	bool exact, used_max_basic = false;
2032
2033	if (vcpu->arch.cpuid_dynamic_bits_dirty)
2034		kvm_update_cpuid_runtime(vcpu);
2035
2036	entry = kvm_find_cpuid_entry_index(vcpu, function, index);
2037	exact = !!entry;
2038
2039	if (!entry && !exact_only) {
2040		entry = get_out_of_range_cpuid_entry(vcpu, &function, index);
2041		used_max_basic = !!entry;
2042	}
2043
2044	if (entry) {
2045		*eax = entry->eax;
2046		*ebx = entry->ebx;
2047		*ecx = entry->ecx;
2048		*edx = entry->edx;
2049		if (function == 7 && index == 0) {
2050			u64 data;
2051			if ((*ebx & (feature_bit(RTM) | feature_bit(HLE))) &&
2052			    !kvm_msr_read(vcpu, MSR_IA32_TSX_CTRL, &data) &&
2053			    (data & TSX_CTRL_CPUID_CLEAR))
2054				*ebx &= ~(feature_bit(RTM) | feature_bit(HLE));
2055		} else if (function == 0x80000007) {
2056			if (kvm_hv_invtsc_suppressed(vcpu))
2057				*edx &= ~feature_bit(CONSTANT_TSC);
2058		} else if (IS_ENABLED(CONFIG_KVM_XEN) &&
2059			   kvm_xen_is_tsc_leaf(vcpu, function)) {
2060			/*
2061			 * Update guest TSC frequency information if necessary.
2062			 * Ignore failures, there is no sane value that can be
2063			 * provided if KVM can't get the TSC frequency.
2064			 */
2065			if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu))
2066				kvm_guest_time_update(vcpu);
2067
2068			if (index == 1) {
2069				*ecx = vcpu->arch.pvclock_tsc_mul;
2070				*edx = vcpu->arch.pvclock_tsc_shift;
2071			} else if (index == 2) {
2072				*eax = vcpu->arch.hw_tsc_khz;
2073			}
2074		}
2075	} else {
2076		*eax = *ebx = *ecx = *edx = 0;
2077		/*
2078		 * When leaf 0BH or 1FH is defined, CL is pass-through
2079		 * and EDX is always the x2APIC ID, even for undefined
2080		 * subleaves. Index 1 will exist iff the leaf is
2081		 * implemented, so we pass through CL iff leaf 1
2082		 * exists. EDX can be copied from any existing index.
2083		 */
2084		if (function == 0xb || function == 0x1f) {
2085			entry = kvm_find_cpuid_entry_index(vcpu, function, 1);
2086			if (entry) {
2087				*ecx = index & 0xff;
2088				*edx = entry->edx;
2089			}
2090		}
2091	}
2092	trace_kvm_cpuid(orig_function, index, *eax, *ebx, *ecx, *edx, exact,
2093			used_max_basic);
2094	return exact;
2095}
2096EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_cpuid);
2097
2098int kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
2099{
2100	u32 eax, ebx, ecx, edx;
2101
2102	if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))
2103		return 1;
2104
2105	eax = kvm_rax_read(vcpu);
2106	ecx = kvm_rcx_read(vcpu);
2107	kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
2108	kvm_rax_write(vcpu, eax);
2109	kvm_rbx_write(vcpu, ebx);
2110	kvm_rcx_write(vcpu, ecx);
2111	kvm_rdx_write(vcpu, edx);
2112	return kvm_skip_emulated_instruction(vcpu);
2113}
2114EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_cpuid);