proton-pack.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Handle detection, reporting and mitigation of Spectre v1, v2, v3a and v4, as
   4 * detailed at:
   5 *
   6 *   https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability
   7 *
   8 * This code was originally written hastily under an awful lot of stress and so
   9 * aspects of it are somewhat hacky. Unfortunately, changing anything in here
  10 * instantly makes me feel ill. Thanks, Jann. Thann.
  11 *
  12 * Copyright (C) 2018 ARM Ltd, All Rights Reserved.
  13 * Copyright (C) 2020 Google LLC
  14 *
  15 * "If there's something strange in your neighbourhood, who you gonna call?"
  16 *
  17 * Authors: Will Deacon <will@kernel.org> and Marc Zyngier <maz@kernel.org>
  18 */
  19
  20#include <linux/arm-smccc.h>
  21#include <linux/bpf.h>
  22#include <linux/cpu.h>
  23#include <linux/device.h>
  24#include <linux/nospec.h>
  25#include <linux/prctl.h>
  26#include <linux/sched/task_stack.h>
  27
  28#include <asm/debug-monitors.h>
  29#include <asm/insn.h>
  30#include <asm/spectre.h>
  31#include <asm/traps.h>
  32#include <asm/vectors.h>
  33#include <asm/virt.h>
  34
  35/*
  36 * We try to ensure that the mitigation state can never change as the result of
  37 * onlining a late CPU.
  38 */
  39static void update_mitigation_state(enum mitigation_state *oldp,
  40				    enum mitigation_state new)
  41{
  42	enum mitigation_state state;
  43
  44	do {
  45		state = READ_ONCE(*oldp);
  46		if (new <= state)
  47			break;
  48
  49		/* Userspace almost certainly can't deal with this. */
  50		if (WARN_ON(system_capabilities_finalized()))
  51			break;
  52	} while (cmpxchg_relaxed(oldp, state, new) != state);
  53}
  54
  55/*
  56 * Spectre v1.
  57 *
  58 * The kernel can't protect userspace for this one: it's each person for
  59 * themselves. Advertise what we're doing and be done with it.
  60 */
  61ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
  62			    char *buf)
  63{
  64	return sprintf(buf, "Mitigation: __user pointer sanitization\n");
  65}
  66
  67/*
  68 * Spectre v2.
  69 *
  70 * This one sucks. A CPU is either:
  71 *
  72 * - Mitigated in hardware and advertised by ID_AA64PFR0_EL1.CSV2.
  73 * - Mitigated in hardware and listed in our "safe list".
  74 * - Mitigated in software by firmware.
  75 * - Mitigated in software by a CPU-specific dance in the kernel and a
  76 *   firmware call at EL2.
  77 * - Vulnerable.
  78 *
  79 * It's not unlikely for different CPUs in a big.LITTLE system to fall into
  80 * different camps.
  81 */
  82static enum mitigation_state spectre_v2_state;
  83
  84static bool __read_mostly __nospectre_v2;
  85static int __init parse_spectre_v2_param(char *str)
  86{
  87	__nospectre_v2 = true;
  88	return 0;
  89}
  90early_param("nospectre_v2", parse_spectre_v2_param);
  91
  92static bool spectre_v2_mitigations_off(void)
  93{
  94	return __nospectre_v2 || cpu_mitigations_off();
  95}
  96
  97static const char *get_bhb_affected_string(enum mitigation_state bhb_state)
  98{
  99	switch (bhb_state) {
 100	case SPECTRE_UNAFFECTED:
 101		return "";
 102	default:
 103	case SPECTRE_VULNERABLE:
 104		return ", but not BHB";
 105	case SPECTRE_MITIGATED:
 106		return ", BHB";
 107	}
 108}
 109
 110static bool _unprivileged_ebpf_enabled(void)
 111{
 112#ifdef CONFIG_BPF_SYSCALL
 113	return !sysctl_unprivileged_bpf_disabled;
 114#else
 115	return false;
 116#endif
 117}
 118
 119ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
 120			    char *buf)
 121{
 122	enum mitigation_state bhb_state = arm64_get_spectre_bhb_state();
 123	const char *bhb_str = get_bhb_affected_string(bhb_state);
 124	const char *v2_str = "Branch predictor hardening";
 125
 126	switch (spectre_v2_state) {
 127	case SPECTRE_UNAFFECTED:
 128		if (bhb_state == SPECTRE_UNAFFECTED)
 129			return sprintf(buf, "Not affected\n");
 130
 131		/*
 132		 * Platforms affected by Spectre-BHB can't report
 133		 * "Not affected" for Spectre-v2.
 134		 */
 135		v2_str = "CSV2";
 136		fallthrough;
 137	case SPECTRE_MITIGATED:
 138		if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
 139			return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n");
 140
 141		return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
 142	case SPECTRE_VULNERABLE:
 143		fallthrough;
 144	default:
 145		return sprintf(buf, "Vulnerable\n");
 146	}
 147}
 148
 149static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void)
 150{
 151	u64 pfr0;
 152	static const struct midr_range spectre_v2_safe_list[] = {
 153		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
 154		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
 155		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
 156		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
 157		MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
 158		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
 159		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
 160		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
 161		{ /* sentinel */ }
 162	};
 163
 164	/* If the CPU has CSV2 set, we're safe */
 165	pfr0 = read_cpuid(ID_AA64PFR0_EL1);
 166	if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_CSV2_SHIFT))
 167		return SPECTRE_UNAFFECTED;
 168
 169	/* Alternatively, we have a list of unaffected CPUs */
 170	if (is_midr_in_range_list(spectre_v2_safe_list))
 171		return SPECTRE_UNAFFECTED;
 172
 173	return SPECTRE_VULNERABLE;
 174}
 175
 176static enum mitigation_state spectre_v2_get_cpu_fw_mitigation_state(void)
 177{
 178	int ret;
 179	struct arm_smccc_res res;
 180
 181	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 182			     ARM_SMCCC_ARCH_WORKAROUND_1, &res);
 183
 184	ret = res.a0;
 185	switch (ret) {
 186	case SMCCC_RET_SUCCESS:
 187		return SPECTRE_MITIGATED;
 188	case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
 189		return SPECTRE_UNAFFECTED;
 190	default:
 191		fallthrough;
 192	case SMCCC_RET_NOT_SUPPORTED:
 193		return SPECTRE_VULNERABLE;
 194	}
 195}
 196
 197bool has_spectre_v2(const struct arm64_cpu_capabilities *entry, int scope)
 198{
 199	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 200
 201	if (spectre_v2_get_cpu_hw_mitigation_state() == SPECTRE_UNAFFECTED)
 202		return false;
 203
 204	if (spectre_v2_get_cpu_fw_mitigation_state() == SPECTRE_UNAFFECTED)
 205		return false;
 206
 207	return true;
 208}
 209
 210enum mitigation_state arm64_get_spectre_v2_state(void)
 211{
 212	return spectre_v2_state;
 213}
 214
 215DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
 216
 217static void install_bp_hardening_cb(bp_hardening_cb_t fn)
 218{
 219	__this_cpu_write(bp_hardening_data.fn, fn);
 220
 221	/*
 222	 * Vinz Clortho takes the hyp_vecs start/end "keys" at
 223	 * the door when we're a guest. Skip the hyp-vectors work.
 224	 */
 225	if (!is_hyp_mode_available())
 226		return;
 227
 228	__this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT);
 229}
 230
 231/* Called during entry so must be noinstr */
 232static noinstr void call_smc_arch_workaround_1(void)
 233{
 234	arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 235}
 236
 237/* Called during entry so must be noinstr */
 238static noinstr void call_hvc_arch_workaround_1(void)
 239{
 240	arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
 241}
 242
 243/* Called during entry so must be noinstr */
 244static noinstr void qcom_link_stack_sanitisation(void)
 245{
 246	u64 tmp;
 247
 248	asm volatile("mov	%0, x30		\n"
 249		     ".rept	16		\n"
 250		     "bl	. + 4		\n"
 251		     ".endr			\n"
 252		     "mov	x30, %0		\n"
 253		     : "=&r" (tmp));
 254}
 255
 256static bp_hardening_cb_t spectre_v2_get_sw_mitigation_cb(void)
 257{
 258	u32 midr = read_cpuid_id();
 259	if (((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR) &&
 260	    ((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR_V1))
 261		return NULL;
 262
 263	return qcom_link_stack_sanitisation;
 264}
 265
 266static enum mitigation_state spectre_v2_enable_fw_mitigation(void)
 267{
 268	bp_hardening_cb_t cb;
 269	enum mitigation_state state;
 270
 271	state = spectre_v2_get_cpu_fw_mitigation_state();
 272	if (state != SPECTRE_MITIGATED)
 273		return state;
 274
 275	if (spectre_v2_mitigations_off())
 276		return SPECTRE_VULNERABLE;
 277
 278	switch (arm_smccc_1_1_get_conduit()) {
 279	case SMCCC_CONDUIT_HVC:
 280		cb = call_hvc_arch_workaround_1;
 281		break;
 282
 283	case SMCCC_CONDUIT_SMC:
 284		cb = call_smc_arch_workaround_1;
 285		break;
 286
 287	default:
 288		return SPECTRE_VULNERABLE;
 289	}
 290
 291	/*
 292	 * Prefer a CPU-specific workaround if it exists. Note that we
 293	 * still rely on firmware for the mitigation at EL2.
 294	 */
 295	cb = spectre_v2_get_sw_mitigation_cb() ?: cb;
 296	install_bp_hardening_cb(cb);
 297	return SPECTRE_MITIGATED;
 298}
 299
 300void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
 301{
 302	enum mitigation_state state;
 303
 304	WARN_ON(preemptible());
 305
 306	state = spectre_v2_get_cpu_hw_mitigation_state();
 307	if (state == SPECTRE_VULNERABLE)
 308		state = spectre_v2_enable_fw_mitigation();
 309
 310	update_mitigation_state(&spectre_v2_state, state);
 311}
 312
 313/*
 314 * Spectre-v3a.
 315 *
 316 * Phew, there's not an awful lot to do here! We just instruct EL2 to use
 317 * an indirect trampoline for the hyp vectors so that guests can't read
 318 * VBAR_EL2 to defeat randomisation of the hypervisor VA layout.
 319 */
 320bool has_spectre_v3a(const struct arm64_cpu_capabilities *entry, int scope)
 321{
 322	static const struct midr_range spectre_v3a_unsafe_list[] = {
 323		MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 324		MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 325		{},
 326	};
 327
 328	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 329	return is_midr_in_range_list(spectre_v3a_unsafe_list);
 330}
 331
 332void spectre_v3a_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
 333{
 334	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
 335
 336	if (this_cpu_has_cap(ARM64_SPECTRE_V3A))
 337		data->slot += HYP_VECTOR_INDIRECT;
 338}
 339
 340/*
 341 * Spectre v4.
 342 *
 343 * If you thought Spectre v2 was nasty, wait until you see this mess. A CPU is
 344 * either:
 345 *
 346 * - Mitigated in hardware and listed in our "safe list".
 347 * - Mitigated in hardware via PSTATE.SSBS.
 348 * - Mitigated in software by firmware (sometimes referred to as SSBD).
 349 *
 350 * Wait, that doesn't sound so bad, does it? Keep reading...
 351 *
 352 * A major source of headaches is that the software mitigation is enabled both
 353 * on a per-task basis, but can also be forced on for the kernel, necessitating
 354 * both context-switch *and* entry/exit hooks. To make it even worse, some CPUs
 355 * allow EL0 to toggle SSBS directly, which can end up with the prctl() state
 356 * being stale when re-entering the kernel. The usual big.LITTLE caveats apply,
 357 * so you can have systems that have both firmware and SSBS mitigations. This
 358 * means we actually have to reject late onlining of CPUs with mitigations if
 359 * all of the currently onlined CPUs are safelisted, as the mitigation tends to
 360 * be opt-in for userspace. Yes, really, the cure is worse than the disease.
 361 *
 362 * The only good part is that if the firmware mitigation is present, then it is
 363 * present for all CPUs, meaning we don't have to worry about late onlining of a
 364 * vulnerable CPU if one of the boot CPUs is using the firmware mitigation.
 365 *
 366 * Give me a VAX-11/780 any day of the week...
 367 */
 368static enum mitigation_state spectre_v4_state;
 369
 370/* This is the per-cpu state tracking whether we need to talk to firmware */
 371DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
 372
 373enum spectre_v4_policy {
 374	SPECTRE_V4_POLICY_MITIGATION_DYNAMIC,
 375	SPECTRE_V4_POLICY_MITIGATION_ENABLED,
 376	SPECTRE_V4_POLICY_MITIGATION_DISABLED,
 377};
 378
 379static enum spectre_v4_policy __read_mostly __spectre_v4_policy;
 380
 381static const struct spectre_v4_param {
 382	const char		*str;
 383	enum spectre_v4_policy	policy;
 384} spectre_v4_params[] = {
 385	{ "force-on",	SPECTRE_V4_POLICY_MITIGATION_ENABLED, },
 386	{ "force-off",	SPECTRE_V4_POLICY_MITIGATION_DISABLED, },
 387	{ "kernel",	SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, },
 388};
 389static int __init parse_spectre_v4_param(char *str)
 390{
 391	int i;
 392
 393	if (!str || !str[0])
 394		return -EINVAL;
 395
 396	for (i = 0; i < ARRAY_SIZE(spectre_v4_params); i++) {
 397		const struct spectre_v4_param *param = &spectre_v4_params[i];
 398
 399		if (strncmp(str, param->str, strlen(param->str)))
 400			continue;
 401
 402		__spectre_v4_policy = param->policy;
 403		return 0;
 404	}
 405
 406	return -EINVAL;
 407}
 408early_param("ssbd", parse_spectre_v4_param);
 409
 410/*
 411 * Because this was all written in a rush by people working in different silos,
 412 * we've ended up with multiple command line options to control the same thing.
 413 * Wrap these up in some helpers, which prefer disabling the mitigation if faced
 414 * with contradictory parameters. The mitigation is always either "off",
 415 * "dynamic" or "on".
 416 */
 417static bool spectre_v4_mitigations_off(void)
 418{
 419	return cpu_mitigations_off() ||
 420	       __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DISABLED;
 421}
 422
 423/* Do we need to toggle the mitigation state on entry to/exit from the kernel? */
 424static bool spectre_v4_mitigations_dynamic(void)
 425{
 426	return !spectre_v4_mitigations_off() &&
 427	       __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DYNAMIC;
 428}
 429
 430static bool spectre_v4_mitigations_on(void)
 431{
 432	return !spectre_v4_mitigations_off() &&
 433	       __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_ENABLED;
 434}
 435
 436ssize_t cpu_show_spec_store_bypass(struct device *dev,
 437				   struct device_attribute *attr, char *buf)
 438{
 439	switch (spectre_v4_state) {
 440	case SPECTRE_UNAFFECTED:
 441		return sprintf(buf, "Not affected\n");
 442	case SPECTRE_MITIGATED:
 443		return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");
 444	case SPECTRE_VULNERABLE:
 445		fallthrough;
 446	default:
 447		return sprintf(buf, "Vulnerable\n");
 448	}
 449}
 450
 451enum mitigation_state arm64_get_spectre_v4_state(void)
 452{
 453	return spectre_v4_state;
 454}
 455
 456static enum mitigation_state spectre_v4_get_cpu_hw_mitigation_state(void)
 457{
 458	static const struct midr_range spectre_v4_safe_list[] = {
 459		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
 460		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
 461		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
 462		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
 463		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
 464		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
 465		{ /* sentinel */ },
 466	};
 467
 468	if (is_midr_in_range_list(spectre_v4_safe_list))
 469		return SPECTRE_UNAFFECTED;
 470
 471	/* CPU features are detected first */
 472	if (this_cpu_has_cap(ARM64_SSBS))
 473		return SPECTRE_MITIGATED;
 474
 475	return SPECTRE_VULNERABLE;
 476}
 477
 478static enum mitigation_state spectre_v4_get_cpu_fw_mitigation_state(void)
 479{
 480	int ret;
 481	struct arm_smccc_res res;
 482
 483	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 484			     ARM_SMCCC_ARCH_WORKAROUND_2, &res);
 485
 486	ret = res.a0;
 487	switch (ret) {
 488	case SMCCC_RET_SUCCESS:
 489		return SPECTRE_MITIGATED;
 490	case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
 491		fallthrough;
 492	case SMCCC_RET_NOT_REQUIRED:
 493		return SPECTRE_UNAFFECTED;
 494	default:
 495		fallthrough;
 496	case SMCCC_RET_NOT_SUPPORTED:
 497		return SPECTRE_VULNERABLE;
 498	}
 499}
 500
 501bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope)
 502{
 503	enum mitigation_state state;
 504
 505	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 506
 507	state = spectre_v4_get_cpu_hw_mitigation_state();
 508	if (state == SPECTRE_VULNERABLE)
 509		state = spectre_v4_get_cpu_fw_mitigation_state();
 510
 511	return state != SPECTRE_UNAFFECTED;
 512}
 513
 514bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr)
 515{
 516	const u32 instr_mask = ~(1U << PSTATE_Imm_shift);
 517	const u32 instr_val = 0xd500401f | PSTATE_SSBS;
 518
 519	if ((instr & instr_mask) != instr_val)
 520		return false;
 521
 522	if (instr & BIT(PSTATE_Imm_shift))
 523		regs->pstate |= PSR_SSBS_BIT;
 524	else
 525		regs->pstate &= ~PSR_SSBS_BIT;
 526
 527	arm64_skip_faulting_instruction(regs, 4);
 528	return true;
 529}
 530
 531static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
 532{
 533	enum mitigation_state state;
 534
 535	/*
 536	 * If the system is mitigated but this CPU doesn't have SSBS, then
 537	 * we must be on the safelist and there's nothing more to do.
 538	 */
 539	state = spectre_v4_get_cpu_hw_mitigation_state();
 540	if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS))
 541		return state;
 542
 543	if (spectre_v4_mitigations_off()) {
 544		sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
 545		set_pstate_ssbs(1);
 546		return SPECTRE_VULNERABLE;
 547	}
 548
 549	/* SCTLR_EL1.DSSBS was initialised to 0 during boot */
 550	set_pstate_ssbs(0);
 551
 552	/*
 553	 * SSBS is self-synchronizing and is intended to affect subsequent
 554	 * speculative instructions, but some CPUs can speculate with a stale
 555	 * value of SSBS.
 556	 *
 557	 * Mitigate this with an unconditional speculation barrier, as CPUs
 558	 * could mis-speculate branches and bypass a conditional barrier.
 559	 */
 560	if (IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386))
 561		spec_bar();
 562
 563	return SPECTRE_MITIGATED;
 564}
 565
 566/*
 567 * Patch a branch over the Spectre-v4 mitigation code with a NOP so that
 568 * we fallthrough and check whether firmware needs to be called on this CPU.
 569 */
 570void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt,
 571						  __le32 *origptr,
 572						  __le32 *updptr, int nr_inst)
 573{
 574	BUG_ON(nr_inst != 1); /* Branch -> NOP */
 575
 576	if (spectre_v4_mitigations_off())
 577		return;
 578
 579	if (cpus_have_cap(ARM64_SSBS))
 580		return;
 581
 582	if (spectre_v4_mitigations_dynamic())
 583		*updptr = cpu_to_le32(aarch64_insn_gen_nop());
 584}
 585
 586/*
 587 * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
 588 * to call into firmware to adjust the mitigation state.
 589 */
 590void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt,
 591					       __le32 *origptr,
 592					       __le32 *updptr, int nr_inst)
 593{
 594	u32 insn;
 595
 596	BUG_ON(nr_inst != 1); /* NOP -> HVC/SMC */
 597
 598	switch (arm_smccc_1_1_get_conduit()) {
 599	case SMCCC_CONDUIT_HVC:
 600		insn = aarch64_insn_get_hvc_value();
 601		break;
 602	case SMCCC_CONDUIT_SMC:
 603		insn = aarch64_insn_get_smc_value();
 604		break;
 605	default:
 606		return;
 607	}
 608
 609	*updptr = cpu_to_le32(insn);
 610}
 611
 612static enum mitigation_state spectre_v4_enable_fw_mitigation(void)
 613{
 614	enum mitigation_state state;
 615
 616	state = spectre_v4_get_cpu_fw_mitigation_state();
 617	if (state != SPECTRE_MITIGATED)
 618		return state;
 619
 620	if (spectre_v4_mitigations_off()) {
 621		arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, false, NULL);
 622		return SPECTRE_VULNERABLE;
 623	}
 624
 625	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, true, NULL);
 626
 627	if (spectre_v4_mitigations_dynamic())
 628		__this_cpu_write(arm64_ssbd_callback_required, 1);
 629
 630	return SPECTRE_MITIGATED;
 631}
 632
 633void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
 634{
 635	enum mitigation_state state;
 636
 637	WARN_ON(preemptible());
 638
 639	state = spectre_v4_enable_hw_mitigation();
 640	if (state == SPECTRE_VULNERABLE)
 641		state = spectre_v4_enable_fw_mitigation();
 642
 643	update_mitigation_state(&spectre_v4_state, state);
 644}
 645
 646static void __update_pstate_ssbs(struct pt_regs *regs, bool state)
 647{
 648	u64 bit = compat_user_mode(regs) ? PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;
 649
 650	if (state)
 651		regs->pstate |= bit;
 652	else
 653		regs->pstate &= ~bit;
 654}
 655
 656void spectre_v4_enable_task_mitigation(struct task_struct *tsk)
 657{
 658	struct pt_regs *regs = task_pt_regs(tsk);
 659	bool ssbs = false, kthread = tsk->flags & PF_KTHREAD;
 660
 661	if (spectre_v4_mitigations_off())
 662		ssbs = true;
 663	else if (spectre_v4_mitigations_dynamic() && !kthread)
 664		ssbs = !test_tsk_thread_flag(tsk, TIF_SSBD);
 665
 666	__update_pstate_ssbs(regs, ssbs);
 667}
 668
 669/*
 670 * The Spectre-v4 mitigation can be controlled via a prctl() from userspace.
 671 * This is interesting because the "speculation disabled" behaviour can be
 672 * configured so that it is preserved across exec(), which means that the
 673 * prctl() may be necessary even when PSTATE.SSBS can be toggled directly
 674 * from userspace.
 675 */
 676static void ssbd_prctl_enable_mitigation(struct task_struct *task)
 677{
 678	task_clear_spec_ssb_noexec(task);
 679	task_set_spec_ssb_disable(task);
 680	set_tsk_thread_flag(task, TIF_SSBD);
 681}
 682
 683static void ssbd_prctl_disable_mitigation(struct task_struct *task)
 684{
 685	task_clear_spec_ssb_noexec(task);
 686	task_clear_spec_ssb_disable(task);
 687	clear_tsk_thread_flag(task, TIF_SSBD);
 688}
 689
 690static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)
 691{
 692	switch (ctrl) {
 693	case PR_SPEC_ENABLE:
 694		/* Enable speculation: disable mitigation */
 695		/*
 696		 * Force disabled speculation prevents it from being
 697		 * re-enabled.
 698		 */
 699		if (task_spec_ssb_force_disable(task))
 700			return -EPERM;
 701
 702		/*
 703		 * If the mitigation is forced on, then speculation is forced
 704		 * off and we again prevent it from being re-enabled.
 705		 */
 706		if (spectre_v4_mitigations_on())
 707			return -EPERM;
 708
 709		ssbd_prctl_disable_mitigation(task);
 710		break;
 711	case PR_SPEC_FORCE_DISABLE:
 712		/* Force disable speculation: force enable mitigation */
 713		/*
 714		 * If the mitigation is forced off, then speculation is forced
 715		 * on and we prevent it from being disabled.
 716		 */
 717		if (spectre_v4_mitigations_off())
 718			return -EPERM;
 719
 720		task_set_spec_ssb_force_disable(task);
 721		fallthrough;
 722	case PR_SPEC_DISABLE:
 723		/* Disable speculation: enable mitigation */
 724		/* Same as PR_SPEC_FORCE_DISABLE */
 725		if (spectre_v4_mitigations_off())
 726			return -EPERM;
 727
 728		ssbd_prctl_enable_mitigation(task);
 729		break;
 730	case PR_SPEC_DISABLE_NOEXEC:
 731		/* Disable speculation until execve(): enable mitigation */
 732		/*
 733		 * If the mitigation state is forced one way or the other, then
 734		 * we must fail now before we try to toggle it on execve().
 735		 */
 736		if (task_spec_ssb_force_disable(task) ||
 737		    spectre_v4_mitigations_off() ||
 738		    spectre_v4_mitigations_on()) {
 739			return -EPERM;
 740		}
 741
 742		ssbd_prctl_enable_mitigation(task);
 743		task_set_spec_ssb_noexec(task);
 744		break;
 745	default:
 746		return -ERANGE;
 747	}
 748
 749	spectre_v4_enable_task_mitigation(task);
 750	return 0;
 751}
 752
 753int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
 754			     unsigned long ctrl)
 755{
 756	switch (which) {
 757	case PR_SPEC_STORE_BYPASS:
 758		return ssbd_prctl_set(task, ctrl);
 759	default:
 760		return -ENODEV;
 761	}
 762}
 763
 764static int ssbd_prctl_get(struct task_struct *task)
 765{
 766	switch (spectre_v4_state) {
 767	case SPECTRE_UNAFFECTED:
 768		return PR_SPEC_NOT_AFFECTED;
 769	case SPECTRE_MITIGATED:
 770		if (spectre_v4_mitigations_on())
 771			return PR_SPEC_NOT_AFFECTED;
 772
 773		if (spectre_v4_mitigations_dynamic())
 774			break;
 775
 776		/* Mitigations are disabled, so we're vulnerable. */
 777		fallthrough;
 778	case SPECTRE_VULNERABLE:
 779		fallthrough;
 780	default:
 781		return PR_SPEC_ENABLE;
 782	}
 783
 784	/* Check the mitigation state for this task */
 785	if (task_spec_ssb_force_disable(task))
 786		return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
 787
 788	if (task_spec_ssb_noexec(task))
 789		return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
 790
 791	if (task_spec_ssb_disable(task))
 792		return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
 793
 794	return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
 795}
 796
 797int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
 798{
 799	switch (which) {
 800	case PR_SPEC_STORE_BYPASS:
 801		return ssbd_prctl_get(task);
 802	default:
 803		return -ENODEV;
 804	}
 805}
 806
 807/*
 808 * Spectre BHB.
 809 *
 810 * A CPU is either:
 811 * - Mitigated by a branchy loop a CPU specific number of times, and listed
 812 *   in our "loop mitigated list".
 813 * - Mitigated in software by the firmware Spectre v2 call.
 814 * - Has the ClearBHB instruction to perform the mitigation.
 815 * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
 816 *   software mitigation in the vectors is needed.
 817 * - Has CSV2.3, so is unaffected.
 818 */
 819static enum mitigation_state spectre_bhb_state;
 820
 821enum mitigation_state arm64_get_spectre_bhb_state(void)
 822{
 823	return spectre_bhb_state;
 824}
 825
 826enum bhb_mitigation_bits {
 827	BHB_LOOP,
 828	BHB_FW,
 829	BHB_HW,
 830	BHB_INSN,
 831};
 832static unsigned long system_bhb_mitigations;
 833
 834/*
 835 * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
 836 * SCOPE_SYSTEM call will give the right answer.
 837 */
 838static bool is_spectre_bhb_safe(int scope)
 839{
 840	static const struct midr_range spectre_bhb_safe_list[] = {
 841		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
 842		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
 843		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
 844		MIDR_ALL_VERSIONS(MIDR_CORTEX_A510),
 845		MIDR_ALL_VERSIONS(MIDR_CORTEX_A520),
 846		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
 847		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
 848		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
 849		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
 850		{},
 851	};
 852	static bool all_safe = true;
 853
 854	if (scope != SCOPE_LOCAL_CPU)
 855		return all_safe;
 856
 857	if (is_midr_in_range_list(spectre_bhb_safe_list))
 858		return true;
 859
 860	all_safe = false;
 861
 862	return false;
 863}
 864
 865static u8 spectre_bhb_loop_affected(void)
 866{
 867	u8 k = 0;
 868
 869	static const struct midr_range spectre_bhb_k132_list[] = {
 870		MIDR_ALL_VERSIONS(MIDR_CORTEX_X3),
 871		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2),
 872		{},
 873	};
 874	static const struct midr_range spectre_bhb_k38_list[] = {
 875		MIDR_ALL_VERSIONS(MIDR_CORTEX_A715),
 876		MIDR_ALL_VERSIONS(MIDR_CORTEX_A720),
 877		MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE),
 878		{},
 879	};
 880	static const struct midr_range spectre_bhb_k32_list[] = {
 881		MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
 882		MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE),
 883		MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
 884		MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
 885		MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C),
 886		MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
 887		MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
 888		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
 889		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
 890		MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
 891		{},
 892	};
 893	static const struct midr_range spectre_bhb_k24_list[] = {
 894		MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
 895		MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE),
 896		MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
 897		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
 898		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_GOLD),
 899		MIDR_ALL_VERSIONS(MIDR_HISI_HIP09),
 900		{},
 901	};
 902	static const struct midr_range spectre_bhb_k11_list[] = {
 903		MIDR_ALL_VERSIONS(MIDR_AMPERE1),
 904		{},
 905	};
 906	static const struct midr_range spectre_bhb_k8_list[] = {
 907		MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
 908		MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
 909		{},
 910	};
 911
 912	if (is_midr_in_range_list(spectre_bhb_k132_list))
 913		k = 132;
 914	else if (is_midr_in_range_list(spectre_bhb_k38_list))
 915		k = 38;
 916	else if (is_midr_in_range_list(spectre_bhb_k32_list))
 917		k = 32;
 918	else if (is_midr_in_range_list(spectre_bhb_k24_list))
 919		k = 24;
 920	else if (is_midr_in_range_list(spectre_bhb_k11_list))
 921		k = 11;
 922	else if (is_midr_in_range_list(spectre_bhb_k8_list))
 923		k =  8;
 924
 925	return k;
 926}
 927
 928static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
 929{
 930	int ret;
 931	struct arm_smccc_res res;
 932
 933	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
 934			     ARM_SMCCC_ARCH_WORKAROUND_3, &res);
 935
 936	ret = res.a0;
 937	switch (ret) {
 938	case SMCCC_RET_SUCCESS:
 939		return SPECTRE_MITIGATED;
 940	case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
 941		return SPECTRE_UNAFFECTED;
 942	default:
 943		fallthrough;
 944	case SMCCC_RET_NOT_SUPPORTED:
 945		return SPECTRE_VULNERABLE;
 946	}
 947}
 948
 949static bool has_spectre_bhb_fw_mitigation(void)
 950{
 951	enum mitigation_state fw_state;
 952	bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE;
 953
 954	fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
 955	return has_smccc && fw_state == SPECTRE_MITIGATED;
 956}
 957
 958static bool supports_ecbhb(int scope)
 959{
 960	u64 mmfr1;
 961
 962	if (scope == SCOPE_LOCAL_CPU)
 963		mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
 964	else
 965		mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
 966
 967	return cpuid_feature_extract_unsigned_field(mmfr1,
 968						    ID_AA64MMFR1_EL1_ECBHB_SHIFT);
 969}
 970
 971static u8 max_bhb_k;
 972
 973bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
 974			     int scope)
 975{
 976	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
 977
 978	if (supports_csv2p3(scope))
 979		return false;
 980
 981	if (is_spectre_bhb_safe(scope))
 982		return false;
 983
 984	/*
 985	 * At this point the core isn't known to be "safe" so we're going to
 986	 * assume it's vulnerable. We still need to update `max_bhb_k` though,
 987	 * but only if we aren't mitigating with clearbhb though.
 988	 */
 989	if (scope == SCOPE_LOCAL_CPU && !supports_clearbhb(SCOPE_LOCAL_CPU))
 990		max_bhb_k = max(max_bhb_k, spectre_bhb_loop_affected());
 991
 992	return true;
 993}
 994
 995u8 get_spectre_bhb_loop_value(void)
 996{
 997	return max_bhb_k;
 998}
 999
1000static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
1001{
1002	const char *v = arm64_get_bp_hardening_vector(slot);
1003
1004	__this_cpu_write(this_cpu_vector, v);
1005
1006	/*
1007	 * When KPTI is in use, the vectors are switched when exiting to
1008	 * user-space.
1009	 */
1010	if (cpus_have_cap(ARM64_UNMAP_KERNEL_AT_EL0))
1011		return;
1012
1013	write_sysreg(v, vbar_el1);
1014	isb();
1015}
1016
1017bool __read_mostly __nospectre_bhb;
1018static int __init parse_spectre_bhb_param(char *str)
1019{
1020	__nospectre_bhb = true;
1021	return 0;
1022}
1023early_param("nospectre_bhb", parse_spectre_bhb_param);
1024
1025void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
1026{
1027	bp_hardening_cb_t cpu_cb;
1028	enum mitigation_state state = SPECTRE_VULNERABLE;
1029	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
1030
1031	if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
1032		return;
1033
1034	if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) {
1035		/* No point mitigating Spectre-BHB alone. */
1036	} else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
1037		/* Do nothing */
1038	} else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
1039		state = SPECTRE_MITIGATED;
1040		set_bit(BHB_HW, &system_bhb_mitigations);
1041	} else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
1042		/*
1043		 * Ensure KVM uses the indirect vector which will have ClearBHB
1044		 * added.
1045		 */
1046		if (!data->slot)
1047			data->slot = HYP_VECTOR_INDIRECT;
1048
1049		this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
1050		state = SPECTRE_MITIGATED;
1051		set_bit(BHB_INSN, &system_bhb_mitigations);
1052	} else if (spectre_bhb_loop_affected()) {
1053		/*
1054		 * Ensure KVM uses the indirect vector which will have the
1055		 * branchy-loop added. A57/A72-r0 will already have selected
1056		 * the spectre-indirect vector, which is sufficient for BHB
1057		 * too.
1058		 */
1059		if (!data->slot)
1060			data->slot = HYP_VECTOR_INDIRECT;
1061
1062		this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
1063		state = SPECTRE_MITIGATED;
1064		set_bit(BHB_LOOP, &system_bhb_mitigations);
1065	} else if (has_spectre_bhb_fw_mitigation()) {
1066		/*
1067		 * Ensure KVM uses one of the spectre bp_hardening
1068		 * vectors. The indirect vector doesn't include the EL3
1069		 * call, so needs upgrading to
1070		 * HYP_VECTOR_SPECTRE_INDIRECT.
1071		 */
1072		if (!data->slot || data->slot == HYP_VECTOR_INDIRECT)
1073			data->slot += 1;
1074
1075		this_cpu_set_vectors(EL1_VECTOR_BHB_FW);
1076
1077		/*
1078		 * The WA3 call in the vectors supersedes the WA1 call
1079		 * made during context-switch. Uninstall any firmware
1080		 * bp_hardening callback.
1081		 */
1082		cpu_cb = spectre_v2_get_sw_mitigation_cb();
1083		if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb)
1084			__this_cpu_write(bp_hardening_data.fn, NULL);
1085
1086		state = SPECTRE_MITIGATED;
1087		set_bit(BHB_FW, &system_bhb_mitigations);
1088	}
1089
1090	update_mitigation_state(&spectre_bhb_state, state);
1091}
1092
1093bool is_spectre_bhb_fw_mitigated(void)
1094{
1095	return test_bit(BHB_FW, &system_bhb_mitigations);
1096}
1097
1098/* Patched to NOP when enabled */
1099void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt,
1100						     __le32 *origptr,
1101						      __le32 *updptr, int nr_inst)
1102{
1103	BUG_ON(nr_inst != 1);
1104
1105	if (test_bit(BHB_LOOP, &system_bhb_mitigations))
1106		*updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1107}
1108
1109/* Patched to NOP when enabled */
1110void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt,
1111						   __le32 *origptr,
1112						   __le32 *updptr, int nr_inst)
1113{
1114	BUG_ON(nr_inst != 1);
1115
1116	if (test_bit(BHB_FW, &system_bhb_mitigations))
1117		*updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1118}
1119
1120/* Patched to correct the immediate */
1121void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt,
1122				   __le32 *origptr, __le32 *updptr, int nr_inst)
1123{
1124	u8 rd;
1125	u32 insn;
1126
1127	BUG_ON(nr_inst != 1); /* MOV -> MOV */
1128
1129	if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
1130		return;
1131
1132	insn = le32_to_cpu(*origptr);
1133	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
1134	insn = aarch64_insn_gen_movewide(rd, max_bhb_k, 0,
1135					 AARCH64_INSN_VARIANT_64BIT,
1136					 AARCH64_INSN_MOVEWIDE_ZERO);
1137	*updptr++ = cpu_to_le32(insn);
1138}
1139
1140/* Patched to mov WA3 when supported */
1141void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt,
1142				   __le32 *origptr, __le32 *updptr, int nr_inst)
1143{
1144	u8 rd;
1145	u32 insn;
1146
1147	BUG_ON(nr_inst != 1); /* MOV -> MOV */
1148
1149	if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||
1150	    !test_bit(BHB_FW, &system_bhb_mitigations))
1151		return;
1152
1153	insn = le32_to_cpu(*origptr);
1154	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
1155
1156	insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR,
1157						  AARCH64_INSN_VARIANT_32BIT,
1158						  AARCH64_INSN_REG_ZR, rd,
1159						  ARM_SMCCC_ARCH_WORKAROUND_3);
1160	if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT))
1161		return;
1162
1163	*updptr++ = cpu_to_le32(insn);
1164}
1165
1166/* Patched to NOP when not supported */
1167void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt,
1168				   __le32 *origptr, __le32 *updptr, int nr_inst)
1169{
1170	BUG_ON(nr_inst != 2);
1171
1172	if (test_bit(BHB_INSN, &system_bhb_mitigations))
1173		return;
1174
1175	*updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1176	*updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
1177}
1178
1179#ifdef CONFIG_BPF_SYSCALL
1180#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n"
1181void unpriv_ebpf_notify(int new_state)
1182{
1183	if (spectre_v2_state == SPECTRE_VULNERABLE ||
1184	    spectre_bhb_state != SPECTRE_MITIGATED)
1185		return;
1186
1187	if (!new_state)
1188		pr_err("WARNING: %s", EBPF_WARN);
1189}
1190#endif
1191
1192void spectre_print_disabled_mitigations(void)
1193{
1194	/* Keep a single copy of the common message suffix to avoid duplication. */
1195	const char *spectre_disabled_suffix = "mitigation disabled by command-line option\n";
1196
1197	if (spectre_v2_mitigations_off())
1198		pr_info("spectre-v2 %s", spectre_disabled_suffix);
1199
1200	if (spectre_v4_mitigations_off())
1201		pr_info("spectre-v4 %s", spectre_disabled_suffix);
1202
1203	if (__nospectre_bhb || cpu_mitigations_off())
1204		pr_info("spectre-bhb %s", spectre_disabled_suffix);
1205}