tsx.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Intel Transactional Synchronization Extensions (TSX) control.
  4 *
  5 * Copyright (C) 2019-2021 Intel Corporation
  6 *
  7 * Author:
  8 *	Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
  9 */
 10
 11#include <linux/cpufeature.h>
 12
 13#include <asm/cmdline.h>
 14#include <asm/cpu.h>
 15#include <asm/msr.h>
 16
 17#include "cpu.h"
 18
 19#undef pr_fmt
 20#define pr_fmt(fmt) "tsx: " fmt
 21
 22enum tsx_ctrl_states {
 23	TSX_CTRL_AUTO,
 24	TSX_CTRL_ENABLE,
 25	TSX_CTRL_DISABLE,
 26	TSX_CTRL_RTM_ALWAYS_ABORT,
 27	TSX_CTRL_NOT_SUPPORTED,
 28};
 29
 30static enum tsx_ctrl_states tsx_ctrl_state __ro_after_init =
 31	IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_AUTO) ? TSX_CTRL_AUTO   :
 32	IS_ENABLED(CONFIG_X86_INTEL_TSX_MODE_OFF) ? TSX_CTRL_DISABLE : TSX_CTRL_ENABLE;
 33
 34static void tsx_disable(void)
 35{
 36	u64 tsx;
 37
 38	rdmsrq(MSR_IA32_TSX_CTRL, tsx);
 39
 40	/* Force all transactions to immediately abort */
 41	tsx |= TSX_CTRL_RTM_DISABLE;
 42
 43	/*
 44	 * Ensure TSX support is not enumerated in CPUID.
 45	 * This is visible to userspace and will ensure they
 46	 * do not waste resources trying TSX transactions that
 47	 * will always abort.
 48	 */
 49	tsx |= TSX_CTRL_CPUID_CLEAR;
 50
 51	wrmsrq(MSR_IA32_TSX_CTRL, tsx);
 52}
 53
 54static void tsx_enable(void)
 55{
 56	u64 tsx;
 57
 58	rdmsrq(MSR_IA32_TSX_CTRL, tsx);
 59
 60	/* Enable the RTM feature in the cpu */
 61	tsx &= ~TSX_CTRL_RTM_DISABLE;
 62
 63	/*
 64	 * Ensure TSX support is enumerated in CPUID.
 65	 * This is visible to userspace and will ensure they
 66	 * can enumerate and use the TSX feature.
 67	 */
 68	tsx &= ~TSX_CTRL_CPUID_CLEAR;
 69
 70	wrmsrq(MSR_IA32_TSX_CTRL, tsx);
 71}
 72
 73static enum tsx_ctrl_states x86_get_tsx_auto_mode(void)
 74{
 75	if (boot_cpu_has_bug(X86_BUG_TAA))
 76		return TSX_CTRL_DISABLE;
 77
 78	return TSX_CTRL_ENABLE;
 79}
 80
 81/*
 82 * Disabling TSX is not a trivial business.
 83 *
 84 * First of all, there's a CPUID bit: X86_FEATURE_RTM_ALWAYS_ABORT
 85 * which says that TSX is practically disabled (all transactions are
 86 * aborted by default). When that bit is set, the kernel unconditionally
 87 * disables TSX.
 88 *
 89 * In order to do that, however, it needs to dance a bit:
 90 *
 91 * 1. The first method to disable it is through MSR_TSX_FORCE_ABORT and
 92 * the MSR is present only when *two* CPUID bits are set:
 93 *
 94 * - X86_FEATURE_RTM_ALWAYS_ABORT
 95 * - X86_FEATURE_TSX_FORCE_ABORT
 96 *
 97 * 2. The second method is for CPUs which do not have the above-mentioned
 98 * MSR: those use a different MSR - MSR_IA32_TSX_CTRL and disable TSX
 99 * through that one. Those CPUs can also have the initially mentioned
100 * CPUID bit X86_FEATURE_RTM_ALWAYS_ABORT set and for those the same strategy
101 * applies: TSX gets disabled unconditionally.
102 *
103 * When either of the two methods are present, the kernel disables TSX and
104 * clears the respective RTM and HLE feature flags.
105 *
106 * An additional twist in the whole thing presents late microcode loading
107 * which, when done, may cause for the X86_FEATURE_RTM_ALWAYS_ABORT CPUID
108 * bit to be set after the update.
109 *
110 * A subsequent hotplug operation on any logical CPU except the BSP will
111 * cause for the supported CPUID feature bits to get re-detected and, if
112 * RTM and HLE get cleared all of a sudden, but, userspace did consult
113 * them before the update, then funny explosions will happen. Long story
114 * short: the kernel doesn't modify CPUID feature bits after booting.
115 *
116 * That's why, this function's call in init_intel() doesn't clear the
117 * feature flags.
118 */
119static void tsx_clear_cpuid(void)
120{
121	u64 msr;
122
123	/*
124	 * MSR_TFA_TSX_CPUID_CLEAR bit is only present when both CPUID
125	 * bits RTM_ALWAYS_ABORT and TSX_FORCE_ABORT are present.
126	 */
127	if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT) &&
128	    boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
129		rdmsrq(MSR_TSX_FORCE_ABORT, msr);
130		msr |= MSR_TFA_TSX_CPUID_CLEAR;
131		wrmsrq(MSR_TSX_FORCE_ABORT, msr);
132	} else if (cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL)) {
133		rdmsrq(MSR_IA32_TSX_CTRL, msr);
134		msr |= TSX_CTRL_CPUID_CLEAR;
135		wrmsrq(MSR_IA32_TSX_CTRL, msr);
136	}
137}
138
139/*
140 * Disable TSX development mode
141 *
142 * When the microcode released in Feb 2022 is applied, TSX will be disabled by
143 * default on some processors. MSR 0x122 (TSX_CTRL) and MSR 0x123
144 * (IA32_MCU_OPT_CTRL) can be used to re-enable TSX for development, doing so is
145 * not recommended for production deployments. In particular, applying MD_CLEAR
146 * flows for mitigation of the Intel TSX Asynchronous Abort (TAA) transient
147 * execution attack may not be effective on these processors when Intel TSX is
148 * enabled with updated microcode.
149 */
150static void tsx_dev_mode_disable(void)
151{
152	u64 mcu_opt_ctrl;
153
154	/* Check if RTM_ALLOW exists */
155	if (!boot_cpu_has_bug(X86_BUG_TAA) ||
156	    !cpu_feature_enabled(X86_FEATURE_MSR_TSX_CTRL) ||
157	    !cpu_feature_enabled(X86_FEATURE_SRBDS_CTRL))
158		return;
159
160	rdmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
161
162	if (mcu_opt_ctrl & RTM_ALLOW) {
163		mcu_opt_ctrl &= ~RTM_ALLOW;
164		wrmsrq(MSR_IA32_MCU_OPT_CTRL, mcu_opt_ctrl);
165		setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT);
166	}
167}
168
169static int __init tsx_parse_cmdline(char *str)
170{
171	if (!str)
172		return -EINVAL;
173
174	if (!strcmp(str, "on")) {
175		tsx_ctrl_state = TSX_CTRL_ENABLE;
176	} else if (!strcmp(str, "off")) {
177		tsx_ctrl_state = TSX_CTRL_DISABLE;
178	} else if (!strcmp(str, "auto")) {
179		tsx_ctrl_state = TSX_CTRL_AUTO;
180	} else {
181		tsx_ctrl_state = TSX_CTRL_DISABLE;
182		pr_err("invalid option, defaulting to off\n");
183	}
184
185	return 0;
186}
187early_param("tsx", tsx_parse_cmdline);
188
189void __init tsx_init(void)
190{
191	tsx_dev_mode_disable();
192
193	/*
194	 * Hardware will always abort a TSX transaction when the CPUID bit
195	 * RTM_ALWAYS_ABORT is set. In this case, it is better not to enumerate
196	 * CPUID.RTM and CPUID.HLE bits. Clear them here.
197	 */
198	if (boot_cpu_has(X86_FEATURE_RTM_ALWAYS_ABORT)) {
199		tsx_ctrl_state = TSX_CTRL_RTM_ALWAYS_ABORT;
200		tsx_clear_cpuid();
201		setup_clear_cpu_cap(X86_FEATURE_RTM);
202		setup_clear_cpu_cap(X86_FEATURE_HLE);
203		return;
204	}
205
206	/*
207	 * TSX is controlled via MSR_IA32_TSX_CTRL.  However, support for this
208	 * MSR is enumerated by ARCH_CAP_TSX_MSR bit in MSR_IA32_ARCH_CAPABILITIES.
209	 *
210	 * TSX control (aka MSR_IA32_TSX_CTRL) is only available after a
211	 * microcode update on CPUs that have their MSR_IA32_ARCH_CAPABILITIES
212	 * bit MDS_NO=1. CPUs with MDS_NO=0 are not planned to get
213	 * MSR_IA32_TSX_CTRL support even after a microcode update. Thus,
214	 * tsx= cmdline requests will do nothing on CPUs without
215	 * MSR_IA32_TSX_CTRL support.
216	 */
217	if (x86_read_arch_cap_msr() & ARCH_CAP_TSX_CTRL_MSR) {
218		setup_force_cpu_cap(X86_FEATURE_MSR_TSX_CTRL);
219	} else {
220		tsx_ctrl_state = TSX_CTRL_NOT_SUPPORTED;
221		return;
222	}
223
224	if (tsx_ctrl_state == TSX_CTRL_AUTO)
225		tsx_ctrl_state = x86_get_tsx_auto_mode();
226
227	if (tsx_ctrl_state == TSX_CTRL_DISABLE) {
228		tsx_disable();
229
230		/*
231		 * tsx_disable() will change the state of the RTM and HLE CPUID
232		 * bits. Clear them here since they are now expected to be not
233		 * set.
234		 */
235		setup_clear_cpu_cap(X86_FEATURE_RTM);
236		setup_clear_cpu_cap(X86_FEATURE_HLE);
237	} else if (tsx_ctrl_state == TSX_CTRL_ENABLE) {
238
239		/*
240		 * HW defaults TSX to be enabled at bootup.
241		 * We may still need the TSX enable support
242		 * during init for special cases like
243		 * kexec after TSX is disabled.
244		 */
245		tsx_enable();
246
247		/*
248		 * tsx_enable() will change the state of the RTM and HLE CPUID
249		 * bits. Force them here since they are now expected to be set.
250		 */
251		setup_force_cpu_cap(X86_FEATURE_RTM);
252		setup_force_cpu_cap(X86_FEATURE_HLE);
253	}
254}
255
256void tsx_ap_init(void)
257{
258	tsx_dev_mode_disable();
259
260	if (tsx_ctrl_state == TSX_CTRL_ENABLE)
261		tsx_enable();
262	else if (tsx_ctrl_state == TSX_CTRL_DISABLE)
263		tsx_disable();
264	else if (tsx_ctrl_state == TSX_CTRL_RTM_ALWAYS_ABORT)
265		/* See comment over that function for more details. */
266		tsx_clear_cpuid();
267}