umwait.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/syscore_ops.h>
  3#include <linux/suspend.h>
  4#include <linux/cpu.h>
  5
  6#include <asm/msr.h>
  7#include <asm/mwait.h>
  8
  9#define UMWAIT_C02_ENABLE	0
 10
 11#define UMWAIT_CTRL_VAL(max_time, c02_disable)				\
 12	(((max_time) & MSR_IA32_UMWAIT_CONTROL_TIME_MASK) |		\
 13	((c02_disable) & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE))
 14
 15/*
 16 * Cache IA32_UMWAIT_CONTROL MSR. This is a systemwide control. By default,
 17 * umwait max time is 100000 in TSC-quanta and C0.2 is enabled
 18 */
 19static u32 umwait_control_cached = UMWAIT_CTRL_VAL(100000, UMWAIT_C02_ENABLE);
 20
 21/*
 22 * Cache the original IA32_UMWAIT_CONTROL MSR value which is configured by
 23 * hardware or BIOS before kernel boot.
 24 */
 25static u32 orig_umwait_control_cached __ro_after_init;
 26
 27/*
 28 * Serialize access to umwait_control_cached and IA32_UMWAIT_CONTROL MSR in
 29 * the sysfs write functions.
 30 */
 31static DEFINE_MUTEX(umwait_lock);
 32
 33static void umwait_update_control_msr(void * unused)
 34{
 35	lockdep_assert_irqs_disabled();
 36	wrmsrq(MSR_IA32_UMWAIT_CONTROL, READ_ONCE(umwait_control_cached));
 37}
 38
 39/*
 40 * The CPU hotplug callback sets the control MSR to the global control
 41 * value.
 42 *
 43 * Disable interrupts so the read of umwait_control_cached and the WRMSR
 44 * are protected against a concurrent sysfs write. Otherwise the sysfs
 45 * write could update the cached value after it had been read on this CPU
 46 * and issue the IPI before the old value had been written. The IPI would
 47 * interrupt, write the new value and after return from IPI the previous
 48 * value would be written by this CPU.
 49 *
 50 * With interrupts disabled the upcoming CPU either sees the new control
 51 * value or the IPI is updating this CPU to the new control value after
 52 * interrupts have been reenabled.
 53 */
 54static int umwait_cpu_online(unsigned int cpu)
 55{
 56	local_irq_disable();
 57	umwait_update_control_msr(NULL);
 58	local_irq_enable();
 59	return 0;
 60}
 61
 62/*
 63 * The CPU hotplug callback sets the control MSR to the original control
 64 * value.
 65 */
 66static int umwait_cpu_offline(unsigned int cpu)
 67{
 68	/*
 69	 * This code is protected by the CPU hotplug already and
 70	 * orig_umwait_control_cached is never changed after it caches
 71	 * the original control MSR value in umwait_init(). So there
 72	 * is no race condition here.
 73	 */
 74	wrmsrq(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached);
 75
 76	return 0;
 77}
 78
 79/*
 80 * On resume, restore IA32_UMWAIT_CONTROL MSR on the boot processor which
 81 * is the only active CPU at this time. The MSR is set up on the APs via the
 82 * CPU hotplug callback.
 83 *
 84 * This function is invoked on resume from suspend and hibernation. On
 85 * resume from suspend the restore should be not required, but we neither
 86 * trust the firmware nor does it matter if the same value is written
 87 * again.
 88 */
 89static void umwait_syscore_resume(void *data)
 90{
 91	umwait_update_control_msr(NULL);
 92}
 93
 94static const struct syscore_ops umwait_syscore_ops = {
 95	.resume	= umwait_syscore_resume,
 96};
 97
 98static struct syscore umwait_syscore = {
 99	.ops = &umwait_syscore_ops,
100};
101
102/* sysfs interface */
103
104/*
105 * When bit 0 in IA32_UMWAIT_CONTROL MSR is 1, C0.2 is disabled.
106 * Otherwise, C0.2 is enabled.
107 */
108static inline bool umwait_ctrl_c02_enabled(u32 ctrl)
109{
110	return !(ctrl & MSR_IA32_UMWAIT_CONTROL_C02_DISABLE);
111}
112
113static inline u32 umwait_ctrl_max_time(u32 ctrl)
114{
115	return ctrl & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
116}
117
118static inline void umwait_update_control(u32 maxtime, bool c02_enable)
119{
120	u32 ctrl = maxtime & MSR_IA32_UMWAIT_CONTROL_TIME_MASK;
121
122	if (!c02_enable)
123		ctrl |= MSR_IA32_UMWAIT_CONTROL_C02_DISABLE;
124
125	WRITE_ONCE(umwait_control_cached, ctrl);
126	/* Propagate to all CPUs */
127	on_each_cpu(umwait_update_control_msr, NULL, 1);
128}
129
130static ssize_t
131enable_c02_show(struct device *dev, struct device_attribute *attr, char *buf)
132{
133	u32 ctrl = READ_ONCE(umwait_control_cached);
134
135	return sprintf(buf, "%d\n", umwait_ctrl_c02_enabled(ctrl));
136}
137
138static ssize_t enable_c02_store(struct device *dev,
139				struct device_attribute *attr,
140				const char *buf, size_t count)
141{
142	bool c02_enable;
143	u32 ctrl;
144	int ret;
145
146	ret = kstrtobool(buf, &c02_enable);
147	if (ret)
148		return ret;
149
150	mutex_lock(&umwait_lock);
151
152	ctrl = READ_ONCE(umwait_control_cached);
153	if (c02_enable != umwait_ctrl_c02_enabled(ctrl))
154		umwait_update_control(ctrl, c02_enable);
155
156	mutex_unlock(&umwait_lock);
157
158	return count;
159}
160static DEVICE_ATTR_RW(enable_c02);
161
162static ssize_t
163max_time_show(struct device *kobj, struct device_attribute *attr, char *buf)
164{
165	u32 ctrl = READ_ONCE(umwait_control_cached);
166
167	return sprintf(buf, "%u\n", umwait_ctrl_max_time(ctrl));
168}
169
170static ssize_t max_time_store(struct device *kobj,
171			      struct device_attribute *attr,
172			      const char *buf, size_t count)
173{
174	u32 max_time, ctrl;
175	int ret;
176
177	ret = kstrtou32(buf, 0, &max_time);
178	if (ret)
179		return ret;
180
181	/* bits[1:0] must be zero */
182	if (max_time & ~MSR_IA32_UMWAIT_CONTROL_TIME_MASK)
183		return -EINVAL;
184
185	mutex_lock(&umwait_lock);
186
187	ctrl = READ_ONCE(umwait_control_cached);
188	if (max_time != umwait_ctrl_max_time(ctrl))
189		umwait_update_control(max_time, umwait_ctrl_c02_enabled(ctrl));
190
191	mutex_unlock(&umwait_lock);
192
193	return count;
194}
195static DEVICE_ATTR_RW(max_time);
196
197static struct attribute *umwait_attrs[] = {
198	&dev_attr_enable_c02.attr,
199	&dev_attr_max_time.attr,
200	NULL
201};
202
203static struct attribute_group umwait_attr_group = {
204	.attrs = umwait_attrs,
205	.name = "umwait_control",
206};
207
208static int __init umwait_init(void)
209{
210	struct device *dev;
211	int ret;
212
213	if (!boot_cpu_has(X86_FEATURE_WAITPKG))
214		return -ENODEV;
215
216	/*
217	 * Cache the original control MSR value before the control MSR is
218	 * changed. This is the only place where orig_umwait_control_cached
219	 * is modified.
220	 */
221	rdmsrq(MSR_IA32_UMWAIT_CONTROL, orig_umwait_control_cached);
222
223	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "umwait:online",
224				umwait_cpu_online, umwait_cpu_offline);
225	if (ret < 0) {
226		/*
227		 * On failure, the control MSR on all CPUs has the
228		 * original control value.
229		 */
230		return ret;
231	}
232
233	register_syscore(&umwait_syscore);
234
235	/*
236	 * Add umwait control interface. Ignore failure, so at least the
237	 * default values are set up in case the machine manages to boot.
238	 */
239	dev = bus_get_dev_root(&cpu_subsys);
240	if (dev) {
241		ret = sysfs_create_group(&dev->kobj, &umwait_attr_group);
242		put_device(dev);
243	}
244	return ret;
245}
246device_initcall(umwait_init);