acpi_pad.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * acpi_pad.c ACPI Processor Aggregator Driver
  4 *
  5 * Copyright (c) 2009, Intel Corporation.
  6 */
  7
  8#include <linux/kernel.h>
  9#include <linux/cpumask.h>
 10#include <linux/module.h>
 11#include <linux/init.h>
 12#include <linux/types.h>
 13#include <linux/kthread.h>
 14#include <uapi/linux/sched/types.h>
 15#include <linux/freezer.h>
 16#include <linux/cpu.h>
 17#include <linux/tick.h>
 18#include <linux/slab.h>
 19#include <linux/acpi.h>
 20#include <linux/perf_event.h>
 21#include <linux/platform_device.h>
 22#include <asm/cpuid/api.h>
 23#include <asm/mwait.h>
 24#include <xen/xen.h>
 25
 26#define ACPI_PROCESSOR_AGGREGATOR_CLASS	"acpi_pad"
 27#define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
 28#define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
 29
 30#define ACPI_PROCESSOR_AGGREGATOR_STATUS_SUCCESS	0
 31#define ACPI_PROCESSOR_AGGREGATOR_STATUS_NO_ACTION	1
 32
 33static DEFINE_MUTEX(isolated_cpus_lock);
 34static DEFINE_MUTEX(round_robin_lock);
 35
 36static unsigned int power_saving_mwait_eax;
 37
 38static unsigned char tsc_detected_unstable;
 39static unsigned char tsc_marked_unstable;
 40
 41static void power_saving_mwait_init(void)
 42{
 43	unsigned int eax, ebx, ecx, edx;
 44	unsigned int highest_cstate = 0;
 45	unsigned int highest_subcstate = 0;
 46	int i;
 47
 48	if (!boot_cpu_has(X86_FEATURE_MWAIT))
 49		return;
 50
 51	cpuid(CPUID_LEAF_MWAIT, &eax, &ebx, &ecx, &edx);
 52
 53	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
 54	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
 55		return;
 56
 57	edx >>= MWAIT_SUBSTATE_SIZE;
 58	for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
 59		if (edx & MWAIT_SUBSTATE_MASK) {
 60			highest_cstate = i;
 61			highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
 62		}
 63	}
 64	power_saving_mwait_eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
 65		(highest_subcstate - 1);
 66
 67#if defined(CONFIG_X86)
 68	switch (boot_cpu_data.x86_vendor) {
 69	case X86_VENDOR_HYGON:
 70	case X86_VENDOR_AMD:
 71	case X86_VENDOR_INTEL:
 72	case X86_VENDOR_ZHAOXIN:
 73	case X86_VENDOR_CENTAUR:
 74		/*
 75		 * AMD Fam10h TSC will tick in all
 76		 * C/P/S0/S1 states when this bit is set.
 77		 */
 78		if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
 79			tsc_detected_unstable = 1;
 80		break;
 81	default:
 82		/* TSC could halt in idle */
 83		tsc_detected_unstable = 1;
 84	}
 85#endif
 86}
 87
 88static unsigned long cpu_weight[NR_CPUS];
 89static int tsk_in_cpu[NR_CPUS] = {[0 ... NR_CPUS-1] = -1};
 90static DECLARE_BITMAP(pad_busy_cpus_bits, NR_CPUS);
 91static void round_robin_cpu(unsigned int tsk_index)
 92{
 93	struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
 94	cpumask_var_t tmp;
 95	int cpu;
 96	unsigned long min_weight = -1;
 97	unsigned long preferred_cpu;
 98
 99	if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
100		return;
101
102	mutex_lock(&round_robin_lock);
103	cpumask_clear(tmp);
104	for_each_cpu(cpu, pad_busy_cpus)
105		cpumask_or(tmp, tmp, topology_sibling_cpumask(cpu));
106	cpumask_andnot(tmp, cpu_online_mask, tmp);
107	/* avoid HT siblings if possible */
108	if (cpumask_empty(tmp))
109		cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus);
110	if (cpumask_empty(tmp)) {
111		mutex_unlock(&round_robin_lock);
112		free_cpumask_var(tmp);
113		return;
114	}
115	for_each_cpu(cpu, tmp) {
116		if (cpu_weight[cpu] < min_weight) {
117			min_weight = cpu_weight[cpu];
118			preferred_cpu = cpu;
119		}
120	}
121
122	if (tsk_in_cpu[tsk_index] != -1)
123		cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
124	tsk_in_cpu[tsk_index] = preferred_cpu;
125	cpumask_set_cpu(preferred_cpu, pad_busy_cpus);
126	cpu_weight[preferred_cpu]++;
127	mutex_unlock(&round_robin_lock);
128
129	set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu));
130
131	free_cpumask_var(tmp);
132}
133
134static void exit_round_robin(unsigned int tsk_index)
135{
136	struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
137
138	if (tsk_in_cpu[tsk_index] != -1) {
139		cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
140		tsk_in_cpu[tsk_index] = -1;
141	}
142}
143
144static unsigned int idle_pct = 5; /* percentage */
145static unsigned int round_robin_time = 1; /* second */
146static int power_saving_thread(void *data)
147{
148	int do_sleep;
149	unsigned int tsk_index = (unsigned long)data;
150	u64 last_jiffies = 0;
151
152	sched_set_fifo_low(current);
153
154	while (!kthread_should_stop()) {
155		unsigned long expire_time;
156
157		/* round robin to cpus */
158		expire_time = last_jiffies + round_robin_time * HZ;
159		if (time_before(expire_time, jiffies)) {
160			last_jiffies = jiffies;
161			round_robin_cpu(tsk_index);
162		}
163
164		do_sleep = 0;
165
166		expire_time = jiffies + HZ * (100 - idle_pct) / 100;
167
168		while (!need_resched()) {
169			if (tsc_detected_unstable && !tsc_marked_unstable) {
170				/* TSC could halt in idle, so notify users */
171				mark_tsc_unstable("TSC halts in idle");
172				tsc_marked_unstable = 1;
173			}
174			local_irq_disable();
175
176			perf_lopwr_cb(true);
177
178			tick_broadcast_enable();
179			tick_broadcast_enter();
180			stop_critical_timings();
181
182			mwait_idle_with_hints(power_saving_mwait_eax, 1);
183
184			start_critical_timings();
185			tick_broadcast_exit();
186
187			perf_lopwr_cb(false);
188
189			local_irq_enable();
190
191			if (time_before(expire_time, jiffies)) {
192				do_sleep = 1;
193				break;
194			}
195		}
196
197		/*
198		 * current sched_rt has threshold for rt task running time.
199		 * When a rt task uses 95% CPU time, the rt thread will be
200		 * scheduled out for 5% CPU time to not starve other tasks. But
201		 * the mechanism only works when all CPUs have RT task running,
202		 * as if one CPU hasn't RT task, RT task from other CPUs will
203		 * borrow CPU time from this CPU and cause RT task use > 95%
204		 * CPU time. To make 'avoid starvation' work, takes a nap here.
205		 */
206		if (unlikely(do_sleep))
207			schedule_timeout_killable(HZ * idle_pct / 100);
208
209		/* If an external event has set the need_resched flag, then
210		 * we need to deal with it, or this loop will continue to
211		 * spin without calling __mwait().
212		 */
213		if (unlikely(need_resched()))
214			schedule();
215	}
216
217	exit_round_robin(tsk_index);
218	return 0;
219}
220
221static struct task_struct *ps_tsks[NR_CPUS];
222static unsigned int ps_tsk_num;
223static int create_power_saving_task(void)
224{
225	int rc;
226
227	ps_tsks[ps_tsk_num] = kthread_run(power_saving_thread,
228		(void *)(unsigned long)ps_tsk_num,
229		"acpi_pad/%d", ps_tsk_num);
230
231	if (IS_ERR(ps_tsks[ps_tsk_num])) {
232		rc = PTR_ERR(ps_tsks[ps_tsk_num]);
233		ps_tsks[ps_tsk_num] = NULL;
234	} else {
235		rc = 0;
236		ps_tsk_num++;
237	}
238
239	return rc;
240}
241
242static void destroy_power_saving_task(void)
243{
244	if (ps_tsk_num > 0) {
245		ps_tsk_num--;
246		kthread_stop(ps_tsks[ps_tsk_num]);
247		ps_tsks[ps_tsk_num] = NULL;
248	}
249}
250
251static void set_power_saving_task_num(unsigned int num)
252{
253	if (num > ps_tsk_num) {
254		while (ps_tsk_num < num) {
255			if (create_power_saving_task())
256				return;
257		}
258	} else if (num < ps_tsk_num) {
259		while (ps_tsk_num > num)
260			destroy_power_saving_task();
261	}
262}
263
264static void acpi_pad_idle_cpus(unsigned int num_cpus)
265{
266	cpus_read_lock();
267
268	num_cpus = min_t(unsigned int, num_cpus, num_online_cpus());
269	set_power_saving_task_num(num_cpus);
270
271	cpus_read_unlock();
272}
273
274static uint32_t acpi_pad_idle_cpus_num(void)
275{
276	return ps_tsk_num;
277}
278
279static ssize_t rrtime_store(struct device *dev,
280	struct device_attribute *attr, const char *buf, size_t count)
281{
282	unsigned long num;
283
284	if (kstrtoul(buf, 0, &num))
285		return -EINVAL;
286	if (num < 1 || num >= 100)
287		return -EINVAL;
288	mutex_lock(&isolated_cpus_lock);
289	round_robin_time = num;
290	mutex_unlock(&isolated_cpus_lock);
291	return count;
292}
293
294static ssize_t rrtime_show(struct device *dev,
295	struct device_attribute *attr, char *buf)
296{
297	return sysfs_emit(buf, "%d\n", round_robin_time);
298}
299static DEVICE_ATTR_RW(rrtime);
300
301static ssize_t idlepct_store(struct device *dev,
302	struct device_attribute *attr, const char *buf, size_t count)
303{
304	unsigned long num;
305
306	if (kstrtoul(buf, 0, &num))
307		return -EINVAL;
308	if (num < 1 || num >= 100)
309		return -EINVAL;
310	mutex_lock(&isolated_cpus_lock);
311	idle_pct = num;
312	mutex_unlock(&isolated_cpus_lock);
313	return count;
314}
315
316static ssize_t idlepct_show(struct device *dev,
317	struct device_attribute *attr, char *buf)
318{
319	return sysfs_emit(buf, "%d\n", idle_pct);
320}
321static DEVICE_ATTR_RW(idlepct);
322
323static ssize_t idlecpus_store(struct device *dev,
324	struct device_attribute *attr, const char *buf, size_t count)
325{
326	unsigned long num;
327
328	if (kstrtoul(buf, 0, &num))
329		return -EINVAL;
330	mutex_lock(&isolated_cpus_lock);
331	acpi_pad_idle_cpus(num);
332	mutex_unlock(&isolated_cpus_lock);
333	return count;
334}
335
336static ssize_t idlecpus_show(struct device *dev,
337	struct device_attribute *attr, char *buf)
338{
339	return cpumap_print_to_pagebuf(false, buf,
340				       to_cpumask(pad_busy_cpus_bits));
341}
342
343static DEVICE_ATTR_RW(idlecpus);
344
345static struct attribute *acpi_pad_attrs[] = {
346	&dev_attr_idlecpus.attr,
347	&dev_attr_idlepct.attr,
348	&dev_attr_rrtime.attr,
349	NULL
350};
351
352ATTRIBUTE_GROUPS(acpi_pad);
353
354/*
355 * Query firmware how many CPUs should be idle
356 * return -1 on failure
357 */
358static int acpi_pad_pur(acpi_handle handle)
359{
360	struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL};
361	union acpi_object *package;
362	int num = -1;
363
364	if (ACPI_FAILURE(acpi_evaluate_object(handle, "_PUR", NULL, &buffer)))
365		return num;
366
367	if (!buffer.length || !buffer.pointer)
368		return num;
369
370	package = buffer.pointer;
371
372	if (package->type == ACPI_TYPE_PACKAGE &&
373		package->package.count == 2 &&
374		package->package.elements[0].integer.value == 1) /* rev 1 */
375
376		num = package->package.elements[1].integer.value;
377
378	kfree(buffer.pointer);
379	return num;
380}
381
382static void acpi_pad_handle_notify(acpi_handle handle)
383{
384	int num_cpus;
385	uint32_t idle_cpus;
386	struct acpi_buffer param = {
387		.length = 4,
388		.pointer = (void *)&idle_cpus,
389	};
390	u32 status;
391
392	mutex_lock(&isolated_cpus_lock);
393	num_cpus = acpi_pad_pur(handle);
394	if (num_cpus < 0) {
395		/* The ACPI specification says that if no action was performed when
396		 * processing the _PUR object, _OST should still be evaluated, albeit
397		 * with a different status code.
398		 */
399		status = ACPI_PROCESSOR_AGGREGATOR_STATUS_NO_ACTION;
400	} else {
401		status = ACPI_PROCESSOR_AGGREGATOR_STATUS_SUCCESS;
402		acpi_pad_idle_cpus(num_cpus);
403	}
404
405	idle_cpus = acpi_pad_idle_cpus_num();
406	acpi_evaluate_ost(handle, ACPI_PROCESSOR_AGGREGATOR_NOTIFY, status, &param);
407	mutex_unlock(&isolated_cpus_lock);
408}
409
410static void acpi_pad_notify(acpi_handle handle, u32 event,
411	void *data)
412{
413	struct acpi_device *adev = data;
414
415	switch (event) {
416	case ACPI_PROCESSOR_AGGREGATOR_NOTIFY:
417		acpi_pad_handle_notify(handle);
418		acpi_bus_generate_netlink_event(adev->pnp.device_class,
419			dev_name(&adev->dev), event, 0);
420		break;
421	default:
422		pr_warn("Unsupported event [0x%x]\n", event);
423		break;
424	}
425}
426
427static int acpi_pad_probe(struct platform_device *pdev)
428{
429	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
430	acpi_status status;
431
432	strscpy(acpi_device_name(adev), ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME);
433	strscpy(acpi_device_class(adev), ACPI_PROCESSOR_AGGREGATOR_CLASS);
434
435	status = acpi_install_notify_handler(adev->handle,
436		ACPI_DEVICE_NOTIFY, acpi_pad_notify, adev);
437
438	if (ACPI_FAILURE(status))
439		return -ENODEV;
440
441	return 0;
442}
443
444static void acpi_pad_remove(struct platform_device *pdev)
445{
446	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
447
448	mutex_lock(&isolated_cpus_lock);
449	acpi_pad_idle_cpus(0);
450	mutex_unlock(&isolated_cpus_lock);
451
452	acpi_remove_notify_handler(adev->handle,
453		ACPI_DEVICE_NOTIFY, acpi_pad_notify);
454}
455
456static const struct acpi_device_id pad_device_ids[] = {
457	{"ACPI000C", 0},
458	{"", 0},
459};
460MODULE_DEVICE_TABLE(acpi, pad_device_ids);
461
462static struct platform_driver acpi_pad_driver = {
463	.probe = acpi_pad_probe,
464	.remove = acpi_pad_remove,
465	.driver = {
466		.dev_groups = acpi_pad_groups,
467		.name = "processor_aggregator",
468		.acpi_match_table = pad_device_ids,
469	},
470};
471
472static int __init acpi_pad_init(void)
473{
474	/* Xen ACPI PAD is used when running as Xen Dom0. */
475	if (xen_initial_domain())
476		return -ENODEV;
477
478	power_saving_mwait_init();
479	if (power_saving_mwait_eax == 0)
480		return -EINVAL;
481
482	return platform_driver_register(&acpi_pad_driver);
483}
484
485static void __exit acpi_pad_exit(void)
486{
487	platform_driver_unregister(&acpi_pad_driver);
488}
489
490module_init(acpi_pad_init);
491module_exit(acpi_pad_exit);
492MODULE_AUTHOR("Shaohua Li<shaohua.li@intel.com>");
493MODULE_DESCRIPTION("ACPI Processor Aggregator Driver");
494MODULE_LICENSE("GPL");