x86_pkg_temp_thermal.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * x86_pkg_temp_thermal driver
  4 * Copyright (c) 2013, Intel Corporation.
  5 */
  6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7
  8#include <linux/module.h>
  9#include <linux/init.h>
 10#include <linux/intel_tcc.h>
 11#include <linux/err.h>
 12#include <linux/param.h>
 13#include <linux/device.h>
 14#include <linux/platform_device.h>
 15#include <linux/cpu.h>
 16#include <linux/smp.h>
 17#include <linux/slab.h>
 18#include <linux/pm.h>
 19#include <linux/thermal.h>
 20#include <linux/debugfs.h>
 21
 22#include <asm/cpu_device_id.h>
 23#include <asm/msr.h>
 24
 25#include "thermal_interrupt.h"
 26
 27/*
 28* Rate control delay: Idea is to introduce denounce effect
 29* This should be long enough to avoid reduce events, when
 30* threshold is set to a temperature, which is constantly
 31* violated, but at the short enough to take any action.
 32* The action can be remove threshold or change it to next
 33* interesting setting. Based on experiments, in around
 34* every 5 seconds under load will give us a significant
 35* temperature change.
 36*/
 37#define PKG_TEMP_THERMAL_NOTIFY_DELAY	5000
 38static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
 39module_param(notify_delay_ms, int, 0644);
 40MODULE_PARM_DESC(notify_delay_ms,
 41	"User space notification delay in milli seconds.");
 42
 43/* Number of trip points in thermal zone. Currently it can't
 44* be more than 2. MSR can allow setting and getting notifications
 45* for only 2 thresholds. This define enforces this, if there
 46* is some wrong values returned by cpuid for number of thresholds.
 47*/
 48#define MAX_NUMBER_OF_TRIPS	2
 49
 50struct zone_device {
 51	int				cpu;
 52	bool				work_scheduled;
 53	u32				msr_pkg_therm_low;
 54	u32				msr_pkg_therm_high;
 55	struct delayed_work		work;
 56	struct thermal_zone_device	*tzone;
 57	struct cpumask			cpumask;
 58};
 59
 60static struct thermal_zone_params pkg_temp_tz_params = {
 61	.no_hwmon	= true,
 62};
 63
 64/* Keep track of how many zone pointers we allocated in init() */
 65static int max_id __read_mostly;
 66/* Array of zone pointers */
 67static struct zone_device **zones;
 68/* Serializes interrupt notification, work and hotplug */
 69static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
 70/* Protects zone operation in the work function against hotplug removal */
 71static DEFINE_MUTEX(thermal_zone_mutex);
 72
 73/* The dynamically assigned cpu hotplug state for module_exit() */
 74static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
 75
 76/* Debug counters to show using debugfs */
 77static struct dentry *debugfs;
 78static unsigned int pkg_interrupt_cnt;
 79static unsigned int pkg_work_cnt;
 80
 81static void pkg_temp_debugfs_init(void)
 82{
 83	debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
 84
 85	debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
 86			   &pkg_interrupt_cnt);
 87	debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
 88			   &pkg_work_cnt);
 89}
 90
 91/*
 92 * Protection:
 93 *
 94 * - cpu hotplug: Read serialized by cpu hotplug lock
 95 *		  Write must hold pkg_temp_lock
 96 *
 97 * - Other callsites: Must hold pkg_temp_lock
 98 */
 99static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
100{
101	int id = topology_logical_die_id(cpu);
102
103	if (id >= 0 && id < max_id)
104		return zones[id];
105	return NULL;
106}
107
108static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
109{
110	struct zone_device *zonedev = thermal_zone_device_priv(tzd);
111	int val, ret;
112
113	ret = intel_tcc_get_temp(zonedev->cpu, &val, true);
114	if (ret < 0)
115		return ret;
116
117	*temp = val * 1000;
118	pr_debug("sys_get_curr_temp %d\n", *temp);
119	return 0;
120}
121
122static int
123sys_set_trip_temp(struct thermal_zone_device *tzd,
124		  const struct thermal_trip *trip, int temp)
125{
126	struct zone_device *zonedev = thermal_zone_device_priv(tzd);
127	unsigned int trip_index = THERMAL_TRIP_PRIV_TO_INT(trip->priv);
128	u32 l, h, mask, shift, intr;
129	int tj_max, val, ret;
130
131	if (temp == THERMAL_TEMP_INVALID)
132		temp = 0;
133
134	tj_max = intel_tcc_get_tjmax(zonedev->cpu);
135	if (tj_max < 0)
136		return tj_max;
137	tj_max *= 1000;
138
139	val = (tj_max - temp)/1000;
140
141	if (trip_index >= MAX_NUMBER_OF_TRIPS || val < 0 || val > 0x7f)
142		return -EINVAL;
143
144	ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
145			   &l, &h);
146	if (ret < 0)
147		return ret;
148
149	if (trip_index) {
150		mask = THERM_MASK_THRESHOLD1;
151		shift = THERM_SHIFT_THRESHOLD1;
152		intr = THERM_INT_THRESHOLD1_ENABLE;
153	} else {
154		mask = THERM_MASK_THRESHOLD0;
155		shift = THERM_SHIFT_THRESHOLD0;
156		intr = THERM_INT_THRESHOLD0_ENABLE;
157	}
158	l &= ~mask;
159	/*
160	* When users space sets a trip temperature == 0, which is indication
161	* that, it is no longer interested in receiving notifications.
162	*/
163	if (!temp) {
164		l &= ~intr;
165	} else {
166		l |= val << shift;
167		l |= intr;
168	}
169
170	return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
171			l, h);
172}
173
174/* Thermal zone callback registry */
175static const struct thermal_zone_device_ops tzone_ops = {
176	.get_temp = sys_get_curr_temp,
177	.set_trip_temp = sys_set_trip_temp,
178};
179
180static bool pkg_thermal_rate_control(void)
181{
182	return true;
183}
184
185/* Enable threshold interrupt on local package/cpu */
186static inline void enable_pkg_thres_interrupt(void)
187{
188	u8 thres_0, thres_1;
189	u32 l, h;
190
191	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
192	/* only enable/disable if it had valid threshold value */
193	thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
194	thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
195	if (thres_0)
196		l |= THERM_INT_THRESHOLD0_ENABLE;
197	if (thres_1)
198		l |= THERM_INT_THRESHOLD1_ENABLE;
199	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
200}
201
202/* Disable threshold interrupt on local package/cpu */
203static inline void disable_pkg_thres_interrupt(void)
204{
205	u32 l, h;
206
207	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
208
209	l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
210	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
211}
212
213static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
214{
215	struct thermal_zone_device *tzone = NULL;
216	int cpu = smp_processor_id();
217	struct zone_device *zonedev;
218
219	mutex_lock(&thermal_zone_mutex);
220	raw_spin_lock_irq(&pkg_temp_lock);
221	++pkg_work_cnt;
222
223	zonedev = pkg_temp_thermal_get_dev(cpu);
224	if (!zonedev) {
225		raw_spin_unlock_irq(&pkg_temp_lock);
226		mutex_unlock(&thermal_zone_mutex);
227		return;
228	}
229	zonedev->work_scheduled = false;
230
231	thermal_clear_package_intr_status(PACKAGE_LEVEL, THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
232	tzone = zonedev->tzone;
233
234	enable_pkg_thres_interrupt();
235	raw_spin_unlock_irq(&pkg_temp_lock);
236
237	/*
238	 * If tzone is not NULL, then thermal_zone_mutex will prevent the
239	 * concurrent removal in the cpu offline callback.
240	 */
241	if (tzone)
242		thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
243
244	mutex_unlock(&thermal_zone_mutex);
245}
246
247static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
248{
249	unsigned long ms = msecs_to_jiffies(notify_delay_ms);
250
251	schedule_delayed_work_on(cpu, work, ms);
252}
253
254static int pkg_thermal_notify(u64 msr_val)
255{
256	int cpu = smp_processor_id();
257	struct zone_device *zonedev;
258	unsigned long flags;
259
260	raw_spin_lock_irqsave(&pkg_temp_lock, flags);
261	++pkg_interrupt_cnt;
262
263	disable_pkg_thres_interrupt();
264
265	/* Work is per package, so scheduling it once is enough. */
266	zonedev = pkg_temp_thermal_get_dev(cpu);
267	if (zonedev && !zonedev->work_scheduled) {
268		zonedev->work_scheduled = true;
269		pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
270	}
271
272	raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
273	return 0;
274}
275
276static int pkg_temp_thermal_trips_init(int cpu, int tj_max,
277				       struct thermal_trip *trips, int num_trips)
278{
279	unsigned long thres_reg_value;
280	u32 mask, shift, eax, edx;
281	int ret, i;
282
283	for (i = 0; i < num_trips; i++) {
284
285		if (i) {
286			mask = THERM_MASK_THRESHOLD1;
287			shift = THERM_SHIFT_THRESHOLD1;
288		} else {
289			mask = THERM_MASK_THRESHOLD0;
290			shift = THERM_SHIFT_THRESHOLD0;
291		}
292
293		ret = rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
294				   &eax, &edx);
295		if (ret < 0)
296			return ret;
297
298		thres_reg_value = (eax & mask) >> shift;
299
300		trips[i].temperature = thres_reg_value ?
301			tj_max - thres_reg_value * 1000 : THERMAL_TEMP_INVALID;
302
303		trips[i].type = THERMAL_TRIP_PASSIVE;
304		trips[i].flags |= THERMAL_TRIP_FLAG_RW_TEMP;
305		trips[i].priv = THERMAL_INT_TO_TRIP_PRIV(i);
306
307		pr_debug("%s: cpu=%d, trip=%d, temp=%d\n",
308			 __func__, cpu, i, trips[i].temperature);
309	}
310
311	return 0;
312}
313
314static int pkg_temp_thermal_device_add(unsigned int cpu)
315{
316	struct thermal_trip trips[MAX_NUMBER_OF_TRIPS] = { 0 };
317	int id = topology_logical_die_id(cpu);
318	u32 eax, ebx, ecx, edx;
319	struct zone_device *zonedev;
320	int thres_count, err;
321	int tj_max;
322
323	if (id >= max_id)
324		return -ENOMEM;
325
326	cpuid(6, &eax, &ebx, &ecx, &edx);
327	thres_count = ebx & 0x07;
328	if (!thres_count)
329		return -ENODEV;
330
331	thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
332
333	tj_max = intel_tcc_get_tjmax(cpu);
334	if (tj_max < 0)
335		return tj_max;
336	tj_max *= 1000;
337
338	zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL);
339	if (!zonedev)
340		return -ENOMEM;
341
342	err = pkg_temp_thermal_trips_init(cpu, tj_max, trips, thres_count);
343	if (err)
344		goto out_kfree_zonedev;
345
346	INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
347	zonedev->cpu = cpu;
348	zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp",
349			trips, thres_count,
350			zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
351	if (IS_ERR(zonedev->tzone)) {
352		err = PTR_ERR(zonedev->tzone);
353		goto out_kfree_zonedev;
354	}
355	err = thermal_zone_device_enable(zonedev->tzone);
356	if (err)
357		goto out_unregister_tz;
358
359	/* Store MSR value for package thermal interrupt, to restore at exit */
360	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
361	      zonedev->msr_pkg_therm_high);
362
363	cpumask_set_cpu(cpu, &zonedev->cpumask);
364	raw_spin_lock_irq(&pkg_temp_lock);
365	zones[id] = zonedev;
366	raw_spin_unlock_irq(&pkg_temp_lock);
367
368	return 0;
369
370out_unregister_tz:
371	thermal_zone_device_unregister(zonedev->tzone);
372out_kfree_zonedev:
373	kfree(zonedev);
374	return err;
375}
376
377static int pkg_thermal_cpu_offline(unsigned int cpu)
378{
379	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
380	bool lastcpu, was_target;
381	int target;
382
383	if (!zonedev)
384		return 0;
385
386	target = cpumask_any_but(&zonedev->cpumask, cpu);
387	cpumask_clear_cpu(cpu, &zonedev->cpumask);
388	lastcpu = target >= nr_cpu_ids;
389	/*
390	 * Remove the sysfs files, if this is the last cpu in the package
391	 * before doing further cleanups.
392	 */
393	if (lastcpu) {
394		struct thermal_zone_device *tzone = zonedev->tzone;
395
396		/*
397		 * We must protect against a work function calling
398		 * thermal_zone_update, after/while unregister. We null out
399		 * the pointer under the zone mutex, so the worker function
400		 * won't try to call.
401		 */
402		mutex_lock(&thermal_zone_mutex);
403		zonedev->tzone = NULL;
404		mutex_unlock(&thermal_zone_mutex);
405
406		thermal_zone_device_unregister(tzone);
407	}
408
409	/* Protect against work and interrupts */
410	raw_spin_lock_irq(&pkg_temp_lock);
411
412	/*
413	 * Check whether this cpu was the current target and store the new
414	 * one. When we drop the lock, then the interrupt notify function
415	 * will see the new target.
416	 */
417	was_target = zonedev->cpu == cpu;
418	zonedev->cpu = target;
419
420	/*
421	 * If this is the last CPU in the package remove the package
422	 * reference from the array and restore the interrupt MSR. When we
423	 * drop the lock neither the interrupt notify function nor the
424	 * worker will see the package anymore.
425	 */
426	if (lastcpu) {
427		zones[topology_logical_die_id(cpu)] = NULL;
428		/* After this point nothing touches the MSR anymore. */
429		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
430		      zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
431	}
432
433	/*
434	 * Check whether there is work scheduled and whether the work is
435	 * targeted at the outgoing CPU.
436	 */
437	if (zonedev->work_scheduled && was_target) {
438		/*
439		 * To cancel the work we need to drop the lock, otherwise
440		 * we might deadlock if the work needs to be flushed.
441		 */
442		raw_spin_unlock_irq(&pkg_temp_lock);
443		cancel_delayed_work_sync(&zonedev->work);
444		raw_spin_lock_irq(&pkg_temp_lock);
445		/*
446		 * If this is not the last cpu in the package and the work
447		 * did not run after we dropped the lock above, then we
448		 * need to reschedule the work, otherwise the interrupt
449		 * stays disabled forever.
450		 */
451		if (!lastcpu && zonedev->work_scheduled)
452			pkg_thermal_schedule_work(target, &zonedev->work);
453	}
454
455	raw_spin_unlock_irq(&pkg_temp_lock);
456
457	/* Final cleanup if this is the last cpu */
458	if (lastcpu)
459		kfree(zonedev);
460
461	return 0;
462}
463
464static int pkg_thermal_cpu_online(unsigned int cpu)
465{
466	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
467	struct cpuinfo_x86 *c = &cpu_data(cpu);
468
469	/* Paranoia check */
470	if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
471		return -ENODEV;
472
473	/* If the package exists, nothing to do */
474	if (zonedev) {
475		cpumask_set_cpu(cpu, &zonedev->cpumask);
476		return 0;
477	}
478	return pkg_temp_thermal_device_add(cpu);
479}
480
481static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
482	X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
483	{}
484};
485MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
486
487static int __init pkg_temp_thermal_init(void)
488{
489	int ret;
490
491	if (!x86_match_cpu(pkg_temp_thermal_ids))
492		return -ENODEV;
493
494	max_id = topology_max_packages() * topology_max_dies_per_package();
495	zones = kcalloc(max_id, sizeof(struct zone_device *),
496			   GFP_KERNEL);
497	if (!zones)
498		return -ENOMEM;
499
500	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
501				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);
502	if (ret < 0)
503		goto err;
504
505	/* Store the state for module exit */
506	pkg_thermal_hp_state = ret;
507
508	platform_thermal_package_notify = pkg_thermal_notify;
509	platform_thermal_package_rate_control = pkg_thermal_rate_control;
510
511	 /* Don't care if it fails */
512	pkg_temp_debugfs_init();
513	return 0;
514
515err:
516	kfree(zones);
517	return ret;
518}
519module_init(pkg_temp_thermal_init)
520
521static void __exit pkg_temp_thermal_exit(void)
522{
523	platform_thermal_package_notify = NULL;
524	platform_thermal_package_rate_control = NULL;
525
526	cpuhp_remove_state(pkg_thermal_hp_state);
527	debugfs_remove_recursive(debugfs);
528	kfree(zones);
529}
530module_exit(pkg_temp_thermal_exit)
531
532MODULE_IMPORT_NS("INTEL_TCC");
533MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
534MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
535MODULE_LICENSE("GPL v2");