rapl.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Support Intel/AMD RAPL energy consumption counters
  4 * Copyright (C) 2013 Google, Inc., Stephane Eranian
  5 *
  6 * Intel RAPL interface is specified in the IA-32 Manual Vol3b
  7 * section 14.7.1 (September 2013)
  8 *
  9 * AMD RAPL interface for Fam17h is described in the public PPR:
 10 * https://bugzilla.kernel.org/show_bug.cgi?id=206537
 11 *
 12 * RAPL provides more controls than just reporting energy consumption
 13 * however here we only expose the 3 energy consumption free running
 14 * counters (pp0, pkg, dram).
 15 *
 16 * Each of those counters increments in a power unit defined by the
 17 * RAPL_POWER_UNIT MSR. On SandyBridge, this unit is 1/(2^16) Joules
 18 * but it can vary.
 19 *
 20 * Counter to rapl events mappings:
 21 *
 22 *  pp0 counter: consumption of all physical cores (power plane 0)
 23 * 	  event: rapl_energy_cores
 24 *    perf code: 0x1
 25 *
 26 *  pkg counter: consumption of the whole processor package
 27 *	  event: rapl_energy_pkg
 28 *    perf code: 0x2
 29 *
 30 * dram counter: consumption of the dram domain (servers only)
 31 *	  event: rapl_energy_dram
 32 *    perf code: 0x3
 33 *
 34 * gpu counter: consumption of the builtin-gpu domain (client only)
 35 *	  event: rapl_energy_gpu
 36 *    perf code: 0x4
 37 *
 38 *  psys counter: consumption of the builtin-psys domain (client only)
 39 *	  event: rapl_energy_psys
 40 *    perf code: 0x5
 41 *
 42 *  core counter: consumption of a single physical core
 43 *	  event: rapl_energy_core (power_core PMU)
 44 *    perf code: 0x1
 45 *
 46 * We manage those counters as free running (read-only). They may be
 47 * use simultaneously by other tools, such as turbostat.
 48 *
 49 * The events only support system-wide mode counting. There is no
 50 * sampling support because it does not make sense and is not
 51 * supported by the RAPL hardware.
 52 *
 53 * Because we want to avoid floating-point operations in the kernel,
 54 * the events are all reported in fixed point arithmetic (32.32).
 55 * Tools must adjust the counts to convert them to Watts using
 56 * the duration of the measurement. Tools may use a function such as
 57 * ldexp(raw_count, -32);
 58 */
 59
 60#define pr_fmt(fmt) "RAPL PMU: " fmt
 61
 62#include <linux/module.h>
 63#include <linux/slab.h>
 64#include <linux/perf_event.h>
 65#include <linux/nospec.h>
 66#include <asm/cpu_device_id.h>
 67#include <asm/intel-family.h>
 68#include <asm/msr.h>
 69#include "perf_event.h"
 70#include "probe.h"
 71
 72MODULE_DESCRIPTION("Support Intel/AMD RAPL energy consumption counters");
 73MODULE_LICENSE("GPL");
 74
 75/*
 76 * RAPL energy status counters
 77 */
 78enum perf_rapl_pkg_events {
 79	PERF_RAPL_PP0 = 0,		/* all cores */
 80	PERF_RAPL_PKG,			/* entire package */
 81	PERF_RAPL_RAM,			/* DRAM */
 82	PERF_RAPL_PP1,			/* gpu */
 83	PERF_RAPL_PSYS,			/* psys */
 84
 85	PERF_RAPL_PKG_EVENTS_MAX,
 86	NR_RAPL_PKG_DOMAINS = PERF_RAPL_PKG_EVENTS_MAX,
 87};
 88
 89#define PERF_RAPL_CORE			0		/* single core */
 90#define PERF_RAPL_CORE_EVENTS_MAX	1
 91#define NR_RAPL_CORE_DOMAINS		PERF_RAPL_CORE_EVENTS_MAX
 92
 93static const char *const rapl_pkg_domain_names[NR_RAPL_PKG_DOMAINS] __initconst = {
 94	"pp0-core",
 95	"package",
 96	"dram",
 97	"pp1-gpu",
 98	"psys",
 99};
100
101static const char *const rapl_core_domain_name __initconst = "core";
102
103/*
104 * event code: LSB 8 bits, passed in attr->config
105 * any other bit is reserved
106 */
107#define RAPL_EVENT_MASK	0xFFULL
108#define RAPL_CNTR_WIDTH 32
109
110#define RAPL_EVENT_ATTR_STR(_name, v, str)					\
111static struct perf_pmu_events_attr event_attr_##v = {				\
112	.attr		= __ATTR(_name, 0444, perf_event_sysfs_show, NULL),	\
113	.id		= 0,							\
114	.event_str	= str,							\
115};
116
117/*
118 * RAPL Package energy counter scope:
119 * 1. AMD/HYGON platforms have a per-PKG package energy counter
120 * 2. For Intel platforms
121 *	2.1. CLX-AP is multi-die and its RAPL MSRs are die-scope
122 *	2.2. Other Intel platforms are single die systems so the scope can be
123 *	     considered as either pkg-scope or die-scope, and we are considering
124 *	     them as die-scope.
125 */
126#define rapl_pkg_pmu_is_pkg_scope()				\
127	(boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||	\
128	 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON)
129
130struct rapl_pmu {
131	raw_spinlock_t		lock;
132	int			n_active;
133	int			cpu;
134	struct list_head	active_list;
135	struct pmu		*pmu;
136	ktime_t			timer_interval;
137	struct hrtimer		hrtimer;
138};
139
140struct rapl_pmus {
141	struct pmu		pmu;
142	unsigned int		nr_rapl_pmu;
143	unsigned int		cntr_mask;
144	struct rapl_pmu		*rapl_pmu[] __counted_by(nr_rapl_pmu);
145};
146
147enum rapl_unit_quirk {
148	RAPL_UNIT_QUIRK_NONE,
149	RAPL_UNIT_QUIRK_INTEL_HSW,
150	RAPL_UNIT_QUIRK_INTEL_SPR,
151};
152
153struct rapl_model {
154	struct perf_msr *rapl_pkg_msrs;
155	struct perf_msr *rapl_core_msrs;
156	unsigned long	pkg_events;
157	unsigned long	core_events;
158	unsigned int	msr_power_unit;
159	enum rapl_unit_quirk	unit_quirk;
160};
161
162 /* 1/2^hw_unit Joule */
163static int rapl_pkg_hw_unit[NR_RAPL_PKG_DOMAINS] __read_mostly;
164static int rapl_core_hw_unit __read_mostly;
165static struct rapl_pmus *rapl_pmus_pkg;
166static struct rapl_pmus *rapl_pmus_core;
167static u64 rapl_timer_ms;
168static struct rapl_model *rapl_model;
169
170/*
171 * Helper function to get the correct topology id according to the
172 * RAPL PMU scope.
173 */
174static inline unsigned int get_rapl_pmu_idx(int cpu, int scope)
175{
176	/*
177	 * Returns unsigned int, which converts the '-1' return value
178	 * (for non-existent mappings in topology map) to UINT_MAX, so
179	 * the error check in the caller is simplified.
180	 */
181	switch (scope) {
182	case PERF_PMU_SCOPE_PKG:
183		return topology_logical_package_id(cpu);
184	case PERF_PMU_SCOPE_DIE:
185		return topology_logical_die_id(cpu);
186	case PERF_PMU_SCOPE_CORE:
187		return topology_logical_core_id(cpu);
188	default:
189		return -EINVAL;
190	}
191}
192
193static inline u64 rapl_read_counter(struct perf_event *event)
194{
195	u64 raw;
196	rdmsrq(event->hw.event_base, raw);
197	return raw;
198}
199
200static inline u64 rapl_scale(u64 v, struct perf_event *event)
201{
202	int hw_unit = rapl_pkg_hw_unit[event->hw.config - 1];
203
204	if (event->pmu->scope == PERF_PMU_SCOPE_CORE)
205		hw_unit = rapl_core_hw_unit;
206
207	/*
208	 * scale delta to smallest unit (1/2^32)
209	 * users must then scale back: count * 1/(1e9*2^32) to get Joules
210	 * or use ldexp(count, -32).
211	 * Watts = Joules/Time delta
212	 */
213	return v << (32 - hw_unit);
214}
215
216static u64 rapl_event_update(struct perf_event *event)
217{
218	struct hw_perf_event *hwc = &event->hw;
219	u64 prev_raw_count, new_raw_count;
220	s64 delta, sdelta;
221	int shift = RAPL_CNTR_WIDTH;
222
223	prev_raw_count = local64_read(&hwc->prev_count);
224	do {
225		rdmsrq(event->hw.event_base, new_raw_count);
226	} while (!local64_try_cmpxchg(&hwc->prev_count,
227				      &prev_raw_count, new_raw_count));
228
229	/*
230	 * Now we have the new raw value and have updated the prev
231	 * timestamp already. We can now calculate the elapsed delta
232	 * (event-)time and add that to the generic event.
233	 *
234	 * Careful, not all hw sign-extends above the physical width
235	 * of the count.
236	 */
237	delta = (new_raw_count << shift) - (prev_raw_count << shift);
238	delta >>= shift;
239
240	sdelta = rapl_scale(delta, event);
241
242	local64_add(sdelta, &event->count);
243
244	return new_raw_count;
245}
246
247static void rapl_start_hrtimer(struct rapl_pmu *pmu)
248{
249       hrtimer_start(&pmu->hrtimer, pmu->timer_interval,
250		     HRTIMER_MODE_REL_PINNED);
251}
252
253static enum hrtimer_restart rapl_hrtimer_handle(struct hrtimer *hrtimer)
254{
255	struct rapl_pmu *rapl_pmu = container_of(hrtimer, struct rapl_pmu, hrtimer);
256	struct perf_event *event;
257	unsigned long flags;
258
259	if (!rapl_pmu->n_active)
260		return HRTIMER_NORESTART;
261
262	raw_spin_lock_irqsave(&rapl_pmu->lock, flags);
263
264	list_for_each_entry(event, &rapl_pmu->active_list, active_entry)
265		rapl_event_update(event);
266
267	raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags);
268
269	hrtimer_forward_now(hrtimer, rapl_pmu->timer_interval);
270
271	return HRTIMER_RESTART;
272}
273
274static void rapl_hrtimer_init(struct rapl_pmu *rapl_pmu)
275{
276	struct hrtimer *hr = &rapl_pmu->hrtimer;
277
278	hrtimer_setup(hr, rapl_hrtimer_handle, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
279}
280
281static void __rapl_pmu_event_start(struct rapl_pmu *rapl_pmu,
282				   struct perf_event *event)
283{
284	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
285		return;
286
287	event->hw.state = 0;
288
289	list_add_tail(&event->active_entry, &rapl_pmu->active_list);
290
291	local64_set(&event->hw.prev_count, rapl_read_counter(event));
292
293	rapl_pmu->n_active++;
294	if (rapl_pmu->n_active == 1)
295		rapl_start_hrtimer(rapl_pmu);
296}
297
298static void rapl_pmu_event_start(struct perf_event *event, int mode)
299{
300	struct rapl_pmu *rapl_pmu = event->pmu_private;
301	unsigned long flags;
302
303	raw_spin_lock_irqsave(&rapl_pmu->lock, flags);
304	__rapl_pmu_event_start(rapl_pmu, event);
305	raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags);
306}
307
308static void rapl_pmu_event_stop(struct perf_event *event, int mode)
309{
310	struct rapl_pmu *rapl_pmu = event->pmu_private;
311	struct hw_perf_event *hwc = &event->hw;
312	unsigned long flags;
313
314	raw_spin_lock_irqsave(&rapl_pmu->lock, flags);
315
316	/* mark event as deactivated and stopped */
317	if (!(hwc->state & PERF_HES_STOPPED)) {
318		WARN_ON_ONCE(rapl_pmu->n_active <= 0);
319		rapl_pmu->n_active--;
320		if (rapl_pmu->n_active == 0)
321			hrtimer_cancel(&rapl_pmu->hrtimer);
322
323		list_del(&event->active_entry);
324
325		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
326		hwc->state |= PERF_HES_STOPPED;
327	}
328
329	/* check if update of sw counter is necessary */
330	if ((mode & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
331		/*
332		 * Drain the remaining delta count out of a event
333		 * that we are disabling:
334		 */
335		rapl_event_update(event);
336		hwc->state |= PERF_HES_UPTODATE;
337	}
338
339	raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags);
340}
341
342static int rapl_pmu_event_add(struct perf_event *event, int mode)
343{
344	struct rapl_pmu *rapl_pmu = event->pmu_private;
345	struct hw_perf_event *hwc = &event->hw;
346	unsigned long flags;
347
348	raw_spin_lock_irqsave(&rapl_pmu->lock, flags);
349
350	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
351
352	if (mode & PERF_EF_START)
353		__rapl_pmu_event_start(rapl_pmu, event);
354
355	raw_spin_unlock_irqrestore(&rapl_pmu->lock, flags);
356
357	return 0;
358}
359
360static void rapl_pmu_event_del(struct perf_event *event, int flags)
361{
362	rapl_pmu_event_stop(event, PERF_EF_UPDATE);
363}
364
365static int rapl_pmu_event_init(struct perf_event *event)
366{
367	u64 cfg = event->attr.config & RAPL_EVENT_MASK;
368	int bit, rapl_pmus_scope, ret = 0;
369	struct rapl_pmu *rapl_pmu;
370	unsigned int rapl_pmu_idx;
371	struct rapl_pmus *rapl_pmus;
372
373	/* only look at RAPL events */
374	if (event->attr.type != event->pmu->type)
375		return -ENOENT;
376
377	/* unsupported modes and filters */
378	if (event->attr.sample_period) /* no sampling */
379		return -EINVAL;
380
381	/* check only supported bits are set */
382	if (event->attr.config & ~RAPL_EVENT_MASK)
383		return -EINVAL;
384
385	if (event->cpu < 0)
386		return -EINVAL;
387
388	rapl_pmus = container_of(event->pmu, struct rapl_pmus, pmu);
389	if (!rapl_pmus)
390		return -EINVAL;
391	rapl_pmus_scope = rapl_pmus->pmu.scope;
392
393	if (rapl_pmus_scope == PERF_PMU_SCOPE_PKG || rapl_pmus_scope == PERF_PMU_SCOPE_DIE) {
394		cfg = array_index_nospec((long)cfg, NR_RAPL_PKG_DOMAINS + 1);
395		if (!cfg || cfg >= NR_RAPL_PKG_DOMAINS + 1)
396			return -EINVAL;
397
398		bit = cfg - 1;
399		event->hw.event_base = rapl_model->rapl_pkg_msrs[bit].msr;
400	} else if (rapl_pmus_scope == PERF_PMU_SCOPE_CORE) {
401		cfg = array_index_nospec((long)cfg, NR_RAPL_CORE_DOMAINS + 1);
402		if (!cfg || cfg >= NR_RAPL_PKG_DOMAINS + 1)
403			return -EINVAL;
404
405		bit = cfg - 1;
406		event->hw.event_base = rapl_model->rapl_core_msrs[bit].msr;
407	} else
408		return -EINVAL;
409
410	/* check event supported */
411	if (!(rapl_pmus->cntr_mask & (1 << bit)))
412		return -EINVAL;
413
414	rapl_pmu_idx = get_rapl_pmu_idx(event->cpu, rapl_pmus_scope);
415	if (rapl_pmu_idx >= rapl_pmus->nr_rapl_pmu)
416		return -EINVAL;
417	/* must be done before validate_group */
418	rapl_pmu = rapl_pmus->rapl_pmu[rapl_pmu_idx];
419	if (!rapl_pmu)
420		return -EINVAL;
421
422	event->pmu_private = rapl_pmu;
423	event->hw.config = cfg;
424	event->hw.idx = bit;
425
426	return ret;
427}
428
429static void rapl_pmu_event_read(struct perf_event *event)
430{
431	rapl_event_update(event);
432}
433
434RAPL_EVENT_ATTR_STR(energy-cores, rapl_cores, "event=0x01");
435RAPL_EVENT_ATTR_STR(energy-pkg  ,   rapl_pkg, "event=0x02");
436RAPL_EVENT_ATTR_STR(energy-ram  ,   rapl_ram, "event=0x03");
437RAPL_EVENT_ATTR_STR(energy-gpu  ,   rapl_gpu, "event=0x04");
438RAPL_EVENT_ATTR_STR(energy-psys,   rapl_psys, "event=0x05");
439RAPL_EVENT_ATTR_STR(energy-core,   rapl_core, "event=0x01");
440
441RAPL_EVENT_ATTR_STR(energy-cores.unit, rapl_cores_unit, "Joules");
442RAPL_EVENT_ATTR_STR(energy-pkg.unit  ,   rapl_pkg_unit, "Joules");
443RAPL_EVENT_ATTR_STR(energy-ram.unit  ,   rapl_ram_unit, "Joules");
444RAPL_EVENT_ATTR_STR(energy-gpu.unit  ,   rapl_gpu_unit, "Joules");
445RAPL_EVENT_ATTR_STR(energy-psys.unit,   rapl_psys_unit, "Joules");
446RAPL_EVENT_ATTR_STR(energy-core.unit,   rapl_core_unit, "Joules");
447
448/*
449 * we compute in 0.23 nJ increments regardless of MSR
450 */
451RAPL_EVENT_ATTR_STR(energy-cores.scale, rapl_cores_scale, "2.3283064365386962890625e-10");
452RAPL_EVENT_ATTR_STR(energy-pkg.scale,     rapl_pkg_scale, "2.3283064365386962890625e-10");
453RAPL_EVENT_ATTR_STR(energy-ram.scale,     rapl_ram_scale, "2.3283064365386962890625e-10");
454RAPL_EVENT_ATTR_STR(energy-gpu.scale,     rapl_gpu_scale, "2.3283064365386962890625e-10");
455RAPL_EVENT_ATTR_STR(energy-psys.scale,   rapl_psys_scale, "2.3283064365386962890625e-10");
456RAPL_EVENT_ATTR_STR(energy-core.scale,   rapl_core_scale, "2.3283064365386962890625e-10");
457
458/*
459 * There are no default events, but we need to create
460 * "events" group (with empty attrs) before updating
461 * it with detected events.
462 */
463static struct attribute *attrs_empty[] = {
464	NULL,
465};
466
467static struct attribute_group rapl_pmu_events_group = {
468	.name = "events",
469	.attrs = attrs_empty,
470};
471
472PMU_FORMAT_ATTR(event, "config:0-7");
473static struct attribute *rapl_formats_attr[] = {
474	&format_attr_event.attr,
475	NULL,
476};
477
478static struct attribute_group rapl_pmu_format_group = {
479	.name = "format",
480	.attrs = rapl_formats_attr,
481};
482
483static const struct attribute_group *rapl_attr_groups[] = {
484	&rapl_pmu_format_group,
485	&rapl_pmu_events_group,
486	NULL,
487};
488
489static const struct attribute_group *rapl_core_attr_groups[] = {
490	&rapl_pmu_format_group,
491	&rapl_pmu_events_group,
492	NULL,
493};
494
495static struct attribute *rapl_events_cores[] = {
496	EVENT_PTR(rapl_cores),
497	EVENT_PTR(rapl_cores_unit),
498	EVENT_PTR(rapl_cores_scale),
499	NULL,
500};
501
502static struct attribute_group rapl_events_cores_group = {
503	.name  = "events",
504	.attrs = rapl_events_cores,
505};
506
507static struct attribute *rapl_events_pkg[] = {
508	EVENT_PTR(rapl_pkg),
509	EVENT_PTR(rapl_pkg_unit),
510	EVENT_PTR(rapl_pkg_scale),
511	NULL,
512};
513
514static struct attribute_group rapl_events_pkg_group = {
515	.name  = "events",
516	.attrs = rapl_events_pkg,
517};
518
519static struct attribute *rapl_events_ram[] = {
520	EVENT_PTR(rapl_ram),
521	EVENT_PTR(rapl_ram_unit),
522	EVENT_PTR(rapl_ram_scale),
523	NULL,
524};
525
526static struct attribute_group rapl_events_ram_group = {
527	.name  = "events",
528	.attrs = rapl_events_ram,
529};
530
531static struct attribute *rapl_events_gpu[] = {
532	EVENT_PTR(rapl_gpu),
533	EVENT_PTR(rapl_gpu_unit),
534	EVENT_PTR(rapl_gpu_scale),
535	NULL,
536};
537
538static struct attribute_group rapl_events_gpu_group = {
539	.name  = "events",
540	.attrs = rapl_events_gpu,
541};
542
543static struct attribute *rapl_events_psys[] = {
544	EVENT_PTR(rapl_psys),
545	EVENT_PTR(rapl_psys_unit),
546	EVENT_PTR(rapl_psys_scale),
547	NULL,
548};
549
550static struct attribute_group rapl_events_psys_group = {
551	.name  = "events",
552	.attrs = rapl_events_psys,
553};
554
555static struct attribute *rapl_events_core[] = {
556	EVENT_PTR(rapl_core),
557	EVENT_PTR(rapl_core_unit),
558	EVENT_PTR(rapl_core_scale),
559	NULL,
560};
561
562static struct attribute_group rapl_events_core_group = {
563	.name  = "events",
564	.attrs = rapl_events_core,
565};
566
567static bool test_msr(int idx, void *data)
568{
569	return test_bit(idx, (unsigned long *) data);
570}
571
572/* Only lower 32bits of the MSR represents the energy counter */
573#define RAPL_MSR_MASK 0xFFFFFFFF
574
575static struct perf_msr intel_rapl_msrs[] = {
576	[PERF_RAPL_PP0]  = { MSR_PP0_ENERGY_STATUS,      &rapl_events_cores_group, test_msr, false, RAPL_MSR_MASK },
577	[PERF_RAPL_PKG]  = { MSR_PKG_ENERGY_STATUS,      &rapl_events_pkg_group,   test_msr, false, RAPL_MSR_MASK },
578	[PERF_RAPL_RAM]  = { MSR_DRAM_ENERGY_STATUS,     &rapl_events_ram_group,   test_msr, false, RAPL_MSR_MASK },
579	[PERF_RAPL_PP1]  = { MSR_PP1_ENERGY_STATUS,      &rapl_events_gpu_group,   test_msr, false, RAPL_MSR_MASK },
580	[PERF_RAPL_PSYS] = { MSR_PLATFORM_ENERGY_STATUS, &rapl_events_psys_group,  test_msr, false, RAPL_MSR_MASK },
581};
582
583static struct perf_msr intel_rapl_spr_msrs[] = {
584	[PERF_RAPL_PP0]  = { MSR_PP0_ENERGY_STATUS,      &rapl_events_cores_group, test_msr, false, RAPL_MSR_MASK },
585	[PERF_RAPL_PKG]  = { MSR_PKG_ENERGY_STATUS,      &rapl_events_pkg_group,   test_msr, false, RAPL_MSR_MASK },
586	[PERF_RAPL_RAM]  = { MSR_DRAM_ENERGY_STATUS,     &rapl_events_ram_group,   test_msr, false, RAPL_MSR_MASK },
587	[PERF_RAPL_PP1]  = { MSR_PP1_ENERGY_STATUS,      &rapl_events_gpu_group,   test_msr, false, RAPL_MSR_MASK },
588	[PERF_RAPL_PSYS] = { MSR_PLATFORM_ENERGY_STATUS, &rapl_events_psys_group,  test_msr, true, RAPL_MSR_MASK },
589};
590
591/*
592 * Force to PERF_RAPL_PKG_EVENTS_MAX size due to:
593 * - perf_msr_probe(PERF_RAPL_PKG_EVENTS_MAX)
594 * - want to use same event codes across both architectures
595 */
596static struct perf_msr amd_rapl_pkg_msrs[] = {
597	[PERF_RAPL_PP0]  = { 0, &rapl_events_cores_group, NULL, false, 0 },
598	[PERF_RAPL_PKG]  = { MSR_AMD_PKG_ENERGY_STATUS,  &rapl_events_pkg_group,   test_msr, false, RAPL_MSR_MASK },
599	[PERF_RAPL_RAM]  = { 0, &rapl_events_ram_group,   NULL, false, 0 },
600	[PERF_RAPL_PP1]  = { 0, &rapl_events_gpu_group,   NULL, false, 0 },
601	[PERF_RAPL_PSYS] = { 0, &rapl_events_psys_group,  NULL, false, 0 },
602};
603
604static struct perf_msr amd_rapl_core_msrs[] = {
605	[PERF_RAPL_CORE] = { MSR_AMD_CORE_ENERGY_STATUS, &rapl_events_core_group,
606				 test_msr, false, RAPL_MSR_MASK },
607};
608
609static int rapl_check_hw_unit(void)
610{
611	u64 msr_rapl_power_unit_bits;
612	int i;
613
614	/* protect rdmsrq() to handle virtualization */
615	if (rdmsrq_safe(rapl_model->msr_power_unit, &msr_rapl_power_unit_bits))
616		return -1;
617	for (i = 0; i < NR_RAPL_PKG_DOMAINS; i++)
618		rapl_pkg_hw_unit[i] = (msr_rapl_power_unit_bits >> 8) & 0x1FULL;
619
620	rapl_core_hw_unit = (msr_rapl_power_unit_bits >> 8) & 0x1FULL;
621
622	switch (rapl_model->unit_quirk) {
623	/*
624	 * DRAM domain on HSW server and KNL has fixed energy unit which can be
625	 * different than the unit from power unit MSR. See
626	 * "Intel Xeon Processor E5-1600 and E5-2600 v3 Product Families, V2
627	 * of 2. Datasheet, September 2014, Reference Number: 330784-001 "
628	 */
629	case RAPL_UNIT_QUIRK_INTEL_HSW:
630		rapl_pkg_hw_unit[PERF_RAPL_RAM] = 16;
631		break;
632	/* SPR uses a fixed energy unit for Psys domain. */
633	case RAPL_UNIT_QUIRK_INTEL_SPR:
634		rapl_pkg_hw_unit[PERF_RAPL_PSYS] = 0;
635		break;
636	default:
637		break;
638	}
639
640	/*
641	 * Calculate the timer rate:
642	 * Use reference of 200W for scaling the timeout to avoid counter
643	 * overflows. 200W = 200 Joules/sec
644	 * Divide interval by 2 to avoid lockstep (2 * 100)
645	 * if hw unit is 32, then we use 2 ms 1/200/2
646	 */
647	rapl_timer_ms = 2;
648	if (rapl_pkg_hw_unit[0] < 32) {
649		rapl_timer_ms = (1000 / (2 * 100));
650		rapl_timer_ms *= (1ULL << (32 - rapl_pkg_hw_unit[0] - 1));
651	}
652	return 0;
653}
654
655static void __init rapl_advertise(void)
656{
657	int i;
658	int num_counters = hweight32(rapl_pmus_pkg->cntr_mask);
659
660	if (rapl_pmus_core)
661		num_counters += hweight32(rapl_pmus_core->cntr_mask);
662
663	pr_info("API unit is 2^-32 Joules, %d fixed counters, %llu ms ovfl timer\n",
664		num_counters, rapl_timer_ms);
665
666	for (i = 0; i < NR_RAPL_PKG_DOMAINS; i++) {
667		if (rapl_pmus_pkg->cntr_mask & (1 << i)) {
668			pr_info("hw unit of domain %s 2^-%d Joules\n",
669				rapl_pkg_domain_names[i], rapl_pkg_hw_unit[i]);
670		}
671	}
672
673	if (rapl_pmus_core && (rapl_pmus_core->cntr_mask & (1 << PERF_RAPL_CORE)))
674		pr_info("hw unit of domain %s 2^-%d Joules\n",
675			rapl_core_domain_name, rapl_core_hw_unit);
676}
677
678static void cleanup_rapl_pmus(struct rapl_pmus *rapl_pmus)
679{
680	int i;
681
682	for (i = 0; i < rapl_pmus->nr_rapl_pmu; i++)
683		kfree(rapl_pmus->rapl_pmu[i]);
684	kfree(rapl_pmus);
685}
686
687static const struct attribute_group *rapl_attr_update[] = {
688	&rapl_events_cores_group,
689	&rapl_events_pkg_group,
690	&rapl_events_ram_group,
691	&rapl_events_gpu_group,
692	&rapl_events_psys_group,
693	NULL,
694};
695
696static const struct attribute_group *rapl_core_attr_update[] = {
697	&rapl_events_core_group,
698	NULL,
699};
700
701static int __init init_rapl_pmu(struct rapl_pmus *rapl_pmus)
702{
703	struct rapl_pmu *rapl_pmu;
704	int idx;
705
706	for (idx = 0; idx < rapl_pmus->nr_rapl_pmu; idx++) {
707		rapl_pmu = kzalloc(sizeof(*rapl_pmu), GFP_KERNEL);
708		if (!rapl_pmu)
709			goto free;
710
711		raw_spin_lock_init(&rapl_pmu->lock);
712		INIT_LIST_HEAD(&rapl_pmu->active_list);
713		rapl_pmu->pmu = &rapl_pmus->pmu;
714		rapl_pmu->timer_interval = ms_to_ktime(rapl_timer_ms);
715		rapl_hrtimer_init(rapl_pmu);
716
717		rapl_pmus->rapl_pmu[idx] = rapl_pmu;
718	}
719
720	return 0;
721free:
722	for (; idx > 0; idx--)
723		kfree(rapl_pmus->rapl_pmu[idx - 1]);
724	return -ENOMEM;
725}
726
727static int __init init_rapl_pmus(struct rapl_pmus **rapl_pmus_ptr, int rapl_pmu_scope,
728				 const struct attribute_group **rapl_attr_groups,
729				 const struct attribute_group **rapl_attr_update)
730{
731	int nr_rapl_pmu = topology_max_packages();
732	struct rapl_pmus *rapl_pmus;
733	int ret;
734
735	/*
736	 * rapl_pmu_scope must be either PKG, DIE or CORE
737	 */
738	if (rapl_pmu_scope == PERF_PMU_SCOPE_DIE)
739		nr_rapl_pmu	*= topology_max_dies_per_package();
740	else if (rapl_pmu_scope == PERF_PMU_SCOPE_CORE)
741		nr_rapl_pmu	*= topology_num_cores_per_package();
742	else if (rapl_pmu_scope != PERF_PMU_SCOPE_PKG)
743		return -EINVAL;
744
745	rapl_pmus = kzalloc(struct_size(rapl_pmus, rapl_pmu, nr_rapl_pmu), GFP_KERNEL);
746	if (!rapl_pmus)
747		return -ENOMEM;
748
749	*rapl_pmus_ptr = rapl_pmus;
750
751	rapl_pmus->nr_rapl_pmu		= nr_rapl_pmu;
752	rapl_pmus->pmu.attr_groups	= rapl_attr_groups;
753	rapl_pmus->pmu.attr_update	= rapl_attr_update;
754	rapl_pmus->pmu.task_ctx_nr	= perf_invalid_context;
755	rapl_pmus->pmu.event_init	= rapl_pmu_event_init;
756	rapl_pmus->pmu.add		= rapl_pmu_event_add;
757	rapl_pmus->pmu.del		= rapl_pmu_event_del;
758	rapl_pmus->pmu.start		= rapl_pmu_event_start;
759	rapl_pmus->pmu.stop		= rapl_pmu_event_stop;
760	rapl_pmus->pmu.read		= rapl_pmu_event_read;
761	rapl_pmus->pmu.scope		= rapl_pmu_scope;
762	rapl_pmus->pmu.module		= THIS_MODULE;
763	rapl_pmus->pmu.capabilities	= PERF_PMU_CAP_NO_EXCLUDE;
764
765	ret = init_rapl_pmu(rapl_pmus);
766	if (ret)
767		kfree(rapl_pmus);
768
769	return ret;
770}
771
772static struct rapl_model model_snb = {
773	.pkg_events	= BIT(PERF_RAPL_PP0) |
774			  BIT(PERF_RAPL_PKG) |
775			  BIT(PERF_RAPL_PP1),
776	.msr_power_unit = MSR_RAPL_POWER_UNIT,
777	.rapl_pkg_msrs	= intel_rapl_msrs,
778};
779
780static struct rapl_model model_snbep = {
781	.pkg_events	= BIT(PERF_RAPL_PP0) |
782			  BIT(PERF_RAPL_PKG) |
783			  BIT(PERF_RAPL_RAM),
784	.msr_power_unit = MSR_RAPL_POWER_UNIT,
785	.rapl_pkg_msrs	= intel_rapl_msrs,
786};
787
788static struct rapl_model model_hsw = {
789	.pkg_events	= BIT(PERF_RAPL_PP0) |
790			  BIT(PERF_RAPL_PKG) |
791			  BIT(PERF_RAPL_RAM) |
792			  BIT(PERF_RAPL_PP1),
793	.msr_power_unit = MSR_RAPL_POWER_UNIT,
794	.rapl_pkg_msrs	= intel_rapl_msrs,
795};
796
797static struct rapl_model model_hsx = {
798	.pkg_events	= BIT(PERF_RAPL_PP0) |
799			  BIT(PERF_RAPL_PKG) |
800			  BIT(PERF_RAPL_RAM),
801	.unit_quirk	= RAPL_UNIT_QUIRK_INTEL_HSW,
802	.msr_power_unit = MSR_RAPL_POWER_UNIT,
803	.rapl_pkg_msrs	= intel_rapl_msrs,
804};
805
806static struct rapl_model model_knl = {
807	.pkg_events	= BIT(PERF_RAPL_PKG) |
808			  BIT(PERF_RAPL_RAM),
809	.unit_quirk	= RAPL_UNIT_QUIRK_INTEL_HSW,
810	.msr_power_unit = MSR_RAPL_POWER_UNIT,
811	.rapl_pkg_msrs	= intel_rapl_msrs,
812};
813
814static struct rapl_model model_skl = {
815	.pkg_events	= BIT(PERF_RAPL_PP0) |
816			  BIT(PERF_RAPL_PKG) |
817			  BIT(PERF_RAPL_RAM) |
818			  BIT(PERF_RAPL_PP1) |
819			  BIT(PERF_RAPL_PSYS),
820	.msr_power_unit = MSR_RAPL_POWER_UNIT,
821	.rapl_pkg_msrs      = intel_rapl_msrs,
822};
823
824static struct rapl_model model_spr = {
825	.pkg_events	= BIT(PERF_RAPL_PP0) |
826			  BIT(PERF_RAPL_PKG) |
827			  BIT(PERF_RAPL_RAM) |
828			  BIT(PERF_RAPL_PSYS),
829	.unit_quirk	= RAPL_UNIT_QUIRK_INTEL_SPR,
830	.msr_power_unit = MSR_RAPL_POWER_UNIT,
831	.rapl_pkg_msrs	= intel_rapl_spr_msrs,
832};
833
834static struct rapl_model model_amd_hygon = {
835	.pkg_events	= BIT(PERF_RAPL_PKG),
836	.core_events	= BIT(PERF_RAPL_CORE),
837	.msr_power_unit = MSR_AMD_RAPL_POWER_UNIT,
838	.rapl_pkg_msrs	= amd_rapl_pkg_msrs,
839	.rapl_core_msrs	= amd_rapl_core_msrs,
840};
841
842static const struct x86_cpu_id rapl_model_match[] __initconst = {
843	X86_MATCH_FEATURE(X86_FEATURE_RAPL,	&model_amd_hygon),
844	X86_MATCH_VFM(INTEL_SANDYBRIDGE,	&model_snb),
845	X86_MATCH_VFM(INTEL_SANDYBRIDGE_X,	&model_snbep),
846	X86_MATCH_VFM(INTEL_IVYBRIDGE,		&model_snb),
847	X86_MATCH_VFM(INTEL_IVYBRIDGE_X,	&model_snbep),
848	X86_MATCH_VFM(INTEL_HASWELL,		&model_hsw),
849	X86_MATCH_VFM(INTEL_HASWELL_X,		&model_hsx),
850	X86_MATCH_VFM(INTEL_HASWELL_L,		&model_hsw),
851	X86_MATCH_VFM(INTEL_HASWELL_G,		&model_hsw),
852	X86_MATCH_VFM(INTEL_BROADWELL,		&model_hsw),
853	X86_MATCH_VFM(INTEL_BROADWELL_G,	&model_hsw),
854	X86_MATCH_VFM(INTEL_BROADWELL_X,	&model_hsx),
855	X86_MATCH_VFM(INTEL_BROADWELL_D,	&model_hsx),
856	X86_MATCH_VFM(INTEL_XEON_PHI_KNL,	&model_knl),
857	X86_MATCH_VFM(INTEL_XEON_PHI_KNM,	&model_knl),
858	X86_MATCH_VFM(INTEL_SKYLAKE_L,		&model_skl),
859	X86_MATCH_VFM(INTEL_SKYLAKE,		&model_skl),
860	X86_MATCH_VFM(INTEL_SKYLAKE_X,		&model_hsx),
861	X86_MATCH_VFM(INTEL_KABYLAKE_L,		&model_skl),
862	X86_MATCH_VFM(INTEL_KABYLAKE,		&model_skl),
863	X86_MATCH_VFM(INTEL_CANNONLAKE_L,	&model_skl),
864	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT,	&model_hsw),
865	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D,	&model_hsw),
866	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS,	&model_hsw),
867	X86_MATCH_VFM(INTEL_ICELAKE_L,		&model_skl),
868	X86_MATCH_VFM(INTEL_ICELAKE,		&model_skl),
869	X86_MATCH_VFM(INTEL_ICELAKE_D,		&model_hsx),
870	X86_MATCH_VFM(INTEL_ICELAKE_X,		&model_hsx),
871	X86_MATCH_VFM(INTEL_COMETLAKE_L,	&model_skl),
872	X86_MATCH_VFM(INTEL_COMETLAKE,		&model_skl),
873	X86_MATCH_VFM(INTEL_TIGERLAKE_L,	&model_skl),
874	X86_MATCH_VFM(INTEL_TIGERLAKE,		&model_skl),
875	X86_MATCH_VFM(INTEL_ALDERLAKE,		&model_skl),
876	X86_MATCH_VFM(INTEL_ALDERLAKE_L,	&model_skl),
877	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&model_skl),
878	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,	&model_spr),
879	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,	&model_spr),
880	X86_MATCH_VFM(INTEL_RAPTORLAKE,		&model_skl),
881	X86_MATCH_VFM(INTEL_RAPTORLAKE_P,	&model_skl),
882	X86_MATCH_VFM(INTEL_RAPTORLAKE_S,	&model_skl),
883	X86_MATCH_VFM(INTEL_METEORLAKE,		&model_skl),
884	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&model_skl),
885	X86_MATCH_VFM(INTEL_ARROWLAKE_H,	&model_skl),
886	X86_MATCH_VFM(INTEL_ARROWLAKE,		&model_skl),
887	X86_MATCH_VFM(INTEL_ARROWLAKE_U,	&model_skl),
888	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&model_skl),
889	{},
890};
891MODULE_DEVICE_TABLE(x86cpu, rapl_model_match);
892
893static int __init rapl_pmu_init(void)
894{
895	const struct x86_cpu_id *id;
896	int rapl_pkg_pmu_scope = PERF_PMU_SCOPE_DIE;
897	int ret;
898
899	if (rapl_pkg_pmu_is_pkg_scope())
900		rapl_pkg_pmu_scope = PERF_PMU_SCOPE_PKG;
901
902	id = x86_match_cpu(rapl_model_match);
903	if (!id)
904		return -ENODEV;
905
906	rapl_model = (struct rapl_model *) id->driver_data;
907
908	ret = rapl_check_hw_unit();
909	if (ret)
910		return ret;
911
912	ret = init_rapl_pmus(&rapl_pmus_pkg, rapl_pkg_pmu_scope, rapl_attr_groups,
913			     rapl_attr_update);
914	if (ret)
915		return ret;
916
917	rapl_pmus_pkg->cntr_mask = perf_msr_probe(rapl_model->rapl_pkg_msrs,
918						  PERF_RAPL_PKG_EVENTS_MAX, false,
919						  (void *) &rapl_model->pkg_events);
920
921	ret = perf_pmu_register(&rapl_pmus_pkg->pmu, "power", -1);
922	if (ret)
923		goto out;
924
925	if (rapl_model->core_events) {
926		ret = init_rapl_pmus(&rapl_pmus_core, PERF_PMU_SCOPE_CORE,
927				     rapl_core_attr_groups,
928				     rapl_core_attr_update);
929		if (ret) {
930			pr_warn("power-core PMU initialization failed (%d)\n", ret);
931			goto core_init_failed;
932		}
933
934		rapl_pmus_core->cntr_mask = perf_msr_probe(rapl_model->rapl_core_msrs,
935						     PERF_RAPL_CORE_EVENTS_MAX, false,
936						     (void *) &rapl_model->core_events);
937
938		ret = perf_pmu_register(&rapl_pmus_core->pmu, "power_core", -1);
939		if (ret) {
940			pr_warn("power-core PMU registration failed (%d)\n", ret);
941			cleanup_rapl_pmus(rapl_pmus_core);
942		}
943	}
944
945core_init_failed:
946	rapl_advertise();
947	return 0;
948
949out:
950	pr_warn("Initialization failed (%d), disabled\n", ret);
951	cleanup_rapl_pmus(rapl_pmus_pkg);
952	return ret;
953}
954module_init(rapl_pmu_init);
955
956static void __exit intel_rapl_exit(void)
957{
958	if (rapl_pmus_core) {
959		perf_pmu_unregister(&rapl_pmus_core->pmu);
960		cleanup_rapl_pmus(rapl_pmus_core);
961	}
962	perf_pmu_unregister(&rapl_pmus_pkg->pmu);
963	cleanup_rapl_pmus(rapl_pmus_pkg);
964}
965module_exit(intel_rapl_exit);