pmu.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <linux/types.h>
  3#include <linux/interrupt.h>
  4
  5#include <asm/msr.h>
  6#include <asm/xen/hypercall.h>
  7#include <xen/xen.h>
  8#include <xen/page.h>
  9#include <xen/interface/xen.h>
 10#include <xen/interface/vcpu.h>
 11#include <xen/interface/xenpmu.h>
 12
 13#include "xen-ops.h"
 14
 15/* x86_pmu.handle_irq definition */
 16#include "../events/perf_event.h"
 17
 18#define XENPMU_IRQ_PROCESSING    1
 19struct xenpmu {
 20	/* Shared page between hypervisor and domain */
 21	struct xen_pmu_data *xenpmu_data;
 22
 23	uint8_t flags;
 24};
 25static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared);
 26#define get_xenpmu_data()    (this_cpu_ptr(&xenpmu_shared)->xenpmu_data)
 27#define get_xenpmu_flags()   (this_cpu_ptr(&xenpmu_shared)->flags)
 28
 29/* Macro for computing address of a PMU MSR bank */
 30#define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \
 31					    (uintptr_t)ctxt->field))
 32
 33/* AMD PMU */
 34#define F15H_NUM_COUNTERS   6
 35#define F10H_NUM_COUNTERS   4
 36
 37static __read_mostly uint32_t amd_counters_base;
 38static __read_mostly uint32_t amd_ctrls_base;
 39static __read_mostly int amd_msr_step;
 40static __read_mostly int k7_counters_mirrored;
 41static __read_mostly int amd_num_counters;
 42
 43/* Intel PMU */
 44#define MSR_TYPE_COUNTER            0
 45#define MSR_TYPE_CTRL               1
 46#define MSR_TYPE_GLOBAL             2
 47#define MSR_TYPE_ARCH_COUNTER       3
 48#define MSR_TYPE_ARCH_CTRL          4
 49
 50/* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
 51#define PMU_GENERAL_NR_SHIFT        8
 52#define PMU_GENERAL_NR_BITS         8
 53#define PMU_GENERAL_NR_MASK         (((1 << PMU_GENERAL_NR_BITS) - 1) \
 54				     << PMU_GENERAL_NR_SHIFT)
 55
 56/* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
 57#define PMU_FIXED_NR_SHIFT          0
 58#define PMU_FIXED_NR_BITS           5
 59#define PMU_FIXED_NR_MASK           (((1 << PMU_FIXED_NR_BITS) - 1) \
 60				     << PMU_FIXED_NR_SHIFT)
 61
 62/* Alias registers (0x4c1) for full-width writes to PMCs */
 63#define MSR_PMC_ALIAS_MASK          (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))
 64
 65#define INTEL_PMC_TYPE_SHIFT        30
 66
 67static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;
 68
 69
 70static void xen_pmu_arch_init(void)
 71{
 72	if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
 73
 74		switch (boot_cpu_data.x86) {
 75		case 0x15:
 76			amd_num_counters = F15H_NUM_COUNTERS;
 77			amd_counters_base = MSR_F15H_PERF_CTR;
 78			amd_ctrls_base = MSR_F15H_PERF_CTL;
 79			amd_msr_step = 2;
 80			k7_counters_mirrored = 1;
 81			break;
 82		case 0x10:
 83		case 0x12:
 84		case 0x14:
 85		case 0x16:
 86		default:
 87			amd_num_counters = F10H_NUM_COUNTERS;
 88			amd_counters_base = MSR_K7_PERFCTR0;
 89			amd_ctrls_base = MSR_K7_EVNTSEL0;
 90			amd_msr_step = 1;
 91			k7_counters_mirrored = 0;
 92			break;
 93		}
 94	} else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
 95		amd_num_counters = F10H_NUM_COUNTERS;
 96		amd_counters_base = MSR_K7_PERFCTR0;
 97		amd_ctrls_base = MSR_K7_EVNTSEL0;
 98		amd_msr_step = 1;
 99		k7_counters_mirrored = 0;
100	} else {
101		uint32_t eax, ebx, ecx, edx;
102
103		cpuid(0xa, &eax, &ebx, &ecx, &edx);
104
105		intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
106			PMU_GENERAL_NR_SHIFT;
107		intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
108			PMU_FIXED_NR_SHIFT;
109	}
110}
111
112static inline uint32_t get_fam15h_addr(u32 addr)
113{
114	switch (addr) {
115	case MSR_K7_PERFCTR0:
116	case MSR_K7_PERFCTR1:
117	case MSR_K7_PERFCTR2:
118	case MSR_K7_PERFCTR3:
119		return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
120	case MSR_K7_EVNTSEL0:
121	case MSR_K7_EVNTSEL1:
122	case MSR_K7_EVNTSEL2:
123	case MSR_K7_EVNTSEL3:
124		return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
125	default:
126		break;
127	}
128
129	return addr;
130}
131
132static bool is_amd_pmu_msr(u32 msr)
133{
134	if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
135	    boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
136		return false;
137
138	if ((msr >= MSR_F15H_PERF_CTL &&
139	     msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) ||
140	    (msr >= MSR_K7_EVNTSEL0 &&
141	     msr < MSR_K7_PERFCTR0 + amd_num_counters))
142		return true;
143
144	return false;
145}
146
147static bool is_intel_pmu_msr(u32 msr_index, int *type, int *index)
148{
149	u32 msr_index_pmc;
150
151	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL &&
152	    boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR &&
153	    boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
154		return false;
155
156	switch (msr_index) {
157	case MSR_CORE_PERF_FIXED_CTR_CTRL:
158	case MSR_IA32_DS_AREA:
159	case MSR_IA32_PEBS_ENABLE:
160		*type = MSR_TYPE_CTRL;
161		return true;
162
163	case MSR_CORE_PERF_GLOBAL_CTRL:
164	case MSR_CORE_PERF_GLOBAL_STATUS:
165	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
166		*type = MSR_TYPE_GLOBAL;
167		return true;
168
169	default:
170
171		if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
172		    (msr_index < MSR_CORE_PERF_FIXED_CTR0 +
173				 intel_num_fixed_counters)) {
174			*index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
175			*type = MSR_TYPE_COUNTER;
176			return true;
177		}
178
179		if ((msr_index >= MSR_P6_EVNTSEL0) &&
180		    (msr_index < MSR_P6_EVNTSEL0 +  intel_num_arch_counters)) {
181			*index = msr_index - MSR_P6_EVNTSEL0;
182			*type = MSR_TYPE_ARCH_CTRL;
183			return true;
184		}
185
186		msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
187		if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
188		    (msr_index_pmc < MSR_IA32_PERFCTR0 +
189				     intel_num_arch_counters)) {
190			*type = MSR_TYPE_ARCH_COUNTER;
191			*index = msr_index_pmc - MSR_IA32_PERFCTR0;
192			return true;
193		}
194		return false;
195	}
196}
197
198static bool xen_intel_pmu_emulate(u32 msr, u64 *val, int type, int index, bool is_read)
199{
200	uint64_t *reg = NULL;
201	struct xen_pmu_intel_ctxt *ctxt;
202	uint64_t *fix_counters;
203	struct xen_pmu_cntr_pair *arch_cntr_pair;
204	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
205	uint8_t xenpmu_flags = get_xenpmu_flags();
206
207
208	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
209		return false;
210
211	ctxt = &xenpmu_data->pmu.c.intel;
212
213	switch (msr) {
214	case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
215		reg = &ctxt->global_ovf_ctrl;
216		break;
217	case MSR_CORE_PERF_GLOBAL_STATUS:
218		reg = &ctxt->global_status;
219		break;
220	case MSR_CORE_PERF_GLOBAL_CTRL:
221		reg = &ctxt->global_ctrl;
222		break;
223	case MSR_CORE_PERF_FIXED_CTR_CTRL:
224		reg = &ctxt->fixed_ctrl;
225		break;
226	default:
227		switch (type) {
228		case MSR_TYPE_COUNTER:
229			fix_counters = field_offset(ctxt, fixed_counters);
230			reg = &fix_counters[index];
231			break;
232		case MSR_TYPE_ARCH_COUNTER:
233			arch_cntr_pair = field_offset(ctxt, arch_counters);
234			reg = &arch_cntr_pair[index].counter;
235			break;
236		case MSR_TYPE_ARCH_CTRL:
237			arch_cntr_pair = field_offset(ctxt, arch_counters);
238			reg = &arch_cntr_pair[index].control;
239			break;
240		default:
241			return false;
242		}
243	}
244
245	if (reg) {
246		if (is_read)
247			*val = *reg;
248		else {
249			*reg = *val;
250
251			if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL)
252				ctxt->global_status &= (~(*val));
253		}
254		return true;
255	}
256
257	return false;
258}
259
260static bool xen_amd_pmu_emulate(u32 msr, u64 *val, bool is_read)
261{
262	uint64_t *reg = NULL;
263	int i, off = 0;
264	struct xen_pmu_amd_ctxt *ctxt;
265	uint64_t *counter_regs, *ctrl_regs;
266	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
267	uint8_t xenpmu_flags = get_xenpmu_flags();
268
269	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
270		return false;
271
272	if (k7_counters_mirrored &&
273	    ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3)))
274		msr = get_fam15h_addr(msr);
275
276	ctxt = &xenpmu_data->pmu.c.amd;
277	for (i = 0; i < amd_num_counters; i++) {
278		if (msr == amd_ctrls_base + off) {
279			ctrl_regs = field_offset(ctxt, ctrls);
280			reg = &ctrl_regs[i];
281			break;
282		} else if (msr == amd_counters_base + off) {
283			counter_regs = field_offset(ctxt, counters);
284			reg = &counter_regs[i];
285			break;
286		}
287		off += amd_msr_step;
288	}
289
290	if (reg) {
291		if (is_read)
292			*val = *reg;
293		else
294			*reg = *val;
295
296		return true;
297	}
298	return false;
299}
300
301bool pmu_msr_chk_emulated(u32 msr, u64 *val, bool is_read)
302{
303	int type, index = 0;
304
305	if (is_amd_pmu_msr(msr))
306		return xen_amd_pmu_emulate(msr, val, is_read);
307
308	if (is_intel_pmu_msr(msr, &type, &index))
309		return xen_intel_pmu_emulate(msr, val, type, index, is_read);
310
311	return false;
312}
313
314static u64 xen_amd_read_pmc(int counter)
315{
316	struct xen_pmu_amd_ctxt *ctxt;
317	uint64_t *counter_regs;
318	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
319	uint8_t xenpmu_flags = get_xenpmu_flags();
320
321	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
322		u32 msr;
323		u64 val;
324
325		msr = amd_counters_base + (counter * amd_msr_step);
326		native_read_msr_safe(msr, &val);
327		return val;
328	}
329
330	ctxt = &xenpmu_data->pmu.c.amd;
331	counter_regs = field_offset(ctxt, counters);
332	return counter_regs[counter];
333}
334
335static u64 xen_intel_read_pmc(int counter)
336{
337	struct xen_pmu_intel_ctxt *ctxt;
338	uint64_t *fixed_counters;
339	struct xen_pmu_cntr_pair *arch_cntr_pair;
340	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
341	uint8_t xenpmu_flags = get_xenpmu_flags();
342
343	if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
344		u32 msr;
345		u64 val;
346
347		if (counter & (1 << INTEL_PMC_TYPE_SHIFT))
348			msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
349		else
350			msr = MSR_IA32_PERFCTR0 + counter;
351
352		native_read_msr_safe(msr, &val);
353		return val;
354	}
355
356	ctxt = &xenpmu_data->pmu.c.intel;
357	if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) {
358		fixed_counters = field_offset(ctxt, fixed_counters);
359		return fixed_counters[counter & 0xffff];
360	}
361
362	arch_cntr_pair = field_offset(ctxt, arch_counters);
363	return arch_cntr_pair[counter].counter;
364}
365
366u64 xen_read_pmc(int counter)
367{
368	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
369		return xen_amd_read_pmc(counter);
370	else
371		return xen_intel_read_pmc(counter);
372}
373
374int pmu_apic_update(uint32_t val)
375{
376	int ret;
377	struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
378
379	if (!xenpmu_data) {
380		pr_warn_once("%s: pmudata not initialized\n", __func__);
381		return -EINVAL;
382	}
383
384	xenpmu_data->pmu.l.lapic_lvtpc = val;
385
386	if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING)
387		return 0;
388
389	ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);
390
391	return ret;
392}
393
394/* perf callbacks */
395static unsigned int xen_guest_state(void)
396{
397	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
398	unsigned int state = 0;
399
400	if (!xenpmu_data) {
401		pr_warn_once("%s: pmudata not initialized\n", __func__);
402		return state;
403	}
404
405	if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
406		return state;
407
408	state |= PERF_GUEST_ACTIVE;
409
410	if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) {
411		if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER)
412			state |= PERF_GUEST_USER;
413	} else if (xenpmu_data->pmu.r.regs.cpl & 3) {
414		state |= PERF_GUEST_USER;
415	}
416
417	return state;
418}
419
420static unsigned long xen_get_guest_ip(void)
421{
422	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
423
424	if (!xenpmu_data) {
425		pr_warn_once("%s: pmudata not initialized\n", __func__);
426		return 0;
427	}
428
429	return xenpmu_data->pmu.r.regs.ip;
430}
431
432static struct perf_guest_info_callbacks xen_guest_cbs = {
433	.state                  = xen_guest_state,
434	.get_ip			= xen_get_guest_ip,
435};
436
437/* Convert registers from Xen's format to Linux' */
438static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
439			     struct pt_regs *regs, uint64_t pmu_flags)
440{
441	regs->ip = xen_regs->ip;
442	regs->cs = xen_regs->cs;
443	regs->sp = xen_regs->sp;
444
445	if (pmu_flags & PMU_SAMPLE_PV) {
446		if (pmu_flags & PMU_SAMPLE_USER)
447			regs->cs |= 3;
448		else
449			regs->cs &= ~3;
450	} else {
451		if (xen_regs->cpl)
452			regs->cs |= 3;
453		else
454			regs->cs &= ~3;
455	}
456}
457
458irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
459{
460	int err, ret = IRQ_NONE;
461	struct pt_regs regs = {0};
462	const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
463	uint8_t xenpmu_flags = get_xenpmu_flags();
464
465	if (!xenpmu_data) {
466		pr_warn_once("%s: pmudata not initialized\n", __func__);
467		return ret;
468	}
469
470	this_cpu_ptr(&xenpmu_shared)->flags =
471		xenpmu_flags | XENPMU_IRQ_PROCESSING;
472	xen_convert_regs(&xenpmu_data->pmu.r.regs, &regs,
473			 xenpmu_data->pmu.pmu_flags);
474	if (x86_pmu.handle_irq(&regs))
475		ret = IRQ_HANDLED;
476
477	/* Write out cached context to HW */
478	err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
479	this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags;
480	if (err) {
481		pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
482		return IRQ_NONE;
483	}
484
485	return ret;
486}
487
488bool is_xen_pmu;
489
490void xen_pmu_init(int cpu)
491{
492	int err;
493	struct xen_pmu_params xp;
494	unsigned long pfn;
495	struct xen_pmu_data *xenpmu_data;
496
497	BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
498
499	if (xen_hvm_domain() || (cpu != 0 && !is_xen_pmu))
500		return;
501
502	xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
503	if (!xenpmu_data) {
504		pr_err("VPMU init: No memory\n");
505		return;
506	}
507	pfn = virt_to_pfn(xenpmu_data);
508
509	xp.val = pfn_to_mfn(pfn);
510	xp.vcpu = cpu;
511	xp.version.maj = XENPMU_VER_MAJ;
512	xp.version.min = XENPMU_VER_MIN;
513	err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
514	if (err)
515		goto fail;
516
517	per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
518	per_cpu(xenpmu_shared, cpu).flags = 0;
519
520	if (!is_xen_pmu) {
521		is_xen_pmu = true;
522		perf_register_guest_info_callbacks(&xen_guest_cbs);
523		xen_pmu_arch_init();
524	}
525
526	return;
527
528fail:
529	if (err == -EOPNOTSUPP || err == -ENOSYS)
530		pr_info_once("VPMU disabled by hypervisor.\n");
531	else
532		pr_info_once("Could not initialize VPMU for cpu %d, error %d\n",
533			cpu, err);
534	free_pages((unsigned long)xenpmu_data, 0);
535}
536
537void xen_pmu_finish(int cpu)
538{
539	struct xen_pmu_params xp;
540
541	if (xen_hvm_domain())
542		return;
543
544	xp.vcpu = cpu;
545	xp.version.maj = XENPMU_VER_MAJ;
546	xp.version.min = XENPMU_VER_MIN;
547
548	(void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);
549
550	free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0);
551	per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL;
552}