apple_m1_cpu_pmu.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * CPU PMU driver for the Apple M1 and derivatives
  4 *
  5 * Copyright (C) 2021 Google LLC
  6 *
  7 * Author: Marc Zyngier <maz@kernel.org>
  8 *
  9 * Most of the information used in this driver was provided by the
 10 * Asahi Linux project. The rest was experimentally discovered.
 11 */
 12
 13#include <linux/of.h>
 14#include <linux/perf/arm_pmu.h>
 15#include <linux/perf/arm_pmuv3.h>
 16#include <linux/platform_device.h>
 17
 18#include <asm/apple_m1_pmu.h>
 19#include <asm/irq_regs.h>
 20#include <asm/perf_event.h>
 21
 22#define M1_PMU_NR_COUNTERS		10
 23
 24#define M1_PMU_CFG_EVENT		GENMASK(7, 0)
 25
 26#define ANY_BUT_0_1			GENMASK(9, 2)
 27#define ONLY_2_TO_7			GENMASK(7, 2)
 28#define ONLY_2_4_6			(BIT(2) | BIT(4) | BIT(6))
 29#define ONLY_5_6_7			(BIT(5) | BIT(6) | BIT(7))
 30
 31/*
 32 * Description of the events we actually know about, as well as those with
 33 * a specific counter affinity. Yes, this is a grand total of two known
 34 * counters, and the rest is anybody's guess.
 35 *
 36 * Not all counters can count all events. Counters #0 and #1 are wired to
 37 * count cycles and instructions respectively, and some events have
 38 * bizarre mappings (every other counter, or even *one* counter). These
 39 * restrictions equally apply to both P and E cores.
 40 *
 41 * It is worth noting that the PMUs attached to P and E cores are likely
 42 * to be different because the underlying uarches are different. At the
 43 * moment, we don't really need to distinguish between the two because we
 44 * know next to nothing about the events themselves, and we already have
 45 * per cpu-type PMU abstractions.
 46 *
 47 * If we eventually find out that the events are different across
 48 * implementations, we'll have to introduce per cpu-type tables.
 49 */
 50enum m1_pmu_events {
 51	M1_PMU_PERFCTR_RETIRE_UOP				= 0x1,
 52	M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE			= 0x2,
 53	M1_PMU_PERFCTR_L1I_TLB_FILL				= 0x4,
 54	M1_PMU_PERFCTR_L1D_TLB_FILL				= 0x5,
 55	M1_PMU_PERFCTR_MMU_TABLE_WALK_INSTRUCTION		= 0x7,
 56	M1_PMU_PERFCTR_MMU_TABLE_WALK_DATA			= 0x8,
 57	M1_PMU_PERFCTR_L2_TLB_MISS_INSTRUCTION			= 0xa,
 58	M1_PMU_PERFCTR_L2_TLB_MISS_DATA				= 0xb,
 59	M1_PMU_PERFCTR_MMU_VIRTUAL_MEMORY_FAULT_NONSPEC		= 0xd,
 60	M1_PMU_PERFCTR_SCHEDULE_UOP				= 0x52,
 61	M1_PMU_PERFCTR_INTERRUPT_PENDING			= 0x6c,
 62	M1_PMU_PERFCTR_MAP_STALL_DISPATCH			= 0x70,
 63	M1_PMU_PERFCTR_MAP_REWIND				= 0x75,
 64	M1_PMU_PERFCTR_MAP_STALL				= 0x76,
 65	M1_PMU_PERFCTR_MAP_INT_UOP				= 0x7c,
 66	M1_PMU_PERFCTR_MAP_LDST_UOP				= 0x7d,
 67	M1_PMU_PERFCTR_MAP_SIMD_UOP				= 0x7e,
 68	M1_PMU_PERFCTR_FLUSH_RESTART_OTHER_NONSPEC		= 0x84,
 69	M1_PMU_PERFCTR_INST_ALL					= 0x8c,
 70	M1_PMU_PERFCTR_INST_BRANCH				= 0x8d,
 71	M1_PMU_PERFCTR_INST_BRANCH_CALL				= 0x8e,
 72	M1_PMU_PERFCTR_INST_BRANCH_RET				= 0x8f,
 73	M1_PMU_PERFCTR_INST_BRANCH_TAKEN			= 0x90,
 74	M1_PMU_PERFCTR_INST_BRANCH_INDIR			= 0x93,
 75	M1_PMU_PERFCTR_INST_BRANCH_COND				= 0x94,
 76	M1_PMU_PERFCTR_INST_INT_LD				= 0x95,
 77	M1_PMU_PERFCTR_INST_INT_ST				= 0x96,
 78	M1_PMU_PERFCTR_INST_INT_ALU				= 0x97,
 79	M1_PMU_PERFCTR_INST_SIMD_LD				= 0x98,
 80	M1_PMU_PERFCTR_INST_SIMD_ST				= 0x99,
 81	M1_PMU_PERFCTR_INST_SIMD_ALU				= 0x9a,
 82	M1_PMU_PERFCTR_INST_LDST				= 0x9b,
 83	M1_PMU_PERFCTR_INST_BARRIER				= 0x9c,
 84	M1_PMU_PERFCTR_UNKNOWN_9f				= 0x9f,
 85	M1_PMU_PERFCTR_L1D_TLB_ACCESS				= 0xa0,
 86	M1_PMU_PERFCTR_L1D_TLB_MISS				= 0xa1,
 87	M1_PMU_PERFCTR_L1D_CACHE_MISS_ST			= 0xa2,
 88	M1_PMU_PERFCTR_L1D_CACHE_MISS_LD			= 0xa3,
 89	M1_PMU_PERFCTR_LD_UNIT_UOP				= 0xa6,
 90	M1_PMU_PERFCTR_ST_UNIT_UOP				= 0xa7,
 91	M1_PMU_PERFCTR_L1D_CACHE_WRITEBACK			= 0xa8,
 92	M1_PMU_PERFCTR_LDST_X64_UOP				= 0xb1,
 93	M1_PMU_PERFCTR_LDST_XPG_UOP				= 0xb2,
 94	M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_SUCC			= 0xb3,
 95	M1_PMU_PERFCTR_ATOMIC_OR_EXCLUSIVE_FAIL			= 0xb4,
 96	M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC		= 0xbf,
 97	M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC		= 0xc0,
 98	M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC			= 0xc1,
 99	M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC	= 0xc4,
100	M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC		= 0xc5,
101	M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC		= 0xc6,
102	M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC		= 0xc8,
103	M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC	= 0xca,
104	M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC			= 0xcb,
105	M1_PMU_PERFCTR_L1I_TLB_MISS_DEMAND			= 0xd4,
106	M1_PMU_PERFCTR_MAP_DISPATCH_BUBBLE			= 0xd6,
107	M1_PMU_PERFCTR_L1I_CACHE_MISS_DEMAND			= 0xdb,
108	M1_PMU_PERFCTR_FETCH_RESTART				= 0xde,
109	M1_PMU_PERFCTR_ST_NT_UOP				= 0xe5,
110	M1_PMU_PERFCTR_LD_NT_UOP				= 0xe6,
111	M1_PMU_PERFCTR_UNKNOWN_f5				= 0xf5,
112	M1_PMU_PERFCTR_UNKNOWN_f6				= 0xf6,
113	M1_PMU_PERFCTR_UNKNOWN_f7				= 0xf7,
114	M1_PMU_PERFCTR_UNKNOWN_f8				= 0xf8,
115	M1_PMU_PERFCTR_UNKNOWN_fd				= 0xfd,
116	M1_PMU_PERFCTR_LAST					= M1_PMU_CFG_EVENT,
117
118	/*
119	 * From this point onwards, these are not actual HW events,
120	 * but attributes that get stored in hw->config_base.
121	 */
122	M1_PMU_CFG_COUNT_USER					= BIT(8),
123	M1_PMU_CFG_COUNT_KERNEL					= BIT(9),
124	M1_PMU_CFG_COUNT_HOST					= BIT(10),
125	M1_PMU_CFG_COUNT_GUEST					= BIT(11),
126};
127
128/*
129 * Per-event affinity table. Most events can be installed on counter
130 * 2-9, but there are a number of exceptions. Note that this table
131 * has been created experimentally, and I wouldn't be surprised if more
132 * counters had strange affinities.
133 */
134static const u16 m1_pmu_event_affinity[M1_PMU_PERFCTR_LAST + 1] = {
135	[0 ... M1_PMU_PERFCTR_LAST]				= ANY_BUT_0_1,
136	[M1_PMU_PERFCTR_RETIRE_UOP]				= BIT(7),
137	[M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE]			= ANY_BUT_0_1 | BIT(0),
138	[M1_PMU_PERFCTR_INST_ALL]				= BIT(7) | BIT(1),
139	[M1_PMU_PERFCTR_INST_BRANCH]				= ONLY_5_6_7,
140	[M1_PMU_PERFCTR_INST_BRANCH_CALL]			= ONLY_5_6_7,
141	[M1_PMU_PERFCTR_INST_BRANCH_RET]			= ONLY_5_6_7,
142	[M1_PMU_PERFCTR_INST_BRANCH_TAKEN]			= ONLY_5_6_7,
143	[M1_PMU_PERFCTR_INST_BRANCH_INDIR]			= ONLY_5_6_7,
144	[M1_PMU_PERFCTR_INST_BRANCH_COND]			= ONLY_5_6_7,
145	[M1_PMU_PERFCTR_INST_INT_LD]				= ONLY_5_6_7,
146	[M1_PMU_PERFCTR_INST_INT_ST]				= BIT(7),
147	[M1_PMU_PERFCTR_INST_INT_ALU]				= BIT(7),
148	[M1_PMU_PERFCTR_INST_SIMD_LD]				= ONLY_5_6_7,
149	[M1_PMU_PERFCTR_INST_SIMD_ST]				= ONLY_5_6_7,
150	[M1_PMU_PERFCTR_INST_SIMD_ALU]				= BIT(7),
151	[M1_PMU_PERFCTR_INST_LDST]				= BIT(7),
152	[M1_PMU_PERFCTR_INST_BARRIER]				= ONLY_5_6_7,
153	[M1_PMU_PERFCTR_UNKNOWN_9f]				= BIT(7),
154	[M1_PMU_PERFCTR_L1D_CACHE_MISS_LD_NONSPEC]		= ONLY_5_6_7,
155	[M1_PMU_PERFCTR_L1D_CACHE_MISS_ST_NONSPEC]		= ONLY_5_6_7,
156	[M1_PMU_PERFCTR_L1D_TLB_MISS_NONSPEC]			= ONLY_5_6_7,
157	[M1_PMU_PERFCTR_ST_MEMORY_ORDER_VIOLATION_NONSPEC]	= ONLY_5_6_7,
158	[M1_PMU_PERFCTR_BRANCH_COND_MISPRED_NONSPEC]		= ONLY_5_6_7,
159	[M1_PMU_PERFCTR_BRANCH_INDIR_MISPRED_NONSPEC]		= ONLY_5_6_7,
160	[M1_PMU_PERFCTR_BRANCH_RET_INDIR_MISPRED_NONSPEC]	= ONLY_5_6_7,
161	[M1_PMU_PERFCTR_BRANCH_CALL_INDIR_MISPRED_NONSPEC]	= ONLY_5_6_7,
162	[M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC]			= ONLY_5_6_7,
163	[M1_PMU_PERFCTR_UNKNOWN_f5]				= ONLY_2_4_6,
164	[M1_PMU_PERFCTR_UNKNOWN_f6]				= ONLY_2_4_6,
165	[M1_PMU_PERFCTR_UNKNOWN_f7]				= ONLY_2_4_6,
166	[M1_PMU_PERFCTR_UNKNOWN_f8]				= ONLY_2_TO_7,
167	[M1_PMU_PERFCTR_UNKNOWN_fd]				= ONLY_2_4_6,
168};
169
170static const unsigned m1_pmu_perf_map[PERF_COUNT_HW_MAX] = {
171	PERF_MAP_ALL_UNSUPPORTED,
172	[PERF_COUNT_HW_CPU_CYCLES]		= M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE,
173	[PERF_COUNT_HW_INSTRUCTIONS]		= M1_PMU_PERFCTR_INST_ALL,
174	[PERF_COUNT_HW_BRANCH_INSTRUCTIONS]	= M1_PMU_PERFCTR_INST_BRANCH,
175	[PERF_COUNT_HW_BRANCH_MISSES]		= M1_PMU_PERFCTR_BRANCH_MISPRED_NONSPEC,
176};
177
178#define M1_PMUV3_EVENT_MAP(pmuv3_event, m1_event)				\
179	[ARMV8_PMUV3_PERFCTR_##pmuv3_event]	= M1_PMU_PERFCTR_##m1_event
180
181static const u16 m1_pmu_pmceid_map[ARMV8_PMUV3_MAX_COMMON_EVENTS] = {
182	[0 ... ARMV8_PMUV3_MAX_COMMON_EVENTS - 1]	= HW_OP_UNSUPPORTED,
183	M1_PMUV3_EVENT_MAP(INST_RETIRED,	INST_ALL),
184	M1_PMUV3_EVENT_MAP(CPU_CYCLES,		CORE_ACTIVE_CYCLE),
185	M1_PMUV3_EVENT_MAP(BR_RETIRED,		INST_BRANCH),
186	M1_PMUV3_EVENT_MAP(BR_MIS_PRED_RETIRED,	BRANCH_MISPRED_NONSPEC),
187};
188
189/* sysfs definitions */
190static ssize_t m1_pmu_events_sysfs_show(struct device *dev,
191					struct device_attribute *attr,
192					char *page)
193{
194	struct perf_pmu_events_attr *pmu_attr;
195
196	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
197
198	return sprintf(page, "event=0x%04llx\n", pmu_attr->id);
199}
200
201#define M1_PMU_EVENT_ATTR(name, config)					\
202	PMU_EVENT_ATTR_ID(name, m1_pmu_events_sysfs_show, config)
203
204static struct attribute *m1_pmu_event_attrs[] = {
205	M1_PMU_EVENT_ATTR(cycles, M1_PMU_PERFCTR_CORE_ACTIVE_CYCLE),
206	M1_PMU_EVENT_ATTR(instructions, M1_PMU_PERFCTR_INST_ALL),
207	NULL,
208};
209
210static const struct attribute_group m1_pmu_events_attr_group = {
211	.name = "events",
212	.attrs = m1_pmu_event_attrs,
213};
214
215PMU_FORMAT_ATTR(event, "config:0-7");
216
217static struct attribute *m1_pmu_format_attrs[] = {
218	&format_attr_event.attr,
219	NULL,
220};
221
222static const struct attribute_group m1_pmu_format_attr_group = {
223	.name = "format",
224	.attrs = m1_pmu_format_attrs,
225};
226
227/* Low level accessors. No synchronisation. */
228#define PMU_READ_COUNTER(_idx)						\
229	case _idx:	return read_sysreg_s(SYS_IMP_APL_PMC## _idx ##_EL1)
230
231#define PMU_WRITE_COUNTER(_val, _idx)					\
232	case _idx:							\
233		write_sysreg_s(_val, SYS_IMP_APL_PMC## _idx ##_EL1);	\
234		return
235
236static u64 m1_pmu_read_hw_counter(unsigned int index)
237{
238	switch (index) {
239		PMU_READ_COUNTER(0);
240		PMU_READ_COUNTER(1);
241		PMU_READ_COUNTER(2);
242		PMU_READ_COUNTER(3);
243		PMU_READ_COUNTER(4);
244		PMU_READ_COUNTER(5);
245		PMU_READ_COUNTER(6);
246		PMU_READ_COUNTER(7);
247		PMU_READ_COUNTER(8);
248		PMU_READ_COUNTER(9);
249	}
250
251	BUG();
252}
253
254static void m1_pmu_write_hw_counter(u64 val, unsigned int index)
255{
256	switch (index) {
257		PMU_WRITE_COUNTER(val, 0);
258		PMU_WRITE_COUNTER(val, 1);
259		PMU_WRITE_COUNTER(val, 2);
260		PMU_WRITE_COUNTER(val, 3);
261		PMU_WRITE_COUNTER(val, 4);
262		PMU_WRITE_COUNTER(val, 5);
263		PMU_WRITE_COUNTER(val, 6);
264		PMU_WRITE_COUNTER(val, 7);
265		PMU_WRITE_COUNTER(val, 8);
266		PMU_WRITE_COUNTER(val, 9);
267	}
268
269	BUG();
270}
271
272#define get_bit_offset(index, mask)	(__ffs(mask) + (index))
273
274static void __m1_pmu_enable_counter(unsigned int index, bool en)
275{
276	u64 val, bit;
277
278	switch (index) {
279	case 0 ... 7:
280		bit = BIT(get_bit_offset(index, PMCR0_CNT_ENABLE_0_7));
281		break;
282	case 8 ... 9:
283		bit = BIT(get_bit_offset(index - 8, PMCR0_CNT_ENABLE_8_9));
284		break;
285	default:
286		BUG();
287	}
288
289	val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
290
291	if (en)
292		val |= bit;
293	else
294		val &= ~bit;
295
296	write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
297}
298
299static void m1_pmu_enable_counter(unsigned int index)
300{
301	__m1_pmu_enable_counter(index, true);
302}
303
304static void m1_pmu_disable_counter(unsigned int index)
305{
306	__m1_pmu_enable_counter(index, false);
307}
308
309static void __m1_pmu_enable_counter_interrupt(unsigned int index, bool en)
310{
311	u64 val, bit;
312
313	switch (index) {
314	case 0 ... 7:
315		bit = BIT(get_bit_offset(index, PMCR0_PMI_ENABLE_0_7));
316		break;
317	case 8 ... 9:
318		bit = BIT(get_bit_offset(index - 8, PMCR0_PMI_ENABLE_8_9));
319		break;
320	default:
321		BUG();
322	}
323
324	val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
325
326	if (en)
327		val |= bit;
328	else
329		val &= ~bit;
330
331	write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
332}
333
334static void m1_pmu_enable_counter_interrupt(unsigned int index)
335{
336	__m1_pmu_enable_counter_interrupt(index, true);
337}
338
339static void m1_pmu_disable_counter_interrupt(unsigned int index)
340{
341	__m1_pmu_enable_counter_interrupt(index, false);
342}
343
344static void __m1_pmu_configure_event_filter(unsigned int index, bool user,
345					    bool kernel, bool host)
346{
347	u64 clear, set, user_bit, kernel_bit;
348
349	switch (index) {
350	case 0 ... 7:
351		user_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL0_0_7));
352		kernel_bit = BIT(get_bit_offset(index, PMCR1_COUNT_A64_EL1_0_7));
353		break;
354	case 8 ... 9:
355		user_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL0_8_9));
356		kernel_bit = BIT(get_bit_offset(index - 8, PMCR1_COUNT_A64_EL1_8_9));
357		break;
358	default:
359		BUG();
360	}
361
362	clear = set = 0;
363	if (user)
364		set |= user_bit;
365	else
366		clear |= user_bit;
367
368	if (kernel)
369		set |= kernel_bit;
370	else
371		clear |= kernel_bit;
372
373	if (host)
374		sysreg_clear_set_s(SYS_IMP_APL_PMCR1_EL1, clear, set);
375	else if (is_kernel_in_hyp_mode())
376		sysreg_clear_set_s(SYS_IMP_APL_PMCR1_EL12, clear, set);
377}
378
379static void __m1_pmu_configure_eventsel(unsigned int index, u8 event)
380{
381	u64 clear = 0, set = 0;
382	int shift;
383
384	/*
385	 * Counters 0 and 1 have fixed events. For anything else,
386	 * place the event at the expected location in the relevant
387	 * register (PMESR0 holds the event configuration for counters
388	 * 2-5, resp. PMESR1 for counters 6-9).
389	 */
390	switch (index) {
391	case 0 ... 1:
392		break;
393	case 2 ... 5:
394		shift = (index - 2) * 8;
395		clear |= (u64)0xff << shift;
396		set |= (u64)event << shift;
397		sysreg_clear_set_s(SYS_IMP_APL_PMESR0_EL1, clear, set);
398		break;
399	case 6 ... 9:
400		shift = (index - 6) * 8;
401		clear |= (u64)0xff << shift;
402		set |= (u64)event << shift;
403		sysreg_clear_set_s(SYS_IMP_APL_PMESR1_EL1, clear, set);
404		break;
405	}
406}
407
408static void m1_pmu_configure_counter(unsigned int index, unsigned long config_base)
409{
410	bool kernel = config_base & M1_PMU_CFG_COUNT_KERNEL;
411	bool guest = config_base & M1_PMU_CFG_COUNT_GUEST;
412	bool host = config_base & M1_PMU_CFG_COUNT_HOST;
413	bool user = config_base & M1_PMU_CFG_COUNT_USER;
414	u8 evt = config_base & M1_PMU_CFG_EVENT;
415
416	__m1_pmu_configure_event_filter(index, user && host, kernel && host, true);
417	__m1_pmu_configure_event_filter(index, user && guest, kernel && guest, false);
418	__m1_pmu_configure_eventsel(index, evt);
419}
420
421/* arm_pmu backend */
422static void m1_pmu_enable_event(struct perf_event *event)
423{
424	bool user, kernel;
425	u8 evt;
426
427	evt = event->hw.config_base & M1_PMU_CFG_EVENT;
428	user = event->hw.config_base & M1_PMU_CFG_COUNT_USER;
429	kernel = event->hw.config_base & M1_PMU_CFG_COUNT_KERNEL;
430
431	m1_pmu_configure_counter(event->hw.idx, event->hw.config_base);
432	m1_pmu_enable_counter(event->hw.idx);
433	m1_pmu_enable_counter_interrupt(event->hw.idx);
434	isb();
435}
436
437static void m1_pmu_disable_event(struct perf_event *event)
438{
439	m1_pmu_disable_counter_interrupt(event->hw.idx);
440	m1_pmu_disable_counter(event->hw.idx);
441	isb();
442}
443
444static irqreturn_t m1_pmu_handle_irq(struct arm_pmu *cpu_pmu)
445{
446	struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events);
447	struct pt_regs *regs;
448	u64 overflow, state;
449	int idx;
450
451	overflow = read_sysreg_s(SYS_IMP_APL_PMSR_EL1);
452	if (!overflow) {
453		/* Spurious interrupt? */
454		state = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
455		state &= ~PMCR0_IACT;
456		write_sysreg_s(state, SYS_IMP_APL_PMCR0_EL1);
457		isb();
458		return IRQ_NONE;
459	}
460
461	cpu_pmu->stop(cpu_pmu);
462
463	regs = get_irq_regs();
464
465	for_each_set_bit(idx, cpu_pmu->cntr_mask, M1_PMU_NR_COUNTERS) {
466		struct perf_event *event = cpuc->events[idx];
467		struct perf_sample_data data;
468
469		if (!event)
470			continue;
471
472		armpmu_event_update(event);
473		perf_sample_data_init(&data, 0, event->hw.last_period);
474		if (!armpmu_event_set_period(event))
475			continue;
476
477		perf_event_overflow(event, &data, regs);
478	}
479
480	cpu_pmu->start(cpu_pmu);
481
482	return IRQ_HANDLED;
483}
484
485static u64 m1_pmu_read_counter(struct perf_event *event)
486{
487	return m1_pmu_read_hw_counter(event->hw.idx);
488}
489
490static void m1_pmu_write_counter(struct perf_event *event, u64 value)
491{
492	m1_pmu_write_hw_counter(value, event->hw.idx);
493	isb();
494}
495
496static int m1_pmu_get_event_idx(struct pmu_hw_events *cpuc,
497				struct perf_event *event)
498{
499	unsigned long evtype = event->hw.config_base & M1_PMU_CFG_EVENT;
500	unsigned long affinity = m1_pmu_event_affinity[evtype];
501	int idx;
502
503	/*
504	 * Place the event on the first free counter that can count
505	 * this event.
506	 *
507	 * We could do a better job if we had a view of all the events
508	 * counting on the PMU at any given time, and by placing the
509	 * most constraining events first.
510	 */
511	for_each_set_bit(idx, &affinity, M1_PMU_NR_COUNTERS) {
512		if (!test_and_set_bit(idx, cpuc->used_mask))
513			return idx;
514	}
515
516	return -EAGAIN;
517}
518
519static void m1_pmu_clear_event_idx(struct pmu_hw_events *cpuc,
520				   struct perf_event *event)
521{
522	clear_bit(event->hw.idx, cpuc->used_mask);
523}
524
525static void __m1_pmu_set_mode(u8 mode)
526{
527	u64 val;
528
529	val = read_sysreg_s(SYS_IMP_APL_PMCR0_EL1);
530	val &= ~(PMCR0_IMODE | PMCR0_IACT);
531	val |= FIELD_PREP(PMCR0_IMODE, mode);
532	write_sysreg_s(val, SYS_IMP_APL_PMCR0_EL1);
533	isb();
534}
535
536static void m1_pmu_start(struct arm_pmu *cpu_pmu)
537{
538	__m1_pmu_set_mode(PMCR0_IMODE_FIQ);
539}
540
541static void m1_pmu_stop(struct arm_pmu *cpu_pmu)
542{
543	__m1_pmu_set_mode(PMCR0_IMODE_OFF);
544}
545
546static int m1_pmu_map_event(struct perf_event *event)
547{
548	/*
549	 * Although the counters are 48bit wide, bit 47 is what
550	 * triggers the overflow interrupt. Advertise the counters
551	 * being 47bit wide to mimick the behaviour of the ARM PMU.
552	 */
553	event->hw.flags |= ARMPMU_EVT_47BIT;
554	return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT);
555}
556
557static int m2_pmu_map_event(struct perf_event *event)
558{
559	/*
560	 * Same deal as the above, except that M2 has 64bit counters.
561	 * Which, as far as we're concerned, actually means 63 bits.
562	 * Yes, this is getting awkward.
563	 */
564	event->hw.flags |= ARMPMU_EVT_63BIT;
565	return armpmu_map_event(event, &m1_pmu_perf_map, NULL, M1_PMU_CFG_EVENT);
566}
567
568static int m1_pmu_map_pmuv3_event(unsigned int eventsel)
569{
570	u16 m1_event = HW_OP_UNSUPPORTED;
571
572	if (eventsel < ARMV8_PMUV3_MAX_COMMON_EVENTS)
573		m1_event = m1_pmu_pmceid_map[eventsel];
574
575	return m1_event == HW_OP_UNSUPPORTED ? -EOPNOTSUPP : m1_event;
576}
577
578static void m1_pmu_init_pmceid(struct arm_pmu *pmu)
579{
580	unsigned int event;
581
582	for (event = 0; event < ARMV8_PMUV3_MAX_COMMON_EVENTS; event++) {
583		if (m1_pmu_map_pmuv3_event(event) >= 0)
584			set_bit(event, pmu->pmceid_bitmap);
585	}
586}
587
588static void m1_pmu_reset(void *info)
589{
590	int i;
591
592	__m1_pmu_set_mode(PMCR0_IMODE_OFF);
593
594	for (i = 0; i < M1_PMU_NR_COUNTERS; i++) {
595		m1_pmu_disable_counter(i);
596		m1_pmu_disable_counter_interrupt(i);
597		m1_pmu_write_hw_counter(0, i);
598	}
599
600	isb();
601}
602
603static int m1_pmu_set_event_filter(struct hw_perf_event *event,
604				   struct perf_event_attr *attr)
605{
606	unsigned long config_base = 0;
607
608	if (!attr->exclude_guest && !is_kernel_in_hyp_mode()) {
609		pr_debug("ARM performance counters do not support mode exclusion\n");
610		return -EOPNOTSUPP;
611	}
612	if (!attr->exclude_kernel)
613		config_base |= M1_PMU_CFG_COUNT_KERNEL;
614	if (!attr->exclude_user)
615		config_base |= M1_PMU_CFG_COUNT_USER;
616	if (!attr->exclude_host)
617		config_base |= M1_PMU_CFG_COUNT_HOST;
618	if (!attr->exclude_guest)
619		config_base |= M1_PMU_CFG_COUNT_GUEST;
620
621	event->config_base = config_base;
622
623	return 0;
624}
625
626static int m1_pmu_init(struct arm_pmu *cpu_pmu, u32 flags)
627{
628	cpu_pmu->handle_irq	  = m1_pmu_handle_irq;
629	cpu_pmu->enable		  = m1_pmu_enable_event;
630	cpu_pmu->disable	  = m1_pmu_disable_event;
631	cpu_pmu->read_counter	  = m1_pmu_read_counter;
632	cpu_pmu->write_counter	  = m1_pmu_write_counter;
633	cpu_pmu->get_event_idx	  = m1_pmu_get_event_idx;
634	cpu_pmu->clear_event_idx  = m1_pmu_clear_event_idx;
635	cpu_pmu->start		  = m1_pmu_start;
636	cpu_pmu->stop		  = m1_pmu_stop;
637
638	if (flags & ARMPMU_EVT_47BIT)
639		cpu_pmu->map_event = m1_pmu_map_event;
640	else if (flags & ARMPMU_EVT_63BIT)
641		cpu_pmu->map_event = m2_pmu_map_event;
642	else
643		return WARN_ON(-EINVAL);
644
645	cpu_pmu->reset		  = m1_pmu_reset;
646	cpu_pmu->set_event_filter = m1_pmu_set_event_filter;
647
648	cpu_pmu->map_pmuv3_event  = m1_pmu_map_pmuv3_event;
649	m1_pmu_init_pmceid(cpu_pmu);
650
651	bitmap_set(cpu_pmu->cntr_mask, 0, M1_PMU_NR_COUNTERS);
652	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = &m1_pmu_events_attr_group;
653	cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = &m1_pmu_format_attr_group;
654	return 0;
655}
656
657/* Device driver gunk */
658static int m1_pmu_ice_init(struct arm_pmu *cpu_pmu)
659{
660	cpu_pmu->name = "apple_icestorm_pmu";
661	return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT);
662}
663
664static int m1_pmu_fire_init(struct arm_pmu *cpu_pmu)
665{
666	cpu_pmu->name = "apple_firestorm_pmu";
667	return m1_pmu_init(cpu_pmu, ARMPMU_EVT_47BIT);
668}
669
670static int m2_pmu_avalanche_init(struct arm_pmu *cpu_pmu)
671{
672	cpu_pmu->name = "apple_avalanche_pmu";
673	return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT);
674}
675
676static int m2_pmu_blizzard_init(struct arm_pmu *cpu_pmu)
677{
678	cpu_pmu->name = "apple_blizzard_pmu";
679	return m1_pmu_init(cpu_pmu, ARMPMU_EVT_63BIT);
680}
681
682static const struct of_device_id m1_pmu_of_device_ids[] = {
683	{ .compatible = "apple,avalanche-pmu",	.data = m2_pmu_avalanche_init, },
684	{ .compatible = "apple,blizzard-pmu",	.data = m2_pmu_blizzard_init, },
685	{ .compatible = "apple,icestorm-pmu",	.data = m1_pmu_ice_init, },
686	{ .compatible = "apple,firestorm-pmu",	.data = m1_pmu_fire_init, },
687	{ },
688};
689MODULE_DEVICE_TABLE(of, m1_pmu_of_device_ids);
690
691static int m1_pmu_device_probe(struct platform_device *pdev)
692{
693	return arm_pmu_device_probe(pdev, m1_pmu_of_device_ids, NULL);
694}
695
696static struct platform_driver m1_pmu_driver = {
697	.driver		= {
698		.name			= "apple-m1-cpu-pmu",
699		.of_match_table		= m1_pmu_of_device_ids,
700		.suppress_bind_attrs	= true,
701	},
702	.probe		= m1_pmu_device_probe,
703};
704
705module_platform_driver(m1_pmu_driver);