arm-ni.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (C) 2022-2024 Arm Limited
  3// NI-700 Network-on-Chip PMU driver
  4
  5#include <linux/acpi.h>
  6#include <linux/bitfield.h>
  7#include <linux/interrupt.h>
  8#include <linux/io.h>
  9#include <linux/io-64-nonatomic-lo-hi.h>
 10#include <linux/kernel.h>
 11#include <linux/module.h>
 12#include <linux/of.h>
 13#include <linux/perf_event.h>
 14#include <linux/platform_device.h>
 15#include <linux/slab.h>
 16
 17/* Common registers */
 18#define NI_NODE_TYPE		0x000
 19#define NI_NODE_TYPE_NODE_ID	GENMASK(31, 16)
 20#define NI_NODE_TYPE_NODE_TYPE	GENMASK(15, 0)
 21
 22#define NI_CHILD_NODE_INFO	0x004
 23#define NI_CHILD_PTR(n)		(0x008 + (n) * 4)
 24#define NI_NUM_SUB_FEATURES	0x100
 25#define NI_SUB_FEATURE_TYPE(n)	(0x108 + (n) * 8)
 26#define NI_SUB_FEATURE_PTR(n)	(0x10c + (n) * 8)
 27
 28#define NI_SUB_FEATURE_TYPE_FCU	0x2
 29
 30#define NI700_PMUSELA		0x00c
 31
 32/* Config node */
 33#define NI_PERIPHERAL_ID0	0xfe0
 34#define NI_PIDR0_PART_7_0	GENMASK(7, 0)
 35#define NI_PERIPHERAL_ID1	0xfe4
 36#define NI_PIDR1_PART_11_8	GENMASK(3, 0)
 37#define NI_PERIPHERAL_ID2	0xfe8
 38#define NI_PIDR2_VERSION	GENMASK(7, 4)
 39
 40/* PMU node */
 41#define NI700_PMEVCNTR(n)	(0x008 + (n) * 8)
 42#define NI700_PMCCNTR_L		0x0f8
 43#define NI_PMEVCNTR(n)		(0x200 + (n) * 8)
 44#define NI_PMCCNTR_L		0x2f8
 45#define NI_PMEVTYPER(n)		(0x400 + (n) * 4)
 46#define NI_PMEVTYPER_NODE_TYPE	GENMASK(12, 9)
 47#define NI_PMEVTYPER_NODE_ID	GENMASK(8, 0)
 48#define NI_PMCNTENSET		0xc00
 49#define NI_PMCNTENCLR		0xc20
 50#define NI_PMINTENSET		0xc40
 51#define NI_PMINTENCLR		0xc60
 52#define NI_PMOVSCLR		0xc80
 53#define NI_PMOVSSET		0xcc0
 54#define NI_PMCFGR		0xe00
 55#define NI_PMCR			0xe04
 56#define NI_PMCR_RESET_CCNT	BIT(2)
 57#define NI_PMCR_RESET_EVCNT	BIT(1)
 58#define NI_PMCR_ENABLE		BIT(0)
 59
 60#define NI_NUM_COUNTERS		8
 61#define NI_CCNT_IDX		31
 62
 63/* Event attributes */
 64#define NI_CONFIG_TYPE		GENMASK_ULL(15, 0)
 65#define NI_CONFIG_NODEID	GENMASK_ULL(31, 16)
 66#define NI_CONFIG_EVENTID	GENMASK_ULL(47, 32)
 67
 68#define NI_EVENT_TYPE(event)	FIELD_GET(NI_CONFIG_TYPE, (event)->attr.config)
 69#define NI_EVENT_NODEID(event)	FIELD_GET(NI_CONFIG_NODEID, (event)->attr.config)
 70#define NI_EVENT_EVENTID(event)	FIELD_GET(NI_CONFIG_EVENTID, (event)->attr.config)
 71
 72enum ni_part {
 73	PART_NI_700 = 0x43b,
 74	PART_NI_710AE = 0x43d,
 75	PART_NOC_S3 = 0x43f,
 76	PART_SI_L1 = 0x455,
 77};
 78
 79enum ni_node_type {
 80	NI_GLOBAL,
 81	NI_VOLTAGE,
 82	NI_POWER,
 83	NI_CLOCK,
 84	NI_ASNI,
 85	NI_AMNI,
 86	NI_PMU,
 87	NI_HSNI,
 88	NI_HMNI,
 89	NI_PMNI,
 90	NI_TSNI,
 91	NI_TMNI,
 92	NI_CMNI = 0x0e,
 93	NI_MCN = 0x63,
 94};
 95
 96struct arm_ni_node {
 97	void __iomem *base;
 98	enum ni_node_type type;
 99	u16 id;
100	u32 num_components;
101};
102
103struct arm_ni_unit {
104	void __iomem *pmusela;
105	enum ni_node_type type;
106	u16 id;
107	bool ns;
108	union {
109		__le64 pmusel;
110		u8 event[8];
111	};
112};
113
114struct arm_ni_cd {
115	void __iomem *pmu_base;
116	u16 id;
117	s8 irq_friend;
118	int num_units;
119	int irq;
120	struct pmu pmu;
121	struct arm_ni_unit *units;
122	struct perf_event *evcnt[NI_NUM_COUNTERS];
123	struct perf_event *ccnt;
124};
125
126struct arm_ni {
127	struct device *dev;
128	void __iomem *base;
129	enum ni_part part;
130	int id;
131	int cpu;
132	int num_cds;
133	struct hlist_node cpuhp_node;
134	struct arm_ni_cd cds[] __counted_by(num_cds);
135};
136
137#define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id])
138#define pmu_to_cd(p) container_of((p), struct arm_ni_cd, pmu)
139
140#define ni_for_each_cd(n, c) \
141	for (struct arm_ni_cd *c = n->cds; c < n->cds + n->num_cds; c++) if (c->pmu_base)
142
143#define cd_for_each_unit(cd, u) \
144	for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++)
145
146static int arm_ni_hp_state;
147
148struct arm_ni_event_attr {
149	struct device_attribute attr;
150	enum ni_node_type type;
151};
152
153#define NI_EVENT_ATTR(_name, _type)					\
154	(&((struct arm_ni_event_attr[]) {{				\
155		.attr = __ATTR(_name, 0444, arm_ni_event_show, NULL),	\
156		.type = _type,						\
157	}})[0].attr.attr)
158
159static ssize_t arm_ni_event_show(struct device *dev,
160				 struct device_attribute *attr, char *buf)
161{
162	struct arm_ni_event_attr *eattr = container_of(attr, typeof(*eattr), attr);
163
164	if (eattr->type == NI_PMU)
165		return sysfs_emit(buf, "type=0x%x\n", eattr->type);
166
167	return sysfs_emit(buf, "type=0x%x,eventid=?,nodeid=?\n", eattr->type);
168}
169
170static umode_t arm_ni_event_attr_is_visible(struct kobject *kobj,
171					    struct attribute *attr, int unused)
172{
173	struct device *dev = kobj_to_dev(kobj);
174	struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));
175	struct arm_ni_event_attr *eattr;
176
177	eattr = container_of(attr, typeof(*eattr), attr.attr);
178
179	cd_for_each_unit(cd, unit) {
180		if (unit->type == eattr->type && unit->ns)
181			return attr->mode;
182	}
183
184	return 0;
185}
186
187static struct attribute *arm_ni_event_attrs[] = {
188	NI_EVENT_ATTR(asni, NI_ASNI),
189	NI_EVENT_ATTR(amni, NI_AMNI),
190	NI_EVENT_ATTR(cycles, NI_PMU),
191	NI_EVENT_ATTR(hsni, NI_HSNI),
192	NI_EVENT_ATTR(hmni, NI_HMNI),
193	NI_EVENT_ATTR(pmni, NI_PMNI),
194	NI_EVENT_ATTR(tsni, NI_TSNI),
195	NI_EVENT_ATTR(tmni, NI_TMNI),
196	NI_EVENT_ATTR(cmni, NI_CMNI),
197	NULL
198};
199
200static const struct attribute_group arm_ni_event_attrs_group = {
201	.name = "events",
202	.attrs = arm_ni_event_attrs,
203	.is_visible = arm_ni_event_attr_is_visible,
204};
205
206struct arm_ni_format_attr {
207	struct device_attribute attr;
208	u64 field;
209};
210
211#define NI_FORMAT_ATTR(_name, _fld)					\
212	(&((struct arm_ni_format_attr[]) {{				\
213		.attr = __ATTR(_name, 0444, arm_ni_format_show, NULL),	\
214		.field = _fld,						\
215	}})[0].attr.attr)
216
217static ssize_t arm_ni_format_show(struct device *dev,
218				  struct device_attribute *attr, char *buf)
219{
220	struct arm_ni_format_attr *fmt = container_of(attr, typeof(*fmt), attr);
221
222	return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field);
223}
224
225static struct attribute *arm_ni_format_attrs[] = {
226	NI_FORMAT_ATTR(type, NI_CONFIG_TYPE),
227	NI_FORMAT_ATTR(nodeid, NI_CONFIG_NODEID),
228	NI_FORMAT_ATTR(eventid, NI_CONFIG_EVENTID),
229	NULL
230};
231
232static const struct attribute_group arm_ni_format_attrs_group = {
233	.name = "format",
234	.attrs = arm_ni_format_attrs,
235};
236
237static ssize_t arm_ni_cpumask_show(struct device *dev,
238				   struct device_attribute *attr, char *buf)
239{
240	struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
241
242	return cpumap_print_to_pagebuf(true, buf, cpumask_of(ni->cpu));
243}
244
245static struct device_attribute arm_ni_cpumask_attr =
246		__ATTR(cpumask, 0444, arm_ni_cpumask_show, NULL);
247
248static ssize_t arm_ni_identifier_show(struct device *dev,
249				      struct device_attribute *attr, char *buf)
250{
251	struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
252	u32 reg = readl_relaxed(ni->base + NI_PERIPHERAL_ID2);
253	int version = FIELD_GET(NI_PIDR2_VERSION, reg);
254
255	return sysfs_emit(buf, "%03x%02x\n", ni->part, version);
256}
257
258static struct device_attribute arm_ni_identifier_attr =
259		__ATTR(identifier, 0444, arm_ni_identifier_show, NULL);
260
261static struct attribute *arm_ni_other_attrs[] = {
262	&arm_ni_cpumask_attr.attr,
263	&arm_ni_identifier_attr.attr,
264	NULL
265};
266
267static const struct attribute_group arm_ni_other_attr_group = {
268	.attrs = arm_ni_other_attrs,
269};
270
271static const struct attribute_group *arm_ni_attr_groups[] = {
272	&arm_ni_event_attrs_group,
273	&arm_ni_format_attrs_group,
274	&arm_ni_other_attr_group,
275	NULL
276};
277
278static void arm_ni_pmu_enable(struct pmu *pmu)
279{
280	writel_relaxed(NI_PMCR_ENABLE, pmu_to_cd(pmu)->pmu_base + NI_PMCR);
281}
282
283static void arm_ni_pmu_disable(struct pmu *pmu)
284{
285	writel_relaxed(0, pmu_to_cd(pmu)->pmu_base + NI_PMCR);
286}
287
288struct arm_ni_val {
289	unsigned int evcnt;
290	unsigned int ccnt;
291};
292
293static bool arm_ni_val_count_event(struct perf_event *evt, struct arm_ni_val *val)
294{
295	if (is_software_event(evt))
296		return true;
297
298	if (NI_EVENT_TYPE(evt) == NI_PMU) {
299		val->ccnt++;
300		return val->ccnt <= 1;
301	}
302
303	val->evcnt++;
304	return val->evcnt <= NI_NUM_COUNTERS;
305}
306
307static int arm_ni_validate_group(struct perf_event *event)
308{
309	struct perf_event *sibling, *leader = event->group_leader;
310	struct arm_ni_val val = { 0 };
311
312	if (leader == event)
313		return 0;
314
315	arm_ni_val_count_event(event, &val);
316	if (!arm_ni_val_count_event(leader, &val))
317		return -EINVAL;
318
319	for_each_sibling_event(sibling, leader) {
320		if (!arm_ni_val_count_event(sibling, &val))
321			return -EINVAL;
322	}
323	return 0;
324}
325
326static bool arm_ni_is_7xx(const struct arm_ni *ni)
327{
328	return ni->part == PART_NI_700 || ni->part == PART_NI_710AE;
329}
330
331static int arm_ni_event_init(struct perf_event *event)
332{
333	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
334	struct arm_ni *ni;
335
336	if (event->attr.type != event->pmu->type)
337		return -ENOENT;
338
339	if (is_sampling_event(event))
340		return -EINVAL;
341
342	ni = cd_to_ni(cd);
343	event->cpu = ni->cpu;
344	event->hw.flags = arm_ni_is_7xx(ni);
345
346	if (NI_EVENT_TYPE(event) == NI_PMU)
347		return arm_ni_validate_group(event);
348
349	cd_for_each_unit(cd, unit) {
350		if (unit->type == NI_EVENT_TYPE(event) &&
351		    unit->id == NI_EVENT_NODEID(event) && unit->ns) {
352			event->hw.config_base = (unsigned long)unit;
353			return arm_ni_validate_group(event);
354		}
355	}
356	return -EINVAL;
357}
358
359static u64 arm_ni_read_ccnt(void __iomem *pmccntr)
360{
361	u64 l, u_old, u_new;
362	int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */
363
364	u_new = readl_relaxed(pmccntr + 4);
365	do {
366		u_old = u_new;
367		l = readl_relaxed(pmccntr);
368		u_new = readl_relaxed(pmccntr + 4);
369	} while (u_new != u_old && --retries);
370	WARN_ON(!retries);
371
372	return (u_new << 32) | l;
373}
374
375static void arm_ni_event_read(struct perf_event *event)
376{
377	struct hw_perf_event *hw = &event->hw;
378	u64 count, prev;
379	bool ccnt = hw->idx == NI_CCNT_IDX;
380
381	do {
382		prev = local64_read(&hw->prev_count);
383		if (ccnt)
384			count = arm_ni_read_ccnt((void __iomem *)event->hw.event_base);
385		else
386			count = readl_relaxed((void __iomem *)event->hw.event_base);
387	} while (local64_cmpxchg(&hw->prev_count, prev, count) != prev);
388
389	count -= prev;
390	if (!ccnt)
391		count = (u32)count;
392	local64_add(count, &event->count);
393}
394
395static void arm_ni_event_start(struct perf_event *event, int flags)
396{
397	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
398
399	writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENSET);
400}
401
402static void arm_ni_event_stop(struct perf_event *event, int flags)
403{
404	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
405
406	writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENCLR);
407	if (flags & PERF_EF_UPDATE)
408		arm_ni_event_read(event);
409}
410
411static void arm_ni_init_ccnt(struct hw_perf_event *hw)
412{
413	local64_set(&hw->prev_count, S64_MIN);
414	lo_hi_writeq_relaxed(S64_MIN, (void __iomem *)hw->event_base);
415}
416
417static void arm_ni_init_evcnt(struct hw_perf_event *hw)
418{
419	local64_set(&hw->prev_count, S32_MIN);
420	writel_relaxed(S32_MIN, (void __iomem *)hw->event_base);
421}
422
423static int arm_ni_event_add(struct perf_event *event, int flags)
424{
425	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
426	struct hw_perf_event *hw = &event->hw;
427	struct arm_ni_unit *unit;
428	enum ni_node_type type = NI_EVENT_TYPE(event);
429	u32 reg;
430
431	if (type == NI_PMU) {
432		if (cd->ccnt)
433			return -ENOSPC;
434		hw->idx = NI_CCNT_IDX;
435		hw->event_base = (unsigned long)cd->pmu_base +
436				 (hw->flags ? NI700_PMCCNTR_L : NI_PMCCNTR_L);
437		cd->ccnt = event;
438		arm_ni_init_ccnt(hw);
439	} else {
440		hw->idx = 0;
441		while (cd->evcnt[hw->idx]) {
442			if (++hw->idx == NI_NUM_COUNTERS)
443				return -ENOSPC;
444		}
445		cd->evcnt[hw->idx] = event;
446		unit = (void *)hw->config_base;
447		unit->event[hw->idx] = NI_EVENT_EVENTID(event);
448		hw->event_base = (unsigned long)cd->pmu_base +
449				 (hw->flags ? NI700_PMEVCNTR(hw->idx) : NI_PMEVCNTR(hw->idx));
450		arm_ni_init_evcnt(hw);
451		lo_hi_writeq_relaxed(le64_to_cpu(unit->pmusel), unit->pmusela);
452
453		reg = FIELD_PREP(NI_PMEVTYPER_NODE_TYPE, type) |
454		      FIELD_PREP(NI_PMEVTYPER_NODE_ID, NI_EVENT_NODEID(event));
455		writel_relaxed(reg, cd->pmu_base + NI_PMEVTYPER(hw->idx));
456	}
457	if (flags & PERF_EF_START)
458		arm_ni_event_start(event, 0);
459	return 0;
460}
461
462static void arm_ni_event_del(struct perf_event *event, int flags)
463{
464	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
465	struct hw_perf_event *hw = &event->hw;
466
467	arm_ni_event_stop(event, PERF_EF_UPDATE);
468
469	if (hw->idx == NI_CCNT_IDX)
470		cd->ccnt = NULL;
471	else
472		cd->evcnt[hw->idx] = NULL;
473}
474
475static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id)
476{
477	struct arm_ni_cd *cd = dev_id;
478	irqreturn_t ret = IRQ_NONE;
479
480	for (;;) {
481		u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR);
482
483		if (reg & (1U << NI_CCNT_IDX)) {
484			ret = IRQ_HANDLED;
485			if (!(WARN_ON(!cd->ccnt))) {
486				arm_ni_event_read(cd->ccnt);
487				arm_ni_init_ccnt(&cd->ccnt->hw);
488			}
489		}
490		for (int i = 0; i < NI_NUM_COUNTERS; i++) {
491			if (!(reg & (1U << i)))
492				continue;
493			ret = IRQ_HANDLED;
494			if (!(WARN_ON(!cd->evcnt[i]))) {
495				arm_ni_event_read(cd->evcnt[i]);
496				arm_ni_init_evcnt(&cd->evcnt[i]->hw);
497			}
498		}
499		writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR);
500		if (!cd->irq_friend)
501			return ret;
502		cd += cd->irq_friend;
503	}
504}
505
506static void __iomem *arm_ni_get_pmusel(struct arm_ni *ni, void __iomem *unit_base)
507{
508	u32 type, ptr, num;
509
510	if (arm_ni_is_7xx(ni))
511		return unit_base + NI700_PMUSELA;
512
513	num = readl_relaxed(unit_base + NI_NUM_SUB_FEATURES);
514	for (int i = 0; i < num; i++) {
515		type = readl_relaxed(unit_base + NI_SUB_FEATURE_TYPE(i));
516		if (type != NI_SUB_FEATURE_TYPE_FCU)
517			continue;
518		ptr = readl_relaxed(unit_base + NI_SUB_FEATURE_PTR(i));
519		return ni->base + ptr;
520	}
521	/* Should be impossible */
522	return NULL;
523}
524
525static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start)
526{
527	struct arm_ni_cd *cd = ni->cds + node->id;
528	const char *name;
529
530	cd->id = node->id;
531	cd->num_units = node->num_components;
532	cd->units = devm_kcalloc(ni->dev, cd->num_units, sizeof(*(cd->units)), GFP_KERNEL);
533	if (!cd->units)
534		return -ENOMEM;
535
536	for (int i = 0; i < cd->num_units; i++) {
537		u32 reg = readl_relaxed(node->base + NI_CHILD_PTR(i));
538		void __iomem *unit_base = ni->base + reg;
539		struct arm_ni_unit *unit = cd->units + i;
540
541		reg = readl_relaxed(unit_base + NI_NODE_TYPE);
542		unit->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg);
543		unit->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg);
544
545		switch (unit->type) {
546		case NI_PMU:
547			reg = readl_relaxed(unit_base + NI_PMCFGR);
548			if (!reg) {
549				dev_info(ni->dev, "No access to PMU %d\n", cd->id);
550				devm_kfree(ni->dev, cd->units);
551				return 0;
552			}
553			unit->ns = true;
554			cd->pmu_base = unit_base;
555			break;
556		case NI_ASNI:
557		case NI_AMNI:
558		case NI_HSNI:
559		case NI_HMNI:
560		case NI_PMNI:
561		case NI_TSNI:
562		case NI_TMNI:
563		case NI_CMNI:
564			unit->pmusela = arm_ni_get_pmusel(ni, unit_base);
565			writel_relaxed(1, unit->pmusela);
566			if (readl_relaxed(unit->pmusela) != 1)
567				dev_info(ni->dev, "No access to node 0x%04x%04x\n", unit->id, unit->type);
568			else
569				unit->ns = true;
570			break;
571		case NI_MCN:
572			break;
573		default:
574			/*
575			 * e.g. FMU - thankfully bits 3:2 of FMU_ERR_FR0 are RES0 so
576			 * can't alias any of the leaf node types we're looking for.
577			 */
578			dev_dbg(ni->dev, "Mystery node 0x%04x%04x\n", unit->id, unit->type);
579			break;
580		}
581	}
582
583	res_start += cd->pmu_base - ni->base;
584	if (!devm_request_mem_region(ni->dev, res_start, SZ_4K, dev_name(ni->dev))) {
585		dev_err(ni->dev, "Failed to request PMU region 0x%llx\n", res_start);
586		return -EBUSY;
587	}
588
589	writel_relaxed(NI_PMCR_RESET_CCNT | NI_PMCR_RESET_EVCNT,
590		       cd->pmu_base + NI_PMCR);
591	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR);
592	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR);
593
594	cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id);
595	if (cd->irq < 0)
596		return cd->irq;
597
598	cd->pmu = (struct pmu) {
599		.module = THIS_MODULE,
600		.parent = ni->dev,
601		.attr_groups = arm_ni_attr_groups,
602		.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
603		.task_ctx_nr = perf_invalid_context,
604		.pmu_enable = arm_ni_pmu_enable,
605		.pmu_disable = arm_ni_pmu_disable,
606		.event_init = arm_ni_event_init,
607		.add = arm_ni_event_add,
608		.del = arm_ni_event_del,
609		.start = arm_ni_event_start,
610		.stop = arm_ni_event_stop,
611		.read = arm_ni_event_read,
612	};
613
614	name = devm_kasprintf(ni->dev, GFP_KERNEL, "arm_ni_%d_cd_%d", ni->id, cd->id);
615	if (!name)
616		return -ENOMEM;
617
618	return perf_pmu_register(&cd->pmu, name, -1);
619}
620
621static void arm_ni_remove(struct platform_device *pdev)
622{
623	struct arm_ni *ni = platform_get_drvdata(pdev);
624
625	ni_for_each_cd(ni, cd) {
626		writel_relaxed(0, cd->pmu_base + NI_PMCR);
627		writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR);
628		perf_pmu_unregister(&cd->pmu);
629	}
630	cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node);
631}
632
633static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node)
634{
635	u32 reg = readl_relaxed(base + NI_NODE_TYPE);
636
637	node->base = base;
638	node->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg);
639	node->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg);
640	node->num_components = readl_relaxed(base + NI_CHILD_NODE_INFO);
641}
642
643static int arm_ni_init_irqs(struct arm_ni *ni)
644{
645	int err;
646
647	ni_for_each_cd(ni, cd) {
648		for (struct arm_ni_cd *prev = cd; prev-- > ni->cds; ) {
649			if (prev->irq == cd->irq) {
650				prev->irq_friend = cd - prev;
651				goto set_inten;
652			}
653		}
654		err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq,
655				       IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN,
656				       dev_name(ni->dev), cd);
657		if (err)
658			return err;
659
660		irq_set_affinity(cd->irq, cpumask_of(ni->cpu));
661set_inten:
662		writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET);
663	}
664
665	ni_for_each_cd(ni, cd)
666		if (!cd->irq_friend)
667			enable_irq(cd->irq);
668	return 0;
669}
670
671static int arm_ni_probe(struct platform_device *pdev)
672{
673	struct arm_ni_node cfg, vd, pd, cd;
674	struct arm_ni *ni;
675	struct resource *res;
676	void __iomem *base;
677	static atomic_t id;
678	int ret, num_cds;
679	u32 reg, part;
680
681	/*
682	 * We want to map the whole configuration space for ease of discovery,
683	 * but the PMU pages are the only ones for which we can honestly claim
684	 * exclusive ownership, so we'll request them explicitly once found.
685	 */
686	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
687	base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
688	if (!base)
689		return -ENOMEM;
690
691	arm_ni_probe_domain(base, &cfg);
692	if (cfg.type != NI_GLOBAL)
693		return -ENODEV;
694
695	reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID0);
696	part = FIELD_GET(NI_PIDR0_PART_7_0, reg);
697	reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID1);
698	part |= FIELD_GET(NI_PIDR1_PART_11_8, reg) << 8;
699
700	switch (part) {
701	case PART_NI_700:
702	case PART_NI_710AE:
703	case PART_NOC_S3:
704	case PART_SI_L1:
705		break;
706	default:
707		dev_WARN(&pdev->dev, "Unknown part number: 0x%03x, this may go badly\n", part);
708		break;
709	}
710
711	num_cds = 0;
712	for (int v = 0; v < cfg.num_components; v++) {
713		reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
714		arm_ni_probe_domain(base + reg, &vd);
715		for (int p = 0; p < vd.num_components; p++) {
716			reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
717			arm_ni_probe_domain(base + reg, &pd);
718			num_cds += pd.num_components;
719		}
720	}
721
722	ni = devm_kzalloc(&pdev->dev, struct_size(ni, cds, num_cds), GFP_KERNEL);
723	if (!ni)
724		return -ENOMEM;
725
726	ni->dev = &pdev->dev;
727	ni->base = base;
728	ni->num_cds = num_cds;
729	ni->part = part;
730	ni->id = atomic_fetch_inc(&id);
731	ni->cpu = cpumask_local_spread(0, dev_to_node(ni->dev));
732	platform_set_drvdata(pdev, ni);
733
734	ret = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node);
735	if (ret)
736		return ret;
737
738	for (int v = 0; v < cfg.num_components; v++) {
739		reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
740		arm_ni_probe_domain(base + reg, &vd);
741		for (int p = 0; p < vd.num_components; p++) {
742			reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
743			arm_ni_probe_domain(base + reg, &pd);
744			for (int c = 0; c < pd.num_components; c++) {
745				reg = readl_relaxed(pd.base + NI_CHILD_PTR(c));
746				arm_ni_probe_domain(base + reg, &cd);
747				ret = arm_ni_init_cd(ni, &cd, res->start);
748				if (ret) {
749					ni->cds[cd.id].pmu_base = NULL;
750					arm_ni_remove(pdev);
751					return ret;
752				}
753			}
754		}
755	}
756
757	ret = arm_ni_init_irqs(ni);
758	if (ret)
759		arm_ni_remove(pdev);
760
761	return ret;
762}
763
764#ifdef CONFIG_OF
765static const struct of_device_id arm_ni_of_match[] = {
766	{ .compatible = "arm,ni-700" },
767	{}
768};
769MODULE_DEVICE_TABLE(of, arm_ni_of_match);
770#endif
771
772#ifdef CONFIG_ACPI
773static const struct acpi_device_id arm_ni_acpi_match[] = {
774	{ "ARMHCB70" },
775	{}
776};
777MODULE_DEVICE_TABLE(acpi, arm_ni_acpi_match);
778#endif
779
780static struct platform_driver arm_ni_driver = {
781	.driver = {
782		.name = "arm-ni",
783		.of_match_table = of_match_ptr(arm_ni_of_match),
784		.acpi_match_table = ACPI_PTR(arm_ni_acpi_match),
785		.suppress_bind_attrs = true,
786	},
787	.probe = arm_ni_probe,
788	.remove = arm_ni_remove,
789};
790
791static void arm_ni_pmu_migrate(struct arm_ni *ni, unsigned int cpu)
792{
793	ni_for_each_cd(ni, cd) {
794		perf_pmu_migrate_context(&cd->pmu, ni->cpu, cpu);
795		irq_set_affinity(cd->irq, cpumask_of(cpu));
796	}
797	ni->cpu = cpu;
798}
799
800static int arm_ni_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
801{
802	struct arm_ni *ni;
803	int node;
804
805	ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node);
806	node = dev_to_node(ni->dev);
807	if (cpu_to_node(ni->cpu) != node && cpu_to_node(cpu) == node)
808		arm_ni_pmu_migrate(ni, cpu);
809	return 0;
810}
811
812static int arm_ni_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
813{
814	struct arm_ni *ni;
815	unsigned int target;
816	int node;
817
818	ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node);
819	if (cpu != ni->cpu)
820		return 0;
821
822	node = dev_to_node(ni->dev);
823	target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
824	if (target >= nr_cpu_ids)
825		target = cpumask_any_but(cpu_online_mask, cpu);
826
827	if (target < nr_cpu_ids)
828		arm_ni_pmu_migrate(ni, target);
829	return 0;
830}
831
832static int __init arm_ni_init(void)
833{
834	int ret;
835
836	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
837				      "perf/arm/ni:online",
838				      arm_ni_pmu_online_cpu,
839				      arm_ni_pmu_offline_cpu);
840	if (ret < 0)
841		return ret;
842
843	arm_ni_hp_state = ret;
844
845	ret = platform_driver_register(&arm_ni_driver);
846	if (ret)
847		cpuhp_remove_multi_state(arm_ni_hp_state);
848	return ret;
849}
850
851static void __exit arm_ni_exit(void)
852{
853	platform_driver_unregister(&arm_ni_driver);
854	cpuhp_remove_multi_state(arm_ni_hp_state);
855}
856
857module_init(arm_ni_init);
858module_exit(arm_ni_exit);
859
860MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
861MODULE_DESCRIPTION("Arm NI-700 PMU driver");
862MODULE_LICENSE("GPL v2");