marvell_cn10k_tad_pmu.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Marvell CN10K LLC-TAD perf driver
  3 *
  4 * Copyright (C) 2021 Marvell
  5 */
  6
  7#define pr_fmt(fmt) "tad_pmu: " fmt
  8
  9#include <linux/io.h>
 10#include <linux/module.h>
 11#include <linux/of.h>
 12#include <linux/cpuhotplug.h>
 13#include <linux/perf_event.h>
 14#include <linux/platform_device.h>
 15#include <linux/acpi.h>
 16
 17#define TAD_PFC_OFFSET		0x800
 18#define TAD_PFC(counter)	(TAD_PFC_OFFSET | (counter << 3))
 19#define TAD_PRF_OFFSET		0x900
 20#define TAD_PRF(counter)	(TAD_PRF_OFFSET | (counter << 3))
 21#define TAD_PRF_CNTSEL_MASK	0xFF
 22#define TAD_MAX_COUNTERS	8
 23
 24#define to_tad_pmu(p) (container_of(p, struct tad_pmu, pmu))
 25
 26struct tad_region {
 27	void __iomem	*base;
 28};
 29
 30struct tad_pmu {
 31	struct pmu pmu;
 32	struct tad_region *regions;
 33	u32 region_cnt;
 34	unsigned int cpu;
 35	struct hlist_node node;
 36	struct perf_event *events[TAD_MAX_COUNTERS];
 37	DECLARE_BITMAP(counters_map, TAD_MAX_COUNTERS);
 38};
 39
 40enum mrvl_tad_pmu_version {
 41	TAD_PMU_V1 = 1,
 42	TAD_PMU_V2,
 43};
 44
 45struct tad_pmu_data {
 46	int id;
 47};
 48
 49static int tad_pmu_cpuhp_state;
 50
 51static void tad_pmu_event_counter_read(struct perf_event *event)
 52{
 53	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
 54	struct hw_perf_event *hwc = &event->hw;
 55	u32 counter_idx = hwc->idx;
 56	u64 prev, new;
 57	int i;
 58
 59	do {
 60		prev = local64_read(&hwc->prev_count);
 61		for (i = 0, new = 0; i < tad_pmu->region_cnt; i++)
 62			new += readq(tad_pmu->regions[i].base +
 63				     TAD_PFC(counter_idx));
 64	} while (local64_cmpxchg(&hwc->prev_count, prev, new) != prev);
 65
 66	local64_add(new - prev, &event->count);
 67}
 68
 69static void tad_pmu_event_counter_stop(struct perf_event *event, int flags)
 70{
 71	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
 72	struct hw_perf_event *hwc = &event->hw;
 73	u32 counter_idx = hwc->idx;
 74	int i;
 75
 76	/* TAD()_PFC() stop counting on the write
 77	 * which sets TAD()_PRF()[CNTSEL] == 0
 78	 */
 79	for (i = 0; i < tad_pmu->region_cnt; i++) {
 80		writeq_relaxed(0, tad_pmu->regions[i].base +
 81			       TAD_PRF(counter_idx));
 82	}
 83
 84	tad_pmu_event_counter_read(event);
 85	hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 86}
 87
 88static void tad_pmu_event_counter_start(struct perf_event *event, int flags)
 89{
 90	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
 91	struct hw_perf_event *hwc = &event->hw;
 92	u32 event_idx = event->attr.config;
 93	u32 counter_idx = hwc->idx;
 94	u64 reg_val;
 95	int i;
 96
 97	hwc->state = 0;
 98
 99	/* Typically TAD_PFC() are zeroed to start counting */
100	for (i = 0; i < tad_pmu->region_cnt; i++)
101		writeq_relaxed(0, tad_pmu->regions[i].base +
102			       TAD_PFC(counter_idx));
103
104	/* TAD()_PFC() start counting on the write
105	 * which sets TAD()_PRF()[CNTSEL] != 0
106	 */
107	for (i = 0; i < tad_pmu->region_cnt; i++) {
108		reg_val = event_idx & 0xFF;
109		writeq_relaxed(reg_val,	tad_pmu->regions[i].base +
110			       TAD_PRF(counter_idx));
111	}
112}
113
114static void tad_pmu_event_counter_del(struct perf_event *event, int flags)
115{
116	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
117	struct hw_perf_event *hwc = &event->hw;
118	int idx = hwc->idx;
119
120	tad_pmu_event_counter_stop(event, flags | PERF_EF_UPDATE);
121	tad_pmu->events[idx] = NULL;
122	clear_bit(idx, tad_pmu->counters_map);
123}
124
125static int tad_pmu_event_counter_add(struct perf_event *event, int flags)
126{
127	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
128	struct hw_perf_event *hwc = &event->hw;
129	int idx;
130
131	/* Get a free counter for this event */
132	idx = find_first_zero_bit(tad_pmu->counters_map, TAD_MAX_COUNTERS);
133	if (idx == TAD_MAX_COUNTERS)
134		return -EAGAIN;
135
136	set_bit(idx, tad_pmu->counters_map);
137
138	hwc->idx = idx;
139	hwc->state = PERF_HES_STOPPED;
140	tad_pmu->events[idx] = event;
141
142	if (flags & PERF_EF_START)
143		tad_pmu_event_counter_start(event, flags);
144
145	return 0;
146}
147
148static int tad_pmu_event_init(struct perf_event *event)
149{
150	struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
151
152	if (event->attr.type != event->pmu->type)
153		return -ENOENT;
154
155	if (!event->attr.disabled)
156		return -EINVAL;
157
158	if (event->state != PERF_EVENT_STATE_OFF)
159		return -EINVAL;
160
161	event->cpu = tad_pmu->cpu;
162	event->hw.idx = -1;
163	event->hw.config_base = event->attr.config;
164
165	return 0;
166}
167
168static ssize_t tad_pmu_event_show(struct device *dev,
169				struct device_attribute *attr, char *page)
170{
171	struct perf_pmu_events_attr *pmu_attr;
172
173	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
174	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
175}
176
177#define TAD_PMU_EVENT_ATTR(name, config)			\
178	PMU_EVENT_ATTR_ID(name, tad_pmu_event_show, config)
179
180static struct attribute *tad_pmu_event_attrs[] = {
181	TAD_PMU_EVENT_ATTR(tad_none, 0x0),
182	TAD_PMU_EVENT_ATTR(tad_req_msh_in_any, 0x1),
183	TAD_PMU_EVENT_ATTR(tad_req_msh_in_mn, 0x2),
184	TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3),
185	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_any, 0x4),
186	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_mn, 0x5),
187	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_exlmn, 0x6),
188	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_dss, 0x7),
189	TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_retry_dss, 0x8),
190	TAD_PMU_EVENT_ATTR(tad_dat_msh_in_any, 0x9),
191	TAD_PMU_EVENT_ATTR(tad_dat_msh_in_dss, 0xa),
192	TAD_PMU_EVENT_ATTR(tad_req_msh_out_any, 0xb),
193	TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_rd, 0xc),
194	TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_wr, 0xd),
195	TAD_PMU_EVENT_ATTR(tad_req_msh_out_evict, 0xe),
196	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_any, 0xf),
197	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_exlmn, 0x10),
198	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_mn, 0x11),
199	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_exlmn, 0x12),
200	TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_mn, 0x13),
201	TAD_PMU_EVENT_ATTR(tad_snp_msh_out_any, 0x14),
202	TAD_PMU_EVENT_ATTR(tad_snp_msh_out_mn, 0x15),
203	TAD_PMU_EVENT_ATTR(tad_snp_msh_out_exlmn, 0x16),
204	TAD_PMU_EVENT_ATTR(tad_dat_msh_out_any, 0x17),
205	TAD_PMU_EVENT_ATTR(tad_dat_msh_out_fill, 0x18),
206	TAD_PMU_EVENT_ATTR(tad_dat_msh_out_dss, 0x19),
207	TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a),
208	TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b),
209	TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c),
210	TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d),
211	TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e),
212	TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f),
213	TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20),
214	TAD_PMU_EVENT_ATTR(tad_dat_rd, 0x21),
215	TAD_PMU_EVENT_ATTR(tad_dat_rd_byp, 0x22),
216	TAD_PMU_EVENT_ATTR(tad_ifb_occ, 0x23),
217	TAD_PMU_EVENT_ATTR(tad_req_occ, 0x24),
218	NULL
219};
220
221static const struct attribute_group tad_pmu_events_attr_group = {
222	.name = "events",
223	.attrs = tad_pmu_event_attrs,
224};
225
226static struct attribute *ody_tad_pmu_event_attrs[] = {
227	TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3),
228	TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a),
229	TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b),
230	TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c),
231	TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d),
232	TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e),
233	TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f),
234	TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20),
235	TAD_PMU_EVENT_ATTR(tad_tot_cycle, 0xFF),
236	NULL
237};
238
239static const struct attribute_group ody_tad_pmu_events_attr_group = {
240	.name = "events",
241	.attrs = ody_tad_pmu_event_attrs,
242};
243
244PMU_FORMAT_ATTR(event, "config:0-7");
245
246static struct attribute *tad_pmu_format_attrs[] = {
247	&format_attr_event.attr,
248	NULL
249};
250
251static struct attribute_group tad_pmu_format_attr_group = {
252	.name = "format",
253	.attrs = tad_pmu_format_attrs,
254};
255
256static ssize_t tad_pmu_cpumask_show(struct device *dev,
257				struct device_attribute *attr, char *buf)
258{
259	struct tad_pmu *tad_pmu = to_tad_pmu(dev_get_drvdata(dev));
260
261	return cpumap_print_to_pagebuf(true, buf, cpumask_of(tad_pmu->cpu));
262}
263
264static DEVICE_ATTR(cpumask, 0444, tad_pmu_cpumask_show, NULL);
265
266static struct attribute *tad_pmu_cpumask_attrs[] = {
267	&dev_attr_cpumask.attr,
268	NULL
269};
270
271static struct attribute_group tad_pmu_cpumask_attr_group = {
272	.attrs = tad_pmu_cpumask_attrs,
273};
274
275static const struct attribute_group *tad_pmu_attr_groups[] = {
276	&tad_pmu_events_attr_group,
277	&tad_pmu_format_attr_group,
278	&tad_pmu_cpumask_attr_group,
279	NULL
280};
281
282static const struct attribute_group *ody_tad_pmu_attr_groups[] = {
283	&ody_tad_pmu_events_attr_group,
284	&tad_pmu_format_attr_group,
285	&tad_pmu_cpumask_attr_group,
286	NULL
287};
288
289static int tad_pmu_probe(struct platform_device *pdev)
290{
291	const struct tad_pmu_data *dev_data;
292	struct device *dev = &pdev->dev;
293	struct tad_region *regions;
294	struct tad_pmu *tad_pmu;
295	struct resource *res;
296	u32 tad_pmu_page_size;
297	u32 tad_page_size;
298	u32 tad_cnt;
299	int version;
300	int i, ret;
301	char *name;
302
303	tad_pmu = devm_kzalloc(&pdev->dev, sizeof(*tad_pmu), GFP_KERNEL);
304	if (!tad_pmu)
305		return -ENOMEM;
306
307	platform_set_drvdata(pdev, tad_pmu);
308
309	dev_data = device_get_match_data(&pdev->dev);
310	if (!dev_data) {
311		dev_err(&pdev->dev, "Error: No device match data found\n");
312		return -ENODEV;
313	}
314	version = dev_data->id;
315
316	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
317	if (!res) {
318		dev_err(&pdev->dev, "Mem resource not found\n");
319		return -ENODEV;
320	}
321
322	ret = device_property_read_u32(dev, "marvell,tad-page-size",
323				       &tad_page_size);
324	if (ret) {
325		dev_err(&pdev->dev, "Can't find tad-page-size property\n");
326		return ret;
327	}
328
329	ret = device_property_read_u32(dev, "marvell,tad-pmu-page-size",
330				       &tad_pmu_page_size);
331	if (ret) {
332		dev_err(&pdev->dev, "Can't find tad-pmu-page-size property\n");
333		return ret;
334	}
335
336	ret = device_property_read_u32(dev, "marvell,tad-cnt", &tad_cnt);
337	if (ret) {
338		dev_err(&pdev->dev, "Can't find tad-cnt property\n");
339		return ret;
340	}
341
342	regions = devm_kcalloc(&pdev->dev, tad_cnt,
343			       sizeof(*regions), GFP_KERNEL);
344	if (!regions)
345		return -ENOMEM;
346
347	/* ioremap the distributed TAD pmu regions */
348	for (i = 0; i < tad_cnt && res->start < res->end; i++) {
349		regions[i].base = devm_ioremap(&pdev->dev,
350					       res->start,
351					       tad_pmu_page_size);
352		if (!regions[i].base) {
353			dev_err(&pdev->dev, "TAD%d ioremap fail\n", i);
354			return -ENOMEM;
355		}
356		res->start += tad_page_size;
357	}
358
359	tad_pmu->regions = regions;
360	tad_pmu->region_cnt = tad_cnt;
361
362	tad_pmu->pmu = (struct pmu) {
363
364		.module		= THIS_MODULE,
365		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE |
366				  PERF_PMU_CAP_NO_INTERRUPT,
367		.task_ctx_nr	= perf_invalid_context,
368
369		.event_init	= tad_pmu_event_init,
370		.add		= tad_pmu_event_counter_add,
371		.del		= tad_pmu_event_counter_del,
372		.start		= tad_pmu_event_counter_start,
373		.stop		= tad_pmu_event_counter_stop,
374		.read		= tad_pmu_event_counter_read,
375	};
376
377	if (version == TAD_PMU_V1)
378		tad_pmu->pmu.attr_groups = tad_pmu_attr_groups;
379	else
380		tad_pmu->pmu.attr_groups = ody_tad_pmu_attr_groups;
381
382	tad_pmu->cpu = raw_smp_processor_id();
383
384	/* Register pmu instance for cpu hotplug */
385	ret = cpuhp_state_add_instance_nocalls(tad_pmu_cpuhp_state,
386					       &tad_pmu->node);
387	if (ret) {
388		dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
389		return ret;
390	}
391
392	name = "tad";
393	ret = perf_pmu_register(&tad_pmu->pmu, name, -1);
394	if (ret)
395		cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state,
396						    &tad_pmu->node);
397
398	return ret;
399}
400
401static void tad_pmu_remove(struct platform_device *pdev)
402{
403	struct tad_pmu *pmu = platform_get_drvdata(pdev);
404
405	cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state,
406						&pmu->node);
407	perf_pmu_unregister(&pmu->pmu);
408}
409
410#if defined(CONFIG_OF) || defined(CONFIG_ACPI)
411static const struct tad_pmu_data tad_pmu_data = {
412	.id   = TAD_PMU_V1,
413};
414#endif
415
416#ifdef CONFIG_ACPI
417static const struct tad_pmu_data tad_pmu_v2_data = {
418	.id   = TAD_PMU_V2,
419};
420#endif
421
422#ifdef CONFIG_OF
423static const struct of_device_id tad_pmu_of_match[] = {
424	{ .compatible = "marvell,cn10k-tad-pmu", .data = &tad_pmu_data },
425	{},
426};
427#endif
428
429#ifdef CONFIG_ACPI
430static const struct acpi_device_id tad_pmu_acpi_match[] = {
431	{"MRVL000B", (kernel_ulong_t)&tad_pmu_data},
432	{"MRVL000D", (kernel_ulong_t)&tad_pmu_v2_data},
433	{},
434};
435MODULE_DEVICE_TABLE(acpi, tad_pmu_acpi_match);
436#endif
437
438static struct platform_driver tad_pmu_driver = {
439	.driver         = {
440		.name   = "cn10k_tad_pmu",
441		.of_match_table = of_match_ptr(tad_pmu_of_match),
442		.acpi_match_table = ACPI_PTR(tad_pmu_acpi_match),
443		.suppress_bind_attrs = true,
444	},
445	.probe          = tad_pmu_probe,
446	.remove         = tad_pmu_remove,
447};
448
449static int tad_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
450{
451	struct tad_pmu *pmu = hlist_entry_safe(node, struct tad_pmu, node);
452	unsigned int target;
453
454	if (cpu != pmu->cpu)
455		return 0;
456
457	target = cpumask_any_but(cpu_online_mask, cpu);
458	if (target >= nr_cpu_ids)
459		return 0;
460
461	perf_pmu_migrate_context(&pmu->pmu, cpu, target);
462	pmu->cpu = target;
463
464	return 0;
465}
466
467static int __init tad_pmu_init(void)
468{
469	int ret;
470
471	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
472				      "perf/cn10k/tadpmu:online",
473				      NULL,
474				      tad_pmu_offline_cpu);
475	if (ret < 0)
476		return ret;
477	tad_pmu_cpuhp_state = ret;
478	ret = platform_driver_register(&tad_pmu_driver);
479	if (ret)
480		cpuhp_remove_multi_state(tad_pmu_cpuhp_state);
481
482	return ret;
483}
484
485static void __exit tad_pmu_exit(void)
486{
487	platform_driver_unregister(&tad_pmu_driver);
488	cpuhp_remove_multi_state(tad_pmu_cpuhp_state);
489}
490
491module_init(tad_pmu_init);
492module_exit(tad_pmu_exit);
493
494MODULE_DESCRIPTION("Marvell CN10K LLC-TAD Perf driver");
495MODULE_AUTHOR("Bhaskara Budiredla <bbudiredla@marvell.com>");
496MODULE_LICENSE("GPL v2");