perfmon.c

  1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright(c) 2020 Intel Corporation. All rights rsvd. */
  3
  4#include <linux/sched/task.h>
  5#include <linux/io-64-nonatomic-lo-hi.h>
  6#include "idxd.h"
  7#include "perfmon.h"
  8
  9/*
 10 * These attributes specify the bits in the config word that the perf
 11 * syscall uses to pass the event ids and categories to perfmon.
 12 */
 13DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3");
 14DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31");
 15
 16/*
 17 * These attributes specify the bits in the config1 word that the perf
 18 * syscall uses to pass filter data to perfmon.
 19 */
 20DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31");
 21DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39");
 22DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43");
 23DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51");
 24DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59");
 25
 26#define PERFMON_FILTERS_START	2
 27#define PERFMON_FILTERS_MAX	5
 28
 29static struct attribute *perfmon_format_attrs[] = {
 30	&format_attr_idxd_event_category.attr,
 31	&format_attr_idxd_event.attr,
 32	&format_attr_idxd_filter_wq.attr,
 33	&format_attr_idxd_filter_tc.attr,
 34	&format_attr_idxd_filter_pgsz.attr,
 35	&format_attr_idxd_filter_sz.attr,
 36	&format_attr_idxd_filter_eng.attr,
 37	NULL,
 38};
 39
 40static struct attribute_group perfmon_format_attr_group = {
 41	.name = "format",
 42	.attrs = perfmon_format_attrs,
 43};
 44
 45static const struct attribute_group *perfmon_attr_groups[] = {
 46	&perfmon_format_attr_group,
 47	NULL,
 48};
 49
 50static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)
 51{
 52	return &idxd_pmu->pmu == event->pmu;
 53}
 54
 55static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,
 56				  struct perf_event *leader,
 57				  bool do_grp)
 58{
 59	struct perf_event *event;
 60	int n, max_count;
 61
 62	max_count = idxd_pmu->n_counters;
 63	n = idxd_pmu->n_events;
 64
 65	if (n >= max_count)
 66		return -EINVAL;
 67
 68	if (is_idxd_event(idxd_pmu, leader)) {
 69		idxd_pmu->event_list[n] = leader;
 70		idxd_pmu->event_list[n]->hw.idx = n;
 71		n++;
 72	}
 73
 74	if (!do_grp)
 75		return n;
 76
 77	for_each_sibling_event(event, leader) {
 78		if (!is_idxd_event(idxd_pmu, event) ||
 79		    event->state <= PERF_EVENT_STATE_OFF)
 80			continue;
 81
 82		if (n >= max_count)
 83			return -EINVAL;
 84
 85		idxd_pmu->event_list[n] = event;
 86		idxd_pmu->event_list[n]->hw.idx = n;
 87		n++;
 88	}
 89
 90	return n;
 91}
 92
 93static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,
 94				    struct perf_event *event, int idx)
 95{
 96	struct idxd_device *idxd = idxd_pmu->idxd;
 97	struct hw_perf_event *hwc = &event->hw;
 98
 99	hwc->idx = idx;
100	hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx));
101	hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx));
102}
103
104static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,
105				struct perf_event *event)
106{
107	int i;
108
109	for (i = 0; i < IDXD_PMU_EVENT_MAX; i++)
110		if (!test_and_set_bit(i, idxd_pmu->used_mask))
111			return i;
112
113	return -EINVAL;
114}
115
116/*
117 * Check whether there are enough counters to satisfy that all the
118 * events in the group can actually be scheduled at the same time.
119 *
120 * To do this, create a fake idxd_pmu object so the event collection
121 * and assignment functions can be used without affecting the internal
122 * state of the real idxd_pmu object.
123 */
124static int perfmon_validate_group(struct idxd_pmu *pmu,
125				  struct perf_event *event)
126{
127	struct perf_event *leader = event->group_leader;
128	struct idxd_pmu *fake_pmu;
129	int i, ret = 0, n, idx;
130
131	fake_pmu = kzalloc(sizeof(*fake_pmu), GFP_KERNEL);
132	if (!fake_pmu)
133		return -ENOMEM;
134
135	fake_pmu->pmu.name = pmu->pmu.name;
136	fake_pmu->n_counters = pmu->n_counters;
137
138	n = perfmon_collect_events(fake_pmu, leader, true);
139	if (n < 0) {
140		ret = n;
141		goto out;
142	}
143
144	fake_pmu->n_events = n;
145	n = perfmon_collect_events(fake_pmu, event, false);
146	if (n < 0) {
147		ret = n;
148		goto out;
149	}
150
151	fake_pmu->n_events = n;
152
153	for (i = 0; i < n; i++) {
154		event = fake_pmu->event_list[i];
155
156		idx = perfmon_assign_event(fake_pmu, event);
157		if (idx < 0) {
158			ret = idx;
159			goto out;
160		}
161	}
162out:
163	kfree(fake_pmu);
164
165	return ret;
166}
167
168static int perfmon_pmu_event_init(struct perf_event *event)
169{
170	struct idxd_device *idxd;
171	int ret = 0;
172
173	idxd = event_to_idxd(event);
174	event->hw.idx = -1;
175
176	if (event->attr.type != event->pmu->type)
177		return -ENOENT;
178
179	/* sampling not supported */
180	if (event->attr.sample_period)
181		return -EINVAL;
182
183	if (event->cpu < 0)
184		return -EINVAL;
185
186	if (event->pmu != &idxd->idxd_pmu->pmu)
187		return -EINVAL;
188
189	event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd));
190	event->hw.config = event->attr.config;
191
192	if (event->group_leader != event)
193		 /* non-group events have themselves as leader */
194		ret = perfmon_validate_group(idxd->idxd_pmu, event);
195
196	return ret;
197}
198
199static inline u64 perfmon_pmu_read_counter(struct perf_event *event)
200{
201	struct hw_perf_event *hwc = &event->hw;
202	struct idxd_device *idxd;
203	int cntr = hwc->idx;
204
205	idxd = event_to_idxd(event);
206
207	return ioread64(CNTRDATA_REG(idxd, cntr));
208}
209
210static void perfmon_pmu_event_update(struct perf_event *event)
211{
212	struct idxd_device *idxd = event_to_idxd(event);
213	u64 prev_raw_count, new_raw_count, delta, p, n;
214	int shift = 64 - idxd->idxd_pmu->counter_width;
215	struct hw_perf_event *hwc = &event->hw;
216
217	prev_raw_count = local64_read(&hwc->prev_count);
218	do {
219		new_raw_count = perfmon_pmu_read_counter(event);
220	} while (!local64_try_cmpxchg(&hwc->prev_count,
221				      &prev_raw_count, new_raw_count));
222	n = (new_raw_count << shift);
223	p = (prev_raw_count << shift);
224
225	delta = ((n - p) >> shift);
226
227	local64_add(delta, &event->count);
228}
229
230void perfmon_counter_overflow(struct idxd_device *idxd)
231{
232	int i, n_counters, max_loop = OVERFLOW_SIZE;
233	struct perf_event *event;
234	unsigned long ovfstatus;
235
236	n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);
237
238	ovfstatus = ioread32(OVFSTATUS_REG(idxd));
239
240	/*
241	 * While updating overflowed counters, other counters behind
242	 * them could overflow and be missed in a given pass.
243	 * Normally this could happen at most n_counters times, but in
244	 * theory a tiny counter width could result in continual
245	 * overflows and endless looping.  max_loop provides a
246	 * failsafe in that highly unlikely case.
247	 */
248	while (ovfstatus && max_loop--) {
249		/* Figure out which counter(s) overflowed */
250		for_each_set_bit(i, &ovfstatus, n_counters) {
251			unsigned long ovfstatus_clear = 0;
252
253			/* Update event->count for overflowed counter */
254			event = idxd->idxd_pmu->event_list[i];
255			perfmon_pmu_event_update(event);
256			/* Writing 1 to OVFSTATUS bit clears it */
257			set_bit(i, &ovfstatus_clear);
258			iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd));
259		}
260
261		ovfstatus = ioread32(OVFSTATUS_REG(idxd));
262	}
263
264	/*
265	 * Should never happen.  If so, it means a counter(s) looped
266	 * around twice while this handler was running.
267	 */
268	WARN_ON_ONCE(ovfstatus);
269}
270
271static inline void perfmon_reset_config(struct idxd_device *idxd)
272{
273	iowrite32(CONFIG_RESET, PERFRST_REG(idxd));
274	iowrite32(0, OVFSTATUS_REG(idxd));
275	iowrite32(0, PERFFRZ_REG(idxd));
276}
277
278static inline void perfmon_reset_counters(struct idxd_device *idxd)
279{
280	iowrite32(CNTR_RESET, PERFRST_REG(idxd));
281}
282
283static inline void perfmon_reset(struct idxd_device *idxd)
284{
285	perfmon_reset_config(idxd);
286	perfmon_reset_counters(idxd);
287}
288
289static void perfmon_pmu_event_start(struct perf_event *event, int mode)
290{
291	u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0;
292	u64 cntr_cfg, cntrdata, event_enc, event_cat = 0;
293	struct hw_perf_event *hwc = &event->hw;
294	union filter_cfg flt_cfg;
295	union event_cfg event_cfg;
296	struct idxd_device *idxd;
297	int cntr;
298
299	idxd = event_to_idxd(event);
300
301	event->hw.idx = hwc->idx;
302	cntr = hwc->idx;
303
304	/* Obtain event category and event value from user space */
305	event_cfg.val = event->attr.config;
306	flt_cfg.val = event->attr.config1;
307	event_cat = event_cfg.event_cat;
308	event_enc = event_cfg.event_enc;
309
310	/* Obtain filter configuration from user space */
311	flt_wq = flt_cfg.wq;
312	flt_tc = flt_cfg.tc;
313	flt_pg_sz = flt_cfg.pg_sz;
314	flt_xfer_sz = flt_cfg.xfer_sz;
315	flt_eng = flt_cfg.eng;
316
317	if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))
318		iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ));
319	if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))
320		iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC));
321	if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))
322		iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ));
323	if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))
324		iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ));
325	if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))
326		iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG));
327
328	/* Read the start value */
329	cntrdata = ioread64(CNTRDATA_REG(idxd, cntr));
330	local64_set(&event->hw.prev_count, cntrdata);
331
332	/* Set counter to event/category */
333	cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT;
334	cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT;
335	/* Set interrupt on overflow and counter enable bits */
336	cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE);
337
338	iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
339}
340
341static void perfmon_pmu_event_stop(struct perf_event *event, int mode)
342{
343	struct hw_perf_event *hwc = &event->hw;
344	struct idxd_device *idxd;
345	int i, cntr = hwc->idx;
346	u64 cntr_cfg;
347
348	idxd = event_to_idxd(event);
349
350	/* remove this event from event list */
351	for (i = 0; i < idxd->idxd_pmu->n_events; i++) {
352		if (event != idxd->idxd_pmu->event_list[i])
353			continue;
354
355		for (++i; i < idxd->idxd_pmu->n_events; i++)
356			idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];
357		--idxd->idxd_pmu->n_events;
358		break;
359	}
360
361	cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr));
362	cntr_cfg &= ~CNTRCFG_ENABLE;
363	iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
364
365	if (mode == PERF_EF_UPDATE)
366		perfmon_pmu_event_update(event);
367
368	event->hw.idx = -1;
369	clear_bit(cntr, idxd->idxd_pmu->used_mask);
370}
371
372static void perfmon_pmu_event_del(struct perf_event *event, int mode)
373{
374	perfmon_pmu_event_stop(event, PERF_EF_UPDATE);
375}
376
377static int perfmon_pmu_event_add(struct perf_event *event, int flags)
378{
379	struct idxd_device *idxd = event_to_idxd(event);
380	struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;
381	struct hw_perf_event *hwc = &event->hw;
382	int idx, n;
383
384	n = perfmon_collect_events(idxd_pmu, event, false);
385	if (n < 0)
386		return n;
387
388	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
389	if (!(flags & PERF_EF_START))
390		hwc->state |= PERF_HES_ARCH;
391
392	idx = perfmon_assign_event(idxd_pmu, event);
393	if (idx < 0)
394		return idx;
395
396	perfmon_assign_hw_event(idxd_pmu, event, idx);
397
398	if (flags & PERF_EF_START)
399		perfmon_pmu_event_start(event, 0);
400
401	idxd_pmu->n_events = n;
402
403	return 0;
404}
405
406static void enable_perfmon_pmu(struct idxd_device *idxd)
407{
408	iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd));
409}
410
411static void disable_perfmon_pmu(struct idxd_device *idxd)
412{
413	iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd));
414}
415
416static void perfmon_pmu_enable(struct pmu *pmu)
417{
418	struct idxd_device *idxd = pmu_to_idxd(pmu);
419
420	enable_perfmon_pmu(idxd);
421}
422
423static void perfmon_pmu_disable(struct pmu *pmu)
424{
425	struct idxd_device *idxd = pmu_to_idxd(pmu);
426
427	disable_perfmon_pmu(idxd);
428}
429
430static void skip_filter(int i)
431{
432	int j;
433
434	for (j = i; j < PERFMON_FILTERS_MAX; j++)
435		perfmon_format_attrs[PERFMON_FILTERS_START + j] =
436			perfmon_format_attrs[PERFMON_FILTERS_START + j + 1];
437}
438
439static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)
440{
441	int i;
442
443	for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) {
444		if (!test_bit(i, &idxd_pmu->supported_filters))
445			skip_filter(i);
446	}
447
448	idxd_pmu->pmu.name		= idxd_pmu->name;
449	idxd_pmu->pmu.attr_groups	= perfmon_attr_groups;
450	idxd_pmu->pmu.task_ctx_nr	= perf_invalid_context;
451	idxd_pmu->pmu.event_init	= perfmon_pmu_event_init;
452	idxd_pmu->pmu.pmu_enable	= perfmon_pmu_enable;
453	idxd_pmu->pmu.pmu_disable	= perfmon_pmu_disable;
454	idxd_pmu->pmu.add		= perfmon_pmu_event_add;
455	idxd_pmu->pmu.del		= perfmon_pmu_event_del;
456	idxd_pmu->pmu.start		= perfmon_pmu_event_start;
457	idxd_pmu->pmu.stop		= perfmon_pmu_event_stop;
458	idxd_pmu->pmu.read		= perfmon_pmu_event_update;
459	idxd_pmu->pmu.capabilities	= PERF_PMU_CAP_NO_EXCLUDE;
460	idxd_pmu->pmu.scope		= PERF_PMU_SCOPE_SYS_WIDE;
461	idxd_pmu->pmu.module		= THIS_MODULE;
462}
463
464void perfmon_pmu_remove(struct idxd_device *idxd)
465{
466	if (!idxd->idxd_pmu)
467		return;
468
469	perf_pmu_unregister(&idxd->idxd_pmu->pmu);
470	kfree(idxd->idxd_pmu);
471	idxd->idxd_pmu = NULL;
472}
473
474int perfmon_pmu_init(struct idxd_device *idxd)
475{
476	union idxd_perfcap perfcap;
477	struct idxd_pmu *idxd_pmu;
478	int rc = -ENODEV;
479
480	/*
481	 * If perfmon_offset or num_counters is 0, it means perfmon is
482	 * not supported on this hardware.
483	 */
484	if (idxd->perfmon_offset == 0)
485		return -ENODEV;
486
487	idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);
488	if (!idxd_pmu)
489		return -ENOMEM;
490
491	idxd_pmu->idxd = idxd;
492	idxd->idxd_pmu = idxd_pmu;
493
494	if (idxd->data->type == IDXD_TYPE_DSA) {
495		rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);
496		if (rc < 0)
497			goto free;
498	} else if (idxd->data->type == IDXD_TYPE_IAX) {
499		rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);
500		if (rc < 0)
501			goto free;
502	} else {
503		goto free;
504	}
505
506	perfmon_reset(idxd);
507
508	perfcap.bits = ioread64(PERFCAP_REG(idxd));
509
510	/*
511	 * If total perf counter is 0, stop further registration.
512	 * This is necessary in order to support driver running on
513	 * guest which does not have pmon support.
514	 */
515	if (perfcap.num_perf_counter == 0)
516		goto free;
517
518	/* A counter width of 0 means it can't count */
519	if (perfcap.counter_width == 0)
520		goto free;
521
522	/* Overflow interrupt and counter freeze support must be available */
523	if (!perfcap.overflow_interrupt || !perfcap.counter_freeze)
524		goto free;
525
526	/* Number of event categories cannot be 0 */
527	if (perfcap.num_event_category == 0)
528		goto free;
529
530	/*
531	 * We don't support per-counter capabilities for now.
532	 */
533	if (perfcap.cap_per_counter)
534		goto free;
535
536	idxd_pmu->n_event_categories = perfcap.num_event_category;
537	idxd_pmu->supported_event_categories = perfcap.global_event_category;
538	idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;
539
540	/* check filter capability.  If 0, then filters are not supported */
541	idxd_pmu->supported_filters = perfcap.filter;
542	if (perfcap.filter)
543		idxd_pmu->n_filters = hweight8(perfcap.filter);
544
545	/* Store the total number of counters categories, and counter width */
546	idxd_pmu->n_counters = perfcap.num_perf_counter;
547	idxd_pmu->counter_width = perfcap.counter_width;
548
549	idxd_pmu_init(idxd_pmu);
550
551	rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);
552	if (rc)
553		goto free;
554
555out:
556	return rc;
557free:
558	kfree(idxd_pmu);
559	idxd->idxd_pmu = NULL;
560
561	goto out;
562}