xe_pmu.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2025 Intel Corporation
  4 */
  5
  6#include <drm/drm_drv.h>
  7#include <linux/device.h>
  8
  9#include "xe_device.h"
 10#include "xe_force_wake.h"
 11#include "xe_gt_idle.h"
 12#include "xe_guc_engine_activity.h"
 13#include "xe_guc_pc.h"
 14#include "xe_hw_engine.h"
 15#include "xe_pm.h"
 16#include "xe_pmu.h"
 17#include "xe_sriov_pf_helpers.h"
 18
 19/**
 20 * DOC: Xe PMU (Performance Monitoring Unit)
 21 *
 22 * Expose events/counters like GT-C6 residency, GT frequency and per-class-engine
 23 * activity to user land via the perf interface. Events are per device.
 24 *
 25 * All events are listed in sysfs:
 26 *
 27 *     $ ls -ld /sys/bus/event_source/devices/xe_*
 28 *     $ ls /sys/bus/event_source/devices/xe_0000_00_02.0/events/
 29 *     $ ls /sys/bus/event_source/devices/xe_0000_00_02.0/format/
 30 *
 31 * The following format parameters are available to read events,
 32 * but only few are valid with each event:
 33 *
 34 *	gt[60:63]		Selects gt for the event
 35 *	engine_class[20:27]	Selects engine-class for event
 36 *	engine_instance[12:19]	Selects the engine-instance for the event
 37 *	function[44:59]		Selects the function of the event (SRIOV enabled)
 38 *
 39 * For engine specific events (engine-*), gt, engine_class and engine_instance parameters must be
 40 * set as populated by DRM_XE_DEVICE_QUERY_ENGINES and function if SRIOV is enabled.
 41 *
 42 * For gt specific events (gt-*) gt parameter must be passed. All other parameters will be 0.
 43 *
 44 * The standard perf tool can be used to grep for a certain event as well.
 45 * Example:
 46 *
 47 *     $ perf list | grep gt-c6
 48 *
 49 * To sample a specific event for a GT at regular intervals:
 50 *
 51 *     $ perf stat -e <event_name,gt=> -I <interval>
 52 */
 53
 54#define XE_PMU_EVENT_GT_MASK			GENMASK_ULL(63, 60)
 55#define XE_PMU_EVENT_FUNCTION_MASK		GENMASK_ULL(59, 44)
 56#define XE_PMU_EVENT_ENGINE_CLASS_MASK		GENMASK_ULL(27, 20)
 57#define XE_PMU_EVENT_ENGINE_INSTANCE_MASK	GENMASK_ULL(19, 12)
 58#define XE_PMU_EVENT_ID_MASK			GENMASK_ULL(11, 0)
 59
 60static unsigned int config_to_event_id(u64 config)
 61{
 62	return FIELD_GET(XE_PMU_EVENT_ID_MASK, config);
 63}
 64
 65static unsigned int config_to_function_id(u64 config)
 66{
 67	return FIELD_GET(XE_PMU_EVENT_FUNCTION_MASK, config);
 68}
 69
 70static unsigned int config_to_engine_class(u64 config)
 71{
 72	return FIELD_GET(XE_PMU_EVENT_ENGINE_CLASS_MASK, config);
 73}
 74
 75static unsigned int config_to_engine_instance(u64 config)
 76{
 77	return FIELD_GET(XE_PMU_EVENT_ENGINE_INSTANCE_MASK, config);
 78}
 79
 80static unsigned int config_to_gt_id(u64 config)
 81{
 82	return FIELD_GET(XE_PMU_EVENT_GT_MASK, config);
 83}
 84
 85#define XE_PMU_EVENT_GT_C6_RESIDENCY		0x01
 86#define XE_PMU_EVENT_ENGINE_ACTIVE_TICKS	0x02
 87#define XE_PMU_EVENT_ENGINE_TOTAL_TICKS		0x03
 88#define XE_PMU_EVENT_GT_ACTUAL_FREQUENCY	0x04
 89#define XE_PMU_EVENT_GT_REQUESTED_FREQUENCY	0x05
 90
 91static struct xe_gt *event_to_gt(struct perf_event *event)
 92{
 93	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
 94	u64 gt = config_to_gt_id(event->attr.config);
 95
 96	return xe_device_get_gt(xe, gt);
 97}
 98
 99static struct xe_hw_engine *event_to_hwe(struct perf_event *event)
100{
101	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
102	struct drm_xe_engine_class_instance eci;
103	u64 config = event->attr.config;
104	struct xe_hw_engine *hwe;
105
106	eci.engine_class = config_to_engine_class(config);
107	eci.engine_instance = config_to_engine_instance(config);
108	eci.gt_id = config_to_gt_id(config);
109
110	hwe = xe_hw_engine_lookup(xe, eci);
111	if (!hwe || xe_hw_engine_is_reserved(hwe))
112		return NULL;
113
114	return hwe;
115}
116
117static bool is_engine_event(u64 config)
118{
119	unsigned int event_id = config_to_event_id(config);
120
121	return (event_id == XE_PMU_EVENT_ENGINE_TOTAL_TICKS ||
122		event_id == XE_PMU_EVENT_ENGINE_ACTIVE_TICKS);
123}
124
125static bool is_gt_frequency_event(struct perf_event *event)
126{
127	u32 id = config_to_event_id(event->attr.config);
128
129	return id == XE_PMU_EVENT_GT_ACTUAL_FREQUENCY ||
130	       id == XE_PMU_EVENT_GT_REQUESTED_FREQUENCY;
131}
132
133static bool event_gt_forcewake(struct perf_event *event)
134{
135	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
136	u64 config = event->attr.config;
137	struct xe_gt *gt;
138	unsigned int *fw_ref;
139
140	if (!is_engine_event(config) && !is_gt_frequency_event(event))
141		return true;
142
143	gt = xe_device_get_gt(xe, config_to_gt_id(config));
144
145	fw_ref = kzalloc(sizeof(*fw_ref), GFP_KERNEL);
146	if (!fw_ref)
147		return false;
148
149	*fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
150	if (!*fw_ref) {
151		kfree(fw_ref);
152		return false;
153	}
154
155	event->pmu_private = fw_ref;
156
157	return true;
158}
159
160static bool event_supported(struct xe_pmu *pmu, unsigned int gt_id,
161			    unsigned int id)
162{
163	struct xe_device *xe = container_of(pmu, typeof(*xe), pmu);
164	struct xe_gt *gt = xe_device_get_gt(xe, gt_id);
165
166	if (!gt)
167		return false;
168
169	return id < sizeof(pmu->supported_events) * BITS_PER_BYTE &&
170		pmu->supported_events & BIT_ULL(id);
171}
172
173static bool event_param_valid(struct perf_event *event)
174{
175	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
176	unsigned int engine_class, engine_instance, function_id;
177	u64 config = event->attr.config;
178	struct xe_gt *gt;
179
180	gt = xe_device_get_gt(xe, config_to_gt_id(config));
181	if (!gt)
182		return false;
183
184	engine_class = config_to_engine_class(config);
185	engine_instance = config_to_engine_instance(config);
186	function_id = config_to_function_id(config);
187
188	switch (config_to_event_id(config)) {
189	case XE_PMU_EVENT_GT_C6_RESIDENCY:
190	case XE_PMU_EVENT_GT_ACTUAL_FREQUENCY:
191	case XE_PMU_EVENT_GT_REQUESTED_FREQUENCY:
192		if (engine_class || engine_instance || function_id)
193			return false;
194		break;
195	case XE_PMU_EVENT_ENGINE_ACTIVE_TICKS:
196	case XE_PMU_EVENT_ENGINE_TOTAL_TICKS:
197		if (!event_to_hwe(event))
198			return false;
199
200		/* PF(0) and total vfs when SRIOV is enabled */
201		if (IS_SRIOV_PF(xe)) {
202			if (function_id > xe_sriov_pf_get_totalvfs(xe))
203				return false;
204		} else if (function_id) {
205			return false;
206		}
207
208		break;
209	}
210
211	return true;
212}
213
214static void xe_pmu_event_destroy(struct perf_event *event)
215{
216	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
217	struct xe_gt *gt;
218	unsigned int *fw_ref = event->pmu_private;
219
220	if (fw_ref) {
221		gt = xe_device_get_gt(xe, config_to_gt_id(event->attr.config));
222		xe_force_wake_put(gt_to_fw(gt), *fw_ref);
223		kfree(fw_ref);
224		event->pmu_private = NULL;
225	}
226
227	drm_WARN_ON(&xe->drm, event->parent);
228	xe_pm_runtime_put(xe);
229	drm_dev_put(&xe->drm);
230}
231
232static int xe_pmu_event_init(struct perf_event *event)
233{
234	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
235	struct xe_pmu *pmu = &xe->pmu;
236	unsigned int id, gt;
237
238	if (!pmu->registered)
239		return -ENODEV;
240
241	if (event->attr.type != event->pmu->type)
242		return -ENOENT;
243
244	/* unsupported modes and filters */
245	if (event->attr.sample_period) /* no sampling */
246		return -EINVAL;
247
248	if (event->cpu < 0)
249		return -EINVAL;
250
251	gt = config_to_gt_id(event->attr.config);
252	id = config_to_event_id(event->attr.config);
253	if (!event_supported(pmu, gt, id))
254		return -ENOENT;
255
256	if (has_branch_stack(event))
257		return -EOPNOTSUPP;
258
259	if (!event_param_valid(event))
260		return -ENOENT;
261
262	if (!event->parent) {
263		drm_dev_get(&xe->drm);
264		xe_pm_runtime_get(xe);
265		if (!event_gt_forcewake(event)) {
266			xe_pm_runtime_put(xe);
267			drm_dev_put(&xe->drm);
268			return -EINVAL;
269		}
270		event->destroy = xe_pmu_event_destroy;
271	}
272
273	return 0;
274}
275
276static u64 read_engine_events(struct xe_gt *gt, struct perf_event *event)
277{
278	struct xe_hw_engine *hwe;
279	unsigned int function_id;
280	u64 config, val = 0;
281
282	config = event->attr.config;
283	function_id = config_to_function_id(config);
284
285	hwe = event_to_hwe(event);
286	if (config_to_event_id(config) == XE_PMU_EVENT_ENGINE_ACTIVE_TICKS)
287		val = xe_guc_engine_activity_active_ticks(&gt->uc.guc, hwe, function_id);
288	else
289		val = xe_guc_engine_activity_total_ticks(&gt->uc.guc, hwe, function_id);
290
291	return val;
292}
293
294static u64 __xe_pmu_event_read(struct perf_event *event)
295{
296	struct xe_gt *gt = event_to_gt(event);
297
298	if (!gt)
299		return 0;
300
301	switch (config_to_event_id(event->attr.config)) {
302	case XE_PMU_EVENT_GT_C6_RESIDENCY:
303		return xe_gt_idle_residency_msec(&gt->gtidle);
304	case XE_PMU_EVENT_ENGINE_ACTIVE_TICKS:
305	case XE_PMU_EVENT_ENGINE_TOTAL_TICKS:
306		return read_engine_events(gt, event);
307	case XE_PMU_EVENT_GT_ACTUAL_FREQUENCY:
308		return xe_guc_pc_get_act_freq(&gt->uc.guc.pc);
309	case XE_PMU_EVENT_GT_REQUESTED_FREQUENCY:
310		return xe_guc_pc_get_cur_freq_fw(&gt->uc.guc.pc);
311	}
312
313	return 0;
314}
315
316static void xe_pmu_event_update(struct perf_event *event)
317{
318	struct hw_perf_event *hwc = &event->hw;
319	u64 prev, new;
320
321	prev = local64_read(&hwc->prev_count);
322	do {
323		new = __xe_pmu_event_read(event);
324	} while (!local64_try_cmpxchg(&hwc->prev_count, &prev, new));
325
326	/*
327	 * GT frequency is not a monotonically increasing counter, so add the
328	 * instantaneous value instead.
329	 */
330	if (is_gt_frequency_event(event))
331		local64_add(new, &event->count);
332	else
333		local64_add(new - prev, &event->count);
334}
335
336static void xe_pmu_event_read(struct perf_event *event)
337{
338	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
339	struct xe_pmu *pmu = &xe->pmu;
340
341	if (!pmu->registered) {
342		event->hw.state = PERF_HES_STOPPED;
343		return;
344	}
345
346	xe_pmu_event_update(event);
347}
348
349static void xe_pmu_enable(struct perf_event *event)
350{
351	/*
352	 * Store the current counter value so we can report the correct delta
353	 * for all listeners. Even when the event was already enabled and has
354	 * an existing non-zero value.
355	 */
356	local64_set(&event->hw.prev_count, __xe_pmu_event_read(event));
357}
358
359static void xe_pmu_event_start(struct perf_event *event, int flags)
360{
361	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
362	struct xe_pmu *pmu = &xe->pmu;
363
364	if (!pmu->registered)
365		return;
366
367	xe_pmu_enable(event);
368	event->hw.state = 0;
369}
370
371static void xe_pmu_event_stop(struct perf_event *event, int flags)
372{
373	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
374	struct xe_pmu *pmu = &xe->pmu;
375
376	if (pmu->registered)
377		if (flags & PERF_EF_UPDATE)
378			xe_pmu_event_update(event);
379
380	event->hw.state = PERF_HES_STOPPED;
381}
382
383static int xe_pmu_event_add(struct perf_event *event, int flags)
384{
385	struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
386	struct xe_pmu *pmu = &xe->pmu;
387
388	if (!pmu->registered)
389		return -ENODEV;
390
391	if (flags & PERF_EF_START)
392		xe_pmu_event_start(event, flags);
393
394	return 0;
395}
396
397static void xe_pmu_event_del(struct perf_event *event, int flags)
398{
399	xe_pmu_event_stop(event, PERF_EF_UPDATE);
400}
401
402PMU_FORMAT_ATTR(gt,			"config:60-63");
403PMU_FORMAT_ATTR(function,		"config:44-59");
404PMU_FORMAT_ATTR(engine_class,		"config:20-27");
405PMU_FORMAT_ATTR(engine_instance,	"config:12-19");
406PMU_FORMAT_ATTR(event,			"config:0-11");
407
408static struct attribute *pmu_format_attrs[] = {
409	&format_attr_event.attr,
410	&format_attr_engine_class.attr,
411	&format_attr_engine_instance.attr,
412	&format_attr_function.attr,
413	&format_attr_gt.attr,
414	NULL,
415};
416
417static const struct attribute_group pmu_format_attr_group = {
418	.name = "format",
419	.attrs = pmu_format_attrs,
420};
421
422static ssize_t event_attr_show(struct device *dev,
423			       struct device_attribute *attr, char *buf)
424{
425	struct perf_pmu_events_attr *pmu_attr =
426		container_of(attr, struct perf_pmu_events_attr, attr);
427
428	return sprintf(buf, "event=%#04llx\n", pmu_attr->id);
429}
430
431#define XE_EVENT_ATTR(name_, v_, id_)					\
432	PMU_EVENT_ATTR(name_, pmu_event_ ## v_, id_, event_attr_show)
433
434#define XE_EVENT_ATTR_UNIT(name_, v_, unit_)				\
435	PMU_EVENT_ATTR_STRING(name_.unit, pmu_event_unit_ ## v_, unit_)
436
437#define XE_EVENT_ATTR_GROUP(v_, id_, ...)				\
438	static struct attribute *pmu_attr_ ##v_[] = {			\
439		__VA_ARGS__,						\
440		NULL							\
441	};								\
442	static umode_t is_visible_##v_(struct kobject *kobj,		\
443				       struct attribute *attr, int idx) \
444	{								\
445		struct perf_pmu_events_attr *pmu_attr;			\
446		struct xe_pmu *pmu;					\
447									\
448		pmu_attr = container_of(attr, typeof(*pmu_attr), attr.attr); \
449		pmu = container_of(dev_get_drvdata(kobj_to_dev(kobj)),	\
450				   typeof(*pmu), base);			\
451									\
452		return event_supported(pmu, 0, id_) ? attr->mode : 0;	\
453	}								\
454	static const struct attribute_group pmu_group_ ##v_ = {		\
455		.name = "events",					\
456		.attrs = pmu_attr_ ## v_,				\
457		.is_visible = is_visible_ ## v_,			\
458	}
459
460#define XE_EVENT_ATTR_SIMPLE(name_, v_, id_, unit_)			\
461	XE_EVENT_ATTR(name_, v_, id_)					\
462	XE_EVENT_ATTR_UNIT(name_, v_, unit_)				\
463	XE_EVENT_ATTR_GROUP(v_, id_, &pmu_event_ ##v_.attr.attr,	\
464			    &pmu_event_unit_ ##v_.attr.attr)
465
466#define XE_EVENT_ATTR_NOUNIT(name_, v_, id_)				\
467	XE_EVENT_ATTR(name_, v_, id_)					\
468	XE_EVENT_ATTR_GROUP(v_, id_, &pmu_event_ ##v_.attr.attr)
469
470XE_EVENT_ATTR_SIMPLE(gt-c6-residency, gt_c6_residency, XE_PMU_EVENT_GT_C6_RESIDENCY, "ms");
471XE_EVENT_ATTR_NOUNIT(engine-active-ticks, engine_active_ticks, XE_PMU_EVENT_ENGINE_ACTIVE_TICKS);
472XE_EVENT_ATTR_NOUNIT(engine-total-ticks, engine_total_ticks, XE_PMU_EVENT_ENGINE_TOTAL_TICKS);
473XE_EVENT_ATTR_SIMPLE(gt-actual-frequency, gt_actual_frequency,
474		     XE_PMU_EVENT_GT_ACTUAL_FREQUENCY, "MHz");
475XE_EVENT_ATTR_SIMPLE(gt-requested-frequency, gt_requested_frequency,
476		     XE_PMU_EVENT_GT_REQUESTED_FREQUENCY, "MHz");
477
478static struct attribute *pmu_empty_event_attrs[] = {
479	/* Empty - all events are added as groups with .attr_update() */
480	NULL,
481};
482
483static const struct attribute_group pmu_events_attr_group = {
484	.name = "events",
485	.attrs = pmu_empty_event_attrs,
486};
487
488static const struct attribute_group *pmu_events_attr_update[] = {
489	&pmu_group_gt_c6_residency,
490	&pmu_group_engine_active_ticks,
491	&pmu_group_engine_total_ticks,
492	&pmu_group_gt_actual_frequency,
493	&pmu_group_gt_requested_frequency,
494	NULL,
495};
496
497static void set_supported_events(struct xe_pmu *pmu)
498{
499	struct xe_device *xe = container_of(pmu, typeof(*xe), pmu);
500	struct xe_gt *gt;
501	int id;
502
503	/* If there are no GTs, don't support any GT-related events */
504	if (xe->info.gt_count == 0)
505		return;
506
507	if (!xe->info.skip_guc_pc) {
508		pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_GT_C6_RESIDENCY);
509		pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_GT_ACTUAL_FREQUENCY);
510		pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_GT_REQUESTED_FREQUENCY);
511	}
512
513	/* Find the first available GT to query engine event capabilities */
514	for_each_gt(gt, xe, id)
515		break;
516
517	if (xe_guc_engine_activity_supported(&gt->uc.guc)) {
518		pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_ENGINE_ACTIVE_TICKS);
519		pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_ENGINE_TOTAL_TICKS);
520	}
521}
522
523/**
524 * xe_pmu_unregister() - Remove/cleanup PMU registration
525 * @arg: Ptr to pmu
526 */
527static void xe_pmu_unregister(void *arg)
528{
529	struct xe_pmu *pmu = arg;
530	struct xe_device *xe = container_of(pmu, typeof(*xe), pmu);
531
532	if (!pmu->registered)
533		return;
534
535	pmu->registered = false;
536
537	perf_pmu_unregister(&pmu->base);
538	kfree(pmu->name);
539}
540
541/**
542 * xe_pmu_register() - Define basic PMU properties for Xe and add event callbacks.
543 * @pmu: the PMU object
544 *
545 * Returns 0 on success and an appropriate error code otherwise
546 */
547int xe_pmu_register(struct xe_pmu *pmu)
548{
549	struct xe_device *xe = container_of(pmu, typeof(*xe), pmu);
550	static const struct attribute_group *attr_groups[] = {
551		&pmu_format_attr_group,
552		&pmu_events_attr_group,
553		NULL
554	};
555	int ret = -ENOMEM;
556	char *name;
557
558	BUILD_BUG_ON(XE_MAX_GT_PER_TILE != XE_PMU_MAX_GT);
559
560	if (IS_SRIOV_VF(xe))
561		return 0;
562
563	name = kasprintf(GFP_KERNEL, "xe_%s",
564			 dev_name(xe->drm.dev));
565	if (!name)
566		goto err;
567
568	/* tools/perf reserves colons as special. */
569	strreplace(name, ':', '_');
570
571	pmu->name		= name;
572	pmu->base.attr_groups	= attr_groups;
573	pmu->base.attr_update	= pmu_events_attr_update;
574	pmu->base.scope		= PERF_PMU_SCOPE_SYS_WIDE;
575	pmu->base.module	= THIS_MODULE;
576	pmu->base.task_ctx_nr	= perf_invalid_context;
577	pmu->base.event_init	= xe_pmu_event_init;
578	pmu->base.add		= xe_pmu_event_add;
579	pmu->base.del		= xe_pmu_event_del;
580	pmu->base.start		= xe_pmu_event_start;
581	pmu->base.stop		= xe_pmu_event_stop;
582	pmu->base.read		= xe_pmu_event_read;
583
584	set_supported_events(pmu);
585
586	ret = perf_pmu_register(&pmu->base, pmu->name, -1);
587	if (ret)
588		goto err_name;
589
590	pmu->registered = true;
591
592	return devm_add_action_or_reset(xe->drm.dev, xe_pmu_unregister, pmu);
593
594err_name:
595	kfree(name);
596err:
597	drm_err(&xe->drm, "Failed to register PMU (ret=%d)!\n", ret);
598
599	return ret;
600}