evsel.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <errno.h>
  3#include <unistd.h>
  4#include <sys/syscall.h>
  5#include <perf/evsel.h>
  6#include <perf/cpumap.h>
  7#include <perf/threadmap.h>
  8#include <linux/hash.h>
  9#include <linux/list.h>
 10#include <internal/evsel.h>
 11#include <linux/zalloc.h>
 12#include <stdlib.h>
 13#include <internal/xyarray.h>
 14#include <internal/cpumap.h>
 15#include <internal/mmap.h>
 16#include <internal/threadmap.h>
 17#include <internal/lib.h>
 18#include <linux/string.h>
 19#include <sys/ioctl.h>
 20#include <sys/mman.h>
 21#include <asm/bug.h>
 22
 23void perf_evsel__init(struct perf_evsel *evsel, struct perf_event_attr *attr,
 24		      int idx)
 25{
 26	INIT_LIST_HEAD(&evsel->node);
 27	INIT_LIST_HEAD(&evsel->per_stream_periods);
 28	evsel->attr = *attr;
 29	evsel->idx  = idx;
 30	evsel->leader = evsel;
 31}
 32
 33struct perf_evsel *perf_evsel__new(struct perf_event_attr *attr)
 34{
 35	struct perf_evsel *evsel = zalloc(sizeof(*evsel));
 36
 37	if (evsel != NULL)
 38		perf_evsel__init(evsel, attr, 0);
 39
 40	return evsel;
 41}
 42
 43void perf_evsel__exit(struct perf_evsel *evsel)
 44{
 45	assert(evsel->fd == NULL);  /* If not fds were not closed. */
 46	assert(evsel->mmap == NULL); /* If not munmap wasn't called. */
 47	assert(evsel->sample_id == NULL); /* If not free_id wasn't called. */
 48	perf_cpu_map__put(evsel->cpus);
 49	perf_cpu_map__put(evsel->pmu_cpus);
 50	perf_thread_map__put(evsel->threads);
 51}
 52
 53void perf_evsel__delete(struct perf_evsel *evsel)
 54{
 55	perf_evsel__exit(evsel);
 56	free(evsel);
 57}
 58
 59#define FD(_evsel, _cpu_map_idx, _thread)				\
 60	((int *)xyarray__entry(_evsel->fd, _cpu_map_idx, _thread))
 61#define MMAP(_evsel, _cpu_map_idx, _thread)				\
 62	(_evsel->mmap ? ((struct perf_mmap *) xyarray__entry(_evsel->mmap, _cpu_map_idx, _thread)) \
 63		      : NULL)
 64
 65int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
 66{
 67	evsel->fd = xyarray__new(ncpus, nthreads, sizeof(int));
 68
 69	if (evsel->fd) {
 70		int idx, thread;
 71
 72		for (idx = 0; idx < ncpus; idx++) {
 73			for (thread = 0; thread < nthreads; thread++) {
 74				int *fd = FD(evsel, idx, thread);
 75
 76				if (fd)
 77					*fd = -1;
 78			}
 79		}
 80	}
 81
 82	return evsel->fd != NULL ? 0 : -ENOMEM;
 83}
 84
 85static int perf_evsel__alloc_mmap(struct perf_evsel *evsel, int ncpus, int nthreads)
 86{
 87	evsel->mmap = xyarray__new(ncpus, nthreads, sizeof(struct perf_mmap));
 88
 89	return evsel->mmap != NULL ? 0 : -ENOMEM;
 90}
 91
 92static int
 93sys_perf_event_open(struct perf_event_attr *attr,
 94		    pid_t pid, struct perf_cpu cpu, int group_fd,
 95		    unsigned long flags)
 96{
 97	return syscall(__NR_perf_event_open, attr, pid, cpu.cpu, group_fd, flags);
 98}
 99
100static int get_group_fd(struct perf_evsel *evsel, int cpu_map_idx, int thread, int *group_fd)
101{
102	struct perf_evsel *leader = evsel->leader;
103	int *fd;
104
105	if (evsel == leader) {
106		*group_fd = -1;
107		return 0;
108	}
109
110	/*
111	 * Leader must be already processed/open,
112	 * if not it's a bug.
113	 */
114	if (!leader->fd)
115		return -ENOTCONN;
116
117	fd = FD(leader, cpu_map_idx, thread);
118	if (fd == NULL || *fd == -1)
119		return -EBADF;
120
121	*group_fd = *fd;
122
123	return 0;
124}
125
126int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus,
127		     struct perf_thread_map *threads)
128{
129	struct perf_cpu cpu;
130	int idx, thread, err = 0;
131
132	if (cpus == NULL) {
133		static struct perf_cpu_map *empty_cpu_map;
134
135		if (empty_cpu_map == NULL) {
136			empty_cpu_map = perf_cpu_map__new_any_cpu();
137			if (empty_cpu_map == NULL)
138				return -ENOMEM;
139		}
140
141		cpus = empty_cpu_map;
142	}
143
144	if (threads == NULL) {
145		static struct perf_thread_map *empty_thread_map;
146
147		if (empty_thread_map == NULL) {
148			empty_thread_map = perf_thread_map__new_dummy();
149			if (empty_thread_map == NULL)
150				return -ENOMEM;
151		}
152
153		threads = empty_thread_map;
154	}
155
156	if (evsel->fd == NULL &&
157	    perf_evsel__alloc_fd(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0)
158		return -ENOMEM;
159
160	perf_cpu_map__for_each_cpu(cpu, idx, cpus) {
161		for (thread = 0; thread < threads->nr; thread++) {
162			int fd, group_fd, *evsel_fd;
163
164			evsel_fd = FD(evsel, idx, thread);
165			if (evsel_fd == NULL) {
166				err = -EINVAL;
167				goto out;
168			}
169
170			err = get_group_fd(evsel, idx, thread, &group_fd);
171			if (err < 0)
172				goto out;
173
174			fd = sys_perf_event_open(&evsel->attr,
175						 threads->map[thread].pid,
176						 cpu, group_fd, 0);
177
178			if (fd < 0) {
179				err = -errno;
180				goto out;
181			}
182
183			*evsel_fd = fd;
184		}
185	}
186out:
187	if (err)
188		perf_evsel__close(evsel);
189
190	return err;
191}
192
193static void perf_evsel__close_fd_cpu(struct perf_evsel *evsel, int cpu_map_idx)
194{
195	int thread;
196
197	for (thread = 0; thread < xyarray__max_y(evsel->fd); ++thread) {
198		int *fd = FD(evsel, cpu_map_idx, thread);
199
200		if (fd && *fd >= 0) {
201			close(*fd);
202			*fd = -1;
203		}
204	}
205}
206
207void perf_evsel__close_fd(struct perf_evsel *evsel)
208{
209	for (int idx = 0; idx < xyarray__max_x(evsel->fd); idx++)
210		perf_evsel__close_fd_cpu(evsel, idx);
211}
212
213void perf_evsel__free_fd(struct perf_evsel *evsel)
214{
215	xyarray__delete(evsel->fd);
216	evsel->fd = NULL;
217}
218
219void perf_evsel__close(struct perf_evsel *evsel)
220{
221	if (evsel->fd == NULL)
222		return;
223
224	perf_evsel__close_fd(evsel);
225	perf_evsel__free_fd(evsel);
226}
227
228void perf_evsel__close_cpu(struct perf_evsel *evsel, int cpu_map_idx)
229{
230	if (evsel->fd == NULL)
231		return;
232
233	perf_evsel__close_fd_cpu(evsel, cpu_map_idx);
234}
235
236void perf_evsel__munmap(struct perf_evsel *evsel)
237{
238	int idx, thread;
239
240	if (evsel->fd == NULL || evsel->mmap == NULL)
241		return;
242
243	for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
244		for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
245			int *fd = FD(evsel, idx, thread);
246
247			if (fd == NULL || *fd < 0)
248				continue;
249
250			perf_mmap__munmap(MMAP(evsel, idx, thread));
251		}
252	}
253
254	xyarray__delete(evsel->mmap);
255	evsel->mmap = NULL;
256}
257
258int perf_evsel__mmap(struct perf_evsel *evsel, int pages)
259{
260	int ret, idx, thread;
261	struct perf_mmap_param mp = {
262		.prot = PROT_READ | PROT_WRITE,
263		.mask = (pages * page_size) - 1,
264	};
265
266	if (evsel->fd == NULL || evsel->mmap)
267		return -EINVAL;
268
269	if (perf_evsel__alloc_mmap(evsel, xyarray__max_x(evsel->fd), xyarray__max_y(evsel->fd)) < 0)
270		return -ENOMEM;
271
272	for (idx = 0; idx < xyarray__max_x(evsel->fd); idx++) {
273		for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
274			int *fd = FD(evsel, idx, thread);
275			struct perf_mmap *map;
276			struct perf_cpu cpu = perf_cpu_map__cpu(evsel->cpus, idx);
277
278			if (fd == NULL || *fd < 0)
279				continue;
280
281			map = MMAP(evsel, idx, thread);
282			perf_mmap__init(map, NULL, false, NULL);
283
284			ret = perf_mmap__mmap(map, &mp, *fd, cpu);
285			if (ret) {
286				perf_evsel__munmap(evsel);
287				return ret;
288			}
289		}
290	}
291
292	return 0;
293}
294
295void *perf_evsel__mmap_base(struct perf_evsel *evsel, int cpu_map_idx, int thread)
296{
297	int *fd = FD(evsel, cpu_map_idx, thread);
298
299	if (fd == NULL || *fd < 0 || MMAP(evsel, cpu_map_idx, thread) == NULL)
300		return NULL;
301
302	return MMAP(evsel, cpu_map_idx, thread)->base;
303}
304
305int perf_evsel__read_size(struct perf_evsel *evsel)
306{
307	u64 read_format = evsel->attr.read_format;
308	int entry = sizeof(u64); /* value */
309	int size = 0;
310	int nr = 1;
311
312	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
313		size += sizeof(u64);
314
315	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
316		size += sizeof(u64);
317
318	if (read_format & PERF_FORMAT_ID)
319		entry += sizeof(u64);
320
321	if (read_format & PERF_FORMAT_LOST)
322		entry += sizeof(u64);
323
324	if (read_format & PERF_FORMAT_GROUP) {
325		nr = evsel->nr_members;
326		size += sizeof(u64);
327	}
328
329	size += entry * nr;
330	return size;
331}
332
333/* This only reads values for the leader */
334static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
335				  int thread, struct perf_counts_values *count)
336{
337	size_t size = perf_evsel__read_size(evsel);
338	int *fd = FD(evsel, cpu_map_idx, thread);
339	u64 read_format = evsel->attr.read_format;
340	u64 *data;
341	int idx = 1;
342
343	if (fd == NULL || *fd < 0)
344		return -EINVAL;
345
346	data = calloc(1, size);
347	if (data == NULL)
348		return -ENOMEM;
349
350	if (readn(*fd, data, size) <= 0) {
351		free(data);
352		return -errno;
353	}
354
355	/*
356	 * This reads only the leader event intentionally since we don't have
357	 * perf counts values for sibling events.
358	 */
359	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
360		count->ena = data[idx++];
361	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
362		count->run = data[idx++];
363
364	/* value is always available */
365	count->val = data[idx++];
366	if (read_format & PERF_FORMAT_ID)
367		count->id = data[idx++];
368	if (read_format & PERF_FORMAT_LOST)
369		count->lost = data[idx++];
370
371	free(data);
372	return 0;
373}
374
375/*
376 * The perf read format is very flexible.  It needs to set the proper
377 * values according to the read format.
378 */
379static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf,
380				      struct perf_counts_values *count)
381{
382	u64 read_format = evsel->attr.read_format;
383	int n = 0;
384
385	count->val = buf[n++];
386
387	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
388		count->ena = buf[n++];
389
390	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
391		count->run = buf[n++];
392
393	if (read_format & PERF_FORMAT_ID)
394		count->id = buf[n++];
395
396	if (read_format & PERF_FORMAT_LOST)
397		count->lost = buf[n++];
398}
399
400int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread,
401		     struct perf_counts_values *count)
402{
403	size_t size = perf_evsel__read_size(evsel);
404	int *fd = FD(evsel, cpu_map_idx, thread);
405	u64 read_format = evsel->attr.read_format;
406	struct perf_counts_values buf;
407
408	memset(count, 0, sizeof(*count));
409
410	if (fd == NULL || *fd < 0)
411		return -EINVAL;
412
413	if (read_format & PERF_FORMAT_GROUP)
414		return perf_evsel__read_group(evsel, cpu_map_idx, thread, count);
415
416	if (MMAP(evsel, cpu_map_idx, thread) &&
417	    !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) &&
418	    !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count))
419		return 0;
420
421	if (readn(*fd, buf.values, size) <= 0)
422		return -errno;
423
424	perf_evsel__adjust_values(evsel, buf.values, count);
425	return 0;
426}
427
428static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg,
429			     int cpu_map_idx, int thread)
430{
431	int *fd = FD(evsel, cpu_map_idx, thread);
432
433	if (fd == NULL || *fd < 0)
434		return -1;
435
436	return ioctl(*fd, ioc, arg);
437}
438
439static int perf_evsel__run_ioctl(struct perf_evsel *evsel,
440				 int ioc,  void *arg,
441				 int cpu_map_idx)
442{
443	int thread;
444
445	for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) {
446		int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread);
447
448		if (err)
449			return err;
450	}
451
452	return 0;
453}
454
455int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
456{
457	return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx);
458}
459
460int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread)
461{
462	struct perf_cpu cpu __maybe_unused;
463	int idx;
464	int err;
465
466	perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) {
467		err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread);
468		if (err)
469			return err;
470	}
471
472	return 0;
473}
474
475int perf_evsel__enable(struct perf_evsel *evsel)
476{
477	int i;
478	int err = 0;
479
480	for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
481		err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, i);
482	return err;
483}
484
485int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx)
486{
487	return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, cpu_map_idx);
488}
489
490int perf_evsel__disable(struct perf_evsel *evsel)
491{
492	int i;
493	int err = 0;
494
495	for (i = 0; i < xyarray__max_x(evsel->fd) && !err; i++)
496		err = perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_DISABLE, NULL, i);
497	return err;
498}
499
500int perf_evsel__apply_filter(struct perf_evsel *evsel, const char *filter)
501{
502	int err = 0, i;
503
504	for (i = 0; i < perf_cpu_map__nr(evsel->cpus) && !err; i++)
505		err = perf_evsel__run_ioctl(evsel,
506				     PERF_EVENT_IOC_SET_FILTER,
507				     (void *)filter, i);
508	return err;
509}
510
511struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel)
512{
513	return evsel->cpus;
514}
515
516struct perf_thread_map *perf_evsel__threads(struct perf_evsel *evsel)
517{
518	return evsel->threads;
519}
520
521struct perf_event_attr *perf_evsel__attr(struct perf_evsel *evsel)
522{
523	return &evsel->attr;
524}
525
526int perf_evsel__alloc_id(struct perf_evsel *evsel, int ncpus, int nthreads)
527{
528	if (ncpus == 0 || nthreads == 0)
529		return 0;
530
531	evsel->sample_id = xyarray__new(ncpus, nthreads, sizeof(struct perf_sample_id));
532	if (evsel->sample_id == NULL)
533		return -ENOMEM;
534
535	evsel->id = zalloc(ncpus * nthreads * sizeof(u64));
536	if (evsel->id == NULL) {
537		xyarray__delete(evsel->sample_id);
538		evsel->sample_id = NULL;
539		return -ENOMEM;
540	}
541
542	return 0;
543}
544
545void perf_evsel__free_id(struct perf_evsel *evsel)
546{
547	struct perf_sample_id_period *pos, *n;
548
549	xyarray__delete(evsel->sample_id);
550	evsel->sample_id = NULL;
551	zfree(&evsel->id);
552	evsel->ids = 0;
553
554	perf_evsel_for_each_per_thread_period_safe(evsel, n, pos) {
555		list_del_init(&pos->node);
556		free(pos);
557	}
558}
559
560bool perf_evsel__attr_has_per_thread_sample_period(struct perf_evsel *evsel)
561{
562	return (evsel->attr.sample_type & PERF_SAMPLE_READ) &&
563		(evsel->attr.sample_type & PERF_SAMPLE_TID) &&
564		evsel->attr.inherit;
565}
566
567u64 *perf_sample_id__get_period_storage(struct perf_sample_id *sid, u32 tid, bool per_thread)
568{
569	struct hlist_head *head;
570	struct perf_sample_id_period *res;
571	int hash;
572
573	if (!per_thread)
574		return &sid->period;
575
576	hash = hash_32(tid, PERF_SAMPLE_ID__HLIST_BITS);
577	head = &sid->periods[hash];
578
579	hlist_for_each_entry(res, head, hnode)
580		if (res->tid == tid)
581			return &res->period;
582
583	if (sid->evsel == NULL)
584		return NULL;
585
586	res = zalloc(sizeof(struct perf_sample_id_period));
587	if (res == NULL)
588		return NULL;
589
590	INIT_LIST_HEAD(&res->node);
591	res->tid = tid;
592
593	list_add_tail(&res->node, &sid->evsel->per_stream_periods);
594	hlist_add_head(&res->hnode, &sid->periods[hash]);
595
596	return &res->period;
597}
598
599void perf_counts_values__scale(struct perf_counts_values *count,
600			       bool scale, __s8 *pscaled)
601{
602	s8 scaled = 0;
603
604	if (scale) {
605		if (count->run == 0) {
606			scaled = -1;
607			count->val = 0;
608		} else if (count->run < count->ena) {
609			scaled = 1;
610			count->val = (u64)((double)count->val * count->ena / count->run);
611		}
612	}
613
614	if (pscaled)
615		*pscaled = scaled;
616}