bpf-event.c

  1// SPDX-License-Identifier: GPL-2.0
  2#include <errno.h>
  3#include <stddef.h>
  4#include <stdint.h>
  5#include <stdio.h>
  6#include <stdlib.h>
  7#include <string.h>
  8#include <bpf/bpf.h>
  9#include <bpf/btf.h>
 10#include <bpf/libbpf.h>
 11#include <linux/bpf.h>
 12#include <linux/btf.h>
 13#include <linux/err.h>
 14#include <linux/perf_event.h>
 15#include <linux/string.h>
 16#include <linux/zalloc.h>
 17#include <internal/lib.h>
 18#include <perf/event.h>
 19#include <symbol/kallsyms.h>
 20#include "bpf-event.h"
 21#include "bpf-utils.h"
 22#include "debug.h"
 23#include "dso.h"
 24#include "symbol.h"
 25#include "machine.h"
 26#include "env.h"
 27#include "session.h"
 28#include "map.h"
 29#include "evlist.h"
 30#include "record.h"
 31#include "util/synthetic-events.h"
 32
 33static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len)
 34{
 35	int ret = 0;
 36	size_t i;
 37
 38	for (i = 0; i < len; i++)
 39		ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
 40	return ret;
 41}
 42
 43static int machine__process_bpf_event_load(struct machine *machine,
 44					   union perf_event *event,
 45					   struct perf_sample *sample __maybe_unused)
 46{
 47	struct bpf_prog_info_node *info_node;
 48	struct perf_env *env = machine->env;
 49	struct perf_bpil *info_linear;
 50	int id = event->bpf.id;
 51	unsigned int i;
 52
 53	/* perf-record, no need to handle bpf-event */
 54	if (env == NULL)
 55		return 0;
 56
 57	info_node = perf_env__find_bpf_prog_info(env, id);
 58	if (!info_node)
 59		return 0;
 60	info_linear = info_node->info_linear;
 61
 62	for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) {
 63		u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms);
 64		u64 addr = addrs[i];
 65		struct map *map = maps__find(machine__kernel_maps(machine), addr);
 66
 67		if (map) {
 68			struct dso *dso = map__dso(map);
 69
 70			dso__set_binary_type(dso, DSO_BINARY_TYPE__BPF_PROG_INFO);
 71			dso__bpf_prog(dso)->id = id;
 72			dso__bpf_prog(dso)->sub_id = i;
 73			dso__bpf_prog(dso)->env = env;
 74			map__put(map);
 75		}
 76	}
 77	return 0;
 78}
 79
 80int machine__process_bpf(struct machine *machine, union perf_event *event,
 81			 struct perf_sample *sample)
 82{
 83	if (dump_trace)
 84		perf_event__fprintf_bpf(event, stdout);
 85
 86	switch (event->bpf.type) {
 87	case PERF_BPF_EVENT_PROG_LOAD:
 88		return machine__process_bpf_event_load(machine, event, sample);
 89
 90	case PERF_BPF_EVENT_PROG_UNLOAD:
 91		/*
 92		 * Do not free bpf_prog_info and btf of the program here,
 93		 * as annotation still need them. They will be freed at
 94		 * the end of the session.
 95		 */
 96		break;
 97	default:
 98		pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
 99		break;
100	}
101	return 0;
102}
103
104static int perf_env__fetch_btf(struct perf_env *env,
105			       u32 btf_id,
106			       struct btf *btf)
107{
108	struct btf_node *node;
109	u32 data_size;
110	const void *data;
111
112	data = btf__raw_data(btf, &data_size);
113
114	node = malloc(data_size + sizeof(struct btf_node));
115	if (!node)
116		return -1;
117
118	node->id = btf_id;
119	node->data_size = data_size;
120	memcpy(node->data, data, data_size);
121
122	if (!perf_env__insert_btf(env, node)) {
123		/* Insertion failed because of a duplicate. */
124		free(node);
125		return -1;
126	}
127	return 0;
128}
129
130static int synthesize_bpf_prog_name(char *buf, int size,
131				    struct bpf_prog_info *info,
132				    struct btf *btf,
133				    u32 sub_id)
134{
135	u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags);
136	void *func_infos = (void *)(uintptr_t)(info->func_info);
137	u32 sub_prog_cnt = info->nr_jited_ksyms;
138	const struct bpf_func_info *finfo;
139	const char *short_name = NULL;
140	const struct btf_type *t;
141	int name_len;
142
143	name_len = snprintf(buf, size, "bpf_prog_");
144	name_len += snprintf_hex(buf + name_len, size - name_len,
145				 prog_tags[sub_id], BPF_TAG_SIZE);
146	if (btf) {
147		finfo = func_infos + sub_id * info->func_info_rec_size;
148		t = btf__type_by_id(btf, finfo->type_id);
149		short_name = btf__name_by_offset(btf, t->name_off);
150	} else if (sub_id == 0 && sub_prog_cnt == 1) {
151		/* no subprog */
152		if (info->name[0])
153			short_name = info->name;
154	} else
155		short_name = "F";
156	if (short_name)
157		name_len += snprintf(buf + name_len, size - name_len,
158				     "_%s", short_name);
159	return name_len;
160}
161
162#ifdef HAVE_LIBBPF_STRINGS_SUPPORT
163
164#define BPF_METADATA_PREFIX "bpf_metadata_"
165#define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1)
166
167static bool name_has_bpf_metadata_prefix(const char **s)
168{
169	if (strncmp(*s, BPF_METADATA_PREFIX, BPF_METADATA_PREFIX_LEN) != 0)
170		return false;
171	*s += BPF_METADATA_PREFIX_LEN;
172	return true;
173}
174
175struct bpf_metadata_map {
176	struct btf *btf;
177	const struct btf_type *datasec;
178	void *rodata;
179	size_t rodata_size;
180	unsigned int num_vars;
181};
182
183static int bpf_metadata_read_map_data(__u32 map_id, struct bpf_metadata_map *map)
184{
185	int map_fd;
186	struct bpf_map_info map_info;
187	__u32 map_info_len;
188	int key;
189	struct btf *btf;
190	const struct btf_type *datasec;
191	struct btf_var_secinfo *vsi;
192	unsigned int vlen, vars;
193	void *rodata;
194
195	map_fd = bpf_map_get_fd_by_id(map_id);
196	if (map_fd < 0)
197		return -1;
198
199	memset(&map_info, 0, sizeof(map_info));
200	map_info_len = sizeof(map_info);
201	if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len) < 0)
202		goto out_close;
203
204	/* If it's not an .rodata map, don't bother. */
205	if (map_info.type != BPF_MAP_TYPE_ARRAY ||
206	    map_info.key_size != sizeof(int) ||
207	    map_info.max_entries != 1 ||
208	    !map_info.btf_value_type_id ||
209	    !strstr(map_info.name, ".rodata")) {
210		goto out_close;
211	}
212
213	btf = btf__load_from_kernel_by_id(map_info.btf_id);
214	if (!btf)
215		goto out_close;
216	datasec = btf__type_by_id(btf, map_info.btf_value_type_id);
217	if (!btf_is_datasec(datasec))
218		goto out_free_btf;
219
220	/*
221	 * If there aren't any variables with the "bpf_metadata_" prefix,
222	 * don't bother.
223	 */
224	vlen = btf_vlen(datasec);
225	vsi = btf_var_secinfos(datasec);
226	vars = 0;
227	for (unsigned int i = 0; i < vlen; i++, vsi++) {
228		const struct btf_type *t_var = btf__type_by_id(btf, vsi->type);
229		const char *name = btf__name_by_offset(btf, t_var->name_off);
230
231		if (name_has_bpf_metadata_prefix(&name))
232			vars++;
233	}
234	if (vars == 0)
235		goto out_free_btf;
236
237	rodata = zalloc(map_info.value_size);
238	if (!rodata)
239		goto out_free_btf;
240	key = 0;
241	if (bpf_map_lookup_elem(map_fd, &key, rodata)) {
242		free(rodata);
243		goto out_free_btf;
244	}
245	close(map_fd);
246
247	map->btf = btf;
248	map->datasec = datasec;
249	map->rodata = rodata;
250	map->rodata_size = map_info.value_size;
251	map->num_vars = vars;
252	return 0;
253
254out_free_btf:
255	btf__free(btf);
256out_close:
257	close(map_fd);
258	return -1;
259}
260
261struct format_btf_ctx {
262	char *buf;
263	size_t buf_size;
264	size_t buf_idx;
265};
266
267static void format_btf_cb(void *arg, const char *fmt, va_list ap)
268{
269	int n;
270	struct format_btf_ctx *ctx = (struct format_btf_ctx *)arg;
271
272	n = vsnprintf(ctx->buf + ctx->buf_idx, ctx->buf_size - ctx->buf_idx,
273		      fmt, ap);
274	ctx->buf_idx += n;
275	if (ctx->buf_idx >= ctx->buf_size)
276		ctx->buf_idx = ctx->buf_size;
277}
278
279static void format_btf_variable(struct btf *btf, char *buf, size_t buf_size,
280				const struct btf_type *t, const void *btf_data)
281{
282	struct format_btf_ctx ctx = {
283		.buf = buf,
284		.buf_idx = 0,
285		.buf_size = buf_size,
286	};
287	const struct btf_dump_type_data_opts opts = {
288		.sz = sizeof(struct btf_dump_type_data_opts),
289		.skip_names = 1,
290		.compact = 1,
291		.emit_strings = 1,
292	};
293	struct btf_dump *d;
294	size_t btf_size;
295
296	d = btf_dump__new(btf, format_btf_cb, &ctx, NULL);
297	btf_size = btf__resolve_size(btf, t->type);
298	btf_dump__dump_type_data(d, t->type, btf_data, btf_size, &opts);
299	btf_dump__free(d);
300}
301
302static void bpf_metadata_fill_event(struct bpf_metadata_map *map,
303				    struct perf_record_bpf_metadata *bpf_metadata_event)
304{
305	struct btf_var_secinfo *vsi;
306	unsigned int i, vlen;
307
308	memset(bpf_metadata_event->prog_name, 0, BPF_PROG_NAME_LEN);
309	vlen = btf_vlen(map->datasec);
310	vsi = btf_var_secinfos(map->datasec);
311
312	for (i = 0; i < vlen; i++, vsi++) {
313		const struct btf_type *t_var = btf__type_by_id(map->btf,
314							       vsi->type);
315		const char *name = btf__name_by_offset(map->btf,
316						       t_var->name_off);
317		const __u64 nr_entries = bpf_metadata_event->nr_entries;
318		struct perf_record_bpf_metadata_entry *entry;
319
320		if (!name_has_bpf_metadata_prefix(&name))
321			continue;
322
323		if (nr_entries >= (__u64)map->num_vars)
324			break;
325
326		entry = &bpf_metadata_event->entries[nr_entries];
327		memset(entry, 0, sizeof(*entry));
328		snprintf(entry->key, BPF_METADATA_KEY_LEN, "%s", name);
329		format_btf_variable(map->btf, entry->value,
330				    BPF_METADATA_VALUE_LEN, t_var,
331				    map->rodata + vsi->offset);
332		bpf_metadata_event->nr_entries++;
333	}
334}
335
336static void bpf_metadata_free_map_data(struct bpf_metadata_map *map)
337{
338	btf__free(map->btf);
339	free(map->rodata);
340}
341
342static struct bpf_metadata *bpf_metadata_alloc(__u32 nr_prog_tags,
343					       __u32 nr_variables)
344{
345	struct bpf_metadata *metadata;
346	size_t event_size;
347
348	metadata = zalloc(sizeof(struct bpf_metadata));
349	if (!metadata)
350		return NULL;
351
352	metadata->prog_names = zalloc(nr_prog_tags * sizeof(char *));
353	if (!metadata->prog_names) {
354		bpf_metadata_free(metadata);
355		return NULL;
356	}
357	for (__u32 prog_index = 0; prog_index < nr_prog_tags; prog_index++) {
358		metadata->prog_names[prog_index] = zalloc(BPF_PROG_NAME_LEN);
359		if (!metadata->prog_names[prog_index]) {
360			bpf_metadata_free(metadata);
361			return NULL;
362		}
363		metadata->nr_prog_names++;
364	}
365
366	event_size = sizeof(metadata->event->bpf_metadata) +
367	    nr_variables * sizeof(metadata->event->bpf_metadata.entries[0]);
368	metadata->event = zalloc(event_size);
369	if (!metadata->event) {
370		bpf_metadata_free(metadata);
371		return NULL;
372	}
373	metadata->event->bpf_metadata = (struct perf_record_bpf_metadata) {
374		.header = {
375			.type = PERF_RECORD_BPF_METADATA,
376			.size = event_size,
377		},
378		.nr_entries = 0,
379	};
380
381	return metadata;
382}
383
384static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info)
385{
386	struct bpf_metadata *metadata;
387	const __u32 *map_ids = (__u32 *)(uintptr_t)info->map_ids;
388
389	for (__u32 map_index = 0; map_index < info->nr_map_ids; map_index++) {
390		struct bpf_metadata_map map;
391
392		if (bpf_metadata_read_map_data(map_ids[map_index], &map) != 0)
393			continue;
394
395		metadata = bpf_metadata_alloc(info->nr_prog_tags, map.num_vars);
396		if (!metadata)
397			continue;
398
399		bpf_metadata_fill_event(&map, &metadata->event->bpf_metadata);
400
401		for (__u32 index = 0; index < info->nr_prog_tags; index++) {
402			synthesize_bpf_prog_name(metadata->prog_names[index],
403						 BPF_PROG_NAME_LEN, info,
404						 map.btf, index);
405		}
406
407		bpf_metadata_free_map_data(&map);
408
409		return metadata;
410	}
411
412	return NULL;
413}
414
415static int synthesize_perf_record_bpf_metadata(const struct bpf_metadata *metadata,
416					       const struct perf_tool *tool,
417					       perf_event__handler_t process,
418					       struct machine *machine)
419{
420	const size_t event_size = metadata->event->header.size;
421	union perf_event *event;
422	int err = 0;
423
424	event = zalloc(event_size + machine->id_hdr_size);
425	if (!event)
426		return -1;
427	memcpy(event, metadata->event, event_size);
428	memset((void *)event + event->header.size, 0, machine->id_hdr_size);
429	event->header.size += machine->id_hdr_size;
430	for (__u32 index = 0; index < metadata->nr_prog_names; index++) {
431		memcpy(event->bpf_metadata.prog_name,
432		       metadata->prog_names[index], BPF_PROG_NAME_LEN);
433		err = perf_tool__process_synth_event(tool, event, machine,
434						     process);
435		if (err != 0)
436			break;
437	}
438
439	free(event);
440	return err;
441}
442
443void bpf_metadata_free(struct bpf_metadata *metadata)
444{
445	if (metadata == NULL)
446		return;
447	for (__u32 index = 0; index < metadata->nr_prog_names; index++)
448		free(metadata->prog_names[index]);
449	free(metadata->prog_names);
450	free(metadata->event);
451	free(metadata);
452}
453
454#else /* HAVE_LIBBPF_STRINGS_SUPPORT */
455
456static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info __maybe_unused)
457{
458	return NULL;
459}
460
461static int synthesize_perf_record_bpf_metadata(const struct bpf_metadata *metadata __maybe_unused,
462					       const struct perf_tool *tool __maybe_unused,
463					       perf_event__handler_t process __maybe_unused,
464					       struct machine *machine __maybe_unused)
465{
466	return 0;
467}
468
469void bpf_metadata_free(struct bpf_metadata *metadata __maybe_unused)
470{
471}
472
473#endif /* HAVE_LIBBPF_STRINGS_SUPPORT */
474
475struct bpf_metadata_final_ctx {
476	const struct perf_tool *tool;
477	perf_event__handler_t process;
478	struct machine *machine;
479};
480
481static void synthesize_final_bpf_metadata_cb(struct bpf_prog_info_node *node,
482					     void *data)
483{
484	struct bpf_metadata_final_ctx *ctx = (struct bpf_metadata_final_ctx *)data;
485	struct bpf_metadata *metadata = node->metadata;
486	int err;
487
488	if (metadata == NULL)
489		return;
490	err = synthesize_perf_record_bpf_metadata(metadata, ctx->tool,
491						  ctx->process, ctx->machine);
492	if (err != 0) {
493		const char *prog_name = metadata->prog_names[0];
494
495		if (prog_name != NULL)
496			pr_warning("Couldn't synthesize final BPF metadata for %s.\n", prog_name);
497		else
498			pr_warning("Couldn't synthesize final BPF metadata.\n");
499	}
500	bpf_metadata_free(metadata);
501	node->metadata = NULL;
502}
503
504void perf_event__synthesize_final_bpf_metadata(struct perf_session *session,
505					       perf_event__handler_t process)
506{
507	struct perf_env *env = &session->header.env;
508	struct bpf_metadata_final_ctx ctx = {
509		.tool = session->tool,
510		.process = process,
511		.machine = &session->machines.host,
512	};
513
514	perf_env__iterate_bpf_prog_info(env, synthesize_final_bpf_metadata_cb,
515					&ctx);
516}
517
518/*
519 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf
520 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And
521 * one PERF_RECORD_KSYMBOL is generated for each sub program.
522 *
523 * Returns:
524 *    0 for success;
525 *   -1 for failures;
526 *   -2 for lack of kernel support.
527 */
528static int perf_event__synthesize_one_bpf_prog(struct perf_session *session,
529					       perf_event__handler_t process,
530					       struct machine *machine,
531					       int fd,
532					       union perf_event *event,
533					       struct record_opts *opts)
534{
535	struct perf_record_ksymbol *ksymbol_event = &event->ksymbol;
536	struct perf_record_bpf_event *bpf_event = &event->bpf;
537	const struct perf_tool *tool = session->tool;
538	struct bpf_prog_info_node *info_node;
539	struct perf_bpil *info_linear;
540	struct bpf_metadata *metadata;
541	struct bpf_prog_info *info;
542	struct btf *btf = NULL;
543	struct perf_env *env;
544	u32 sub_prog_cnt, i;
545	int err = 0;
546	u64 arrays;
547
548	/*
549	 * for perf-record and perf-report use header.env;
550	 * otherwise, use global perf_env.
551	 */
552	env = perf_session__env(session);
553
554	arrays = 1UL << PERF_BPIL_JITED_KSYMS;
555	arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS;
556	arrays |= 1UL << PERF_BPIL_FUNC_INFO;
557	arrays |= 1UL << PERF_BPIL_PROG_TAGS;
558	arrays |= 1UL << PERF_BPIL_JITED_INSNS;
559	arrays |= 1UL << PERF_BPIL_LINE_INFO;
560	arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO;
561	arrays |= 1UL << PERF_BPIL_MAP_IDS;
562
563	info_linear = get_bpf_prog_info_linear(fd, arrays);
564	if (IS_ERR_OR_NULL(info_linear)) {
565		info_linear = NULL;
566		pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
567		return -1;
568	}
569
570	if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) {
571		free(info_linear);
572		pr_debug("%s: the kernel is too old, aborting\n", __func__);
573		return -2;
574	}
575
576	info = &info_linear->info;
577	if (!info->jited_ksyms) {
578		free(info_linear);
579		return -1;
580	}
581
582	/* number of ksyms, func_lengths, and tags should match */
583	sub_prog_cnt = info->nr_jited_ksyms;
584	if (sub_prog_cnt != info->nr_prog_tags ||
585	    sub_prog_cnt != info->nr_jited_func_lens) {
586		free(info_linear);
587		return -1;
588	}
589
590	/* check BTF func info support */
591	if (info->btf_id && info->nr_func_info && info->func_info_rec_size) {
592		/* btf func info number should be same as sub_prog_cnt */
593		if (sub_prog_cnt != info->nr_func_info) {
594			pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__);
595			free(info_linear);
596			return -1;
597		}
598		btf = btf__load_from_kernel_by_id(info->btf_id);
599		if (libbpf_get_error(btf)) {
600			pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id);
601			err = -1;
602			goto out;
603		}
604		perf_env__fetch_btf(env, info->btf_id, btf);
605	}
606
607	/* Synthesize PERF_RECORD_KSYMBOL */
608	for (i = 0; i < sub_prog_cnt; i++) {
609		__u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
610		__u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
611		int name_len;
612
613		*ksymbol_event = (struct perf_record_ksymbol) {
614			.header = {
615				.type = PERF_RECORD_KSYMBOL,
616				.size = offsetof(struct perf_record_ksymbol, name),
617			},
618			.addr = prog_addrs[i],
619			.len = prog_lens[i],
620			.ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
621			.flags = 0,
622		};
623
624		name_len = synthesize_bpf_prog_name(ksymbol_event->name,
625						    KSYM_NAME_LEN, info, btf, i);
626		ksymbol_event->header.size += PERF_ALIGN(name_len + 1,
627							 sizeof(u64));
628
629		memset((void *)event + event->header.size, 0, machine->id_hdr_size);
630		event->header.size += machine->id_hdr_size;
631		err = perf_tool__process_synth_event(tool, event,
632						     machine, process);
633	}
634
635	if (!opts->no_bpf_event) {
636		/* Synthesize PERF_RECORD_BPF_EVENT */
637		*bpf_event = (struct perf_record_bpf_event) {
638			.header = {
639				.type = PERF_RECORD_BPF_EVENT,
640				.size = sizeof(struct perf_record_bpf_event),
641			},
642			.type = PERF_BPF_EVENT_PROG_LOAD,
643			.flags = 0,
644			.id = info->id,
645		};
646		memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE);
647		memset((void *)event + event->header.size, 0, machine->id_hdr_size);
648		event->header.size += machine->id_hdr_size;
649
650		/* save bpf_prog_info to env */
651		info_node = malloc(sizeof(struct bpf_prog_info_node));
652		if (!info_node) {
653			err = -1;
654			goto out;
655		}
656
657		info_node->info_linear = info_linear;
658		info_node->metadata = NULL;
659		if (!perf_env__insert_bpf_prog_info(env, info_node)) {
660			/*
661			 * Insert failed, likely because of a duplicate event
662			 * made by the sideband thread. Ignore synthesizing the
663			 * metadata.
664			 */
665			free(info_node);
666			goto out;
667		}
668		/* info_linear is now owned by info_node and shouldn't be freed below. */
669		info_linear = NULL;
670
671		/*
672		 * process after saving bpf_prog_info to env, so that
673		 * required information is ready for look up
674		 */
675		err = perf_tool__process_synth_event(tool, event,
676						     machine, process);
677
678		/* Synthesize PERF_RECORD_BPF_METADATA */
679		metadata = bpf_metadata_create(info);
680		if (metadata != NULL) {
681			err = synthesize_perf_record_bpf_metadata(metadata,
682								  tool, process,
683								  machine);
684			bpf_metadata_free(metadata);
685		}
686	}
687
688out:
689	free(info_linear);
690	btf__free(btf);
691	return err ? -1 : 0;
692}
693
694struct kallsyms_parse {
695	union perf_event	*event;
696	perf_event__handler_t	 process;
697	struct machine		*machine;
698	const struct perf_tool	*tool;
699};
700
701static int
702process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data)
703{
704	struct machine *machine = data->machine;
705	union perf_event *event = data->event;
706	struct perf_record_ksymbol *ksymbol;
707	int len;
708
709	ksymbol = &event->ksymbol;
710
711	*ksymbol = (struct perf_record_ksymbol) {
712		.header = {
713			.type = PERF_RECORD_KSYMBOL,
714			.size = offsetof(struct perf_record_ksymbol, name),
715		},
716		.addr      = addr,
717		.len       = page_size,
718		.ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
719		.flags     = 0,
720	};
721
722	len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name);
723	ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64));
724	memset((void *) event + event->header.size, 0, machine->id_hdr_size);
725	event->header.size += machine->id_hdr_size;
726
727	return perf_tool__process_synth_event(data->tool, event, machine,
728					      data->process);
729}
730
731static int
732kallsyms_process_symbol(void *data, const char *_name,
733			char type __maybe_unused, u64 start)
734{
735	char disp[KSYM_NAME_LEN];
736	char *module, *name;
737	unsigned long id;
738	int err = 0;
739
740	module = strchr(_name, '\t');
741	if (!module)
742		return 0;
743
744	/* We are going after [bpf] module ... */
745	if (strcmp(module + 1, "[bpf]"))
746		return 0;
747
748	name = memdup(_name, (module - _name) + 1);
749	if (!name)
750		return -ENOMEM;
751
752	name[module - _name] = 0;
753
754	/* .. and only for trampolines and dispatchers */
755	if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) ||
756	    (sscanf(name, "bpf_dispatcher_%s", disp) == 1))
757		err = process_bpf_image(name, start, data);
758
759	free(name);
760	return err;
761}
762
763int perf_event__synthesize_bpf_events(struct perf_session *session,
764				      perf_event__handler_t process,
765				      struct machine *machine,
766				      struct record_opts *opts)
767{
768	const char *kallsyms_filename = "/proc/kallsyms";
769	struct kallsyms_parse arg;
770	union perf_event *event;
771	__u32 id = 0;
772	int err;
773	int fd;
774
775	if (opts->no_bpf_event)
776		return 0;
777
778	event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size);
779	if (!event)
780		return -1;
781
782	/* Synthesize all the bpf programs in system. */
783	while (true) {
784		err = bpf_prog_get_next_id(id, &id);
785		if (err) {
786			if (errno == ENOENT) {
787				err = 0;
788				break;
789			}
790			pr_debug("%s: can't get next program: %s%s\n",
791				 __func__, strerror(errno),
792				 errno == EINVAL ? " -- kernel too old?" : "");
793			/* don't report error on old kernel or EPERM  */
794			err = (errno == EINVAL || errno == EPERM) ? 0 : -1;
795			break;
796		}
797		fd = bpf_prog_get_fd_by_id(id);
798		if (fd < 0) {
799			pr_debug("%s: failed to get fd for prog_id %u\n",
800				 __func__, id);
801			continue;
802		}
803
804		err = perf_event__synthesize_one_bpf_prog(session, process,
805							  machine, fd,
806							  event, opts);
807		close(fd);
808		if (err) {
809			/* do not return error for old kernel */
810			if (err == -2)
811				err = 0;
812			break;
813		}
814	}
815
816	/* Synthesize all the bpf images - trampolines/dispatchers. */
817	if (symbol_conf.kallsyms_name != NULL)
818		kallsyms_filename = symbol_conf.kallsyms_name;
819
820	arg = (struct kallsyms_parse) {
821		.event   = event,
822		.process = process,
823		.machine = machine,
824		.tool    = session->tool,
825	};
826
827	if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) {
828		pr_err("%s: failed to synthesize bpf images: %s\n",
829		       __func__, strerror(errno));
830	}
831
832	free(event);
833	return err;
834}
835
836static int perf_env__add_bpf_info(struct perf_env *env, u32 id)
837{
838	struct bpf_prog_info_node *info_node;
839	struct perf_bpil *info_linear;
840	struct btf *btf = NULL;
841	u64 arrays;
842	u32 btf_id;
843	int fd, err = 0;
844
845	fd = bpf_prog_get_fd_by_id(id);
846	if (fd < 0)
847		return -EINVAL;
848
849	arrays = 1UL << PERF_BPIL_JITED_KSYMS;
850	arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS;
851	arrays |= 1UL << PERF_BPIL_FUNC_INFO;
852	arrays |= 1UL << PERF_BPIL_PROG_TAGS;
853	arrays |= 1UL << PERF_BPIL_JITED_INSNS;
854	arrays |= 1UL << PERF_BPIL_LINE_INFO;
855	arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO;
856	arrays |= 1UL << PERF_BPIL_MAP_IDS;
857
858	info_linear = get_bpf_prog_info_linear(fd, arrays);
859	if (IS_ERR_OR_NULL(info_linear)) {
860		pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
861		err = PTR_ERR(info_linear);
862		goto out;
863	}
864
865	btf_id = info_linear->info.btf_id;
866
867	info_node = malloc(sizeof(struct bpf_prog_info_node));
868	if (info_node) {
869		info_node->info_linear = info_linear;
870		info_node->metadata = bpf_metadata_create(&info_linear->info);
871		if (!perf_env__insert_bpf_prog_info(env, info_node)) {
872			pr_debug("%s: duplicate add bpf info request for id %u\n",
873				 __func__, btf_id);
874			free(info_linear);
875			free(info_node);
876			goto out;
877		}
878	} else {
879		free(info_linear);
880		err = -ENOMEM;
881		goto out;
882	}
883
884	if (btf_id == 0)
885		goto out;
886
887	btf = btf__load_from_kernel_by_id(btf_id);
888	if (!btf) {
889		err = -errno;
890		pr_debug("%s: failed to get BTF of id %u %d\n", __func__, btf_id, err);
891	} else {
892		perf_env__fetch_btf(env, btf_id, btf);
893	}
894
895out:
896	btf__free(btf);
897	close(fd);
898	return err;
899}
900
901static int bpf_event__sb_cb(union perf_event *event, void *data)
902{
903	struct perf_env *env = data;
904	int ret = 0;
905
906	if (event->header.type != PERF_RECORD_BPF_EVENT)
907		return -1;
908
909	switch (event->bpf.type) {
910	case PERF_BPF_EVENT_PROG_LOAD:
911		ret = perf_env__add_bpf_info(env, event->bpf.id);
912
913	case PERF_BPF_EVENT_PROG_UNLOAD:
914		/*
915		 * Do not free bpf_prog_info and btf of the program here,
916		 * as annotation still need them. They will be freed at
917		 * the end of the session.
918		 */
919		break;
920	default:
921		pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
922		break;
923	}
924
925	return ret;
926}
927
928int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env)
929{
930	struct perf_event_attr attr = {
931		.type	          = PERF_TYPE_SOFTWARE,
932		.config           = PERF_COUNT_SW_DUMMY,
933		.sample_id_all    = 1,
934		.watermark        = 1,
935		.bpf_event        = 1,
936		.size	   = sizeof(attr), /* to capture ABI version */
937	};
938
939	/*
940	 * Older gcc versions don't support designated initializers, like above,
941	 * for unnamed union members, such as the following:
942	 */
943	attr.wakeup_watermark = 1;
944
945	return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env);
946}
947
948void __bpf_event__print_bpf_prog_info(struct bpf_prog_info *info,
949				      struct perf_env *env,
950				      FILE *fp)
951{
952	__u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
953	__u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
954	char name[KSYM_NAME_LEN];
955	struct btf *btf = NULL;
956	u32 sub_prog_cnt, i;
957
958	sub_prog_cnt = info->nr_jited_ksyms;
959	if (sub_prog_cnt != info->nr_prog_tags ||
960	    sub_prog_cnt != info->nr_jited_func_lens)
961		return;
962
963	if (info->btf_id) {
964		struct btf_node *node;
965
966		node = __perf_env__find_btf(env, info->btf_id);
967		if (node)
968			btf = btf__new((__u8 *)(node->data),
969				       node->data_size);
970	}
971
972	if (sub_prog_cnt == 1) {
973		synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0);
974		fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n",
975			info->id, name, prog_addrs[0], prog_lens[0]);
976		goto out;
977	}
978
979	fprintf(fp, "# bpf_prog_info %u:\n", info->id);
980	for (i = 0; i < sub_prog_cnt; i++) {
981		synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i);
982
983		fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n",
984			i, name, prog_addrs[i], prog_lens[i]);
985	}
986out:
987	btf__free(btf);
988}