prog.c

   1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2/* Copyright (C) 2017-2018 Netronome Systems, Inc. */
   3
   4#ifndef _GNU_SOURCE
   5#define _GNU_SOURCE
   6#endif
   7#include <errno.h>
   8#include <fcntl.h>
   9#include <signal.h>
  10#include <stdarg.h>
  11#include <stdio.h>
  12#include <stdlib.h>
  13#include <string.h>
  14#include <time.h>
  15#include <unistd.h>
  16#include <net/if.h>
  17#include <sys/ioctl.h>
  18#include <sys/types.h>
  19#include <sys/stat.h>
  20#include <sys/syscall.h>
  21#include <dirent.h>
  22
  23#include <linux/err.h>
  24#include <linux/perf_event.h>
  25#include <linux/sizes.h>
  26#include <linux/keyctl.h>
  27
  28#include <bpf/bpf.h>
  29#include <bpf/btf.h>
  30#include <bpf/hashmap.h>
  31#include <bpf/libbpf.h>
  32#include <bpf/libbpf_internal.h>
  33#include <bpf/skel_internal.h>
  34
  35#include "cfg.h"
  36#include "main.h"
  37#include "xlated_dumper.h"
  38
  39#define BPF_METADATA_PREFIX "bpf_metadata_"
  40#define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1)
  41
  42enum dump_mode {
  43	DUMP_JITED,
  44	DUMP_XLATED,
  45};
  46
  47static const bool attach_types[] = {
  48	[BPF_SK_SKB_STREAM_PARSER] = true,
  49	[BPF_SK_SKB_STREAM_VERDICT] = true,
  50	[BPF_SK_SKB_VERDICT] = true,
  51	[BPF_SK_MSG_VERDICT] = true,
  52	[BPF_FLOW_DISSECTOR] = true,
  53	[__MAX_BPF_ATTACH_TYPE] = false,
  54};
  55
  56/* Textual representations traditionally used by the program and kept around
  57 * for the sake of backwards compatibility.
  58 */
  59static const char * const attach_type_strings[] = {
  60	[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
  61	[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
  62	[BPF_SK_SKB_VERDICT] = "skb_verdict",
  63	[BPF_SK_MSG_VERDICT] = "msg_verdict",
  64	[__MAX_BPF_ATTACH_TYPE] = NULL,
  65};
  66
  67static struct hashmap *prog_table;
  68
  69static enum bpf_attach_type parse_attach_type(const char *str)
  70{
  71	enum bpf_attach_type type;
  72
  73	for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {
  74		if (attach_types[type]) {
  75			const char *attach_type_str;
  76
  77			attach_type_str = libbpf_bpf_attach_type_str(type);
  78			if (!strcmp(str, attach_type_str))
  79				return type;
  80		}
  81
  82		if (attach_type_strings[type] &&
  83		    is_prefix(str, attach_type_strings[type]))
  84			return type;
  85	}
  86
  87	return __MAX_BPF_ATTACH_TYPE;
  88}
  89
  90static int prep_prog_info(struct bpf_prog_info *const info, enum dump_mode mode,
  91			  void **info_data, size_t *const info_data_sz)
  92{
  93	struct bpf_prog_info holder = {};
  94	size_t needed = 0;
  95	void *ptr;
  96
  97	if (mode == DUMP_JITED) {
  98		holder.jited_prog_len = info->jited_prog_len;
  99		needed += info->jited_prog_len;
 100	} else {
 101		holder.xlated_prog_len = info->xlated_prog_len;
 102		needed += info->xlated_prog_len;
 103	}
 104
 105	holder.nr_jited_ksyms = info->nr_jited_ksyms;
 106	needed += info->nr_jited_ksyms * sizeof(__u64);
 107
 108	holder.nr_jited_func_lens = info->nr_jited_func_lens;
 109	needed += info->nr_jited_func_lens * sizeof(__u32);
 110
 111	holder.nr_func_info = info->nr_func_info;
 112	holder.func_info_rec_size = info->func_info_rec_size;
 113	needed += info->nr_func_info * info->func_info_rec_size;
 114
 115	holder.nr_line_info = info->nr_line_info;
 116	holder.line_info_rec_size = info->line_info_rec_size;
 117	needed += info->nr_line_info * info->line_info_rec_size;
 118
 119	holder.nr_jited_line_info = info->nr_jited_line_info;
 120	holder.jited_line_info_rec_size = info->jited_line_info_rec_size;
 121	needed += info->nr_jited_line_info * info->jited_line_info_rec_size;
 122
 123	if (needed > *info_data_sz) {
 124		ptr = realloc(*info_data, needed);
 125		if (!ptr)
 126			return -1;
 127
 128		*info_data = ptr;
 129		*info_data_sz = needed;
 130	}
 131	ptr = *info_data;
 132
 133	if (mode == DUMP_JITED) {
 134		holder.jited_prog_insns = ptr_to_u64(ptr);
 135		ptr += holder.jited_prog_len;
 136	} else {
 137		holder.xlated_prog_insns = ptr_to_u64(ptr);
 138		ptr += holder.xlated_prog_len;
 139	}
 140
 141	holder.jited_ksyms = ptr_to_u64(ptr);
 142	ptr += holder.nr_jited_ksyms * sizeof(__u64);
 143
 144	holder.jited_func_lens = ptr_to_u64(ptr);
 145	ptr += holder.nr_jited_func_lens * sizeof(__u32);
 146
 147	holder.func_info = ptr_to_u64(ptr);
 148	ptr += holder.nr_func_info * holder.func_info_rec_size;
 149
 150	holder.line_info = ptr_to_u64(ptr);
 151	ptr += holder.nr_line_info * holder.line_info_rec_size;
 152
 153	holder.jited_line_info = ptr_to_u64(ptr);
 154	ptr += holder.nr_jited_line_info * holder.jited_line_info_rec_size;
 155
 156	*info = holder;
 157	return 0;
 158}
 159
 160static void print_boot_time(__u64 nsecs, char *buf, unsigned int size)
 161{
 162	struct timespec real_time_ts, boot_time_ts;
 163	time_t wallclock_secs;
 164	struct tm load_tm;
 165
 166	buf[--size] = '\0';
 167
 168	if (clock_gettime(CLOCK_REALTIME, &real_time_ts) ||
 169	    clock_gettime(CLOCK_BOOTTIME, &boot_time_ts)) {
 170		perror("Can't read clocks");
 171		snprintf(buf, size, "%llu", nsecs / 1000000000);
 172		return;
 173	}
 174
 175	wallclock_secs = (real_time_ts.tv_sec - boot_time_ts.tv_sec) +
 176		(real_time_ts.tv_nsec - boot_time_ts.tv_nsec + nsecs) /
 177		1000000000;
 178
 179
 180	if (!localtime_r(&wallclock_secs, &load_tm)) {
 181		snprintf(buf, size, "%llu", nsecs / 1000000000);
 182		return;
 183	}
 184
 185	if (json_output)
 186		strftime(buf, size, "%s", &load_tm);
 187	else
 188		strftime(buf, size, "%FT%T%z", &load_tm);
 189}
 190
 191static void show_prog_maps(int fd, __u32 num_maps)
 192{
 193	struct bpf_prog_info info = {};
 194	__u32 len = sizeof(info);
 195	__u32 map_ids[num_maps];
 196	unsigned int i;
 197	int err;
 198
 199	info.nr_map_ids = num_maps;
 200	info.map_ids = ptr_to_u64(map_ids);
 201
 202	err = bpf_prog_get_info_by_fd(fd, &info, &len);
 203	if (err || !info.nr_map_ids)
 204		return;
 205
 206	if (json_output) {
 207		jsonw_name(json_wtr, "map_ids");
 208		jsonw_start_array(json_wtr);
 209		for (i = 0; i < info.nr_map_ids; i++)
 210			jsonw_uint(json_wtr, map_ids[i]);
 211		jsonw_end_array(json_wtr);
 212	} else {
 213		printf("  map_ids ");
 214		for (i = 0; i < info.nr_map_ids; i++)
 215			printf("%u%s", map_ids[i],
 216			       i == info.nr_map_ids - 1 ? "" : ",");
 217	}
 218}
 219
 220static void *find_metadata(int prog_fd, struct bpf_map_info *map_info)
 221{
 222	struct bpf_prog_info prog_info;
 223	__u32 prog_info_len;
 224	__u32 map_info_len;
 225	void *value = NULL;
 226	__u32 *map_ids;
 227	int nr_maps;
 228	int key = 0;
 229	int map_fd;
 230	int ret;
 231	__u32 i;
 232
 233	memset(&prog_info, 0, sizeof(prog_info));
 234	prog_info_len = sizeof(prog_info);
 235	ret = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &prog_info_len);
 236	if (ret)
 237		return NULL;
 238
 239	if (!prog_info.nr_map_ids)
 240		return NULL;
 241
 242	map_ids = calloc(prog_info.nr_map_ids, sizeof(__u32));
 243	if (!map_ids)
 244		return NULL;
 245
 246	nr_maps = prog_info.nr_map_ids;
 247	memset(&prog_info, 0, sizeof(prog_info));
 248	prog_info.nr_map_ids = nr_maps;
 249	prog_info.map_ids = ptr_to_u64(map_ids);
 250	prog_info_len = sizeof(prog_info);
 251
 252	ret = bpf_prog_get_info_by_fd(prog_fd, &prog_info, &prog_info_len);
 253	if (ret)
 254		goto free_map_ids;
 255
 256	for (i = 0; i < prog_info.nr_map_ids; i++) {
 257		map_fd = bpf_map_get_fd_by_id(map_ids[i]);
 258		if (map_fd < 0)
 259			goto free_map_ids;
 260
 261		memset(map_info, 0, sizeof(*map_info));
 262		map_info_len = sizeof(*map_info);
 263		ret = bpf_map_get_info_by_fd(map_fd, map_info, &map_info_len);
 264		if (ret < 0) {
 265			close(map_fd);
 266			goto free_map_ids;
 267		}
 268
 269		if (map_info->type != BPF_MAP_TYPE_ARRAY ||
 270		    map_info->key_size != sizeof(int) ||
 271		    map_info->max_entries != 1 ||
 272		    !map_info->btf_value_type_id ||
 273		    !strstr(map_info->name, ".rodata")) {
 274			close(map_fd);
 275			continue;
 276		}
 277
 278		value = malloc(map_info->value_size);
 279		if (!value) {
 280			close(map_fd);
 281			goto free_map_ids;
 282		}
 283
 284		if (bpf_map_lookup_elem(map_fd, &key, value)) {
 285			close(map_fd);
 286			free(value);
 287			value = NULL;
 288			goto free_map_ids;
 289		}
 290
 291		close(map_fd);
 292		break;
 293	}
 294
 295free_map_ids:
 296	free(map_ids);
 297	return value;
 298}
 299
 300static bool has_metadata_prefix(const char *s)
 301{
 302	return strncmp(s, BPF_METADATA_PREFIX, BPF_METADATA_PREFIX_LEN) == 0;
 303}
 304
 305static void show_prog_metadata(int fd, __u32 num_maps)
 306{
 307	const struct btf_type *t_datasec, *t_var;
 308	struct bpf_map_info map_info;
 309	struct btf_var_secinfo *vsi;
 310	bool printed_header = false;
 311	unsigned int i, vlen;
 312	void *value = NULL;
 313	const char *name;
 314	struct btf *btf;
 315	int err;
 316
 317	if (!num_maps)
 318		return;
 319
 320	memset(&map_info, 0, sizeof(map_info));
 321	value = find_metadata(fd, &map_info);
 322	if (!value)
 323		return;
 324
 325	btf = btf__load_from_kernel_by_id(map_info.btf_id);
 326	if (!btf)
 327		goto out_free;
 328
 329	t_datasec = btf__type_by_id(btf, map_info.btf_value_type_id);
 330	if (!btf_is_datasec(t_datasec))
 331		goto out_free;
 332
 333	vlen = btf_vlen(t_datasec);
 334	vsi = btf_var_secinfos(t_datasec);
 335
 336	/* We don't proceed to check the kinds of the elements of the DATASEC.
 337	 * The verifier enforces them to be BTF_KIND_VAR.
 338	 */
 339
 340	if (json_output) {
 341		struct btf_dumper d = {
 342			.btf = btf,
 343			.jw = json_wtr,
 344			.is_plain_text = false,
 345		};
 346
 347		for (i = 0; i < vlen; i++, vsi++) {
 348			t_var = btf__type_by_id(btf, vsi->type);
 349			name = btf__name_by_offset(btf, t_var->name_off);
 350
 351			if (!has_metadata_prefix(name))
 352				continue;
 353
 354			if (!printed_header) {
 355				jsonw_name(json_wtr, "metadata");
 356				jsonw_start_object(json_wtr);
 357				printed_header = true;
 358			}
 359
 360			jsonw_name(json_wtr, name + BPF_METADATA_PREFIX_LEN);
 361			err = btf_dumper_type(&d, t_var->type, value + vsi->offset);
 362			if (err) {
 363				p_err("btf dump failed: %d", err);
 364				break;
 365			}
 366		}
 367		if (printed_header)
 368			jsonw_end_object(json_wtr);
 369	} else {
 370		json_writer_t *btf_wtr;
 371		struct btf_dumper d = {
 372			.btf = btf,
 373			.is_plain_text = true,
 374		};
 375
 376		for (i = 0; i < vlen; i++, vsi++) {
 377			t_var = btf__type_by_id(btf, vsi->type);
 378			name = btf__name_by_offset(btf, t_var->name_off);
 379
 380			if (!has_metadata_prefix(name))
 381				continue;
 382
 383			if (!printed_header) {
 384				printf("\tmetadata:");
 385
 386				btf_wtr = jsonw_new(stdout);
 387				if (!btf_wtr) {
 388					p_err("jsonw alloc failed");
 389					goto out_free;
 390				}
 391				d.jw = btf_wtr,
 392
 393				printed_header = true;
 394			}
 395
 396			printf("\n\t\t%s = ", name + BPF_METADATA_PREFIX_LEN);
 397
 398			jsonw_reset(btf_wtr);
 399			err = btf_dumper_type(&d, t_var->type, value + vsi->offset);
 400			if (err) {
 401				p_err("btf dump failed: %d", err);
 402				break;
 403			}
 404		}
 405		if (printed_header)
 406			jsonw_destroy(&btf_wtr);
 407	}
 408
 409out_free:
 410	btf__free(btf);
 411	free(value);
 412}
 413
 414static void print_prog_header_json(struct bpf_prog_info *info, int fd)
 415{
 416	const char *prog_type_str;
 417	char prog_name[MAX_PROG_FULL_NAME];
 418
 419	jsonw_uint_field(json_wtr, "id", info->id);
 420	prog_type_str = libbpf_bpf_prog_type_str(info->type);
 421
 422	if (prog_type_str)
 423		jsonw_string_field(json_wtr, "type", prog_type_str);
 424	else
 425		jsonw_uint_field(json_wtr, "type", info->type);
 426
 427	if (*info->name) {
 428		get_prog_full_name(info, fd, prog_name, sizeof(prog_name));
 429		jsonw_string_field(json_wtr, "name", prog_name);
 430	}
 431
 432	jsonw_name(json_wtr, "tag");
 433	jsonw_printf(json_wtr, "\"" BPF_TAG_FMT "\"",
 434		     info->tag[0], info->tag[1], info->tag[2], info->tag[3],
 435		     info->tag[4], info->tag[5], info->tag[6], info->tag[7]);
 436
 437	jsonw_bool_field(json_wtr, "gpl_compatible", info->gpl_compatible);
 438	if (info->run_time_ns) {
 439		jsonw_uint_field(json_wtr, "run_time_ns", info->run_time_ns);
 440		jsonw_uint_field(json_wtr, "run_cnt", info->run_cnt);
 441	}
 442	if (info->recursion_misses)
 443		jsonw_uint_field(json_wtr, "recursion_misses", info->recursion_misses);
 444}
 445
 446static void print_prog_json(struct bpf_prog_info *info, int fd, bool orphaned)
 447{
 448	char *memlock;
 449
 450	jsonw_start_object(json_wtr);
 451	print_prog_header_json(info, fd);
 452	print_dev_json(info->ifindex, info->netns_dev, info->netns_ino);
 453
 454	if (info->load_time) {
 455		char buf[32];
 456
 457		print_boot_time(info->load_time, buf, sizeof(buf));
 458
 459		/* Piggy back on load_time, since 0 uid is a valid one */
 460		jsonw_name(json_wtr, "loaded_at");
 461		jsonw_printf(json_wtr, "%s", buf);
 462		jsonw_uint_field(json_wtr, "uid", info->created_by_uid);
 463	}
 464
 465	jsonw_bool_field(json_wtr, "orphaned", orphaned);
 466	jsonw_uint_field(json_wtr, "bytes_xlated", info->xlated_prog_len);
 467
 468	if (info->jited_prog_len) {
 469		jsonw_bool_field(json_wtr, "jited", true);
 470		jsonw_uint_field(json_wtr, "bytes_jited", info->jited_prog_len);
 471	} else {
 472		jsonw_bool_field(json_wtr, "jited", false);
 473	}
 474
 475	memlock = get_fdinfo(fd, "memlock");
 476	if (memlock)
 477		jsonw_int_field(json_wtr, "bytes_memlock", atoll(memlock));
 478	free(memlock);
 479
 480	if (info->nr_map_ids)
 481		show_prog_maps(fd, info->nr_map_ids);
 482
 483	if (info->btf_id)
 484		jsonw_int_field(json_wtr, "btf_id", info->btf_id);
 485
 486	if (!hashmap__empty(prog_table)) {
 487		struct hashmap_entry *entry;
 488
 489		jsonw_name(json_wtr, "pinned");
 490		jsonw_start_array(json_wtr);
 491		hashmap__for_each_key_entry(prog_table, entry, info->id)
 492			jsonw_string(json_wtr, entry->pvalue);
 493		jsonw_end_array(json_wtr);
 494	}
 495
 496	emit_obj_refs_json(refs_table, info->id, json_wtr);
 497
 498	show_prog_metadata(fd, info->nr_map_ids);
 499
 500	jsonw_end_object(json_wtr);
 501}
 502
 503static void print_prog_header_plain(struct bpf_prog_info *info, int fd)
 504{
 505	const char *prog_type_str;
 506	char prog_name[MAX_PROG_FULL_NAME];
 507
 508	printf("%u: ", info->id);
 509	prog_type_str = libbpf_bpf_prog_type_str(info->type);
 510	if (prog_type_str)
 511		printf("%s  ", prog_type_str);
 512	else
 513		printf("type %u  ", info->type);
 514
 515	if (*info->name) {
 516		get_prog_full_name(info, fd, prog_name, sizeof(prog_name));
 517		printf("name %s  ", prog_name);
 518	}
 519
 520	printf("tag ");
 521	fprint_hex(stdout, info->tag, BPF_TAG_SIZE, "");
 522	print_dev_plain(info->ifindex, info->netns_dev, info->netns_ino);
 523	printf("%s", info->gpl_compatible ? "  gpl" : "");
 524	if (info->run_time_ns)
 525		printf(" run_time_ns %llu run_cnt %llu",
 526		       info->run_time_ns, info->run_cnt);
 527	if (info->recursion_misses)
 528		printf(" recursion_misses %llu", info->recursion_misses);
 529	printf("\n");
 530}
 531
 532static void print_prog_plain(struct bpf_prog_info *info, int fd, bool orphaned)
 533{
 534	char *memlock;
 535
 536	print_prog_header_plain(info, fd);
 537
 538	if (info->load_time) {
 539		char buf[32];
 540
 541		print_boot_time(info->load_time, buf, sizeof(buf));
 542
 543		/* Piggy back on load_time, since 0 uid is a valid one */
 544		printf("\tloaded_at %s  uid %u\n", buf, info->created_by_uid);
 545	}
 546
 547	printf("\txlated %uB", info->xlated_prog_len);
 548
 549	if (info->jited_prog_len)
 550		printf("  jited %uB", info->jited_prog_len);
 551	else
 552		printf("  not jited");
 553
 554	memlock = get_fdinfo(fd, "memlock");
 555	if (memlock)
 556		printf("  memlock %sB", memlock);
 557	free(memlock);
 558
 559	if (orphaned)
 560		printf("  orphaned");
 561
 562	if (info->nr_map_ids)
 563		show_prog_maps(fd, info->nr_map_ids);
 564
 565	if (!hashmap__empty(prog_table)) {
 566		struct hashmap_entry *entry;
 567
 568		hashmap__for_each_key_entry(prog_table, entry, info->id)
 569			printf("\n\tpinned %s", (char *)entry->pvalue);
 570	}
 571
 572	if (info->btf_id)
 573		printf("\n\tbtf_id %u", info->btf_id);
 574
 575	emit_obj_refs_plain(refs_table, info->id, "\n\tpids ");
 576
 577	printf("\n");
 578
 579	show_prog_metadata(fd, info->nr_map_ids);
 580}
 581
 582static int show_prog(int fd)
 583{
 584	struct bpf_prog_info info = {};
 585	__u32 len = sizeof(info);
 586	int err;
 587
 588	err = bpf_prog_get_info_by_fd(fd, &info, &len);
 589	if (err && err != -ENODEV) {
 590		p_err("can't get prog info: %s", strerror(errno));
 591		return -1;
 592	}
 593
 594	if (json_output)
 595		print_prog_json(&info, fd, err == -ENODEV);
 596	else
 597		print_prog_plain(&info, fd, err == -ENODEV);
 598
 599	return 0;
 600}
 601
 602static int do_show_subset(int argc, char **argv)
 603{
 604	int *fds = NULL;
 605	int nb_fds, i;
 606	int err = -1;
 607
 608	fds = malloc(sizeof(int));
 609	if (!fds) {
 610		p_err("mem alloc failed");
 611		return -1;
 612	}
 613	nb_fds = prog_parse_fds(&argc, &argv, &fds);
 614	if (nb_fds < 1)
 615		goto exit_free;
 616
 617	if (json_output && nb_fds > 1)
 618		jsonw_start_array(json_wtr);	/* root array */
 619	for (i = 0; i < nb_fds; i++) {
 620		err = show_prog(fds[i]);
 621		if (err) {
 622			for (; i < nb_fds; i++)
 623				close(fds[i]);
 624			break;
 625		}
 626		close(fds[i]);
 627	}
 628	if (json_output && nb_fds > 1)
 629		jsonw_end_array(json_wtr);	/* root array */
 630
 631exit_free:
 632	free(fds);
 633	return err;
 634}
 635
 636static int do_show(int argc, char **argv)
 637{
 638	__u32 id = 0;
 639	int err;
 640	int fd;
 641
 642	if (show_pinned) {
 643		prog_table = hashmap__new(hash_fn_for_key_as_id,
 644					  equal_fn_for_key_as_id, NULL);
 645		if (IS_ERR(prog_table)) {
 646			p_err("failed to create hashmap for pinned paths");
 647			return -1;
 648		}
 649		build_pinned_obj_table(prog_table, BPF_OBJ_PROG);
 650	}
 651	build_obj_refs_table(&refs_table, BPF_OBJ_PROG);
 652
 653	if (argc == 2)
 654		return do_show_subset(argc, argv);
 655
 656	if (argc)
 657		return BAD_ARG();
 658
 659	if (json_output)
 660		jsonw_start_array(json_wtr);
 661	while (true) {
 662		err = bpf_prog_get_next_id(id, &id);
 663		if (err) {
 664			if (errno == ENOENT) {
 665				err = 0;
 666				break;
 667			}
 668			p_err("can't get next program: %s%s", strerror(errno),
 669			      errno == EINVAL ? " -- kernel too old?" : "");
 670			err = -1;
 671			break;
 672		}
 673
 674		fd = bpf_prog_get_fd_by_id(id);
 675		if (fd < 0) {
 676			if (errno == ENOENT)
 677				continue;
 678			p_err("can't get prog by id (%u): %s",
 679			      id, strerror(errno));
 680			err = -1;
 681			break;
 682		}
 683
 684		err = show_prog(fd);
 685		close(fd);
 686		if (err)
 687			break;
 688	}
 689
 690	if (json_output)
 691		jsonw_end_array(json_wtr);
 692
 693	delete_obj_refs_table(refs_table);
 694
 695	if (show_pinned)
 696		delete_pinned_obj_table(prog_table);
 697
 698	return err;
 699}
 700
 701static int
 702prog_dump(struct bpf_prog_info *info, enum dump_mode mode,
 703	  char *filepath, bool opcodes, bool visual, bool linum)
 704{
 705	struct bpf_prog_linfo *prog_linfo = NULL;
 706	const char *disasm_opt = NULL;
 707	struct dump_data dd = {};
 708	void *func_info = NULL;
 709	struct btf *btf = NULL;
 710	char func_sig[1024];
 711	unsigned char *buf;
 712	__u32 member_len;
 713	int fd, err = -1;
 714	ssize_t n;
 715
 716	if (mode == DUMP_JITED) {
 717		if (info->jited_prog_len == 0 || !info->jited_prog_insns) {
 718			p_err("error retrieving jit dump: no instructions returned or kernel.kptr_restrict set?");
 719			return -1;
 720		}
 721		buf = u64_to_ptr(info->jited_prog_insns);
 722		member_len = info->jited_prog_len;
 723	} else {	/* DUMP_XLATED */
 724		if (info->xlated_prog_len == 0 || !info->xlated_prog_insns) {
 725			p_err("error retrieving insn dump: kernel.kptr_restrict set?");
 726			return -1;
 727		}
 728		buf = u64_to_ptr(info->xlated_prog_insns);
 729		member_len = info->xlated_prog_len;
 730	}
 731
 732	if (info->btf_id) {
 733		btf = btf__load_from_kernel_by_id(info->btf_id);
 734		if (!btf) {
 735			p_err("failed to get btf");
 736			return -1;
 737		}
 738	}
 739
 740	func_info = u64_to_ptr(info->func_info);
 741
 742	if (info->nr_line_info) {
 743		prog_linfo = bpf_prog_linfo__new(info);
 744		if (!prog_linfo)
 745			p_info("error in processing bpf_line_info.  continue without it.");
 746	}
 747
 748	if (filepath) {
 749		fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600);
 750		if (fd < 0) {
 751			p_err("can't open file %s: %s", filepath,
 752			      strerror(errno));
 753			goto exit_free;
 754		}
 755
 756		n = write(fd, buf, member_len);
 757		close(fd);
 758		if (n != (ssize_t)member_len) {
 759			p_err("error writing output file: %s",
 760			      n < 0 ? strerror(errno) : "short write");
 761			goto exit_free;
 762		}
 763
 764		if (json_output)
 765			jsonw_null(json_wtr);
 766	} else if (mode == DUMP_JITED) {
 767		const char *name = NULL;
 768
 769		if (info->ifindex) {
 770			name = ifindex_to_arch(info->ifindex, info->netns_dev,
 771					       info->netns_ino, &disasm_opt);
 772			if (!name)
 773				goto exit_free;
 774		}
 775
 776		if (info->nr_jited_func_lens && info->jited_func_lens) {
 777			struct kernel_sym *sym = NULL;
 778			struct bpf_func_info *record;
 779			char sym_name[SYM_MAX_NAME];
 780			unsigned char *img = buf;
 781			__u64 *ksyms = NULL;
 782			__u32 *lens;
 783			__u32 i;
 784			if (info->nr_jited_ksyms) {
 785				kernel_syms_load(&dd);
 786				ksyms = u64_to_ptr(info->jited_ksyms);
 787			}
 788
 789			if (json_output)
 790				jsonw_start_array(json_wtr);
 791
 792			lens = u64_to_ptr(info->jited_func_lens);
 793			for (i = 0; i < info->nr_jited_func_lens; i++) {
 794				if (ksyms) {
 795					sym = kernel_syms_search(&dd, ksyms[i]);
 796					if (sym)
 797						sprintf(sym_name, "%s", sym->name);
 798					else
 799						sprintf(sym_name, "0x%016llx", ksyms[i]);
 800				} else {
 801					strcpy(sym_name, "unknown");
 802				}
 803
 804				if (func_info) {
 805					record = func_info + i * info->func_info_rec_size;
 806					btf_dumper_type_only(btf, record->type_id,
 807							     func_sig,
 808							     sizeof(func_sig));
 809				}
 810
 811				if (json_output) {
 812					jsonw_start_object(json_wtr);
 813					if (func_info && func_sig[0] != '\0') {
 814						jsonw_name(json_wtr, "proto");
 815						jsonw_string(json_wtr, func_sig);
 816					}
 817					jsonw_name(json_wtr, "name");
 818					jsonw_string(json_wtr, sym_name);
 819					jsonw_name(json_wtr, "insns");
 820				} else {
 821					if (func_info && func_sig[0] != '\0')
 822						printf("%s:\n", func_sig);
 823					printf("%s:\n", sym_name);
 824				}
 825
 826				if (ksyms) {
 827					if (disasm_print_insn(img, lens[i], opcodes,
 828							      name, disasm_opt, btf,
 829							      prog_linfo, ksyms[i], i,
 830							      linum))
 831						goto exit_free;
 832				} else {
 833					if (disasm_print_insn(img, lens[i], opcodes,
 834							      name, disasm_opt, btf,
 835							      NULL, 0, 0, false))
 836						goto exit_free;
 837				}
 838
 839				img += lens[i];
 840
 841				if (json_output)
 842					jsonw_end_object(json_wtr);
 843				else
 844					printf("\n");
 845			}
 846
 847			if (json_output)
 848				jsonw_end_array(json_wtr);
 849		} else {
 850			if (disasm_print_insn(buf, member_len, opcodes, name,
 851					      disasm_opt, btf, NULL, 0, 0,
 852					      false))
 853				goto exit_free;
 854		}
 855	} else {
 856		kernel_syms_load(&dd);
 857		dd.nr_jited_ksyms = info->nr_jited_ksyms;
 858		dd.jited_ksyms = u64_to_ptr(info->jited_ksyms);
 859		dd.btf = btf;
 860		dd.func_info = func_info;
 861		dd.finfo_rec_size = info->func_info_rec_size;
 862		dd.prog_linfo = prog_linfo;
 863
 864		if (json_output)
 865			dump_xlated_json(&dd, buf, member_len, opcodes, linum);
 866		else if (visual)
 867			dump_xlated_cfg(&dd, buf, member_len, opcodes, linum);
 868		else
 869			dump_xlated_plain(&dd, buf, member_len, opcodes, linum);
 870		kernel_syms_destroy(&dd);
 871	}
 872
 873	err = 0;
 874
 875exit_free:
 876	btf__free(btf);
 877	bpf_prog_linfo__free(prog_linfo);
 878	return err;
 879}
 880
 881static int do_dump(int argc, char **argv)
 882{
 883	struct bpf_prog_info info;
 884	__u32 info_len = sizeof(info);
 885	size_t info_data_sz = 0;
 886	void *info_data = NULL;
 887	char *filepath = NULL;
 888	bool opcodes = false;
 889	bool visual = false;
 890	enum dump_mode mode;
 891	bool linum = false;
 892	int nb_fds, i = 0;
 893	int *fds = NULL;
 894	int err = -1;
 895
 896	if (is_prefix(*argv, "jited")) {
 897		if (disasm_init())
 898			return -1;
 899		mode = DUMP_JITED;
 900	} else if (is_prefix(*argv, "xlated")) {
 901		mode = DUMP_XLATED;
 902	} else {
 903		p_err("expected 'xlated' or 'jited', got: %s", *argv);
 904		return -1;
 905	}
 906	NEXT_ARG();
 907
 908	if (argc < 2)
 909		usage();
 910
 911	fds = malloc(sizeof(int));
 912	if (!fds) {
 913		p_err("mem alloc failed");
 914		return -1;
 915	}
 916	nb_fds = prog_parse_fds(&argc, &argv, &fds);
 917	if (nb_fds < 1)
 918		goto exit_free;
 919
 920	while (argc) {
 921		if (is_prefix(*argv, "file")) {
 922			NEXT_ARG();
 923			if (!argc) {
 924				p_err("expected file path");
 925				goto exit_close;
 926			}
 927			if (nb_fds > 1) {
 928				p_err("several programs matched");
 929				goto exit_close;
 930			}
 931
 932			filepath = *argv;
 933			NEXT_ARG();
 934		} else if (is_prefix(*argv, "opcodes")) {
 935			opcodes = true;
 936			NEXT_ARG();
 937		} else if (is_prefix(*argv, "visual")) {
 938			if (nb_fds > 1) {
 939				p_err("several programs matched");
 940				goto exit_close;
 941			}
 942
 943			visual = true;
 944			NEXT_ARG();
 945		} else if (is_prefix(*argv, "linum")) {
 946			linum = true;
 947			NEXT_ARG();
 948		} else {
 949			usage();
 950			goto exit_close;
 951		}
 952	}
 953
 954	if (filepath && (opcodes || visual || linum)) {
 955		p_err("'file' is not compatible with 'opcodes', 'visual', or 'linum'");
 956		goto exit_close;
 957	}
 958	if (json_output && visual) {
 959		p_err("'visual' is not compatible with JSON output");
 960		goto exit_close;
 961	}
 962
 963	if (json_output && nb_fds > 1)
 964		jsonw_start_array(json_wtr);	/* root array */
 965	for (i = 0; i < nb_fds; i++) {
 966		memset(&info, 0, sizeof(info));
 967
 968		err = bpf_prog_get_info_by_fd(fds[i], &info, &info_len);
 969		if (err) {
 970			p_err("can't get prog info: %s", strerror(errno));
 971			break;
 972		}
 973
 974		err = prep_prog_info(&info, mode, &info_data, &info_data_sz);
 975		if (err) {
 976			p_err("can't grow prog info_data");
 977			break;
 978		}
 979
 980		err = bpf_prog_get_info_by_fd(fds[i], &info, &info_len);
 981		if (err) {
 982			p_err("can't get prog info: %s", strerror(errno));
 983			break;
 984		}
 985
 986		if (json_output && nb_fds > 1) {
 987			jsonw_start_object(json_wtr);	/* prog object */
 988			print_prog_header_json(&info, fds[i]);
 989			jsonw_name(json_wtr, "insns");
 990		} else if (nb_fds > 1) {
 991			print_prog_header_plain(&info, fds[i]);
 992		}
 993
 994		err = prog_dump(&info, mode, filepath, opcodes, visual, linum);
 995
 996		if (json_output && nb_fds > 1)
 997			jsonw_end_object(json_wtr);	/* prog object */
 998		else if (i != nb_fds - 1 && nb_fds > 1)
 999			printf("\n");
1000
1001		if (err)
1002			break;
1003		close(fds[i]);
1004	}
1005	if (json_output && nb_fds > 1)
1006		jsonw_end_array(json_wtr);	/* root array */
1007
1008exit_close:
1009	for (; i < nb_fds; i++)
1010		close(fds[i]);
1011exit_free:
1012	free(info_data);
1013	free(fds);
1014	return err;
1015}
1016
1017static int do_pin(int argc, char **argv)
1018{
1019	int err;
1020
1021	err = do_pin_any(argc, argv, prog_parse_fd);
1022	if (!err && json_output)
1023		jsonw_null(json_wtr);
1024	return err;
1025}
1026
1027struct map_replace {
1028	int idx;
1029	int fd;
1030	char *name;
1031};
1032
1033static int map_replace_compar(const void *p1, const void *p2)
1034{
1035	const struct map_replace *a = p1, *b = p2;
1036
1037	return a->idx - b->idx;
1038}
1039
1040static int parse_attach_detach_args(int argc, char **argv, int *progfd,
1041				    enum bpf_attach_type *attach_type,
1042				    int *mapfd)
1043{
1044	if (!REQ_ARGS(3))
1045		return -EINVAL;
1046
1047	*progfd = prog_parse_fd(&argc, &argv);
1048	if (*progfd < 0)
1049		return *progfd;
1050
1051	*attach_type = parse_attach_type(*argv);
1052	if (*attach_type == __MAX_BPF_ATTACH_TYPE) {
1053		p_err("invalid attach/detach type");
1054		return -EINVAL;
1055	}
1056
1057	if (*attach_type == BPF_FLOW_DISSECTOR) {
1058		*mapfd = 0;
1059		return 0;
1060	}
1061
1062	NEXT_ARG();
1063	if (!REQ_ARGS(2))
1064		return -EINVAL;
1065
1066	*mapfd = map_parse_fd(&argc, &argv, 0);
1067	if (*mapfd < 0)
1068		return *mapfd;
1069
1070	return 0;
1071}
1072
1073static int do_attach(int argc, char **argv)
1074{
1075	enum bpf_attach_type attach_type;
1076	int err, progfd;
1077	int mapfd;
1078
1079	err = parse_attach_detach_args(argc, argv,
1080				       &progfd, &attach_type, &mapfd);
1081	if (err)
1082		return err;
1083
1084	err = bpf_prog_attach(progfd, mapfd, attach_type, 0);
1085	if (err) {
1086		p_err("failed prog attach to map");
1087		return -EINVAL;
1088	}
1089
1090	if (json_output)
1091		jsonw_null(json_wtr);
1092	return 0;
1093}
1094
1095static int do_detach(int argc, char **argv)
1096{
1097	enum bpf_attach_type attach_type;
1098	int err, progfd;
1099	int mapfd;
1100
1101	err = parse_attach_detach_args(argc, argv,
1102				       &progfd, &attach_type, &mapfd);
1103	if (err)
1104		return err;
1105
1106	err = bpf_prog_detach2(progfd, mapfd, attach_type);
1107	if (err) {
1108		p_err("failed prog detach from map");
1109		return -EINVAL;
1110	}
1111
1112	if (json_output)
1113		jsonw_null(json_wtr);
1114	return 0;
1115}
1116
1117enum prog_tracelog_mode {
1118	TRACE_STDOUT,
1119	TRACE_STDERR,
1120};
1121
1122static int
1123prog_tracelog_stream(int prog_fd, enum prog_tracelog_mode mode)
1124{
1125	FILE *file = mode == TRACE_STDOUT ? stdout : stderr;
1126	int stream_id = mode == TRACE_STDOUT ? 1 : 2;
1127	char buf[512];
1128	int ret;
1129
1130	ret = 0;
1131	do {
1132		ret = bpf_prog_stream_read(prog_fd, stream_id, buf, sizeof(buf), NULL);
1133		if (ret > 0)
1134			fwrite(buf, sizeof(buf[0]), ret, file);
1135	} while (ret > 0);
1136
1137	fflush(file);
1138	return ret ? -1 : 0;
1139}
1140
1141static int do_tracelog_any(int argc, char **argv)
1142{
1143	enum prog_tracelog_mode mode;
1144	int fd;
1145
1146	if (argc == 0)
1147		return do_tracelog(argc, argv);
1148	if (!is_prefix(*argv, "stdout") && !is_prefix(*argv, "stderr"))
1149		usage();
1150	mode = is_prefix(*argv, "stdout") ? TRACE_STDOUT : TRACE_STDERR;
1151	NEXT_ARG();
1152
1153	if (!REQ_ARGS(2))
1154		return -1;
1155
1156	fd = prog_parse_fd(&argc, &argv);
1157	if (fd < 0)
1158		return -1;
1159
1160	return prog_tracelog_stream(fd, mode);
1161}
1162
1163static int check_single_stdin(char *file_data_in, char *file_ctx_in)
1164{
1165	if (file_data_in && file_ctx_in &&
1166	    !strcmp(file_data_in, "-") && !strcmp(file_ctx_in, "-")) {
1167		p_err("cannot use standard input for both data_in and ctx_in");
1168		return -1;
1169	}
1170
1171	return 0;
1172}
1173
1174static int get_run_data(const char *fname, void **data_ptr, unsigned int *size)
1175{
1176	size_t block_size = 256;
1177	size_t buf_size = block_size;
1178	size_t nb_read = 0;
1179	void *tmp;
1180	FILE *f;
1181
1182	if (!fname) {
1183		*data_ptr = NULL;
1184		*size = 0;
1185		return 0;
1186	}
1187
1188	if (!strcmp(fname, "-"))
1189		f = stdin;
1190	else
1191		f = fopen(fname, "r");
1192	if (!f) {
1193		p_err("failed to open %s: %s", fname, strerror(errno));
1194		return -1;
1195	}
1196
1197	*data_ptr = malloc(block_size);
1198	if (!*data_ptr) {
1199		p_err("failed to allocate memory for data_in/ctx_in: %s",
1200		      strerror(errno));
1201		goto err_fclose;
1202	}
1203
1204	while ((nb_read += fread(*data_ptr + nb_read, 1, block_size, f))) {
1205		if (feof(f))
1206			break;
1207		if (ferror(f)) {
1208			p_err("failed to read data_in/ctx_in from %s: %s",
1209			      fname, strerror(errno));
1210			goto err_free;
1211		}
1212		if (nb_read > buf_size - block_size) {
1213			if (buf_size == UINT32_MAX) {
1214				p_err("data_in/ctx_in is too long (max: %u)",
1215				      UINT32_MAX);
1216				goto err_free;
1217			}
1218			/* No space for fread()-ing next chunk; realloc() */
1219			buf_size *= 2;
1220			tmp = realloc(*data_ptr, buf_size);
1221			if (!tmp) {
1222				p_err("failed to reallocate data_in/ctx_in: %s",
1223				      strerror(errno));
1224				goto err_free;
1225			}
1226			*data_ptr = tmp;
1227		}
1228	}
1229	if (f != stdin)
1230		fclose(f);
1231
1232	*size = nb_read;
1233	return 0;
1234
1235err_free:
1236	free(*data_ptr);
1237	*data_ptr = NULL;
1238err_fclose:
1239	if (f != stdin)
1240		fclose(f);
1241	return -1;
1242}
1243
1244static void hex_print(void *data, unsigned int size, FILE *f)
1245{
1246	size_t i, j;
1247	char c;
1248
1249	for (i = 0; i < size; i += 16) {
1250		/* Row offset */
1251		fprintf(f, "%07zx\t", i);
1252
1253		/* Hexadecimal values */
1254		for (j = i; j < i + 16 && j < size; j++)
1255			fprintf(f, "%02x%s", *(uint8_t *)(data + j),
1256				j % 2 ? " " : "");
1257		for (; j < i + 16; j++)
1258			fprintf(f, "  %s", j % 2 ? " " : "");
1259
1260		/* ASCII values (if relevant), '.' otherwise */
1261		fprintf(f, "| ");
1262		for (j = i; j < i + 16 && j < size; j++) {
1263			c = *(char *)(data + j);
1264			if (c < ' ' || c > '~')
1265				c = '.';
1266			fprintf(f, "%c%s", c, j == i + 7 ? " " : "");
1267		}
1268
1269		fprintf(f, "\n");
1270	}
1271}
1272
1273static int
1274print_run_output(void *data, unsigned int size, const char *fname,
1275		 const char *json_key)
1276{
1277	size_t nb_written;
1278	FILE *f;
1279
1280	if (!fname)
1281		return 0;
1282
1283	if (!strcmp(fname, "-")) {
1284		f = stdout;
1285		if (json_output) {
1286			jsonw_name(json_wtr, json_key);
1287			print_data_json(data, size);
1288		} else {
1289			hex_print(data, size, f);
1290		}
1291		return 0;
1292	}
1293
1294	f = fopen(fname, "w");
1295	if (!f) {
1296		p_err("failed to open %s: %s", fname, strerror(errno));
1297		return -1;
1298	}
1299
1300	nb_written = fwrite(data, 1, size, f);
1301	fclose(f);
1302	if (nb_written != size) {
1303		p_err("failed to write output data/ctx: %s", strerror(errno));
1304		return -1;
1305	}
1306
1307	return 0;
1308}
1309
1310static int alloc_run_data(void **data_ptr, unsigned int size_out)
1311{
1312	*data_ptr = calloc(size_out, 1);
1313	if (!*data_ptr) {
1314		p_err("failed to allocate memory for output data/ctx: %s",
1315		      strerror(errno));
1316		return -1;
1317	}
1318
1319	return 0;
1320}
1321
1322static int do_run(int argc, char **argv)
1323{
1324	char *data_fname_in = NULL, *data_fname_out = NULL;
1325	char *ctx_fname_in = NULL, *ctx_fname_out = NULL;
1326	const unsigned int default_size = SZ_32K;
1327	void *data_in = NULL, *data_out = NULL;
1328	void *ctx_in = NULL, *ctx_out = NULL;
1329	unsigned int repeat = 1;
1330	int fd, err;
1331	LIBBPF_OPTS(bpf_test_run_opts, test_attr);
1332
1333	if (!REQ_ARGS(4))
1334		return -1;
1335
1336	fd = prog_parse_fd(&argc, &argv);
1337	if (fd < 0)
1338		return -1;
1339
1340	while (argc) {
1341		if (detect_common_prefix(*argv, "data_in", "data_out",
1342					 "data_size_out", NULL))
1343			return -1;
1344		if (detect_common_prefix(*argv, "ctx_in", "ctx_out",
1345					 "ctx_size_out", NULL))
1346			return -1;
1347
1348		if (is_prefix(*argv, "data_in")) {
1349			NEXT_ARG();
1350			if (!REQ_ARGS(1))
1351				return -1;
1352
1353			data_fname_in = GET_ARG();
1354			if (check_single_stdin(data_fname_in, ctx_fname_in))
1355				return -1;
1356		} else if (is_prefix(*argv, "data_out")) {
1357			NEXT_ARG();
1358			if (!REQ_ARGS(1))
1359				return -1;
1360
1361			data_fname_out = GET_ARG();
1362		} else if (is_prefix(*argv, "data_size_out")) {
1363			char *endptr;
1364
1365			NEXT_ARG();
1366			if (!REQ_ARGS(1))
1367				return -1;
1368
1369			test_attr.data_size_out = strtoul(*argv, &endptr, 0);
1370			if (*endptr) {
1371				p_err("can't parse %s as output data size",
1372				      *argv);
1373				return -1;
1374			}
1375			NEXT_ARG();
1376		} else if (is_prefix(*argv, "ctx_in")) {
1377			NEXT_ARG();
1378			if (!REQ_ARGS(1))
1379				return -1;
1380
1381			ctx_fname_in = GET_ARG();
1382			if (check_single_stdin(data_fname_in, ctx_fname_in))
1383				return -1;
1384		} else if (is_prefix(*argv, "ctx_out")) {
1385			NEXT_ARG();
1386			if (!REQ_ARGS(1))
1387				return -1;
1388
1389			ctx_fname_out = GET_ARG();
1390		} else if (is_prefix(*argv, "ctx_size_out")) {
1391			char *endptr;
1392
1393			NEXT_ARG();
1394			if (!REQ_ARGS(1))
1395				return -1;
1396
1397			test_attr.ctx_size_out = strtoul(*argv, &endptr, 0);
1398			if (*endptr) {
1399				p_err("can't parse %s as output context size",
1400				      *argv);
1401				return -1;
1402			}
1403			NEXT_ARG();
1404		} else if (is_prefix(*argv, "repeat")) {
1405			char *endptr;
1406
1407			NEXT_ARG();
1408			if (!REQ_ARGS(1))
1409				return -1;
1410
1411			repeat = strtoul(*argv, &endptr, 0);
1412			if (*endptr) {
1413				p_err("can't parse %s as repeat number",
1414				      *argv);
1415				return -1;
1416			}
1417			NEXT_ARG();
1418		} else {
1419			p_err("expected no more arguments, 'data_in', 'data_out', 'data_size_out', 'ctx_in', 'ctx_out', 'ctx_size_out' or 'repeat', got: '%s'?",
1420			      *argv);
1421			return -1;
1422		}
1423	}
1424
1425	err = get_run_data(data_fname_in, &data_in, &test_attr.data_size_in);
1426	if (err)
1427		return -1;
1428
1429	if (data_in) {
1430		if (!test_attr.data_size_out)
1431			test_attr.data_size_out = default_size;
1432		err = alloc_run_data(&data_out, test_attr.data_size_out);
1433		if (err)
1434			goto free_data_in;
1435	}
1436
1437	err = get_run_data(ctx_fname_in, &ctx_in, &test_attr.ctx_size_in);
1438	if (err)
1439		goto free_data_out;
1440
1441	if (ctx_in) {
1442		if (!test_attr.ctx_size_out)
1443			test_attr.ctx_size_out = default_size;
1444		err = alloc_run_data(&ctx_out, test_attr.ctx_size_out);
1445		if (err)
1446			goto free_ctx_in;
1447	}
1448
1449	test_attr.repeat	= repeat;
1450	test_attr.data_in	= data_in;
1451	test_attr.data_out	= data_out;
1452	test_attr.ctx_in	= ctx_in;
1453	test_attr.ctx_out	= ctx_out;
1454
1455	err = bpf_prog_test_run_opts(fd, &test_attr);
1456	if (err) {
1457		p_err("failed to run program: %s", strerror(errno));
1458		goto free_ctx_out;
1459	}
1460
1461	err = 0;
1462
1463	if (json_output)
1464		jsonw_start_object(json_wtr);	/* root */
1465
1466	/* Do not exit on errors occurring when printing output data/context,
1467	 * we still want to print return value and duration for program run.
1468	 */
1469	if (test_attr.data_size_out)
1470		err += print_run_output(test_attr.data_out,
1471					test_attr.data_size_out,
1472					data_fname_out, "data_out");
1473	if (test_attr.ctx_size_out)
1474		err += print_run_output(test_attr.ctx_out,
1475					test_attr.ctx_size_out,
1476					ctx_fname_out, "ctx_out");
1477
1478	if (json_output) {
1479		jsonw_uint_field(json_wtr, "retval", test_attr.retval);
1480		jsonw_uint_field(json_wtr, "duration", test_attr.duration);
1481		jsonw_end_object(json_wtr);	/* root */
1482	} else {
1483		fprintf(stdout, "Return value: %u, duration%s: %uns\n",
1484			test_attr.retval,
1485			repeat > 1 ? " (average)" : "", test_attr.duration);
1486	}
1487
1488free_ctx_out:
1489	free(ctx_out);
1490free_ctx_in:
1491	free(ctx_in);
1492free_data_out:
1493	free(data_out);
1494free_data_in:
1495	free(data_in);
1496
1497	return err;
1498}
1499
1500static int
1501get_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
1502		      enum bpf_attach_type *expected_attach_type)
1503{
1504	libbpf_print_fn_t print_backup;
1505	int ret;
1506
1507	ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type);
1508	if (!ret)
1509		return ret;
1510
1511	/* libbpf_prog_type_by_name() failed, let's re-run with debug level */
1512	print_backup = libbpf_set_print(print_all_levels);
1513	ret = libbpf_prog_type_by_name(name, prog_type, expected_attach_type);
1514	libbpf_set_print(print_backup);
1515
1516	return ret;
1517}
1518
1519static int
1520auto_attach_program(struct bpf_program *prog, const char *path)
1521{
1522	struct bpf_link *link;
1523	int err;
1524
1525	link = bpf_program__attach(prog);
1526	if (!link) {
1527		p_info("Program %s does not support autoattach, falling back to pinning",
1528		       bpf_program__name(prog));
1529		return bpf_obj_pin(bpf_program__fd(prog), path);
1530	}
1531
1532	err = bpf_link__pin(link, path);
1533	bpf_link__destroy(link);
1534	return err;
1535}
1536
1537static int
1538auto_attach_programs(struct bpf_object *obj, const char *path)
1539{
1540	struct bpf_program *prog;
1541	char buf[PATH_MAX];
1542	int err;
1543
1544	bpf_object__for_each_program(prog, obj) {
1545		err = pathname_concat(buf, sizeof(buf), path, bpf_program__name(prog));
1546		if (err)
1547			goto err_unpin_programs;
1548
1549		err = auto_attach_program(prog, buf);
1550		if (err)
1551			goto err_unpin_programs;
1552	}
1553
1554	return 0;
1555
1556err_unpin_programs:
1557	while ((prog = bpf_object__prev_program(obj, prog))) {
1558		if (pathname_concat(buf, sizeof(buf), path, bpf_program__name(prog)))
1559			continue;
1560
1561		bpf_program__unpin(prog, buf);
1562	}
1563
1564	return err;
1565}
1566
1567static int load_with_options(int argc, char **argv, bool first_prog_only)
1568{
1569	enum bpf_prog_type common_prog_type = BPF_PROG_TYPE_UNSPEC;
1570	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,
1571		.relaxed_maps = relaxed_maps,
1572	);
1573	enum bpf_attach_type expected_attach_type;
1574	struct map_replace *map_replace = NULL;
1575	struct bpf_program *prog = NULL, *pos;
1576	unsigned int old_map_fds = 0;
1577	const char *pinmaps = NULL;
1578	__u32 xdpmeta_ifindex = 0;
1579	__u32 offload_ifindex = 0;
1580	bool auto_attach = false;
1581	struct bpf_object *obj;
1582	struct bpf_map *map;
1583	const char *pinfile;
1584	unsigned int i, j;
1585	const char *file;
1586	int idx, err;
1587
1588
1589	if (!REQ_ARGS(2))
1590		return -1;
1591	file = GET_ARG();
1592	pinfile = GET_ARG();
1593
1594	while (argc) {
1595		if (is_prefix(*argv, "type")) {
1596			NEXT_ARG();
1597
1598			if (common_prog_type != BPF_PROG_TYPE_UNSPEC) {
1599				p_err("program type already specified");
1600				goto err_free_reuse_maps;
1601			}
1602			if (!REQ_ARGS(1))
1603				goto err_free_reuse_maps;
1604
1605			err = libbpf_prog_type_by_name(*argv, &common_prog_type,
1606						       &expected_attach_type);
1607			if (err < 0) {
1608				/* Put a '/' at the end of type to appease libbpf */
1609				char *type = malloc(strlen(*argv) + 2);
1610
1611				if (!type) {
1612					p_err("mem alloc failed");
1613					goto err_free_reuse_maps;
1614				}
1615				*type = 0;
1616				strcat(type, *argv);
1617				strcat(type, "/");
1618
1619				err = get_prog_type_by_name(type, &common_prog_type,
1620							    &expected_attach_type);
1621				free(type);
1622				if (err < 0)
1623					goto err_free_reuse_maps;
1624			}
1625
1626			NEXT_ARG();
1627		} else if (is_prefix(*argv, "map")) {
1628			void *new_map_replace;
1629			char *endptr, *name;
1630			int fd;
1631
1632			NEXT_ARG();
1633
1634			if (!REQ_ARGS(4))
1635				goto err_free_reuse_maps;
1636
1637			if (is_prefix(*argv, "idx")) {
1638				NEXT_ARG();
1639
1640				idx = strtoul(*argv, &endptr, 0);
1641				if (*endptr) {
1642					p_err("can't parse %s as IDX", *argv);
1643					goto err_free_reuse_maps;
1644				}
1645				name = NULL;
1646			} else if (is_prefix(*argv, "name")) {
1647				NEXT_ARG();
1648
1649				name = *argv;
1650				idx = -1;
1651			} else {
1652				p_err("expected 'idx' or 'name', got: '%s'?",
1653				      *argv);
1654				goto err_free_reuse_maps;
1655			}
1656			NEXT_ARG();
1657
1658			fd = map_parse_fd(&argc, &argv, 0);
1659			if (fd < 0)
1660				goto err_free_reuse_maps;
1661
1662			new_map_replace = libbpf_reallocarray(map_replace,
1663							      old_map_fds + 1,
1664							      sizeof(*map_replace));
1665			if (!new_map_replace) {
1666				p_err("mem alloc failed");
1667				goto err_free_reuse_maps;
1668			}
1669			map_replace = new_map_replace;
1670
1671			map_replace[old_map_fds].idx = idx;
1672			map_replace[old_map_fds].name = name;
1673			map_replace[old_map_fds].fd = fd;
1674			old_map_fds++;
1675		} else if (is_prefix(*argv, "dev")) {
1676			p_info("Warning: 'bpftool prog load [...] dev <ifname>' syntax is deprecated.\n"
1677			       "Going further, please use 'offload_dev <ifname>' to offload program to device.\n"
1678			       "For applications using XDP hints only, use 'xdpmeta_dev <ifname>'.");
1679			goto offload_dev;
1680		} else if (is_prefix(*argv, "offload_dev")) {
1681offload_dev:
1682			NEXT_ARG();
1683
1684			if (offload_ifindex) {
1685				p_err("offload_dev already specified");
1686				goto err_free_reuse_maps;
1687			} else if (xdpmeta_ifindex) {
1688				p_err("xdpmeta_dev and offload_dev are mutually exclusive");
1689				goto err_free_reuse_maps;
1690			}
1691			if (!REQ_ARGS(1))
1692				goto err_free_reuse_maps;
1693
1694			offload_ifindex = if_nametoindex(*argv);
1695			if (!offload_ifindex) {
1696				p_err("unrecognized netdevice '%s': %s",
1697				      *argv, strerror(errno));
1698				goto err_free_reuse_maps;
1699			}
1700			NEXT_ARG();
1701		} else if (is_prefix(*argv, "xdpmeta_dev")) {
1702			NEXT_ARG();
1703
1704			if (xdpmeta_ifindex) {
1705				p_err("xdpmeta_dev already specified");
1706				goto err_free_reuse_maps;
1707			} else if (offload_ifindex) {
1708				p_err("xdpmeta_dev and offload_dev are mutually exclusive");
1709				goto err_free_reuse_maps;
1710			}
1711			if (!REQ_ARGS(1))
1712				goto err_free_reuse_maps;
1713
1714			xdpmeta_ifindex = if_nametoindex(*argv);
1715			if (!xdpmeta_ifindex) {
1716				p_err("unrecognized netdevice '%s': %s",
1717				      *argv, strerror(errno));
1718				goto err_free_reuse_maps;
1719			}
1720			NEXT_ARG();
1721		} else if (is_prefix(*argv, "pinmaps")) {
1722			NEXT_ARG();
1723
1724			if (!REQ_ARGS(1))
1725				goto err_free_reuse_maps;
1726
1727			pinmaps = GET_ARG();
1728		} else if (is_prefix(*argv, "autoattach")) {
1729			auto_attach = true;
1730			NEXT_ARG();
1731		} else if (is_prefix(*argv, "kernel_btf")) {
1732			NEXT_ARG();
1733
1734			if (!REQ_ARGS(1))
1735				goto err_free_reuse_maps;
1736
1737			open_opts.btf_custom_path = GET_ARG();
1738		} else {
1739			p_err("expected no more arguments, "
1740			      "'type', 'map', 'offload_dev', 'xdpmeta_dev', 'pinmaps', "
1741			      "'autoattach', or 'kernel_btf', got: '%s'?",
1742			      *argv);
1743			goto err_free_reuse_maps;
1744		}
1745	}
1746
1747	set_max_rlimit();
1748
1749	if (verifier_logs)
1750		/* log_level1 + log_level2 + stats, but not stable UAPI */
1751		open_opts.kernel_log_level = 1 + 2 + 4;
1752
1753	obj = bpf_object__open_file(file, &open_opts);
1754	if (!obj) {
1755		p_err("failed to open object file");
1756		goto err_free_reuse_maps;
1757	}
1758
1759	bpf_object__for_each_program(pos, obj) {
1760		enum bpf_prog_type prog_type = common_prog_type;
1761
1762		if (prog_type == BPF_PROG_TYPE_UNSPEC) {
1763			const char *sec_name = bpf_program__section_name(pos);
1764
1765			err = get_prog_type_by_name(sec_name, &prog_type,
1766						    &expected_attach_type);
1767			if (err < 0)
1768				goto err_close_obj;
1769		}
1770
1771		if (prog_type == BPF_PROG_TYPE_XDP && xdpmeta_ifindex) {
1772			bpf_program__set_flags(pos, BPF_F_XDP_DEV_BOUND_ONLY);
1773			bpf_program__set_ifindex(pos, xdpmeta_ifindex);
1774		} else {
1775			bpf_program__set_ifindex(pos, offload_ifindex);
1776		}
1777		if (bpf_program__type(pos) != prog_type)
1778			bpf_program__set_type(pos, prog_type);
1779		bpf_program__set_expected_attach_type(pos, expected_attach_type);
1780	}
1781
1782	qsort(map_replace, old_map_fds, sizeof(*map_replace),
1783	      map_replace_compar);
1784
1785	/* After the sort maps by name will be first on the list, because they
1786	 * have idx == -1.  Resolve them.
1787	 */
1788	j = 0;
1789	while (j < old_map_fds && map_replace[j].name) {
1790		i = 0;
1791		bpf_object__for_each_map(map, obj) {
1792			if (!strcmp(bpf_map__name(map), map_replace[j].name)) {
1793				map_replace[j].idx = i;
1794				break;
1795			}
1796			i++;
1797		}
1798		if (map_replace[j].idx == -1) {
1799			p_err("unable to find map '%s'", map_replace[j].name);
1800			goto err_close_obj;
1801		}
1802		j++;
1803	}
1804	/* Resort if any names were resolved */
1805	if (j)
1806		qsort(map_replace, old_map_fds, sizeof(*map_replace),
1807		      map_replace_compar);
1808
1809	/* Set ifindex and name reuse */
1810	j = 0;
1811	idx = 0;
1812	bpf_object__for_each_map(map, obj) {
1813		if (bpf_map__type(map) != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
1814			bpf_map__set_ifindex(map, offload_ifindex);
1815
1816		if (j < old_map_fds && idx == map_replace[j].idx) {
1817			err = bpf_map__reuse_fd(map, map_replace[j++].fd);
1818			if (err) {
1819				p_err("unable to set up map reuse: %d", err);
1820				goto err_close_obj;
1821			}
1822
1823			/* Next reuse wants to apply to the same map */
1824			if (j < old_map_fds && map_replace[j].idx == idx) {
1825				p_err("replacement for map idx %d specified more than once",
1826				      idx);
1827				goto err_close_obj;
1828			}
1829		}
1830
1831		idx++;
1832	}
1833	if (j < old_map_fds) {
1834		p_err("map idx '%d' not used", map_replace[j].idx);
1835		goto err_close_obj;
1836	}
1837
1838	err = bpf_object__load(obj);
1839	if (err) {
1840		p_err("failed to load object file");
1841		goto err_close_obj;
1842	}
1843
1844	if (first_prog_only)
1845		err = mount_bpffs_for_file(pinfile);
1846	else
1847		err = create_and_mount_bpffs_dir(pinfile);
1848	if (err)
1849		goto err_close_obj;
1850
1851	if (first_prog_only) {
1852		prog = bpf_object__next_program(obj, NULL);
1853		if (!prog) {
1854			p_err("object file doesn't contain any bpf program");
1855			goto err_close_obj;
1856		}
1857
1858		if (auto_attach)
1859			err = auto_attach_program(prog, pinfile);
1860		else
1861			err = bpf_obj_pin(bpf_program__fd(prog), pinfile);
1862		if (err) {
1863			p_err("failed to pin program %s",
1864			      bpf_program__section_name(prog));
1865			goto err_close_obj;
1866		}
1867	} else {
1868		if (auto_attach)
1869			err = auto_attach_programs(obj, pinfile);
1870		else
1871			err = bpf_object__pin_programs(obj, pinfile);
1872		if (err) {
1873			p_err("failed to pin all programs");
1874			goto err_close_obj;
1875		}
1876	}
1877
1878	if (pinmaps) {
1879		err = create_and_mount_bpffs_dir(pinmaps);
1880		if (err)
1881			goto err_unpin;
1882
1883		err = bpf_object__pin_maps(obj, pinmaps);
1884		if (err) {
1885			p_err("failed to pin all maps");
1886			goto err_unpin;
1887		}
1888	}
1889
1890	if (json_output)
1891		jsonw_null(json_wtr);
1892
1893	bpf_object__close(obj);
1894	for (i = 0; i < old_map_fds; i++)
1895		close(map_replace[i].fd);
1896	free(map_replace);
1897
1898	return 0;
1899
1900err_unpin:
1901	if (first_prog_only)
1902		unlink(pinfile);
1903	else
1904		bpf_object__unpin_programs(obj, pinfile);
1905err_close_obj:
1906	bpf_object__close(obj);
1907err_free_reuse_maps:
1908	for (i = 0; i < old_map_fds; i++)
1909		close(map_replace[i].fd);
1910	free(map_replace);
1911	return -1;
1912}
1913
1914static int count_open_fds(void)
1915{
1916	DIR *dp = opendir("/proc/self/fd");
1917	struct dirent *de;
1918	int cnt = -3;
1919
1920	if (!dp)
1921		return -1;
1922
1923	while ((de = readdir(dp)))
1924		cnt++;
1925
1926	closedir(dp);
1927	return cnt;
1928}
1929
1930static int try_loader(struct gen_loader_opts *gen)
1931{
1932	struct bpf_load_and_run_opts opts = {};
1933	struct bpf_loader_ctx *ctx;
1934	char sig_buf[MAX_SIG_SIZE];
1935	__u8 prog_sha[SHA256_DIGEST_LENGTH];
1936	int ctx_sz = sizeof(*ctx) + 64 * max(sizeof(struct bpf_map_desc),
1937					     sizeof(struct bpf_prog_desc));
1938	int log_buf_sz = (1u << 24) - 1;
1939	int err, fds_before, fd_delta;
1940	char *log_buf = NULL;
1941
1942	ctx = alloca(ctx_sz);
1943	memset(ctx, 0, ctx_sz);
1944	ctx->sz = ctx_sz;
1945	if (verifier_logs) {
1946		ctx->log_level = 1 + 2 + 4;
1947		ctx->log_size = log_buf_sz;
1948		log_buf = malloc(log_buf_sz);
1949		if (!log_buf)
1950			return -ENOMEM;
1951		ctx->log_buf = (long) log_buf;
1952	}
1953	opts.ctx = ctx;
1954	opts.data = gen->data;
1955	opts.data_sz = gen->data_sz;
1956	opts.insns = gen->insns;
1957	opts.insns_sz = gen->insns_sz;
1958	fds_before = count_open_fds();
1959
1960	if (sign_progs) {
1961		opts.excl_prog_hash = prog_sha;
1962		opts.excl_prog_hash_sz = sizeof(prog_sha);
1963		opts.signature = sig_buf;
1964		opts.signature_sz = MAX_SIG_SIZE;
1965		opts.keyring_id = KEY_SPEC_SESSION_KEYRING;
1966
1967		err = bpftool_prog_sign(&opts);
1968		if (err < 0) {
1969			p_err("failed to sign program");
1970			goto out;
1971		}
1972
1973		err = register_session_key(cert_path);
1974		if (err < 0) {
1975			p_err("failed to add session key");
1976			goto out;
1977		}
1978	}
1979	err = bpf_load_and_run(&opts);
1980	fd_delta = count_open_fds() - fds_before;
1981	if (err < 0 || verifier_logs) {
1982		fprintf(stderr, "err %d\n%s\n%s", err, opts.errstr, log_buf);
1983		if (fd_delta && err < 0)
1984			fprintf(stderr, "loader prog leaked %d FDs\n",
1985				fd_delta);
1986	}
1987out:
1988	free(log_buf);
1989	return err;
1990}
1991
1992static int do_loader(int argc, char **argv)
1993{
1994	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts);
1995	DECLARE_LIBBPF_OPTS(gen_loader_opts, gen);
1996	struct bpf_object *obj;
1997	const char *file;
1998	int err = 0;
1999
2000	if (!REQ_ARGS(1))
2001		return -1;
2002	file = GET_ARG();
2003
2004	if (verifier_logs)
2005		/* log_level1 + log_level2 + stats, but not stable UAPI */
2006		open_opts.kernel_log_level = 1 + 2 + 4;
2007
2008	obj = bpf_object__open_file(file, &open_opts);
2009	if (!obj) {
2010		err = -1;
2011		p_err("failed to open object file");
2012		goto err_close_obj;
2013	}
2014
2015	if (sign_progs)
2016		gen.gen_hash = true;
2017
2018	err = bpf_object__gen_loader(obj, &gen);
2019	if (err)
2020		goto err_close_obj;
2021
2022	err = bpf_object__load(obj);
2023	if (err) {
2024		p_err("failed to load object file");
2025		goto err_close_obj;
2026	}
2027
2028	if (verifier_logs) {
2029		struct dump_data dd = {};
2030
2031		kernel_syms_load(&dd);
2032		dump_xlated_plain(&dd, (void *)gen.insns, gen.insns_sz, false, false);
2033		kernel_syms_destroy(&dd);
2034	}
2035	err = try_loader(&gen);
2036err_close_obj:
2037	bpf_object__close(obj);
2038	return err;
2039}
2040
2041static int do_load(int argc, char **argv)
2042{
2043	if (use_loader)
2044		return do_loader(argc, argv);
2045	return load_with_options(argc, argv, true);
2046}
2047
2048static int do_loadall(int argc, char **argv)
2049{
2050	return load_with_options(argc, argv, false);
2051}
2052
2053#ifdef BPFTOOL_WITHOUT_SKELETONS
2054
2055static int do_profile(int argc, char **argv)
2056{
2057	p_err("bpftool prog profile command is not supported. Please build bpftool with clang >= 10.0.0");
2058	return 0;
2059}
2060
2061#else /* BPFTOOL_WITHOUT_SKELETONS */
2062
2063#include "profiler.skel.h"
2064
2065struct profile_metric {
2066	const char *name;
2067	struct bpf_perf_event_value val;
2068	struct perf_event_attr attr;
2069	bool selected;
2070
2071	/* calculate ratios like instructions per cycle */
2072	const int ratio_metric; /* 0 for N/A, 1 for index 0 (cycles) */
2073	const char *ratio_desc;
2074	const float ratio_mul;
2075} metrics[] = {
2076	{
2077		.name = "cycles",
2078		.attr = {
2079			.type = PERF_TYPE_HARDWARE,
2080			.config = PERF_COUNT_HW_CPU_CYCLES,
2081			.exclude_user = 1,
2082		},
2083	},
2084	{
2085		.name = "instructions",
2086		.attr = {
2087			.type = PERF_TYPE_HARDWARE,
2088			.config = PERF_COUNT_HW_INSTRUCTIONS,
2089			.exclude_user = 1,
2090		},
2091		.ratio_metric = 1,
2092		.ratio_desc = "insns per cycle",
2093		.ratio_mul = 1.0,
2094	},
2095	{
2096		.name = "l1d_loads",
2097		.attr = {
2098			.type = PERF_TYPE_HW_CACHE,
2099			.config =
2100				PERF_COUNT_HW_CACHE_L1D |
2101				(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2102				(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16),
2103			.exclude_user = 1,
2104		},
2105	},
2106	{
2107		.name = "llc_misses",
2108		.attr = {
2109			.type = PERF_TYPE_HW_CACHE,
2110			.config =
2111				PERF_COUNT_HW_CACHE_LL |
2112				(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2113				(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2114			.exclude_user = 1
2115		},
2116		.ratio_metric = 2,
2117		.ratio_desc = "LLC misses per million insns",
2118		.ratio_mul = 1e6,
2119	},
2120	{
2121		.name = "itlb_misses",
2122		.attr = {
2123			.type = PERF_TYPE_HW_CACHE,
2124			.config =
2125				PERF_COUNT_HW_CACHE_ITLB |
2126				(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2127				(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2128			.exclude_user = 1
2129		},
2130		.ratio_metric = 2,
2131		.ratio_desc = "itlb misses per million insns",
2132		.ratio_mul = 1e6,
2133	},
2134	{
2135		.name = "dtlb_misses",
2136		.attr = {
2137			.type = PERF_TYPE_HW_CACHE,
2138			.config =
2139				PERF_COUNT_HW_CACHE_DTLB |
2140				(PERF_COUNT_HW_CACHE_OP_READ << 8) |
2141				(PERF_COUNT_HW_CACHE_RESULT_MISS << 16),
2142			.exclude_user = 1
2143		},
2144		.ratio_metric = 2,
2145		.ratio_desc = "dtlb misses per million insns",
2146		.ratio_mul = 1e6,
2147	},
2148};
2149
2150static __u64 profile_total_count;
2151
2152#define MAX_NUM_PROFILE_METRICS 4
2153
2154static int profile_parse_metrics(int argc, char **argv)
2155{
2156	unsigned int metric_cnt;
2157	int selected_cnt = 0;
2158	unsigned int i;
2159
2160	metric_cnt = ARRAY_SIZE(metrics);
2161
2162	while (argc > 0) {
2163		for (i = 0; i < metric_cnt; i++) {
2164			if (is_prefix(argv[0], metrics[i].name)) {
2165				if (!metrics[i].selected)
2166					selected_cnt++;
2167				metrics[i].selected = true;
2168				break;
2169			}
2170		}
2171		if (i == metric_cnt) {
2172			p_err("unknown metric %s", argv[0]);
2173			return -1;
2174		}
2175		NEXT_ARG();
2176	}
2177	if (selected_cnt > MAX_NUM_PROFILE_METRICS) {
2178		p_err("too many (%d) metrics, please specify no more than %d metrics at a time",
2179		      selected_cnt, MAX_NUM_PROFILE_METRICS);
2180		return -1;
2181	}
2182	return selected_cnt;
2183}
2184
2185static void profile_read_values(struct profiler_bpf *obj)
2186{
2187	__u32 m, cpu, num_cpu = obj->rodata->num_cpu;
2188	int reading_map_fd, count_map_fd;
2189	__u64 counts[num_cpu];
2190	__u32 key = 0;
2191	int err;
2192
2193	reading_map_fd = bpf_map__fd(obj->maps.accum_readings);
2194	count_map_fd = bpf_map__fd(obj->maps.counts);
2195	if (reading_map_fd < 0 || count_map_fd < 0) {
2196		p_err("failed to get fd for map");
2197		return;
2198	}
2199
2200	err = bpf_map_lookup_elem(count_map_fd, &key, counts);
2201	if (err) {
2202		p_err("failed to read count_map: %s", strerror(errno));
2203		return;
2204	}
2205
2206	profile_total_count = 0;
2207	for (cpu = 0; cpu < num_cpu; cpu++)
2208		profile_total_count += counts[cpu];
2209
2210	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2211		struct bpf_perf_event_value values[num_cpu];
2212
2213		if (!metrics[m].selected)
2214			continue;
2215
2216		err = bpf_map_lookup_elem(reading_map_fd, &key, values);
2217		if (err) {
2218			p_err("failed to read reading_map: %s",
2219			      strerror(errno));
2220			return;
2221		}
2222		for (cpu = 0; cpu < num_cpu; cpu++) {
2223			metrics[m].val.counter += values[cpu].counter;
2224			metrics[m].val.enabled += values[cpu].enabled;
2225			metrics[m].val.running += values[cpu].running;
2226		}
2227		key++;
2228	}
2229}
2230
2231static void profile_print_readings_json(void)
2232{
2233	__u32 m;
2234
2235	jsonw_start_array(json_wtr);
2236	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2237		if (!metrics[m].selected)
2238			continue;
2239		jsonw_start_object(json_wtr);
2240		jsonw_string_field(json_wtr, "metric", metrics[m].name);
2241		jsonw_lluint_field(json_wtr, "run_cnt", profile_total_count);
2242		jsonw_lluint_field(json_wtr, "value", metrics[m].val.counter);
2243		jsonw_lluint_field(json_wtr, "enabled", metrics[m].val.enabled);
2244		jsonw_lluint_field(json_wtr, "running", metrics[m].val.running);
2245
2246		jsonw_end_object(json_wtr);
2247	}
2248	jsonw_end_array(json_wtr);
2249}
2250
2251static void profile_print_readings_plain(void)
2252{
2253	__u32 m;
2254
2255	printf("\n%18llu %-20s\n", profile_total_count, "run_cnt");
2256	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2257		struct bpf_perf_event_value *val = &metrics[m].val;
2258		int r;
2259
2260		if (!metrics[m].selected)
2261			continue;
2262		printf("%18llu %-20s", val->counter, metrics[m].name);
2263
2264		r = metrics[m].ratio_metric - 1;
2265		if (r >= 0 && metrics[r].selected &&
2266		    metrics[r].val.counter > 0) {
2267			printf("# %8.2f %-30s",
2268			       val->counter * metrics[m].ratio_mul /
2269			       metrics[r].val.counter,
2270			       metrics[m].ratio_desc);
2271		} else {
2272			printf("%-41s", "");
2273		}
2274
2275		if (val->enabled > val->running)
2276			printf("(%4.2f%%)",
2277			       val->running * 100.0 / val->enabled);
2278		printf("\n");
2279	}
2280}
2281
2282static void profile_print_readings(void)
2283{
2284	if (json_output)
2285		profile_print_readings_json();
2286	else
2287		profile_print_readings_plain();
2288}
2289
2290static char *profile_target_name(int tgt_fd)
2291{
2292	struct bpf_func_info func_info = {};
2293	struct bpf_prog_info info = {};
2294	__u32 info_len = sizeof(info);
2295	const struct btf_type *t;
2296	__u32 func_info_rec_size;
2297	struct btf *btf = NULL;
2298	char *name = NULL;
2299	int err;
2300
2301	err = bpf_prog_get_info_by_fd(tgt_fd, &info, &info_len);
2302	if (err) {
2303		p_err("failed to get info for prog FD %d", tgt_fd);
2304		goto out;
2305	}
2306
2307	if (info.btf_id == 0) {
2308		p_err("prog FD %d doesn't have valid btf", tgt_fd);
2309		goto out;
2310	}
2311
2312	func_info_rec_size = info.func_info_rec_size;
2313	if (info.nr_func_info == 0) {
2314		p_err("found 0 func_info for prog FD %d", tgt_fd);
2315		goto out;
2316	}
2317
2318	memset(&info, 0, sizeof(info));
2319	info.nr_func_info = 1;
2320	info.func_info_rec_size = func_info_rec_size;
2321	info.func_info = ptr_to_u64(&func_info);
2322
2323	err = bpf_prog_get_info_by_fd(tgt_fd, &info, &info_len);
2324	if (err) {
2325		p_err("failed to get func_info for prog FD %d", tgt_fd);
2326		goto out;
2327	}
2328
2329	btf = btf__load_from_kernel_by_id(info.btf_id);
2330	if (!btf) {
2331		p_err("failed to load btf for prog FD %d", tgt_fd);
2332		goto out;
2333	}
2334
2335	t = btf__type_by_id(btf, func_info.type_id);
2336	if (!t) {
2337		p_err("btf %u doesn't have type %u",
2338		      info.btf_id, func_info.type_id);
2339		goto out;
2340	}
2341	name = strdup(btf__name_by_offset(btf, t->name_off));
2342out:
2343	btf__free(btf);
2344	return name;
2345}
2346
2347static struct profiler_bpf *profile_obj;
2348static int profile_tgt_fd = -1;
2349static char *profile_tgt_name;
2350static int *profile_perf_events;
2351static int profile_perf_event_cnt;
2352
2353static void profile_close_perf_events(struct profiler_bpf *obj)
2354{
2355	int i;
2356
2357	for (i = profile_perf_event_cnt - 1; i >= 0; i--)
2358		close(profile_perf_events[i]);
2359
2360	free(profile_perf_events);
2361	profile_perf_event_cnt = 0;
2362}
2363
2364static int profile_open_perf_event(int mid, int cpu, int map_fd)
2365{
2366	int pmu_fd;
2367
2368	pmu_fd = syscall(__NR_perf_event_open, &metrics[mid].attr,
2369			 -1 /*pid*/, cpu, -1 /*group_fd*/, 0);
2370	if (pmu_fd < 0) {
2371		if (errno == ENODEV) {
2372			p_info("cpu %d may be offline, skip %s profiling.",
2373				cpu, metrics[mid].name);
2374			profile_perf_event_cnt++;
2375			return 0;
2376		}
2377		return -1;
2378	}
2379
2380	if (bpf_map_update_elem(map_fd,
2381				&profile_perf_event_cnt,
2382				&pmu_fd, BPF_ANY) ||
2383	    ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0)) {
2384		close(pmu_fd);
2385		return -1;
2386	}
2387
2388	profile_perf_events[profile_perf_event_cnt++] = pmu_fd;
2389	return 0;
2390}
2391
2392static int profile_open_perf_events(struct profiler_bpf *obj)
2393{
2394	unsigned int cpu, m;
2395	int map_fd;
2396
2397	profile_perf_events = calloc(
2398		obj->rodata->num_cpu * obj->rodata->num_metric, sizeof(int));
2399	if (!profile_perf_events) {
2400		p_err("failed to allocate memory for perf_event array: %s",
2401		      strerror(errno));
2402		return -1;
2403	}
2404	map_fd = bpf_map__fd(obj->maps.events);
2405	if (map_fd < 0) {
2406		p_err("failed to get fd for events map");
2407		return -1;
2408	}
2409
2410	for (m = 0; m < ARRAY_SIZE(metrics); m++) {
2411		if (!metrics[m].selected)
2412			continue;
2413		for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) {
2414			if (profile_open_perf_event(m, cpu, map_fd)) {
2415				p_err("failed to create event %s on cpu %u",
2416				      metrics[m].name, cpu);
2417				return -1;
2418			}
2419		}
2420	}
2421	return 0;
2422}
2423
2424static void profile_print_and_cleanup(void)
2425{
2426	profile_close_perf_events(profile_obj);
2427	profile_read_values(profile_obj);
2428	profile_print_readings();
2429	profiler_bpf__destroy(profile_obj);
2430
2431	close(profile_tgt_fd);
2432	free(profile_tgt_name);
2433}
2434
2435static void int_exit(int signo)
2436{
2437	profile_print_and_cleanup();
2438	exit(0);
2439}
2440
2441static int do_profile(int argc, char **argv)
2442{
2443	int num_metric, num_cpu, err = -1;
2444	struct bpf_program *prog;
2445	unsigned long duration;
2446	char *endptr;
2447
2448	/* we at least need two args for the prog and one metric */
2449	if (!REQ_ARGS(3))
2450		return -EINVAL;
2451
2452	/* parse target fd */
2453	profile_tgt_fd = prog_parse_fd(&argc, &argv);
2454	if (profile_tgt_fd < 0) {
2455		p_err("failed to parse fd");
2456		return -1;
2457	}
2458
2459	/* parse profiling optional duration */
2460	if (argc > 2 && is_prefix(argv[0], "duration")) {
2461		NEXT_ARG();
2462		duration = strtoul(*argv, &endptr, 0);
2463		if (*endptr)
2464			usage();
2465		NEXT_ARG();
2466	} else {
2467		duration = UINT_MAX;
2468	}
2469
2470	num_metric = profile_parse_metrics(argc, argv);
2471	if (num_metric <= 0)
2472		goto out;
2473
2474	num_cpu = libbpf_num_possible_cpus();
2475	if (num_cpu <= 0) {
2476		p_err("failed to identify number of CPUs");
2477		goto out;
2478	}
2479
2480	profile_obj = profiler_bpf__open();
2481	if (!profile_obj) {
2482		p_err("failed to open and/or load BPF object");
2483		goto out;
2484	}
2485
2486	profile_obj->rodata->num_cpu = num_cpu;
2487	profile_obj->rodata->num_metric = num_metric;
2488
2489	/* adjust map sizes */
2490	bpf_map__set_max_entries(profile_obj->maps.events, num_metric * num_cpu);
2491	bpf_map__set_max_entries(profile_obj->maps.fentry_readings, num_metric);
2492	bpf_map__set_max_entries(profile_obj->maps.accum_readings, num_metric);
2493	bpf_map__set_max_entries(profile_obj->maps.counts, 1);
2494
2495	/* change target name */
2496	profile_tgt_name = profile_target_name(profile_tgt_fd);
2497	if (!profile_tgt_name)
2498		goto out;
2499
2500	bpf_object__for_each_program(prog, profile_obj->obj) {
2501		err = bpf_program__set_attach_target(prog, profile_tgt_fd,
2502						     profile_tgt_name);
2503		if (err) {
2504			p_err("failed to set attach target\n");
2505			goto out;
2506		}
2507	}
2508
2509	set_max_rlimit();
2510	err = profiler_bpf__load(profile_obj);
2511	if (err) {
2512		p_err("failed to load profile_obj");
2513		goto out;
2514	}
2515
2516	err = profile_open_perf_events(profile_obj);
2517	if (err)
2518		goto out;
2519
2520	err = profiler_bpf__attach(profile_obj);
2521	if (err) {
2522		p_err("failed to attach profile_obj");
2523		goto out;
2524	}
2525	signal(SIGINT, int_exit);
2526
2527	sleep(duration);
2528	profile_print_and_cleanup();
2529	return 0;
2530
2531out:
2532	profile_close_perf_events(profile_obj);
2533	if (profile_obj)
2534		profiler_bpf__destroy(profile_obj);
2535	close(profile_tgt_fd);
2536	free(profile_tgt_name);
2537	return err;
2538}
2539
2540#endif /* BPFTOOL_WITHOUT_SKELETONS */
2541
2542static int do_help(int argc, char **argv)
2543{
2544	if (json_output) {
2545		jsonw_null(json_wtr);
2546		return 0;
2547	}
2548
2549	fprintf(stderr,
2550		"Usage: %1$s %2$s { show | list } [PROG]\n"
2551		"       %1$s %2$s dump xlated PROG [{ file FILE | [opcodes] [linum] [visual] }]\n"
2552		"       %1$s %2$s dump jited  PROG [{ file FILE | [opcodes] [linum] }]\n"
2553		"       %1$s %2$s pin   PROG FILE\n"
2554		"       %1$s %2$s { load | loadall } OBJ  PATH \\\n"
2555		"                         [type TYPE] [{ offload_dev | xdpmeta_dev } NAME] \\\n"
2556		"                         [map { idx IDX | name NAME } MAP]\\\n"
2557		"                         [pinmaps MAP_DIR]\n"
2558		"                         [autoattach]\n"
2559		"                         [kernel_btf BTF_FILE]\n"
2560		"       %1$s %2$s attach PROG ATTACH_TYPE [MAP]\n"
2561		"       %1$s %2$s detach PROG ATTACH_TYPE [MAP]\n"
2562		"       %1$s %2$s run PROG \\\n"
2563		"                         data_in FILE \\\n"
2564		"                         [data_out FILE [data_size_out L]] \\\n"
2565		"                         [ctx_in FILE [ctx_out FILE [ctx_size_out M]]] \\\n"
2566		"                         [repeat N]\n"
2567		"       %1$s %2$s profile PROG [duration DURATION] METRICs\n"
2568		"       %1$s %2$s tracelog\n"
2569		"       %1$s %2$s tracelog { stdout | stderr } PROG\n"
2570		"       %1$s %2$s help\n"
2571		"\n"
2572		"       " HELP_SPEC_MAP "\n"
2573		"       " HELP_SPEC_PROGRAM "\n"
2574		"       TYPE := { socket | kprobe | kretprobe | classifier | action |\n"
2575		"                 tracepoint | raw_tracepoint | xdp | perf_event | cgroup/skb |\n"
2576		"                 cgroup/sock | cgroup/dev | lwt_in | lwt_out | lwt_xmit |\n"
2577		"                 lwt_seg6local | sockops | sk_skb | sk_msg | lirc_mode2 |\n"
2578		"                 sk_reuseport | flow_dissector | cgroup/sysctl |\n"
2579		"                 cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n"
2580		"                 cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n"
2581		"                 cgroup/connect_unix | cgroup/getpeername4 | cgroup/getpeername6 |\n"
2582		"                 cgroup/getpeername_unix | cgroup/getsockname4 | cgroup/getsockname6 |\n"
2583		"                 cgroup/getsockname_unix | cgroup/sendmsg4 | cgroup/sendmsg6 |\n"
2584		"                 cgroup/sendmsg_unix | cgroup/recvmsg4 | cgroup/recvmsg6 | cgroup/recvmsg_unix |\n"
2585		"                 cgroup/getsockopt | cgroup/setsockopt | cgroup/sock_release |\n"
2586		"                 struct_ops | fentry | fexit | freplace | sk_lookup }\n"
2587		"       ATTACH_TYPE := { sk_msg_verdict | sk_skb_verdict | sk_skb_stream_verdict |\n"
2588		"                        sk_skb_stream_parser | flow_dissector }\n"
2589		"       METRIC := { cycles | instructions | l1d_loads | llc_misses | itlb_misses | dtlb_misses }\n"
2590		"       " HELP_SPEC_OPTIONS " |\n"
2591		"                    {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} |\n"
2592		"                    {-L|--use-loader} | [ {-S|--sign } {-k} <private_key.pem> {-i} <certificate.x509> ] \n"
2593		"",
2594		bin_name, argv[-2]);
2595
2596	return 0;
2597}
2598
2599static const struct cmd cmds[] = {
2600	{ "show",	do_show },
2601	{ "list",	do_show },
2602	{ "help",	do_help },
2603	{ "dump",	do_dump },
2604	{ "pin",	do_pin },
2605	{ "load",	do_load },
2606	{ "loadall",	do_loadall },
2607	{ "attach",	do_attach },
2608	{ "detach",	do_detach },
2609	{ "tracelog",	do_tracelog_any },
2610	{ "run",	do_run },
2611	{ "profile",	do_profile },
2612	{ 0 }
2613};
2614
2615int do_prog(int argc, char **argv)
2616{
2617	return cmd_select(cmds, argc, argv, do_help);
2618}