veristat.c

   1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
   3#define _GNU_SOURCE
   4#include <argp.h>
   5#include <libgen.h>
   6#include <ctype.h>
   7#include <string.h>
   8#include <stdlib.h>
   9#include <sched.h>
  10#include <pthread.h>
  11#include <dirent.h>
  12#include <signal.h>
  13#include <fcntl.h>
  14#include <unistd.h>
  15#include <sys/time.h>
  16#include <sys/sysinfo.h>
  17#include <sys/stat.h>
  18#include <bpf/libbpf.h>
  19#include <bpf/btf.h>
  20#include <bpf/bpf.h>
  21#include <libelf.h>
  22#include <gelf.h>
  23#include <float.h>
  24#include <math.h>
  25#include <limits.h>
  26#include <assert.h>
  27
  28#ifndef ARRAY_SIZE
  29#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
  30#endif
  31
  32#ifndef max
  33#define max(a, b) ((a) > (b) ? (a) : (b))
  34#endif
  35
  36#ifndef min
  37#define min(a, b) ((a) < (b) ? (a) : (b))
  38#endif
  39
  40enum stat_id {
  41	VERDICT,
  42	DURATION,
  43	TOTAL_INSNS,
  44	TOTAL_STATES,
  45	PEAK_STATES,
  46	MAX_STATES_PER_INSN,
  47	MARK_READ_MAX_LEN,
  48	SIZE,
  49	JITED_SIZE,
  50	STACK,
  51	PROG_TYPE,
  52	ATTACH_TYPE,
  53	MEMORY_PEAK,
  54
  55	FILE_NAME,
  56	PROG_NAME,
  57
  58	ALL_STATS_CNT,
  59	NUM_STATS_CNT = FILE_NAME - VERDICT,
  60};
  61
  62/* In comparison mode each stat can specify up to four different values:
  63 *   - A side value;
  64 *   - B side value;
  65 *   - absolute diff value;
  66 *   - relative (percentage) diff value.
  67 *
  68 * When specifying stat specs in comparison mode, user can use one of the
  69 * following variant suffixes to specify which exact variant should be used for
  70 * ordering or filtering:
  71 *   - `_a` for A side value;
  72 *   - `_b` for B side value;
  73 *   - `_diff` for absolute diff value;
  74 *   - `_pct` for relative (percentage) diff value.
  75 *
  76 * If no variant suffix is provided, then `_b` (control data) is assumed.
  77 *
  78 * As an example, let's say instructions stat has the following output:
  79 *
  80 * Insns (A)  Insns (B)  Insns   (DIFF)
  81 * ---------  ---------  --------------
  82 * 21547      20920       -627 (-2.91%)
  83 *
  84 * Then:
  85 *   - 21547 is A side value (insns_a);
  86 *   - 20920 is B side value (insns_b);
  87 *   - -627 is absolute diff value (insns_diff);
  88 *   - -2.91% is relative diff value (insns_pct).
  89 *
  90 * For verdict there is no verdict_pct variant.
  91 * For file and program name, _a and _b variants are equivalent and there are
  92 * no _diff or _pct variants.
  93 */
  94enum stat_variant {
  95	VARIANT_A,
  96	VARIANT_B,
  97	VARIANT_DIFF,
  98	VARIANT_PCT,
  99};
 100
 101struct verif_stats {
 102	char *file_name;
 103	char *prog_name;
 104
 105	long stats[NUM_STATS_CNT];
 106};
 107
 108/* joined comparison mode stats */
 109struct verif_stats_join {
 110	char *file_name;
 111	char *prog_name;
 112
 113	const struct verif_stats *stats_a;
 114	const struct verif_stats *stats_b;
 115};
 116
 117struct stat_specs {
 118	int spec_cnt;
 119	enum stat_id ids[ALL_STATS_CNT];
 120	enum stat_variant variants[ALL_STATS_CNT];
 121	bool asc[ALL_STATS_CNT];
 122	bool abs[ALL_STATS_CNT];
 123	int lens[ALL_STATS_CNT * 3]; /* 3x for comparison mode */
 124};
 125
 126enum resfmt {
 127	RESFMT_TABLE,
 128	RESFMT_TABLE_CALCLEN, /* fake format to pre-calculate table's column widths */
 129	RESFMT_CSV,
 130};
 131
 132enum filter_kind {
 133	FILTER_NAME,
 134	FILTER_STAT,
 135};
 136
 137enum operator_kind {
 138	OP_EQ,		/* == or = */
 139	OP_NEQ,		/* != or <> */
 140	OP_LT,		/* < */
 141	OP_LE,		/* <= */
 142	OP_GT,		/* > */
 143	OP_GE,		/* >= */
 144};
 145
 146struct filter {
 147	enum filter_kind kind;
 148	/* FILTER_NAME */
 149	char *any_glob;
 150	char *file_glob;
 151	char *prog_glob;
 152	/* FILTER_STAT */
 153	enum operator_kind op;
 154	int stat_id;
 155	enum stat_variant stat_var;
 156	long value;
 157	bool abs;
 158};
 159
 160struct rvalue {
 161	enum { INTEGRAL, ENUMERATOR } type;
 162	union {
 163		long long ivalue;
 164		char *svalue;
 165	};
 166};
 167
 168struct field_access {
 169	enum { FIELD_NAME, ARRAY_INDEX } type;
 170	union {
 171		char *name;
 172		struct rvalue index;
 173	};
 174};
 175
 176struct var_preset {
 177	struct field_access *atoms;
 178	int atom_count;
 179	char *full_name;
 180	struct rvalue value;
 181	bool applied;
 182};
 183
 184enum dump_mode {
 185	DUMP_NONE = 0,
 186	DUMP_XLATED = 1,
 187	DUMP_JITED = 2,
 188};
 189
 190static struct env {
 191	char **filenames;
 192	int filename_cnt;
 193	bool verbose;
 194	bool debug;
 195	bool quiet;
 196	bool force_checkpoints;
 197	bool force_reg_invariants;
 198	enum resfmt out_fmt;
 199	bool show_version;
 200	bool comparison_mode;
 201	bool replay_mode;
 202	int top_n;
 203
 204	int log_level;
 205	int log_size;
 206	bool log_fixed;
 207
 208	struct verif_stats *prog_stats;
 209	int prog_stat_cnt;
 210
 211	/* baseline_stats is allocated and used only in comparison mode */
 212	struct verif_stats *baseline_stats;
 213	int baseline_stat_cnt;
 214
 215	struct verif_stats_join *join_stats;
 216	int join_stat_cnt;
 217
 218	struct stat_specs output_spec;
 219	struct stat_specs sort_spec;
 220
 221	struct filter *allow_filters;
 222	struct filter *deny_filters;
 223	int allow_filter_cnt;
 224	int deny_filter_cnt;
 225
 226	int files_processed;
 227	int files_skipped;
 228	int progs_processed;
 229	int progs_skipped;
 230	int top_src_lines;
 231	struct var_preset *presets;
 232	int npresets;
 233	char orig_cgroup[PATH_MAX];
 234	char stat_cgroup[PATH_MAX];
 235	int memory_peak_fd;
 236	__u32 dump_mode;
 237} env;
 238
 239static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
 240{
 241	if (!env.verbose)
 242		return 0;
 243	if (level == LIBBPF_DEBUG  && !env.debug)
 244		return 0;
 245	return vfprintf(stderr, format, args);
 246}
 247
 248#define log_errno(fmt, ...) log_errno_aux(__FILE__, __LINE__, fmt, ##__VA_ARGS__)
 249
 250__attribute__((format(printf, 3, 4)))
 251static int log_errno_aux(const char *file, int line, const char *fmt, ...)
 252{
 253	int err = -errno;
 254	va_list ap;
 255
 256	va_start(ap, fmt);
 257	fprintf(stderr, "%s:%d: ", file, line);
 258	vfprintf(stderr, fmt, ap);
 259	fprintf(stderr, " failed with error '%s'.\n", strerror(errno));
 260	va_end(ap);
 261	return err;
 262}
 263
 264#ifndef VERISTAT_VERSION
 265#define VERISTAT_VERSION "<kernel>"
 266#endif
 267
 268const char *argp_program_version = "veristat v" VERISTAT_VERSION;
 269const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
 270const char argp_program_doc[] =
 271"veristat    BPF verifier stats collection and comparison tool.\n"
 272"\n"
 273"USAGE: veristat <obj-file> [<obj-file>...]\n"
 274"   OR: veristat -C <baseline.csv> <comparison.csv>\n"
 275"   OR: veristat -R <results.csv>\n"
 276"   OR: veristat -vl2 <to_analyze.bpf.o>\n";
 277
 278enum {
 279	OPT_LOG_FIXED = 1000,
 280	OPT_LOG_SIZE = 1001,
 281	OPT_DUMP = 1002,
 282};
 283
 284static const struct argp_option opts[] = {
 285	{ NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" },
 286	{ "version", 'V', NULL, 0, "Print version" },
 287	{ "verbose", 'v', NULL, 0, "Verbose mode" },
 288	{ "debug", 'd', NULL, 0, "Debug mode (turns on libbpf debug logging)" },
 289	{ "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode, 2 for full verification log)" },
 290	{ "log-fixed", OPT_LOG_FIXED, NULL, 0, "Disable verifier log rotation" },
 291	{ "log-size", OPT_LOG_SIZE, "BYTES", 0, "Customize verifier log size (default to 16MB)" },
 292	{ "top-n", 'n', "N", 0, "Emit only up to first N results." },
 293	{ "quiet", 'q', NULL, 0, "Quiet mode" },
 294	{ "emit", 'e', "SPEC", 0, "Specify stats to be emitted" },
 295	{ "sort", 's', "SPEC", 0, "Specify sort order" },
 296	{ "output-format", 'o', "FMT", 0, "Result output format (table, csv), default is table." },
 297	{ "compare", 'C', NULL, 0, "Comparison mode" },
 298	{ "replay", 'R', NULL, 0, "Replay mode" },
 299	{ "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." },
 300	{ "test-states", 't', NULL, 0,
 301	  "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" },
 302	{ "test-reg-invariants", 'r', NULL, 0,
 303	  "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },
 304	{ "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" },
 305	{ "set-global-vars", 'G', "GLOBAL", 0, "Set global variables provided in the expression, for example \"var1 = 1\"" },
 306	{ "dump", OPT_DUMP, "DUMP_MODE", OPTION_ARG_OPTIONAL, "Print BPF program dump (xlated, jited)" },
 307	{},
 308};
 309
 310static int parse_stats(const char *stats_str, struct stat_specs *specs);
 311static int append_filter(struct filter **filters, int *cnt, const char *str);
 312static int append_filter_file(const char *path);
 313static int append_var_preset(struct var_preset **presets, int *cnt, const char *expr);
 314static int append_var_preset_file(const char *filename);
 315static int append_file(const char *path);
 316static int append_file_from_file(const char *path);
 317
 318static error_t parse_arg(int key, char *arg, struct argp_state *state)
 319{
 320	int err;
 321
 322	switch (key) {
 323	case 'h':
 324		argp_state_help(state, stderr, ARGP_HELP_STD_HELP);
 325		break;
 326	case 'V':
 327		env.show_version = true;
 328		break;
 329	case 'v':
 330		env.verbose = true;
 331		break;
 332	case 'd':
 333		env.debug = true;
 334		env.verbose = true;
 335		break;
 336	case 'q':
 337		env.quiet = true;
 338		break;
 339	case 'e':
 340		err = parse_stats(arg, &env.output_spec);
 341		if (err)
 342			return err;
 343		break;
 344	case 's':
 345		err = parse_stats(arg, &env.sort_spec);
 346		if (err)
 347			return err;
 348		break;
 349	case 'o':
 350		if (strcmp(arg, "table") == 0) {
 351			env.out_fmt = RESFMT_TABLE;
 352		} else if (strcmp(arg, "csv") == 0) {
 353			env.out_fmt = RESFMT_CSV;
 354		} else {
 355			fprintf(stderr, "Unrecognized output format '%s'\n", arg);
 356			return -EINVAL;
 357		}
 358		break;
 359	case 'l':
 360		errno = 0;
 361		env.log_level = strtol(arg, NULL, 10);
 362		if (errno) {
 363			fprintf(stderr, "invalid log level: %s\n", arg);
 364			argp_usage(state);
 365		}
 366		break;
 367	case OPT_LOG_FIXED:
 368		env.log_fixed = true;
 369		break;
 370	case OPT_LOG_SIZE:
 371		errno = 0;
 372		env.log_size = strtol(arg, NULL, 10);
 373		if (errno) {
 374			fprintf(stderr, "invalid log size: %s\n", arg);
 375			argp_usage(state);
 376		}
 377		break;
 378	case 't':
 379		env.force_checkpoints = true;
 380		break;
 381	case 'r':
 382		env.force_reg_invariants = true;
 383		break;
 384	case 'n':
 385		errno = 0;
 386		env.top_n = strtol(arg, NULL, 10);
 387		if (errno) {
 388			fprintf(stderr, "invalid top N specifier: %s\n", arg);
 389			argp_usage(state);
 390		}
 391		break;
 392	case 'C':
 393		env.comparison_mode = true;
 394		break;
 395	case 'R':
 396		env.replay_mode = true;
 397		break;
 398	case 'f':
 399		if (arg[0] == '@')
 400			err = append_filter_file(arg + 1);
 401		else if (arg[0] == '!')
 402			err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1);
 403		else
 404			err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg);
 405		if (err) {
 406			fprintf(stderr, "Failed to collect program filter expressions: %d\n", err);
 407			return err;
 408		}
 409		break;
 410	case 'S':
 411		errno = 0;
 412		env.top_src_lines = strtol(arg, NULL, 10);
 413		if (errno) {
 414			fprintf(stderr, "invalid top lines N specifier: %s\n", arg);
 415			argp_usage(state);
 416		}
 417		break;
 418	case 'G': {
 419		if (arg[0] == '@')
 420			err = append_var_preset_file(arg + 1);
 421		else
 422			err = append_var_preset(&env.presets, &env.npresets, arg);
 423		if (err) {
 424			fprintf(stderr, "Failed to parse global variable presets: %s\n", arg);
 425			return err;
 426		}
 427		break;
 428	}
 429	case ARGP_KEY_ARG:
 430		if (arg[0] == '@')
 431			err = append_file_from_file(arg + 1);
 432		else
 433			err = append_file(arg);
 434		if (err) {
 435			fprintf(stderr, "Failed to collect BPF object files: %d\n", err);
 436			return err;
 437		}
 438		break;
 439	case OPT_DUMP:
 440		if (!arg || strcasecmp(arg, "xlated") == 0) {
 441			env.dump_mode |= DUMP_XLATED;
 442		} else if (strcasecmp(arg, "jited") == 0) {
 443			env.dump_mode |= DUMP_JITED;
 444		} else {
 445			fprintf(stderr, "Unrecognized dump mode '%s'\n", arg);
 446			return -EINVAL;
 447		}
 448		break;
 449	default:
 450		return ARGP_ERR_UNKNOWN;
 451	}
 452	return 0;
 453}
 454
 455static const struct argp argp = {
 456	.options = opts,
 457	.parser = parse_arg,
 458	.doc = argp_program_doc,
 459};
 460
 461
 462/* Adapted from perf/util/string.c */
 463static bool glob_matches(const char *str, const char *pat)
 464{
 465	while (*str && *pat && *pat != '*') {
 466		if (*str != *pat)
 467			return false;
 468		str++;
 469		pat++;
 470	}
 471	/* Check wild card */
 472	if (*pat == '*') {
 473		while (*pat == '*')
 474			pat++;
 475		if (!*pat) /* Tail wild card matches all */
 476			return true;
 477		while (*str)
 478			if (glob_matches(str++, pat))
 479				return true;
 480	}
 481	return !*str && !*pat;
 482}
 483
 484static bool is_bpf_obj_file(const char *path) {
 485	Elf64_Ehdr *ehdr;
 486	int fd, err = -EINVAL;
 487	Elf *elf = NULL;
 488
 489	fd = open(path, O_RDONLY | O_CLOEXEC);
 490	if (fd < 0)
 491		return true; /* we'll fail later and propagate error */
 492
 493	/* ensure libelf is initialized */
 494	(void)elf_version(EV_CURRENT);
 495
 496	elf = elf_begin(fd, ELF_C_READ, NULL);
 497	if (!elf)
 498		goto cleanup;
 499
 500	if (elf_kind(elf) != ELF_K_ELF || gelf_getclass(elf) != ELFCLASS64)
 501		goto cleanup;
 502
 503	ehdr = elf64_getehdr(elf);
 504	/* Old LLVM set e_machine to EM_NONE */
 505	if (!ehdr || ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF))
 506		goto cleanup;
 507
 508	err = 0;
 509cleanup:
 510	if (elf)
 511		elf_end(elf);
 512	close(fd);
 513	return err == 0;
 514}
 515
 516static bool should_process_file_prog(const char *filename, const char *prog_name)
 517{
 518	struct filter *f;
 519	int i, allow_cnt = 0;
 520
 521	for (i = 0; i < env.deny_filter_cnt; i++) {
 522		f = &env.deny_filters[i];
 523		if (f->kind != FILTER_NAME)
 524			continue;
 525
 526		if (f->any_glob && glob_matches(filename, f->any_glob))
 527			return false;
 528		if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob))
 529			return false;
 530		if (f->file_glob && glob_matches(filename, f->file_glob))
 531			return false;
 532		if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob))
 533			return false;
 534	}
 535
 536	for (i = 0; i < env.allow_filter_cnt; i++) {
 537		f = &env.allow_filters[i];
 538		if (f->kind != FILTER_NAME)
 539			continue;
 540
 541		allow_cnt++;
 542		if (f->any_glob) {
 543			if (glob_matches(filename, f->any_glob))
 544				return true;
 545			/* If we don't know program name yet, any_glob filter
 546			 * has to assume that current BPF object file might be
 547			 * relevant; we'll check again later on after opening
 548			 * BPF object file, at which point program name will
 549			 * be known finally.
 550			 */
 551			if (!prog_name || glob_matches(prog_name, f->any_glob))
 552				return true;
 553		} else {
 554			if (f->file_glob && !glob_matches(filename, f->file_glob))
 555				continue;
 556			if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob))
 557				continue;
 558			return true;
 559		}
 560	}
 561
 562	/* if there are no file/prog name allow filters, allow all progs,
 563	 * unless they are denied earlier explicitly
 564	 */
 565	return allow_cnt == 0;
 566}
 567
 568static struct {
 569	enum operator_kind op_kind;
 570	const char *op_str;
 571} operators[] = {
 572	/* Order of these definitions matter to avoid situations like '<'
 573	 * matching part of what is actually a '<>' operator. That is,
 574	 * substrings should go last.
 575	 */
 576	{ OP_EQ, "==" },
 577	{ OP_NEQ, "!=" },
 578	{ OP_NEQ, "<>" },
 579	{ OP_LE, "<=" },
 580	{ OP_LT, "<" },
 581	{ OP_GE, ">=" },
 582	{ OP_GT, ">" },
 583	{ OP_EQ, "=" },
 584};
 585
 586static bool parse_stat_id_var(const char *name, size_t len, int *id,
 587			      enum stat_variant *var, bool *is_abs);
 588
 589static int append_filter(struct filter **filters, int *cnt, const char *str)
 590{
 591	struct filter *f;
 592	void *tmp;
 593	const char *p;
 594	int i;
 595
 596	tmp = realloc(*filters, (*cnt + 1) * sizeof(**filters));
 597	if (!tmp)
 598		return -ENOMEM;
 599	*filters = tmp;
 600
 601	f = &(*filters)[*cnt];
 602	memset(f, 0, sizeof(*f));
 603
 604	/* First, let's check if it's a stats filter of the following form:
 605	 * <stat><op><value, where:
 606	 *   - <stat> is one of supported numerical stats (verdict is also
 607	 *     considered numerical, failure == 0, success == 1);
 608	 *   - <op> is comparison operator (see `operators` definitions);
 609	 *   - <value> is an integer (or failure/success, or false/true as
 610	 *     special aliases for 0 and 1, respectively).
 611	 * If the form doesn't match what user provided, we assume file/prog
 612	 * glob filter.
 613	 */
 614	for (i = 0; i < ARRAY_SIZE(operators); i++) {
 615		enum stat_variant var;
 616		int id;
 617		long val;
 618		const char *end = str;
 619		const char *op_str;
 620		bool is_abs;
 621
 622		op_str = operators[i].op_str;
 623		p = strstr(str, op_str);
 624		if (!p)
 625			continue;
 626
 627		if (!parse_stat_id_var(str, p - str, &id, &var, &is_abs)) {
 628			fprintf(stderr, "Unrecognized stat name in '%s'!\n", str);
 629			return -EINVAL;
 630		}
 631		if (id >= FILE_NAME) {
 632			fprintf(stderr, "Non-integer stat is specified in '%s'!\n", str);
 633			return -EINVAL;
 634		}
 635
 636		p += strlen(op_str);
 637
 638		if (strcasecmp(p, "true") == 0 ||
 639		    strcasecmp(p, "t") == 0 ||
 640		    strcasecmp(p, "success") == 0 ||
 641		    strcasecmp(p, "succ") == 0 ||
 642		    strcasecmp(p, "s") == 0 ||
 643		    strcasecmp(p, "match") == 0 ||
 644		    strcasecmp(p, "m") == 0) {
 645			val = 1;
 646		} else if (strcasecmp(p, "false") == 0 ||
 647			   strcasecmp(p, "f") == 0 ||
 648			   strcasecmp(p, "failure") == 0 ||
 649			   strcasecmp(p, "fail") == 0 ||
 650			   strcasecmp(p, "mismatch") == 0 ||
 651			   strcasecmp(p, "mis") == 0) {
 652			val = 0;
 653		} else {
 654			errno = 0;
 655			val = strtol(p, (char **)&end, 10);
 656			if (errno || end == p || *end != '\0' ) {
 657				fprintf(stderr, "Invalid integer value in '%s'!\n", str);
 658				return -EINVAL;
 659			}
 660		}
 661
 662		f->kind = FILTER_STAT;
 663		f->stat_id = id;
 664		f->stat_var = var;
 665		f->op = operators[i].op_kind;
 666		f->abs = true;
 667		f->value = val;
 668
 669		*cnt += 1;
 670		return 0;
 671	}
 672
 673	/* File/prog filter can be specified either as '<glob>' or
 674	 * '<file-glob>/<prog-glob>'. In the former case <glob> is applied to
 675	 * both file and program names. This seems to be way more useful in
 676	 * practice. If user needs full control, they can use '/<prog-glob>'
 677	 * form to glob just program name, or '<file-glob>/' to glob only file
 678	 * name. But usually common <glob> seems to be the most useful and
 679	 * ergonomic way.
 680	 */
 681	f->kind = FILTER_NAME;
 682	p = strchr(str, '/');
 683	if (!p) {
 684		f->any_glob = strdup(str);
 685		if (!f->any_glob)
 686			return -ENOMEM;
 687	} else {
 688		if (str != p) {
 689			/* non-empty file glob */
 690			f->file_glob = strndup(str, p - str);
 691			if (!f->file_glob)
 692				return -ENOMEM;
 693		}
 694		if (strlen(p + 1) > 0) {
 695			/* non-empty prog glob */
 696			f->prog_glob = strdup(p + 1);
 697			if (!f->prog_glob) {
 698				free(f->file_glob);
 699				f->file_glob = NULL;
 700				return -ENOMEM;
 701			}
 702		}
 703	}
 704
 705	*cnt += 1;
 706	return 0;
 707}
 708
 709static int append_filter_file(const char *path)
 710{
 711	char buf[1024];
 712	FILE *f;
 713	int err = 0;
 714
 715	f = fopen(path, "r");
 716	if (!f) {
 717		err = -errno;
 718		fprintf(stderr, "Failed to open filters in '%s': %s\n", path, strerror(-err));
 719		return err;
 720	}
 721
 722	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
 723		/* lines starting with # are comments, skip them */
 724		if (buf[0] == '\0' || buf[0] == '#')
 725			continue;
 726		/* lines starting with ! are negative match filters */
 727		if (buf[0] == '!')
 728			err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1);
 729		else
 730			err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf);
 731		if (err)
 732			goto cleanup;
 733	}
 734
 735cleanup:
 736	fclose(f);
 737	return err;
 738}
 739
 740static const struct stat_specs default_output_spec = {
 741	.spec_cnt = 8,
 742	.ids = {
 743		FILE_NAME, PROG_NAME, VERDICT, DURATION,
 744		TOTAL_INSNS, TOTAL_STATES, SIZE, JITED_SIZE
 745	},
 746};
 747
 748static int append_file(const char *path)
 749{
 750	void *tmp;
 751
 752	tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
 753	if (!tmp)
 754		return -ENOMEM;
 755	env.filenames = tmp;
 756	env.filenames[env.filename_cnt] = strdup(path);
 757	if (!env.filenames[env.filename_cnt])
 758		return -ENOMEM;
 759	env.filename_cnt++;
 760	return 0;
 761}
 762
 763static int append_file_from_file(const char *path)
 764{
 765	char buf[1024];
 766	int err = 0;
 767	FILE *f;
 768
 769	f = fopen(path, "r");
 770	if (!f) {
 771		err = -errno;
 772		fprintf(stderr, "Failed to open object files list in '%s': %s\n",
 773			path, strerror(errno));
 774		return err;
 775	}
 776
 777	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
 778		/* lines starting with # are comments, skip them */
 779		if (buf[0] == '\0' || buf[0] == '#')
 780			continue;
 781		err = append_file(buf);
 782		if (err)
 783			goto cleanup;
 784	}
 785
 786cleanup:
 787	fclose(f);
 788	return err;
 789}
 790
 791static const struct stat_specs default_csv_output_spec = {
 792	.spec_cnt = 15,
 793	.ids = {
 794		FILE_NAME, PROG_NAME, VERDICT, DURATION,
 795		TOTAL_INSNS, TOTAL_STATES, PEAK_STATES,
 796		MAX_STATES_PER_INSN, MARK_READ_MAX_LEN,
 797		SIZE, JITED_SIZE, PROG_TYPE, ATTACH_TYPE,
 798		STACK, MEMORY_PEAK,
 799	},
 800};
 801
 802static const struct stat_specs default_sort_spec = {
 803	.spec_cnt = 2,
 804	.ids = {
 805		FILE_NAME, PROG_NAME,
 806	},
 807	.asc = { true, true, },
 808};
 809
 810/* sorting for comparison mode to join two data sets */
 811static const struct stat_specs join_sort_spec = {
 812	.spec_cnt = 2,
 813	.ids = {
 814		FILE_NAME, PROG_NAME,
 815	},
 816	.asc = { true, true, },
 817};
 818
 819static struct stat_def {
 820	const char *header;
 821	const char *names[4];
 822	bool asc_by_default;
 823	bool left_aligned;
 824} stat_defs[] = {
 825	[FILE_NAME] = { "File", {"file_name", "filename", "file"}, true /* asc */, true /* left */ },
 826	[PROG_NAME] = { "Program", {"prog_name", "progname", "prog"}, true /* asc */, true /* left */ },
 827	[VERDICT] = { "Verdict", {"verdict"}, true /* asc: failure, success */, true /* left */ },
 828	[DURATION] = { "Duration (us)", {"duration", "dur"}, },
 829	[TOTAL_INSNS] = { "Insns", {"total_insns", "insns"}, },
 830	[TOTAL_STATES] = { "States", {"total_states", "states"}, },
 831	[PEAK_STATES] = { "Peak states", {"peak_states"}, },
 832	[MAX_STATES_PER_INSN] = { "Max states per insn", {"max_states_per_insn"}, },
 833	[MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, },
 834	[SIZE] = { "Program size", {"prog_size"}, },
 835	[JITED_SIZE] = { "Jited size", {"prog_size_jited"}, },
 836	[STACK] = {"Stack depth", {"stack_depth", "stack"}, },
 837	[PROG_TYPE] = { "Program type", {"prog_type"}, },
 838	[ATTACH_TYPE] = { "Attach type", {"attach_type", }, },
 839	[MEMORY_PEAK] = { "Peak memory (MiB)", {"mem_peak", }, },
 840};
 841
 842static bool parse_stat_id_var(const char *name, size_t len, int *id,
 843			      enum stat_variant *var, bool *is_abs)
 844{
 845	static const char *var_sfxs[] = {
 846		[VARIANT_A] = "_a",
 847		[VARIANT_B] = "_b",
 848		[VARIANT_DIFF] = "_diff",
 849		[VARIANT_PCT] = "_pct",
 850	};
 851	int i, j, k;
 852
 853	/* |<stat>| means we take absolute value of given stat */
 854	*is_abs = false;
 855	if (len > 2 && name[0] == '|' && name[len - 1] == '|') {
 856		*is_abs = true;
 857		name += 1;
 858		len -= 2;
 859	}
 860
 861	for (i = 0; i < ARRAY_SIZE(stat_defs); i++) {
 862		struct stat_def *def = &stat_defs[i];
 863		size_t alias_len, sfx_len;
 864		const char *alias;
 865
 866		for (j = 0; j < ARRAY_SIZE(stat_defs[i].names); j++) {
 867			alias = def->names[j];
 868			if (!alias)
 869				continue;
 870
 871			alias_len = strlen(alias);
 872			if (strncmp(name, alias, alias_len) != 0)
 873				continue;
 874
 875			if (alias_len == len) {
 876				/* If no variant suffix is specified, we
 877				 * assume control group (just in case we are
 878				 * in comparison mode. Variant is ignored in
 879				 * non-comparison mode.
 880				 */
 881				*var = VARIANT_B;
 882				*id = i;
 883				return true;
 884			}
 885
 886			for (k = 0; k < ARRAY_SIZE(var_sfxs); k++) {
 887				sfx_len = strlen(var_sfxs[k]);
 888				if (alias_len + sfx_len != len)
 889					continue;
 890
 891				if (strncmp(name + alias_len, var_sfxs[k], sfx_len) == 0) {
 892					*var = (enum stat_variant)k;
 893					*id = i;
 894					return true;
 895				}
 896			}
 897		}
 898	}
 899
 900	return false;
 901}
 902
 903static bool is_asc_sym(char c)
 904{
 905	return c == '^';
 906}
 907
 908static bool is_desc_sym(char c)
 909{
 910	return c == 'v' || c == 'V' || c == '.' || c == '!' || c == '_';
 911}
 912
 913static char *rtrim(char *str)
 914{
 915	int i;
 916
 917	for (i = strlen(str) - 1; i > 0; --i) {
 918		if (!isspace(str[i]))
 919			break;
 920		str[i] = '\0';
 921	}
 922	return str;
 923}
 924
 925static int parse_stat(const char *stat_name, struct stat_specs *specs)
 926{
 927	int id;
 928	bool has_order = false, is_asc = false, is_abs = false;
 929	size_t len = strlen(stat_name);
 930	enum stat_variant var;
 931
 932	if (specs->spec_cnt >= ARRAY_SIZE(specs->ids)) {
 933		fprintf(stderr, "Can't specify more than %zd stats\n", ARRAY_SIZE(specs->ids));
 934		return -E2BIG;
 935	}
 936
 937	if (len > 1 && (is_asc_sym(stat_name[len - 1]) || is_desc_sym(stat_name[len - 1]))) {
 938		has_order = true;
 939		is_asc = is_asc_sym(stat_name[len - 1]);
 940		len -= 1;
 941	}
 942
 943	if (!parse_stat_id_var(stat_name, len, &id, &var, &is_abs)) {
 944		fprintf(stderr, "Unrecognized stat name '%s'\n", stat_name);
 945		return -ESRCH;
 946	}
 947
 948	specs->ids[specs->spec_cnt] = id;
 949	specs->variants[specs->spec_cnt] = var;
 950	specs->asc[specs->spec_cnt] = has_order ? is_asc : stat_defs[id].asc_by_default;
 951	specs->abs[specs->spec_cnt] = is_abs;
 952	specs->spec_cnt++;
 953
 954	return 0;
 955}
 956
 957static int parse_stats(const char *stats_str, struct stat_specs *specs)
 958{
 959	char *input, *state = NULL, *next;
 960	int err, cnt = 0;
 961
 962	input = strdup(stats_str);
 963	if (!input)
 964		return -ENOMEM;
 965
 966	while ((next = strtok_r(cnt++ ? NULL : input, ",", &state))) {
 967		err = parse_stat(next, specs);
 968		if (err) {
 969			free(input);
 970			return err;
 971		}
 972	}
 973
 974	free(input);
 975	return 0;
 976}
 977
 978static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt)
 979{
 980	int i;
 981
 982	if (!stats)
 983		return;
 984
 985	for (i = 0; i < stat_cnt; i++) {
 986		free(stats[i].file_name);
 987		free(stats[i].prog_name);
 988	}
 989	free(stats);
 990}
 991
 992static char verif_log_buf[64 * 1024];
 993
 994#define MAX_PARSED_LOG_LINES 100
 995
 996static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s)
 997{
 998	const char *cur;
 999	int pos, lines, sub_stack, cnt = 0;
1000	char *state = NULL, *token, stack[512];
1001
1002	buf[buf_sz - 1] = '\0';
1003
1004	for (pos = strlen(buf) - 1, lines = 0; pos >= 0 && lines < MAX_PARSED_LOG_LINES; lines++) {
1005		/* find previous endline or otherwise take the start of log buf */
1006		for (cur = &buf[pos]; cur > buf && cur[0] != '\n'; cur--, pos--) {
1007		}
1008		/* next time start from end of previous line (or pos goes to <0) */
1009		pos--;
1010		/* if we found endline, point right after endline symbol;
1011		 * otherwise, stay at the beginning of log buf
1012		 */
1013		if (cur[0] == '\n')
1014			cur++;
1015
1016		if (1 == sscanf(cur, "verification time %ld usec\n", &s->stats[DURATION]))
1017			continue;
1018		if (5 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld",
1019				&s->stats[TOTAL_INSNS],
1020				&s->stats[MAX_STATES_PER_INSN],
1021				&s->stats[TOTAL_STATES],
1022				&s->stats[PEAK_STATES],
1023				&s->stats[MARK_READ_MAX_LEN]))
1024			continue;
1025
1026		if (1 == sscanf(cur, "stack depth %511s", stack))
1027			continue;
1028	}
1029	while ((token = strtok_r(cnt++ ? NULL : stack, "+", &state))) {
1030		if (sscanf(token, "%d", &sub_stack) == 0)
1031			break;
1032		s->stats[STACK] += sub_stack;
1033	}
1034	return 0;
1035}
1036
1037struct line_cnt {
1038	char *line;
1039	int cnt;
1040};
1041
1042static int str_cmp(const void *a, const void *b)
1043{
1044	const char **str1 = (const char **)a;
1045	const char **str2 = (const char **)b;
1046
1047	return strcmp(*str1, *str2);
1048}
1049
1050static int line_cnt_cmp(const void *a, const void *b)
1051{
1052	const struct line_cnt *a_cnt = (const struct line_cnt *)a;
1053	const struct line_cnt *b_cnt = (const struct line_cnt *)b;
1054
1055	if (a_cnt->cnt != b_cnt->cnt)
1056		return a_cnt->cnt > b_cnt->cnt ? -1 : 1;
1057	return strcmp(a_cnt->line, b_cnt->line);
1058}
1059
1060static int print_top_src_lines(char * const buf, size_t buf_sz, const char *prog_name)
1061{
1062	int lines_cap = 0;
1063	int lines_size = 0;
1064	char **lines = NULL;
1065	char *line = NULL;
1066	char *state;
1067	struct line_cnt *freq = NULL;
1068	struct line_cnt *cur;
1069	int unique_lines;
1070	int err = 0;
1071	int i;
1072
1073	while ((line = strtok_r(line ? NULL : buf, "\n", &state))) {
1074		if (strncmp(line, "; ", 2) != 0)
1075			continue;
1076		line += 2;
1077
1078		if (lines_size == lines_cap) {
1079			char **tmp;
1080
1081			lines_cap = max(16, lines_cap * 2);
1082			tmp = realloc(lines, lines_cap * sizeof(*tmp));
1083			if (!tmp) {
1084				err = -ENOMEM;
1085				goto cleanup;
1086			}
1087			lines = tmp;
1088		}
1089		lines[lines_size] = line;
1090		lines_size++;
1091	}
1092
1093	if (lines_size == 0)
1094		goto cleanup;
1095
1096	qsort(lines, lines_size, sizeof(*lines), str_cmp);
1097
1098	freq = calloc(lines_size, sizeof(*freq));
1099	if (!freq) {
1100		err = -ENOMEM;
1101		goto cleanup;
1102	}
1103
1104	cur = freq;
1105	cur->line = lines[0];
1106	cur->cnt = 1;
1107	for (i = 1; i < lines_size; ++i) {
1108		if (strcmp(lines[i], cur->line) != 0) {
1109			cur++;
1110			cur->line = lines[i];
1111			cur->cnt = 0;
1112		}
1113		cur->cnt++;
1114	}
1115	unique_lines = cur - freq + 1;
1116
1117	qsort(freq, unique_lines, sizeof(struct line_cnt), line_cnt_cmp);
1118
1119	printf("Top source lines (%s):\n", prog_name);
1120	for (i = 0; i < min(unique_lines, env.top_src_lines); ++i) {
1121		const char *src_code = freq[i].line;
1122		const char *src_line = NULL;
1123		char *split = strrchr(freq[i].line, '@');
1124
1125		if (split) {
1126			src_line = split + 1;
1127
1128			while (*src_line && isspace(*src_line))
1129				src_line++;
1130
1131			while (split > src_code && isspace(*split))
1132				split--;
1133			*split = '\0';
1134		}
1135
1136		if (src_line)
1137			printf("%5d: (%s)\t%s\n", freq[i].cnt, src_line, src_code);
1138		else
1139			printf("%5d: %s\n", freq[i].cnt, src_code);
1140	}
1141	printf("\n");
1142
1143cleanup:
1144	free(freq);
1145	free(lines);
1146	return err;
1147}
1148
1149static int guess_prog_type_by_ctx_name(const char *ctx_name,
1150				       enum bpf_prog_type *prog_type,
1151				       enum bpf_attach_type *attach_type)
1152{
1153	/* We need to guess program type based on its declared context type.
1154	 * This guess can't be perfect as many different program types might
1155	 * share the same context type.  So we can only hope to reasonably
1156	 * well guess this and get lucky.
1157	 *
1158	 * Just in case, we support both UAPI-side type names and
1159	 * kernel-internal names.
1160	 */
1161	static struct {
1162		const char *uapi_name;
1163		const char *kern_name;
1164		enum bpf_prog_type prog_type;
1165		enum bpf_attach_type attach_type;
1166	} ctx_map[] = {
1167		/* __sk_buff is most ambiguous, we assume TC program */
1168		{ "__sk_buff", "sk_buff", BPF_PROG_TYPE_SCHED_CLS },
1169		{ "bpf_sock", "sock", BPF_PROG_TYPE_CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND },
1170		{ "bpf_sock_addr", "bpf_sock_addr_kern",  BPF_PROG_TYPE_CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND },
1171		{ "bpf_sock_ops", "bpf_sock_ops_kern", BPF_PROG_TYPE_SOCK_OPS, BPF_CGROUP_SOCK_OPS },
1172		{ "sk_msg_md", "sk_msg", BPF_PROG_TYPE_SK_MSG, BPF_SK_MSG_VERDICT },
1173		{ "bpf_cgroup_dev_ctx", "bpf_cgroup_dev_ctx", BPF_PROG_TYPE_CGROUP_DEVICE, BPF_CGROUP_DEVICE },
1174		{ "bpf_sysctl", "bpf_sysctl_kern", BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_CGROUP_SYSCTL },
1175		{ "bpf_sockopt", "bpf_sockopt_kern", BPF_PROG_TYPE_CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT },
1176		{ "sk_reuseport_md", "sk_reuseport_kern", BPF_PROG_TYPE_SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE },
1177		{ "bpf_sk_lookup", "bpf_sk_lookup_kern", BPF_PROG_TYPE_SK_LOOKUP, BPF_SK_LOOKUP },
1178		{ "xdp_md", "xdp_buff", BPF_PROG_TYPE_XDP, BPF_XDP },
1179		/* tracing types with no expected attach type */
1180		{ "bpf_user_pt_regs_t", "pt_regs", BPF_PROG_TYPE_KPROBE },
1181		{ "bpf_perf_event_data", "bpf_perf_event_data_kern", BPF_PROG_TYPE_PERF_EVENT },
1182		/* raw_tp programs use u64[] from kernel side, we don't want
1183		 * to match on that, probably; so NULL for kern-side type
1184		 */
1185		{ "bpf_raw_tracepoint_args", NULL, BPF_PROG_TYPE_RAW_TRACEPOINT },
1186	};
1187	int i;
1188
1189	if (!ctx_name)
1190		return -EINVAL;
1191
1192	for (i = 0; i < ARRAY_SIZE(ctx_map); i++) {
1193		if (strcmp(ctx_map[i].uapi_name, ctx_name) == 0 ||
1194		    (ctx_map[i].kern_name && strcmp(ctx_map[i].kern_name, ctx_name) == 0)) {
1195			*prog_type = ctx_map[i].prog_type;
1196			*attach_type = ctx_map[i].attach_type;
1197			return 0;
1198		}
1199	}
1200
1201	return -ESRCH;
1202}
1203
1204/* Make sure only target program is referenced from struct_ops map,
1205 * otherwise libbpf would automatically set autocreate for all
1206 * referenced programs.
1207 * See libbpf.c:bpf_object_adjust_struct_ops_autoload.
1208 */
1209static void mask_unrelated_struct_ops_progs(struct bpf_object *obj,
1210					    struct bpf_map *map,
1211					    struct bpf_program *prog)
1212{
1213	struct btf *btf = bpf_object__btf(obj);
1214	const struct btf_type *t, *mt;
1215	struct btf_member *m;
1216	int i, moff;
1217	size_t data_sz, ptr_sz = sizeof(void *);
1218	void *data;
1219
1220	t = btf__type_by_id(btf, bpf_map__btf_value_type_id(map));
1221	if (!btf_is_struct(t))
1222		return;
1223
1224	data = bpf_map__initial_value(map, &data_sz);
1225	for (i = 0; i < btf_vlen(t); i++) {
1226		m = &btf_members(t)[i];
1227		mt = btf__type_by_id(btf, m->type);
1228		if (!btf_is_ptr(mt))
1229			continue;
1230		moff = m->offset / 8;
1231		if (moff + ptr_sz > data_sz)
1232			continue;
1233		if (memcmp(data + moff, &prog, ptr_sz) == 0)
1234			continue;
1235		memset(data + moff, 0, ptr_sz);
1236	}
1237}
1238
1239static void fixup_obj(struct bpf_object *obj, struct bpf_program *prog, const char *filename)
1240{
1241	struct bpf_map *map;
1242
1243	bpf_object__for_each_map(map, obj) {
1244		/* disable pinning */
1245		bpf_map__set_pin_path(map, NULL);
1246
1247		/* fix up map size, if necessary */
1248		switch (bpf_map__type(map)) {
1249		case BPF_MAP_TYPE_SK_STORAGE:
1250		case BPF_MAP_TYPE_TASK_STORAGE:
1251		case BPF_MAP_TYPE_INODE_STORAGE:
1252		case BPF_MAP_TYPE_CGROUP_STORAGE:
1253		case BPF_MAP_TYPE_CGRP_STORAGE:
1254			break;
1255		case BPF_MAP_TYPE_STRUCT_OPS:
1256			mask_unrelated_struct_ops_progs(obj, map, prog);
1257			break;
1258		default:
1259			if (bpf_map__max_entries(map) == 0)
1260				bpf_map__set_max_entries(map, 1);
1261		}
1262	}
1263
1264	/* SEC(freplace) programs can't be loaded with veristat as is,
1265	 * but we can try guessing their target program's expected type by
1266	 * looking at the type of program's first argument and substituting
1267	 * corresponding program type
1268	 */
1269	if (bpf_program__type(prog) == BPF_PROG_TYPE_EXT) {
1270		const struct btf *btf = bpf_object__btf(obj);
1271		const char *prog_name = bpf_program__name(prog);
1272		enum bpf_prog_type prog_type;
1273		enum bpf_attach_type attach_type;
1274		const struct btf_type *t;
1275		const char *ctx_name;
1276		int id;
1277
1278		if (!btf)
1279			goto skip_freplace_fixup;
1280
1281		id = btf__find_by_name_kind(btf, prog_name, BTF_KIND_FUNC);
1282		t = btf__type_by_id(btf, id);
1283		t = btf__type_by_id(btf, t->type);
1284		if (!btf_is_func_proto(t) || btf_vlen(t) != 1)
1285			goto skip_freplace_fixup;
1286
1287		/* context argument is a pointer to a struct/typedef */
1288		t = btf__type_by_id(btf, btf_params(t)[0].type);
1289		while (t && btf_is_mod(t))
1290			t = btf__type_by_id(btf, t->type);
1291		if (!t || !btf_is_ptr(t))
1292			goto skip_freplace_fixup;
1293		t = btf__type_by_id(btf, t->type);
1294		while (t && btf_is_mod(t))
1295			t = btf__type_by_id(btf, t->type);
1296		if (!t)
1297			goto skip_freplace_fixup;
1298
1299		ctx_name = btf__name_by_offset(btf, t->name_off);
1300
1301		if (guess_prog_type_by_ctx_name(ctx_name, &prog_type, &attach_type) == 0) {
1302			bpf_program__set_type(prog, prog_type);
1303			bpf_program__set_expected_attach_type(prog, attach_type);
1304
1305			if (!env.quiet) {
1306				fprintf(stderr, "Using guessed program type '%s' for %s/%s...\n",
1307					libbpf_bpf_prog_type_str(prog_type),
1308					filename, prog_name);
1309			}
1310		} else {
1311			if (!env.quiet) {
1312				fprintf(stderr, "Failed to guess program type for freplace program with context type name '%s' for %s/%s. Consider using canonical type names to help veristat...\n",
1313					ctx_name, filename, prog_name);
1314			}
1315		}
1316	}
1317skip_freplace_fixup:
1318	return;
1319}
1320
1321static int max_verifier_log_size(void)
1322{
1323	const int SMALL_LOG_SIZE = UINT_MAX >> 8;
1324	const int BIG_LOG_SIZE = UINT_MAX >> 2;
1325	struct bpf_insn insns[] = {
1326		{ .code = BPF_ALU | BPF_MOV | BPF_X, .dst_reg = BPF_REG_0, },
1327		{ .code  = BPF_JMP | BPF_EXIT, },
1328	};
1329	LIBBPF_OPTS(bpf_prog_load_opts, opts,
1330		    .log_size = BIG_LOG_SIZE,
1331		    .log_buf = (void *)-1,
1332		    .log_level = 4
1333	);
1334	int ret, insn_cnt = ARRAY_SIZE(insns);
1335	static int log_size;
1336
1337	if (log_size != 0)
1338		return log_size;
1339
1340	ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
1341
1342	if (ret == -EFAULT)
1343		log_size = BIG_LOG_SIZE;
1344	else /* ret == -EINVAL, big log size is not supported by the verifier */
1345		log_size = SMALL_LOG_SIZE;
1346
1347	return log_size;
1348}
1349
1350static bool output_stat_enabled(int id)
1351{
1352	int i;
1353
1354	for (i = 0; i < env.output_spec.spec_cnt; i++)
1355		if (env.output_spec.ids[i] == id)
1356			return true;
1357	return false;
1358}
1359
1360__attribute__((format(printf, 2, 3)))
1361static int write_one_line(const char *file, const char *fmt, ...)
1362{
1363	int err, saved_errno;
1364	va_list ap;
1365	FILE *f;
1366
1367	f = fopen(file, "w");
1368	if (!f)
1369		return -1;
1370
1371	va_start(ap, fmt);
1372	errno = 0;
1373	err = vfprintf(f, fmt, ap);
1374	saved_errno = errno;
1375	va_end(ap);
1376	fclose(f);
1377	errno = saved_errno;
1378	return err < 0 ? -1 : 0;
1379}
1380
1381__attribute__((format(scanf, 3, 4)))
1382static int scanf_one_line(const char *file, int fields_expected, const char *fmt, ...)
1383{
1384	int res = 0, saved_errno = 0;
1385	char *line = NULL;
1386	size_t line_len;
1387	va_list ap;
1388	FILE *f;
1389
1390	f = fopen(file, "r");
1391	if (!f)
1392		return -1;
1393
1394	va_start(ap, fmt);
1395	while (getline(&line, &line_len, f) > 0) {
1396		res = vsscanf(line, fmt, ap);
1397		if (res == fields_expected)
1398			goto out;
1399	}
1400	if (ferror(f)) {
1401		saved_errno = errno;
1402		res = -1;
1403	}
1404
1405out:
1406	va_end(ap);
1407	free(line);
1408	fclose(f);
1409	errno = saved_errno;
1410	return res;
1411}
1412
1413static void destroy_stat_cgroup(void)
1414{
1415	char buf[PATH_MAX];
1416	int err;
1417
1418	close(env.memory_peak_fd);
1419
1420	if (env.orig_cgroup[0]) {
1421		snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.orig_cgroup);
1422		err = write_one_line(buf, "%d\n", getpid());
1423		if (err < 0)
1424			log_errno("moving self to original cgroup %s\n", env.orig_cgroup);
1425	}
1426
1427	if (env.stat_cgroup[0]) {
1428		err = rmdir(env.stat_cgroup);
1429		if (err < 0)
1430			log_errno("deletion of cgroup %s", env.stat_cgroup);
1431	}
1432
1433	env.memory_peak_fd = -1;
1434	env.orig_cgroup[0] = 0;
1435	env.stat_cgroup[0] = 0;
1436}
1437
1438/*
1439 * Creates a cgroup at /sys/fs/cgroup/veristat-accounting-<pid>,
1440 * moves current process to this cgroup.
1441 */
1442static void create_stat_cgroup(void)
1443{
1444	char cgroup_fs_mount[4096];
1445	char buf[4096];
1446	int err;
1447
1448	env.memory_peak_fd = -1;
1449
1450	if (!output_stat_enabled(MEMORY_PEAK))
1451		return;
1452
1453	err = scanf_one_line("/proc/self/mounts", 2, "%*s %4095s cgroup2 %s",
1454			     cgroup_fs_mount, buf);
1455	if (err != 2) {
1456		if (err < 0)
1457			log_errno("reading /proc/self/mounts");
1458		else if (!env.quiet)
1459			fprintf(stderr, "Can't find cgroupfs v2 mount point.\n");
1460		goto err_out;
1461	}
1462
1463	/* cgroup-v2.rst promises the line "0::<group>" for cgroups v2 */
1464	err = scanf_one_line("/proc/self/cgroup", 1, "0::%4095s", buf);
1465	if (err != 1) {
1466		if (err < 0)
1467			log_errno("reading /proc/self/cgroup");
1468		else if (!env.quiet)
1469			fprintf(stderr, "Can't infer veristat process cgroup.");
1470		goto err_out;
1471	}
1472
1473	snprintf(env.orig_cgroup, sizeof(env.orig_cgroup), "%s/%s", cgroup_fs_mount, buf);
1474
1475	snprintf(buf, sizeof(buf), "%s/veristat-accounting-%d", cgroup_fs_mount, getpid());
1476	err = mkdir(buf, 0777);
1477	if (err < 0) {
1478		log_errno("creation of cgroup %s", buf);
1479		goto err_out;
1480	}
1481	strcpy(env.stat_cgroup, buf);
1482
1483	snprintf(buf, sizeof(buf), "%s/cgroup.procs", env.stat_cgroup);
1484	err = write_one_line(buf, "%d\n", getpid());
1485	if (err < 0) {
1486		log_errno("entering cgroup %s", buf);
1487		goto err_out;
1488	}
1489
1490	snprintf(buf, sizeof(buf), "%s/memory.peak", env.stat_cgroup);
1491	env.memory_peak_fd = open(buf, O_RDWR | O_APPEND);
1492	if (env.memory_peak_fd < 0) {
1493		log_errno("opening %s", buf);
1494		goto err_out;
1495	}
1496
1497	return;
1498
1499err_out:
1500	if (!env.quiet)
1501		fprintf(stderr, "Memory usage metric unavailable.\n");
1502	destroy_stat_cgroup();
1503}
1504
1505/* Current value of /sys/fs/cgroup/veristat-accounting-<pid>/memory.peak */
1506static long cgroup_memory_peak(void)
1507{
1508	long err, memory_peak;
1509	char buf[32];
1510
1511	if (env.memory_peak_fd < 0)
1512		return -1;
1513
1514	err = pread(env.memory_peak_fd, buf, sizeof(buf) - 1, 0);
1515	if (err <= 0) {
1516		log_errno("pread(%s/memory.peak)", env.stat_cgroup);
1517		return -1;
1518	}
1519
1520	buf[err] = 0;
1521	errno = 0;
1522	memory_peak = strtoll(buf, NULL, 10);
1523	if (errno) {
1524		log_errno("%s/memory.peak:strtoll(%s)", env.stat_cgroup, buf);
1525		return -1;
1526	}
1527
1528	return memory_peak;
1529}
1530
1531static int reset_stat_cgroup(void)
1532{
1533	char buf[] = "r\n";
1534	int err;
1535
1536	if (env.memory_peak_fd < 0)
1537		return -1;
1538
1539	err = pwrite(env.memory_peak_fd, buf, sizeof(buf), 0);
1540	if (err <= 0) {
1541		log_errno("pwrite(%s/memory.peak)", env.stat_cgroup);
1542		return -1;
1543	}
1544	return 0;
1545}
1546
1547static int parse_rvalue(const char *val, struct rvalue *rvalue)
1548{
1549	long long value;
1550	char *val_end;
1551
1552	if (val[0] == '-' || isdigit(val[0])) {
1553		/* must be a number */
1554		errno = 0;
1555		value = strtoll(val, &val_end, 0);
1556		if (errno == ERANGE) {
1557			errno = 0;
1558			value = strtoull(val, &val_end, 0);
1559		}
1560		if (errno || *val_end != '\0') {
1561			fprintf(stderr, "Failed to parse value '%s'\n", val);
1562			return -EINVAL;
1563		}
1564		rvalue->ivalue = value;
1565		rvalue->type = INTEGRAL;
1566	} else {
1567		/* if not a number, consider it enum value */
1568		rvalue->svalue = strdup(val);
1569		if (!rvalue->svalue)
1570			return -ENOMEM;
1571		rvalue->type = ENUMERATOR;
1572	}
1573	return 0;
1574}
1575
1576static void dump(__u32 prog_id, enum dump_mode mode, const char *file_name, const char *prog_name)
1577{
1578	char command[64], buf[4096];
1579	FILE *fp;
1580	int status;
1581
1582	status = system("command -v bpftool > /dev/null 2>&1");
1583	if (status != 0) {
1584		fprintf(stderr, "bpftool is not available, can't print program dump\n");
1585		return;
1586	}
1587	snprintf(command, sizeof(command), "bpftool prog dump %s id %u",
1588		 mode == DUMP_JITED ? "jited" : "xlated", prog_id);
1589	fp = popen(command, "r");
1590	if (!fp) {
1591		fprintf(stderr, "bpftool failed with error: %d\n", errno);
1592		return;
1593	}
1594
1595	printf("DUMP (%s) %s/%s:\n", mode == DUMP_JITED ? "JITED" : "XLATED", file_name, prog_name);
1596	while (fgets(buf, sizeof(buf), fp))
1597		fputs(buf, stdout);
1598	fprintf(stdout, "\n");
1599
1600	if (ferror(fp))
1601		fprintf(stderr, "Failed to dump BPF prog with error: %d\n", errno);
1602
1603	pclose(fp);
1604}
1605
1606static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog)
1607{
1608	const char *base_filename = basename(strdupa(filename));
1609	const char *prog_name = bpf_program__name(prog);
1610	long mem_peak_a, mem_peak_b, mem_peak = -1;
1611	char *buf;
1612	int buf_sz, log_level;
1613	struct verif_stats *stats;
1614	struct bpf_prog_info info;
1615	__u32 info_len = sizeof(info);
1616	int err = 0, cgroup_err;
1617	void *tmp;
1618	int fd;
1619
1620	if (!should_process_file_prog(base_filename, bpf_program__name(prog))) {
1621		env.progs_skipped++;
1622		return 0;
1623	}
1624
1625	tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats));
1626	if (!tmp)
1627		return -ENOMEM;
1628	env.prog_stats = tmp;
1629	stats = &env.prog_stats[env.prog_stat_cnt++];
1630	memset(stats, 0, sizeof(*stats));
1631
1632	if (env.verbose || env.top_src_lines > 0) {
1633		buf_sz = env.log_size ? env.log_size : max_verifier_log_size();
1634		buf = malloc(buf_sz);
1635		if (!buf)
1636			return -ENOMEM;
1637		/* ensure we always request stats */
1638		log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0);
1639		/* --top-src-lines needs verifier log */
1640		if (env.top_src_lines > 0 && env.log_level == 0)
1641			log_level |= 2;
1642	} else {
1643		buf = verif_log_buf;
1644		buf_sz = sizeof(verif_log_buf);
1645		/* request only verifier stats */
1646		log_level = 4 | (env.log_fixed ? 8 : 0);
1647	}
1648	verif_log_buf[0] = '\0';
1649
1650	bpf_program__set_log_buf(prog, buf, buf_sz);
1651	bpf_program__set_log_level(prog, log_level);
1652
1653	/* increase chances of successful BPF object loading */
1654	fixup_obj(obj, prog, base_filename);
1655
1656	if (env.force_checkpoints)
1657		bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_STATE_FREQ);
1658	if (env.force_reg_invariants)
1659		bpf_program__set_flags(prog, bpf_program__flags(prog) | BPF_F_TEST_REG_INVARIANTS);
1660
1661	err = bpf_object__prepare(obj);
1662	if (!err) {
1663		cgroup_err = reset_stat_cgroup();
1664		mem_peak_a = cgroup_memory_peak();
1665		err = bpf_object__load(obj);
1666		mem_peak_b = cgroup_memory_peak();
1667		if (!cgroup_err && mem_peak_a >= 0 && mem_peak_b >= 0)
1668			mem_peak = mem_peak_b - mem_peak_a;
1669	}
1670	env.progs_processed++;
1671
1672	stats->file_name = strdup(base_filename);
1673	stats->prog_name = strdup(bpf_program__name(prog));
1674	stats->stats[VERDICT] = err == 0; /* 1 - success, 0 - failure */
1675	stats->stats[SIZE] = bpf_program__insn_cnt(prog);
1676	stats->stats[PROG_TYPE] = bpf_program__type(prog);
1677	stats->stats[ATTACH_TYPE] = bpf_program__expected_attach_type(prog);
1678	stats->stats[MEMORY_PEAK] = mem_peak < 0 ? -1 : mem_peak / (1024 * 1024);
1679
1680	memset(&info, 0, info_len);
1681	fd = bpf_program__fd(prog);
1682	if (fd > 0 && bpf_prog_get_info_by_fd(fd, &info, &info_len) == 0) {
1683		stats->stats[JITED_SIZE] = info.jited_prog_len;
1684		if (env.dump_mode & DUMP_JITED)
1685			dump(info.id, DUMP_JITED, base_filename, prog_name);
1686		if (env.dump_mode & DUMP_XLATED)
1687			dump(info.id, DUMP_XLATED, base_filename, prog_name);
1688	}
1689
1690	parse_verif_log(buf, buf_sz, stats);
1691
1692	if (env.verbose) {
1693		printf("PROCESSING %s/%s, DURATION US: %ld, VERDICT: %s, VERIFIER LOG:\n%s\n",
1694		       filename, prog_name, stats->stats[DURATION],
1695		       err ? "failure" : "success", buf);
1696	}
1697	if (env.top_src_lines > 0)
1698		print_top_src_lines(buf, buf_sz, stats->prog_name);
1699
1700	if (verif_log_buf != buf)
1701		free(buf);
1702
1703	return 0;
1704}
1705
1706static int append_preset_atom(struct var_preset *preset, char *value, bool is_index)
1707{
1708	struct field_access *tmp;
1709	int i = preset->atom_count;
1710	int err;
1711
1712	tmp = reallocarray(preset->atoms, i + 1, sizeof(*preset->atoms));
1713	if (!tmp)
1714		return -ENOMEM;
1715
1716	preset->atoms = tmp;
1717	preset->atom_count++;
1718
1719	if (is_index) {
1720		preset->atoms[i].type = ARRAY_INDEX;
1721		err = parse_rvalue(value, &preset->atoms[i].index);
1722		if (err)
1723			return err;
1724	} else {
1725		preset->atoms[i].type = FIELD_NAME;
1726		preset->atoms[i].name = strdup(value);
1727		if (!preset->atoms[i].name)
1728			return -ENOMEM;
1729	}
1730	return 0;
1731}
1732
1733static int parse_var_atoms(const char *full_var, struct var_preset *preset)
1734{
1735	char expr[256], var[256], *name, *saveptr;
1736	int n, len, off, err;
1737
1738	snprintf(expr, sizeof(expr), "%s", full_var);
1739	preset->atom_count = 0;
1740	while ((name = strtok_r(preset->atom_count ? NULL : expr, ".", &saveptr))) {
1741		len = strlen(name);
1742		/* parse variable name */
1743		if (sscanf(name, "%[a-zA-Z0-9_] %n", var, &off) != 1) {
1744			fprintf(stderr, "Can't parse %s\n", name);
1745			return -EINVAL;
1746		}
1747		err = append_preset_atom(preset, var, false);
1748		if (err)
1749			return err;
1750
1751		/* parse optional array indexes */
1752		while (off < len) {
1753			if (sscanf(name + off, " [ %[a-zA-Z0-9_] ] %n", var, &n) != 1) {
1754				fprintf(stderr, "Can't parse %s as index\n", name + off);
1755				return -EINVAL;
1756			}
1757			err = append_preset_atom(preset, var, true);
1758			if (err)
1759				return err;
1760			off += n;
1761		}
1762	}
1763	return 0;
1764}
1765
1766static int append_var_preset(struct var_preset **presets, int *cnt, const char *expr)
1767{
1768	void *tmp;
1769	struct var_preset *cur;
1770	char var[256], val[256];
1771	int n, err;
1772
1773	tmp = realloc(*presets, (*cnt + 1) * sizeof(**presets));
1774	if (!tmp)
1775		return -ENOMEM;
1776	*presets = tmp;
1777	cur = &(*presets)[*cnt];
1778	memset(cur, 0, sizeof(*cur));
1779	(*cnt)++;
1780
1781	if (sscanf(expr, " %[][a-zA-Z0-9_. ] = %s %n", var, val, &n) != 2 || n != strlen(expr)) {
1782		fprintf(stderr, "Failed to parse expression '%s'\n", expr);
1783		return -EINVAL;
1784	}
1785	/* Remove trailing spaces from var, as scanf may add those */
1786	rtrim(var);
1787
1788	err = parse_rvalue(val, &cur->value);
1789	if (err)
1790		return err;
1791
1792	cur->full_name = strdup(var);
1793	if (!cur->full_name)
1794		return -ENOMEM;
1795
1796	err = parse_var_atoms(var, cur);
1797	if (err)
1798		return err;
1799
1800	return 0;
1801}
1802
1803static int append_var_preset_file(const char *filename)
1804{
1805	char buf[1024];
1806	FILE *f;
1807	int err = 0;
1808
1809	f = fopen(filename, "rt");
1810	if (!f) {
1811		err = -errno;
1812		fprintf(stderr, "Failed to open presets in '%s': %s\n", filename, strerror(-err));
1813		return -EINVAL;
1814	}
1815
1816	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
1817		if (buf[0] == '\0' || buf[0] == '#')
1818			continue;
1819
1820		err = append_var_preset(&env.presets, &env.npresets, buf);
1821		if (err)
1822			goto cleanup;
1823	}
1824
1825cleanup:
1826	fclose(f);
1827	return err;
1828}
1829
1830static bool is_signed_type(const struct btf_type *t)
1831{
1832	if (btf_is_int(t))
1833		return btf_int_encoding(t) & BTF_INT_SIGNED;
1834	if (btf_is_any_enum(t))
1835		return btf_kflag(t);
1836	return true;
1837}
1838
1839static int enum_value_from_name(const struct btf *btf, const struct btf_type *t,
1840				const char *evalue, long long *retval)
1841{
1842	if (btf_is_enum(t)) {
1843		struct btf_enum *e = btf_enum(t);
1844		int i, n = btf_vlen(t);
1845
1846		for (i = 0; i < n; ++i, ++e) {
1847			const char *cur_name = btf__name_by_offset(btf, e->name_off);
1848
1849			if (strcmp(cur_name, evalue) == 0) {
1850				*retval = e->val;
1851				return 0;
1852			}
1853		}
1854	} else if (btf_is_enum64(t)) {
1855		struct btf_enum64 *e = btf_enum64(t);
1856		int i, n = btf_vlen(t);
1857
1858		for (i = 0; i < n; ++i, ++e) {
1859			const char *cur_name = btf__name_by_offset(btf, e->name_off);
1860			__u64 value =  btf_enum64_value(e);
1861
1862			if (strcmp(cur_name, evalue) == 0) {
1863				*retval = value;
1864				return 0;
1865			}
1866		}
1867	}
1868	return -EINVAL;
1869}
1870
1871static bool is_preset_supported(const struct btf_type *t)
1872{
1873	return btf_is_int(t) || btf_is_enum(t) || btf_is_enum64(t);
1874}
1875
1876static int find_enum_value(const struct btf *btf, const char *name, long long *value)
1877{
1878	const struct btf_type *t;
1879	int cnt, i;
1880	long long lvalue;
1881
1882	cnt = btf__type_cnt(btf);
1883	for (i = 1; i != cnt; ++i) {
1884		t = btf__type_by_id(btf, i);
1885
1886		if (!btf_is_any_enum(t))
1887			continue;
1888
1889		if (enum_value_from_name(btf, t, name, &lvalue) == 0) {
1890			*value = lvalue;
1891			return 0;
1892		}
1893	}
1894	return -ESRCH;
1895}
1896
1897static int resolve_rvalue(struct btf *btf, const struct rvalue *rvalue, long long *result)
1898{
1899	int err = 0;
1900
1901	switch (rvalue->type) {
1902	case INTEGRAL:
1903		*result = rvalue->ivalue;
1904		return 0;
1905	case ENUMERATOR:
1906		err = find_enum_value(btf, rvalue->svalue, result);
1907		if (err) {
1908			fprintf(stderr, "Can't resolve enum value %s\n", rvalue->svalue);
1909			return err;
1910		}
1911		return 0;
1912	default:
1913		fprintf(stderr, "Unknown rvalue type\n");
1914		return -EOPNOTSUPP;
1915	}
1916	return 0;
1917}
1918
1919static int adjust_var_secinfo_array(struct btf *btf, int tid, struct field_access *atom,
1920				    const char *array_name, struct btf_var_secinfo *sinfo)
1921{
1922	const struct btf_type *t;
1923	struct btf_array *barr;
1924	long long idx;
1925	int err;
1926
1927	tid = btf__resolve_type(btf, tid);
1928	t = btf__type_by_id(btf, tid);
1929	if (!btf_is_array(t)) {
1930		fprintf(stderr, "Array index is not expected for %s\n",
1931			array_name);
1932		return -EINVAL;
1933	}
1934	barr = btf_array(t);
1935	err = resolve_rvalue(btf, &atom->index, &idx);
1936	if (err)
1937		return err;
1938	if (idx < 0 || idx >= barr->nelems) {
1939		fprintf(stderr, "Array index %lld is out of bounds [0, %u): %s\n",
1940			idx, barr->nelems, array_name);
1941		return -EINVAL;
1942	}
1943	sinfo->size = btf__resolve_size(btf, barr->type);
1944	sinfo->offset += sinfo->size * idx;
1945	sinfo->type = btf__resolve_type(btf, barr->type);
1946	return 0;
1947}
1948
1949static int adjust_var_secinfo_member(const struct btf *btf,
1950				     const struct btf_type *parent_type,
1951				     __u32 parent_offset,
1952				     const char *member_name,
1953				     struct btf_var_secinfo *sinfo)
1954{
1955	int i;
1956
1957	if (!btf_is_composite(parent_type)) {
1958		fprintf(stderr, "Can't resolve field %s for non-composite type\n", member_name);
1959		return -EINVAL;
1960	}
1961
1962	for (i = 0; i < btf_vlen(parent_type); ++i) {
1963		const struct btf_member *member;
1964		const struct btf_type *member_type;
1965		int tid, off;
1966
1967		member = btf_members(parent_type) + i;
1968		tid =  btf__resolve_type(btf, member->type);
1969		if (tid < 0)
1970			return -EINVAL;
1971
1972		member_type = btf__type_by_id(btf, tid);
1973		off = parent_offset + member->offset;
1974		if (member->name_off) {
1975			const char *name = btf__name_by_offset(btf, member->name_off);
1976
1977			if (strcmp(member_name, name) == 0) {
1978				if (btf_member_bitfield_size(parent_type, i) != 0) {
1979					fprintf(stderr, "Bitfield presets are not supported %s\n",
1980						name);
1981					return -EINVAL;
1982				}
1983				sinfo->offset += off / 8;
1984				sinfo->type = tid;
1985				sinfo->size = member_type->size;
1986				return 0;
1987			}
1988		} else if (btf_is_composite(member_type)) {
1989			int err;
1990
1991			err = adjust_var_secinfo_member(btf, member_type, off,
1992							member_name, sinfo);
1993			if (!err)
1994				return 0;
1995		}
1996	}
1997
1998	return -ESRCH;
1999}
2000
2001static int adjust_var_secinfo(struct btf *btf, const struct btf_type *t,
2002			      struct btf_var_secinfo *sinfo, struct var_preset *preset)
2003{
2004	const struct btf_type *base_type;
2005	const char *prev_name;
2006	int err, i;
2007	int tid;
2008
2009	assert(preset->atom_count > 0);
2010	assert(preset->atoms[0].type == FIELD_NAME);
2011
2012	tid = btf__resolve_type(btf, t->type);
2013	base_type = btf__type_by_id(btf, tid);
2014	prev_name = preset->atoms[0].name;
2015
2016	for (i = 1; i < preset->atom_count; ++i) {
2017		struct field_access *atom = preset->atoms + i;
2018
2019		switch (atom->type) {
2020		case ARRAY_INDEX:
2021			err = adjust_var_secinfo_array(btf, tid, atom, prev_name, sinfo);
2022			break;
2023		case FIELD_NAME:
2024			err = adjust_var_secinfo_member(btf, base_type, 0, atom->name, sinfo);
2025			if (err == -ESRCH)
2026				fprintf(stderr, "Can't find '%s'\n", atom->name);
2027			prev_name = atom->name;
2028			break;
2029		default:
2030			fprintf(stderr, "Unknown field_access type\n");
2031			return -EOPNOTSUPP;
2032		}
2033		if (err)
2034			return err;
2035		base_type = btf__type_by_id(btf, sinfo->type);
2036		tid = sinfo->type;
2037	}
2038
2039	return 0;
2040}
2041
2042static int set_global_var(struct bpf_object *obj, struct btf *btf,
2043			  struct bpf_map *map, struct btf_var_secinfo *sinfo,
2044			  struct var_preset *preset)
2045{
2046	const struct btf_type *base_type;
2047	void *ptr;
2048	long long value = preset->value.ivalue;
2049	size_t size;
2050
2051	base_type = btf__type_by_id(btf, btf__resolve_type(btf, sinfo->type));
2052	if (!base_type) {
2053		fprintf(stderr, "Failed to resolve type %d\n", sinfo->type);
2054		return -EINVAL;
2055	}
2056	if (!is_preset_supported(base_type)) {
2057		fprintf(stderr, "Can't set %s. Only ints and enums are supported\n",
2058			preset->full_name);
2059		return -EINVAL;
2060	}
2061
2062	if (preset->value.type == ENUMERATOR) {
2063		if (btf_is_any_enum(base_type)) {
2064			if (enum_value_from_name(btf, base_type, preset->value.svalue, &value)) {
2065				fprintf(stderr,
2066					"Failed to find integer value for enum element %s\n",
2067					preset->value.svalue);
2068				return -EINVAL;
2069			}
2070		} else {
2071			fprintf(stderr, "Value %s is not supported for type %s\n",
2072				preset->value.svalue,
2073				btf__name_by_offset(btf, base_type->name_off));
2074			return -EINVAL;
2075		}
2076	}
2077
2078	/* Check if value fits into the target variable size */
2079	if  (sinfo->size < sizeof(value)) {
2080		bool is_signed = is_signed_type(base_type);
2081		__u32 unsigned_bits = sinfo->size * 8 - (is_signed ? 1 : 0);
2082		long long max_val = 1ll << unsigned_bits;
2083
2084		if (value >= max_val || value < -max_val) {
2085			fprintf(stderr,
2086				"Variable %s value %lld is out of range [%lld; %lld]\n",
2087				btf__name_by_offset(btf, base_type->name_off), value,
2088				is_signed ? -max_val : 0, max_val - 1);
2089			return -EINVAL;
2090		}
2091	}
2092
2093	ptr = bpf_map__initial_value(map, &size);
2094	if (!ptr || sinfo->offset + sinfo->size > size)
2095		return -EINVAL;
2096
2097	if (__BYTE_ORDER == __LITTLE_ENDIAN) {
2098		memcpy(ptr + sinfo->offset, &value, sinfo->size);
2099	} else { /* __BYTE_ORDER == __BIG_ENDIAN */
2100		__u8 src_offset = sizeof(value) - sinfo->size;
2101
2102		memcpy(ptr + sinfo->offset, (void *)&value + src_offset, sinfo->size);
2103	}
2104	return 0;
2105}
2106
2107static int set_global_vars(struct bpf_object *obj, struct var_preset *presets, int npresets)
2108{
2109	struct btf_var_secinfo *sinfo;
2110	const char *sec_name;
2111	const struct btf_type *t;
2112	struct bpf_map *map;
2113	struct btf *btf;
2114	int i, j, k, n, cnt, err = 0;
2115
2116	if (npresets == 0)
2117		return 0;
2118
2119	btf = bpf_object__btf(obj);
2120	if (!btf)
2121		return -EINVAL;
2122
2123	cnt = btf__type_cnt(btf);
2124	for (i = 1; i != cnt; ++i) {
2125		t = btf__type_by_id(btf, i);
2126
2127		if (!btf_is_datasec(t))
2128			continue;
2129
2130		sinfo = btf_var_secinfos(t);
2131		sec_name = btf__name_by_offset(btf, t->name_off);
2132		map = bpf_object__find_map_by_name(obj, sec_name);
2133		if (!map)
2134			continue;
2135
2136		n = btf_vlen(t);
2137		for (j = 0; j < n; ++j, ++sinfo) {
2138			const struct btf_type *var_type = btf__type_by_id(btf, sinfo->type);
2139			const char *var_name;
2140
2141			if (!btf_is_var(var_type))
2142				continue;
2143
2144			var_name = btf__name_by_offset(btf, var_type->name_off);
2145
2146			for (k = 0; k < npresets; ++k) {
2147				struct btf_var_secinfo tmp_sinfo;
2148
2149				if (strcmp(var_name, presets[k].atoms[0].name) != 0)
2150					continue;
2151
2152				if (presets[k].applied) {
2153					fprintf(stderr, "Variable %s is set more than once",
2154						var_name);
2155					return -EINVAL;
2156				}
2157				tmp_sinfo = *sinfo;
2158				err = adjust_var_secinfo(btf, var_type,
2159							 &tmp_sinfo, presets + k);
2160				if (err)
2161					return err;
2162
2163				err = set_global_var(obj, btf, map, &tmp_sinfo, presets + k);
2164				if (err)
2165					return err;
2166
2167				presets[k].applied = true;
2168			}
2169		}
2170	}
2171	for (i = 0; i < npresets; ++i) {
2172		if (!presets[i].applied) {
2173			fprintf(stderr, "Global variable preset %s has not been applied\n",
2174				presets[i].full_name);
2175			err = -EINVAL;
2176		}
2177		presets[i].applied = false;
2178	}
2179	return err;
2180}
2181
2182static int process_obj(const char *filename)
2183{
2184	const char *base_filename = basename(strdupa(filename));
2185	struct bpf_object *obj = NULL, *tobj;
2186	struct bpf_program *prog, *tprog, *lprog;
2187	libbpf_print_fn_t old_libbpf_print_fn;
2188	LIBBPF_OPTS(bpf_object_open_opts, opts);
2189	int err = 0, prog_cnt = 0;
2190
2191	if (!should_process_file_prog(base_filename, NULL)) {
2192		if (env.verbose)
2193			printf("Skipping '%s' due to filters...\n", filename);
2194		env.files_skipped++;
2195		return 0;
2196	}
2197	if (!is_bpf_obj_file(filename)) {
2198		if (env.verbose)
2199			printf("Skipping '%s' as it's not a BPF object file...\n", filename);
2200		env.files_skipped++;
2201		return 0;
2202	}
2203
2204	if (!env.quiet && env.out_fmt == RESFMT_TABLE)
2205		printf("Processing '%s'...\n", base_filename);
2206
2207	old_libbpf_print_fn = libbpf_set_print(libbpf_print_fn);
2208	obj = bpf_object__open_file(filename, &opts);
2209	if (!obj) {
2210		/* if libbpf can't open BPF object file, it could be because
2211		 * that BPF object file is incomplete and has to be statically
2212		 * linked into a final BPF object file; instead of bailing
2213		 * out, report it into stderr, mark it as skipped, and
2214		 * proceed
2215		 */
2216		fprintf(stderr, "Failed to open '%s': %d\n", filename, -errno);
2217		env.files_skipped++;
2218		err = 0;
2219		goto cleanup;
2220	}
2221
2222	env.files_processed++;
2223
2224	bpf_object__for_each_program(prog, obj) {
2225		prog_cnt++;
2226	}
2227
2228	if (prog_cnt == 1) {
2229		prog = bpf_object__next_program(obj, NULL);
2230		bpf_program__set_autoload(prog, true);
2231		err = set_global_vars(obj, env.presets, env.npresets);
2232		if (err) {
2233			fprintf(stderr, "Failed to set global variables %d\n", err);
2234			goto cleanup;
2235		}
2236		process_prog(filename, obj, prog);
2237		goto cleanup;
2238	}
2239
2240	bpf_object__for_each_program(prog, obj) {
2241		const char *prog_name = bpf_program__name(prog);
2242
2243		tobj = bpf_object__open_file(filename, &opts);
2244		if (!tobj) {
2245			err = -errno;
2246			fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
2247			goto cleanup;
2248		}
2249
2250		err = set_global_vars(tobj, env.presets, env.npresets);
2251		if (err) {
2252			fprintf(stderr, "Failed to set global variables %d\n", err);
2253			goto cleanup;
2254		}
2255
2256		lprog = NULL;
2257		bpf_object__for_each_program(tprog, tobj) {
2258			const char *tprog_name = bpf_program__name(tprog);
2259
2260			if (strcmp(prog_name, tprog_name) == 0) {
2261				bpf_program__set_autoload(tprog, true);
2262				lprog = tprog;
2263			} else {
2264				bpf_program__set_autoload(tprog, false);
2265			}
2266		}
2267
2268		process_prog(filename, tobj, lprog);
2269		bpf_object__close(tobj);
2270	}
2271
2272cleanup:
2273	bpf_object__close(obj);
2274	libbpf_set_print(old_libbpf_print_fn);
2275	return err;
2276}
2277
2278static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2,
2279		    enum stat_id id, bool asc, bool abs)
2280{
2281	int cmp = 0;
2282
2283	switch (id) {
2284	case FILE_NAME:
2285		cmp = strcmp(s1->file_name, s2->file_name);
2286		break;
2287	case PROG_NAME:
2288		cmp = strcmp(s1->prog_name, s2->prog_name);
2289		break;
2290	case ATTACH_TYPE:
2291	case PROG_TYPE:
2292	case SIZE:
2293	case JITED_SIZE:
2294	case STACK:
2295	case VERDICT:
2296	case DURATION:
2297	case TOTAL_INSNS:
2298	case TOTAL_STATES:
2299	case PEAK_STATES:
2300	case MAX_STATES_PER_INSN:
2301	case MEMORY_PEAK:
2302	case MARK_READ_MAX_LEN: {
2303		long v1 = s1->stats[id];
2304		long v2 = s2->stats[id];
2305
2306		if (abs) {
2307			v1 = v1 < 0 ? -v1 : v1;
2308			v2 = v2 < 0 ? -v2 : v2;
2309		}
2310
2311		if (v1 != v2)
2312			cmp = v1 < v2 ? -1 : 1;
2313		break;
2314	}
2315	default:
2316		fprintf(stderr, "Unrecognized stat #%d\n", id);
2317		exit(1);
2318	}
2319
2320	return asc ? cmp : -cmp;
2321}
2322
2323static int cmp_prog_stats(const void *v1, const void *v2)
2324{
2325	const struct verif_stats *s1 = v1, *s2 = v2;
2326	int i, cmp;
2327
2328	for (i = 0; i < env.sort_spec.spec_cnt; i++) {
2329		cmp = cmp_stat(s1, s2, env.sort_spec.ids[i],
2330			       env.sort_spec.asc[i], env.sort_spec.abs[i]);
2331		if (cmp != 0)
2332			return cmp;
2333	}
2334
2335	/* always disambiguate with file+prog, which are unique */
2336	cmp = strcmp(s1->file_name, s2->file_name);
2337	if (cmp != 0)
2338		return cmp;
2339	return strcmp(s1->prog_name, s2->prog_name);
2340}
2341
2342static void fetch_join_stat_value(const struct verif_stats_join *s,
2343				  enum stat_id id, enum stat_variant var,
2344				  const char **str_val,
2345				  double *num_val)
2346{
2347	long v1, v2;
2348
2349	if (id == FILE_NAME) {
2350		*str_val = s->file_name;
2351		return;
2352	}
2353	if (id == PROG_NAME) {
2354		*str_val = s->prog_name;
2355		return;
2356	}
2357
2358	v1 = s->stats_a ? s->stats_a->stats[id] : 0;
2359	v2 = s->stats_b ? s->stats_b->stats[id] : 0;
2360
2361	switch (var) {
2362	case VARIANT_A:
2363		if (!s->stats_a)
2364			*num_val = -DBL_MAX;
2365		else
2366			*num_val = s->stats_a->stats[id];
2367		return;
2368	case VARIANT_B:
2369		if (!s->stats_b)
2370			*num_val = -DBL_MAX;
2371		else
2372			*num_val = s->stats_b->stats[id];
2373		return;
2374	case VARIANT_DIFF:
2375		if (!s->stats_a || !s->stats_b)
2376			*num_val = -DBL_MAX;
2377		else if (id == VERDICT)
2378			*num_val = v1 == v2 ? 1.0 /* MATCH */ : 0.0 /* MISMATCH */;
2379		else
2380			*num_val = (double)(v2 - v1);
2381		return;
2382	case VARIANT_PCT:
2383		if (!s->stats_a || !s->stats_b) {
2384			*num_val = -DBL_MAX;
2385		} else if (v1 == 0) {
2386			if (v1 == v2)
2387				*num_val = 0.0;
2388			else
2389				*num_val = v2 < v1 ? -100.0 : 100.0;
2390		} else {
2391			 *num_val = (v2 - v1) * 100.0 / v1;
2392		}
2393		return;
2394	}
2395}
2396
2397static int cmp_join_stat(const struct verif_stats_join *s1,
2398			 const struct verif_stats_join *s2,
2399			 enum stat_id id, enum stat_variant var,
2400			 bool asc, bool abs)
2401{
2402	const char *str1 = NULL, *str2 = NULL;
2403	double v1 = 0.0, v2 = 0.0;
2404	int cmp = 0;
2405
2406	fetch_join_stat_value(s1, id, var, &str1, &v1);
2407	fetch_join_stat_value(s2, id, var, &str2, &v2);
2408
2409	if (abs) {
2410		v1 = fabs(v1);
2411		v2 = fabs(v2);
2412	}
2413
2414	if (str1)
2415		cmp = strcmp(str1, str2);
2416	else if (v1 != v2)
2417		cmp = v1 < v2 ? -1 : 1;
2418
2419	return asc ? cmp : -cmp;
2420}
2421
2422static int cmp_join_stats(const void *v1, const void *v2)
2423{
2424	const struct verif_stats_join *s1 = v1, *s2 = v2;
2425	int i, cmp;
2426
2427	for (i = 0; i < env.sort_spec.spec_cnt; i++) {
2428		cmp = cmp_join_stat(s1, s2,
2429				    env.sort_spec.ids[i],
2430				    env.sort_spec.variants[i],
2431				    env.sort_spec.asc[i],
2432				    env.sort_spec.abs[i]);
2433		if (cmp != 0)
2434			return cmp;
2435	}
2436
2437	/* always disambiguate with file+prog, which are unique */
2438	cmp = strcmp(s1->file_name, s2->file_name);
2439	if (cmp != 0)
2440		return cmp;
2441	return strcmp(s1->prog_name, s2->prog_name);
2442}
2443
2444#define HEADER_CHAR '-'
2445#define COLUMN_SEP "  "
2446
2447static void output_header_underlines(void)
2448{
2449	int i, j, len;
2450
2451	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2452		len = env.output_spec.lens[i];
2453
2454		printf("%s", i == 0 ? "" : COLUMN_SEP);
2455		for (j = 0; j < len; j++)
2456			printf("%c", HEADER_CHAR);
2457	}
2458	printf("\n");
2459}
2460
2461static void output_headers(enum resfmt fmt)
2462{
2463	const char *fmt_str;
2464	int i, len;
2465
2466	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2467		int id = env.output_spec.ids[i];
2468		int *max_len = &env.output_spec.lens[i];
2469
2470		switch (fmt) {
2471		case RESFMT_TABLE_CALCLEN:
2472			len = snprintf(NULL, 0, "%s", stat_defs[id].header);
2473			if (len > *max_len)
2474				*max_len = len;
2475			break;
2476		case RESFMT_TABLE:
2477			fmt_str = stat_defs[id].left_aligned ? "%s%-*s" : "%s%*s";
2478			printf(fmt_str, i == 0 ? "" : COLUMN_SEP,  *max_len, stat_defs[id].header);
2479			if (i == env.output_spec.spec_cnt - 1)
2480				printf("\n");
2481			break;
2482		case RESFMT_CSV:
2483			printf("%s%s", i == 0 ? "" : ",", stat_defs[id].names[0]);
2484			if (i == env.output_spec.spec_cnt - 1)
2485				printf("\n");
2486			break;
2487		}
2488	}
2489
2490	if (fmt == RESFMT_TABLE)
2491		output_header_underlines();
2492}
2493
2494static void prepare_value(const struct verif_stats *s, enum stat_id id,
2495			  const char **str, long *val)
2496{
2497	switch (id) {
2498	case FILE_NAME:
2499		*str = s ? s->file_name : "N/A";
2500		break;
2501	case PROG_NAME:
2502		*str = s ? s->prog_name : "N/A";
2503		break;
2504	case VERDICT:
2505		if (!s)
2506			*str = "N/A";
2507		else
2508			*str = s->stats[VERDICT] ? "success" : "failure";
2509		break;
2510	case ATTACH_TYPE:
2511		if (!s)
2512			*str = "N/A";
2513		else
2514			*str = libbpf_bpf_attach_type_str(s->stats[ATTACH_TYPE]) ?: "N/A";
2515		break;
2516	case PROG_TYPE:
2517		if (!s)
2518			*str = "N/A";
2519		else
2520			*str = libbpf_bpf_prog_type_str(s->stats[PROG_TYPE]) ?: "N/A";
2521		break;
2522	case DURATION:
2523	case TOTAL_INSNS:
2524	case TOTAL_STATES:
2525	case PEAK_STATES:
2526	case MAX_STATES_PER_INSN:
2527	case MARK_READ_MAX_LEN:
2528	case STACK:
2529	case SIZE:
2530	case JITED_SIZE:
2531	case MEMORY_PEAK:
2532		*val = s ? s->stats[id] : 0;
2533		break;
2534	default:
2535		fprintf(stderr, "Unrecognized stat #%d\n", id);
2536		exit(1);
2537	}
2538}
2539
2540static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last)
2541{
2542	int i;
2543
2544	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2545		int id = env.output_spec.ids[i];
2546		int *max_len = &env.output_spec.lens[i], len;
2547		const char *str = NULL;
2548		long val = 0;
2549
2550		prepare_value(s, id, &str, &val);
2551
2552		switch (fmt) {
2553		case RESFMT_TABLE_CALCLEN:
2554			if (str)
2555				len = snprintf(NULL, 0, "%s", str);
2556			else
2557				len = snprintf(NULL, 0, "%ld", val);
2558			if (len > *max_len)
2559				*max_len = len;
2560			break;
2561		case RESFMT_TABLE:
2562			if (str)
2563				printf("%s%-*s", i == 0 ? "" : COLUMN_SEP, *max_len, str);
2564			else
2565				printf("%s%*ld", i == 0 ? "" : COLUMN_SEP,  *max_len, val);
2566			if (i == env.output_spec.spec_cnt - 1)
2567				printf("\n");
2568			break;
2569		case RESFMT_CSV:
2570			if (str)
2571				printf("%s%s", i == 0 ? "" : ",", str);
2572			else
2573				printf("%s%ld", i == 0 ? "" : ",", val);
2574			if (i == env.output_spec.spec_cnt - 1)
2575				printf("\n");
2576			break;
2577		}
2578	}
2579
2580	if (last && fmt == RESFMT_TABLE) {
2581		output_header_underlines();
2582		printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n",
2583		       env.files_processed, env.progs_processed, env.files_skipped, env.progs_skipped);
2584	}
2585}
2586
2587static int parse_stat_value(const char *str, enum stat_id id, struct verif_stats *st)
2588{
2589	switch (id) {
2590	case FILE_NAME:
2591		st->file_name = strdup(str);
2592		if (!st->file_name)
2593			return -ENOMEM;
2594		break;
2595	case PROG_NAME:
2596		st->prog_name = strdup(str);
2597		if (!st->prog_name)
2598			return -ENOMEM;
2599		break;
2600	case VERDICT:
2601		if (strcmp(str, "success") == 0) {
2602			st->stats[VERDICT] = true;
2603		} else if (strcmp(str, "failure") == 0) {
2604			st->stats[VERDICT] = false;
2605		} else {
2606			fprintf(stderr, "Unrecognized verification verdict '%s'\n", str);
2607			return -EINVAL;
2608		}
2609		break;
2610	case DURATION:
2611	case TOTAL_INSNS:
2612	case TOTAL_STATES:
2613	case PEAK_STATES:
2614	case MAX_STATES_PER_INSN:
2615	case MARK_READ_MAX_LEN:
2616	case SIZE:
2617	case JITED_SIZE:
2618	case MEMORY_PEAK:
2619	case STACK: {
2620		long val;
2621		int err, n;
2622
2623		if (sscanf(str, "%ld %n", &val, &n) != 1 || n != strlen(str)) {
2624			err = -errno;
2625			fprintf(stderr, "Failed to parse '%s' as integer\n", str);
2626			return err;
2627		}
2628
2629		st->stats[id] = val;
2630		break;
2631	}
2632	case PROG_TYPE: {
2633		enum bpf_prog_type prog_type = 0;
2634		const char *type;
2635
2636		while ((type = libbpf_bpf_prog_type_str(prog_type)))  {
2637			if (strcmp(type, str) == 0) {
2638				st->stats[id] = prog_type;
2639				break;
2640			}
2641			prog_type++;
2642		}
2643
2644		if (!type) {
2645			fprintf(stderr, "Unrecognized prog type %s\n", str);
2646			return -EINVAL;
2647		}
2648		break;
2649	}
2650	case ATTACH_TYPE: {
2651		enum bpf_attach_type attach_type = 0;
2652		const char *type;
2653
2654		while ((type = libbpf_bpf_attach_type_str(attach_type)))  {
2655			if (strcmp(type, str) == 0) {
2656				st->stats[id] = attach_type;
2657				break;
2658			}
2659			attach_type++;
2660		}
2661
2662		if (!type) {
2663			fprintf(stderr, "Unrecognized attach type %s\n", str);
2664			return -EINVAL;
2665		}
2666		break;
2667	}
2668	default:
2669		fprintf(stderr, "Unrecognized stat #%d\n", id);
2670		return -EINVAL;
2671	}
2672	return 0;
2673}
2674
2675static int parse_stats_csv(const char *filename, struct stat_specs *specs,
2676			   struct verif_stats **statsp, int *stat_cntp)
2677{
2678	char line[4096];
2679	FILE *f;
2680	int err = 0;
2681	bool header = true;
2682
2683	f = fopen(filename, "r");
2684	if (!f) {
2685		err = -errno;
2686		fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
2687		return err;
2688	}
2689
2690	*stat_cntp = 0;
2691
2692	while (fgets(line, sizeof(line), f)) {
2693		char *input = line, *state = NULL, *next;
2694		struct verif_stats *st = NULL;
2695		int col = 0, cnt = 0;
2696
2697		if (!header) {
2698			void *tmp;
2699
2700			tmp = realloc(*statsp, (*stat_cntp + 1) * sizeof(**statsp));
2701			if (!tmp) {
2702				err = -ENOMEM;
2703				goto cleanup;
2704			}
2705			*statsp = tmp;
2706
2707			st = &(*statsp)[*stat_cntp];
2708			memset(st, 0, sizeof(*st));
2709
2710			*stat_cntp += 1;
2711		}
2712
2713		while ((next = strtok_r(cnt++ ? NULL : input, ",\n", &state))) {
2714			if (header) {
2715				/* for the first line, set up spec stats */
2716				err = parse_stat(next, specs);
2717				if (err)
2718					goto cleanup;
2719				continue;
2720			}
2721
2722			/* for all other lines, parse values based on spec */
2723			if (col >= specs->spec_cnt) {
2724				fprintf(stderr, "Found extraneous column #%d in row #%d of '%s'\n",
2725					col, *stat_cntp, filename);
2726				err = -EINVAL;
2727				goto cleanup;
2728			}
2729			err = parse_stat_value(next, specs->ids[col], st);
2730			if (err)
2731				goto cleanup;
2732			col++;
2733		}
2734
2735		if (header) {
2736			header = false;
2737			continue;
2738		}
2739
2740		if (col < specs->spec_cnt) {
2741			fprintf(stderr, "Not enough columns in row #%d in '%s'\n",
2742				*stat_cntp, filename);
2743			err = -EINVAL;
2744			goto cleanup;
2745		}
2746
2747		if (!st->file_name || !st->prog_name) {
2748			fprintf(stderr, "Row #%d in '%s' is missing file and/or program name\n",
2749				*stat_cntp, filename);
2750			err = -EINVAL;
2751			goto cleanup;
2752		}
2753
2754		/* in comparison mode we can only check filters after we
2755		 * parsed entire line; if row should be ignored we pretend we
2756		 * never parsed it
2757		 */
2758		if (!should_process_file_prog(st->file_name, st->prog_name)) {
2759			free(st->file_name);
2760			free(st->prog_name);
2761			*stat_cntp -= 1;
2762		}
2763	}
2764
2765	if (!feof(f)) {
2766		err = -errno;
2767		fprintf(stderr, "Failed I/O for '%s': %d\n", filename, err);
2768	}
2769
2770cleanup:
2771	fclose(f);
2772	return err;
2773}
2774
2775/* empty/zero stats for mismatched rows */
2776static const struct verif_stats fallback_stats = { .file_name = "", .prog_name = "" };
2777
2778static bool is_key_stat(enum stat_id id)
2779{
2780	return id == FILE_NAME || id == PROG_NAME;
2781}
2782
2783static void output_comp_header_underlines(void)
2784{
2785	int i, j, k;
2786
2787	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2788		int id = env.output_spec.ids[i];
2789		int max_j = is_key_stat(id) ? 1 : 3;
2790
2791		for (j = 0; j < max_j; j++) {
2792			int len = env.output_spec.lens[3 * i + j];
2793
2794			printf("%s", i + j == 0 ? "" : COLUMN_SEP);
2795
2796			for (k = 0; k < len; k++)
2797				printf("%c", HEADER_CHAR);
2798		}
2799	}
2800	printf("\n");
2801}
2802
2803static void output_comp_headers(enum resfmt fmt)
2804{
2805	static const char *table_sfxs[3] = {" (A)", " (B)", " (DIFF)"};
2806	static const char *name_sfxs[3] = {"_base", "_comp", "_diff"};
2807	int i, j, len;
2808
2809	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2810		int id = env.output_spec.ids[i];
2811		/* key stats don't have A/B/DIFF columns, they are common for both data sets */
2812		int max_j = is_key_stat(id) ? 1 : 3;
2813
2814		for (j = 0; j < max_j; j++) {
2815			int *max_len = &env.output_spec.lens[3 * i + j];
2816			bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1);
2817			const char *sfx;
2818
2819			switch (fmt) {
2820			case RESFMT_TABLE_CALCLEN:
2821				sfx = is_key_stat(id) ? "" : table_sfxs[j];
2822				len = snprintf(NULL, 0, "%s%s", stat_defs[id].header, sfx);
2823				if (len > *max_len)
2824					*max_len = len;
2825				break;
2826			case RESFMT_TABLE:
2827				sfx = is_key_stat(id) ? "" : table_sfxs[j];
2828				printf("%s%-*s%s", i + j == 0 ? "" : COLUMN_SEP,
2829				       *max_len - (int)strlen(sfx), stat_defs[id].header, sfx);
2830				if (last)
2831					printf("\n");
2832				break;
2833			case RESFMT_CSV:
2834				sfx = is_key_stat(id) ? "" : name_sfxs[j];
2835				printf("%s%s%s", i + j == 0 ? "" : ",", stat_defs[id].names[0], sfx);
2836				if (last)
2837					printf("\n");
2838				break;
2839			}
2840		}
2841	}
2842
2843	if (fmt == RESFMT_TABLE)
2844		output_comp_header_underlines();
2845}
2846
2847static void output_comp_stats(const struct verif_stats_join *join_stats,
2848			      enum resfmt fmt, bool last)
2849{
2850	const struct verif_stats *base = join_stats->stats_a;
2851	const struct verif_stats *comp = join_stats->stats_b;
2852	char base_buf[1024] = {}, comp_buf[1024] = {}, diff_buf[1024] = {};
2853	int i;
2854
2855	for (i = 0; i < env.output_spec.spec_cnt; i++) {
2856		int id = env.output_spec.ids[i], len;
2857		int *max_len_base = &env.output_spec.lens[3 * i + 0];
2858		int *max_len_comp = &env.output_spec.lens[3 * i + 1];
2859		int *max_len_diff = &env.output_spec.lens[3 * i + 2];
2860		const char *base_str = NULL, *comp_str = NULL;
2861		long base_val = 0, comp_val = 0, diff_val = 0;
2862
2863		prepare_value(base, id, &base_str, &base_val);
2864		prepare_value(comp, id, &comp_str, &comp_val);
2865
2866		/* normalize all the outputs to be in string buffers for simplicity */
2867		if (is_key_stat(id)) {
2868			/* key stats (file and program name) are always strings */
2869			if (base)
2870				snprintf(base_buf, sizeof(base_buf), "%s", base_str);
2871			else
2872				snprintf(base_buf, sizeof(base_buf), "%s", comp_str);
2873		} else if (base_str) {
2874			snprintf(base_buf, sizeof(base_buf), "%s", base_str);
2875			snprintf(comp_buf, sizeof(comp_buf), "%s", comp_str);
2876			if (!base || !comp)
2877				snprintf(diff_buf, sizeof(diff_buf), "%s", "N/A");
2878			else if (strcmp(base_str, comp_str) == 0)
2879				snprintf(diff_buf, sizeof(diff_buf), "%s", "MATCH");
2880			else
2881				snprintf(diff_buf, sizeof(diff_buf), "%s", "MISMATCH");
2882		} else {
2883			double p = 0.0;
2884
2885			if (base)
2886				snprintf(base_buf, sizeof(base_buf), "%ld", base_val);
2887			else
2888				snprintf(base_buf, sizeof(base_buf), "%s", "N/A");
2889			if (comp)
2890				snprintf(comp_buf, sizeof(comp_buf), "%ld", comp_val);
2891			else
2892				snprintf(comp_buf, sizeof(comp_buf), "%s", "N/A");
2893
2894			diff_val = comp_val - base_val;
2895			if (!base || !comp) {
2896				snprintf(diff_buf, sizeof(diff_buf), "%s", "N/A");
2897			} else {
2898				if (base_val == 0) {
2899					if (comp_val == base_val)
2900						p = 0.0; /* avoid +0 (+100%) case */
2901					else
2902						p = comp_val < base_val ? -100.0 : 100.0;
2903				} else {
2904					 p = diff_val * 100.0 / base_val;
2905				}
2906				snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)", diff_val, p);
2907			}
2908		}
2909
2910		switch (fmt) {
2911		case RESFMT_TABLE_CALCLEN:
2912			len = strlen(base_buf);
2913			if (len > *max_len_base)
2914				*max_len_base = len;
2915			if (!is_key_stat(id)) {
2916				len = strlen(comp_buf);
2917				if (len > *max_len_comp)
2918					*max_len_comp = len;
2919				len = strlen(diff_buf);
2920				if (len > *max_len_diff)
2921					*max_len_diff = len;
2922			}
2923			break;
2924		case RESFMT_TABLE: {
2925			/* string outputs are left-aligned, number outputs are right-aligned */
2926			const char *fmt = base_str ? "%s%-*s" : "%s%*s";
2927
2928			printf(fmt, i == 0 ? "" : COLUMN_SEP, *max_len_base, base_buf);
2929			if (!is_key_stat(id)) {
2930				printf(fmt, COLUMN_SEP, *max_len_comp, comp_buf);
2931				printf(fmt, COLUMN_SEP, *max_len_diff, diff_buf);
2932			}
2933			if (i == env.output_spec.spec_cnt - 1)
2934				printf("\n");
2935			break;
2936		}
2937		case RESFMT_CSV:
2938			printf("%s%s", i == 0 ? "" : ",", base_buf);
2939			if (!is_key_stat(id)) {
2940				printf("%s%s", i == 0 ? "" : ",", comp_buf);
2941				printf("%s%s", i == 0 ? "" : ",", diff_buf);
2942			}
2943			if (i == env.output_spec.spec_cnt - 1)
2944				printf("\n");
2945			break;
2946		}
2947	}
2948
2949	if (last && fmt == RESFMT_TABLE)
2950		output_comp_header_underlines();
2951}
2952
2953static int cmp_stats_key(const struct verif_stats *base, const struct verif_stats *comp)
2954{
2955	int r;
2956
2957	r = strcmp(base->file_name, comp->file_name);
2958	if (r != 0)
2959		return r;
2960	return strcmp(base->prog_name, comp->prog_name);
2961}
2962
2963static bool is_join_stat_filter_matched(struct filter *f, const struct verif_stats_join *stats)
2964{
2965	static const double eps = 1e-9;
2966	const char *str = NULL;
2967	double value = 0.0;
2968
2969	fetch_join_stat_value(stats, f->stat_id, f->stat_var, &str, &value);
2970
2971	if (f->abs)
2972		value = fabs(value);
2973
2974	switch (f->op) {
2975	case OP_EQ: return value > f->value - eps && value < f->value + eps;
2976	case OP_NEQ: return value < f->value - eps || value > f->value + eps;
2977	case OP_LT: return value < f->value - eps;
2978	case OP_LE: return value <= f->value + eps;
2979	case OP_GT: return value > f->value + eps;
2980	case OP_GE: return value >= f->value - eps;
2981	}
2982
2983	fprintf(stderr, "BUG: unknown filter op %d!\n", f->op);
2984	return false;
2985}
2986
2987static bool should_output_join_stats(const struct verif_stats_join *stats)
2988{
2989	struct filter *f;
2990	int i, allow_cnt = 0;
2991
2992	for (i = 0; i < env.deny_filter_cnt; i++) {
2993		f = &env.deny_filters[i];
2994		if (f->kind != FILTER_STAT)
2995			continue;
2996
2997		if (is_join_stat_filter_matched(f, stats))
2998			return false;
2999	}
3000
3001	for (i = 0; i < env.allow_filter_cnt; i++) {
3002		f = &env.allow_filters[i];
3003		if (f->kind != FILTER_STAT)
3004			continue;
3005		allow_cnt++;
3006
3007		if (is_join_stat_filter_matched(f, stats))
3008			return true;
3009	}
3010
3011	/* if there are no stat allowed filters, pass everything through */
3012	return allow_cnt == 0;
3013}
3014
3015static int handle_comparison_mode(void)
3016{
3017	struct stat_specs base_specs = {}, comp_specs = {};
3018	struct stat_specs tmp_sort_spec;
3019	enum resfmt cur_fmt;
3020	int err, i, j, last_idx, cnt;
3021
3022	if (env.filename_cnt != 2) {
3023		fprintf(stderr, "Comparison mode expects exactly two input CSV files!\n\n");
3024		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
3025		return -EINVAL;
3026	}
3027
3028	err = parse_stats_csv(env.filenames[0], &base_specs,
3029			      &env.baseline_stats, &env.baseline_stat_cnt);
3030	if (err) {
3031		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);
3032		return err;
3033	}
3034	err = parse_stats_csv(env.filenames[1], &comp_specs,
3035			      &env.prog_stats, &env.prog_stat_cnt);
3036	if (err) {
3037		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err);
3038		return err;
3039	}
3040
3041	/* To keep it simple we validate that the set and order of stats in
3042	 * both CSVs are exactly the same. This can be lifted with a bit more
3043	 * pre-processing later.
3044	 */
3045	if (base_specs.spec_cnt != comp_specs.spec_cnt) {
3046		fprintf(stderr, "Number of stats in '%s' and '%s' differs (%d != %d)!\n",
3047			env.filenames[0], env.filenames[1],
3048			base_specs.spec_cnt, comp_specs.spec_cnt);
3049		return -EINVAL;
3050	}
3051	for (i = 0; i < base_specs.spec_cnt; i++) {
3052		if (base_specs.ids[i] != comp_specs.ids[i]) {
3053			fprintf(stderr, "Stats composition differs between '%s' and '%s' (%s != %s)!\n",
3054				env.filenames[0], env.filenames[1],
3055				stat_defs[base_specs.ids[i]].names[0],
3056				stat_defs[comp_specs.ids[i]].names[0]);
3057			return -EINVAL;
3058		}
3059	}
3060
3061	/* Replace user-specified sorting spec with file+prog sorting rule to
3062	 * be able to join two datasets correctly. Once we are done, we will
3063	 * restore the original sort spec.
3064	 */
3065	tmp_sort_spec = env.sort_spec;
3066	env.sort_spec = join_sort_spec;
3067	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
3068	qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats);
3069	env.sort_spec = tmp_sort_spec;
3070
3071	/* Join two datasets together. If baseline and comparison datasets
3072	 * have different subset of rows (we match by 'object + prog' as
3073	 * a unique key) then assume empty/missing/zero value for rows that
3074	 * are missing in the opposite data set.
3075	 */
3076	i = j = 0;
3077	while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) {
3078		const struct verif_stats *base, *comp;
3079		struct verif_stats_join *join;
3080		void *tmp;
3081		int r;
3082
3083		base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats;
3084		comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats;
3085
3086		if (!base->file_name || !base->prog_name) {
3087			fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n",
3088				i, env.filenames[0]);
3089			return -EINVAL;
3090		}
3091		if (!comp->file_name || !comp->prog_name) {
3092			fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n",
3093				j, env.filenames[1]);
3094			return -EINVAL;
3095		}
3096
3097		tmp = realloc(env.join_stats, (env.join_stat_cnt + 1) * sizeof(*env.join_stats));
3098		if (!tmp)
3099			return -ENOMEM;
3100		env.join_stats = tmp;
3101
3102		join = &env.join_stats[env.join_stat_cnt];
3103		memset(join, 0, sizeof(*join));
3104
3105		r = cmp_stats_key(base, comp);
3106		if (r == 0) {
3107			join->file_name = base->file_name;
3108			join->prog_name = base->prog_name;
3109			join->stats_a = base;
3110			join->stats_b = comp;
3111			i++;
3112			j++;
3113		} else if (base != &fallback_stats && (comp == &fallback_stats || r < 0)) {
3114			join->file_name = base->file_name;
3115			join->prog_name = base->prog_name;
3116			join->stats_a = base;
3117			join->stats_b = NULL;
3118			i++;
3119		} else if (comp != &fallback_stats && (base == &fallback_stats || r > 0)) {
3120			join->file_name = comp->file_name;
3121			join->prog_name = comp->prog_name;
3122			join->stats_a = NULL;
3123			join->stats_b = comp;
3124			j++;
3125		} else {
3126			fprintf(stderr, "%s:%d: should never reach here i=%i, j=%i",
3127				__FILE__, __LINE__, i, j);
3128			return -EINVAL;
3129		}
3130		env.join_stat_cnt += 1;
3131	}
3132
3133	/* now sort joined results according to sort spec */
3134	qsort(env.join_stats, env.join_stat_cnt, sizeof(*env.join_stats), cmp_join_stats);
3135
3136	/* for human-readable table output we need to do extra pass to
3137	 * calculate column widths, so we substitute current output format
3138	 * with RESFMT_TABLE_CALCLEN and later revert it back to RESFMT_TABLE
3139	 * and do everything again.
3140	 */
3141	if (env.out_fmt == RESFMT_TABLE)
3142		cur_fmt = RESFMT_TABLE_CALCLEN;
3143	else
3144		cur_fmt = env.out_fmt;
3145
3146one_more_time:
3147	output_comp_headers(cur_fmt);
3148
3149	last_idx = -1;
3150	cnt = 0;
3151	for (i = 0; i < env.join_stat_cnt; i++) {
3152		const struct verif_stats_join *join = &env.join_stats[i];
3153
3154		if (!should_output_join_stats(join))
3155			continue;
3156
3157		if (env.top_n && cnt >= env.top_n)
3158			break;
3159
3160		if (cur_fmt == RESFMT_TABLE_CALCLEN)
3161			last_idx = i;
3162
3163		output_comp_stats(join, cur_fmt, i == last_idx);
3164
3165		cnt++;
3166	}
3167
3168	if (cur_fmt == RESFMT_TABLE_CALCLEN) {
3169		cur_fmt = RESFMT_TABLE;
3170		goto one_more_time; /* ... this time with feeling */
3171	}
3172
3173	return 0;
3174}
3175
3176static bool is_stat_filter_matched(struct filter *f, const struct verif_stats *stats)
3177{
3178	long value = stats->stats[f->stat_id];
3179
3180	if (f->abs)
3181		value = value < 0 ? -value : value;
3182
3183	switch (f->op) {
3184	case OP_EQ: return value == f->value;
3185	case OP_NEQ: return value != f->value;
3186	case OP_LT: return value < f->value;
3187	case OP_LE: return value <= f->value;
3188	case OP_GT: return value > f->value;
3189	case OP_GE: return value >= f->value;
3190	}
3191
3192	fprintf(stderr, "BUG: unknown filter op %d!\n", f->op);
3193	return false;
3194}
3195
3196static bool should_output_stats(const struct verif_stats *stats)
3197{
3198	struct filter *f;
3199	int i, allow_cnt = 0;
3200
3201	for (i = 0; i < env.deny_filter_cnt; i++) {
3202		f = &env.deny_filters[i];
3203		if (f->kind != FILTER_STAT)
3204			continue;
3205
3206		if (is_stat_filter_matched(f, stats))
3207			return false;
3208	}
3209
3210	for (i = 0; i < env.allow_filter_cnt; i++) {
3211		f = &env.allow_filters[i];
3212		if (f->kind != FILTER_STAT)
3213			continue;
3214		allow_cnt++;
3215
3216		if (is_stat_filter_matched(f, stats))
3217			return true;
3218	}
3219
3220	/* if there are no stat allowed filters, pass everything through */
3221	return allow_cnt == 0;
3222}
3223
3224static void output_prog_stats(void)
3225{
3226	const struct verif_stats *stats;
3227	int i, last_stat_idx = 0, cnt = 0;
3228
3229	if (env.out_fmt == RESFMT_TABLE) {
3230		/* calculate column widths */
3231		output_headers(RESFMT_TABLE_CALCLEN);
3232		for (i = 0; i < env.prog_stat_cnt; i++) {
3233			stats = &env.prog_stats[i];
3234			if (!should_output_stats(stats))
3235				continue;
3236			output_stats(stats, RESFMT_TABLE_CALCLEN, false);
3237			last_stat_idx = i;
3238		}
3239	}
3240
3241	/* actually output the table */
3242	output_headers(env.out_fmt);
3243	for (i = 0; i < env.prog_stat_cnt; i++) {
3244		stats = &env.prog_stats[i];
3245		if (!should_output_stats(stats))
3246			continue;
3247		if (env.top_n && cnt >= env.top_n)
3248			break;
3249		output_stats(stats, env.out_fmt, i == last_stat_idx);
3250		cnt++;
3251	}
3252}
3253
3254static int handle_verif_mode(void)
3255{
3256	int i, err = 0;
3257
3258	if (env.filename_cnt == 0) {
3259		fprintf(stderr, "Please provide path to BPF object file!\n\n");
3260		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
3261		return -EINVAL;
3262	}
3263
3264	create_stat_cgroup();
3265	for (i = 0; i < env.filename_cnt; i++) {
3266		err = process_obj(env.filenames[i]);
3267		if (err) {
3268			fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err);
3269			goto out;
3270		}
3271	}
3272
3273	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
3274
3275	output_prog_stats();
3276
3277out:
3278	destroy_stat_cgroup();
3279	return err;
3280}
3281
3282static int handle_replay_mode(void)
3283{
3284	struct stat_specs specs = {};
3285	int err;
3286
3287	if (env.filename_cnt != 1) {
3288		fprintf(stderr, "Replay mode expects exactly one input CSV file!\n\n");
3289		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
3290		return -EINVAL;
3291	}
3292
3293	err = parse_stats_csv(env.filenames[0], &specs,
3294			      &env.prog_stats, &env.prog_stat_cnt);
3295	if (err) {
3296		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);
3297		return err;
3298	}
3299
3300	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
3301
3302	output_prog_stats();
3303
3304	return 0;
3305}
3306
3307int main(int argc, char **argv)
3308{
3309	int err = 0, i, j;
3310
3311	if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
3312		return 1;
3313
3314	if (env.show_version) {
3315		printf("%s\n", argp_program_version);
3316		return 0;
3317	}
3318
3319	if (env.verbose && env.quiet) {
3320		fprintf(stderr, "Verbose and quiet modes are incompatible, please specify just one or neither!\n\n");
3321		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
3322		return 1;
3323	}
3324	if (env.verbose && env.log_level == 0)
3325		env.log_level = 1;
3326
3327	if (env.output_spec.spec_cnt == 0) {
3328		if (env.out_fmt == RESFMT_CSV)
3329			env.output_spec = default_csv_output_spec;
3330		else
3331			env.output_spec = default_output_spec;
3332	}
3333	if (env.sort_spec.spec_cnt == 0)
3334		env.sort_spec = default_sort_spec;
3335
3336	if (env.comparison_mode && env.replay_mode) {
3337		fprintf(stderr, "Can't specify replay and comparison mode at the same time!\n\n");
3338		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
3339		return 1;
3340	}
3341
3342	if (env.comparison_mode)
3343		err = handle_comparison_mode();
3344	else if (env.replay_mode)
3345		err = handle_replay_mode();
3346	else
3347		err = handle_verif_mode();
3348
3349	free_verif_stats(env.prog_stats, env.prog_stat_cnt);
3350	free_verif_stats(env.baseline_stats, env.baseline_stat_cnt);
3351	free(env.join_stats);
3352	for (i = 0; i < env.filename_cnt; i++)
3353		free(env.filenames[i]);
3354	free(env.filenames);
3355	for (i = 0; i < env.allow_filter_cnt; i++) {
3356		free(env.allow_filters[i].any_glob);
3357		free(env.allow_filters[i].file_glob);
3358		free(env.allow_filters[i].prog_glob);
3359	}
3360	free(env.allow_filters);
3361	for (i = 0; i < env.deny_filter_cnt; i++) {
3362		free(env.deny_filters[i].any_glob);
3363		free(env.deny_filters[i].file_glob);
3364		free(env.deny_filters[i].prog_glob);
3365	}
3366	free(env.deny_filters);
3367	for (i = 0; i < env.npresets; ++i) {
3368		free(env.presets[i].full_name);
3369		for (j = 0; j < env.presets[i].atom_count; ++j) {
3370			switch (env.presets[i].atoms[j].type) {
3371			case FIELD_NAME:
3372				free(env.presets[i].atoms[j].name);
3373				break;
3374			case ARRAY_INDEX:
3375				if (env.presets[i].atoms[j].index.type == ENUMERATOR)
3376					free(env.presets[i].atoms[j].index.svalue);
3377				break;
3378			}
3379		}
3380		free(env.presets[i].atoms);
3381	}
3382	free(env.presets);
3383	return -err;
3384}