FILE
bpf_counter.c
File preview with syntax highlighting.
linux-6.19.14/tools/perf/util/bpf_counter.c
1// SPDX-License-Identifier: GPL-2.0 2 3/* Copyright (c) 2019 Facebook */ 4 5#include <assert.h> 6#include <limits.h> 7#include <unistd.h> 8#include <sys/file.h> 9#include <sys/resource.h> 10#include <sys/time.h> 11#include <linux/err.h> 12#include <linux/list.h> 13#include <linux/zalloc.h> 14#include <api/fs/fs.h> 15#include <bpf/bpf.h> 16#include <bpf/btf.h> 17#include <perf/bpf_perf.h> 18 19#include "bpf_counter.h" 20#include "bpf-utils.h" 21#include "counts.h" 22#include "debug.h" 23#include "evsel.h" 24#include "evlist.h" 25#include "target.h" 26#include "cgroup.h" 27#include "cpumap.h" 28#include "thread_map.h" 29 30#include "bpf_skel/bpf_prog_profiler.skel.h" 31#include "bpf_skel/bperf_u.h" 32#include "bpf_skel/bperf_leader.skel.h" 33#include "bpf_skel/bperf_follower.skel.h" 34 35struct bpf_counter { 36 void *skel; 37 struct list_head list; 38}; 39 40#define ATTR_MAP_SIZE 16 41 42static void *u64_to_ptr(__u64 ptr) 43{ 44 return (void *)(unsigned long)ptr; 45} 46 47 48void set_max_rlimit(void) 49{ 50 struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; 51 52 setrlimit(RLIMIT_MEMLOCK, &rinf); 53} 54 55static __u32 bpf_link_get_id(int fd) 56{ 57 struct bpf_link_info link_info = { .id = 0, }; 58 __u32 link_info_len = sizeof(link_info); 59 60 bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); 61 return link_info.id; 62} 63 64static __u32 bpf_link_get_prog_id(int fd) 65{ 66 struct bpf_link_info link_info = { .id = 0, }; 67 __u32 link_info_len = sizeof(link_info); 68 69 bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); 70 return link_info.prog_id; 71} 72 73static __u32 bpf_map_get_id(int fd) 74{ 75 struct bpf_map_info map_info = { .id = 0, }; 76 __u32 map_info_len = sizeof(map_info); 77 78 bpf_obj_get_info_by_fd(fd, &map_info, &map_info_len); 79 return map_info.id; 80} 81 82/* trigger the leader program on a cpu */ 83int bperf_trigger_reading(int prog_fd, int cpu) 84{ 85 DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, 86 .ctx_in = NULL, 87 .ctx_size_in = 0, 88 .flags = BPF_F_TEST_RUN_ON_CPU, 89 .cpu = cpu, 90 .retval = 0, 91 ); 92 93 return bpf_prog_test_run_opts(prog_fd, &opts); 94} 95 96static struct bpf_counter *bpf_counter_alloc(void) 97{ 98 struct bpf_counter *counter; 99 100 counter = zalloc(sizeof(*counter)); 101 if (counter) 102 INIT_LIST_HEAD(&counter->list); 103 return counter; 104} 105 106static int bpf_program_profiler__destroy(struct evsel *evsel) 107{ 108 struct bpf_counter *counter, *tmp; 109 110 list_for_each_entry_safe(counter, tmp, 111 &evsel->bpf_counter_list, list) { 112 list_del_init(&counter->list); 113 bpf_prog_profiler_bpf__destroy(counter->skel); 114 free(counter); 115 } 116 assert(list_empty(&evsel->bpf_counter_list)); 117 118 return 0; 119} 120 121static char *bpf_target_prog_name(int tgt_fd) 122{ 123 struct bpf_func_info *func_info; 124 struct perf_bpil *info_linear; 125 const struct btf_type *t; 126 struct btf *btf = NULL; 127 char *name = NULL; 128 129 info_linear = get_bpf_prog_info_linear(tgt_fd, 1UL << PERF_BPIL_FUNC_INFO); 130 if (IS_ERR_OR_NULL(info_linear)) { 131 pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd); 132 return NULL; 133 } 134 135 if (info_linear->info.btf_id == 0) { 136 pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd); 137 goto out; 138 } 139 140 btf = btf__load_from_kernel_by_id(info_linear->info.btf_id); 141 if (libbpf_get_error(btf)) { 142 pr_debug("failed to load btf for prog FD %d\n", tgt_fd); 143 goto out; 144 } 145 146 func_info = u64_to_ptr(info_linear->info.func_info); 147 t = btf__type_by_id(btf, func_info[0].type_id); 148 if (!t) { 149 pr_debug("btf %d doesn't have type %d\n", 150 info_linear->info.btf_id, func_info[0].type_id); 151 goto out; 152 } 153 name = strdup(btf__name_by_offset(btf, t->name_off)); 154out: 155 btf__free(btf); 156 free(info_linear); 157 return name; 158} 159 160static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id) 161{ 162 struct bpf_prog_profiler_bpf *skel; 163 struct bpf_counter *counter; 164 struct bpf_program *prog; 165 char *prog_name = NULL; 166 int prog_fd; 167 int err; 168 169 prog_fd = bpf_prog_get_fd_by_id(prog_id); 170 if (prog_fd < 0) { 171 pr_err("Failed to open fd for bpf prog %u\n", prog_id); 172 return -1; 173 } 174 counter = bpf_counter_alloc(); 175 if (!counter) { 176 close(prog_fd); 177 return -1; 178 } 179 180 skel = bpf_prog_profiler_bpf__open(); 181 if (!skel) { 182 pr_err("Failed to open bpf skeleton\n"); 183 goto err_out; 184 } 185 186 skel->rodata->num_cpu = evsel__nr_cpus(evsel); 187 188 bpf_map__set_max_entries(skel->maps.events, evsel__nr_cpus(evsel)); 189 bpf_map__set_max_entries(skel->maps.fentry_readings, 1); 190 bpf_map__set_max_entries(skel->maps.accum_readings, 1); 191 192 prog_name = bpf_target_prog_name(prog_fd); 193 if (!prog_name) { 194 pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id); 195 goto err_out; 196 } 197 198 bpf_object__for_each_program(prog, skel->obj) { 199 err = bpf_program__set_attach_target(prog, prog_fd, prog_name); 200 if (err) { 201 pr_err("bpf_program__set_attach_target failed.\n" 202 "Does bpf prog %u have BTF?\n", prog_id); 203 goto err_out; 204 } 205 } 206 set_max_rlimit(); 207 err = bpf_prog_profiler_bpf__load(skel); 208 if (err) { 209 pr_err("bpf_prog_profiler_bpf__load failed\n"); 210 goto err_out; 211 } 212 213 assert(skel != NULL); 214 counter->skel = skel; 215 list_add(&counter->list, &evsel->bpf_counter_list); 216 free(prog_name); 217 close(prog_fd); 218 return 0; 219err_out: 220 bpf_prog_profiler_bpf__destroy(skel); 221 free(prog_name); 222 free(counter); 223 close(prog_fd); 224 return -1; 225} 226 227static int bpf_program_profiler__load(struct evsel *evsel, struct target *target) 228{ 229 char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p; 230 u32 prog_id; 231 int ret; 232 233 bpf_str_ = bpf_str = strdup(target->bpf_str); 234 if (!bpf_str) 235 return -1; 236 237 while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) { 238 prog_id = strtoul(tok, &p, 10); 239 if (prog_id == 0 || prog_id == UINT_MAX || 240 (*p != '\0' && *p != ',')) { 241 pr_err("Failed to parse bpf prog ids %s\n", 242 target->bpf_str); 243 free(bpf_str_); 244 return -1; 245 } 246 247 ret = bpf_program_profiler_load_one(evsel, prog_id); 248 if (ret) { 249 bpf_program_profiler__destroy(evsel); 250 free(bpf_str_); 251 return -1; 252 } 253 bpf_str = NULL; 254 } 255 free(bpf_str_); 256 return 0; 257} 258 259static int bpf_program_profiler__enable(struct evsel *evsel) 260{ 261 struct bpf_counter *counter; 262 int ret; 263 264 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 265 assert(counter->skel != NULL); 266 ret = bpf_prog_profiler_bpf__attach(counter->skel); 267 if (ret) { 268 bpf_program_profiler__destroy(evsel); 269 return ret; 270 } 271 } 272 return 0; 273} 274 275static int bpf_program_profiler__disable(struct evsel *evsel) 276{ 277 struct bpf_counter *counter; 278 279 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 280 assert(counter->skel != NULL); 281 bpf_prog_profiler_bpf__detach(counter->skel); 282 } 283 return 0; 284} 285 286static int bpf_program_profiler__read(struct evsel *evsel) 287{ 288 // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible 289 // Sometimes possible > online, like on a Ryzen 3900X that has 24 290 // threads but its possible showed 0-31 -acme 291 int num_cpu_bpf = libbpf_num_possible_cpus(); 292 struct bpf_perf_event_value values[num_cpu_bpf]; 293 struct bpf_counter *counter; 294 struct perf_counts_values *counts; 295 int reading_map_fd; 296 __u32 key = 0; 297 int err, idx, bpf_cpu; 298 299 if (list_empty(&evsel->bpf_counter_list)) 300 return -EAGAIN; 301 302 perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) { 303 counts = perf_counts(evsel->counts, idx, 0); 304 counts->val = 0; 305 counts->ena = 0; 306 counts->run = 0; 307 } 308 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 309 struct bpf_prog_profiler_bpf *skel = counter->skel; 310 311 assert(skel != NULL); 312 reading_map_fd = bpf_map__fd(skel->maps.accum_readings); 313 314 err = bpf_map_lookup_elem(reading_map_fd, &key, values); 315 if (err) { 316 pr_err("failed to read value\n"); 317 return err; 318 } 319 320 for (bpf_cpu = 0; bpf_cpu < num_cpu_bpf; bpf_cpu++) { 321 idx = perf_cpu_map__idx(evsel__cpus(evsel), 322 (struct perf_cpu){.cpu = bpf_cpu}); 323 if (idx == -1) 324 continue; 325 counts = perf_counts(evsel->counts, idx, 0); 326 counts->val += values[bpf_cpu].counter; 327 counts->ena += values[bpf_cpu].enabled; 328 counts->run += values[bpf_cpu].running; 329 } 330 } 331 return 0; 332} 333 334static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu_map_idx, 335 int fd) 336{ 337 struct bpf_prog_profiler_bpf *skel; 338 struct bpf_counter *counter; 339 int cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx).cpu; 340 int ret; 341 342 list_for_each_entry(counter, &evsel->bpf_counter_list, list) { 343 skel = counter->skel; 344 assert(skel != NULL); 345 346 ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events), 347 &cpu, &fd, BPF_ANY); 348 if (ret) 349 return ret; 350 } 351 return 0; 352} 353 354struct bpf_counter_ops bpf_program_profiler_ops = { 355 .load = bpf_program_profiler__load, 356 .enable = bpf_program_profiler__enable, 357 .disable = bpf_program_profiler__disable, 358 .read = bpf_program_profiler__read, 359 .destroy = bpf_program_profiler__destroy, 360 .install_pe = bpf_program_profiler__install_pe, 361}; 362 363static bool bperf_attr_map_compatible(int attr_map_fd) 364{ 365 struct bpf_map_info map_info = {0}; 366 __u32 map_info_len = sizeof(map_info); 367 int err; 368 369 err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len); 370 371 if (err) 372 return false; 373 return (map_info.key_size == sizeof(struct perf_event_attr)) && 374 (map_info.value_size == sizeof(struct perf_event_attr_map_entry)); 375} 376 377static int bperf_lock_attr_map(struct target *target) 378{ 379 char path[PATH_MAX]; 380 int map_fd, err; 381 382 if (target->attr_map) { 383 scnprintf(path, PATH_MAX, "%s", target->attr_map); 384 } else { 385 scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(), 386 BPF_PERF_DEFAULT_ATTR_MAP_PATH); 387 } 388 389 if (access(path, F_OK)) { 390 map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, 391 sizeof(struct perf_event_attr), 392 sizeof(struct perf_event_attr_map_entry), 393 ATTR_MAP_SIZE, NULL); 394 if (map_fd < 0) 395 return -1; 396 397 err = bpf_obj_pin(map_fd, path); 398 if (err) { 399 /* someone pinned the map in parallel? */ 400 close(map_fd); 401 map_fd = bpf_obj_get(path); 402 if (map_fd < 0) 403 return -1; 404 } 405 } else { 406 map_fd = bpf_obj_get(path); 407 if (map_fd < 0) 408 return -1; 409 } 410 411 if (!bperf_attr_map_compatible(map_fd)) { 412 close(map_fd); 413 return -1; 414 415 } 416 err = flock(map_fd, LOCK_EX); 417 if (err) { 418 close(map_fd); 419 return -1; 420 } 421 return map_fd; 422} 423 424static int bperf_check_target(struct evsel *evsel, 425 struct target *target, 426 enum bperf_filter_type *filter_type, 427 __u32 *filter_entry_cnt) 428{ 429 if (evsel->core.leader->nr_members > 1) { 430 pr_err("bpf managed perf events do not yet support groups.\n"); 431 return -1; 432 } 433 434 /* determine filter type based on target */ 435 if (target->system_wide) { 436 *filter_type = BPERF_FILTER_GLOBAL; 437 *filter_entry_cnt = 1; 438 } else if (target->cpu_list) { 439 *filter_type = BPERF_FILTER_CPU; 440 *filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel)); 441 } else if (target->tid) { 442 *filter_type = BPERF_FILTER_PID; 443 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); 444 } else if (target->pid || evsel->evlist->workload.pid != -1) { 445 *filter_type = BPERF_FILTER_TGID; 446 *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); 447 } else { 448 pr_err("bpf managed perf events do not yet support these targets.\n"); 449 return -1; 450 } 451 452 return 0; 453} 454 455static __u32 filter_entry_cnt; 456 457static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, 458 struct perf_event_attr_map_entry *entry) 459{ 460 struct bperf_leader_bpf *skel = bperf_leader_bpf__open(); 461 int link_fd, diff_map_fd, err; 462 struct bpf_link *link = NULL; 463 struct perf_thread_map *threads; 464 465 if (!skel) { 466 pr_err("Failed to open leader skeleton\n"); 467 return -1; 468 } 469 470 bpf_map__set_max_entries(skel->maps.events, libbpf_num_possible_cpus()); 471 err = bperf_leader_bpf__load(skel); 472 if (err) { 473 pr_err("Failed to load leader skeleton\n"); 474 goto out; 475 } 476 477 link = bpf_program__attach(skel->progs.on_switch); 478 if (IS_ERR(link)) { 479 pr_err("Failed to attach leader program\n"); 480 err = PTR_ERR(link); 481 goto out; 482 } 483 484 link_fd = bpf_link__fd(link); 485 diff_map_fd = bpf_map__fd(skel->maps.diff_readings); 486 entry->link_id = bpf_link_get_id(link_fd); 487 entry->diff_map_id = bpf_map_get_id(diff_map_fd); 488 err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY); 489 assert(err == 0); 490 491 evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id); 492 assert(evsel->bperf_leader_link_fd >= 0); 493 494 /* 495 * save leader_skel for install_pe, which is called within 496 * following evsel__open_per_cpu call 497 */ 498 evsel->leader_skel = skel; 499 assert(!perf_cpu_map__has_any_cpu_or_is_empty(evsel->core.cpus)); 500 /* Always open system wide. */ 501 threads = thread_map__new_by_tid(-1); 502 evsel__open(evsel, evsel->core.cpus, threads); 503 perf_thread_map__put(threads); 504 505out: 506 bperf_leader_bpf__destroy(skel); 507 bpf_link__destroy(link); 508 return err; 509} 510 511static int bperf_attach_follower_program(struct bperf_follower_bpf *skel, 512 enum bperf_filter_type filter_type, 513 bool inherit) 514{ 515 struct bpf_link *link; 516 int err = 0; 517 518 if ((filter_type == BPERF_FILTER_PID || 519 filter_type == BPERF_FILTER_TGID) && inherit) 520 /* attach all follower bpf progs to enable event inheritance */ 521 err = bperf_follower_bpf__attach(skel); 522 else { 523 link = bpf_program__attach(skel->progs.fexit_XXX); 524 if (IS_ERR(link)) 525 err = PTR_ERR(link); 526 } 527 528 return err; 529} 530 531static int bperf__load(struct evsel *evsel, struct target *target) 532{ 533 struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff}; 534 int attr_map_fd, diff_map_fd = -1, err; 535 enum bperf_filter_type filter_type; 536 __u32 i; 537 538 if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt)) 539 return -1; 540 541 evsel->bperf_leader_prog_fd = -1; 542 evsel->bperf_leader_link_fd = -1; 543 544 /* 545 * Step 1: hold a fd on the leader program and the bpf_link, if 546 * the program is not already gone, reload the program. 547 * Use flock() to ensure exclusive access to the perf_event_attr 548 * map. 549 */ 550 attr_map_fd = bperf_lock_attr_map(target); 551 if (attr_map_fd < 0) { 552 pr_err("Failed to lock perf_event_attr map\n"); 553 return -1; 554 } 555 556 err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry); 557 if (err) { 558 err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY); 559 if (err) 560 goto out; 561 } 562 563 evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id); 564 if (evsel->bperf_leader_link_fd < 0 && 565 bperf_reload_leader_program(evsel, attr_map_fd, &entry)) { 566 err = -1; 567 goto out; 568 } 569 /* 570 * The bpf_link holds reference to the leader program, and the 571 * leader program holds reference to the maps. Therefore, if 572 * link_id is valid, diff_map_id should also be valid. 573 */ 574 evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id( 575 bpf_link_get_prog_id(evsel->bperf_leader_link_fd)); 576 assert(evsel->bperf_leader_prog_fd >= 0); 577 578 diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id); 579 assert(diff_map_fd >= 0); 580 581 /* 582 * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check 583 * whether the kernel support it 584 */ 585 err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0); 586 if (err) { 587 pr_err("The kernel does not support test_run for raw_tp BPF programs.\n" 588 "Therefore, --use-bpf might show inaccurate readings\n"); 589 goto out; 590 } 591 592 /* Step 2: load the follower skeleton */ 593 evsel->follower_skel = bperf_follower_bpf__open(); 594 if (!evsel->follower_skel) { 595 err = -1; 596 pr_err("Failed to open follower skeleton\n"); 597 goto out; 598 } 599 600 /* attach fexit program to the leader program */ 601 bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX, 602 evsel->bperf_leader_prog_fd, "on_switch"); 603 604 /* connect to leader diff_reading map */ 605 bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd); 606 607 /* set up reading map */ 608 bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings, 609 filter_entry_cnt); 610 err = bperf_follower_bpf__load(evsel->follower_skel); 611 if (err) { 612 pr_err("Failed to load follower skeleton\n"); 613 bperf_follower_bpf__destroy(evsel->follower_skel); 614 evsel->follower_skel = NULL; 615 goto out; 616 } 617 618 for (i = 0; i < filter_entry_cnt; i++) { 619 int filter_map_fd; 620 __u32 key; 621 struct bperf_filter_value fval = { i, 0 }; 622 623 if (filter_type == BPERF_FILTER_PID || 624 filter_type == BPERF_FILTER_TGID) 625 key = perf_thread_map__pid(evsel->core.threads, i); 626 else if (filter_type == BPERF_FILTER_CPU) 627 key = perf_cpu_map__cpu(evsel->core.cpus, i).cpu; 628 else 629 break; 630 631 filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter); 632 bpf_map_update_elem(filter_map_fd, &key, &fval, BPF_ANY); 633 } 634 635 evsel->follower_skel->bss->type = filter_type; 636 evsel->follower_skel->bss->inherit = target->inherit; 637 638 err = bperf_attach_follower_program(evsel->follower_skel, filter_type, 639 target->inherit); 640 641out: 642 if (err && evsel->bperf_leader_link_fd >= 0) 643 close(evsel->bperf_leader_link_fd); 644 if (err && evsel->bperf_leader_prog_fd >= 0) 645 close(evsel->bperf_leader_prog_fd); 646 if (diff_map_fd >= 0) 647 close(diff_map_fd); 648 649 flock(attr_map_fd, LOCK_UN); 650 close(attr_map_fd); 651 652 return err; 653} 654 655static int bperf__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) 656{ 657 struct bperf_leader_bpf *skel = evsel->leader_skel; 658 int cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx).cpu; 659 660 return bpf_map_update_elem(bpf_map__fd(skel->maps.events), 661 &cpu, &fd, BPF_ANY); 662} 663 664/* 665 * trigger the leader prog on each cpu, so the accum_reading map could get 666 * the latest readings. 667 */ 668static int bperf_sync_counters(struct evsel *evsel) 669{ 670 struct perf_cpu cpu; 671 int idx; 672 673 perf_cpu_map__for_each_cpu(cpu, idx, evsel->core.cpus) 674 bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu.cpu); 675 676 return 0; 677} 678 679static int bperf__enable(struct evsel *evsel) 680{ 681 evsel->follower_skel->bss->enabled = 1; 682 return 0; 683} 684 685static int bperf__disable(struct evsel *evsel) 686{ 687 evsel->follower_skel->bss->enabled = 0; 688 return 0; 689} 690 691static int bperf__read(struct evsel *evsel) 692{ 693 struct bperf_follower_bpf *skel = evsel->follower_skel; 694 __u32 num_cpu_bpf = cpu__max_cpu().cpu; 695 struct bpf_perf_event_value values[num_cpu_bpf]; 696 struct perf_counts_values *counts; 697 int reading_map_fd, err = 0; 698 __u32 i; 699 int j; 700 701 bperf_sync_counters(evsel); 702 reading_map_fd = bpf_map__fd(skel->maps.accum_readings); 703 704 for (i = 0; i < filter_entry_cnt; i++) { 705 struct perf_cpu entry; 706 __u32 cpu; 707 708 err = bpf_map_lookup_elem(reading_map_fd, &i, values); 709 if (err) 710 goto out; 711 switch (evsel->follower_skel->bss->type) { 712 case BPERF_FILTER_GLOBAL: 713 assert(i == 0); 714 715 perf_cpu_map__for_each_cpu(entry, j, evsel__cpus(evsel)) { 716 counts = perf_counts(evsel->counts, j, 0); 717 counts->val = values[entry.cpu].counter; 718 counts->ena = values[entry.cpu].enabled; 719 counts->run = values[entry.cpu].running; 720 } 721 break; 722 case BPERF_FILTER_CPU: 723 cpu = perf_cpu_map__cpu(evsel__cpus(evsel), i).cpu; 724 assert(cpu >= 0); 725 counts = perf_counts(evsel->counts, i, 0); 726 counts->val = values[cpu].counter; 727 counts->ena = values[cpu].enabled; 728 counts->run = values[cpu].running; 729 break; 730 case BPERF_FILTER_PID: 731 case BPERF_FILTER_TGID: 732 counts = perf_counts(evsel->counts, 0, i); 733 counts->val = 0; 734 counts->ena = 0; 735 counts->run = 0; 736 737 for (cpu = 0; cpu < num_cpu_bpf; cpu++) { 738 counts->val += values[cpu].counter; 739 counts->ena += values[cpu].enabled; 740 counts->run += values[cpu].running; 741 } 742 break; 743 default: 744 break; 745 } 746 } 747out: 748 return err; 749} 750 751static int bperf__destroy(struct evsel *evsel) 752{ 753 bperf_follower_bpf__destroy(evsel->follower_skel); 754 close(evsel->bperf_leader_prog_fd); 755 close(evsel->bperf_leader_link_fd); 756 return 0; 757} 758 759/* 760 * bperf: share hardware PMCs with BPF 761 * 762 * perf uses performance monitoring counters (PMC) to monitor system 763 * performance. The PMCs are limited hardware resources. For example, 764 * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu. 765 * 766 * Modern data center systems use these PMCs in many different ways: 767 * system level monitoring, (maybe nested) container level monitoring, per 768 * process monitoring, profiling (in sample mode), etc. In some cases, 769 * there are more active perf_events than available hardware PMCs. To allow 770 * all perf_events to have a chance to run, it is necessary to do expensive 771 * time multiplexing of events. 772 * 773 * On the other hand, many monitoring tools count the common metrics 774 * (cycles, instructions). It is a waste to have multiple tools create 775 * multiple perf_events of "cycles" and occupy multiple PMCs. 776 * 777 * bperf tries to reduce such wastes by allowing multiple perf_events of 778 * "cycles" or "instructions" (at different scopes) to share PMUs. Instead 779 * of having each perf-stat session to read its own perf_events, bperf uses 780 * BPF programs to read the perf_events and aggregate readings to BPF maps. 781 * Then, the perf-stat session(s) reads the values from these BPF maps. 782 * 783 * || 784 * shared progs and maps <- || -> per session progs and maps 785 * || 786 * --------------- || 787 * | perf_events | || 788 * --------------- fexit || ----------------- 789 * | --------||----> | follower prog | 790 * --------------- / || --- ----------------- 791 * cs -> | leader prog |/ ||/ | | 792 * --> --------------- /|| -------------- ------------------ 793 * / | | / || | filter map | | accum_readings | 794 * / ------------ ------------ || -------------- ------------------ 795 * | | prev map | | diff map | || | 796 * | ------------ ------------ || | 797 * \ || | 798 * = \ ==================================================== | ============ 799 * \ / user space 800 * \ / 801 * \ / 802 * BPF_PROG_TEST_RUN BPF_MAP_LOOKUP_ELEM 803 * \ / 804 * \ / 805 * \------ perf-stat ----------------------/ 806 * 807 * The figure above shows the architecture of bperf. Note that the figure 808 * is divided into 3 regions: shared progs and maps (top left), per session 809 * progs and maps (top right), and user space (bottom). 810 * 811 * The leader prog is triggered on each context switch (cs). The leader 812 * prog reads perf_events and stores the difference (current_reading - 813 * previous_reading) to the diff map. For the same metric, e.g. "cycles", 814 * multiple perf-stat sessions share the same leader prog. 815 * 816 * Each perf-stat session creates a follower prog as fexit program to the 817 * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38) 818 * follower progs to the same leader prog. The follower prog checks current 819 * task and processor ID to decide whether to add the value from the diff 820 * map to its accumulated reading map (accum_readings). 821 * 822 * Finally, perf-stat user space reads the value from accum_reading map. 823 * 824 * Besides context switch, it is also necessary to trigger the leader prog 825 * before perf-stat reads the value. Otherwise, the accum_reading map may 826 * not have the latest reading from the perf_events. This is achieved by 827 * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU. 828 * 829 * Comment before the definition of struct perf_event_attr_map_entry 830 * describes how different sessions of perf-stat share information about 831 * the leader prog. 832 */ 833 834struct bpf_counter_ops bperf_ops = { 835 .load = bperf__load, 836 .enable = bperf__enable, 837 .disable = bperf__disable, 838 .read = bperf__read, 839 .install_pe = bperf__install_pe, 840 .destroy = bperf__destroy, 841}; 842 843extern struct bpf_counter_ops bperf_cgrp_ops; 844 845static bool bpf_counter_skip(struct evsel *evsel) 846{ 847 return evsel->bpf_counter_ops == NULL; 848} 849 850int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) 851{ 852 if (bpf_counter_skip(evsel)) 853 return 0; 854 return evsel->bpf_counter_ops->install_pe(evsel, cpu_map_idx, fd); 855} 856 857int bpf_counter__load(struct evsel *evsel, struct target *target) 858{ 859 if (target->bpf_str) 860 evsel->bpf_counter_ops = &bpf_program_profiler_ops; 861 else if (cgrp_event_expanded && target->use_bpf) 862 evsel->bpf_counter_ops = &bperf_cgrp_ops; 863 else if (target->use_bpf || evsel->bpf_counter || 864 evsel__match_bpf_counter_events(evsel->name)) 865 evsel->bpf_counter_ops = &bperf_ops; 866 867 if (evsel->bpf_counter_ops) 868 return evsel->bpf_counter_ops->load(evsel, target); 869 return 0; 870} 871 872int bpf_counter__enable(struct evsel *evsel) 873{ 874 if (bpf_counter_skip(evsel)) 875 return 0; 876 return evsel->bpf_counter_ops->enable(evsel); 877} 878 879int bpf_counter__disable(struct evsel *evsel) 880{ 881 if (bpf_counter_skip(evsel)) 882 return 0; 883 return evsel->bpf_counter_ops->disable(evsel); 884} 885 886int bpf_counter__read(struct evsel *evsel) 887{ 888 if (bpf_counter_skip(evsel)) 889 return -EAGAIN; 890 return evsel->bpf_counter_ops->read(evsel); 891} 892 893void bpf_counter__destroy(struct evsel *evsel) 894{ 895 if (bpf_counter_skip(evsel)) 896 return; 897 evsel->bpf_counter_ops->destroy(evsel); 898 evsel->bpf_counter_ops = NULL; 899 evsel->bpf_skel = NULL; 900}
~/
Read-only file preview.