FILE
usdt.c
File preview with syntax highlighting.
linux-6.19.14/tools/lib/bpf/usdt.c
1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3#include <ctype.h> 4#include <stdio.h> 5#include <stdlib.h> 6#include <string.h> 7#include <libelf.h> 8#include <gelf.h> 9#include <unistd.h> 10#include <linux/ptrace.h> 11#include <linux/kernel.h> 12 13/* s8 will be marked as poison while it's a reg of riscv */ 14#if defined(__riscv) 15#define rv_s8 s8 16#endif 17 18#include "bpf.h" 19#include "libbpf.h" 20#include "libbpf_common.h" 21#include "libbpf_internal.h" 22#include "hashmap.h" 23 24/* libbpf's USDT support consists of BPF-side state/code and user-space 25 * state/code working together in concert. BPF-side parts are defined in 26 * usdt.bpf.h header library. User-space state is encapsulated by struct 27 * usdt_manager and all the supporting code centered around usdt_manager. 28 * 29 * usdt.bpf.h defines two BPF maps that usdt_manager expects: USDT spec map 30 * and IP-to-spec-ID map, which is auxiliary map necessary for kernels that 31 * don't support BPF cookie (see below). These two maps are implicitly 32 * embedded into user's end BPF object file when user's code included 33 * usdt.bpf.h. This means that libbpf doesn't do anything special to create 34 * these USDT support maps. They are created by normal libbpf logic of 35 * instantiating BPF maps when opening and loading BPF object. 36 * 37 * As such, libbpf is basically unaware of the need to do anything 38 * USDT-related until the very first call to bpf_program__attach_usdt(), which 39 * can be called by user explicitly or happen automatically during skeleton 40 * attach (or, equivalently, through generic bpf_program__attach() call). At 41 * this point, libbpf will instantiate and initialize struct usdt_manager and 42 * store it in bpf_object. USDT manager is per-BPF object construct, as each 43 * independent BPF object might or might not have USDT programs, and thus all 44 * the expected USDT-related state. There is no coordination between two 45 * bpf_object in parts of USDT attachment, they are oblivious of each other's 46 * existence and libbpf is just oblivious, dealing with bpf_object-specific 47 * USDT state. 48 * 49 * Quick crash course on USDTs. 50 * 51 * From user-space application's point of view, USDT is essentially just 52 * a slightly special function call that normally has zero overhead, unless it 53 * is being traced by some external entity (e.g, BPF-based tool). Here's how 54 * a typical application can trigger USDT probe: 55 * 56 * #include <sys/sdt.h> // provided by systemtap-sdt-devel package 57 * // folly also provide similar functionality in folly/tracing/StaticTracepoint.h 58 * 59 * STAP_PROBE3(my_usdt_provider, my_usdt_probe_name, 123, x, &y); 60 * 61 * USDT is identified by its <provider-name>:<probe-name> pair of names. Each 62 * individual USDT has a fixed number of arguments (3 in the above example) 63 * and specifies values of each argument as if it was a function call. 64 * 65 * USDT call is actually not a function call, but is instead replaced by 66 * a single NOP instruction (thus zero overhead, effectively). But in addition 67 * to that, those USDT macros generate special SHT_NOTE ELF records in 68 * .note.stapsdt ELF section. Here's an example USDT definition as emitted by 69 * `readelf -n <binary>`: 70 * 71 * stapsdt 0x00000089 NT_STAPSDT (SystemTap probe descriptors) 72 * Provider: test 73 * Name: usdt12 74 * Location: 0x0000000000549df3, Base: 0x00000000008effa4, Semaphore: 0x0000000000a4606e 75 * Arguments: -4@-1204(%rbp) -4@%edi -8@-1216(%rbp) -8@%r8 -4@$5 -8@%r9 8@%rdx 8@%r10 -4@$-9 -2@%cx -2@%ax -1@%sil 76 * 77 * In this case we have USDT test:usdt12 with 12 arguments. 78 * 79 * Location and base are offsets used to calculate absolute IP address of that 80 * NOP instruction that kernel can replace with an interrupt instruction to 81 * trigger instrumentation code (BPF program for all that we care about). 82 * 83 * Semaphore above is an optional feature. It records an address of a 2-byte 84 * refcount variable (normally in '.probes' ELF section) used for signaling if 85 * there is anything that is attached to USDT. This is useful for user 86 * applications if, for example, they need to prepare some arguments that are 87 * passed only to USDTs and preparation is expensive. By checking if USDT is 88 * "activated", an application can avoid paying those costs unnecessarily. 89 * Recent enough kernel has built-in support for automatically managing this 90 * refcount, which libbpf expects and relies on. If USDT is defined without 91 * associated semaphore, this value will be zero. See selftests for semaphore 92 * examples. 93 * 94 * Arguments is the most interesting part. This USDT specification string is 95 * providing information about all the USDT arguments and their locations. The 96 * part before @ sign defined byte size of the argument (1, 2, 4, or 8) and 97 * whether the argument is signed or unsigned (negative size means signed). 98 * The part after @ sign is assembly-like definition of argument location 99 * (see [0] for more details). Technically, assembler can provide some pretty 100 * advanced definitions, but libbpf is currently supporting three most common 101 * cases: 102 * 1) immediate constant, see 5th and 9th args above (-4@$5 and -4@-9); 103 * 2) register value, e.g., 8@%rdx, which means "unsigned 8-byte integer 104 * whose value is in register %rdx"; 105 * 3) memory dereference addressed by register, e.g., -4@-1204(%rbp), which 106 * specifies signed 32-bit integer stored at offset -1204 bytes from 107 * memory address stored in %rbp. 108 * 109 * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation 110 * 111 * During attachment, libbpf parses all the relevant USDT specifications and 112 * prepares `struct usdt_spec` (USDT spec), which is then provided to BPF-side 113 * code through spec map. This allows BPF applications to quickly fetch the 114 * actual value at runtime using a simple BPF-side code. 115 * 116 * With basics out of the way, let's go over less immediately obvious aspects 117 * of supporting USDTs. 118 * 119 * First, there is no special USDT BPF program type. It is actually just 120 * a uprobe BPF program (which for kernel, at least currently, is just a kprobe 121 * program, so BPF_PROG_TYPE_KPROBE program type). With the only difference 122 * that uprobe is usually attached at the function entry, while USDT will 123 * normally be somewhere inside the function. But it should always be 124 * pointing to NOP instruction, which makes such uprobes the fastest uprobe 125 * kind. 126 * 127 * Second, it's important to realize that such STAP_PROBEn(provider, name, ...) 128 * macro invocations can end up being inlined many-many times, depending on 129 * specifics of each individual user application. So single conceptual USDT 130 * (identified by provider:name pair of identifiers) is, generally speaking, 131 * multiple uprobe locations (USDT call sites) in different places in user 132 * application. Further, again due to inlining, each USDT call site might end 133 * up having the same argument #N be located in a different place. In one call 134 * site it could be a constant, in another will end up in a register, and in 135 * yet another could be some other register or even somewhere on the stack. 136 * 137 * As such, "attaching to USDT" means (in general case) attaching the same 138 * uprobe BPF program to multiple target locations in user application, each 139 * potentially having a completely different USDT spec associated with it. 140 * To wire all this up together libbpf allocates a unique integer spec ID for 141 * each unique USDT spec. Spec IDs are allocated as sequential small integers 142 * so that they can be used as keys in array BPF map (for performance reasons). 143 * Spec ID allocation and accounting is big part of what usdt_manager is 144 * about. This state has to be maintained per-BPF object and coordinate 145 * between different USDT attachments within the same BPF object. 146 * 147 * Spec ID is the key in spec BPF map, value is the actual USDT spec layed out 148 * as struct usdt_spec. Each invocation of BPF program at runtime needs to 149 * know its associated spec ID. It gets it either through BPF cookie, which 150 * libbpf sets to spec ID during attach time, or, if kernel is too old to 151 * support BPF cookie, through IP-to-spec-ID map that libbpf maintains in such 152 * case. The latter means that some modes of operation can't be supported 153 * without BPF cookie. Such a mode is attaching to shared library "generically", 154 * without specifying target process. In such case, it's impossible to 155 * calculate absolute IP addresses for IP-to-spec-ID map, and thus such mode 156 * is not supported without BPF cookie support. 157 * 158 * Note that libbpf is using BPF cookie functionality for its own internal 159 * needs, so user itself can't rely on BPF cookie feature. To that end, libbpf 160 * provides conceptually equivalent USDT cookie support. It's still u64 161 * user-provided value that can be associated with USDT attachment. Note that 162 * this will be the same value for all USDT call sites within the same single 163 * *logical* USDT attachment. This makes sense because to user attaching to 164 * USDT is a single BPF program triggered for singular USDT probe. The fact 165 * that this is done at multiple actual locations is a mostly hidden 166 * implementation details. This USDT cookie value can be fetched with 167 * bpf_usdt_cookie(ctx) API provided by usdt.bpf.h 168 * 169 * Lastly, while single USDT can have tons of USDT call sites, it doesn't 170 * necessarily have that many different USDT specs. It very well might be 171 * that 1000 USDT call sites only need 5 different USDT specs, because all the 172 * arguments are typically contained in a small set of registers or stack 173 * locations. As such, it's wasteful to allocate as many USDT spec IDs as 174 * there are USDT call sites. So libbpf tries to be frugal and performs 175 * on-the-fly deduplication during a single USDT attachment to only allocate 176 * the minimal required amount of unique USDT specs (and thus spec IDs). This 177 * is trivially achieved by using USDT spec string (Arguments string from USDT 178 * note) as a lookup key in a hashmap. USDT spec string uniquely defines 179 * everything about how to fetch USDT arguments, so two USDT call sites 180 * sharing USDT spec string can safely share the same USDT spec and spec ID. 181 * Note, this spec string deduplication is happening only during the same USDT 182 * attachment, so each USDT spec shares the same USDT cookie value. This is 183 * not generally true for other USDT attachments within the same BPF object, 184 * as even if USDT spec string is the same, USDT cookie value can be 185 * different. It was deemed excessive to try to deduplicate across independent 186 * USDT attachments by taking into account USDT spec string *and* USDT cookie 187 * value, which would complicate spec ID accounting significantly for little 188 * gain. 189 */ 190 191#define USDT_BASE_SEC ".stapsdt.base" 192#define USDT_SEMA_SEC ".probes" 193#define USDT_NOTE_SEC ".note.stapsdt" 194#define USDT_NOTE_TYPE 3 195#define USDT_NOTE_NAME "stapsdt" 196 197/* should match exactly enum __bpf_usdt_arg_type from usdt.bpf.h */ 198enum usdt_arg_type { 199 USDT_ARG_CONST, 200 USDT_ARG_REG, 201 USDT_ARG_REG_DEREF, 202 USDT_ARG_SIB, 203}; 204 205/* should match exactly struct __bpf_usdt_arg_spec from usdt.bpf.h */ 206struct usdt_arg_spec { 207 __u64 val_off; 208#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 209 enum usdt_arg_type arg_type: 8; 210 __u16 idx_reg_off: 12; 211 __u16 scale_bitshift: 4; 212 __u8 __reserved: 8; /* keep reg_off offset stable */ 213#else 214 __u8 __reserved: 8; /* keep reg_off offset stable */ 215 __u16 idx_reg_off: 12; 216 __u16 scale_bitshift: 4; 217 enum usdt_arg_type arg_type: 8; 218#endif 219 short reg_off; 220 bool arg_signed; 221 char arg_bitshift; 222}; 223 224/* should match BPF_USDT_MAX_ARG_CNT in usdt.bpf.h */ 225#define USDT_MAX_ARG_CNT 12 226 227/* should match struct __bpf_usdt_spec from usdt.bpf.h */ 228struct usdt_spec { 229 struct usdt_arg_spec args[USDT_MAX_ARG_CNT]; 230 __u64 usdt_cookie; 231 short arg_cnt; 232}; 233 234struct usdt_note { 235 const char *provider; 236 const char *name; 237 /* USDT args specification string, e.g.: 238 * "-4@%esi -4@-24(%rbp) -4@%ecx 2@%ax 8@%rdx" 239 */ 240 const char *args; 241 long loc_addr; 242 long base_addr; 243 long sema_addr; 244}; 245 246struct usdt_target { 247 long abs_ip; 248 long rel_ip; 249 long sema_off; 250 struct usdt_spec spec; 251 const char *spec_str; 252}; 253 254struct usdt_manager { 255 struct bpf_map *specs_map; 256 struct bpf_map *ip_to_spec_id_map; 257 258 int *free_spec_ids; 259 size_t free_spec_cnt; 260 size_t next_free_spec_id; 261 262 bool has_bpf_cookie; 263 bool has_sema_refcnt; 264 bool has_uprobe_multi; 265}; 266 267struct usdt_manager *usdt_manager_new(struct bpf_object *obj) 268{ 269 static const char *ref_ctr_sysfs_path = "/sys/bus/event_source/devices/uprobe/format/ref_ctr_offset"; 270 struct usdt_manager *man; 271 struct bpf_map *specs_map, *ip_to_spec_id_map; 272 273 specs_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_specs"); 274 ip_to_spec_id_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_ip_to_spec_id"); 275 if (!specs_map || !ip_to_spec_id_map) { 276 pr_warn("usdt: failed to find USDT support BPF maps, did you forget to include bpf/usdt.bpf.h?\n"); 277 return ERR_PTR(-ESRCH); 278 } 279 280 man = calloc(1, sizeof(*man)); 281 if (!man) 282 return ERR_PTR(-ENOMEM); 283 284 man->specs_map = specs_map; 285 man->ip_to_spec_id_map = ip_to_spec_id_map; 286 287 /* Detect if BPF cookie is supported for kprobes. 288 * We don't need IP-to-ID mapping if we can use BPF cookies. 289 * Added in: 7adfc6c9b315 ("bpf: Add bpf_get_attach_cookie() BPF helper to access bpf_cookie value") 290 */ 291 man->has_bpf_cookie = kernel_supports(obj, FEAT_BPF_COOKIE); 292 293 /* Detect kernel support for automatic refcounting of USDT semaphore. 294 * If this is not supported, USDTs with semaphores will not be supported. 295 * Added in: a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe") 296 */ 297 man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0; 298 299 /* 300 * Detect kernel support for uprobe multi link to be used for attaching 301 * usdt probes. 302 */ 303 man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK); 304 return man; 305} 306 307void usdt_manager_free(struct usdt_manager *man) 308{ 309 if (IS_ERR_OR_NULL(man)) 310 return; 311 312 free(man->free_spec_ids); 313 free(man); 314} 315 316static int sanity_check_usdt_elf(Elf *elf, const char *path) 317{ 318 GElf_Ehdr ehdr; 319 int endianness; 320 321 if (elf_kind(elf) != ELF_K_ELF) { 322 pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path); 323 return -EBADF; 324 } 325 326 switch (gelf_getclass(elf)) { 327 case ELFCLASS64: 328 if (sizeof(void *) != 8) { 329 pr_warn("usdt: attaching to 64-bit ELF binary '%s' is not supported\n", path); 330 return -EBADF; 331 } 332 break; 333 case ELFCLASS32: 334 if (sizeof(void *) != 4) { 335 pr_warn("usdt: attaching to 32-bit ELF binary '%s' is not supported\n", path); 336 return -EBADF; 337 } 338 break; 339 default: 340 pr_warn("usdt: unsupported ELF class for '%s'\n", path); 341 return -EBADF; 342 } 343 344 if (!gelf_getehdr(elf, &ehdr)) 345 return -EINVAL; 346 347 if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) { 348 pr_warn("usdt: unsupported type of ELF binary '%s' (%d), only ET_EXEC and ET_DYN are supported\n", 349 path, ehdr.e_type); 350 return -EBADF; 351 } 352 353#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 354 endianness = ELFDATA2LSB; 355#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 356 endianness = ELFDATA2MSB; 357#else 358# error "Unrecognized __BYTE_ORDER__" 359#endif 360 if (endianness != ehdr.e_ident[EI_DATA]) { 361 pr_warn("usdt: ELF endianness mismatch for '%s'\n", path); 362 return -EBADF; 363 } 364 365 return 0; 366} 367 368static int find_elf_sec_by_name(Elf *elf, const char *sec_name, GElf_Shdr *shdr, Elf_Scn **scn) 369{ 370 Elf_Scn *sec = NULL; 371 size_t shstrndx; 372 373 if (elf_getshdrstrndx(elf, &shstrndx)) 374 return -EINVAL; 375 376 /* check if ELF is corrupted and avoid calling elf_strptr if yes */ 377 if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) 378 return -EINVAL; 379 380 while ((sec = elf_nextscn(elf, sec)) != NULL) { 381 char *name; 382 383 if (!gelf_getshdr(sec, shdr)) 384 return -EINVAL; 385 386 name = elf_strptr(elf, shstrndx, shdr->sh_name); 387 if (name && strcmp(sec_name, name) == 0) { 388 *scn = sec; 389 return 0; 390 } 391 } 392 393 return -ENOENT; 394} 395 396struct elf_seg { 397 long start; 398 long end; 399 long offset; 400 bool is_exec; 401}; 402 403static int cmp_elf_segs(const void *_a, const void *_b) 404{ 405 const struct elf_seg *a = _a; 406 const struct elf_seg *b = _b; 407 408 return a->start < b->start ? -1 : 1; 409} 410 411static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, size_t *seg_cnt) 412{ 413 GElf_Phdr phdr; 414 size_t n; 415 int i, err; 416 struct elf_seg *seg; 417 void *tmp; 418 419 *seg_cnt = 0; 420 421 if (elf_getphdrnum(elf, &n)) { 422 err = -errno; 423 return err; 424 } 425 426 for (i = 0; i < n; i++) { 427 if (!gelf_getphdr(elf, i, &phdr)) { 428 err = -errno; 429 return err; 430 } 431 432 pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n", 433 i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset, 434 (long)phdr.p_type, (long)phdr.p_flags); 435 if (phdr.p_type != PT_LOAD) 436 continue; 437 438 tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs)); 439 if (!tmp) 440 return -ENOMEM; 441 442 *segs = tmp; 443 seg = *segs + *seg_cnt; 444 (*seg_cnt)++; 445 446 seg->start = phdr.p_vaddr; 447 seg->end = phdr.p_vaddr + phdr.p_memsz; 448 seg->offset = phdr.p_offset; 449 seg->is_exec = phdr.p_flags & PF_X; 450 } 451 452 if (*seg_cnt == 0) { 453 pr_warn("usdt: failed to find PT_LOAD program headers in '%s'\n", path); 454 return -ESRCH; 455 } 456 457 qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs); 458 return 0; 459} 460 461static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt) 462{ 463 char path[PATH_MAX], line[PATH_MAX], mode[16]; 464 size_t seg_start, seg_end, seg_off; 465 struct elf_seg *seg; 466 int tmp_pid, i, err; 467 FILE *f; 468 469 *seg_cnt = 0; 470 471 /* Handle containerized binaries only accessible from 472 * /proc/<pid>/root/<path>. They will be reported as just /<path> in 473 * /proc/<pid>/maps. 474 */ 475 if (sscanf(lib_path, "/proc/%d/root%s", &tmp_pid, path) == 2 && pid == tmp_pid) 476 goto proceed; 477 478 if (!realpath(lib_path, path)) { 479 pr_warn("usdt: failed to get absolute path of '%s' (err %s), using path as is...\n", 480 lib_path, errstr(-errno)); 481 libbpf_strlcpy(path, lib_path, sizeof(path)); 482 } 483 484proceed: 485 sprintf(line, "/proc/%d/maps", pid); 486 f = fopen(line, "re"); 487 if (!f) { 488 err = -errno; 489 pr_warn("usdt: failed to open '%s' to get base addr of '%s': %s\n", 490 line, lib_path, errstr(err)); 491 return err; 492 } 493 494 /* We need to handle lines with no path at the end: 495 * 496 * 7f5c6f5d1000-7f5c6f5d3000 rw-p 001c7000 08:04 21238613 /usr/lib64/libc-2.17.so 497 * 7f5c6f5d3000-7f5c6f5d8000 rw-p 00000000 00:00 0 498 * 7f5c6f5d8000-7f5c6f5d9000 r-xp 00000000 103:01 362990598 /data/users/andriin/linux/tools/bpf/usdt/libhello_usdt.so 499 */ 500 while (fscanf(f, "%zx-%zx %s %zx %*s %*d%[^\n]\n", 501 &seg_start, &seg_end, mode, &seg_off, line) == 5) { 502 void *tmp; 503 504 /* to handle no path case (see above) we need to capture line 505 * without skipping any whitespaces. So we need to strip 506 * leading whitespaces manually here 507 */ 508 i = 0; 509 while (isblank(line[i])) 510 i++; 511 if (strcmp(line + i, path) != 0) 512 continue; 513 514 pr_debug("usdt: discovered segment for lib '%s': addrs %zx-%zx mode %s offset %zx\n", 515 path, seg_start, seg_end, mode, seg_off); 516 517 /* ignore non-executable sections for shared libs */ 518 if (mode[2] != 'x') 519 continue; 520 521 tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs)); 522 if (!tmp) { 523 err = -ENOMEM; 524 goto err_out; 525 } 526 527 *segs = tmp; 528 seg = *segs + *seg_cnt; 529 *seg_cnt += 1; 530 531 seg->start = seg_start; 532 seg->end = seg_end; 533 seg->offset = seg_off; 534 seg->is_exec = true; 535 } 536 537 if (*seg_cnt == 0) { 538 pr_warn("usdt: failed to find '%s' (resolved to '%s') within PID %d memory mappings\n", 539 lib_path, path, pid); 540 err = -ESRCH; 541 goto err_out; 542 } 543 544 qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs); 545 err = 0; 546err_out: 547 fclose(f); 548 return err; 549} 550 551static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long virtaddr) 552{ 553 struct elf_seg *seg; 554 int i; 555 556 /* for ELF binaries (both executables and shared libraries), we are 557 * given virtual address (absolute for executables, relative for 558 * libraries) which should match address range of [seg_start, seg_end) 559 */ 560 for (i = 0, seg = segs; i < seg_cnt; i++, seg++) { 561 if (seg->start <= virtaddr && virtaddr < seg->end) 562 return seg; 563 } 564 return NULL; 565} 566 567static struct elf_seg *find_vma_seg(struct elf_seg *segs, size_t seg_cnt, long offset) 568{ 569 struct elf_seg *seg; 570 int i; 571 572 /* for VMA segments from /proc/<pid>/maps file, provided "address" is 573 * actually a file offset, so should be fall within logical 574 * offset-based range of [offset_start, offset_end) 575 */ 576 for (i = 0, seg = segs; i < seg_cnt; i++, seg++) { 577 if (seg->offset <= offset && offset < seg->offset + (seg->end - seg->start)) 578 return seg; 579 } 580 return NULL; 581} 582 583static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off, 584 size_t desc_off, struct usdt_note *usdt_note); 585 586static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie); 587 588static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *path, pid_t pid, 589 const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie, 590 struct usdt_target **out_targets, size_t *out_target_cnt) 591{ 592 size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0; 593 struct elf_seg *segs = NULL, *vma_segs = NULL; 594 struct usdt_target *targets = NULL, *target; 595 long base_addr = 0; 596 Elf_Scn *notes_scn, *base_scn; 597 GElf_Shdr base_shdr, notes_shdr; 598 GElf_Ehdr ehdr; 599 GElf_Nhdr nhdr; 600 Elf_Data *data; 601 int err; 602 603 *out_targets = NULL; 604 *out_target_cnt = 0; 605 606 err = find_elf_sec_by_name(elf, USDT_NOTE_SEC, ¬es_shdr, ¬es_scn); 607 if (err) { 608 pr_warn("usdt: no USDT notes section (%s) found in '%s'\n", USDT_NOTE_SEC, path); 609 return err; 610 } 611 612 if (notes_shdr.sh_type != SHT_NOTE || !gelf_getehdr(elf, &ehdr)) { 613 pr_warn("usdt: invalid USDT notes section (%s) in '%s'\n", USDT_NOTE_SEC, path); 614 return -EINVAL; 615 } 616 617 err = parse_elf_segs(elf, path, &segs, &seg_cnt); 618 if (err) { 619 pr_warn("usdt: failed to process ELF program segments for '%s': %s\n", 620 path, errstr(err)); 621 goto err_out; 622 } 623 624 /* .stapsdt.base ELF section is optional, but is used for prelink 625 * offset compensation (see a big comment further below) 626 */ 627 if (find_elf_sec_by_name(elf, USDT_BASE_SEC, &base_shdr, &base_scn) == 0) 628 base_addr = base_shdr.sh_addr; 629 630 data = elf_getdata(notes_scn, 0); 631 off = 0; 632 while ((off = gelf_getnote(data, off, &nhdr, &name_off, &desc_off)) > 0) { 633 long usdt_abs_ip, usdt_rel_ip, usdt_sema_off = 0; 634 struct usdt_note note; 635 struct elf_seg *seg = NULL; 636 void *tmp; 637 638 err = parse_usdt_note(&nhdr, data->d_buf, name_off, desc_off, ¬e); 639 if (err) 640 goto err_out; 641 642 if (strcmp(note.provider, usdt_provider) != 0 || strcmp(note.name, usdt_name) != 0) 643 continue; 644 645 /* We need to compensate "prelink effect". See [0] for details, 646 * relevant parts quoted here: 647 * 648 * Each SDT probe also expands into a non-allocated ELF note. You can 649 * find this by looking at SHT_NOTE sections and decoding the format; 650 * see below for details. Because the note is non-allocated, it means 651 * there is no runtime cost, and also preserved in both stripped files 652 * and .debug files. 653 * 654 * However, this means that prelink won't adjust the note's contents 655 * for address offsets. Instead, this is done via the .stapsdt.base 656 * section. This is a special section that is added to the text. We 657 * will only ever have one of these sections in a final link and it 658 * will only ever be one byte long. Nothing about this section itself 659 * matters, we just use it as a marker to detect prelink address 660 * adjustments. 661 * 662 * Each probe note records the link-time address of the .stapsdt.base 663 * section alongside the probe PC address. The decoder compares the 664 * base address stored in the note with the .stapsdt.base section's 665 * sh_addr. Initially these are the same, but the section header will 666 * be adjusted by prelink. So the decoder applies the difference to 667 * the probe PC address to get the correct prelinked PC address; the 668 * same adjustment is applied to the semaphore address, if any. 669 * 670 * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation 671 */ 672 usdt_abs_ip = note.loc_addr; 673 if (base_addr && note.base_addr) 674 usdt_abs_ip += base_addr - note.base_addr; 675 676 /* When attaching uprobes (which is what USDTs basically are) 677 * kernel expects file offset to be specified, not a relative 678 * virtual address, so we need to translate virtual address to 679 * file offset, for both ET_EXEC and ET_DYN binaries. 680 */ 681 seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip); 682 if (!seg) { 683 err = -ESRCH; 684 pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n", 685 usdt_provider, usdt_name, path, usdt_abs_ip); 686 goto err_out; 687 } 688 if (!seg->is_exec) { 689 err = -ESRCH; 690 pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n", 691 path, seg->start, seg->end, usdt_provider, usdt_name, 692 usdt_abs_ip); 693 goto err_out; 694 } 695 /* translate from virtual address to file offset */ 696 usdt_rel_ip = usdt_abs_ip - seg->start + seg->offset; 697 698 if (ehdr.e_type == ET_DYN && !man->has_bpf_cookie) { 699 /* If we don't have BPF cookie support but need to 700 * attach to a shared library, we'll need to know and 701 * record absolute addresses of attach points due to 702 * the need to lookup USDT spec by absolute IP of 703 * triggered uprobe. Doing this resolution is only 704 * possible when we have a specific PID of the process 705 * that's using specified shared library. BPF cookie 706 * removes the absolute address limitation as we don't 707 * need to do this lookup (we just use BPF cookie as 708 * an index of USDT spec), so for newer kernels with 709 * BPF cookie support libbpf supports USDT attachment 710 * to shared libraries with no PID filter. 711 */ 712 if (pid < 0) { 713 pr_warn("usdt: attaching to shared libraries without specific PID is not supported on current kernel\n"); 714 err = -ENOTSUP; 715 goto err_out; 716 } 717 718 /* vma_segs are lazily initialized only if necessary */ 719 if (vma_seg_cnt == 0) { 720 err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt); 721 if (err) { 722 pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %s\n", 723 pid, path, errstr(err)); 724 goto err_out; 725 } 726 } 727 728 seg = find_vma_seg(vma_segs, vma_seg_cnt, usdt_rel_ip); 729 if (!seg) { 730 err = -ESRCH; 731 pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n", 732 usdt_provider, usdt_name, path, usdt_rel_ip); 733 goto err_out; 734 } 735 736 usdt_abs_ip = seg->start - seg->offset + usdt_rel_ip; 737 } 738 739 pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n", 740 usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path, 741 note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args, 742 seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0); 743 744 /* Adjust semaphore address to be a file offset */ 745 if (note.sema_addr) { 746 if (!man->has_sema_refcnt) { 747 pr_warn("usdt: kernel doesn't support USDT semaphore refcounting for '%s:%s' in '%s'\n", 748 usdt_provider, usdt_name, path); 749 err = -ENOTSUP; 750 goto err_out; 751 } 752 753 seg = find_elf_seg(segs, seg_cnt, note.sema_addr); 754 if (!seg) { 755 err = -ESRCH; 756 pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n", 757 usdt_provider, usdt_name, path, note.sema_addr); 758 goto err_out; 759 } 760 if (seg->is_exec) { 761 err = -ESRCH; 762 pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n", 763 path, seg->start, seg->end, usdt_provider, usdt_name, 764 note.sema_addr); 765 goto err_out; 766 } 767 768 usdt_sema_off = note.sema_addr - seg->start + seg->offset; 769 770 pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n", 771 usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", 772 path, note.sema_addr, note.base_addr, usdt_sema_off, 773 seg->start, seg->end, seg->offset); 774 } 775 776 /* Record adjusted addresses and offsets and parse USDT spec */ 777 tmp = libbpf_reallocarray(targets, target_cnt + 1, sizeof(*targets)); 778 if (!tmp) { 779 err = -ENOMEM; 780 goto err_out; 781 } 782 targets = tmp; 783 784 target = &targets[target_cnt]; 785 memset(target, 0, sizeof(*target)); 786 787 target->abs_ip = usdt_abs_ip; 788 target->rel_ip = usdt_rel_ip; 789 target->sema_off = usdt_sema_off; 790 791 /* notes.args references strings from ELF itself, so they can 792 * be referenced safely until elf_end() call 793 */ 794 target->spec_str = note.args; 795 796 err = parse_usdt_spec(&target->spec, ¬e, usdt_cookie); 797 if (err) 798 goto err_out; 799 800 target_cnt++; 801 } 802 803 *out_targets = targets; 804 *out_target_cnt = target_cnt; 805 err = target_cnt; 806 807err_out: 808 free(segs); 809 free(vma_segs); 810 if (err < 0) 811 free(targets); 812 return err; 813} 814 815struct bpf_link_usdt { 816 struct bpf_link link; 817 818 struct usdt_manager *usdt_man; 819 820 size_t spec_cnt; 821 int *spec_ids; 822 823 size_t uprobe_cnt; 824 struct { 825 long abs_ip; 826 struct bpf_link *link; 827 } *uprobes; 828 829 struct bpf_link *multi_link; 830}; 831 832static int bpf_link_usdt_detach(struct bpf_link *link) 833{ 834 struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); 835 struct usdt_manager *man = usdt_link->usdt_man; 836 int i; 837 838 bpf_link__destroy(usdt_link->multi_link); 839 840 /* When having multi_link, uprobe_cnt is 0 */ 841 for (i = 0; i < usdt_link->uprobe_cnt; i++) { 842 /* detach underlying uprobe link */ 843 bpf_link__destroy(usdt_link->uprobes[i].link); 844 /* there is no need to update specs map because it will be 845 * unconditionally overwritten on subsequent USDT attaches, 846 * but if BPF cookies are not used we need to remove entry 847 * from ip_to_spec_id map, otherwise we'll run into false 848 * conflicting IP errors 849 */ 850 if (!man->has_bpf_cookie) { 851 /* not much we can do about errors here */ 852 (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), 853 &usdt_link->uprobes[i].abs_ip); 854 } 855 } 856 857 /* try to return the list of previously used spec IDs to usdt_manager 858 * for future reuse for subsequent USDT attaches 859 */ 860 if (!man->free_spec_ids) { 861 /* if there were no free spec IDs yet, just transfer our IDs */ 862 man->free_spec_ids = usdt_link->spec_ids; 863 man->free_spec_cnt = usdt_link->spec_cnt; 864 usdt_link->spec_ids = NULL; 865 } else { 866 /* otherwise concat IDs */ 867 size_t new_cnt = man->free_spec_cnt + usdt_link->spec_cnt; 868 int *new_free_ids; 869 870 new_free_ids = libbpf_reallocarray(man->free_spec_ids, new_cnt, 871 sizeof(*new_free_ids)); 872 /* If we couldn't resize free_spec_ids, we'll just leak 873 * a bunch of free IDs; this is very unlikely to happen and if 874 * system is so exhausted on memory, it's the least of user's 875 * concerns, probably. 876 * So just do our best here to return those IDs to usdt_manager. 877 * Another edge case when we can legitimately get NULL is when 878 * new_cnt is zero, which can happen in some edge cases, so we 879 * need to be careful about that. 880 */ 881 if (new_free_ids || new_cnt == 0) { 882 memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids, 883 usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids)); 884 man->free_spec_ids = new_free_ids; 885 man->free_spec_cnt = new_cnt; 886 } 887 } 888 889 return 0; 890} 891 892static void bpf_link_usdt_dealloc(struct bpf_link *link) 893{ 894 struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); 895 896 free(usdt_link->spec_ids); 897 free(usdt_link->uprobes); 898 free(usdt_link); 899} 900 901static size_t specs_hash_fn(long key, void *ctx) 902{ 903 return str_hash((char *)key); 904} 905 906static bool specs_equal_fn(long key1, long key2, void *ctx) 907{ 908 return strcmp((char *)key1, (char *)key2) == 0; 909} 910 911static int allocate_spec_id(struct usdt_manager *man, struct hashmap *specs_hash, 912 struct bpf_link_usdt *link, struct usdt_target *target, 913 int *spec_id, bool *is_new) 914{ 915 long tmp; 916 void *new_ids; 917 int err; 918 919 /* check if we already allocated spec ID for this spec string */ 920 if (hashmap__find(specs_hash, target->spec_str, &tmp)) { 921 *spec_id = tmp; 922 *is_new = false; 923 return 0; 924 } 925 926 /* otherwise it's a new ID that needs to be set up in specs map and 927 * returned back to usdt_manager when USDT link is detached 928 */ 929 new_ids = libbpf_reallocarray(link->spec_ids, link->spec_cnt + 1, sizeof(*link->spec_ids)); 930 if (!new_ids) 931 return -ENOMEM; 932 link->spec_ids = new_ids; 933 934 /* get next free spec ID, giving preference to free list, if not empty */ 935 if (man->free_spec_cnt) { 936 *spec_id = man->free_spec_ids[man->free_spec_cnt - 1]; 937 938 /* cache spec ID for current spec string for future lookups */ 939 err = hashmap__add(specs_hash, target->spec_str, *spec_id); 940 if (err) 941 return err; 942 943 man->free_spec_cnt--; 944 } else { 945 /* don't allocate spec ID bigger than what fits in specs map */ 946 if (man->next_free_spec_id >= bpf_map__max_entries(man->specs_map)) 947 return -E2BIG; 948 949 *spec_id = man->next_free_spec_id; 950 951 /* cache spec ID for current spec string for future lookups */ 952 err = hashmap__add(specs_hash, target->spec_str, *spec_id); 953 if (err) 954 return err; 955 956 man->next_free_spec_id++; 957 } 958 959 /* remember new spec ID in the link for later return back to free list on detach */ 960 link->spec_ids[link->spec_cnt] = *spec_id; 961 link->spec_cnt++; 962 *is_new = true; 963 return 0; 964} 965 966struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct bpf_program *prog, 967 pid_t pid, const char *path, 968 const char *usdt_provider, const char *usdt_name, 969 __u64 usdt_cookie) 970{ 971 unsigned long *offsets = NULL, *ref_ctr_offsets = NULL; 972 int i, err, spec_map_fd, ip_map_fd; 973 LIBBPF_OPTS(bpf_uprobe_opts, opts); 974 struct hashmap *specs_hash = NULL; 975 struct bpf_link_usdt *link = NULL; 976 struct usdt_target *targets = NULL; 977 __u64 *cookies = NULL; 978 struct elf_fd elf_fd; 979 size_t target_cnt; 980 981 spec_map_fd = bpf_map__fd(man->specs_map); 982 ip_map_fd = bpf_map__fd(man->ip_to_spec_id_map); 983 984 err = elf_open(path, &elf_fd); 985 if (err) 986 return libbpf_err_ptr(err); 987 988 err = sanity_check_usdt_elf(elf_fd.elf, path); 989 if (err) 990 goto err_out; 991 992 /* normalize PID filter */ 993 if (pid < 0) 994 pid = -1; 995 else if (pid == 0) 996 pid = getpid(); 997 998 /* discover USDT in given binary, optionally limiting 999 * activations to a given PID, if pid > 0 1000 */ 1001 err = collect_usdt_targets(man, elf_fd.elf, path, pid, usdt_provider, usdt_name, 1002 usdt_cookie, &targets, &target_cnt); 1003 if (err <= 0) { 1004 err = (err == 0) ? -ENOENT : err; 1005 goto err_out; 1006 } 1007 1008 specs_hash = hashmap__new(specs_hash_fn, specs_equal_fn, NULL); 1009 if (IS_ERR(specs_hash)) { 1010 err = PTR_ERR(specs_hash); 1011 goto err_out; 1012 } 1013 1014 link = calloc(1, sizeof(*link)); 1015 if (!link) { 1016 err = -ENOMEM; 1017 goto err_out; 1018 } 1019 1020 link->usdt_man = man; 1021 link->link.detach = &bpf_link_usdt_detach; 1022 link->link.dealloc = &bpf_link_usdt_dealloc; 1023 1024 if (man->has_uprobe_multi) { 1025 offsets = calloc(target_cnt, sizeof(*offsets)); 1026 cookies = calloc(target_cnt, sizeof(*cookies)); 1027 ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets)); 1028 1029 if (!offsets || !ref_ctr_offsets || !cookies) { 1030 err = -ENOMEM; 1031 goto err_out; 1032 } 1033 } else { 1034 link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); 1035 if (!link->uprobes) { 1036 err = -ENOMEM; 1037 goto err_out; 1038 } 1039 } 1040 1041 for (i = 0; i < target_cnt; i++) { 1042 struct usdt_target *target = &targets[i]; 1043 struct bpf_link *uprobe_link; 1044 bool is_new; 1045 int spec_id; 1046 1047 /* Spec ID can be either reused or newly allocated. If it is 1048 * newly allocated, we'll need to fill out spec map, otherwise 1049 * entire spec should be valid and can be just used by a new 1050 * uprobe. We reuse spec when USDT arg spec is identical. We 1051 * also never share specs between two different USDT 1052 * attachments ("links"), so all the reused specs already 1053 * share USDT cookie value implicitly. 1054 */ 1055 err = allocate_spec_id(man, specs_hash, link, target, &spec_id, &is_new); 1056 if (err) 1057 goto err_out; 1058 1059 if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) { 1060 err = -errno; 1061 pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %s\n", 1062 spec_id, usdt_provider, usdt_name, path, errstr(err)); 1063 goto err_out; 1064 } 1065 if (!man->has_bpf_cookie && 1066 bpf_map_update_elem(ip_map_fd, &target->abs_ip, &spec_id, BPF_NOEXIST)) { 1067 err = -errno; 1068 if (err == -EEXIST) { 1069 pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n", 1070 spec_id, usdt_provider, usdt_name, path); 1071 } else { 1072 pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n", 1073 target->abs_ip, spec_id, usdt_provider, usdt_name, 1074 path, errstr(err)); 1075 } 1076 goto err_out; 1077 } 1078 1079 if (man->has_uprobe_multi) { 1080 offsets[i] = target->rel_ip; 1081 ref_ctr_offsets[i] = target->sema_off; 1082 cookies[i] = spec_id; 1083 } else { 1084 opts.ref_ctr_offset = target->sema_off; 1085 opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; 1086 uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, 1087 target->rel_ip, &opts); 1088 err = libbpf_get_error(uprobe_link); 1089 if (err) { 1090 pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %s\n", 1091 i, usdt_provider, usdt_name, path, errstr(err)); 1092 goto err_out; 1093 } 1094 1095 link->uprobes[i].link = uprobe_link; 1096 link->uprobes[i].abs_ip = target->abs_ip; 1097 link->uprobe_cnt++; 1098 } 1099 } 1100 1101 if (man->has_uprobe_multi) { 1102 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi, 1103 .ref_ctr_offsets = ref_ctr_offsets, 1104 .offsets = offsets, 1105 .cookies = cookies, 1106 .cnt = target_cnt, 1107 ); 1108 1109 link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path, 1110 NULL, &opts_multi); 1111 if (!link->multi_link) { 1112 err = -errno; 1113 pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %s\n", 1114 usdt_provider, usdt_name, path, errstr(err)); 1115 goto err_out; 1116 } 1117 1118 free(offsets); 1119 free(ref_ctr_offsets); 1120 free(cookies); 1121 } 1122 1123 free(targets); 1124 hashmap__free(specs_hash); 1125 elf_close(&elf_fd); 1126 return &link->link; 1127 1128err_out: 1129 free(offsets); 1130 free(ref_ctr_offsets); 1131 free(cookies); 1132 1133 if (link) 1134 bpf_link__destroy(&link->link); 1135 free(targets); 1136 hashmap__free(specs_hash); 1137 elf_close(&elf_fd); 1138 return libbpf_err_ptr(err); 1139} 1140 1141/* Parse out USDT ELF note from '.note.stapsdt' section. 1142 * Logic inspired by perf's code. 1143 */ 1144static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off, 1145 struct usdt_note *note) 1146{ 1147 const char *provider, *name, *args; 1148 long addrs[3]; 1149 size_t len; 1150 1151 /* sanity check USDT note name and type first */ 1152 if (strncmp(data + name_off, USDT_NOTE_NAME, nhdr->n_namesz) != 0) 1153 return -EINVAL; 1154 if (nhdr->n_type != USDT_NOTE_TYPE) 1155 return -EINVAL; 1156 1157 /* sanity check USDT note contents ("description" in ELF terminology) */ 1158 len = nhdr->n_descsz; 1159 data = data + desc_off; 1160 1161 /* +3 is the very minimum required to store three empty strings */ 1162 if (len < sizeof(addrs) + 3) 1163 return -EINVAL; 1164 1165 /* get location, base, and semaphore addrs */ 1166 memcpy(&addrs, data, sizeof(addrs)); 1167 1168 /* parse string fields: provider, name, args */ 1169 provider = data + sizeof(addrs); 1170 1171 name = (const char *)memchr(provider, '\0', data + len - provider); 1172 if (!name) /* non-zero-terminated provider */ 1173 return -EINVAL; 1174 name++; 1175 if (name >= data + len || *name == '\0') /* missing or empty name */ 1176 return -EINVAL; 1177 1178 args = memchr(name, '\0', data + len - name); 1179 if (!args) /* non-zero-terminated name */ 1180 return -EINVAL; 1181 ++args; 1182 if (args >= data + len) /* missing arguments spec */ 1183 return -EINVAL; 1184 1185 note->provider = provider; 1186 note->name = name; 1187 if (*args == '\0' || *args == ':') 1188 note->args = ""; 1189 else 1190 note->args = args; 1191 note->loc_addr = addrs[0]; 1192 note->base_addr = addrs[1]; 1193 note->sema_addr = addrs[2]; 1194 1195 return 0; 1196} 1197 1198static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz); 1199 1200static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie) 1201{ 1202 struct usdt_arg_spec *arg; 1203 const char *s; 1204 int arg_sz, len; 1205 1206 spec->usdt_cookie = usdt_cookie; 1207 spec->arg_cnt = 0; 1208 1209 s = note->args; 1210 while (s[0]) { 1211 if (spec->arg_cnt >= USDT_MAX_ARG_CNT) { 1212 pr_warn("usdt: too many USDT arguments (> %d) for '%s:%s' with args spec '%s'\n", 1213 USDT_MAX_ARG_CNT, note->provider, note->name, note->args); 1214 return -E2BIG; 1215 } 1216 1217 arg = &spec->args[spec->arg_cnt]; 1218 len = parse_usdt_arg(s, spec->arg_cnt, arg, &arg_sz); 1219 if (len < 0) 1220 return len; 1221 1222 arg->arg_signed = arg_sz < 0; 1223 if (arg_sz < 0) 1224 arg_sz = -arg_sz; 1225 1226 switch (arg_sz) { 1227 case 1: case 2: case 4: case 8: 1228 arg->arg_bitshift = 64 - arg_sz * 8; 1229 break; 1230 default: 1231 pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n", 1232 spec->arg_cnt, s, arg_sz); 1233 return -EINVAL; 1234 } 1235 1236 s += len; 1237 spec->arg_cnt++; 1238 } 1239 1240 return 0; 1241} 1242 1243/* Architecture-specific logic for parsing USDT argument location specs */ 1244 1245#if defined(__x86_64__) || defined(__i386__) 1246 1247static int calc_pt_regs_off(const char *reg_name) 1248{ 1249 static struct { 1250 const char *names[4]; 1251 size_t pt_regs_off; 1252 } reg_map[] = { 1253#ifdef __x86_64__ 1254#define reg_off(reg64, reg32) offsetof(struct pt_regs, reg64) 1255#else 1256#define reg_off(reg64, reg32) offsetof(struct pt_regs, reg32) 1257#endif 1258 { {"rip", "eip", "", ""}, reg_off(rip, eip) }, 1259 { {"rax", "eax", "ax", "al"}, reg_off(rax, eax) }, 1260 { {"rbx", "ebx", "bx", "bl"}, reg_off(rbx, ebx) }, 1261 { {"rcx", "ecx", "cx", "cl"}, reg_off(rcx, ecx) }, 1262 { {"rdx", "edx", "dx", "dl"}, reg_off(rdx, edx) }, 1263 { {"rsi", "esi", "si", "sil"}, reg_off(rsi, esi) }, 1264 { {"rdi", "edi", "di", "dil"}, reg_off(rdi, edi) }, 1265 { {"rbp", "ebp", "bp", "bpl"}, reg_off(rbp, ebp) }, 1266 { {"rsp", "esp", "sp", "spl"}, reg_off(rsp, esp) }, 1267#undef reg_off 1268#ifdef __x86_64__ 1269 { {"r8", "r8d", "r8w", "r8b"}, offsetof(struct pt_regs, r8) }, 1270 { {"r9", "r9d", "r9w", "r9b"}, offsetof(struct pt_regs, r9) }, 1271 { {"r10", "r10d", "r10w", "r10b"}, offsetof(struct pt_regs, r10) }, 1272 { {"r11", "r11d", "r11w", "r11b"}, offsetof(struct pt_regs, r11) }, 1273 { {"r12", "r12d", "r12w", "r12b"}, offsetof(struct pt_regs, r12) }, 1274 { {"r13", "r13d", "r13w", "r13b"}, offsetof(struct pt_regs, r13) }, 1275 { {"r14", "r14d", "r14w", "r14b"}, offsetof(struct pt_regs, r14) }, 1276 { {"r15", "r15d", "r15w", "r15b"}, offsetof(struct pt_regs, r15) }, 1277#endif 1278 }; 1279 int i, j; 1280 1281 for (i = 0; i < ARRAY_SIZE(reg_map); i++) { 1282 for (j = 0; j < ARRAY_SIZE(reg_map[i].names); j++) { 1283 if (strcmp(reg_name, reg_map[i].names[j]) == 0) 1284 return reg_map[i].pt_regs_off; 1285 } 1286 } 1287 1288 pr_warn("usdt: unrecognized register '%s'\n", reg_name); 1289 return -ENOENT; 1290} 1291 1292static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) 1293{ 1294 char reg_name[16] = {0}, idx_reg_name[16] = {0}; 1295 int len, reg_off, idx_reg_off, scale = 1; 1296 long off = 0; 1297 1298 if (sscanf(arg_str, " %d @ %ld ( %%%15[^,] , %%%15[^,] , %d ) %n", 1299 arg_sz, &off, reg_name, idx_reg_name, &scale, &len) == 5 || 1300 sscanf(arg_str, " %d @ ( %%%15[^,] , %%%15[^,] , %d ) %n", 1301 arg_sz, reg_name, idx_reg_name, &scale, &len) == 4 || 1302 sscanf(arg_str, " %d @ %ld ( %%%15[^,] , %%%15[^)] ) %n", 1303 arg_sz, &off, reg_name, idx_reg_name, &len) == 4 || 1304 sscanf(arg_str, " %d @ ( %%%15[^,] , %%%15[^)] ) %n", 1305 arg_sz, reg_name, idx_reg_name, &len) == 3 1306 ) { 1307 /* 1308 * Scale Index Base case: 1309 * 1@-96(%rbp,%rax,8) 1310 * 1@(%rbp,%rax,8) 1311 * 1@-96(%rbp,%rax) 1312 * 1@(%rbp,%rax) 1313 */ 1314 arg->arg_type = USDT_ARG_SIB; 1315 arg->val_off = off; 1316 1317 reg_off = calc_pt_regs_off(reg_name); 1318 if (reg_off < 0) 1319 return reg_off; 1320 arg->reg_off = reg_off; 1321 1322 idx_reg_off = calc_pt_regs_off(idx_reg_name); 1323 if (idx_reg_off < 0) 1324 return idx_reg_off; 1325 arg->idx_reg_off = idx_reg_off; 1326 1327 /* validate scale factor and set fields directly */ 1328 switch (scale) { 1329 case 1: arg->scale_bitshift = 0; break; 1330 case 2: arg->scale_bitshift = 1; break; 1331 case 4: arg->scale_bitshift = 2; break; 1332 case 8: arg->scale_bitshift = 3; break; 1333 default: 1334 pr_warn("usdt: invalid SIB scale %d, expected 1, 2, 4, 8\n", scale); 1335 return -EINVAL; 1336 } 1337 } else if (sscanf(arg_str, " %d @ %ld ( %%%15[^)] ) %n", 1338 arg_sz, &off, reg_name, &len) == 3) { 1339 /* Memory dereference case, e.g., -4@-20(%rbp) */ 1340 arg->arg_type = USDT_ARG_REG_DEREF; 1341 arg->val_off = off; 1342 reg_off = calc_pt_regs_off(reg_name); 1343 if (reg_off < 0) 1344 return reg_off; 1345 arg->reg_off = reg_off; 1346 } else if (sscanf(arg_str, " %d @ ( %%%15[^)] ) %n", arg_sz, reg_name, &len) == 2) { 1347 /* Memory dereference case without offset, e.g., 8@(%rsp) */ 1348 arg->arg_type = USDT_ARG_REG_DEREF; 1349 arg->val_off = 0; 1350 reg_off = calc_pt_regs_off(reg_name); 1351 if (reg_off < 0) 1352 return reg_off; 1353 arg->reg_off = reg_off; 1354 } else if (sscanf(arg_str, " %d @ %%%15s %n", arg_sz, reg_name, &len) == 2) { 1355 /* Register read case, e.g., -4@%eax */ 1356 arg->arg_type = USDT_ARG_REG; 1357 /* register read has no memory offset */ 1358 arg->val_off = 0; 1359 1360 reg_off = calc_pt_regs_off(reg_name); 1361 if (reg_off < 0) 1362 return reg_off; 1363 arg->reg_off = reg_off; 1364 } else if (sscanf(arg_str, " %d @ $%ld %n", arg_sz, &off, &len) == 2) { 1365 /* Constant value case, e.g., 4@$71 */ 1366 arg->arg_type = USDT_ARG_CONST; 1367 arg->val_off = off; 1368 arg->reg_off = 0; 1369 } else { 1370 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); 1371 return -EINVAL; 1372 } 1373 1374 return len; 1375} 1376 1377#elif defined(__s390x__) 1378 1379static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) 1380{ 1381 unsigned int reg; 1382 int len; 1383 long off; 1384 1385 if (sscanf(arg_str, " %d @ %ld ( %%r%u ) %n", arg_sz, &off, ®, &len) == 3) { 1386 /* Memory dereference case, e.g., -2@-28(%r15) */ 1387 arg->arg_type = USDT_ARG_REG_DEREF; 1388 arg->val_off = off; 1389 if (reg > 15) { 1390 pr_warn("usdt: unrecognized register '%%r%u'\n", reg); 1391 return -EINVAL; 1392 } 1393 arg->reg_off = offsetof(user_pt_regs, gprs[reg]); 1394 } else if (sscanf(arg_str, " %d @ %%r%u %n", arg_sz, ®, &len) == 2) { 1395 /* Register read case, e.g., -8@%r0 */ 1396 arg->arg_type = USDT_ARG_REG; 1397 arg->val_off = 0; 1398 if (reg > 15) { 1399 pr_warn("usdt: unrecognized register '%%r%u'\n", reg); 1400 return -EINVAL; 1401 } 1402 arg->reg_off = offsetof(user_pt_regs, gprs[reg]); 1403 } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { 1404 /* Constant value case, e.g., 4@71 */ 1405 arg->arg_type = USDT_ARG_CONST; 1406 arg->val_off = off; 1407 arg->reg_off = 0; 1408 } else { 1409 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); 1410 return -EINVAL; 1411 } 1412 1413 return len; 1414} 1415 1416#elif defined(__aarch64__) 1417 1418static int calc_pt_regs_off(const char *reg_name) 1419{ 1420 int reg_num; 1421 1422 if (sscanf(reg_name, "x%d", ®_num) == 1) { 1423 if (reg_num >= 0 && reg_num < 31) 1424 return offsetof(struct user_pt_regs, regs[reg_num]); 1425 } else if (strcmp(reg_name, "sp") == 0) { 1426 return offsetof(struct user_pt_regs, sp); 1427 } 1428 pr_warn("usdt: unrecognized register '%s'\n", reg_name); 1429 return -ENOENT; 1430} 1431 1432static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) 1433{ 1434 char reg_name[16]; 1435 int len, reg_off; 1436 long off; 1437 1438 if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , %ld ] %n", arg_sz, reg_name, &off, &len) == 3) { 1439 /* Memory dereference case, e.g., -4@[sp, 96] */ 1440 arg->arg_type = USDT_ARG_REG_DEREF; 1441 arg->val_off = off; 1442 reg_off = calc_pt_regs_off(reg_name); 1443 if (reg_off < 0) 1444 return reg_off; 1445 arg->reg_off = reg_off; 1446 } else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) { 1447 /* Memory dereference case, e.g., -4@[sp] */ 1448 arg->arg_type = USDT_ARG_REG_DEREF; 1449 arg->val_off = 0; 1450 reg_off = calc_pt_regs_off(reg_name); 1451 if (reg_off < 0) 1452 return reg_off; 1453 arg->reg_off = reg_off; 1454 } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { 1455 /* Constant value case, e.g., 4@5 */ 1456 arg->arg_type = USDT_ARG_CONST; 1457 arg->val_off = off; 1458 arg->reg_off = 0; 1459 } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { 1460 /* Register read case, e.g., -8@x4 */ 1461 arg->arg_type = USDT_ARG_REG; 1462 arg->val_off = 0; 1463 reg_off = calc_pt_regs_off(reg_name); 1464 if (reg_off < 0) 1465 return reg_off; 1466 arg->reg_off = reg_off; 1467 } else { 1468 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); 1469 return -EINVAL; 1470 } 1471 1472 return len; 1473} 1474 1475#elif defined(__riscv) 1476 1477static int calc_pt_regs_off(const char *reg_name) 1478{ 1479 static struct { 1480 const char *name; 1481 size_t pt_regs_off; 1482 } reg_map[] = { 1483 { "ra", offsetof(struct user_regs_struct, ra) }, 1484 { "sp", offsetof(struct user_regs_struct, sp) }, 1485 { "gp", offsetof(struct user_regs_struct, gp) }, 1486 { "tp", offsetof(struct user_regs_struct, tp) }, 1487 { "a0", offsetof(struct user_regs_struct, a0) }, 1488 { "a1", offsetof(struct user_regs_struct, a1) }, 1489 { "a2", offsetof(struct user_regs_struct, a2) }, 1490 { "a3", offsetof(struct user_regs_struct, a3) }, 1491 { "a4", offsetof(struct user_regs_struct, a4) }, 1492 { "a5", offsetof(struct user_regs_struct, a5) }, 1493 { "a6", offsetof(struct user_regs_struct, a6) }, 1494 { "a7", offsetof(struct user_regs_struct, a7) }, 1495 { "s0", offsetof(struct user_regs_struct, s0) }, 1496 { "s1", offsetof(struct user_regs_struct, s1) }, 1497 { "s2", offsetof(struct user_regs_struct, s2) }, 1498 { "s3", offsetof(struct user_regs_struct, s3) }, 1499 { "s4", offsetof(struct user_regs_struct, s4) }, 1500 { "s5", offsetof(struct user_regs_struct, s5) }, 1501 { "s6", offsetof(struct user_regs_struct, s6) }, 1502 { "s7", offsetof(struct user_regs_struct, s7) }, 1503 { "s8", offsetof(struct user_regs_struct, rv_s8) }, 1504 { "s9", offsetof(struct user_regs_struct, s9) }, 1505 { "s10", offsetof(struct user_regs_struct, s10) }, 1506 { "s11", offsetof(struct user_regs_struct, s11) }, 1507 { "t0", offsetof(struct user_regs_struct, t0) }, 1508 { "t1", offsetof(struct user_regs_struct, t1) }, 1509 { "t2", offsetof(struct user_regs_struct, t2) }, 1510 { "t3", offsetof(struct user_regs_struct, t3) }, 1511 { "t4", offsetof(struct user_regs_struct, t4) }, 1512 { "t5", offsetof(struct user_regs_struct, t5) }, 1513 { "t6", offsetof(struct user_regs_struct, t6) }, 1514 }; 1515 int i; 1516 1517 for (i = 0; i < ARRAY_SIZE(reg_map); i++) { 1518 if (strcmp(reg_name, reg_map[i].name) == 0) 1519 return reg_map[i].pt_regs_off; 1520 } 1521 1522 pr_warn("usdt: unrecognized register '%s'\n", reg_name); 1523 return -ENOENT; 1524} 1525 1526static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) 1527{ 1528 char reg_name[16]; 1529 int len, reg_off; 1530 long off; 1531 1532 if (sscanf(arg_str, " %d @ %ld ( %15[a-z0-9] ) %n", arg_sz, &off, reg_name, &len) == 3) { 1533 /* Memory dereference case, e.g., -8@-88(s0) */ 1534 arg->arg_type = USDT_ARG_REG_DEREF; 1535 arg->val_off = off; 1536 reg_off = calc_pt_regs_off(reg_name); 1537 if (reg_off < 0) 1538 return reg_off; 1539 arg->reg_off = reg_off; 1540 } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) { 1541 /* Constant value case, e.g., 4@5 */ 1542 arg->arg_type = USDT_ARG_CONST; 1543 arg->val_off = off; 1544 arg->reg_off = 0; 1545 } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { 1546 /* Register read case, e.g., -8@a1 */ 1547 arg->arg_type = USDT_ARG_REG; 1548 arg->val_off = 0; 1549 reg_off = calc_pt_regs_off(reg_name); 1550 if (reg_off < 0) 1551 return reg_off; 1552 arg->reg_off = reg_off; 1553 } else { 1554 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); 1555 return -EINVAL; 1556 } 1557 1558 return len; 1559} 1560 1561#elif defined(__arm__) 1562 1563static int calc_pt_regs_off(const char *reg_name) 1564{ 1565 static struct { 1566 const char *name; 1567 size_t pt_regs_off; 1568 } reg_map[] = { 1569 { "r0", offsetof(struct pt_regs, uregs[0]) }, 1570 { "r1", offsetof(struct pt_regs, uregs[1]) }, 1571 { "r2", offsetof(struct pt_regs, uregs[2]) }, 1572 { "r3", offsetof(struct pt_regs, uregs[3]) }, 1573 { "r4", offsetof(struct pt_regs, uregs[4]) }, 1574 { "r5", offsetof(struct pt_regs, uregs[5]) }, 1575 { "r6", offsetof(struct pt_regs, uregs[6]) }, 1576 { "r7", offsetof(struct pt_regs, uregs[7]) }, 1577 { "r8", offsetof(struct pt_regs, uregs[8]) }, 1578 { "r9", offsetof(struct pt_regs, uregs[9]) }, 1579 { "r10", offsetof(struct pt_regs, uregs[10]) }, 1580 { "fp", offsetof(struct pt_regs, uregs[11]) }, 1581 { "ip", offsetof(struct pt_regs, uregs[12]) }, 1582 { "sp", offsetof(struct pt_regs, uregs[13]) }, 1583 { "lr", offsetof(struct pt_regs, uregs[14]) }, 1584 { "pc", offsetof(struct pt_regs, uregs[15]) }, 1585 }; 1586 int i; 1587 1588 for (i = 0; i < ARRAY_SIZE(reg_map); i++) { 1589 if (strcmp(reg_name, reg_map[i].name) == 0) 1590 return reg_map[i].pt_regs_off; 1591 } 1592 1593 pr_warn("usdt: unrecognized register '%s'\n", reg_name); 1594 return -ENOENT; 1595} 1596 1597static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) 1598{ 1599 char reg_name[16]; 1600 int len, reg_off; 1601 long off; 1602 1603 if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , #%ld ] %n", 1604 arg_sz, reg_name, &off, &len) == 3) { 1605 /* Memory dereference case, e.g., -4@[fp, #96] */ 1606 arg->arg_type = USDT_ARG_REG_DEREF; 1607 arg->val_off = off; 1608 reg_off = calc_pt_regs_off(reg_name); 1609 if (reg_off < 0) 1610 return reg_off; 1611 arg->reg_off = reg_off; 1612 } else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) { 1613 /* Memory dereference case, e.g., -4@[sp] */ 1614 arg->arg_type = USDT_ARG_REG_DEREF; 1615 arg->val_off = 0; 1616 reg_off = calc_pt_regs_off(reg_name); 1617 if (reg_off < 0) 1618 return reg_off; 1619 arg->reg_off = reg_off; 1620 } else if (sscanf(arg_str, " %d @ #%ld %n", arg_sz, &off, &len) == 2) { 1621 /* Constant value case, e.g., 4@#5 */ 1622 arg->arg_type = USDT_ARG_CONST; 1623 arg->val_off = off; 1624 arg->reg_off = 0; 1625 } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) { 1626 /* Register read case, e.g., -8@r4 */ 1627 arg->arg_type = USDT_ARG_REG; 1628 arg->val_off = 0; 1629 reg_off = calc_pt_regs_off(reg_name); 1630 if (reg_off < 0) 1631 return reg_off; 1632 arg->reg_off = reg_off; 1633 } else { 1634 pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); 1635 return -EINVAL; 1636 } 1637 1638 return len; 1639} 1640 1641#else 1642 1643static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz) 1644{ 1645 pr_warn("usdt: libbpf doesn't support USDTs on current architecture\n"); 1646 return -ENOTSUP; 1647} 1648 1649#endif
~/
Read-only file preview.