FILE
bpf_trace.c
File preview with syntax highlighting.
linux-6.19.14/kernel/trace/bpf_trace.c
1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 */ 5#include <linux/kernel.h> 6#include <linux/types.h> 7#include <linux/slab.h> 8#include <linux/bpf.h> 9#include <linux/bpf_verifier.h> 10#include <linux/bpf_perf_event.h> 11#include <linux/btf.h> 12#include <linux/filter.h> 13#include <linux/uaccess.h> 14#include <linux/ctype.h> 15#include <linux/kprobes.h> 16#include <linux/spinlock.h> 17#include <linux/syscalls.h> 18#include <linux/error-injection.h> 19#include <linux/btf_ids.h> 20#include <linux/bpf_lsm.h> 21#include <linux/fprobe.h> 22#include <linux/bsearch.h> 23#include <linux/sort.h> 24#include <linux/key.h> 25#include <linux/namei.h> 26 27#include <net/bpf_sk_storage.h> 28 29#include <uapi/linux/bpf.h> 30#include <uapi/linux/btf.h> 31 32#include <asm/tlb.h> 33 34#include "trace_probe.h" 35#include "trace.h" 36 37#define CREATE_TRACE_POINTS 38#include "bpf_trace.h" 39 40#define bpf_event_rcu_dereference(p) \ 41 rcu_dereference_protected(p, lockdep_is_held(&bpf_event_mutex)) 42 43#define MAX_UPROBE_MULTI_CNT (1U << 20) 44#define MAX_KPROBE_MULTI_CNT (1U << 20) 45 46#ifdef CONFIG_MODULES 47struct bpf_trace_module { 48 struct module *module; 49 struct list_head list; 50}; 51 52static LIST_HEAD(bpf_trace_modules); 53static DEFINE_MUTEX(bpf_module_mutex); 54 55static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 56{ 57 struct bpf_raw_event_map *btp, *ret = NULL; 58 struct bpf_trace_module *btm; 59 unsigned int i; 60 61 mutex_lock(&bpf_module_mutex); 62 list_for_each_entry(btm, &bpf_trace_modules, list) { 63 for (i = 0; i < btm->module->num_bpf_raw_events; ++i) { 64 btp = &btm->module->bpf_raw_events[i]; 65 if (!strcmp(btp->tp->name, name)) { 66 if (try_module_get(btm->module)) 67 ret = btp; 68 goto out; 69 } 70 } 71 } 72out: 73 mutex_unlock(&bpf_module_mutex); 74 return ret; 75} 76#else 77static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name) 78{ 79 return NULL; 80} 81#endif /* CONFIG_MODULES */ 82 83u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 84u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); 85 86static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, 87 u64 flags, const struct btf **btf, 88 s32 *btf_id); 89static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx); 90static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx); 91 92static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx); 93static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx); 94 95/** 96 * trace_call_bpf - invoke BPF program 97 * @call: tracepoint event 98 * @ctx: opaque context pointer 99 * 100 * kprobe handlers execute BPF programs via this helper. 101 * Can be used from static tracepoints in the future. 102 * 103 * Return: BPF programs always return an integer which is interpreted by 104 * kprobe handler as: 105 * 0 - return from kprobe (event is filtered out) 106 * 1 - store kprobe event into ring buffer 107 * Other values are reserved and currently alias to 1 108 */ 109unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) 110{ 111 unsigned int ret; 112 113 cant_sleep(); 114 115 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { 116 /* 117 * since some bpf program is already running on this cpu, 118 * don't call into another bpf program (same or different) 119 * and don't send kprobe event into ring-buffer, 120 * so return zero here 121 */ 122 rcu_read_lock(); 123 bpf_prog_inc_misses_counters(rcu_dereference(call->prog_array)); 124 rcu_read_unlock(); 125 ret = 0; 126 goto out; 127 } 128 129 /* 130 * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock 131 * to all call sites, we did a bpf_prog_array_valid() there to check 132 * whether call->prog_array is empty or not, which is 133 * a heuristic to speed up execution. 134 * 135 * If bpf_prog_array_valid() fetched prog_array was 136 * non-NULL, we go into trace_call_bpf() and do the actual 137 * proper rcu_dereference() under RCU lock. 138 * If it turns out that prog_array is NULL then, we bail out. 139 * For the opposite, if the bpf_prog_array_valid() fetched pointer 140 * was NULL, you'll skip the prog_array with the risk of missing 141 * out of events when it was updated in between this and the 142 * rcu_dereference() which is accepted risk. 143 */ 144 rcu_read_lock(); 145 ret = bpf_prog_run_array(rcu_dereference(call->prog_array), 146 ctx, bpf_prog_run); 147 rcu_read_unlock(); 148 149 out: 150 __this_cpu_dec(bpf_prog_active); 151 152 return ret; 153} 154 155#ifdef CONFIG_BPF_KPROBE_OVERRIDE 156BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc) 157{ 158 regs_set_return_value(regs, rc); 159 override_function_with_return(regs); 160 return 0; 161} 162 163static const struct bpf_func_proto bpf_override_return_proto = { 164 .func = bpf_override_return, 165 .gpl_only = true, 166 .ret_type = RET_INTEGER, 167 .arg1_type = ARG_PTR_TO_CTX, 168 .arg2_type = ARG_ANYTHING, 169}; 170#endif 171 172static __always_inline int 173bpf_probe_read_user_common(void *dst, u32 size, const void __user *unsafe_ptr) 174{ 175 int ret; 176 177 ret = copy_from_user_nofault(dst, unsafe_ptr, size); 178 if (unlikely(ret < 0)) 179 memset(dst, 0, size); 180 return ret; 181} 182 183BPF_CALL_3(bpf_probe_read_user, void *, dst, u32, size, 184 const void __user *, unsafe_ptr) 185{ 186 return bpf_probe_read_user_common(dst, size, unsafe_ptr); 187} 188 189const struct bpf_func_proto bpf_probe_read_user_proto = { 190 .func = bpf_probe_read_user, 191 .gpl_only = true, 192 .ret_type = RET_INTEGER, 193 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 194 .arg2_type = ARG_CONST_SIZE_OR_ZERO, 195 .arg3_type = ARG_ANYTHING, 196}; 197 198static __always_inline int 199bpf_probe_read_user_str_common(void *dst, u32 size, 200 const void __user *unsafe_ptr) 201{ 202 int ret; 203 204 /* 205 * NB: We rely on strncpy_from_user() not copying junk past the NUL 206 * terminator into `dst`. 207 * 208 * strncpy_from_user() does long-sized strides in the fast path. If the 209 * strncpy does not mask out the bytes after the NUL in `unsafe_ptr`, 210 * then there could be junk after the NUL in `dst`. If user takes `dst` 211 * and keys a hash map with it, then semantically identical strings can 212 * occupy multiple entries in the map. 213 */ 214 ret = strncpy_from_user_nofault(dst, unsafe_ptr, size); 215 if (unlikely(ret < 0)) 216 memset(dst, 0, size); 217 return ret; 218} 219 220BPF_CALL_3(bpf_probe_read_user_str, void *, dst, u32, size, 221 const void __user *, unsafe_ptr) 222{ 223 return bpf_probe_read_user_str_common(dst, size, unsafe_ptr); 224} 225 226const struct bpf_func_proto bpf_probe_read_user_str_proto = { 227 .func = bpf_probe_read_user_str, 228 .gpl_only = true, 229 .ret_type = RET_INTEGER, 230 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 231 .arg2_type = ARG_CONST_SIZE_OR_ZERO, 232 .arg3_type = ARG_ANYTHING, 233}; 234 235BPF_CALL_3(bpf_probe_read_kernel, void *, dst, u32, size, 236 const void *, unsafe_ptr) 237{ 238 return bpf_probe_read_kernel_common(dst, size, unsafe_ptr); 239} 240 241const struct bpf_func_proto bpf_probe_read_kernel_proto = { 242 .func = bpf_probe_read_kernel, 243 .gpl_only = true, 244 .ret_type = RET_INTEGER, 245 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 246 .arg2_type = ARG_CONST_SIZE_OR_ZERO, 247 .arg3_type = ARG_ANYTHING, 248}; 249 250static __always_inline int 251bpf_probe_read_kernel_str_common(void *dst, u32 size, const void *unsafe_ptr) 252{ 253 int ret; 254 255 /* 256 * The strncpy_from_kernel_nofault() call will likely not fill the 257 * entire buffer, but that's okay in this circumstance as we're probing 258 * arbitrary memory anyway similar to bpf_probe_read_*() and might 259 * as well probe the stack. Thus, memory is explicitly cleared 260 * only in error case, so that improper users ignoring return 261 * code altogether don't copy garbage; otherwise length of string 262 * is returned that can be used for bpf_perf_event_output() et al. 263 */ 264 ret = strncpy_from_kernel_nofault(dst, unsafe_ptr, size); 265 if (unlikely(ret < 0)) 266 memset(dst, 0, size); 267 return ret; 268} 269 270BPF_CALL_3(bpf_probe_read_kernel_str, void *, dst, u32, size, 271 const void *, unsafe_ptr) 272{ 273 return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr); 274} 275 276const struct bpf_func_proto bpf_probe_read_kernel_str_proto = { 277 .func = bpf_probe_read_kernel_str, 278 .gpl_only = true, 279 .ret_type = RET_INTEGER, 280 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 281 .arg2_type = ARG_CONST_SIZE_OR_ZERO, 282 .arg3_type = ARG_ANYTHING, 283}; 284 285#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 286BPF_CALL_3(bpf_probe_read_compat, void *, dst, u32, size, 287 const void *, unsafe_ptr) 288{ 289 if ((unsigned long)unsafe_ptr < TASK_SIZE) { 290 return bpf_probe_read_user_common(dst, size, 291 (__force void __user *)unsafe_ptr); 292 } 293 return bpf_probe_read_kernel_common(dst, size, unsafe_ptr); 294} 295 296static const struct bpf_func_proto bpf_probe_read_compat_proto = { 297 .func = bpf_probe_read_compat, 298 .gpl_only = true, 299 .ret_type = RET_INTEGER, 300 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 301 .arg2_type = ARG_CONST_SIZE_OR_ZERO, 302 .arg3_type = ARG_ANYTHING, 303}; 304 305BPF_CALL_3(bpf_probe_read_compat_str, void *, dst, u32, size, 306 const void *, unsafe_ptr) 307{ 308 if ((unsigned long)unsafe_ptr < TASK_SIZE) { 309 return bpf_probe_read_user_str_common(dst, size, 310 (__force void __user *)unsafe_ptr); 311 } 312 return bpf_probe_read_kernel_str_common(dst, size, unsafe_ptr); 313} 314 315static const struct bpf_func_proto bpf_probe_read_compat_str_proto = { 316 .func = bpf_probe_read_compat_str, 317 .gpl_only = true, 318 .ret_type = RET_INTEGER, 319 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 320 .arg2_type = ARG_CONST_SIZE_OR_ZERO, 321 .arg3_type = ARG_ANYTHING, 322}; 323#endif /* CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE */ 324 325BPF_CALL_3(bpf_probe_write_user, void __user *, unsafe_ptr, const void *, src, 326 u32, size) 327{ 328 /* 329 * Ensure we're in user context which is safe for the helper to 330 * run. This helper has no business in a kthread. 331 * 332 * access_ok() should prevent writing to non-user memory, but in 333 * some situations (nommu, temporary switch, etc) access_ok() does 334 * not provide enough validation, hence the check on KERNEL_DS. 335 * 336 * nmi_uaccess_okay() ensures the probe is not run in an interim 337 * state, when the task or mm are switched. This is specifically 338 * required to prevent the use of temporary mm. 339 */ 340 341 if (unlikely(in_interrupt() || 342 current->flags & (PF_KTHREAD | PF_EXITING))) 343 return -EPERM; 344 if (unlikely(!nmi_uaccess_okay())) 345 return -EPERM; 346 347 return copy_to_user_nofault(unsafe_ptr, src, size); 348} 349 350static const struct bpf_func_proto bpf_probe_write_user_proto = { 351 .func = bpf_probe_write_user, 352 .gpl_only = true, 353 .ret_type = RET_INTEGER, 354 .arg1_type = ARG_ANYTHING, 355 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 356 .arg3_type = ARG_CONST_SIZE, 357}; 358 359#define MAX_TRACE_PRINTK_VARARGS 3 360#define BPF_TRACE_PRINTK_SIZE 1024 361 362BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1, 363 u64, arg2, u64, arg3) 364{ 365 u64 args[MAX_TRACE_PRINTK_VARARGS] = { arg1, arg2, arg3 }; 366 struct bpf_bprintf_data data = { 367 .get_bin_args = true, 368 .get_buf = true, 369 }; 370 int ret; 371 372 ret = bpf_bprintf_prepare(fmt, fmt_size, args, 373 MAX_TRACE_PRINTK_VARARGS, &data); 374 if (ret < 0) 375 return ret; 376 377 ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args); 378 379 trace_bpf_trace_printk(data.buf); 380 381 bpf_bprintf_cleanup(&data); 382 383 return ret; 384} 385 386static const struct bpf_func_proto bpf_trace_printk_proto = { 387 .func = bpf_trace_printk, 388 .gpl_only = true, 389 .ret_type = RET_INTEGER, 390 .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, 391 .arg2_type = ARG_CONST_SIZE, 392}; 393 394static void __set_printk_clr_event(struct work_struct *work) 395{ 396 /* 397 * This program might be calling bpf_trace_printk, 398 * so enable the associated bpf_trace/bpf_trace_printk event. 399 * Repeat this each time as it is possible a user has 400 * disabled bpf_trace_printk events. By loading a program 401 * calling bpf_trace_printk() however the user has expressed 402 * the intent to see such events. 403 */ 404 if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1)) 405 pr_warn_ratelimited("could not enable bpf_trace_printk events"); 406} 407static DECLARE_WORK(set_printk_work, __set_printk_clr_event); 408 409const struct bpf_func_proto *bpf_get_trace_printk_proto(void) 410{ 411 schedule_work(&set_printk_work); 412 return &bpf_trace_printk_proto; 413} 414 415BPF_CALL_4(bpf_trace_vprintk, char *, fmt, u32, fmt_size, const void *, args, 416 u32, data_len) 417{ 418 struct bpf_bprintf_data data = { 419 .get_bin_args = true, 420 .get_buf = true, 421 }; 422 int ret, num_args; 423 424 if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 || 425 (data_len && !args)) 426 return -EINVAL; 427 num_args = data_len / 8; 428 429 ret = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data); 430 if (ret < 0) 431 return ret; 432 433 ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt, data.bin_args); 434 435 trace_bpf_trace_printk(data.buf); 436 437 bpf_bprintf_cleanup(&data); 438 439 return ret; 440} 441 442static const struct bpf_func_proto bpf_trace_vprintk_proto = { 443 .func = bpf_trace_vprintk, 444 .gpl_only = true, 445 .ret_type = RET_INTEGER, 446 .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, 447 .arg2_type = ARG_CONST_SIZE, 448 .arg3_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, 449 .arg4_type = ARG_CONST_SIZE_OR_ZERO, 450}; 451 452const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void) 453{ 454 schedule_work(&set_printk_work); 455 return &bpf_trace_vprintk_proto; 456} 457 458BPF_CALL_5(bpf_seq_printf, struct seq_file *, m, char *, fmt, u32, fmt_size, 459 const void *, args, u32, data_len) 460{ 461 struct bpf_bprintf_data data = { 462 .get_bin_args = true, 463 }; 464 int err, num_args; 465 466 if (data_len & 7 || data_len > MAX_BPRINTF_VARARGS * 8 || 467 (data_len && !args)) 468 return -EINVAL; 469 num_args = data_len / 8; 470 471 err = bpf_bprintf_prepare(fmt, fmt_size, args, num_args, &data); 472 if (err < 0) 473 return err; 474 475 seq_bprintf(m, fmt, data.bin_args); 476 477 bpf_bprintf_cleanup(&data); 478 479 return seq_has_overflowed(m) ? -EOVERFLOW : 0; 480} 481 482BTF_ID_LIST_SINGLE(btf_seq_file_ids, struct, seq_file) 483 484static const struct bpf_func_proto bpf_seq_printf_proto = { 485 .func = bpf_seq_printf, 486 .gpl_only = true, 487 .ret_type = RET_INTEGER, 488 .arg1_type = ARG_PTR_TO_BTF_ID, 489 .arg1_btf_id = &btf_seq_file_ids[0], 490 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 491 .arg3_type = ARG_CONST_SIZE, 492 .arg4_type = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY, 493 .arg5_type = ARG_CONST_SIZE_OR_ZERO, 494}; 495 496BPF_CALL_3(bpf_seq_write, struct seq_file *, m, const void *, data, u32, len) 497{ 498 return seq_write(m, data, len) ? -EOVERFLOW : 0; 499} 500 501static const struct bpf_func_proto bpf_seq_write_proto = { 502 .func = bpf_seq_write, 503 .gpl_only = true, 504 .ret_type = RET_INTEGER, 505 .arg1_type = ARG_PTR_TO_BTF_ID, 506 .arg1_btf_id = &btf_seq_file_ids[0], 507 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 508 .arg3_type = ARG_CONST_SIZE_OR_ZERO, 509}; 510 511BPF_CALL_4(bpf_seq_printf_btf, struct seq_file *, m, struct btf_ptr *, ptr, 512 u32, btf_ptr_size, u64, flags) 513{ 514 const struct btf *btf; 515 s32 btf_id; 516 int ret; 517 518 ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); 519 if (ret) 520 return ret; 521 522 return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags); 523} 524 525static const struct bpf_func_proto bpf_seq_printf_btf_proto = { 526 .func = bpf_seq_printf_btf, 527 .gpl_only = true, 528 .ret_type = RET_INTEGER, 529 .arg1_type = ARG_PTR_TO_BTF_ID, 530 .arg1_btf_id = &btf_seq_file_ids[0], 531 .arg2_type = ARG_PTR_TO_MEM | MEM_RDONLY, 532 .arg3_type = ARG_CONST_SIZE_OR_ZERO, 533 .arg4_type = ARG_ANYTHING, 534}; 535 536static __always_inline int 537get_map_perf_counter(struct bpf_map *map, u64 flags, 538 u64 *value, u64 *enabled, u64 *running) 539{ 540 struct bpf_array *array = container_of(map, struct bpf_array, map); 541 unsigned int cpu = smp_processor_id(); 542 u64 index = flags & BPF_F_INDEX_MASK; 543 struct bpf_event_entry *ee; 544 545 if (unlikely(flags & ~(BPF_F_INDEX_MASK))) 546 return -EINVAL; 547 if (index == BPF_F_CURRENT_CPU) 548 index = cpu; 549 if (unlikely(index >= array->map.max_entries)) 550 return -E2BIG; 551 552 ee = READ_ONCE(array->ptrs[index]); 553 if (!ee) 554 return -ENOENT; 555 556 return perf_event_read_local(ee->event, value, enabled, running); 557} 558 559BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags) 560{ 561 u64 value = 0; 562 int err; 563 564 err = get_map_perf_counter(map, flags, &value, NULL, NULL); 565 /* 566 * this api is ugly since we miss [-22..-2] range of valid 567 * counter values, but that's uapi 568 */ 569 if (err) 570 return err; 571 return value; 572} 573 574const struct bpf_func_proto bpf_perf_event_read_proto = { 575 .func = bpf_perf_event_read, 576 .gpl_only = true, 577 .ret_type = RET_INTEGER, 578 .arg1_type = ARG_CONST_MAP_PTR, 579 .arg2_type = ARG_ANYTHING, 580}; 581 582BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags, 583 struct bpf_perf_event_value *, buf, u32, size) 584{ 585 int err = -EINVAL; 586 587 if (unlikely(size != sizeof(struct bpf_perf_event_value))) 588 goto clear; 589 err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled, 590 &buf->running); 591 if (unlikely(err)) 592 goto clear; 593 return 0; 594clear: 595 memset(buf, 0, size); 596 return err; 597} 598 599static const struct bpf_func_proto bpf_perf_event_read_value_proto = { 600 .func = bpf_perf_event_read_value, 601 .gpl_only = true, 602 .ret_type = RET_INTEGER, 603 .arg1_type = ARG_CONST_MAP_PTR, 604 .arg2_type = ARG_ANYTHING, 605 .arg3_type = ARG_PTR_TO_UNINIT_MEM, 606 .arg4_type = ARG_CONST_SIZE, 607}; 608 609const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void) 610{ 611 return &bpf_perf_event_read_value_proto; 612} 613 614static __always_inline u64 615__bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, 616 u64 flags, struct perf_raw_record *raw, 617 struct perf_sample_data *sd) 618{ 619 struct bpf_array *array = container_of(map, struct bpf_array, map); 620 unsigned int cpu = smp_processor_id(); 621 u64 index = flags & BPF_F_INDEX_MASK; 622 struct bpf_event_entry *ee; 623 struct perf_event *event; 624 625 if (index == BPF_F_CURRENT_CPU) 626 index = cpu; 627 if (unlikely(index >= array->map.max_entries)) 628 return -E2BIG; 629 630 ee = READ_ONCE(array->ptrs[index]); 631 if (!ee) 632 return -ENOENT; 633 634 event = ee->event; 635 if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || 636 event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) 637 return -EINVAL; 638 639 if (unlikely(event->oncpu != cpu)) 640 return -EOPNOTSUPP; 641 642 perf_sample_save_raw_data(sd, event, raw); 643 644 return perf_event_output(event, sd, regs); 645} 646 647/* 648 * Support executing tracepoints in normal, irq, and nmi context that each call 649 * bpf_perf_event_output 650 */ 651struct bpf_trace_sample_data { 652 struct perf_sample_data sds[3]; 653}; 654 655static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_trace_sds); 656static DEFINE_PER_CPU(int, bpf_trace_nest_level); 657BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, 658 u64, flags, void *, data, u64, size) 659{ 660 struct bpf_trace_sample_data *sds; 661 struct perf_raw_record raw = { 662 .frag = { 663 .size = size, 664 .data = data, 665 }, 666 }; 667 struct perf_sample_data *sd; 668 int nest_level, err; 669 670 preempt_disable(); 671 sds = this_cpu_ptr(&bpf_trace_sds); 672 nest_level = this_cpu_inc_return(bpf_trace_nest_level); 673 674 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(sds->sds))) { 675 err = -EBUSY; 676 goto out; 677 } 678 679 sd = &sds->sds[nest_level - 1]; 680 681 if (unlikely(flags & ~(BPF_F_INDEX_MASK))) { 682 err = -EINVAL; 683 goto out; 684 } 685 686 perf_sample_data_init(sd, 0, 0); 687 688 err = __bpf_perf_event_output(regs, map, flags, &raw, sd); 689out: 690 this_cpu_dec(bpf_trace_nest_level); 691 preempt_enable(); 692 return err; 693} 694 695static const struct bpf_func_proto bpf_perf_event_output_proto = { 696 .func = bpf_perf_event_output, 697 .gpl_only = true, 698 .ret_type = RET_INTEGER, 699 .arg1_type = ARG_PTR_TO_CTX, 700 .arg2_type = ARG_CONST_MAP_PTR, 701 .arg3_type = ARG_ANYTHING, 702 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, 703 .arg5_type = ARG_CONST_SIZE_OR_ZERO, 704}; 705 706static DEFINE_PER_CPU(int, bpf_event_output_nest_level); 707struct bpf_nested_pt_regs { 708 struct pt_regs regs[3]; 709}; 710static DEFINE_PER_CPU(struct bpf_nested_pt_regs, bpf_pt_regs); 711static DEFINE_PER_CPU(struct bpf_trace_sample_data, bpf_misc_sds); 712 713u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, 714 void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) 715{ 716 struct perf_raw_frag frag = { 717 .copy = ctx_copy, 718 .size = ctx_size, 719 .data = ctx, 720 }; 721 struct perf_raw_record raw = { 722 .frag = { 723 { 724 .next = ctx_size ? &frag : NULL, 725 }, 726 .size = meta_size, 727 .data = meta, 728 }, 729 }; 730 struct perf_sample_data *sd; 731 struct pt_regs *regs; 732 int nest_level; 733 u64 ret; 734 735 preempt_disable(); 736 nest_level = this_cpu_inc_return(bpf_event_output_nest_level); 737 738 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bpf_misc_sds.sds))) { 739 ret = -EBUSY; 740 goto out; 741 } 742 sd = this_cpu_ptr(&bpf_misc_sds.sds[nest_level - 1]); 743 regs = this_cpu_ptr(&bpf_pt_regs.regs[nest_level - 1]); 744 745 perf_fetch_caller_regs(regs); 746 perf_sample_data_init(sd, 0, 0); 747 748 ret = __bpf_perf_event_output(regs, map, flags, &raw, sd); 749out: 750 this_cpu_dec(bpf_event_output_nest_level); 751 preempt_enable(); 752 return ret; 753} 754 755BPF_CALL_0(bpf_get_current_task) 756{ 757 return (long) current; 758} 759 760const struct bpf_func_proto bpf_get_current_task_proto = { 761 .func = bpf_get_current_task, 762 .gpl_only = true, 763 .ret_type = RET_INTEGER, 764}; 765 766BPF_CALL_0(bpf_get_current_task_btf) 767{ 768 return (unsigned long) current; 769} 770 771const struct bpf_func_proto bpf_get_current_task_btf_proto = { 772 .func = bpf_get_current_task_btf, 773 .gpl_only = true, 774 .ret_type = RET_PTR_TO_BTF_ID_TRUSTED, 775 .ret_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 776}; 777 778BPF_CALL_1(bpf_task_pt_regs, struct task_struct *, task) 779{ 780 return (unsigned long) task_pt_regs(task); 781} 782 783BTF_ID_LIST_SINGLE(bpf_task_pt_regs_ids, struct, pt_regs) 784 785const struct bpf_func_proto bpf_task_pt_regs_proto = { 786 .func = bpf_task_pt_regs, 787 .gpl_only = true, 788 .arg1_type = ARG_PTR_TO_BTF_ID, 789 .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], 790 .ret_type = RET_PTR_TO_BTF_ID, 791 .ret_btf_id = &bpf_task_pt_regs_ids[0], 792}; 793 794struct send_signal_irq_work { 795 struct irq_work irq_work; 796 struct task_struct *task; 797 u32 sig; 798 enum pid_type type; 799 bool has_siginfo; 800 struct kernel_siginfo info; 801}; 802 803static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); 804 805static void do_bpf_send_signal(struct irq_work *entry) 806{ 807 struct send_signal_irq_work *work; 808 struct kernel_siginfo *siginfo; 809 810 work = container_of(entry, struct send_signal_irq_work, irq_work); 811 siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV; 812 813 group_send_sig_info(work->sig, siginfo, work->task, work->type); 814 put_task_struct(work->task); 815} 816 817static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value) 818{ 819 struct send_signal_irq_work *work = NULL; 820 struct kernel_siginfo info; 821 struct kernel_siginfo *siginfo; 822 823 if (!task) { 824 task = current; 825 siginfo = SEND_SIG_PRIV; 826 } else { 827 clear_siginfo(&info); 828 info.si_signo = sig; 829 info.si_errno = 0; 830 info.si_code = SI_KERNEL; 831 info.si_pid = 0; 832 info.si_uid = 0; 833 info.si_value.sival_ptr = (void *)(unsigned long)value; 834 siginfo = &info; 835 } 836 837 /* Similar to bpf_probe_write_user, task needs to be 838 * in a sound condition and kernel memory access be 839 * permitted in order to send signal to the current 840 * task. 841 */ 842 if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING))) 843 return -EPERM; 844 if (unlikely(!nmi_uaccess_okay())) 845 return -EPERM; 846 /* Task should not be pid=1 to avoid kernel panic. */ 847 if (unlikely(is_global_init(task))) 848 return -EPERM; 849 850 if (preempt_count() != 0 || irqs_disabled()) { 851 /* Do an early check on signal validity. Otherwise, 852 * the error is lost in deferred irq_work. 853 */ 854 if (unlikely(!valid_signal(sig))) 855 return -EINVAL; 856 857 work = this_cpu_ptr(&send_signal_work); 858 if (irq_work_is_busy(&work->irq_work)) 859 return -EBUSY; 860 861 /* Add the current task, which is the target of sending signal, 862 * to the irq_work. The current task may change when queued 863 * irq works get executed. 864 */ 865 work->task = get_task_struct(task); 866 work->has_siginfo = siginfo == &info; 867 if (work->has_siginfo) 868 copy_siginfo(&work->info, &info); 869 work->sig = sig; 870 work->type = type; 871 irq_work_queue(&work->irq_work); 872 return 0; 873 } 874 875 return group_send_sig_info(sig, siginfo, task, type); 876} 877 878BPF_CALL_1(bpf_send_signal, u32, sig) 879{ 880 return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0); 881} 882 883const struct bpf_func_proto bpf_send_signal_proto = { 884 .func = bpf_send_signal, 885 .gpl_only = false, 886 .ret_type = RET_INTEGER, 887 .arg1_type = ARG_ANYTHING, 888}; 889 890BPF_CALL_1(bpf_send_signal_thread, u32, sig) 891{ 892 return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0); 893} 894 895const struct bpf_func_proto bpf_send_signal_thread_proto = { 896 .func = bpf_send_signal_thread, 897 .gpl_only = false, 898 .ret_type = RET_INTEGER, 899 .arg1_type = ARG_ANYTHING, 900}; 901 902BPF_CALL_3(bpf_d_path, const struct path *, path, char *, buf, u32, sz) 903{ 904 struct path copy; 905 long len; 906 char *p; 907 908 if (!sz) 909 return 0; 910 911 /* 912 * The path pointer is verified as trusted and safe to use, 913 * but let's double check it's valid anyway to workaround 914 * potentially broken verifier. 915 */ 916 len = copy_from_kernel_nofault(©, path, sizeof(*path)); 917 if (len < 0) 918 return len; 919 920 p = d_path(©, buf, sz); 921 if (IS_ERR(p)) { 922 len = PTR_ERR(p); 923 } else { 924 len = buf + sz - p; 925 memmove(buf, p, len); 926 } 927 928 return len; 929} 930 931BTF_SET_START(btf_allowlist_d_path) 932#ifdef CONFIG_SECURITY 933BTF_ID(func, security_file_permission) 934BTF_ID(func, security_inode_getattr) 935BTF_ID(func, security_file_open) 936#endif 937#ifdef CONFIG_SECURITY_PATH 938BTF_ID(func, security_path_truncate) 939#endif 940BTF_ID(func, vfs_truncate) 941BTF_ID(func, vfs_fallocate) 942BTF_ID(func, dentry_open) 943BTF_ID(func, vfs_getattr) 944BTF_ID(func, filp_close) 945BTF_SET_END(btf_allowlist_d_path) 946 947static bool bpf_d_path_allowed(const struct bpf_prog *prog) 948{ 949 if (prog->type == BPF_PROG_TYPE_TRACING && 950 prog->expected_attach_type == BPF_TRACE_ITER) 951 return true; 952 953 if (prog->type == BPF_PROG_TYPE_LSM) 954 return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id); 955 956 return btf_id_set_contains(&btf_allowlist_d_path, 957 prog->aux->attach_btf_id); 958} 959 960BTF_ID_LIST_SINGLE(bpf_d_path_btf_ids, struct, path) 961 962static const struct bpf_func_proto bpf_d_path_proto = { 963 .func = bpf_d_path, 964 .gpl_only = false, 965 .ret_type = RET_INTEGER, 966 .arg1_type = ARG_PTR_TO_BTF_ID, 967 .arg1_btf_id = &bpf_d_path_btf_ids[0], 968 .arg2_type = ARG_PTR_TO_MEM | MEM_WRITE, 969 .arg3_type = ARG_CONST_SIZE_OR_ZERO, 970 .allowed = bpf_d_path_allowed, 971}; 972 973#define BTF_F_ALL (BTF_F_COMPACT | BTF_F_NONAME | \ 974 BTF_F_PTR_RAW | BTF_F_ZERO) 975 976static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size, 977 u64 flags, const struct btf **btf, 978 s32 *btf_id) 979{ 980 const struct btf_type *t; 981 982 if (unlikely(flags & ~(BTF_F_ALL))) 983 return -EINVAL; 984 985 if (btf_ptr_size != sizeof(struct btf_ptr)) 986 return -EINVAL; 987 988 *btf = bpf_get_btf_vmlinux(); 989 990 if (IS_ERR_OR_NULL(*btf)) 991 return IS_ERR(*btf) ? PTR_ERR(*btf) : -EINVAL; 992 993 if (ptr->type_id > 0) 994 *btf_id = ptr->type_id; 995 else 996 return -EINVAL; 997 998 if (*btf_id > 0) 999 t = btf_type_by_id(*btf, *btf_id); 1000 if (*btf_id <= 0 || !t) 1001 return -ENOENT; 1002 1003 return 0; 1004} 1005 1006BPF_CALL_5(bpf_snprintf_btf, char *, str, u32, str_size, struct btf_ptr *, ptr, 1007 u32, btf_ptr_size, u64, flags) 1008{ 1009 const struct btf *btf; 1010 s32 btf_id; 1011 int ret; 1012 1013 ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id); 1014 if (ret) 1015 return ret; 1016 1017 return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size, 1018 flags); 1019} 1020 1021const struct bpf_func_proto bpf_snprintf_btf_proto = { 1022 .func = bpf_snprintf_btf, 1023 .gpl_only = false, 1024 .ret_type = RET_INTEGER, 1025 .arg1_type = ARG_PTR_TO_MEM | MEM_WRITE, 1026 .arg2_type = ARG_CONST_SIZE, 1027 .arg3_type = ARG_PTR_TO_MEM | MEM_RDONLY, 1028 .arg4_type = ARG_CONST_SIZE, 1029 .arg5_type = ARG_ANYTHING, 1030}; 1031 1032BPF_CALL_1(bpf_get_func_ip_tracing, void *, ctx) 1033{ 1034 /* This helper call is inlined by verifier. */ 1035 return ((u64 *)ctx)[-2]; 1036} 1037 1038static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = { 1039 .func = bpf_get_func_ip_tracing, 1040 .gpl_only = true, 1041 .ret_type = RET_INTEGER, 1042 .arg1_type = ARG_PTR_TO_CTX, 1043}; 1044 1045static inline unsigned long get_entry_ip(unsigned long fentry_ip) 1046{ 1047#ifdef CONFIG_X86_KERNEL_IBT 1048 if (is_endbr((void *)(fentry_ip - ENDBR_INSN_SIZE))) 1049 fentry_ip -= ENDBR_INSN_SIZE; 1050#endif 1051 return fentry_ip; 1052} 1053 1054BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs) 1055{ 1056 struct bpf_trace_run_ctx *run_ctx __maybe_unused; 1057 struct kprobe *kp; 1058 1059#ifdef CONFIG_UPROBES 1060 run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); 1061 if (run_ctx->is_uprobe) 1062 return ((struct uprobe_dispatch_data *)current->utask->vaddr)->bp_addr; 1063#endif 1064 1065 kp = kprobe_running(); 1066 1067 if (!kp || !(kp->flags & KPROBE_FLAG_ON_FUNC_ENTRY)) 1068 return 0; 1069 1070 return get_entry_ip((uintptr_t)kp->addr); 1071} 1072 1073static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe = { 1074 .func = bpf_get_func_ip_kprobe, 1075 .gpl_only = true, 1076 .ret_type = RET_INTEGER, 1077 .arg1_type = ARG_PTR_TO_CTX, 1078}; 1079 1080BPF_CALL_1(bpf_get_func_ip_kprobe_multi, struct pt_regs *, regs) 1081{ 1082 return bpf_kprobe_multi_entry_ip(current->bpf_ctx); 1083} 1084 1085static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe_multi = { 1086 .func = bpf_get_func_ip_kprobe_multi, 1087 .gpl_only = false, 1088 .ret_type = RET_INTEGER, 1089 .arg1_type = ARG_PTR_TO_CTX, 1090}; 1091 1092BPF_CALL_1(bpf_get_attach_cookie_kprobe_multi, struct pt_regs *, regs) 1093{ 1094 return bpf_kprobe_multi_cookie(current->bpf_ctx); 1095} 1096 1097static const struct bpf_func_proto bpf_get_attach_cookie_proto_kmulti = { 1098 .func = bpf_get_attach_cookie_kprobe_multi, 1099 .gpl_only = false, 1100 .ret_type = RET_INTEGER, 1101 .arg1_type = ARG_PTR_TO_CTX, 1102}; 1103 1104BPF_CALL_1(bpf_get_func_ip_uprobe_multi, struct pt_regs *, regs) 1105{ 1106 return bpf_uprobe_multi_entry_ip(current->bpf_ctx); 1107} 1108 1109static const struct bpf_func_proto bpf_get_func_ip_proto_uprobe_multi = { 1110 .func = bpf_get_func_ip_uprobe_multi, 1111 .gpl_only = false, 1112 .ret_type = RET_INTEGER, 1113 .arg1_type = ARG_PTR_TO_CTX, 1114}; 1115 1116BPF_CALL_1(bpf_get_attach_cookie_uprobe_multi, struct pt_regs *, regs) 1117{ 1118 return bpf_uprobe_multi_cookie(current->bpf_ctx); 1119} 1120 1121static const struct bpf_func_proto bpf_get_attach_cookie_proto_umulti = { 1122 .func = bpf_get_attach_cookie_uprobe_multi, 1123 .gpl_only = false, 1124 .ret_type = RET_INTEGER, 1125 .arg1_type = ARG_PTR_TO_CTX, 1126}; 1127 1128BPF_CALL_1(bpf_get_attach_cookie_trace, void *, ctx) 1129{ 1130 struct bpf_trace_run_ctx *run_ctx; 1131 1132 run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); 1133 return run_ctx->bpf_cookie; 1134} 1135 1136static const struct bpf_func_proto bpf_get_attach_cookie_proto_trace = { 1137 .func = bpf_get_attach_cookie_trace, 1138 .gpl_only = false, 1139 .ret_type = RET_INTEGER, 1140 .arg1_type = ARG_PTR_TO_CTX, 1141}; 1142 1143BPF_CALL_1(bpf_get_attach_cookie_pe, struct bpf_perf_event_data_kern *, ctx) 1144{ 1145 return ctx->event->bpf_cookie; 1146} 1147 1148static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = { 1149 .func = bpf_get_attach_cookie_pe, 1150 .gpl_only = false, 1151 .ret_type = RET_INTEGER, 1152 .arg1_type = ARG_PTR_TO_CTX, 1153}; 1154 1155BPF_CALL_1(bpf_get_attach_cookie_tracing, void *, ctx) 1156{ 1157 struct bpf_trace_run_ctx *run_ctx; 1158 1159 run_ctx = container_of(current->bpf_ctx, struct bpf_trace_run_ctx, run_ctx); 1160 return run_ctx->bpf_cookie; 1161} 1162 1163static const struct bpf_func_proto bpf_get_attach_cookie_proto_tracing = { 1164 .func = bpf_get_attach_cookie_tracing, 1165 .gpl_only = false, 1166 .ret_type = RET_INTEGER, 1167 .arg1_type = ARG_PTR_TO_CTX, 1168}; 1169 1170BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags) 1171{ 1172 static const u32 br_entry_size = sizeof(struct perf_branch_entry); 1173 u32 entry_cnt = size / br_entry_size; 1174 1175 entry_cnt = static_call(perf_snapshot_branch_stack)(buf, entry_cnt); 1176 1177 if (unlikely(flags)) 1178 return -EINVAL; 1179 1180 if (!entry_cnt) 1181 return -ENOENT; 1182 1183 return entry_cnt * br_entry_size; 1184} 1185 1186const struct bpf_func_proto bpf_get_branch_snapshot_proto = { 1187 .func = bpf_get_branch_snapshot, 1188 .gpl_only = true, 1189 .ret_type = RET_INTEGER, 1190 .arg1_type = ARG_PTR_TO_UNINIT_MEM, 1191 .arg2_type = ARG_CONST_SIZE_OR_ZERO, 1192}; 1193 1194BPF_CALL_3(get_func_arg, void *, ctx, u32, n, u64 *, value) 1195{ 1196 /* This helper call is inlined by verifier. */ 1197 u64 nr_args = ((u64 *)ctx)[-1]; 1198 1199 if ((u64) n >= nr_args) 1200 return -EINVAL; 1201 *value = ((u64 *)ctx)[n]; 1202 return 0; 1203} 1204 1205static const struct bpf_func_proto bpf_get_func_arg_proto = { 1206 .func = get_func_arg, 1207 .ret_type = RET_INTEGER, 1208 .arg1_type = ARG_PTR_TO_CTX, 1209 .arg2_type = ARG_ANYTHING, 1210 .arg3_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, 1211 .arg3_size = sizeof(u64), 1212}; 1213 1214BPF_CALL_2(get_func_ret, void *, ctx, u64 *, value) 1215{ 1216 /* This helper call is inlined by verifier. */ 1217 u64 nr_args = ((u64 *)ctx)[-1]; 1218 1219 *value = ((u64 *)ctx)[nr_args]; 1220 return 0; 1221} 1222 1223static const struct bpf_func_proto bpf_get_func_ret_proto = { 1224 .func = get_func_ret, 1225 .ret_type = RET_INTEGER, 1226 .arg1_type = ARG_PTR_TO_CTX, 1227 .arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_UNINIT | MEM_WRITE | MEM_ALIGNED, 1228 .arg2_size = sizeof(u64), 1229}; 1230 1231BPF_CALL_1(get_func_arg_cnt, void *, ctx) 1232{ 1233 /* This helper call is inlined by verifier. */ 1234 return ((u64 *)ctx)[-1]; 1235} 1236 1237static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = { 1238 .func = get_func_arg_cnt, 1239 .ret_type = RET_INTEGER, 1240 .arg1_type = ARG_PTR_TO_CTX, 1241}; 1242 1243static const struct bpf_func_proto * 1244bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1245{ 1246 const struct bpf_func_proto *func_proto; 1247 1248 switch (func_id) { 1249 case BPF_FUNC_get_smp_processor_id: 1250 return &bpf_get_smp_processor_id_proto; 1251#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE 1252 case BPF_FUNC_probe_read: 1253 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ? 1254 NULL : &bpf_probe_read_compat_proto; 1255 case BPF_FUNC_probe_read_str: 1256 return security_locked_down(LOCKDOWN_BPF_READ_KERNEL) < 0 ? 1257 NULL : &bpf_probe_read_compat_str_proto; 1258#endif 1259 case BPF_FUNC_get_func_ip: 1260 return &bpf_get_func_ip_proto_tracing; 1261 default: 1262 break; 1263 } 1264 1265 func_proto = bpf_base_func_proto(func_id, prog); 1266 if (func_proto) 1267 return func_proto; 1268 1269 if (!bpf_token_capable(prog->aux->token, CAP_SYS_ADMIN)) 1270 return NULL; 1271 1272 switch (func_id) { 1273 case BPF_FUNC_probe_write_user: 1274 return security_locked_down(LOCKDOWN_BPF_WRITE_USER) < 0 ? 1275 NULL : &bpf_probe_write_user_proto; 1276 default: 1277 return NULL; 1278 } 1279} 1280 1281static bool is_kprobe_multi(const struct bpf_prog *prog) 1282{ 1283 return prog->expected_attach_type == BPF_TRACE_KPROBE_MULTI || 1284 prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION; 1285} 1286 1287static inline bool is_kprobe_session(const struct bpf_prog *prog) 1288{ 1289 return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION; 1290} 1291 1292static inline bool is_uprobe_multi(const struct bpf_prog *prog) 1293{ 1294 return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI || 1295 prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION; 1296} 1297 1298static inline bool is_uprobe_session(const struct bpf_prog *prog) 1299{ 1300 return prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION; 1301} 1302 1303static const struct bpf_func_proto * 1304kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1305{ 1306 switch (func_id) { 1307 case BPF_FUNC_perf_event_output: 1308 return &bpf_perf_event_output_proto; 1309 case BPF_FUNC_get_stackid: 1310 return &bpf_get_stackid_proto; 1311 case BPF_FUNC_get_stack: 1312 return prog->sleepable ? &bpf_get_stack_sleepable_proto : &bpf_get_stack_proto; 1313#ifdef CONFIG_BPF_KPROBE_OVERRIDE 1314 case BPF_FUNC_override_return: 1315 return &bpf_override_return_proto; 1316#endif 1317 case BPF_FUNC_get_func_ip: 1318 if (is_kprobe_multi(prog)) 1319 return &bpf_get_func_ip_proto_kprobe_multi; 1320 if (is_uprobe_multi(prog)) 1321 return &bpf_get_func_ip_proto_uprobe_multi; 1322 return &bpf_get_func_ip_proto_kprobe; 1323 case BPF_FUNC_get_attach_cookie: 1324 if (is_kprobe_multi(prog)) 1325 return &bpf_get_attach_cookie_proto_kmulti; 1326 if (is_uprobe_multi(prog)) 1327 return &bpf_get_attach_cookie_proto_umulti; 1328 return &bpf_get_attach_cookie_proto_trace; 1329 default: 1330 return bpf_tracing_func_proto(func_id, prog); 1331 } 1332} 1333 1334/* bpf+kprobe programs can access fields of 'struct pt_regs' */ 1335static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 1336 const struct bpf_prog *prog, 1337 struct bpf_insn_access_aux *info) 1338{ 1339 if (off < 0 || off >= sizeof(struct pt_regs)) 1340 return false; 1341 if (off % size != 0) 1342 return false; 1343 /* 1344 * Assertion for 32 bit to make sure last 8 byte access 1345 * (BPF_DW) to the last 4 byte member is disallowed. 1346 */ 1347 if (off + size > sizeof(struct pt_regs)) 1348 return false; 1349 1350 if (type == BPF_WRITE) 1351 prog->aux->kprobe_write_ctx = true; 1352 1353 return true; 1354} 1355 1356const struct bpf_verifier_ops kprobe_verifier_ops = { 1357 .get_func_proto = kprobe_prog_func_proto, 1358 .is_valid_access = kprobe_prog_is_valid_access, 1359}; 1360 1361const struct bpf_prog_ops kprobe_prog_ops = { 1362}; 1363 1364BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map, 1365 u64, flags, void *, data, u64, size) 1366{ 1367 struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1368 1369 /* 1370 * r1 points to perf tracepoint buffer where first 8 bytes are hidden 1371 * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it 1372 * from there and call the same bpf_perf_event_output() helper inline. 1373 */ 1374 return ____bpf_perf_event_output(regs, map, flags, data, size); 1375} 1376 1377static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { 1378 .func = bpf_perf_event_output_tp, 1379 .gpl_only = true, 1380 .ret_type = RET_INTEGER, 1381 .arg1_type = ARG_PTR_TO_CTX, 1382 .arg2_type = ARG_CONST_MAP_PTR, 1383 .arg3_type = ARG_ANYTHING, 1384 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, 1385 .arg5_type = ARG_CONST_SIZE_OR_ZERO, 1386}; 1387 1388BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, 1389 u64, flags) 1390{ 1391 struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1392 1393 /* 1394 * Same comment as in bpf_perf_event_output_tp(), only that this time 1395 * the other helper's function body cannot be inlined due to being 1396 * external, thus we need to call raw helper function. 1397 */ 1398 return bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1399 flags, 0, 0); 1400} 1401 1402static const struct bpf_func_proto bpf_get_stackid_proto_tp = { 1403 .func = bpf_get_stackid_tp, 1404 .gpl_only = true, 1405 .ret_type = RET_INTEGER, 1406 .arg1_type = ARG_PTR_TO_CTX, 1407 .arg2_type = ARG_CONST_MAP_PTR, 1408 .arg3_type = ARG_ANYTHING, 1409}; 1410 1411BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size, 1412 u64, flags) 1413{ 1414 struct pt_regs *regs = *(struct pt_regs **)tp_buff; 1415 1416 return bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1417 (unsigned long) size, flags, 0); 1418} 1419 1420static const struct bpf_func_proto bpf_get_stack_proto_tp = { 1421 .func = bpf_get_stack_tp, 1422 .gpl_only = true, 1423 .ret_type = RET_INTEGER, 1424 .arg1_type = ARG_PTR_TO_CTX, 1425 .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1426 .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1427 .arg4_type = ARG_ANYTHING, 1428}; 1429 1430static const struct bpf_func_proto * 1431tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1432{ 1433 switch (func_id) { 1434 case BPF_FUNC_perf_event_output: 1435 return &bpf_perf_event_output_proto_tp; 1436 case BPF_FUNC_get_stackid: 1437 return &bpf_get_stackid_proto_tp; 1438 case BPF_FUNC_get_stack: 1439 return &bpf_get_stack_proto_tp; 1440 case BPF_FUNC_get_attach_cookie: 1441 return &bpf_get_attach_cookie_proto_trace; 1442 default: 1443 return bpf_tracing_func_proto(func_id, prog); 1444 } 1445} 1446 1447static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type, 1448 const struct bpf_prog *prog, 1449 struct bpf_insn_access_aux *info) 1450{ 1451 if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE) 1452 return false; 1453 if (type != BPF_READ) 1454 return false; 1455 if (off % size != 0) 1456 return false; 1457 1458 BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64)); 1459 return true; 1460} 1461 1462const struct bpf_verifier_ops tracepoint_verifier_ops = { 1463 .get_func_proto = tp_prog_func_proto, 1464 .is_valid_access = tp_prog_is_valid_access, 1465}; 1466 1467const struct bpf_prog_ops tracepoint_prog_ops = { 1468}; 1469 1470BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx, 1471 struct bpf_perf_event_value *, buf, u32, size) 1472{ 1473 int err = -EINVAL; 1474 1475 if (unlikely(size != sizeof(struct bpf_perf_event_value))) 1476 goto clear; 1477 err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled, 1478 &buf->running); 1479 if (unlikely(err)) 1480 goto clear; 1481 return 0; 1482clear: 1483 memset(buf, 0, size); 1484 return err; 1485} 1486 1487static const struct bpf_func_proto bpf_perf_prog_read_value_proto = { 1488 .func = bpf_perf_prog_read_value, 1489 .gpl_only = true, 1490 .ret_type = RET_INTEGER, 1491 .arg1_type = ARG_PTR_TO_CTX, 1492 .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1493 .arg3_type = ARG_CONST_SIZE, 1494}; 1495 1496BPF_CALL_4(bpf_read_branch_records, struct bpf_perf_event_data_kern *, ctx, 1497 void *, buf, u32, size, u64, flags) 1498{ 1499 static const u32 br_entry_size = sizeof(struct perf_branch_entry); 1500 struct perf_branch_stack *br_stack = ctx->data->br_stack; 1501 u32 to_copy; 1502 1503 if (unlikely(flags & ~BPF_F_GET_BRANCH_RECORDS_SIZE)) 1504 return -EINVAL; 1505 1506 if (unlikely(!(ctx->data->sample_flags & PERF_SAMPLE_BRANCH_STACK))) 1507 return -ENOENT; 1508 1509 if (unlikely(!br_stack)) 1510 return -ENOENT; 1511 1512 if (flags & BPF_F_GET_BRANCH_RECORDS_SIZE) 1513 return br_stack->nr * br_entry_size; 1514 1515 if (!buf || (size % br_entry_size != 0)) 1516 return -EINVAL; 1517 1518 to_copy = min_t(u32, br_stack->nr * br_entry_size, size); 1519 memcpy(buf, br_stack->entries, to_copy); 1520 1521 return to_copy; 1522} 1523 1524static const struct bpf_func_proto bpf_read_branch_records_proto = { 1525 .func = bpf_read_branch_records, 1526 .gpl_only = true, 1527 .ret_type = RET_INTEGER, 1528 .arg1_type = ARG_PTR_TO_CTX, 1529 .arg2_type = ARG_PTR_TO_MEM_OR_NULL | MEM_WRITE, 1530 .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1531 .arg4_type = ARG_ANYTHING, 1532}; 1533 1534static const struct bpf_func_proto * 1535pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1536{ 1537 switch (func_id) { 1538 case BPF_FUNC_perf_event_output: 1539 return &bpf_perf_event_output_proto_tp; 1540 case BPF_FUNC_get_stackid: 1541 return &bpf_get_stackid_proto_pe; 1542 case BPF_FUNC_get_stack: 1543 return &bpf_get_stack_proto_pe; 1544 case BPF_FUNC_perf_prog_read_value: 1545 return &bpf_perf_prog_read_value_proto; 1546 case BPF_FUNC_read_branch_records: 1547 return &bpf_read_branch_records_proto; 1548 case BPF_FUNC_get_attach_cookie: 1549 return &bpf_get_attach_cookie_proto_pe; 1550 default: 1551 return bpf_tracing_func_proto(func_id, prog); 1552 } 1553} 1554 1555/* 1556 * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp 1557 * to avoid potential recursive reuse issue when/if tracepoints are added 1558 * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack. 1559 * 1560 * Since raw tracepoints run despite bpf_prog_active, support concurrent usage 1561 * in normal, irq, and nmi context. 1562 */ 1563struct bpf_raw_tp_regs { 1564 struct pt_regs regs[3]; 1565}; 1566static DEFINE_PER_CPU(struct bpf_raw_tp_regs, bpf_raw_tp_regs); 1567static DEFINE_PER_CPU(int, bpf_raw_tp_nest_level); 1568static struct pt_regs *get_bpf_raw_tp_regs(void) 1569{ 1570 struct bpf_raw_tp_regs *tp_regs = this_cpu_ptr(&bpf_raw_tp_regs); 1571 int nest_level = this_cpu_inc_return(bpf_raw_tp_nest_level); 1572 1573 if (nest_level > ARRAY_SIZE(tp_regs->regs)) { 1574 this_cpu_dec(bpf_raw_tp_nest_level); 1575 return ERR_PTR(-EBUSY); 1576 } 1577 1578 return &tp_regs->regs[nest_level - 1]; 1579} 1580 1581static void put_bpf_raw_tp_regs(void) 1582{ 1583 this_cpu_dec(bpf_raw_tp_nest_level); 1584} 1585 1586BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args, 1587 struct bpf_map *, map, u64, flags, void *, data, u64, size) 1588{ 1589 struct pt_regs *regs = get_bpf_raw_tp_regs(); 1590 int ret; 1591 1592 if (IS_ERR(regs)) 1593 return PTR_ERR(regs); 1594 1595 perf_fetch_caller_regs(regs); 1596 ret = ____bpf_perf_event_output(regs, map, flags, data, size); 1597 1598 put_bpf_raw_tp_regs(); 1599 return ret; 1600} 1601 1602static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = { 1603 .func = bpf_perf_event_output_raw_tp, 1604 .gpl_only = true, 1605 .ret_type = RET_INTEGER, 1606 .arg1_type = ARG_PTR_TO_CTX, 1607 .arg2_type = ARG_CONST_MAP_PTR, 1608 .arg3_type = ARG_ANYTHING, 1609 .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, 1610 .arg5_type = ARG_CONST_SIZE_OR_ZERO, 1611}; 1612 1613extern const struct bpf_func_proto bpf_skb_output_proto; 1614extern const struct bpf_func_proto bpf_xdp_output_proto; 1615extern const struct bpf_func_proto bpf_xdp_get_buff_len_trace_proto; 1616 1617BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args, 1618 struct bpf_map *, map, u64, flags) 1619{ 1620 struct pt_regs *regs = get_bpf_raw_tp_regs(); 1621 int ret; 1622 1623 if (IS_ERR(regs)) 1624 return PTR_ERR(regs); 1625 1626 perf_fetch_caller_regs(regs); 1627 /* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */ 1628 ret = bpf_get_stackid((unsigned long) regs, (unsigned long) map, 1629 flags, 0, 0); 1630 put_bpf_raw_tp_regs(); 1631 return ret; 1632} 1633 1634static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = { 1635 .func = bpf_get_stackid_raw_tp, 1636 .gpl_only = true, 1637 .ret_type = RET_INTEGER, 1638 .arg1_type = ARG_PTR_TO_CTX, 1639 .arg2_type = ARG_CONST_MAP_PTR, 1640 .arg3_type = ARG_ANYTHING, 1641}; 1642 1643BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args, 1644 void *, buf, u32, size, u64, flags) 1645{ 1646 struct pt_regs *regs = get_bpf_raw_tp_regs(); 1647 int ret; 1648 1649 if (IS_ERR(regs)) 1650 return PTR_ERR(regs); 1651 1652 perf_fetch_caller_regs(regs); 1653 ret = bpf_get_stack((unsigned long) regs, (unsigned long) buf, 1654 (unsigned long) size, flags, 0); 1655 put_bpf_raw_tp_regs(); 1656 return ret; 1657} 1658 1659static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = { 1660 .func = bpf_get_stack_raw_tp, 1661 .gpl_only = true, 1662 .ret_type = RET_INTEGER, 1663 .arg1_type = ARG_PTR_TO_CTX, 1664 .arg2_type = ARG_PTR_TO_UNINIT_MEM, 1665 .arg3_type = ARG_CONST_SIZE_OR_ZERO, 1666 .arg4_type = ARG_ANYTHING, 1667}; 1668 1669static const struct bpf_func_proto * 1670raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1671{ 1672 switch (func_id) { 1673 case BPF_FUNC_perf_event_output: 1674 return &bpf_perf_event_output_proto_raw_tp; 1675 case BPF_FUNC_get_stackid: 1676 return &bpf_get_stackid_proto_raw_tp; 1677 case BPF_FUNC_get_stack: 1678 return &bpf_get_stack_proto_raw_tp; 1679 case BPF_FUNC_get_attach_cookie: 1680 return &bpf_get_attach_cookie_proto_tracing; 1681 default: 1682 return bpf_tracing_func_proto(func_id, prog); 1683 } 1684} 1685 1686const struct bpf_func_proto * 1687tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) 1688{ 1689 const struct bpf_func_proto *fn; 1690 1691 switch (func_id) { 1692#ifdef CONFIG_NET 1693 case BPF_FUNC_skb_output: 1694 return &bpf_skb_output_proto; 1695 case BPF_FUNC_xdp_output: 1696 return &bpf_xdp_output_proto; 1697 case BPF_FUNC_skc_to_tcp6_sock: 1698 return &bpf_skc_to_tcp6_sock_proto; 1699 case BPF_FUNC_skc_to_tcp_sock: 1700 return &bpf_skc_to_tcp_sock_proto; 1701 case BPF_FUNC_skc_to_tcp_timewait_sock: 1702 return &bpf_skc_to_tcp_timewait_sock_proto; 1703 case BPF_FUNC_skc_to_tcp_request_sock: 1704 return &bpf_skc_to_tcp_request_sock_proto; 1705 case BPF_FUNC_skc_to_udp6_sock: 1706 return &bpf_skc_to_udp6_sock_proto; 1707 case BPF_FUNC_skc_to_unix_sock: 1708 return &bpf_skc_to_unix_sock_proto; 1709 case BPF_FUNC_skc_to_mptcp_sock: 1710 return &bpf_skc_to_mptcp_sock_proto; 1711 case BPF_FUNC_sk_storage_get: 1712 return &bpf_sk_storage_get_tracing_proto; 1713 case BPF_FUNC_sk_storage_delete: 1714 return &bpf_sk_storage_delete_tracing_proto; 1715 case BPF_FUNC_sock_from_file: 1716 return &bpf_sock_from_file_proto; 1717 case BPF_FUNC_get_socket_cookie: 1718 return &bpf_get_socket_ptr_cookie_proto; 1719 case BPF_FUNC_xdp_get_buff_len: 1720 return &bpf_xdp_get_buff_len_trace_proto; 1721#endif 1722 case BPF_FUNC_seq_printf: 1723 return prog->expected_attach_type == BPF_TRACE_ITER ? 1724 &bpf_seq_printf_proto : 1725 NULL; 1726 case BPF_FUNC_seq_write: 1727 return prog->expected_attach_type == BPF_TRACE_ITER ? 1728 &bpf_seq_write_proto : 1729 NULL; 1730 case BPF_FUNC_seq_printf_btf: 1731 return prog->expected_attach_type == BPF_TRACE_ITER ? 1732 &bpf_seq_printf_btf_proto : 1733 NULL; 1734 case BPF_FUNC_d_path: 1735 return &bpf_d_path_proto; 1736 case BPF_FUNC_get_func_arg: 1737 return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_proto : NULL; 1738 case BPF_FUNC_get_func_ret: 1739 return bpf_prog_has_trampoline(prog) ? &bpf_get_func_ret_proto : NULL; 1740 case BPF_FUNC_get_func_arg_cnt: 1741 return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL; 1742 case BPF_FUNC_get_attach_cookie: 1743 if (prog->type == BPF_PROG_TYPE_TRACING && 1744 prog->expected_attach_type == BPF_TRACE_RAW_TP) 1745 return &bpf_get_attach_cookie_proto_tracing; 1746 return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto_tracing : NULL; 1747 default: 1748 fn = raw_tp_prog_func_proto(func_id, prog); 1749 if (!fn && prog->expected_attach_type == BPF_TRACE_ITER) 1750 fn = bpf_iter_get_func_proto(func_id, prog); 1751 return fn; 1752 } 1753} 1754 1755static bool raw_tp_prog_is_valid_access(int off, int size, 1756 enum bpf_access_type type, 1757 const struct bpf_prog *prog, 1758 struct bpf_insn_access_aux *info) 1759{ 1760 return bpf_tracing_ctx_access(off, size, type); 1761} 1762 1763static bool tracing_prog_is_valid_access(int off, int size, 1764 enum bpf_access_type type, 1765 const struct bpf_prog *prog, 1766 struct bpf_insn_access_aux *info) 1767{ 1768 return bpf_tracing_btf_ctx_access(off, size, type, prog, info); 1769} 1770 1771int __weak bpf_prog_test_run_tracing(struct bpf_prog *prog, 1772 const union bpf_attr *kattr, 1773 union bpf_attr __user *uattr) 1774{ 1775 return -ENOTSUPP; 1776} 1777 1778const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { 1779 .get_func_proto = raw_tp_prog_func_proto, 1780 .is_valid_access = raw_tp_prog_is_valid_access, 1781}; 1782 1783const struct bpf_prog_ops raw_tracepoint_prog_ops = { 1784#ifdef CONFIG_NET 1785 .test_run = bpf_prog_test_run_raw_tp, 1786#endif 1787}; 1788 1789const struct bpf_verifier_ops tracing_verifier_ops = { 1790 .get_func_proto = tracing_prog_func_proto, 1791 .is_valid_access = tracing_prog_is_valid_access, 1792}; 1793 1794const struct bpf_prog_ops tracing_prog_ops = { 1795 .test_run = bpf_prog_test_run_tracing, 1796}; 1797 1798static bool raw_tp_writable_prog_is_valid_access(int off, int size, 1799 enum bpf_access_type type, 1800 const struct bpf_prog *prog, 1801 struct bpf_insn_access_aux *info) 1802{ 1803 if (off == 0) { 1804 if (size != sizeof(u64) || type != BPF_READ) 1805 return false; 1806 info->reg_type = PTR_TO_TP_BUFFER; 1807 } 1808 return raw_tp_prog_is_valid_access(off, size, type, prog, info); 1809} 1810 1811const struct bpf_verifier_ops raw_tracepoint_writable_verifier_ops = { 1812 .get_func_proto = raw_tp_prog_func_proto, 1813 .is_valid_access = raw_tp_writable_prog_is_valid_access, 1814}; 1815 1816const struct bpf_prog_ops raw_tracepoint_writable_prog_ops = { 1817}; 1818 1819static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, 1820 const struct bpf_prog *prog, 1821 struct bpf_insn_access_aux *info) 1822{ 1823 const int size_u64 = sizeof(u64); 1824 1825 if (off < 0 || off >= sizeof(struct bpf_perf_event_data)) 1826 return false; 1827 if (type != BPF_READ) 1828 return false; 1829 if (off % size != 0) { 1830 if (sizeof(unsigned long) != 4) 1831 return false; 1832 if (size != 8) 1833 return false; 1834 if (off % size != 4) 1835 return false; 1836 } 1837 1838 switch (off) { 1839 case bpf_ctx_range(struct bpf_perf_event_data, sample_period): 1840 bpf_ctx_record_field_size(info, size_u64); 1841 if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 1842 return false; 1843 break; 1844 case bpf_ctx_range(struct bpf_perf_event_data, addr): 1845 bpf_ctx_record_field_size(info, size_u64); 1846 if (!bpf_ctx_narrow_access_ok(off, size, size_u64)) 1847 return false; 1848 break; 1849 default: 1850 if (size != sizeof(long)) 1851 return false; 1852 } 1853 1854 return true; 1855} 1856 1857static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, 1858 const struct bpf_insn *si, 1859 struct bpf_insn *insn_buf, 1860 struct bpf_prog *prog, u32 *target_size) 1861{ 1862 struct bpf_insn *insn = insn_buf; 1863 1864 switch (si->off) { 1865 case offsetof(struct bpf_perf_event_data, sample_period): 1866 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 1867 data), si->dst_reg, si->src_reg, 1868 offsetof(struct bpf_perf_event_data_kern, data)); 1869 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 1870 bpf_target_off(struct perf_sample_data, period, 8, 1871 target_size)); 1872 break; 1873 case offsetof(struct bpf_perf_event_data, addr): 1874 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 1875 data), si->dst_reg, si->src_reg, 1876 offsetof(struct bpf_perf_event_data_kern, data)); 1877 *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, 1878 bpf_target_off(struct perf_sample_data, addr, 8, 1879 target_size)); 1880 break; 1881 default: 1882 *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, 1883 regs), si->dst_reg, si->src_reg, 1884 offsetof(struct bpf_perf_event_data_kern, regs)); 1885 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg, 1886 si->off); 1887 break; 1888 } 1889 1890 return insn - insn_buf; 1891} 1892 1893const struct bpf_verifier_ops perf_event_verifier_ops = { 1894 .get_func_proto = pe_prog_func_proto, 1895 .is_valid_access = pe_prog_is_valid_access, 1896 .convert_ctx_access = pe_prog_convert_ctx_access, 1897}; 1898 1899const struct bpf_prog_ops perf_event_prog_ops = { 1900}; 1901 1902static DEFINE_MUTEX(bpf_event_mutex); 1903 1904#define BPF_TRACE_MAX_PROGS 64 1905 1906int perf_event_attach_bpf_prog(struct perf_event *event, 1907 struct bpf_prog *prog, 1908 u64 bpf_cookie) 1909{ 1910 struct bpf_prog_array *old_array; 1911 struct bpf_prog_array *new_array; 1912 int ret = -EEXIST; 1913 1914 /* 1915 * Kprobe override only works if they are on the function entry, 1916 * and only if they are on the opt-in list. 1917 */ 1918 if (prog->kprobe_override && 1919 (!trace_kprobe_on_func_entry(event->tp_event) || 1920 !trace_kprobe_error_injectable(event->tp_event))) 1921 return -EINVAL; 1922 1923 mutex_lock(&bpf_event_mutex); 1924 1925 if (event->prog) 1926 goto unlock; 1927 1928 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1929 if (old_array && 1930 bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) { 1931 ret = -E2BIG; 1932 goto unlock; 1933 } 1934 1935 ret = bpf_prog_array_copy(old_array, NULL, prog, bpf_cookie, &new_array); 1936 if (ret < 0) 1937 goto unlock; 1938 1939 /* set the new array to event->tp_event and set event->prog */ 1940 event->prog = prog; 1941 event->bpf_cookie = bpf_cookie; 1942 rcu_assign_pointer(event->tp_event->prog_array, new_array); 1943 bpf_prog_array_free_sleepable(old_array); 1944 1945unlock: 1946 mutex_unlock(&bpf_event_mutex); 1947 return ret; 1948} 1949 1950void perf_event_detach_bpf_prog(struct perf_event *event) 1951{ 1952 struct bpf_prog_array *old_array; 1953 struct bpf_prog_array *new_array; 1954 struct bpf_prog *prog = NULL; 1955 int ret; 1956 1957 mutex_lock(&bpf_event_mutex); 1958 1959 if (!event->prog) 1960 goto unlock; 1961 1962 old_array = bpf_event_rcu_dereference(event->tp_event->prog_array); 1963 if (!old_array) 1964 goto put; 1965 1966 ret = bpf_prog_array_copy(old_array, event->prog, NULL, 0, &new_array); 1967 if (ret < 0) { 1968 bpf_prog_array_delete_safe(old_array, event->prog); 1969 } else { 1970 rcu_assign_pointer(event->tp_event->prog_array, new_array); 1971 bpf_prog_array_free_sleepable(old_array); 1972 } 1973 1974put: 1975 prog = event->prog; 1976 event->prog = NULL; 1977 1978unlock: 1979 mutex_unlock(&bpf_event_mutex); 1980 1981 if (prog) { 1982 /* 1983 * It could be that the bpf_prog is not sleepable (and will be freed 1984 * via normal RCU), but is called from a point that supports sleepable 1985 * programs and uses tasks-trace-RCU. 1986 */ 1987 synchronize_rcu_tasks_trace(); 1988 1989 bpf_prog_put(prog); 1990 } 1991} 1992 1993int perf_event_query_prog_array(struct perf_event *event, void __user *info) 1994{ 1995 struct perf_event_query_bpf __user *uquery = info; 1996 struct perf_event_query_bpf query = {}; 1997 struct bpf_prog_array *progs; 1998 u32 *ids, prog_cnt, ids_len; 1999 int ret; 2000 2001 if (!perfmon_capable()) 2002 return -EPERM; 2003 if (event->attr.type != PERF_TYPE_TRACEPOINT) 2004 return -EINVAL; 2005 if (copy_from_user(&query, uquery, sizeof(query))) 2006 return -EFAULT; 2007 2008 ids_len = query.ids_len; 2009 if (ids_len > BPF_TRACE_MAX_PROGS) 2010 return -E2BIG; 2011 ids = kcalloc(ids_len, sizeof(u32), GFP_USER | __GFP_NOWARN); 2012 if (!ids) 2013 return -ENOMEM; 2014 /* 2015 * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which 2016 * is required when user only wants to check for uquery->prog_cnt. 2017 * There is no need to check for it since the case is handled 2018 * gracefully in bpf_prog_array_copy_info. 2019 */ 2020 2021 mutex_lock(&bpf_event_mutex); 2022 progs = bpf_event_rcu_dereference(event->tp_event->prog_array); 2023 ret = bpf_prog_array_copy_info(progs, ids, ids_len, &prog_cnt); 2024 mutex_unlock(&bpf_event_mutex); 2025 2026 if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) || 2027 copy_to_user(uquery->ids, ids, ids_len * sizeof(u32))) 2028 ret = -EFAULT; 2029 2030 kfree(ids); 2031 return ret; 2032} 2033 2034extern struct bpf_raw_event_map __start__bpf_raw_tp[]; 2035extern struct bpf_raw_event_map __stop__bpf_raw_tp[]; 2036 2037struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name) 2038{ 2039 struct bpf_raw_event_map *btp = __start__bpf_raw_tp; 2040 2041 for (; btp < __stop__bpf_raw_tp; btp++) { 2042 if (!strcmp(btp->tp->name, name)) 2043 return btp; 2044 } 2045 2046 return bpf_get_raw_tracepoint_module(name); 2047} 2048 2049void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp) 2050{ 2051 struct module *mod; 2052 2053 guard(rcu)(); 2054 mod = __module_address((unsigned long)btp); 2055 module_put(mod); 2056} 2057 2058static __always_inline 2059void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args) 2060{ 2061 struct bpf_prog *prog = link->link.prog; 2062 struct bpf_run_ctx *old_run_ctx; 2063 struct bpf_trace_run_ctx run_ctx; 2064 2065 cant_sleep(); 2066 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) { 2067 bpf_prog_inc_misses_counter(prog); 2068 goto out; 2069 } 2070 2071 run_ctx.bpf_cookie = link->cookie; 2072 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); 2073 2074 rcu_read_lock(); 2075 (void) bpf_prog_run(prog, args); 2076 rcu_read_unlock(); 2077 2078 bpf_reset_run_ctx(old_run_ctx); 2079out: 2080 this_cpu_dec(*(prog->active)); 2081} 2082 2083#define UNPACK(...) __VA_ARGS__ 2084#define REPEAT_1(FN, DL, X, ...) FN(X) 2085#define REPEAT_2(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__) 2086#define REPEAT_3(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__) 2087#define REPEAT_4(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__) 2088#define REPEAT_5(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__) 2089#define REPEAT_6(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__) 2090#define REPEAT_7(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__) 2091#define REPEAT_8(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__) 2092#define REPEAT_9(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__) 2093#define REPEAT_10(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__) 2094#define REPEAT_11(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__) 2095#define REPEAT_12(FN, DL, X, ...) FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__) 2096#define REPEAT(X, FN, DL, ...) REPEAT_##X(FN, DL, __VA_ARGS__) 2097 2098#define SARG(X) u64 arg##X 2099#define COPY(X) args[X] = arg##X 2100 2101#define __DL_COM (,) 2102#define __DL_SEM (;) 2103 2104#define __SEQ_0_11 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 2105 2106#define BPF_TRACE_DEFN_x(x) \ 2107 void bpf_trace_run##x(struct bpf_raw_tp_link *link, \ 2108 REPEAT(x, SARG, __DL_COM, __SEQ_0_11)) \ 2109 { \ 2110 u64 args[x]; \ 2111 REPEAT(x, COPY, __DL_SEM, __SEQ_0_11); \ 2112 __bpf_trace_run(link, args); \ 2113 } \ 2114 EXPORT_SYMBOL_GPL(bpf_trace_run##x) 2115BPF_TRACE_DEFN_x(1); 2116BPF_TRACE_DEFN_x(2); 2117BPF_TRACE_DEFN_x(3); 2118BPF_TRACE_DEFN_x(4); 2119BPF_TRACE_DEFN_x(5); 2120BPF_TRACE_DEFN_x(6); 2121BPF_TRACE_DEFN_x(7); 2122BPF_TRACE_DEFN_x(8); 2123BPF_TRACE_DEFN_x(9); 2124BPF_TRACE_DEFN_x(10); 2125BPF_TRACE_DEFN_x(11); 2126BPF_TRACE_DEFN_x(12); 2127 2128int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link) 2129{ 2130 struct tracepoint *tp = btp->tp; 2131 struct bpf_prog *prog = link->link.prog; 2132 2133 /* 2134 * check that program doesn't access arguments beyond what's 2135 * available in this tracepoint 2136 */ 2137 if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64)) 2138 return -EINVAL; 2139 2140 if (prog->aux->max_tp_access > btp->writable_size) 2141 return -EINVAL; 2142 2143 return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func, link); 2144} 2145 2146int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link) 2147{ 2148 return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link); 2149} 2150 2151int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, 2152 u32 *fd_type, const char **buf, 2153 u64 *probe_offset, u64 *probe_addr, 2154 unsigned long *missed) 2155{ 2156 bool is_tracepoint, is_syscall_tp; 2157 struct bpf_prog *prog; 2158 int flags, err = 0; 2159 2160 prog = event->prog; 2161 if (!prog) 2162 return -ENOENT; 2163 2164 /* not supporting BPF_PROG_TYPE_PERF_EVENT yet */ 2165 if (prog->type == BPF_PROG_TYPE_PERF_EVENT) 2166 return -EOPNOTSUPP; 2167 2168 *prog_id = prog->aux->id; 2169 flags = event->tp_event->flags; 2170 is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT; 2171 is_syscall_tp = is_syscall_trace_event(event->tp_event); 2172 2173 if (is_tracepoint || is_syscall_tp) { 2174 *buf = is_tracepoint ? event->tp_event->tp->name 2175 : event->tp_event->name; 2176 /* We allow NULL pointer for tracepoint */ 2177 if (fd_type) 2178 *fd_type = BPF_FD_TYPE_TRACEPOINT; 2179 if (probe_offset) 2180 *probe_offset = 0x0; 2181 if (probe_addr) 2182 *probe_addr = 0x0; 2183 } else { 2184 /* kprobe/uprobe */ 2185 err = -EOPNOTSUPP; 2186#ifdef CONFIG_KPROBE_EVENTS 2187 if (flags & TRACE_EVENT_FL_KPROBE) 2188 err = bpf_get_kprobe_info(event, fd_type, buf, 2189 probe_offset, probe_addr, missed, 2190 event->attr.type == PERF_TYPE_TRACEPOINT); 2191#endif 2192#ifdef CONFIG_UPROBE_EVENTS 2193 if (flags & TRACE_EVENT_FL_UPROBE) 2194 err = bpf_get_uprobe_info(event, fd_type, buf, 2195 probe_offset, probe_addr, 2196 event->attr.type == PERF_TYPE_TRACEPOINT); 2197#endif 2198 } 2199 2200 return err; 2201} 2202 2203static int __init send_signal_irq_work_init(void) 2204{ 2205 int cpu; 2206 struct send_signal_irq_work *work; 2207 2208 for_each_possible_cpu(cpu) { 2209 work = per_cpu_ptr(&send_signal_work, cpu); 2210 init_irq_work(&work->irq_work, do_bpf_send_signal); 2211 } 2212 return 0; 2213} 2214 2215subsys_initcall(send_signal_irq_work_init); 2216 2217#ifdef CONFIG_MODULES 2218static int bpf_event_notify(struct notifier_block *nb, unsigned long op, 2219 void *module) 2220{ 2221 struct bpf_trace_module *btm, *tmp; 2222 struct module *mod = module; 2223 int ret = 0; 2224 2225 if (mod->num_bpf_raw_events == 0 || 2226 (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING)) 2227 goto out; 2228 2229 mutex_lock(&bpf_module_mutex); 2230 2231 switch (op) { 2232 case MODULE_STATE_COMING: 2233 btm = kzalloc(sizeof(*btm), GFP_KERNEL); 2234 if (btm) { 2235 btm->module = module; 2236 list_add(&btm->list, &bpf_trace_modules); 2237 } else { 2238 ret = -ENOMEM; 2239 } 2240 break; 2241 case MODULE_STATE_GOING: 2242 list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) { 2243 if (btm->module == module) { 2244 list_del(&btm->list); 2245 kfree(btm); 2246 break; 2247 } 2248 } 2249 break; 2250 } 2251 2252 mutex_unlock(&bpf_module_mutex); 2253 2254out: 2255 return notifier_from_errno(ret); 2256} 2257 2258static struct notifier_block bpf_module_nb = { 2259 .notifier_call = bpf_event_notify, 2260}; 2261 2262static int __init bpf_event_init(void) 2263{ 2264 register_module_notifier(&bpf_module_nb); 2265 return 0; 2266} 2267 2268fs_initcall(bpf_event_init); 2269#endif /* CONFIG_MODULES */ 2270 2271struct bpf_session_run_ctx { 2272 struct bpf_run_ctx run_ctx; 2273 bool is_return; 2274 void *data; 2275}; 2276 2277#ifdef CONFIG_FPROBE 2278struct bpf_kprobe_multi_link { 2279 struct bpf_link link; 2280 struct fprobe fp; 2281 unsigned long *addrs; 2282 u64 *cookies; 2283 u32 cnt; 2284 u32 mods_cnt; 2285 struct module **mods; 2286}; 2287 2288struct bpf_kprobe_multi_run_ctx { 2289 struct bpf_session_run_ctx session_ctx; 2290 struct bpf_kprobe_multi_link *link; 2291 unsigned long entry_ip; 2292}; 2293 2294struct user_syms { 2295 const char **syms; 2296 char *buf; 2297}; 2298 2299#ifndef CONFIG_HAVE_FTRACE_REGS_HAVING_PT_REGS 2300static DEFINE_PER_CPU(struct pt_regs, bpf_kprobe_multi_pt_regs); 2301#define bpf_kprobe_multi_pt_regs_ptr() this_cpu_ptr(&bpf_kprobe_multi_pt_regs) 2302#else 2303#define bpf_kprobe_multi_pt_regs_ptr() (NULL) 2304#endif 2305 2306static unsigned long ftrace_get_entry_ip(unsigned long fentry_ip) 2307{ 2308 unsigned long ip = ftrace_get_symaddr(fentry_ip); 2309 2310 return ip ? : fentry_ip; 2311} 2312 2313static int copy_user_syms(struct user_syms *us, unsigned long __user *usyms, u32 cnt) 2314{ 2315 unsigned long __user usymbol; 2316 const char **syms = NULL; 2317 char *buf = NULL, *p; 2318 int err = -ENOMEM; 2319 unsigned int i; 2320 2321 syms = kvmalloc_array(cnt, sizeof(*syms), GFP_KERNEL); 2322 if (!syms) 2323 goto error; 2324 2325 buf = kvmalloc_array(cnt, KSYM_NAME_LEN, GFP_KERNEL); 2326 if (!buf) 2327 goto error; 2328 2329 for (p = buf, i = 0; i < cnt; i++) { 2330 if (__get_user(usymbol, usyms + i)) { 2331 err = -EFAULT; 2332 goto error; 2333 } 2334 err = strncpy_from_user(p, (const char __user *) usymbol, KSYM_NAME_LEN); 2335 if (err == KSYM_NAME_LEN) 2336 err = -E2BIG; 2337 if (err < 0) 2338 goto error; 2339 syms[i] = p; 2340 p += err + 1; 2341 } 2342 2343 us->syms = syms; 2344 us->buf = buf; 2345 return 0; 2346 2347error: 2348 if (err) { 2349 kvfree(syms); 2350 kvfree(buf); 2351 } 2352 return err; 2353} 2354 2355static void kprobe_multi_put_modules(struct module **mods, u32 cnt) 2356{ 2357 u32 i; 2358 2359 for (i = 0; i < cnt; i++) 2360 module_put(mods[i]); 2361} 2362 2363static void free_user_syms(struct user_syms *us) 2364{ 2365 kvfree(us->syms); 2366 kvfree(us->buf); 2367} 2368 2369static void bpf_kprobe_multi_link_release(struct bpf_link *link) 2370{ 2371 struct bpf_kprobe_multi_link *kmulti_link; 2372 2373 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); 2374 unregister_fprobe(&kmulti_link->fp); 2375 kprobe_multi_put_modules(kmulti_link->mods, kmulti_link->mods_cnt); 2376} 2377 2378static void bpf_kprobe_multi_link_dealloc(struct bpf_link *link) 2379{ 2380 struct bpf_kprobe_multi_link *kmulti_link; 2381 2382 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); 2383 kvfree(kmulti_link->addrs); 2384 kvfree(kmulti_link->cookies); 2385 kfree(kmulti_link->mods); 2386 kfree(kmulti_link); 2387} 2388 2389static int bpf_kprobe_multi_link_fill_link_info(const struct bpf_link *link, 2390 struct bpf_link_info *info) 2391{ 2392 u64 __user *ucookies = u64_to_user_ptr(info->kprobe_multi.cookies); 2393 u64 __user *uaddrs = u64_to_user_ptr(info->kprobe_multi.addrs); 2394 struct bpf_kprobe_multi_link *kmulti_link; 2395 u32 ucount = info->kprobe_multi.count; 2396 int err = 0, i; 2397 2398 if (!uaddrs ^ !ucount) 2399 return -EINVAL; 2400 if (ucookies && !ucount) 2401 return -EINVAL; 2402 2403 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); 2404 info->kprobe_multi.count = kmulti_link->cnt; 2405 info->kprobe_multi.flags = kmulti_link->link.flags; 2406 info->kprobe_multi.missed = kmulti_link->fp.nmissed; 2407 2408 if (!uaddrs) 2409 return 0; 2410 if (ucount < kmulti_link->cnt) 2411 err = -ENOSPC; 2412 else 2413 ucount = kmulti_link->cnt; 2414 2415 if (ucookies) { 2416 if (kmulti_link->cookies) { 2417 if (copy_to_user(ucookies, kmulti_link->cookies, ucount * sizeof(u64))) 2418 return -EFAULT; 2419 } else { 2420 for (i = 0; i < ucount; i++) { 2421 if (put_user(0, ucookies + i)) 2422 return -EFAULT; 2423 } 2424 } 2425 } 2426 2427 if (kallsyms_show_value(current_cred())) { 2428 if (copy_to_user(uaddrs, kmulti_link->addrs, ucount * sizeof(u64))) 2429 return -EFAULT; 2430 } else { 2431 for (i = 0; i < ucount; i++) { 2432 if (put_user(0, uaddrs + i)) 2433 return -EFAULT; 2434 } 2435 } 2436 return err; 2437} 2438 2439#ifdef CONFIG_PROC_FS 2440static void bpf_kprobe_multi_show_fdinfo(const struct bpf_link *link, 2441 struct seq_file *seq) 2442{ 2443 struct bpf_kprobe_multi_link *kmulti_link; 2444 bool has_cookies; 2445 2446 kmulti_link = container_of(link, struct bpf_kprobe_multi_link, link); 2447 has_cookies = !!kmulti_link->cookies; 2448 2449 seq_printf(seq, 2450 "kprobe_cnt:\t%u\n" 2451 "missed:\t%lu\n", 2452 kmulti_link->cnt, 2453 kmulti_link->fp.nmissed); 2454 2455 seq_printf(seq, "%s\t %s\n", "cookie", "func"); 2456 for (int i = 0; i < kmulti_link->cnt; i++) { 2457 seq_printf(seq, 2458 "%llu\t %pS\n", 2459 has_cookies ? kmulti_link->cookies[i] : 0, 2460 (void *)kmulti_link->addrs[i]); 2461 } 2462} 2463#endif 2464 2465static const struct bpf_link_ops bpf_kprobe_multi_link_lops = { 2466 .release = bpf_kprobe_multi_link_release, 2467 .dealloc_deferred = bpf_kprobe_multi_link_dealloc, 2468 .fill_link_info = bpf_kprobe_multi_link_fill_link_info, 2469#ifdef CONFIG_PROC_FS 2470 .show_fdinfo = bpf_kprobe_multi_show_fdinfo, 2471#endif 2472}; 2473 2474static void bpf_kprobe_multi_cookie_swap(void *a, void *b, int size, const void *priv) 2475{ 2476 const struct bpf_kprobe_multi_link *link = priv; 2477 unsigned long *addr_a = a, *addr_b = b; 2478 u64 *cookie_a, *cookie_b; 2479 2480 cookie_a = link->cookies + (addr_a - link->addrs); 2481 cookie_b = link->cookies + (addr_b - link->addrs); 2482 2483 /* swap addr_a/addr_b and cookie_a/cookie_b values */ 2484 swap(*addr_a, *addr_b); 2485 swap(*cookie_a, *cookie_b); 2486} 2487 2488static int bpf_kprobe_multi_addrs_cmp(const void *a, const void *b) 2489{ 2490 const unsigned long *addr_a = a, *addr_b = b; 2491 2492 if (*addr_a == *addr_b) 2493 return 0; 2494 return *addr_a < *addr_b ? -1 : 1; 2495} 2496 2497static int bpf_kprobe_multi_cookie_cmp(const void *a, const void *b, const void *priv) 2498{ 2499 return bpf_kprobe_multi_addrs_cmp(a, b); 2500} 2501 2502static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx) 2503{ 2504 struct bpf_kprobe_multi_run_ctx *run_ctx; 2505 struct bpf_kprobe_multi_link *link; 2506 u64 *cookie, entry_ip; 2507 unsigned long *addr; 2508 2509 if (WARN_ON_ONCE(!ctx)) 2510 return 0; 2511 run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, 2512 session_ctx.run_ctx); 2513 link = run_ctx->link; 2514 if (!link->cookies) 2515 return 0; 2516 entry_ip = run_ctx->entry_ip; 2517 addr = bsearch(&entry_ip, link->addrs, link->cnt, sizeof(entry_ip), 2518 bpf_kprobe_multi_addrs_cmp); 2519 if (!addr) 2520 return 0; 2521 cookie = link->cookies + (addr - link->addrs); 2522 return *cookie; 2523} 2524 2525static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx) 2526{ 2527 struct bpf_kprobe_multi_run_ctx *run_ctx; 2528 2529 run_ctx = container_of(current->bpf_ctx, struct bpf_kprobe_multi_run_ctx, 2530 session_ctx.run_ctx); 2531 return run_ctx->entry_ip; 2532} 2533 2534static __always_inline int 2535kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link, 2536 unsigned long entry_ip, struct ftrace_regs *fregs, 2537 bool is_return, void *data) 2538{ 2539 struct bpf_kprobe_multi_run_ctx run_ctx = { 2540 .session_ctx = { 2541 .is_return = is_return, 2542 .data = data, 2543 }, 2544 .link = link, 2545 .entry_ip = entry_ip, 2546 }; 2547 struct bpf_run_ctx *old_run_ctx; 2548 struct pt_regs *regs; 2549 int err; 2550 2551 /* 2552 * graph tracer framework ensures we won't migrate, so there is no need 2553 * to use migrate_disable for bpf_prog_run again. The check here just for 2554 * __this_cpu_inc_return. 2555 */ 2556 cant_sleep(); 2557 2558 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { 2559 bpf_prog_inc_misses_counter(link->link.prog); 2560 err = 1; 2561 goto out; 2562 } 2563 2564 rcu_read_lock(); 2565 regs = ftrace_partial_regs(fregs, bpf_kprobe_multi_pt_regs_ptr()); 2566 old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); 2567 err = bpf_prog_run(link->link.prog, regs); 2568 bpf_reset_run_ctx(old_run_ctx); 2569 ftrace_partial_regs_update(fregs, bpf_kprobe_multi_pt_regs_ptr()); 2570 rcu_read_unlock(); 2571 2572 out: 2573 __this_cpu_dec(bpf_prog_active); 2574 return err; 2575} 2576 2577static int 2578kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip, 2579 unsigned long ret_ip, struct ftrace_regs *fregs, 2580 void *data) 2581{ 2582 struct bpf_kprobe_multi_link *link; 2583 int err; 2584 2585 link = container_of(fp, struct bpf_kprobe_multi_link, fp); 2586 err = kprobe_multi_link_prog_run(link, ftrace_get_entry_ip(fentry_ip), 2587 fregs, false, data); 2588 return is_kprobe_session(link->link.prog) ? err : 0; 2589} 2590 2591static void 2592kprobe_multi_link_exit_handler(struct fprobe *fp, unsigned long fentry_ip, 2593 unsigned long ret_ip, struct ftrace_regs *fregs, 2594 void *data) 2595{ 2596 struct bpf_kprobe_multi_link *link; 2597 2598 link = container_of(fp, struct bpf_kprobe_multi_link, fp); 2599 kprobe_multi_link_prog_run(link, ftrace_get_entry_ip(fentry_ip), 2600 fregs, true, data); 2601} 2602 2603static int symbols_cmp_r(const void *a, const void *b, const void *priv) 2604{ 2605 const char **str_a = (const char **) a; 2606 const char **str_b = (const char **) b; 2607 2608 return strcmp(*str_a, *str_b); 2609} 2610 2611struct multi_symbols_sort { 2612 const char **funcs; 2613 u64 *cookies; 2614}; 2615 2616static void symbols_swap_r(void *a, void *b, int size, const void *priv) 2617{ 2618 const struct multi_symbols_sort *data = priv; 2619 const char **name_a = a, **name_b = b; 2620 2621 swap(*name_a, *name_b); 2622 2623 /* If defined, swap also related cookies. */ 2624 if (data->cookies) { 2625 u64 *cookie_a, *cookie_b; 2626 2627 cookie_a = data->cookies + (name_a - data->funcs); 2628 cookie_b = data->cookies + (name_b - data->funcs); 2629 swap(*cookie_a, *cookie_b); 2630 } 2631} 2632 2633struct modules_array { 2634 struct module **mods; 2635 int mods_cnt; 2636 int mods_cap; 2637}; 2638 2639static int add_module(struct modules_array *arr, struct module *mod) 2640{ 2641 struct module **mods; 2642 2643 if (arr->mods_cnt == arr->mods_cap) { 2644 arr->mods_cap = max(16, arr->mods_cap * 3 / 2); 2645 mods = krealloc_array(arr->mods, arr->mods_cap, sizeof(*mods), GFP_KERNEL); 2646 if (!mods) 2647 return -ENOMEM; 2648 arr->mods = mods; 2649 } 2650 2651 arr->mods[arr->mods_cnt] = mod; 2652 arr->mods_cnt++; 2653 return 0; 2654} 2655 2656static bool has_module(struct modules_array *arr, struct module *mod) 2657{ 2658 int i; 2659 2660 for (i = arr->mods_cnt - 1; i >= 0; i--) { 2661 if (arr->mods[i] == mod) 2662 return true; 2663 } 2664 return false; 2665} 2666 2667static int get_modules_for_addrs(struct module ***mods, unsigned long *addrs, u32 addrs_cnt) 2668{ 2669 struct modules_array arr = {}; 2670 u32 i, err = 0; 2671 2672 for (i = 0; i < addrs_cnt; i++) { 2673 bool skip_add = false; 2674 struct module *mod; 2675 2676 scoped_guard(rcu) { 2677 mod = __module_address(addrs[i]); 2678 /* Either no module or it's already stored */ 2679 if (!mod || has_module(&arr, mod)) { 2680 skip_add = true; 2681 break; /* scoped_guard */ 2682 } 2683 if (!try_module_get(mod)) 2684 err = -EINVAL; 2685 } 2686 if (skip_add) 2687 continue; 2688 if (err) 2689 break; 2690 err = add_module(&arr, mod); 2691 if (err) { 2692 module_put(mod); 2693 break; 2694 } 2695 } 2696 2697 /* We return either err < 0 in case of error, ... */ 2698 if (err) { 2699 kprobe_multi_put_modules(arr.mods, arr.mods_cnt); 2700 kfree(arr.mods); 2701 return err; 2702 } 2703 2704 /* or number of modules found if everything is ok. */ 2705 *mods = arr.mods; 2706 return arr.mods_cnt; 2707} 2708 2709static int addrs_check_error_injection_list(unsigned long *addrs, u32 cnt) 2710{ 2711 u32 i; 2712 2713 for (i = 0; i < cnt; i++) { 2714 if (!within_error_injection_list(addrs[i])) 2715 return -EINVAL; 2716 } 2717 return 0; 2718} 2719 2720int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 2721{ 2722 struct bpf_kprobe_multi_link *link = NULL; 2723 struct bpf_link_primer link_primer; 2724 void __user *ucookies; 2725 unsigned long *addrs; 2726 u32 flags, cnt, size; 2727 void __user *uaddrs; 2728 u64 *cookies = NULL; 2729 void __user *usyms; 2730 int err; 2731 2732 /* no support for 32bit archs yet */ 2733 if (sizeof(u64) != sizeof(void *)) 2734 return -EOPNOTSUPP; 2735 2736 if (attr->link_create.flags) 2737 return -EINVAL; 2738 2739 if (!is_kprobe_multi(prog)) 2740 return -EINVAL; 2741 2742 /* kprobe_multi is not allowed to be sleepable. */ 2743 if (prog->sleepable) 2744 return -EINVAL; 2745 2746 /* Writing to context is not allowed for kprobes. */ 2747 if (prog->aux->kprobe_write_ctx) 2748 return -EINVAL; 2749 2750 flags = attr->link_create.kprobe_multi.flags; 2751 if (flags & ~BPF_F_KPROBE_MULTI_RETURN) 2752 return -EINVAL; 2753 2754 uaddrs = u64_to_user_ptr(attr->link_create.kprobe_multi.addrs); 2755 usyms = u64_to_user_ptr(attr->link_create.kprobe_multi.syms); 2756 if (!!uaddrs == !!usyms) 2757 return -EINVAL; 2758 2759 cnt = attr->link_create.kprobe_multi.cnt; 2760 if (!cnt) 2761 return -EINVAL; 2762 if (cnt > MAX_KPROBE_MULTI_CNT) 2763 return -E2BIG; 2764 2765 size = cnt * sizeof(*addrs); 2766 addrs = kvmalloc_array(cnt, sizeof(*addrs), GFP_KERNEL); 2767 if (!addrs) 2768 return -ENOMEM; 2769 2770 ucookies = u64_to_user_ptr(attr->link_create.kprobe_multi.cookies); 2771 if (ucookies) { 2772 cookies = kvmalloc_array(cnt, sizeof(*addrs), GFP_KERNEL); 2773 if (!cookies) { 2774 err = -ENOMEM; 2775 goto error; 2776 } 2777 if (copy_from_user(cookies, ucookies, size)) { 2778 err = -EFAULT; 2779 goto error; 2780 } 2781 } 2782 2783 if (uaddrs) { 2784 if (copy_from_user(addrs, uaddrs, size)) { 2785 err = -EFAULT; 2786 goto error; 2787 } 2788 } else { 2789 struct multi_symbols_sort data = { 2790 .cookies = cookies, 2791 }; 2792 struct user_syms us; 2793 2794 err = copy_user_syms(&us, usyms, cnt); 2795 if (err) 2796 goto error; 2797 2798 if (cookies) 2799 data.funcs = us.syms; 2800 2801 sort_r(us.syms, cnt, sizeof(*us.syms), symbols_cmp_r, 2802 symbols_swap_r, &data); 2803 2804 err = ftrace_lookup_symbols(us.syms, cnt, addrs); 2805 free_user_syms(&us); 2806 if (err) 2807 goto error; 2808 } 2809 2810 if (prog->kprobe_override && addrs_check_error_injection_list(addrs, cnt)) { 2811 err = -EINVAL; 2812 goto error; 2813 } 2814 2815 link = kzalloc(sizeof(*link), GFP_KERNEL); 2816 if (!link) { 2817 err = -ENOMEM; 2818 goto error; 2819 } 2820 2821 bpf_link_init(&link->link, BPF_LINK_TYPE_KPROBE_MULTI, 2822 &bpf_kprobe_multi_link_lops, prog, attr->link_create.attach_type); 2823 2824 err = bpf_link_prime(&link->link, &link_primer); 2825 if (err) 2826 goto error; 2827 2828 if (!(flags & BPF_F_KPROBE_MULTI_RETURN)) 2829 link->fp.entry_handler = kprobe_multi_link_handler; 2830 if ((flags & BPF_F_KPROBE_MULTI_RETURN) || is_kprobe_session(prog)) 2831 link->fp.exit_handler = kprobe_multi_link_exit_handler; 2832 if (is_kprobe_session(prog)) 2833 link->fp.entry_data_size = sizeof(u64); 2834 2835 link->addrs = addrs; 2836 link->cookies = cookies; 2837 link->cnt = cnt; 2838 link->link.flags = flags; 2839 2840 if (cookies) { 2841 /* 2842 * Sorting addresses will trigger sorting cookies as well 2843 * (check bpf_kprobe_multi_cookie_swap). This way we can 2844 * find cookie based on the address in bpf_get_attach_cookie 2845 * helper. 2846 */ 2847 sort_r(addrs, cnt, sizeof(*addrs), 2848 bpf_kprobe_multi_cookie_cmp, 2849 bpf_kprobe_multi_cookie_swap, 2850 link); 2851 } 2852 2853 err = get_modules_for_addrs(&link->mods, addrs, cnt); 2854 if (err < 0) { 2855 bpf_link_cleanup(&link_primer); 2856 return err; 2857 } 2858 link->mods_cnt = err; 2859 2860 err = register_fprobe_ips(&link->fp, addrs, cnt); 2861 if (err) { 2862 kprobe_multi_put_modules(link->mods, link->mods_cnt); 2863 bpf_link_cleanup(&link_primer); 2864 return err; 2865 } 2866 2867 return bpf_link_settle(&link_primer); 2868 2869error: 2870 kfree(link); 2871 kvfree(addrs); 2872 kvfree(cookies); 2873 return err; 2874} 2875#else /* !CONFIG_FPROBE */ 2876int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 2877{ 2878 return -EOPNOTSUPP; 2879} 2880static u64 bpf_kprobe_multi_cookie(struct bpf_run_ctx *ctx) 2881{ 2882 return 0; 2883} 2884static u64 bpf_kprobe_multi_entry_ip(struct bpf_run_ctx *ctx) 2885{ 2886 return 0; 2887} 2888#endif 2889 2890#ifdef CONFIG_UPROBES 2891struct bpf_uprobe_multi_link; 2892 2893struct bpf_uprobe { 2894 struct bpf_uprobe_multi_link *link; 2895 loff_t offset; 2896 unsigned long ref_ctr_offset; 2897 u64 cookie; 2898 struct uprobe *uprobe; 2899 struct uprobe_consumer consumer; 2900 bool session; 2901}; 2902 2903struct bpf_uprobe_multi_link { 2904 struct path path; 2905 struct bpf_link link; 2906 u32 cnt; 2907 struct bpf_uprobe *uprobes; 2908 struct task_struct *task; 2909}; 2910 2911struct bpf_uprobe_multi_run_ctx { 2912 struct bpf_session_run_ctx session_ctx; 2913 unsigned long entry_ip; 2914 struct bpf_uprobe *uprobe; 2915}; 2916 2917static void bpf_uprobe_unregister(struct bpf_uprobe *uprobes, u32 cnt) 2918{ 2919 u32 i; 2920 2921 for (i = 0; i < cnt; i++) 2922 uprobe_unregister_nosync(uprobes[i].uprobe, &uprobes[i].consumer); 2923 2924 if (cnt) 2925 uprobe_unregister_sync(); 2926} 2927 2928static void bpf_uprobe_multi_link_release(struct bpf_link *link) 2929{ 2930 struct bpf_uprobe_multi_link *umulti_link; 2931 2932 umulti_link = container_of(link, struct bpf_uprobe_multi_link, link); 2933 bpf_uprobe_unregister(umulti_link->uprobes, umulti_link->cnt); 2934 if (umulti_link->task) 2935 put_task_struct(umulti_link->task); 2936 path_put(&umulti_link->path); 2937} 2938 2939static void bpf_uprobe_multi_link_dealloc(struct bpf_link *link) 2940{ 2941 struct bpf_uprobe_multi_link *umulti_link; 2942 2943 umulti_link = container_of(link, struct bpf_uprobe_multi_link, link); 2944 kvfree(umulti_link->uprobes); 2945 kfree(umulti_link); 2946} 2947 2948static int bpf_uprobe_multi_link_fill_link_info(const struct bpf_link *link, 2949 struct bpf_link_info *info) 2950{ 2951 u64 __user *uref_ctr_offsets = u64_to_user_ptr(info->uprobe_multi.ref_ctr_offsets); 2952 u64 __user *ucookies = u64_to_user_ptr(info->uprobe_multi.cookies); 2953 u64 __user *uoffsets = u64_to_user_ptr(info->uprobe_multi.offsets); 2954 u64 __user *upath = u64_to_user_ptr(info->uprobe_multi.path); 2955 u32 upath_size = info->uprobe_multi.path_size; 2956 struct bpf_uprobe_multi_link *umulti_link; 2957 u32 ucount = info->uprobe_multi.count; 2958 int err = 0, i; 2959 char *p, *buf; 2960 long left = 0; 2961 2962 if (!upath ^ !upath_size) 2963 return -EINVAL; 2964 2965 if ((uoffsets || uref_ctr_offsets || ucookies) && !ucount) 2966 return -EINVAL; 2967 2968 umulti_link = container_of(link, struct bpf_uprobe_multi_link, link); 2969 info->uprobe_multi.count = umulti_link->cnt; 2970 info->uprobe_multi.flags = umulti_link->link.flags; 2971 info->uprobe_multi.pid = umulti_link->task ? 2972 task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0; 2973 2974 upath_size = upath_size ? min_t(u32, upath_size, PATH_MAX) : PATH_MAX; 2975 buf = kmalloc(upath_size, GFP_KERNEL); 2976 if (!buf) 2977 return -ENOMEM; 2978 p = d_path(&umulti_link->path, buf, upath_size); 2979 if (IS_ERR(p)) { 2980 kfree(buf); 2981 return PTR_ERR(p); 2982 } 2983 upath_size = buf + upath_size - p; 2984 2985 if (upath) 2986 left = copy_to_user(upath, p, upath_size); 2987 kfree(buf); 2988 if (left) 2989 return -EFAULT; 2990 info->uprobe_multi.path_size = upath_size; 2991 2992 if (!uoffsets && !ucookies && !uref_ctr_offsets) 2993 return 0; 2994 2995 if (ucount < umulti_link->cnt) 2996 err = -ENOSPC; 2997 else 2998 ucount = umulti_link->cnt; 2999 3000 for (i = 0; i < ucount; i++) { 3001 if (uoffsets && 3002 put_user(umulti_link->uprobes[i].offset, uoffsets + i)) 3003 return -EFAULT; 3004 if (uref_ctr_offsets && 3005 put_user(umulti_link->uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) 3006 return -EFAULT; 3007 if (ucookies && 3008 put_user(umulti_link->uprobes[i].cookie, ucookies + i)) 3009 return -EFAULT; 3010 } 3011 3012 return err; 3013} 3014 3015#ifdef CONFIG_PROC_FS 3016static void bpf_uprobe_multi_show_fdinfo(const struct bpf_link *link, 3017 struct seq_file *seq) 3018{ 3019 struct bpf_uprobe_multi_link *umulti_link; 3020 char *p, *buf; 3021 pid_t pid; 3022 3023 umulti_link = container_of(link, struct bpf_uprobe_multi_link, link); 3024 3025 buf = kmalloc(PATH_MAX, GFP_KERNEL); 3026 if (!buf) 3027 return; 3028 3029 p = d_path(&umulti_link->path, buf, PATH_MAX); 3030 if (IS_ERR(p)) { 3031 kfree(buf); 3032 return; 3033 } 3034 3035 pid = umulti_link->task ? 3036 task_pid_nr_ns(umulti_link->task, task_active_pid_ns(current)) : 0; 3037 seq_printf(seq, 3038 "uprobe_cnt:\t%u\n" 3039 "pid:\t%u\n" 3040 "path:\t%s\n", 3041 umulti_link->cnt, pid, p); 3042 3043 seq_printf(seq, "%s\t %s\t %s\n", "cookie", "offset", "ref_ctr_offset"); 3044 for (int i = 0; i < umulti_link->cnt; i++) { 3045 seq_printf(seq, 3046 "%llu\t %#llx\t %#lx\n", 3047 umulti_link->uprobes[i].cookie, 3048 umulti_link->uprobes[i].offset, 3049 umulti_link->uprobes[i].ref_ctr_offset); 3050 } 3051 3052 kfree(buf); 3053} 3054#endif 3055 3056static const struct bpf_link_ops bpf_uprobe_multi_link_lops = { 3057 .release = bpf_uprobe_multi_link_release, 3058 .dealloc_deferred = bpf_uprobe_multi_link_dealloc, 3059 .fill_link_info = bpf_uprobe_multi_link_fill_link_info, 3060#ifdef CONFIG_PROC_FS 3061 .show_fdinfo = bpf_uprobe_multi_show_fdinfo, 3062#endif 3063}; 3064 3065static int uprobe_prog_run(struct bpf_uprobe *uprobe, 3066 unsigned long entry_ip, 3067 struct pt_regs *regs, 3068 bool is_return, void *data) 3069{ 3070 struct bpf_uprobe_multi_link *link = uprobe->link; 3071 struct bpf_uprobe_multi_run_ctx run_ctx = { 3072 .session_ctx = { 3073 .is_return = is_return, 3074 .data = data, 3075 }, 3076 .entry_ip = entry_ip, 3077 .uprobe = uprobe, 3078 }; 3079 struct bpf_prog *prog = link->link.prog; 3080 bool sleepable = prog->sleepable; 3081 struct bpf_run_ctx *old_run_ctx; 3082 int err; 3083 3084 if (link->task && !same_thread_group(current, link->task)) 3085 return 0; 3086 3087 if (sleepable) 3088 rcu_read_lock_trace(); 3089 else 3090 rcu_read_lock(); 3091 3092 migrate_disable(); 3093 3094 old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx); 3095 err = bpf_prog_run(link->link.prog, regs); 3096 bpf_reset_run_ctx(old_run_ctx); 3097 3098 migrate_enable(); 3099 3100 if (sleepable) 3101 rcu_read_unlock_trace(); 3102 else 3103 rcu_read_unlock(); 3104 return err; 3105} 3106 3107static bool 3108uprobe_multi_link_filter(struct uprobe_consumer *con, struct mm_struct *mm) 3109{ 3110 struct bpf_uprobe *uprobe; 3111 3112 uprobe = container_of(con, struct bpf_uprobe, consumer); 3113 return uprobe->link->task->mm == mm; 3114} 3115 3116static int 3117uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs, 3118 __u64 *data) 3119{ 3120 struct bpf_uprobe *uprobe; 3121 int ret; 3122 3123 uprobe = container_of(con, struct bpf_uprobe, consumer); 3124 ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data); 3125 if (uprobe->session) 3126 return ret ? UPROBE_HANDLER_IGNORE : 0; 3127 return 0; 3128} 3129 3130static int 3131uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, struct pt_regs *regs, 3132 __u64 *data) 3133{ 3134 struct bpf_uprobe *uprobe; 3135 3136 uprobe = container_of(con, struct bpf_uprobe, consumer); 3137 uprobe_prog_run(uprobe, func, regs, true, data); 3138 return 0; 3139} 3140 3141static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx) 3142{ 3143 struct bpf_uprobe_multi_run_ctx *run_ctx; 3144 3145 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, 3146 session_ctx.run_ctx); 3147 return run_ctx->entry_ip; 3148} 3149 3150static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) 3151{ 3152 struct bpf_uprobe_multi_run_ctx *run_ctx; 3153 3154 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, 3155 session_ctx.run_ctx); 3156 return run_ctx->uprobe->cookie; 3157} 3158 3159int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3160{ 3161 struct bpf_uprobe_multi_link *link = NULL; 3162 unsigned long __user *uref_ctr_offsets; 3163 struct bpf_link_primer link_primer; 3164 struct bpf_uprobe *uprobes = NULL; 3165 struct task_struct *task = NULL; 3166 unsigned long __user *uoffsets; 3167 u64 __user *ucookies; 3168 void __user *upath; 3169 u32 flags, cnt, i; 3170 struct path path; 3171 char *name; 3172 pid_t pid; 3173 int err; 3174 3175 /* no support for 32bit archs yet */ 3176 if (sizeof(u64) != sizeof(void *)) 3177 return -EOPNOTSUPP; 3178 3179 if (attr->link_create.flags) 3180 return -EINVAL; 3181 3182 if (!is_uprobe_multi(prog)) 3183 return -EINVAL; 3184 3185 flags = attr->link_create.uprobe_multi.flags; 3186 if (flags & ~BPF_F_UPROBE_MULTI_RETURN) 3187 return -EINVAL; 3188 3189 /* 3190 * path, offsets and cnt are mandatory, 3191 * ref_ctr_offsets and cookies are optional 3192 */ 3193 upath = u64_to_user_ptr(attr->link_create.uprobe_multi.path); 3194 uoffsets = u64_to_user_ptr(attr->link_create.uprobe_multi.offsets); 3195 cnt = attr->link_create.uprobe_multi.cnt; 3196 pid = attr->link_create.uprobe_multi.pid; 3197 3198 if (!upath || !uoffsets || !cnt || pid < 0) 3199 return -EINVAL; 3200 if (cnt > MAX_UPROBE_MULTI_CNT) 3201 return -E2BIG; 3202 3203 uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets); 3204 ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies); 3205 3206 name = strndup_user(upath, PATH_MAX); 3207 if (IS_ERR(name)) { 3208 err = PTR_ERR(name); 3209 return err; 3210 } 3211 3212 err = kern_path(name, LOOKUP_FOLLOW, &path); 3213 kfree(name); 3214 if (err) 3215 return err; 3216 3217 if (!d_is_reg(path.dentry)) { 3218 err = -EBADF; 3219 goto error_path_put; 3220 } 3221 3222 if (pid) { 3223 rcu_read_lock(); 3224 task = get_pid_task(find_vpid(pid), PIDTYPE_TGID); 3225 rcu_read_unlock(); 3226 if (!task) { 3227 err = -ESRCH; 3228 goto error_path_put; 3229 } 3230 } 3231 3232 err = -ENOMEM; 3233 3234 link = kzalloc(sizeof(*link), GFP_KERNEL); 3235 uprobes = kvcalloc(cnt, sizeof(*uprobes), GFP_KERNEL); 3236 3237 if (!uprobes || !link) 3238 goto error_free; 3239 3240 for (i = 0; i < cnt; i++) { 3241 if (__get_user(uprobes[i].offset, uoffsets + i)) { 3242 err = -EFAULT; 3243 goto error_free; 3244 } 3245 if (uprobes[i].offset < 0) { 3246 err = -EINVAL; 3247 goto error_free; 3248 } 3249 if (uref_ctr_offsets && __get_user(uprobes[i].ref_ctr_offset, uref_ctr_offsets + i)) { 3250 err = -EFAULT; 3251 goto error_free; 3252 } 3253 if (ucookies && __get_user(uprobes[i].cookie, ucookies + i)) { 3254 err = -EFAULT; 3255 goto error_free; 3256 } 3257 3258 uprobes[i].link = link; 3259 3260 if (!(flags & BPF_F_UPROBE_MULTI_RETURN)) 3261 uprobes[i].consumer.handler = uprobe_multi_link_handler; 3262 if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog)) 3263 uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler; 3264 if (is_uprobe_session(prog)) 3265 uprobes[i].session = true; 3266 if (pid) 3267 uprobes[i].consumer.filter = uprobe_multi_link_filter; 3268 } 3269 3270 link->cnt = cnt; 3271 link->uprobes = uprobes; 3272 link->path = path; 3273 link->task = task; 3274 link->link.flags = flags; 3275 3276 bpf_link_init(&link->link, BPF_LINK_TYPE_UPROBE_MULTI, 3277 &bpf_uprobe_multi_link_lops, prog, attr->link_create.attach_type); 3278 3279 for (i = 0; i < cnt; i++) { 3280 uprobes[i].uprobe = uprobe_register(d_real_inode(link->path.dentry), 3281 uprobes[i].offset, 3282 uprobes[i].ref_ctr_offset, 3283 &uprobes[i].consumer); 3284 if (IS_ERR(uprobes[i].uprobe)) { 3285 err = PTR_ERR(uprobes[i].uprobe); 3286 link->cnt = i; 3287 goto error_unregister; 3288 } 3289 } 3290 3291 err = bpf_link_prime(&link->link, &link_primer); 3292 if (err) 3293 goto error_unregister; 3294 3295 return bpf_link_settle(&link_primer); 3296 3297error_unregister: 3298 bpf_uprobe_unregister(uprobes, link->cnt); 3299 3300error_free: 3301 kvfree(uprobes); 3302 kfree(link); 3303 if (task) 3304 put_task_struct(task); 3305error_path_put: 3306 path_put(&path); 3307 return err; 3308} 3309#else /* !CONFIG_UPROBES */ 3310int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 3311{ 3312 return -EOPNOTSUPP; 3313} 3314static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) 3315{ 3316 return 0; 3317} 3318static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx) 3319{ 3320 return 0; 3321} 3322#endif /* CONFIG_UPROBES */ 3323 3324__bpf_kfunc_start_defs(); 3325 3326__bpf_kfunc bool bpf_session_is_return(void) 3327{ 3328 struct bpf_session_run_ctx *session_ctx; 3329 3330 session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx); 3331 return session_ctx->is_return; 3332} 3333 3334__bpf_kfunc __u64 *bpf_session_cookie(void) 3335{ 3336 struct bpf_session_run_ctx *session_ctx; 3337 3338 session_ctx = container_of(current->bpf_ctx, struct bpf_session_run_ctx, run_ctx); 3339 return session_ctx->data; 3340} 3341 3342__bpf_kfunc_end_defs(); 3343 3344BTF_KFUNCS_START(kprobe_multi_kfunc_set_ids) 3345BTF_ID_FLAGS(func, bpf_session_is_return) 3346BTF_ID_FLAGS(func, bpf_session_cookie) 3347BTF_KFUNCS_END(kprobe_multi_kfunc_set_ids) 3348 3349static int bpf_kprobe_multi_filter(const struct bpf_prog *prog, u32 kfunc_id) 3350{ 3351 if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id)) 3352 return 0; 3353 3354 if (!is_kprobe_session(prog) && !is_uprobe_session(prog)) 3355 return -EACCES; 3356 3357 return 0; 3358} 3359 3360static const struct btf_kfunc_id_set bpf_kprobe_multi_kfunc_set = { 3361 .owner = THIS_MODULE, 3362 .set = &kprobe_multi_kfunc_set_ids, 3363 .filter = bpf_kprobe_multi_filter, 3364}; 3365 3366static int __init bpf_kprobe_multi_kfuncs_init(void) 3367{ 3368 return register_btf_kfunc_id_set(BPF_PROG_TYPE_KPROBE, &bpf_kprobe_multi_kfunc_set); 3369} 3370 3371late_initcall(bpf_kprobe_multi_kfuncs_init); 3372 3373typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struct *tsk); 3374 3375/* 3376 * The __always_inline is to make sure the compiler doesn't 3377 * generate indirect calls into callbacks, which is expensive, 3378 * on some kernel configurations. This allows compiler to put 3379 * direct calls into all the specific callback implementations 3380 * (copy_user_data_sleepable, copy_user_data_nofault, and so on) 3381 */ 3382static __always_inline int __bpf_dynptr_copy_str(struct bpf_dynptr *dptr, u64 doff, u64 size, 3383 const void *unsafe_src, 3384 copy_fn_t str_copy_fn, 3385 struct task_struct *tsk) 3386{ 3387 struct bpf_dynptr_kern *dst; 3388 u64 chunk_sz, off; 3389 void *dst_slice; 3390 int cnt, err; 3391 char buf[256]; 3392 3393 dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size); 3394 if (likely(dst_slice)) 3395 return str_copy_fn(dst_slice, unsafe_src, size, tsk); 3396 3397 dst = (struct bpf_dynptr_kern *)dptr; 3398 if (bpf_dynptr_check_off_len(dst, doff, size)) 3399 return -E2BIG; 3400 3401 for (off = 0; off < size; off += chunk_sz - 1) { 3402 chunk_sz = min_t(u64, sizeof(buf), size - off); 3403 /* Expect str_copy_fn to return count of copied bytes, including 3404 * zero terminator. Next iteration increment off by chunk_sz - 1 to 3405 * overwrite NUL. 3406 */ 3407 cnt = str_copy_fn(buf, unsafe_src + off, chunk_sz, tsk); 3408 if (cnt < 0) 3409 return cnt; 3410 err = __bpf_dynptr_write(dst, doff + off, buf, cnt, 0); 3411 if (err) 3412 return err; 3413 if (cnt < chunk_sz || chunk_sz == 1) /* we are done */ 3414 return off + cnt; 3415 } 3416 return off; 3417} 3418 3419static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u64 doff, 3420 u64 size, const void *unsafe_src, 3421 copy_fn_t copy_fn, struct task_struct *tsk) 3422{ 3423 struct bpf_dynptr_kern *dst; 3424 void *dst_slice; 3425 char buf[256]; 3426 u64 off, chunk_sz; 3427 int err; 3428 3429 dst_slice = bpf_dynptr_slice_rdwr(dptr, doff, NULL, size); 3430 if (likely(dst_slice)) 3431 return copy_fn(dst_slice, unsafe_src, size, tsk); 3432 3433 dst = (struct bpf_dynptr_kern *)dptr; 3434 if (bpf_dynptr_check_off_len(dst, doff, size)) 3435 return -E2BIG; 3436 3437 for (off = 0; off < size; off += chunk_sz) { 3438 chunk_sz = min_t(u64, sizeof(buf), size - off); 3439 err = copy_fn(buf, unsafe_src + off, chunk_sz, tsk); 3440 if (err) 3441 return err; 3442 err = __bpf_dynptr_write(dst, doff + off, buf, chunk_sz, 0); 3443 if (err) 3444 return err; 3445 } 3446 return 0; 3447} 3448 3449static __always_inline int copy_user_data_nofault(void *dst, const void *unsafe_src, 3450 u32 size, struct task_struct *tsk) 3451{ 3452 return copy_from_user_nofault(dst, (const void __user *)unsafe_src, size); 3453} 3454 3455static __always_inline int copy_user_data_sleepable(void *dst, const void *unsafe_src, 3456 u32 size, struct task_struct *tsk) 3457{ 3458 int ret; 3459 3460 if (!tsk) { /* Read from the current task */ 3461 ret = copy_from_user(dst, (const void __user *)unsafe_src, size); 3462 if (ret) 3463 return -EFAULT; 3464 return 0; 3465 } 3466 3467 ret = access_process_vm(tsk, (unsigned long)unsafe_src, dst, size, 0); 3468 if (ret != size) 3469 return -EFAULT; 3470 return 0; 3471} 3472 3473static __always_inline int copy_kernel_data_nofault(void *dst, const void *unsafe_src, 3474 u32 size, struct task_struct *tsk) 3475{ 3476 return copy_from_kernel_nofault(dst, unsafe_src, size); 3477} 3478 3479static __always_inline int copy_user_str_nofault(void *dst, const void *unsafe_src, 3480 u32 size, struct task_struct *tsk) 3481{ 3482 return strncpy_from_user_nofault(dst, (const void __user *)unsafe_src, size); 3483} 3484 3485static __always_inline int copy_user_str_sleepable(void *dst, const void *unsafe_src, 3486 u32 size, struct task_struct *tsk) 3487{ 3488 int ret; 3489 3490 if (unlikely(size == 0)) 3491 return 0; 3492 3493 if (tsk) { 3494 ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_src, dst, size, 0); 3495 } else { 3496 ret = strncpy_from_user(dst, (const void __user *)unsafe_src, size - 1); 3497 /* strncpy_from_user does not guarantee NUL termination */ 3498 if (ret >= 0) 3499 ((char *)dst)[ret] = '\0'; 3500 } 3501 3502 if (ret < 0) 3503 return ret; 3504 return ret + 1; 3505} 3506 3507static __always_inline int copy_kernel_str_nofault(void *dst, const void *unsafe_src, 3508 u32 size, struct task_struct *tsk) 3509{ 3510 return strncpy_from_kernel_nofault(dst, unsafe_src, size); 3511} 3512 3513__bpf_kfunc_start_defs(); 3514 3515__bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type, 3516 u64 value) 3517{ 3518 if (type != PIDTYPE_PID && type != PIDTYPE_TGID) 3519 return -EINVAL; 3520 3521 return bpf_send_signal_common(sig, type, task, value); 3522} 3523 3524__bpf_kfunc int bpf_probe_read_user_dynptr(struct bpf_dynptr *dptr, u64 off, 3525 u64 size, const void __user *unsafe_ptr__ign) 3526{ 3527 return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign, 3528 copy_user_data_nofault, NULL); 3529} 3530 3531__bpf_kfunc int bpf_probe_read_kernel_dynptr(struct bpf_dynptr *dptr, u64 off, 3532 u64 size, const void *unsafe_ptr__ign) 3533{ 3534 return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign, 3535 copy_kernel_data_nofault, NULL); 3536} 3537 3538__bpf_kfunc int bpf_probe_read_user_str_dynptr(struct bpf_dynptr *dptr, u64 off, 3539 u64 size, const void __user *unsafe_ptr__ign) 3540{ 3541 return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign, 3542 copy_user_str_nofault, NULL); 3543} 3544 3545__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(struct bpf_dynptr *dptr, u64 off, 3546 u64 size, const void *unsafe_ptr__ign) 3547{ 3548 return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign, 3549 copy_kernel_str_nofault, NULL); 3550} 3551 3552__bpf_kfunc int bpf_copy_from_user_dynptr(struct bpf_dynptr *dptr, u64 off, 3553 u64 size, const void __user *unsafe_ptr__ign) 3554{ 3555 return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign, 3556 copy_user_data_sleepable, NULL); 3557} 3558 3559__bpf_kfunc int bpf_copy_from_user_str_dynptr(struct bpf_dynptr *dptr, u64 off, 3560 u64 size, const void __user *unsafe_ptr__ign) 3561{ 3562 return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign, 3563 copy_user_str_sleepable, NULL); 3564} 3565 3566__bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u64 off, 3567 u64 size, const void __user *unsafe_ptr__ign, 3568 struct task_struct *tsk) 3569{ 3570 return __bpf_dynptr_copy(dptr, off, size, (const void *)unsafe_ptr__ign, 3571 copy_user_data_sleepable, tsk); 3572} 3573 3574__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(struct bpf_dynptr *dptr, u64 off, 3575 u64 size, const void __user *unsafe_ptr__ign, 3576 struct task_struct *tsk) 3577{ 3578 return __bpf_dynptr_copy_str(dptr, off, size, (const void *)unsafe_ptr__ign, 3579 copy_user_str_sleepable, tsk); 3580} 3581 3582__bpf_kfunc_end_defs();
~/
Read-only file preview.