FILE
kprobes.c
File preview with syntax highlighting.
linux-6.19.14/kernel/kprobes.c
1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Kernel Probes (KProbes) 4 * 5 * Copyright (C) IBM Corporation, 2002, 2004 6 * 7 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 8 * Probes initial implementation (includes suggestions from 9 * Rusty Russell). 10 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 11 * hlists and exceptions notifier as suggested by Andi Kleen. 12 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 13 * interface to access function arguments. 14 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 15 * exceptions notifier to be first on the priority list. 16 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 17 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 18 * <prasanna@in.ibm.com> added function-return probes. 19 */ 20 21#define pr_fmt(fmt) "kprobes: " fmt 22 23#include <linux/kprobes.h> 24#include <linux/hash.h> 25#include <linux/init.h> 26#include <linux/slab.h> 27#include <linux/stddef.h> 28#include <linux/export.h> 29#include <linux/kallsyms.h> 30#include <linux/freezer.h> 31#include <linux/seq_file.h> 32#include <linux/debugfs.h> 33#include <linux/sysctl.h> 34#include <linux/kdebug.h> 35#include <linux/memory.h> 36#include <linux/ftrace.h> 37#include <linux/cpu.h> 38#include <linux/jump_label.h> 39#include <linux/static_call.h> 40#include <linux/perf_event.h> 41#include <linux/execmem.h> 42#include <linux/cleanup.h> 43 44#include <asm/sections.h> 45#include <asm/cacheflush.h> 46#include <asm/errno.h> 47#include <linux/uaccess.h> 48 49#define KPROBE_HASH_BITS 6 50#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 51 52#if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL) 53#define kprobe_sysctls_init() do { } while (0) 54#endif 55 56static int kprobes_initialized; 57/* kprobe_table can be accessed by 58 * - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held. 59 * Or 60 * - RCU hlist traversal under disabling preempt (breakpoint handlers) 61 */ 62static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 63 64/* NOTE: change this value only with 'kprobe_mutex' held */ 65static bool kprobes_all_disarmed; 66 67/* This protects 'kprobe_table' and 'optimizing_list' */ 68static DEFINE_MUTEX(kprobe_mutex); 69static DEFINE_PER_CPU(struct kprobe *, kprobe_instance); 70 71kprobe_opcode_t * __weak kprobe_lookup_name(const char *name, 72 unsigned int __unused) 73{ 74 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name))); 75} 76 77/* 78 * Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where 79 * kprobes can not probe. 80 */ 81static LIST_HEAD(kprobe_blacklist); 82 83#ifdef __ARCH_WANT_KPROBES_INSN_SLOT 84/* 85 * 'kprobe::ainsn.insn' points to the copy of the instruction to be 86 * single-stepped. x86_64, POWER4 and above have no-exec support and 87 * stepping on the instruction on a vmalloced/kmalloced/data page 88 * is a recipe for disaster 89 */ 90struct kprobe_insn_page { 91 struct list_head list; 92 kprobe_opcode_t *insns; /* Page of instruction slots */ 93 struct kprobe_insn_cache *cache; 94 int nused; 95 int ngarbage; 96 char slot_used[]; 97}; 98 99static int slots_per_page(struct kprobe_insn_cache *c) 100{ 101 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t)); 102} 103 104enum kprobe_slot_state { 105 SLOT_CLEAN = 0, 106 SLOT_DIRTY = 1, 107 SLOT_USED = 2, 108}; 109 110void __weak *alloc_insn_page(void) 111{ 112 /* 113 * Use execmem_alloc() so this page is within +/- 2GB of where the 114 * kernel image and loaded module images reside. This is required 115 * for most of the architectures. 116 * (e.g. x86-64 needs this to handle the %rip-relative fixups.) 117 */ 118 return execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE); 119} 120 121static void free_insn_page(void *page) 122{ 123 execmem_free(page); 124} 125 126struct kprobe_insn_cache kprobe_insn_slots = { 127 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex), 128 .alloc = alloc_insn_page, 129 .free = free_insn_page, 130 .sym = KPROBE_INSN_PAGE_SYM, 131 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages), 132 .insn_size = MAX_INSN_SIZE, 133 .nr_garbage = 0, 134}; 135static int collect_garbage_slots(struct kprobe_insn_cache *c); 136 137/** 138 * __get_insn_slot - Find a slot on an executable page for an instruction. 139 * @c: Pointer to kprobe instruction cache 140 * 141 * Description: Locates available slot on existing executable pages, 142 * allocates an executable page if there's no room on existing ones. 143 * Return: Pointer to instruction slot on success, NULL on failure. 144 */ 145kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c) 146{ 147 struct kprobe_insn_page *kip; 148 149 /* Since the slot array is not protected by rcu, we need a mutex */ 150 guard(mutex)(&c->mutex); 151 do { 152 guard(rcu)(); 153 list_for_each_entry_rcu(kip, &c->pages, list) { 154 if (kip->nused < slots_per_page(c)) { 155 int i; 156 157 for (i = 0; i < slots_per_page(c); i++) { 158 if (kip->slot_used[i] == SLOT_CLEAN) { 159 kip->slot_used[i] = SLOT_USED; 160 kip->nused++; 161 return kip->insns + (i * c->insn_size); 162 } 163 } 164 /* kip->nused is broken. Fix it. */ 165 kip->nused = slots_per_page(c); 166 WARN_ON(1); 167 } 168 } 169 /* If there are any garbage slots, collect it and try again. */ 170 } while (c->nr_garbage && collect_garbage_slots(c) == 0); 171 172 /* All out of space. Need to allocate a new page. */ 173 kip = kmalloc(struct_size(kip, slot_used, slots_per_page(c)), GFP_KERNEL); 174 if (!kip) 175 return NULL; 176 177 kip->insns = c->alloc(); 178 if (!kip->insns) { 179 kfree(kip); 180 return NULL; 181 } 182 INIT_LIST_HEAD(&kip->list); 183 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c)); 184 kip->slot_used[0] = SLOT_USED; 185 kip->nused = 1; 186 kip->ngarbage = 0; 187 kip->cache = c; 188 list_add_rcu(&kip->list, &c->pages); 189 190 /* Record the perf ksymbol register event after adding the page */ 191 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns, 192 PAGE_SIZE, false, c->sym); 193 194 return kip->insns; 195} 196 197/* Return true if all garbages are collected, otherwise false. */ 198static bool collect_one_slot(struct kprobe_insn_page *kip, int idx) 199{ 200 kip->slot_used[idx] = SLOT_CLEAN; 201 kip->nused--; 202 if (kip->nused != 0) 203 return false; 204 205 /* 206 * Page is no longer in use. Free it unless 207 * it's the last one. We keep the last one 208 * so as not to have to set it up again the 209 * next time somebody inserts a probe. 210 */ 211 if (!list_is_singular(&kip->list)) { 212 /* 213 * Record perf ksymbol unregister event before removing 214 * the page. 215 */ 216 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, 217 (unsigned long)kip->insns, PAGE_SIZE, true, 218 kip->cache->sym); 219 list_del_rcu(&kip->list); 220 synchronize_rcu(); 221 kip->cache->free(kip->insns); 222 kfree(kip); 223 } 224 return true; 225} 226 227static int collect_garbage_slots(struct kprobe_insn_cache *c) 228{ 229 struct kprobe_insn_page *kip, *next; 230 231 /* Ensure no-one is interrupted on the garbages */ 232 synchronize_rcu(); 233 234 list_for_each_entry_safe(kip, next, &c->pages, list) { 235 int i; 236 237 if (kip->ngarbage == 0) 238 continue; 239 kip->ngarbage = 0; /* we will collect all garbages */ 240 for (i = 0; i < slots_per_page(c); i++) { 241 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i)) 242 break; 243 } 244 } 245 c->nr_garbage = 0; 246 return 0; 247} 248 249static long __find_insn_page(struct kprobe_insn_cache *c, 250 kprobe_opcode_t *slot, struct kprobe_insn_page **pkip) 251{ 252 struct kprobe_insn_page *kip = NULL; 253 long idx; 254 255 guard(rcu)(); 256 list_for_each_entry_rcu(kip, &c->pages, list) { 257 idx = ((long)slot - (long)kip->insns) / 258 (c->insn_size * sizeof(kprobe_opcode_t)); 259 if (idx >= 0 && idx < slots_per_page(c)) { 260 *pkip = kip; 261 return idx; 262 } 263 } 264 /* Could not find this slot. */ 265 WARN_ON(1); 266 *pkip = NULL; 267 return -1; 268} 269 270void __free_insn_slot(struct kprobe_insn_cache *c, 271 kprobe_opcode_t *slot, int dirty) 272{ 273 struct kprobe_insn_page *kip = NULL; 274 long idx; 275 276 guard(mutex)(&c->mutex); 277 idx = __find_insn_page(c, slot, &kip); 278 /* Mark and sweep: this may sleep */ 279 if (kip) { 280 /* Check double free */ 281 WARN_ON(kip->slot_used[idx] != SLOT_USED); 282 if (dirty) { 283 kip->slot_used[idx] = SLOT_DIRTY; 284 kip->ngarbage++; 285 if (++c->nr_garbage > slots_per_page(c)) 286 collect_garbage_slots(c); 287 } else { 288 collect_one_slot(kip, idx); 289 } 290 } 291} 292 293/* 294 * Check given address is on the page of kprobe instruction slots. 295 * This will be used for checking whether the address on a stack 296 * is on a text area or not. 297 */ 298bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr) 299{ 300 struct kprobe_insn_page *kip; 301 bool ret = false; 302 303 rcu_read_lock(); 304 list_for_each_entry_rcu(kip, &c->pages, list) { 305 if (addr >= (unsigned long)kip->insns && 306 addr < (unsigned long)kip->insns + PAGE_SIZE) { 307 ret = true; 308 break; 309 } 310 } 311 rcu_read_unlock(); 312 313 return ret; 314} 315 316int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum, 317 unsigned long *value, char *type, char *sym) 318{ 319 struct kprobe_insn_page *kip; 320 int ret = -ERANGE; 321 322 rcu_read_lock(); 323 list_for_each_entry_rcu(kip, &c->pages, list) { 324 if ((*symnum)--) 325 continue; 326 strscpy(sym, c->sym, KSYM_NAME_LEN); 327 *type = 't'; 328 *value = (unsigned long)kip->insns; 329 ret = 0; 330 break; 331 } 332 rcu_read_unlock(); 333 334 return ret; 335} 336 337#ifdef CONFIG_OPTPROBES 338void __weak *alloc_optinsn_page(void) 339{ 340 return alloc_insn_page(); 341} 342 343void __weak free_optinsn_page(void *page) 344{ 345 free_insn_page(page); 346} 347 348/* For optimized_kprobe buffer */ 349struct kprobe_insn_cache kprobe_optinsn_slots = { 350 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex), 351 .alloc = alloc_optinsn_page, 352 .free = free_optinsn_page, 353 .sym = KPROBE_OPTINSN_PAGE_SYM, 354 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages), 355 /* .insn_size is initialized later */ 356 .nr_garbage = 0, 357}; 358#endif /* CONFIG_OPTPROBES */ 359#endif /* __ARCH_WANT_KPROBES_INSN_SLOT */ 360 361/* We have preemption disabled.. so it is safe to use __ versions */ 362static inline void set_kprobe_instance(struct kprobe *kp) 363{ 364 __this_cpu_write(kprobe_instance, kp); 365} 366 367static inline void reset_kprobe_instance(void) 368{ 369 __this_cpu_write(kprobe_instance, NULL); 370} 371 372/* 373 * This routine is called either: 374 * - under the 'kprobe_mutex' - during kprobe_[un]register(). 375 * OR 376 * - with preemption disabled - from architecture specific code. 377 */ 378struct kprobe *get_kprobe(void *addr) 379{ 380 struct hlist_head *head; 381 struct kprobe *p; 382 383 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 384 hlist_for_each_entry_rcu(p, head, hlist, 385 lockdep_is_held(&kprobe_mutex)) { 386 if (p->addr == addr) 387 return p; 388 } 389 390 return NULL; 391} 392NOKPROBE_SYMBOL(get_kprobe); 393 394static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs); 395 396/* Return true if 'p' is an aggregator */ 397static inline bool kprobe_aggrprobe(struct kprobe *p) 398{ 399 return p->pre_handler == aggr_pre_handler; 400} 401 402/* Return true if 'p' is unused */ 403static inline bool kprobe_unused(struct kprobe *p) 404{ 405 return kprobe_aggrprobe(p) && kprobe_disabled(p) && 406 list_empty(&p->list); 407} 408 409/* Keep all fields in the kprobe consistent. */ 410static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p) 411{ 412 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t)); 413 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn)); 414} 415 416#ifdef CONFIG_OPTPROBES 417/* NOTE: This is protected by 'kprobe_mutex'. */ 418static bool kprobes_allow_optimization; 419 420/* 421 * Call all 'kprobe::pre_handler' on the list, but ignores its return value. 422 * This must be called from arch-dep optimized caller. 423 */ 424void opt_pre_handler(struct kprobe *p, struct pt_regs *regs) 425{ 426 struct kprobe *kp; 427 428 list_for_each_entry_rcu(kp, &p->list, list) { 429 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 430 set_kprobe_instance(kp); 431 kp->pre_handler(kp, regs); 432 } 433 reset_kprobe_instance(); 434 } 435} 436NOKPROBE_SYMBOL(opt_pre_handler); 437 438/* Free optimized instructions and optimized_kprobe */ 439static void free_aggr_kprobe(struct kprobe *p) 440{ 441 struct optimized_kprobe *op; 442 443 op = container_of(p, struct optimized_kprobe, kp); 444 arch_remove_optimized_kprobe(op); 445 arch_remove_kprobe(p); 446 kfree(op); 447} 448 449/* Return true if the kprobe is ready for optimization. */ 450static inline int kprobe_optready(struct kprobe *p) 451{ 452 struct optimized_kprobe *op; 453 454 if (kprobe_aggrprobe(p)) { 455 op = container_of(p, struct optimized_kprobe, kp); 456 return arch_prepared_optinsn(&op->optinsn); 457 } 458 459 return 0; 460} 461 462/* Return true if the kprobe is disarmed. Note: p must be on hash list */ 463bool kprobe_disarmed(struct kprobe *p) 464{ 465 struct optimized_kprobe *op; 466 467 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */ 468 if (!kprobe_aggrprobe(p)) 469 return kprobe_disabled(p); 470 471 op = container_of(p, struct optimized_kprobe, kp); 472 473 return kprobe_disabled(p) && list_empty(&op->list); 474} 475 476/* Return true if the probe is queued on (un)optimizing lists */ 477static bool kprobe_queued(struct kprobe *p) 478{ 479 struct optimized_kprobe *op; 480 481 if (kprobe_aggrprobe(p)) { 482 op = container_of(p, struct optimized_kprobe, kp); 483 if (!list_empty(&op->list)) 484 return true; 485 } 486 return false; 487} 488 489/* 490 * Return an optimized kprobe whose optimizing code replaces 491 * instructions including 'addr' (exclude breakpoint). 492 */ 493static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr) 494{ 495 int i; 496 struct kprobe *p = NULL; 497 struct optimized_kprobe *op; 498 499 /* Don't check i == 0, since that is a breakpoint case. */ 500 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) 501 p = get_kprobe(addr - i); 502 503 if (p && kprobe_optready(p)) { 504 op = container_of(p, struct optimized_kprobe, kp); 505 if (arch_within_optimized_kprobe(op, addr)) 506 return p; 507 } 508 509 return NULL; 510} 511 512/* Optimization staging list, protected by 'kprobe_mutex' */ 513static LIST_HEAD(optimizing_list); 514static LIST_HEAD(unoptimizing_list); 515static LIST_HEAD(freeing_list); 516 517static void kprobe_optimizer(struct work_struct *work); 518static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer); 519#define OPTIMIZE_DELAY 5 520 521/* 522 * Optimize (replace a breakpoint with a jump) kprobes listed on 523 * 'optimizing_list'. 524 */ 525static void do_optimize_kprobes(void) 526{ 527 lockdep_assert_held(&text_mutex); 528 /* 529 * The optimization/unoptimization refers 'online_cpus' via 530 * stop_machine() and cpu-hotplug modifies the 'online_cpus'. 531 * And same time, 'text_mutex' will be held in cpu-hotplug and here. 532 * This combination can cause a deadlock (cpu-hotplug tries to lock 533 * 'text_mutex' but stop_machine() can not be done because 534 * the 'online_cpus' has been changed) 535 * To avoid this deadlock, caller must have locked cpu-hotplug 536 * for preventing cpu-hotplug outside of 'text_mutex' locking. 537 */ 538 lockdep_assert_cpus_held(); 539 540 /* Optimization never be done when disarmed */ 541 if (kprobes_all_disarmed || !kprobes_allow_optimization || 542 list_empty(&optimizing_list)) 543 return; 544 545 arch_optimize_kprobes(&optimizing_list); 546} 547 548/* 549 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint 550 * if need) kprobes listed on 'unoptimizing_list'. 551 */ 552static void do_unoptimize_kprobes(void) 553{ 554 struct optimized_kprobe *op, *tmp; 555 556 lockdep_assert_held(&text_mutex); 557 /* See comment in do_optimize_kprobes() */ 558 lockdep_assert_cpus_held(); 559 560 if (!list_empty(&unoptimizing_list)) 561 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list); 562 563 /* Loop on 'freeing_list' for disarming and removing from kprobe hash list */ 564 list_for_each_entry_safe(op, tmp, &freeing_list, list) { 565 /* Switching from detour code to origin */ 566 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 567 /* Disarm probes if marked disabled and not gone */ 568 if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp)) 569 arch_disarm_kprobe(&op->kp); 570 if (kprobe_unused(&op->kp)) { 571 /* 572 * Remove unused probes from hash list. After waiting 573 * for synchronization, these probes are reclaimed. 574 * (reclaiming is done by do_free_cleaned_kprobes().) 575 */ 576 hlist_del_rcu(&op->kp.hlist); 577 } else 578 list_del_init(&op->list); 579 } 580} 581 582/* Reclaim all kprobes on the 'freeing_list' */ 583static void do_free_cleaned_kprobes(void) 584{ 585 struct optimized_kprobe *op, *tmp; 586 587 list_for_each_entry_safe(op, tmp, &freeing_list, list) { 588 list_del_init(&op->list); 589 if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) { 590 /* 591 * This must not happen, but if there is a kprobe 592 * still in use, keep it on kprobes hash list. 593 */ 594 continue; 595 } 596 free_aggr_kprobe(&op->kp); 597 } 598} 599 600/* Start optimizer after OPTIMIZE_DELAY passed */ 601static void kick_kprobe_optimizer(void) 602{ 603 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY); 604} 605 606/* Kprobe jump optimizer */ 607static void kprobe_optimizer(struct work_struct *work) 608{ 609 guard(mutex)(&kprobe_mutex); 610 611 scoped_guard(cpus_read_lock) { 612 guard(mutex)(&text_mutex); 613 614 /* 615 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed) 616 * kprobes before waiting for quiesence period. 617 */ 618 do_unoptimize_kprobes(); 619 620 /* 621 * Step 2: Wait for quiesence period to ensure all potentially 622 * preempted tasks to have normally scheduled. Because optprobe 623 * may modify multiple instructions, there is a chance that Nth 624 * instruction is preempted. In that case, such tasks can return 625 * to 2nd-Nth byte of jump instruction. This wait is for avoiding it. 626 * Note that on non-preemptive kernel, this is transparently converted 627 * to synchronoze_sched() to wait for all interrupts to have completed. 628 */ 629 synchronize_rcu_tasks(); 630 631 /* Step 3: Optimize kprobes after quiesence period */ 632 do_optimize_kprobes(); 633 634 /* Step 4: Free cleaned kprobes after quiesence period */ 635 do_free_cleaned_kprobes(); 636 } 637 638 /* Step 5: Kick optimizer again if needed */ 639 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) 640 kick_kprobe_optimizer(); 641} 642 643static void wait_for_kprobe_optimizer_locked(void) 644{ 645 lockdep_assert_held(&kprobe_mutex); 646 647 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) { 648 mutex_unlock(&kprobe_mutex); 649 650 /* This will also make 'optimizing_work' execute immmediately */ 651 flush_delayed_work(&optimizing_work); 652 /* 'optimizing_work' might not have been queued yet, relax */ 653 cpu_relax(); 654 655 mutex_lock(&kprobe_mutex); 656 } 657} 658 659/* Wait for completing optimization and unoptimization */ 660void wait_for_kprobe_optimizer(void) 661{ 662 guard(mutex)(&kprobe_mutex); 663 664 wait_for_kprobe_optimizer_locked(); 665} 666 667bool optprobe_queued_unopt(struct optimized_kprobe *op) 668{ 669 struct optimized_kprobe *_op; 670 671 list_for_each_entry(_op, &unoptimizing_list, list) { 672 if (op == _op) 673 return true; 674 } 675 676 return false; 677} 678 679/* Optimize kprobe if p is ready to be optimized */ 680static void optimize_kprobe(struct kprobe *p) 681{ 682 struct optimized_kprobe *op; 683 684 /* Check if the kprobe is disabled or not ready for optimization. */ 685 if (!kprobe_optready(p) || !kprobes_allow_optimization || 686 (kprobe_disabled(p) || kprobes_all_disarmed)) 687 return; 688 689 /* kprobes with 'post_handler' can not be optimized */ 690 if (p->post_handler) 691 return; 692 693 op = container_of(p, struct optimized_kprobe, kp); 694 695 /* Check there is no other kprobes at the optimized instructions */ 696 if (arch_check_optimized_kprobe(op) < 0) 697 return; 698 699 /* Check if it is already optimized. */ 700 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) { 701 if (optprobe_queued_unopt(op)) { 702 /* This is under unoptimizing. Just dequeue the probe */ 703 list_del_init(&op->list); 704 } 705 return; 706 } 707 op->kp.flags |= KPROBE_FLAG_OPTIMIZED; 708 709 /* 710 * On the 'unoptimizing_list' and 'optimizing_list', 711 * 'op' must have OPTIMIZED flag 712 */ 713 if (WARN_ON_ONCE(!list_empty(&op->list))) 714 return; 715 716 list_add(&op->list, &optimizing_list); 717 kick_kprobe_optimizer(); 718} 719 720/* Short cut to direct unoptimizing */ 721static void force_unoptimize_kprobe(struct optimized_kprobe *op) 722{ 723 lockdep_assert_cpus_held(); 724 arch_unoptimize_kprobe(op); 725 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 726} 727 728/* Unoptimize a kprobe if p is optimized */ 729static void unoptimize_kprobe(struct kprobe *p, bool force) 730{ 731 struct optimized_kprobe *op; 732 733 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p)) 734 return; /* This is not an optprobe nor optimized */ 735 736 op = container_of(p, struct optimized_kprobe, kp); 737 if (!kprobe_optimized(p)) 738 return; 739 740 if (!list_empty(&op->list)) { 741 if (optprobe_queued_unopt(op)) { 742 /* Queued in unoptimizing queue */ 743 if (force) { 744 /* 745 * Forcibly unoptimize the kprobe here, and queue it 746 * in the freeing list for release afterwards. 747 */ 748 force_unoptimize_kprobe(op); 749 list_move(&op->list, &freeing_list); 750 } 751 } else { 752 /* Dequeue from the optimizing queue */ 753 list_del_init(&op->list); 754 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 755 } 756 return; 757 } 758 759 /* Optimized kprobe case */ 760 if (force) { 761 /* Forcibly update the code: this is a special case */ 762 force_unoptimize_kprobe(op); 763 } else { 764 list_add(&op->list, &unoptimizing_list); 765 kick_kprobe_optimizer(); 766 } 767} 768 769/* Cancel unoptimizing for reusing */ 770static int reuse_unused_kprobe(struct kprobe *ap) 771{ 772 struct optimized_kprobe *op; 773 774 /* 775 * Unused kprobe MUST be on the way of delayed unoptimizing (means 776 * there is still a relative jump) and disabled. 777 */ 778 op = container_of(ap, struct optimized_kprobe, kp); 779 WARN_ON_ONCE(list_empty(&op->list)); 780 /* Enable the probe again */ 781 ap->flags &= ~KPROBE_FLAG_DISABLED; 782 /* Optimize it again. (remove from 'op->list') */ 783 if (!kprobe_optready(ap)) 784 return -EINVAL; 785 786 optimize_kprobe(ap); 787 return 0; 788} 789 790/* Remove optimized instructions */ 791static void kill_optimized_kprobe(struct kprobe *p) 792{ 793 struct optimized_kprobe *op; 794 795 op = container_of(p, struct optimized_kprobe, kp); 796 if (!list_empty(&op->list)) 797 /* Dequeue from the (un)optimization queue */ 798 list_del_init(&op->list); 799 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED; 800 801 if (kprobe_unused(p)) { 802 /* 803 * Unused kprobe is on unoptimizing or freeing list. We move it 804 * to freeing_list and let the kprobe_optimizer() remove it from 805 * the kprobe hash list and free it. 806 */ 807 if (optprobe_queued_unopt(op)) 808 list_move(&op->list, &freeing_list); 809 } 810 811 /* Don't touch the code, because it is already freed. */ 812 arch_remove_optimized_kprobe(op); 813} 814 815static inline 816void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p) 817{ 818 if (!kprobe_ftrace(p)) 819 arch_prepare_optimized_kprobe(op, p); 820} 821 822/* Try to prepare optimized instructions */ 823static void prepare_optimized_kprobe(struct kprobe *p) 824{ 825 struct optimized_kprobe *op; 826 827 op = container_of(p, struct optimized_kprobe, kp); 828 __prepare_optimized_kprobe(op, p); 829} 830 831/* Allocate new optimized_kprobe and try to prepare optimized instructions. */ 832static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 833{ 834 struct optimized_kprobe *op; 835 836 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL); 837 if (!op) 838 return NULL; 839 840 INIT_LIST_HEAD(&op->list); 841 op->kp.addr = p->addr; 842 __prepare_optimized_kprobe(op, p); 843 844 return &op->kp; 845} 846 847static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p); 848 849/* 850 * Prepare an optimized_kprobe and optimize it. 851 * NOTE: 'p' must be a normal registered kprobe. 852 */ 853static void try_to_optimize_kprobe(struct kprobe *p) 854{ 855 struct kprobe *ap; 856 struct optimized_kprobe *op; 857 858 /* Impossible to optimize ftrace-based kprobe. */ 859 if (kprobe_ftrace(p)) 860 return; 861 862 /* For preparing optimization, jump_label_text_reserved() is called. */ 863 guard(cpus_read_lock)(); 864 guard(jump_label_lock)(); 865 guard(mutex)(&text_mutex); 866 867 ap = alloc_aggr_kprobe(p); 868 if (!ap) 869 return; 870 871 op = container_of(ap, struct optimized_kprobe, kp); 872 if (!arch_prepared_optinsn(&op->optinsn)) { 873 /* If failed to setup optimizing, fallback to kprobe. */ 874 arch_remove_optimized_kprobe(op); 875 kfree(op); 876 return; 877 } 878 879 init_aggr_kprobe(ap, p); 880 optimize_kprobe(ap); /* This just kicks optimizer thread. */ 881} 882 883static void optimize_all_kprobes(void) 884{ 885 struct hlist_head *head; 886 struct kprobe *p; 887 unsigned int i; 888 889 guard(mutex)(&kprobe_mutex); 890 /* If optimization is already allowed, just return. */ 891 if (kprobes_allow_optimization) 892 return; 893 894 cpus_read_lock(); 895 kprobes_allow_optimization = true; 896 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 897 head = &kprobe_table[i]; 898 hlist_for_each_entry(p, head, hlist) 899 if (!kprobe_disabled(p)) 900 optimize_kprobe(p); 901 } 902 cpus_read_unlock(); 903 pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n"); 904} 905 906#ifdef CONFIG_SYSCTL 907static void unoptimize_all_kprobes(void) 908{ 909 struct hlist_head *head; 910 struct kprobe *p; 911 unsigned int i; 912 913 guard(mutex)(&kprobe_mutex); 914 /* If optimization is already prohibited, just return. */ 915 if (!kprobes_allow_optimization) 916 return; 917 918 cpus_read_lock(); 919 kprobes_allow_optimization = false; 920 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 921 head = &kprobe_table[i]; 922 hlist_for_each_entry(p, head, hlist) { 923 if (!kprobe_disabled(p)) 924 unoptimize_kprobe(p, false); 925 } 926 } 927 cpus_read_unlock(); 928 /* Wait for unoptimizing completion. */ 929 wait_for_kprobe_optimizer_locked(); 930 pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n"); 931} 932 933static DEFINE_MUTEX(kprobe_sysctl_mutex); 934static int sysctl_kprobes_optimization; 935static int proc_kprobes_optimization_handler(const struct ctl_table *table, 936 int write, void *buffer, 937 size_t *length, loff_t *ppos) 938{ 939 int ret; 940 941 guard(mutex)(&kprobe_sysctl_mutex); 942 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0; 943 ret = proc_dointvec_minmax(table, write, buffer, length, ppos); 944 945 if (sysctl_kprobes_optimization) 946 optimize_all_kprobes(); 947 else 948 unoptimize_all_kprobes(); 949 950 return ret; 951} 952 953static const struct ctl_table kprobe_sysctls[] = { 954 { 955 .procname = "kprobes-optimization", 956 .data = &sysctl_kprobes_optimization, 957 .maxlen = sizeof(int), 958 .mode = 0644, 959 .proc_handler = proc_kprobes_optimization_handler, 960 .extra1 = SYSCTL_ZERO, 961 .extra2 = SYSCTL_ONE, 962 }, 963}; 964 965static void __init kprobe_sysctls_init(void) 966{ 967 register_sysctl_init("debug", kprobe_sysctls); 968} 969#endif /* CONFIG_SYSCTL */ 970 971/* Put a breakpoint for a probe. */ 972static void __arm_kprobe(struct kprobe *p) 973{ 974 struct kprobe *_p; 975 976 lockdep_assert_held(&text_mutex); 977 978 /* Find the overlapping optimized kprobes. */ 979 _p = get_optimized_kprobe(p->addr); 980 if (unlikely(_p)) 981 /* Fallback to unoptimized kprobe */ 982 unoptimize_kprobe(_p, true); 983 984 arch_arm_kprobe(p); 985 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */ 986} 987 988/* Remove the breakpoint of a probe. */ 989static void __disarm_kprobe(struct kprobe *p, bool reopt) 990{ 991 struct kprobe *_p; 992 993 lockdep_assert_held(&text_mutex); 994 995 /* Try to unoptimize */ 996 unoptimize_kprobe(p, kprobes_all_disarmed); 997 998 if (!kprobe_queued(p)) { 999 arch_disarm_kprobe(p); 1000 /* If another kprobe was blocked, re-optimize it. */ 1001 _p = get_optimized_kprobe(p->addr); 1002 if (unlikely(_p) && reopt) 1003 optimize_kprobe(_p); 1004 } 1005 /* 1006 * TODO: Since unoptimization and real disarming will be done by 1007 * the worker thread, we can not check whether another probe are 1008 * unoptimized because of this probe here. It should be re-optimized 1009 * by the worker thread. 1010 */ 1011} 1012 1013#else /* !CONFIG_OPTPROBES */ 1014 1015#define optimize_kprobe(p) do {} while (0) 1016#define unoptimize_kprobe(p, f) do {} while (0) 1017#define kill_optimized_kprobe(p) do {} while (0) 1018#define prepare_optimized_kprobe(p) do {} while (0) 1019#define try_to_optimize_kprobe(p) do {} while (0) 1020#define __arm_kprobe(p) arch_arm_kprobe(p) 1021#define __disarm_kprobe(p, o) arch_disarm_kprobe(p) 1022#define kprobe_disarmed(p) kprobe_disabled(p) 1023#define wait_for_kprobe_optimizer_locked() \ 1024 lockdep_assert_held(&kprobe_mutex) 1025 1026static int reuse_unused_kprobe(struct kprobe *ap) 1027{ 1028 /* 1029 * If the optimized kprobe is NOT supported, the aggr kprobe is 1030 * released at the same time that the last aggregated kprobe is 1031 * unregistered. 1032 * Thus there should be no chance to reuse unused kprobe. 1033 */ 1034 WARN_ON_ONCE(1); 1035 return -EINVAL; 1036} 1037 1038static void free_aggr_kprobe(struct kprobe *p) 1039{ 1040 arch_remove_kprobe(p); 1041 kfree(p); 1042} 1043 1044static struct kprobe *alloc_aggr_kprobe(struct kprobe *p) 1045{ 1046 return kzalloc(sizeof(struct kprobe), GFP_KERNEL); 1047} 1048#endif /* CONFIG_OPTPROBES */ 1049 1050#ifdef CONFIG_KPROBES_ON_FTRACE 1051static struct ftrace_ops kprobe_ftrace_ops __read_mostly = { 1052 .func = kprobe_ftrace_handler, 1053 .flags = FTRACE_OPS_FL_SAVE_REGS, 1054}; 1055 1056static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = { 1057 .func = kprobe_ftrace_handler, 1058 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY, 1059}; 1060 1061static int kprobe_ipmodify_enabled; 1062static int kprobe_ftrace_enabled; 1063bool kprobe_ftrace_disabled; 1064 1065static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, 1066 int *cnt) 1067{ 1068 int ret; 1069 1070 lockdep_assert_held(&kprobe_mutex); 1071 1072 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0); 1073 if (ret < 0) 1074 return ret; 1075 1076 if (*cnt == 0) { 1077 ret = register_ftrace_function(ops); 1078 if (ret < 0) { 1079 /* 1080 * At this point, sinec ops is not registered, we should be sefe from 1081 * registering empty filter. 1082 */ 1083 ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); 1084 return ret; 1085 } 1086 } 1087 1088 (*cnt)++; 1089 return ret; 1090} 1091 1092static int arm_kprobe_ftrace(struct kprobe *p) 1093{ 1094 bool ipmodify = (p->post_handler != NULL); 1095 1096 return __arm_kprobe_ftrace(p, 1097 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops, 1098 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); 1099} 1100 1101static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops, 1102 int *cnt) 1103{ 1104 int ret; 1105 1106 lockdep_assert_held(&kprobe_mutex); 1107 if (unlikely(kprobe_ftrace_disabled)) { 1108 /* Now ftrace is disabled forever, disarm is already done. */ 1109 return 0; 1110 } 1111 1112 if (*cnt == 1) { 1113 ret = unregister_ftrace_function(ops); 1114 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret)) 1115 return ret; 1116 } 1117 1118 (*cnt)--; 1119 1120 ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0); 1121 WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n", 1122 p->addr, ret); 1123 return ret; 1124} 1125 1126static int disarm_kprobe_ftrace(struct kprobe *p) 1127{ 1128 bool ipmodify = (p->post_handler != NULL); 1129 1130 return __disarm_kprobe_ftrace(p, 1131 ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops, 1132 ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled); 1133} 1134 1135void kprobe_ftrace_kill(void) 1136{ 1137 kprobe_ftrace_disabled = true; 1138} 1139#else /* !CONFIG_KPROBES_ON_FTRACE */ 1140static inline int arm_kprobe_ftrace(struct kprobe *p) 1141{ 1142 return -ENODEV; 1143} 1144 1145static inline int disarm_kprobe_ftrace(struct kprobe *p) 1146{ 1147 return -ENODEV; 1148} 1149#endif 1150 1151static int prepare_kprobe(struct kprobe *p) 1152{ 1153 /* Must ensure p->addr is really on ftrace */ 1154 if (kprobe_ftrace(p)) 1155 return arch_prepare_kprobe_ftrace(p); 1156 1157 return arch_prepare_kprobe(p); 1158} 1159 1160static int arm_kprobe(struct kprobe *kp) 1161{ 1162 if (unlikely(kprobe_ftrace(kp))) 1163 return arm_kprobe_ftrace(kp); 1164 1165 guard(cpus_read_lock)(); 1166 guard(mutex)(&text_mutex); 1167 __arm_kprobe(kp); 1168 return 0; 1169} 1170 1171static int disarm_kprobe(struct kprobe *kp, bool reopt) 1172{ 1173 if (unlikely(kprobe_ftrace(kp))) 1174 return disarm_kprobe_ftrace(kp); 1175 1176 guard(cpus_read_lock)(); 1177 guard(mutex)(&text_mutex); 1178 __disarm_kprobe(kp, reopt); 1179 return 0; 1180} 1181 1182/* 1183 * Aggregate handlers for multiple kprobes support - these handlers 1184 * take care of invoking the individual kprobe handlers on p->list 1185 */ 1186static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 1187{ 1188 struct kprobe *kp; 1189 1190 list_for_each_entry_rcu(kp, &p->list, list) { 1191 if (kp->pre_handler && likely(!kprobe_disabled(kp))) { 1192 set_kprobe_instance(kp); 1193 if (kp->pre_handler(kp, regs)) 1194 return 1; 1195 } 1196 reset_kprobe_instance(); 1197 } 1198 return 0; 1199} 1200NOKPROBE_SYMBOL(aggr_pre_handler); 1201 1202static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 1203 unsigned long flags) 1204{ 1205 struct kprobe *kp; 1206 1207 list_for_each_entry_rcu(kp, &p->list, list) { 1208 if (kp->post_handler && likely(!kprobe_disabled(kp))) { 1209 set_kprobe_instance(kp); 1210 kp->post_handler(kp, regs, flags); 1211 reset_kprobe_instance(); 1212 } 1213 } 1214} 1215NOKPROBE_SYMBOL(aggr_post_handler); 1216 1217/* Walks the list and increments 'nmissed' if 'p' has child probes. */ 1218void kprobes_inc_nmissed_count(struct kprobe *p) 1219{ 1220 struct kprobe *kp; 1221 1222 if (!kprobe_aggrprobe(p)) { 1223 p->nmissed++; 1224 } else { 1225 list_for_each_entry_rcu(kp, &p->list, list) 1226 kp->nmissed++; 1227 } 1228} 1229NOKPROBE_SYMBOL(kprobes_inc_nmissed_count); 1230 1231static struct kprobe kprobe_busy = { 1232 .addr = (void *) get_kprobe, 1233}; 1234 1235void kprobe_busy_begin(void) 1236{ 1237 struct kprobe_ctlblk *kcb; 1238 1239 preempt_disable(); 1240 __this_cpu_write(current_kprobe, &kprobe_busy); 1241 kcb = get_kprobe_ctlblk(); 1242 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 1243} 1244 1245void kprobe_busy_end(void) 1246{ 1247 __this_cpu_write(current_kprobe, NULL); 1248 preempt_enable(); 1249} 1250 1251/* Add the new probe to 'ap->list'. */ 1252static int add_new_kprobe(struct kprobe *ap, struct kprobe *p) 1253{ 1254 if (p->post_handler) 1255 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */ 1256 1257 list_add_rcu(&p->list, &ap->list); 1258 if (p->post_handler && !ap->post_handler) 1259 ap->post_handler = aggr_post_handler; 1260 1261 return 0; 1262} 1263 1264/* 1265 * Fill in the required fields of the aggregator kprobe. Replace the 1266 * earlier kprobe in the hlist with the aggregator kprobe. 1267 */ 1268static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 1269{ 1270 /* Copy the insn slot of 'p' to 'ap'. */ 1271 copy_kprobe(p, ap); 1272 flush_insn_slot(ap); 1273 ap->addr = p->addr; 1274 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED; 1275 ap->pre_handler = aggr_pre_handler; 1276 /* We don't care the kprobe which has gone. */ 1277 if (p->post_handler && !kprobe_gone(p)) 1278 ap->post_handler = aggr_post_handler; 1279 1280 INIT_LIST_HEAD(&ap->list); 1281 INIT_HLIST_NODE(&ap->hlist); 1282 1283 list_add_rcu(&p->list, &ap->list); 1284 hlist_replace_rcu(&p->hlist, &ap->hlist); 1285} 1286 1287/* 1288 * This registers the second or subsequent kprobe at the same address. 1289 */ 1290static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p) 1291{ 1292 int ret = 0; 1293 struct kprobe *ap = orig_p; 1294 1295 scoped_guard(cpus_read_lock) { 1296 /* For preparing optimization, jump_label_text_reserved() is called */ 1297 guard(jump_label_lock)(); 1298 guard(mutex)(&text_mutex); 1299 1300 if (!kprobe_aggrprobe(orig_p)) { 1301 /* If 'orig_p' is not an 'aggr_kprobe', create new one. */ 1302 ap = alloc_aggr_kprobe(orig_p); 1303 if (!ap) 1304 return -ENOMEM; 1305 init_aggr_kprobe(ap, orig_p); 1306 } else if (kprobe_unused(ap)) { 1307 /* This probe is going to die. Rescue it */ 1308 ret = reuse_unused_kprobe(ap); 1309 if (ret) 1310 return ret; 1311 } 1312 1313 if (kprobe_gone(ap)) { 1314 /* 1315 * Attempting to insert new probe at the same location that 1316 * had a probe in the module vaddr area which already 1317 * freed. So, the instruction slot has already been 1318 * released. We need a new slot for the new probe. 1319 */ 1320 ret = arch_prepare_kprobe(ap); 1321 if (ret) 1322 /* 1323 * Even if fail to allocate new slot, don't need to 1324 * free the 'ap'. It will be used next time, or 1325 * freed by unregister_kprobe(). 1326 */ 1327 return ret; 1328 1329 /* Prepare optimized instructions if possible. */ 1330 prepare_optimized_kprobe(ap); 1331 1332 /* 1333 * Clear gone flag to prevent allocating new slot again, and 1334 * set disabled flag because it is not armed yet. 1335 */ 1336 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE) 1337 | KPROBE_FLAG_DISABLED; 1338 } 1339 1340 /* Copy the insn slot of 'p' to 'ap'. */ 1341 copy_kprobe(ap, p); 1342 ret = add_new_kprobe(ap, p); 1343 } 1344 1345 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) { 1346 ap->flags &= ~KPROBE_FLAG_DISABLED; 1347 if (!kprobes_all_disarmed) { 1348 /* Arm the breakpoint again. */ 1349 ret = arm_kprobe(ap); 1350 if (ret) { 1351 ap->flags |= KPROBE_FLAG_DISABLED; 1352 list_del_rcu(&p->list); 1353 synchronize_rcu(); 1354 } 1355 } 1356 } 1357 return ret; 1358} 1359 1360bool __weak arch_within_kprobe_blacklist(unsigned long addr) 1361{ 1362 /* The '__kprobes' functions and entry code must not be probed. */ 1363 return addr >= (unsigned long)__kprobes_text_start && 1364 addr < (unsigned long)__kprobes_text_end; 1365} 1366 1367static bool __within_kprobe_blacklist(unsigned long addr) 1368{ 1369 struct kprobe_blacklist_entry *ent; 1370 1371 if (arch_within_kprobe_blacklist(addr)) 1372 return true; 1373 /* 1374 * If 'kprobe_blacklist' is defined, check the address and 1375 * reject any probe registration in the prohibited area. 1376 */ 1377 list_for_each_entry(ent, &kprobe_blacklist, list) { 1378 if (addr >= ent->start_addr && addr < ent->end_addr) 1379 return true; 1380 } 1381 return false; 1382} 1383 1384bool within_kprobe_blacklist(unsigned long addr) 1385{ 1386 char symname[KSYM_NAME_LEN], *p; 1387 1388 if (__within_kprobe_blacklist(addr)) 1389 return true; 1390 1391 /* Check if the address is on a suffixed-symbol */ 1392 if (!lookup_symbol_name(addr, symname)) { 1393 p = strchr(symname, '.'); 1394 if (!p) 1395 return false; 1396 *p = '\0'; 1397 addr = (unsigned long)kprobe_lookup_name(symname, 0); 1398 if (addr) 1399 return __within_kprobe_blacklist(addr); 1400 } 1401 return false; 1402} 1403 1404/* 1405 * arch_adjust_kprobe_addr - adjust the address 1406 * @addr: symbol base address 1407 * @offset: offset within the symbol 1408 * @on_func_entry: was this @addr+@offset on the function entry 1409 * 1410 * Typically returns @addr + @offset, except for special cases where the 1411 * function might be prefixed by a CFI landing pad, in that case any offset 1412 * inside the landing pad is mapped to the first 'real' instruction of the 1413 * symbol. 1414 * 1415 * Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C 1416 * instruction at +0. 1417 */ 1418kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr, 1419 unsigned long offset, 1420 bool *on_func_entry) 1421{ 1422 *on_func_entry = !offset; 1423 return (kprobe_opcode_t *)(addr + offset); 1424} 1425 1426/* 1427 * If 'symbol_name' is specified, look it up and add the 'offset' 1428 * to it. This way, we can specify a relative address to a symbol. 1429 * This returns encoded errors if it fails to look up symbol or invalid 1430 * combination of parameters. 1431 */ 1432static kprobe_opcode_t * 1433_kprobe_addr(kprobe_opcode_t *addr, const char *symbol_name, 1434 unsigned long offset, bool *on_func_entry) 1435{ 1436 if ((symbol_name && addr) || (!symbol_name && !addr)) 1437 return ERR_PTR(-EINVAL); 1438 1439 if (symbol_name) { 1440 /* 1441 * Input: @sym + @offset 1442 * Output: @addr + @offset 1443 * 1444 * NOTE: kprobe_lookup_name() does *NOT* fold the offset 1445 * argument into it's output! 1446 */ 1447 addr = kprobe_lookup_name(symbol_name, offset); 1448 if (!addr) 1449 return ERR_PTR(-ENOENT); 1450 } 1451 1452 /* 1453 * So here we have @addr + @offset, displace it into a new 1454 * @addr' + @offset' where @addr' is the symbol start address. 1455 */ 1456 addr = (void *)addr + offset; 1457 if (!kallsyms_lookup_size_offset((unsigned long)addr, NULL, &offset)) 1458 return ERR_PTR(-ENOENT); 1459 addr = (void *)addr - offset; 1460 1461 /* 1462 * Then ask the architecture to re-combine them, taking care of 1463 * magical function entry details while telling us if this was indeed 1464 * at the start of the function. 1465 */ 1466 addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry); 1467 if (!addr) 1468 return ERR_PTR(-EINVAL); 1469 1470 return addr; 1471} 1472 1473static kprobe_opcode_t *kprobe_addr(struct kprobe *p) 1474{ 1475 bool on_func_entry; 1476 1477 return _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry); 1478} 1479 1480/* 1481 * Check the 'p' is valid and return the aggregator kprobe 1482 * at the same address. 1483 */ 1484static struct kprobe *__get_valid_kprobe(struct kprobe *p) 1485{ 1486 struct kprobe *ap, *list_p; 1487 1488 lockdep_assert_held(&kprobe_mutex); 1489 1490 ap = get_kprobe(p->addr); 1491 if (unlikely(!ap)) 1492 return NULL; 1493 1494 if (p == ap) 1495 return ap; 1496 1497 list_for_each_entry(list_p, &ap->list, list) 1498 if (list_p == p) 1499 /* kprobe p is a valid probe */ 1500 return ap; 1501 1502 return NULL; 1503} 1504 1505/* 1506 * Warn and return error if the kprobe is being re-registered since 1507 * there must be a software bug. 1508 */ 1509static inline int warn_kprobe_rereg(struct kprobe *p) 1510{ 1511 guard(mutex)(&kprobe_mutex); 1512 1513 if (WARN_ON_ONCE(__get_valid_kprobe(p))) 1514 return -EINVAL; 1515 1516 return 0; 1517} 1518 1519static int check_ftrace_location(struct kprobe *p) 1520{ 1521 unsigned long addr = (unsigned long)p->addr; 1522 1523 if (ftrace_location(addr) == addr) { 1524#ifdef CONFIG_KPROBES_ON_FTRACE 1525 p->flags |= KPROBE_FLAG_FTRACE; 1526#else 1527 return -EINVAL; 1528#endif 1529 } 1530 return 0; 1531} 1532 1533static bool is_cfi_preamble_symbol(unsigned long addr) 1534{ 1535 char symbuf[KSYM_NAME_LEN]; 1536 1537 if (lookup_symbol_name(addr, symbuf)) 1538 return false; 1539 1540 return str_has_prefix(symbuf, "__cfi_") || 1541 str_has_prefix(symbuf, "__pfx_"); 1542} 1543 1544static int check_kprobe_address_safe(struct kprobe *p, 1545 struct module **probed_mod) 1546{ 1547 int ret; 1548 1549 ret = check_ftrace_location(p); 1550 if (ret) 1551 return ret; 1552 1553 guard(jump_label_lock)(); 1554 1555 /* Ensure the address is in a text area, and find a module if exists. */ 1556 *probed_mod = NULL; 1557 if (!core_kernel_text((unsigned long) p->addr)) { 1558 guard(rcu)(); 1559 *probed_mod = __module_text_address((unsigned long) p->addr); 1560 if (!(*probed_mod)) 1561 return -EINVAL; 1562 1563 /* 1564 * We must hold a refcount of the probed module while updating 1565 * its code to prohibit unexpected unloading. 1566 */ 1567 if (unlikely(!try_module_get(*probed_mod))) 1568 return -ENOENT; 1569 } 1570 /* Ensure it is not in reserved area. */ 1571 if (in_gate_area_no_mm((unsigned long) p->addr) || 1572 within_kprobe_blacklist((unsigned long) p->addr) || 1573 jump_label_text_reserved(p->addr, p->addr) || 1574 static_call_text_reserved(p->addr, p->addr) || 1575 find_bug((unsigned long)p->addr) || 1576 is_cfi_preamble_symbol((unsigned long)p->addr)) { 1577 module_put(*probed_mod); 1578 return -EINVAL; 1579 } 1580 1581 /* Get module refcount and reject __init functions for loaded modules. */ 1582 if (IS_ENABLED(CONFIG_MODULES) && *probed_mod) { 1583 /* 1584 * If the module freed '.init.text', we couldn't insert 1585 * kprobes in there. 1586 */ 1587 if (within_module_init((unsigned long)p->addr, *probed_mod) && 1588 !module_is_coming(*probed_mod)) { 1589 module_put(*probed_mod); 1590 return -ENOENT; 1591 } 1592 } 1593 1594 return 0; 1595} 1596 1597static int __register_kprobe(struct kprobe *p) 1598{ 1599 int ret; 1600 struct kprobe *old_p; 1601 1602 guard(mutex)(&kprobe_mutex); 1603 1604 old_p = get_kprobe(p->addr); 1605 if (old_p) 1606 /* Since this may unoptimize 'old_p', locking 'text_mutex'. */ 1607 return register_aggr_kprobe(old_p, p); 1608 1609 scoped_guard(cpus_read_lock) { 1610 /* Prevent text modification */ 1611 guard(mutex)(&text_mutex); 1612 ret = prepare_kprobe(p); 1613 if (ret) 1614 return ret; 1615 } 1616 1617 INIT_HLIST_NODE(&p->hlist); 1618 hlist_add_head_rcu(&p->hlist, 1619 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 1620 1621 if (!kprobes_all_disarmed && !kprobe_disabled(p)) { 1622 ret = arm_kprobe(p); 1623 if (ret) { 1624 hlist_del_rcu(&p->hlist); 1625 synchronize_rcu(); 1626 } 1627 } 1628 1629 /* Try to optimize kprobe */ 1630 try_to_optimize_kprobe(p); 1631 return 0; 1632} 1633 1634int register_kprobe(struct kprobe *p) 1635{ 1636 int ret; 1637 struct module *probed_mod; 1638 kprobe_opcode_t *addr; 1639 bool on_func_entry; 1640 1641 /* Canonicalize probe address from symbol */ 1642 addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry); 1643 if (IS_ERR(addr)) 1644 return PTR_ERR(addr); 1645 p->addr = addr; 1646 1647 ret = warn_kprobe_rereg(p); 1648 if (ret) 1649 return ret; 1650 1651 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ 1652 p->flags &= KPROBE_FLAG_DISABLED; 1653 if (on_func_entry) 1654 p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY; 1655 p->nmissed = 0; 1656 INIT_LIST_HEAD(&p->list); 1657 1658 ret = check_kprobe_address_safe(p, &probed_mod); 1659 if (ret) 1660 return ret; 1661 1662 ret = __register_kprobe(p); 1663 1664 if (probed_mod) 1665 module_put(probed_mod); 1666 1667 return ret; 1668} 1669EXPORT_SYMBOL_GPL(register_kprobe); 1670 1671/* Check if all probes on the 'ap' are disabled. */ 1672static bool aggr_kprobe_disabled(struct kprobe *ap) 1673{ 1674 struct kprobe *kp; 1675 1676 lockdep_assert_held(&kprobe_mutex); 1677 1678 list_for_each_entry(kp, &ap->list, list) 1679 if (!kprobe_disabled(kp)) 1680 /* 1681 * Since there is an active probe on the list, 1682 * we can't disable this 'ap'. 1683 */ 1684 return false; 1685 1686 return true; 1687} 1688 1689static struct kprobe *__disable_kprobe(struct kprobe *p) 1690{ 1691 struct kprobe *orig_p; 1692 int ret; 1693 1694 lockdep_assert_held(&kprobe_mutex); 1695 1696 /* Get an original kprobe for return */ 1697 orig_p = __get_valid_kprobe(p); 1698 if (unlikely(orig_p == NULL)) 1699 return ERR_PTR(-EINVAL); 1700 1701 if (kprobe_disabled(p)) 1702 return orig_p; 1703 1704 /* Disable probe if it is a child probe */ 1705 if (p != orig_p) 1706 p->flags |= KPROBE_FLAG_DISABLED; 1707 1708 /* Try to disarm and disable this/parent probe */ 1709 if (p == orig_p || aggr_kprobe_disabled(orig_p)) { 1710 /* 1711 * Don't be lazy here. Even if 'kprobes_all_disarmed' 1712 * is false, 'orig_p' might not have been armed yet. 1713 * Note arm_all_kprobes() __tries__ to arm all kprobes 1714 * on the best effort basis. 1715 */ 1716 if (!kprobes_all_disarmed && !kprobe_disabled(orig_p)) { 1717 ret = disarm_kprobe(orig_p, true); 1718 if (ret) { 1719 p->flags &= ~KPROBE_FLAG_DISABLED; 1720 return ERR_PTR(ret); 1721 } 1722 } 1723 orig_p->flags |= KPROBE_FLAG_DISABLED; 1724 } 1725 1726 return orig_p; 1727} 1728 1729/* 1730 * Unregister a kprobe without a scheduler synchronization. 1731 */ 1732static int __unregister_kprobe_top(struct kprobe *p) 1733{ 1734 struct kprobe *ap, *list_p; 1735 1736 /* Disable kprobe. This will disarm it if needed. */ 1737 ap = __disable_kprobe(p); 1738 if (IS_ERR(ap)) 1739 return PTR_ERR(ap); 1740 1741 WARN_ON(ap != p && !kprobe_aggrprobe(ap)); 1742 1743 /* 1744 * If the probe is an independent(and non-optimized) kprobe 1745 * (not an aggrprobe), the last kprobe on the aggrprobe, or 1746 * kprobe is already disarmed, just remove from the hash list. 1747 */ 1748 if (ap == p || 1749 (list_is_singular(&ap->list) && kprobe_disarmed(ap))) { 1750 /* 1751 * !disarmed could be happen if the probe is under delayed 1752 * unoptimizing. 1753 */ 1754 hlist_del_rcu(&ap->hlist); 1755 return 0; 1756 } 1757 1758 /* If disabling probe has special handlers, update aggrprobe */ 1759 if (p->post_handler && !kprobe_gone(p)) { 1760 list_for_each_entry(list_p, &ap->list, list) { 1761 if ((list_p != p) && (list_p->post_handler)) 1762 break; 1763 } 1764 /* No other probe has post_handler */ 1765 if (list_entry_is_head(list_p, &ap->list, list)) { 1766 /* 1767 * For the kprobe-on-ftrace case, we keep the 1768 * post_handler setting to identify this aggrprobe 1769 * armed with kprobe_ipmodify_ops. 1770 */ 1771 if (!kprobe_ftrace(ap)) 1772 ap->post_handler = NULL; 1773 } 1774 } 1775 1776 /* 1777 * Remove from the aggrprobe: this path will do nothing in 1778 * __unregister_kprobe_bottom(). 1779 */ 1780 list_del_rcu(&p->list); 1781 if (!kprobe_disabled(ap) && !kprobes_all_disarmed) 1782 /* 1783 * Try to optimize this probe again, because post 1784 * handler may have been changed. 1785 */ 1786 optimize_kprobe(ap); 1787 return 0; 1788 1789} 1790 1791static void __unregister_kprobe_bottom(struct kprobe *p) 1792{ 1793 struct kprobe *ap; 1794 1795 if (list_empty(&p->list)) 1796 /* This is an independent kprobe */ 1797 arch_remove_kprobe(p); 1798 else if (list_is_singular(&p->list)) { 1799 /* This is the last child of an aggrprobe */ 1800 ap = list_entry(p->list.next, struct kprobe, list); 1801 list_del(&p->list); 1802 free_aggr_kprobe(ap); 1803 } 1804 /* Otherwise, do nothing. */ 1805} 1806 1807int register_kprobes(struct kprobe **kps, int num) 1808{ 1809 int i, ret = 0; 1810 1811 if (num <= 0) 1812 return -EINVAL; 1813 for (i = 0; i < num; i++) { 1814 ret = register_kprobe(kps[i]); 1815 if (ret < 0) { 1816 if (i > 0) 1817 unregister_kprobes(kps, i); 1818 break; 1819 } 1820 } 1821 return ret; 1822} 1823EXPORT_SYMBOL_GPL(register_kprobes); 1824 1825void unregister_kprobe(struct kprobe *p) 1826{ 1827 unregister_kprobes(&p, 1); 1828} 1829EXPORT_SYMBOL_GPL(unregister_kprobe); 1830 1831void unregister_kprobes(struct kprobe **kps, int num) 1832{ 1833 int i; 1834 1835 if (num <= 0) 1836 return; 1837 scoped_guard(mutex, &kprobe_mutex) { 1838 for (i = 0; i < num; i++) 1839 if (__unregister_kprobe_top(kps[i]) < 0) 1840 kps[i]->addr = NULL; 1841 } 1842 synchronize_rcu(); 1843 for (i = 0; i < num; i++) 1844 if (kps[i]->addr) 1845 __unregister_kprobe_bottom(kps[i]); 1846} 1847EXPORT_SYMBOL_GPL(unregister_kprobes); 1848 1849int __weak kprobe_exceptions_notify(struct notifier_block *self, 1850 unsigned long val, void *data) 1851{ 1852 return NOTIFY_DONE; 1853} 1854NOKPROBE_SYMBOL(kprobe_exceptions_notify); 1855 1856static struct notifier_block kprobe_exceptions_nb = { 1857 .notifier_call = kprobe_exceptions_notify, 1858 .priority = 0x7fffffff /* we need to be notified first */ 1859}; 1860 1861#ifdef CONFIG_KRETPROBES 1862 1863#if !defined(CONFIG_KRETPROBE_ON_RETHOOK) 1864 1865/* callbacks for objpool of kretprobe instances */ 1866static int kretprobe_init_inst(void *nod, void *context) 1867{ 1868 struct kretprobe_instance *ri = nod; 1869 1870 ri->rph = context; 1871 return 0; 1872} 1873static int kretprobe_fini_pool(struct objpool_head *head, void *context) 1874{ 1875 kfree(context); 1876 return 0; 1877} 1878 1879static void free_rp_inst_rcu(struct rcu_head *head) 1880{ 1881 struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu); 1882 struct kretprobe_holder *rph = ri->rph; 1883 1884 objpool_drop(ri, &rph->pool); 1885} 1886NOKPROBE_SYMBOL(free_rp_inst_rcu); 1887 1888static void recycle_rp_inst(struct kretprobe_instance *ri) 1889{ 1890 struct kretprobe *rp = get_kretprobe(ri); 1891 1892 if (likely(rp)) 1893 objpool_push(ri, &rp->rph->pool); 1894 else 1895 call_rcu(&ri->rcu, free_rp_inst_rcu); 1896} 1897NOKPROBE_SYMBOL(recycle_rp_inst); 1898 1899/* 1900 * This function is called from delayed_put_task_struct() when a task is 1901 * dead and cleaned up to recycle any kretprobe instances associated with 1902 * this task. These left over instances represent probed functions that 1903 * have been called but will never return. 1904 */ 1905void kprobe_flush_task(struct task_struct *tk) 1906{ 1907 struct kretprobe_instance *ri; 1908 struct llist_node *node; 1909 1910 /* Early boot, not yet initialized. */ 1911 if (unlikely(!kprobes_initialized)) 1912 return; 1913 1914 kprobe_busy_begin(); 1915 1916 node = __llist_del_all(&tk->kretprobe_instances); 1917 while (node) { 1918 ri = container_of(node, struct kretprobe_instance, llist); 1919 node = node->next; 1920 1921 recycle_rp_inst(ri); 1922 } 1923 1924 kprobe_busy_end(); 1925} 1926NOKPROBE_SYMBOL(kprobe_flush_task); 1927 1928static inline void free_rp_inst(struct kretprobe *rp) 1929{ 1930 struct kretprobe_holder *rph = rp->rph; 1931 1932 if (!rph) 1933 return; 1934 rp->rph = NULL; 1935 objpool_fini(&rph->pool); 1936} 1937 1938/* This assumes the 'tsk' is the current task or the is not running. */ 1939static kprobe_opcode_t *__kretprobe_find_ret_addr(struct task_struct *tsk, 1940 struct llist_node **cur) 1941{ 1942 struct kretprobe_instance *ri = NULL; 1943 struct llist_node *node = *cur; 1944 1945 if (!node) 1946 node = tsk->kretprobe_instances.first; 1947 else 1948 node = node->next; 1949 1950 while (node) { 1951 ri = container_of(node, struct kretprobe_instance, llist); 1952 if (ri->ret_addr != kretprobe_trampoline_addr()) { 1953 *cur = node; 1954 return ri->ret_addr; 1955 } 1956 node = node->next; 1957 } 1958 return NULL; 1959} 1960NOKPROBE_SYMBOL(__kretprobe_find_ret_addr); 1961 1962/** 1963 * kretprobe_find_ret_addr -- Find correct return address modified by kretprobe 1964 * @tsk: Target task 1965 * @fp: A frame pointer 1966 * @cur: a storage of the loop cursor llist_node pointer for next call 1967 * 1968 * Find the correct return address modified by a kretprobe on @tsk in unsigned 1969 * long type. If it finds the return address, this returns that address value, 1970 * or this returns 0. 1971 * The @tsk must be 'current' or a task which is not running. @fp is a hint 1972 * to get the currect return address - which is compared with the 1973 * kretprobe_instance::fp field. The @cur is a loop cursor for searching the 1974 * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the 1975 * first call, but '@cur' itself must NOT NULL. 1976 */ 1977unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp, 1978 struct llist_node **cur) 1979{ 1980 struct kretprobe_instance *ri; 1981 kprobe_opcode_t *ret; 1982 1983 if (WARN_ON_ONCE(!cur)) 1984 return 0; 1985 1986 do { 1987 ret = __kretprobe_find_ret_addr(tsk, cur); 1988 if (!ret) 1989 break; 1990 ri = container_of(*cur, struct kretprobe_instance, llist); 1991 } while (ri->fp != fp); 1992 1993 return (unsigned long)ret; 1994} 1995NOKPROBE_SYMBOL(kretprobe_find_ret_addr); 1996 1997void __weak arch_kretprobe_fixup_return(struct pt_regs *regs, 1998 kprobe_opcode_t *correct_ret_addr) 1999{ 2000 /* 2001 * Do nothing by default. Please fill this to update the fake return 2002 * address on the stack with the correct one on each arch if possible. 2003 */ 2004} 2005 2006unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs, 2007 void *frame_pointer) 2008{ 2009 struct kretprobe_instance *ri = NULL; 2010 struct llist_node *first, *node = NULL; 2011 kprobe_opcode_t *correct_ret_addr; 2012 struct kretprobe *rp; 2013 2014 /* Find correct address and all nodes for this frame. */ 2015 correct_ret_addr = __kretprobe_find_ret_addr(current, &node); 2016 if (!correct_ret_addr) { 2017 pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n"); 2018 BUG_ON(1); 2019 } 2020 2021 /* 2022 * Set the return address as the instruction pointer, because if the 2023 * user handler calls stack_trace_save_regs() with this 'regs', 2024 * the stack trace will start from the instruction pointer. 2025 */ 2026 instruction_pointer_set(regs, (unsigned long)correct_ret_addr); 2027 2028 /* Run the user handler of the nodes. */ 2029 first = current->kretprobe_instances.first; 2030 while (first) { 2031 ri = container_of(first, struct kretprobe_instance, llist); 2032 2033 if (WARN_ON_ONCE(ri->fp != frame_pointer)) 2034 break; 2035 2036 rp = get_kretprobe(ri); 2037 if (rp && rp->handler) { 2038 struct kprobe *prev = kprobe_running(); 2039 2040 __this_cpu_write(current_kprobe, &rp->kp); 2041 ri->ret_addr = correct_ret_addr; 2042 rp->handler(ri, regs); 2043 __this_cpu_write(current_kprobe, prev); 2044 } 2045 if (first == node) 2046 break; 2047 2048 first = first->next; 2049 } 2050 2051 arch_kretprobe_fixup_return(regs, correct_ret_addr); 2052 2053 /* Unlink all nodes for this frame. */ 2054 first = current->kretprobe_instances.first; 2055 current->kretprobe_instances.first = node->next; 2056 node->next = NULL; 2057 2058 /* Recycle free instances. */ 2059 while (first) { 2060 ri = container_of(first, struct kretprobe_instance, llist); 2061 first = first->next; 2062 2063 recycle_rp_inst(ri); 2064 } 2065 2066 return (unsigned long)correct_ret_addr; 2067} 2068NOKPROBE_SYMBOL(__kretprobe_trampoline_handler) 2069 2070/* 2071 * This kprobe pre_handler is registered with every kretprobe. When probe 2072 * hits it will set up the return probe. 2073 */ 2074static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) 2075{ 2076 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 2077 struct kretprobe_holder *rph = rp->rph; 2078 struct kretprobe_instance *ri; 2079 2080 ri = objpool_pop(&rph->pool); 2081 if (!ri) { 2082 rp->nmissed++; 2083 return 0; 2084 } 2085 2086 if (rp->entry_handler && rp->entry_handler(ri, regs)) { 2087 objpool_push(ri, &rph->pool); 2088 return 0; 2089 } 2090 2091 arch_prepare_kretprobe(ri, regs); 2092 2093 __llist_add(&ri->llist, ¤t->kretprobe_instances); 2094 2095 return 0; 2096} 2097NOKPROBE_SYMBOL(pre_handler_kretprobe); 2098#else /* CONFIG_KRETPROBE_ON_RETHOOK */ 2099/* 2100 * This kprobe pre_handler is registered with every kretprobe. When probe 2101 * hits it will set up the return probe. 2102 */ 2103static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) 2104{ 2105 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 2106 struct kretprobe_instance *ri; 2107 struct rethook_node *rhn; 2108 2109 rhn = rethook_try_get(rp->rh); 2110 if (!rhn) { 2111 rp->nmissed++; 2112 return 0; 2113 } 2114 2115 ri = container_of(rhn, struct kretprobe_instance, node); 2116 2117 if (rp->entry_handler && rp->entry_handler(ri, regs)) 2118 rethook_recycle(rhn); 2119 else 2120 rethook_hook(rhn, regs, kprobe_ftrace(p)); 2121 2122 return 0; 2123} 2124NOKPROBE_SYMBOL(pre_handler_kretprobe); 2125 2126static void kretprobe_rethook_handler(struct rethook_node *rh, void *data, 2127 unsigned long ret_addr, 2128 struct pt_regs *regs) 2129{ 2130 struct kretprobe *rp = (struct kretprobe *)data; 2131 struct kretprobe_instance *ri; 2132 struct kprobe_ctlblk *kcb; 2133 2134 /* The data must NOT be null. This means rethook data structure is broken. */ 2135 if (WARN_ON_ONCE(!data) || !rp->handler) 2136 return; 2137 2138 __this_cpu_write(current_kprobe, &rp->kp); 2139 kcb = get_kprobe_ctlblk(); 2140 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 2141 2142 ri = container_of(rh, struct kretprobe_instance, node); 2143 rp->handler(ri, regs); 2144 2145 __this_cpu_write(current_kprobe, NULL); 2146} 2147NOKPROBE_SYMBOL(kretprobe_rethook_handler); 2148 2149#endif /* !CONFIG_KRETPROBE_ON_RETHOOK */ 2150 2151/** 2152 * kprobe_on_func_entry() -- check whether given address is function entry 2153 * @addr: Target address 2154 * @sym: Target symbol name 2155 * @offset: The offset from the symbol or the address 2156 * 2157 * This checks whether the given @addr+@offset or @sym+@offset is on the 2158 * function entry address or not. 2159 * This returns 0 if it is the function entry, or -EINVAL if it is not. 2160 * And also it returns -ENOENT if it fails the symbol or address lookup. 2161 * Caller must pass @addr or @sym (either one must be NULL), or this 2162 * returns -EINVAL. 2163 */ 2164int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset) 2165{ 2166 bool on_func_entry; 2167 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset, &on_func_entry); 2168 2169 if (IS_ERR(kp_addr)) 2170 return PTR_ERR(kp_addr); 2171 2172 if (!on_func_entry) 2173 return -EINVAL; 2174 2175 return 0; 2176} 2177 2178int register_kretprobe(struct kretprobe *rp) 2179{ 2180 int ret; 2181 int i; 2182 void *addr; 2183 2184 ret = kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset); 2185 if (ret) 2186 return ret; 2187 2188 /* If only 'rp->kp.addr' is specified, check reregistering kprobes */ 2189 if (rp->kp.addr && warn_kprobe_rereg(&rp->kp)) 2190 return -EINVAL; 2191 2192 if (kretprobe_blacklist_size) { 2193 addr = kprobe_addr(&rp->kp); 2194 if (IS_ERR(addr)) 2195 return PTR_ERR(addr); 2196 2197 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 2198 if (kretprobe_blacklist[i].addr == addr) 2199 return -EINVAL; 2200 } 2201 } 2202 2203 if (rp->data_size > KRETPROBE_MAX_DATA_SIZE) 2204 return -E2BIG; 2205 2206 rp->kp.pre_handler = pre_handler_kretprobe; 2207 rp->kp.post_handler = NULL; 2208 2209 /* Pre-allocate memory for max kretprobe instances */ 2210 if (rp->maxactive <= 0) 2211 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus()); 2212 2213#ifdef CONFIG_KRETPROBE_ON_RETHOOK 2214 rp->rh = rethook_alloc((void *)rp, kretprobe_rethook_handler, 2215 sizeof(struct kretprobe_instance) + 2216 rp->data_size, rp->maxactive); 2217 if (IS_ERR(rp->rh)) 2218 return PTR_ERR(rp->rh); 2219 2220 rp->nmissed = 0; 2221 /* Establish function entry probe point */ 2222 ret = register_kprobe(&rp->kp); 2223 if (ret != 0) { 2224 rethook_free(rp->rh); 2225 rp->rh = NULL; 2226 } 2227#else /* !CONFIG_KRETPROBE_ON_RETHOOK */ 2228 rp->rph = kzalloc(sizeof(struct kretprobe_holder), GFP_KERNEL); 2229 if (!rp->rph) 2230 return -ENOMEM; 2231 2232 if (objpool_init(&rp->rph->pool, rp->maxactive, rp->data_size + 2233 sizeof(struct kretprobe_instance), GFP_KERNEL, 2234 rp->rph, kretprobe_init_inst, kretprobe_fini_pool)) { 2235 kfree(rp->rph); 2236 rp->rph = NULL; 2237 return -ENOMEM; 2238 } 2239 rcu_assign_pointer(rp->rph->rp, rp); 2240 rp->nmissed = 0; 2241 /* Establish function entry probe point */ 2242 ret = register_kprobe(&rp->kp); 2243 if (ret != 0) 2244 free_rp_inst(rp); 2245#endif 2246 return ret; 2247} 2248EXPORT_SYMBOL_GPL(register_kretprobe); 2249 2250int register_kretprobes(struct kretprobe **rps, int num) 2251{ 2252 int ret = 0, i; 2253 2254 if (num <= 0) 2255 return -EINVAL; 2256 for (i = 0; i < num; i++) { 2257 ret = register_kretprobe(rps[i]); 2258 if (ret < 0) { 2259 if (i > 0) 2260 unregister_kretprobes(rps, i); 2261 break; 2262 } 2263 } 2264 return ret; 2265} 2266EXPORT_SYMBOL_GPL(register_kretprobes); 2267 2268void unregister_kretprobe(struct kretprobe *rp) 2269{ 2270 unregister_kretprobes(&rp, 1); 2271} 2272EXPORT_SYMBOL_GPL(unregister_kretprobe); 2273 2274void unregister_kretprobes(struct kretprobe **rps, int num) 2275{ 2276 int i; 2277 2278 if (num <= 0) 2279 return; 2280 for (i = 0; i < num; i++) { 2281 guard(mutex)(&kprobe_mutex); 2282 2283 if (__unregister_kprobe_top(&rps[i]->kp) < 0) 2284 rps[i]->kp.addr = NULL; 2285#ifdef CONFIG_KRETPROBE_ON_RETHOOK 2286 rethook_free(rps[i]->rh); 2287#else 2288 rcu_assign_pointer(rps[i]->rph->rp, NULL); 2289#endif 2290 } 2291 2292 synchronize_rcu(); 2293 for (i = 0; i < num; i++) { 2294 if (rps[i]->kp.addr) { 2295 __unregister_kprobe_bottom(&rps[i]->kp); 2296#ifndef CONFIG_KRETPROBE_ON_RETHOOK 2297 free_rp_inst(rps[i]); 2298#endif 2299 } 2300 } 2301} 2302EXPORT_SYMBOL_GPL(unregister_kretprobes); 2303 2304#else /* CONFIG_KRETPROBES */ 2305int register_kretprobe(struct kretprobe *rp) 2306{ 2307 return -EOPNOTSUPP; 2308} 2309EXPORT_SYMBOL_GPL(register_kretprobe); 2310 2311int register_kretprobes(struct kretprobe **rps, int num) 2312{ 2313 return -EOPNOTSUPP; 2314} 2315EXPORT_SYMBOL_GPL(register_kretprobes); 2316 2317void unregister_kretprobe(struct kretprobe *rp) 2318{ 2319} 2320EXPORT_SYMBOL_GPL(unregister_kretprobe); 2321 2322void unregister_kretprobes(struct kretprobe **rps, int num) 2323{ 2324} 2325EXPORT_SYMBOL_GPL(unregister_kretprobes); 2326 2327static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs) 2328{ 2329 return 0; 2330} 2331NOKPROBE_SYMBOL(pre_handler_kretprobe); 2332 2333#endif /* CONFIG_KRETPROBES */ 2334 2335/* Set the kprobe gone and remove its instruction buffer. */ 2336static void kill_kprobe(struct kprobe *p) 2337{ 2338 struct kprobe *kp; 2339 2340 lockdep_assert_held(&kprobe_mutex); 2341 2342 /* 2343 * The module is going away. We should disarm the kprobe which 2344 * is using ftrace, because ftrace framework is still available at 2345 * 'MODULE_STATE_GOING' notification. 2346 */ 2347 if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed) 2348 disarm_kprobe_ftrace(p); 2349 2350 p->flags |= KPROBE_FLAG_GONE; 2351 if (kprobe_aggrprobe(p)) { 2352 /* 2353 * If this is an aggr_kprobe, we have to list all the 2354 * chained probes and mark them GONE. 2355 */ 2356 list_for_each_entry(kp, &p->list, list) 2357 kp->flags |= KPROBE_FLAG_GONE; 2358 p->post_handler = NULL; 2359 kill_optimized_kprobe(p); 2360 } 2361 /* 2362 * Here, we can remove insn_slot safely, because no thread calls 2363 * the original probed function (which will be freed soon) any more. 2364 */ 2365 arch_remove_kprobe(p); 2366} 2367 2368/* Disable one kprobe */ 2369int disable_kprobe(struct kprobe *kp) 2370{ 2371 struct kprobe *p; 2372 2373 guard(mutex)(&kprobe_mutex); 2374 2375 /* Disable this kprobe */ 2376 p = __disable_kprobe(kp); 2377 2378 return IS_ERR(p) ? PTR_ERR(p) : 0; 2379} 2380EXPORT_SYMBOL_GPL(disable_kprobe); 2381 2382/* Enable one kprobe */ 2383int enable_kprobe(struct kprobe *kp) 2384{ 2385 int ret = 0; 2386 struct kprobe *p; 2387 2388 guard(mutex)(&kprobe_mutex); 2389 2390 /* Check whether specified probe is valid. */ 2391 p = __get_valid_kprobe(kp); 2392 if (unlikely(p == NULL)) 2393 return -EINVAL; 2394 2395 if (kprobe_gone(kp)) 2396 /* This kprobe has gone, we couldn't enable it. */ 2397 return -EINVAL; 2398 2399 if (p != kp) 2400 kp->flags &= ~KPROBE_FLAG_DISABLED; 2401 2402 if (!kprobes_all_disarmed && kprobe_disabled(p)) { 2403 p->flags &= ~KPROBE_FLAG_DISABLED; 2404 ret = arm_kprobe(p); 2405 if (ret) { 2406 p->flags |= KPROBE_FLAG_DISABLED; 2407 if (p != kp) 2408 kp->flags |= KPROBE_FLAG_DISABLED; 2409 } 2410 } 2411 return ret; 2412} 2413EXPORT_SYMBOL_GPL(enable_kprobe); 2414 2415/* Caller must NOT call this in usual path. This is only for critical case */ 2416void dump_kprobe(struct kprobe *kp) 2417{ 2418 pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n", 2419 kp->symbol_name, kp->offset, kp->addr); 2420} 2421NOKPROBE_SYMBOL(dump_kprobe); 2422 2423int kprobe_add_ksym_blacklist(unsigned long entry) 2424{ 2425 struct kprobe_blacklist_entry *ent; 2426 unsigned long offset = 0, size = 0; 2427 2428 if (!kernel_text_address(entry) || 2429 !kallsyms_lookup_size_offset(entry, &size, &offset)) 2430 return -EINVAL; 2431 2432 ent = kmalloc(sizeof(*ent), GFP_KERNEL); 2433 if (!ent) 2434 return -ENOMEM; 2435 ent->start_addr = entry; 2436 ent->end_addr = entry + size; 2437 INIT_LIST_HEAD(&ent->list); 2438 list_add_tail(&ent->list, &kprobe_blacklist); 2439 2440 return (int)size; 2441} 2442 2443/* Add all symbols in given area into kprobe blacklist */ 2444int kprobe_add_area_blacklist(unsigned long start, unsigned long end) 2445{ 2446 unsigned long entry; 2447 int ret = 0; 2448 2449 for (entry = start; entry < end; entry += ret) { 2450 ret = kprobe_add_ksym_blacklist(entry); 2451 if (ret < 0) 2452 return ret; 2453 if (ret == 0) /* In case of alias symbol */ 2454 ret = 1; 2455 } 2456 return 0; 2457} 2458 2459int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value, 2460 char *type, char *sym) 2461{ 2462 return -ERANGE; 2463} 2464 2465int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type, 2466 char *sym) 2467{ 2468#ifdef __ARCH_WANT_KPROBES_INSN_SLOT 2469 if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym)) 2470 return 0; 2471#ifdef CONFIG_OPTPROBES 2472 if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym)) 2473 return 0; 2474#endif 2475#endif 2476 if (!arch_kprobe_get_kallsym(&symnum, value, type, sym)) 2477 return 0; 2478 return -ERANGE; 2479} 2480 2481int __init __weak arch_populate_kprobe_blacklist(void) 2482{ 2483 return 0; 2484} 2485 2486/* 2487 * Lookup and populate the kprobe_blacklist. 2488 * 2489 * Unlike the kretprobe blacklist, we'll need to determine 2490 * the range of addresses that belong to the said functions, 2491 * since a kprobe need not necessarily be at the beginning 2492 * of a function. 2493 */ 2494static int __init populate_kprobe_blacklist(unsigned long *start, 2495 unsigned long *end) 2496{ 2497 unsigned long entry; 2498 unsigned long *iter; 2499 int ret; 2500 2501 for (iter = start; iter < end; iter++) { 2502 entry = (unsigned long)dereference_symbol_descriptor((void *)*iter); 2503 ret = kprobe_add_ksym_blacklist(entry); 2504 if (ret == -EINVAL) 2505 continue; 2506 if (ret < 0) 2507 return ret; 2508 } 2509 2510 /* Symbols in '__kprobes_text' are blacklisted */ 2511 ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start, 2512 (unsigned long)__kprobes_text_end); 2513 if (ret) 2514 return ret; 2515 2516 /* Symbols in 'noinstr' section are blacklisted */ 2517 ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start, 2518 (unsigned long)__noinstr_text_end); 2519 2520 return ret ? : arch_populate_kprobe_blacklist(); 2521} 2522 2523#ifdef CONFIG_MODULES 2524/* Remove all symbols in given area from kprobe blacklist */ 2525static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end) 2526{ 2527 struct kprobe_blacklist_entry *ent, *n; 2528 2529 list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) { 2530 if (ent->start_addr < start || ent->start_addr >= end) 2531 continue; 2532 list_del(&ent->list); 2533 kfree(ent); 2534 } 2535} 2536 2537static void kprobe_remove_ksym_blacklist(unsigned long entry) 2538{ 2539 kprobe_remove_area_blacklist(entry, entry + 1); 2540} 2541 2542static void add_module_kprobe_blacklist(struct module *mod) 2543{ 2544 unsigned long start, end; 2545 int i; 2546 2547 if (mod->kprobe_blacklist) { 2548 for (i = 0; i < mod->num_kprobe_blacklist; i++) 2549 kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]); 2550 } 2551 2552 start = (unsigned long)mod->kprobes_text_start; 2553 if (start) { 2554 end = start + mod->kprobes_text_size; 2555 kprobe_add_area_blacklist(start, end); 2556 } 2557 2558 start = (unsigned long)mod->noinstr_text_start; 2559 if (start) { 2560 end = start + mod->noinstr_text_size; 2561 kprobe_add_area_blacklist(start, end); 2562 } 2563} 2564 2565static void remove_module_kprobe_blacklist(struct module *mod) 2566{ 2567 unsigned long start, end; 2568 int i; 2569 2570 if (mod->kprobe_blacklist) { 2571 for (i = 0; i < mod->num_kprobe_blacklist; i++) 2572 kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]); 2573 } 2574 2575 start = (unsigned long)mod->kprobes_text_start; 2576 if (start) { 2577 end = start + mod->kprobes_text_size; 2578 kprobe_remove_area_blacklist(start, end); 2579 } 2580 2581 start = (unsigned long)mod->noinstr_text_start; 2582 if (start) { 2583 end = start + mod->noinstr_text_size; 2584 kprobe_remove_area_blacklist(start, end); 2585 } 2586} 2587 2588/* Module notifier call back, checking kprobes on the module */ 2589static int kprobes_module_callback(struct notifier_block *nb, 2590 unsigned long val, void *data) 2591{ 2592 struct module *mod = data; 2593 struct hlist_head *head; 2594 struct kprobe *p; 2595 unsigned int i; 2596 int checkcore = (val == MODULE_STATE_GOING); 2597 2598 guard(mutex)(&kprobe_mutex); 2599 2600 if (val == MODULE_STATE_COMING) 2601 add_module_kprobe_blacklist(mod); 2602 2603 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE) 2604 return NOTIFY_DONE; 2605 2606 /* 2607 * When 'MODULE_STATE_GOING' was notified, both of module '.text' and 2608 * '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was 2609 * notified, only '.init.text' section would be freed. We need to 2610 * disable kprobes which have been inserted in the sections. 2611 */ 2612 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2613 head = &kprobe_table[i]; 2614 hlist_for_each_entry(p, head, hlist) 2615 if (within_module_init((unsigned long)p->addr, mod) || 2616 (checkcore && 2617 within_module_core((unsigned long)p->addr, mod))) { 2618 /* 2619 * The vaddr this probe is installed will soon 2620 * be vfreed buy not synced to disk. Hence, 2621 * disarming the breakpoint isn't needed. 2622 * 2623 * Note, this will also move any optimized probes 2624 * that are pending to be removed from their 2625 * corresponding lists to the 'freeing_list' and 2626 * will not be touched by the delayed 2627 * kprobe_optimizer() work handler. 2628 */ 2629 kill_kprobe(p); 2630 } 2631 } 2632 if (val == MODULE_STATE_GOING) 2633 remove_module_kprobe_blacklist(mod); 2634 return NOTIFY_DONE; 2635} 2636 2637static struct notifier_block kprobe_module_nb = { 2638 .notifier_call = kprobes_module_callback, 2639 .priority = 0 2640}; 2641 2642static int kprobe_register_module_notifier(void) 2643{ 2644 return register_module_notifier(&kprobe_module_nb); 2645} 2646#else 2647static int kprobe_register_module_notifier(void) 2648{ 2649 return 0; 2650} 2651#endif /* CONFIG_MODULES */ 2652 2653void kprobe_free_init_mem(void) 2654{ 2655 void *start = (void *)(&__init_begin); 2656 void *end = (void *)(&__init_end); 2657 struct hlist_head *head; 2658 struct kprobe *p; 2659 int i; 2660 2661 guard(mutex)(&kprobe_mutex); 2662 2663 /* Kill all kprobes on initmem because the target code has been freed. */ 2664 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2665 head = &kprobe_table[i]; 2666 hlist_for_each_entry(p, head, hlist) { 2667 if (start <= (void *)p->addr && (void *)p->addr < end) 2668 kill_kprobe(p); 2669 } 2670 } 2671} 2672 2673static int __init init_kprobes(void) 2674{ 2675 int i, err; 2676 2677 /* FIXME allocate the probe table, currently defined statically */ 2678 /* initialize all list heads */ 2679 for (i = 0; i < KPROBE_TABLE_SIZE; i++) 2680 INIT_HLIST_HEAD(&kprobe_table[i]); 2681 2682 err = populate_kprobe_blacklist(__start_kprobe_blacklist, 2683 __stop_kprobe_blacklist); 2684 if (err) 2685 pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err); 2686 2687 if (kretprobe_blacklist_size) { 2688 /* lookup the function address from its name */ 2689 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 2690 kretprobe_blacklist[i].addr = 2691 kprobe_lookup_name(kretprobe_blacklist[i].name, 0); 2692 if (!kretprobe_blacklist[i].addr) 2693 pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n", 2694 kretprobe_blacklist[i].name); 2695 } 2696 } 2697 2698 /* By default, kprobes are armed */ 2699 kprobes_all_disarmed = false; 2700 2701#if defined(CONFIG_OPTPROBES) && defined(__ARCH_WANT_KPROBES_INSN_SLOT) 2702 /* Init 'kprobe_optinsn_slots' for allocation */ 2703 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE; 2704#endif 2705 2706 err = arch_init_kprobes(); 2707 if (!err) 2708 err = register_die_notifier(&kprobe_exceptions_nb); 2709 if (!err) 2710 err = kprobe_register_module_notifier(); 2711 2712 kprobes_initialized = (err == 0); 2713 kprobe_sysctls_init(); 2714 return err; 2715} 2716early_initcall(init_kprobes); 2717 2718#if defined(CONFIG_OPTPROBES) 2719static int __init init_optprobes(void) 2720{ 2721 /* 2722 * Enable kprobe optimization - this kicks the optimizer which 2723 * depends on synchronize_rcu_tasks() and ksoftirqd, that is 2724 * not spawned in early initcall. So delay the optimization. 2725 */ 2726 optimize_all_kprobes(); 2727 2728 return 0; 2729} 2730subsys_initcall(init_optprobes); 2731#endif 2732 2733#ifdef CONFIG_DEBUG_FS 2734static void report_probe(struct seq_file *pi, struct kprobe *p, 2735 const char *sym, int offset, char *modname, struct kprobe *pp) 2736{ 2737 char *kprobe_type; 2738 void *addr = p->addr; 2739 2740 if (p->pre_handler == pre_handler_kretprobe) 2741 kprobe_type = "r"; 2742 else 2743 kprobe_type = "k"; 2744 2745 if (!kallsyms_show_value(pi->file->f_cred)) 2746 addr = NULL; 2747 2748 if (sym) 2749 seq_printf(pi, "%px %s %s+0x%x %s ", 2750 addr, kprobe_type, sym, offset, 2751 (modname ? modname : " ")); 2752 else /* try to use %pS */ 2753 seq_printf(pi, "%px %s %pS ", 2754 addr, kprobe_type, p->addr); 2755 2756 if (!pp) 2757 pp = p; 2758 seq_printf(pi, "%s%s%s%s\n", 2759 (kprobe_gone(p) ? "[GONE]" : ""), 2760 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""), 2761 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""), 2762 (kprobe_ftrace(pp) ? "[FTRACE]" : "")); 2763} 2764 2765static void *kprobe_seq_start(struct seq_file *f, loff_t *pos) 2766{ 2767 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 2768} 2769 2770static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 2771{ 2772 (*pos)++; 2773 if (*pos >= KPROBE_TABLE_SIZE) 2774 return NULL; 2775 return pos; 2776} 2777 2778static void kprobe_seq_stop(struct seq_file *f, void *v) 2779{ 2780 /* Nothing to do */ 2781} 2782 2783static int show_kprobe_addr(struct seq_file *pi, void *v) 2784{ 2785 struct hlist_head *head; 2786 struct kprobe *p, *kp; 2787 const char *sym; 2788 unsigned int i = *(loff_t *) v; 2789 unsigned long offset = 0; 2790 char *modname, namebuf[KSYM_NAME_LEN]; 2791 2792 head = &kprobe_table[i]; 2793 preempt_disable(); 2794 hlist_for_each_entry_rcu(p, head, hlist) { 2795 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 2796 &offset, &modname, namebuf); 2797 if (kprobe_aggrprobe(p)) { 2798 list_for_each_entry_rcu(kp, &p->list, list) 2799 report_probe(pi, kp, sym, offset, modname, p); 2800 } else 2801 report_probe(pi, p, sym, offset, modname, NULL); 2802 } 2803 preempt_enable(); 2804 return 0; 2805} 2806 2807static const struct seq_operations kprobes_sops = { 2808 .start = kprobe_seq_start, 2809 .next = kprobe_seq_next, 2810 .stop = kprobe_seq_stop, 2811 .show = show_kprobe_addr 2812}; 2813 2814DEFINE_SEQ_ATTRIBUTE(kprobes); 2815 2816/* kprobes/blacklist -- shows which functions can not be probed */ 2817static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos) 2818{ 2819 mutex_lock(&kprobe_mutex); 2820 return seq_list_start(&kprobe_blacklist, *pos); 2821} 2822 2823static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos) 2824{ 2825 return seq_list_next(v, &kprobe_blacklist, pos); 2826} 2827 2828static int kprobe_blacklist_seq_show(struct seq_file *m, void *v) 2829{ 2830 struct kprobe_blacklist_entry *ent = 2831 list_entry(v, struct kprobe_blacklist_entry, list); 2832 2833 /* 2834 * If '/proc/kallsyms' is not showing kernel address, we won't 2835 * show them here either. 2836 */ 2837 if (!kallsyms_show_value(m->file->f_cred)) 2838 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL, 2839 (void *)ent->start_addr); 2840 else 2841 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr, 2842 (void *)ent->end_addr, (void *)ent->start_addr); 2843 return 0; 2844} 2845 2846static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v) 2847{ 2848 mutex_unlock(&kprobe_mutex); 2849} 2850 2851static const struct seq_operations kprobe_blacklist_sops = { 2852 .start = kprobe_blacklist_seq_start, 2853 .next = kprobe_blacklist_seq_next, 2854 .stop = kprobe_blacklist_seq_stop, 2855 .show = kprobe_blacklist_seq_show, 2856}; 2857DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist); 2858 2859static int arm_all_kprobes(void) 2860{ 2861 struct hlist_head *head; 2862 struct kprobe *p; 2863 unsigned int i, total = 0, errors = 0; 2864 int err, ret = 0; 2865 2866 guard(mutex)(&kprobe_mutex); 2867 2868 /* If kprobes are armed, just return */ 2869 if (!kprobes_all_disarmed) 2870 return 0; 2871 2872 /* 2873 * optimize_kprobe() called by arm_kprobe() checks 2874 * kprobes_all_disarmed, so set kprobes_all_disarmed before 2875 * arm_kprobe. 2876 */ 2877 kprobes_all_disarmed = false; 2878 /* Arming kprobes doesn't optimize kprobe itself */ 2879 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2880 head = &kprobe_table[i]; 2881 /* Arm all kprobes on a best-effort basis */ 2882 hlist_for_each_entry(p, head, hlist) { 2883 if (!kprobe_disabled(p)) { 2884 err = arm_kprobe(p); 2885 if (err) { 2886 errors++; 2887 ret = err; 2888 } 2889 total++; 2890 } 2891 } 2892 } 2893 2894 if (errors) 2895 pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n", 2896 errors, total); 2897 else 2898 pr_info("Kprobes globally enabled\n"); 2899 2900 return ret; 2901} 2902 2903static int disarm_all_kprobes(void) 2904{ 2905 struct hlist_head *head; 2906 struct kprobe *p; 2907 unsigned int i, total = 0, errors = 0; 2908 int err, ret = 0; 2909 2910 guard(mutex)(&kprobe_mutex); 2911 2912 /* If kprobes are already disarmed, just return */ 2913 if (kprobes_all_disarmed) 2914 return 0; 2915 2916 kprobes_all_disarmed = true; 2917 2918 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 2919 head = &kprobe_table[i]; 2920 /* Disarm all kprobes on a best-effort basis */ 2921 hlist_for_each_entry(p, head, hlist) { 2922 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) { 2923 err = disarm_kprobe(p, false); 2924 if (err) { 2925 errors++; 2926 ret = err; 2927 } 2928 total++; 2929 } 2930 } 2931 } 2932 2933 if (errors) 2934 pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n", 2935 errors, total); 2936 else 2937 pr_info("Kprobes globally disabled\n"); 2938 2939 /* Wait for disarming all kprobes by optimizer */ 2940 wait_for_kprobe_optimizer_locked(); 2941 return ret; 2942} 2943 2944/* 2945 * XXX: The debugfs bool file interface doesn't allow for callbacks 2946 * when the bool state is switched. We can reuse that facility when 2947 * available 2948 */ 2949static ssize_t read_enabled_file_bool(struct file *file, 2950 char __user *user_buf, size_t count, loff_t *ppos) 2951{ 2952 char buf[3]; 2953 2954 if (!kprobes_all_disarmed) 2955 buf[0] = '1'; 2956 else 2957 buf[0] = '0'; 2958 buf[1] = '\n'; 2959 buf[2] = 0x00; 2960 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 2961} 2962 2963static ssize_t write_enabled_file_bool(struct file *file, 2964 const char __user *user_buf, size_t count, loff_t *ppos) 2965{ 2966 bool enable; 2967 int ret; 2968 2969 ret = kstrtobool_from_user(user_buf, count, &enable); 2970 if (ret) 2971 return ret; 2972 2973 ret = enable ? arm_all_kprobes() : disarm_all_kprobes(); 2974 if (ret) 2975 return ret; 2976 2977 return count; 2978} 2979 2980static const struct file_operations fops_kp = { 2981 .read = read_enabled_file_bool, 2982 .write = write_enabled_file_bool, 2983 .llseek = default_llseek, 2984}; 2985 2986static int __init debugfs_kprobe_init(void) 2987{ 2988 struct dentry *dir; 2989 2990 dir = debugfs_create_dir("kprobes", NULL); 2991 2992 debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops); 2993 2994 debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp); 2995 2996 debugfs_create_file("blacklist", 0400, dir, NULL, 2997 &kprobe_blacklist_fops); 2998 2999 return 0; 3000} 3001 3002late_initcall(debugfs_kprobe_init); 3003#endif /* CONFIG_DEBUG_FS */
~/
Read-only file preview.