FILE
ftrace.c
File preview with syntax highlighting.
linux-6.19.14/kernel/trace/ftrace.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Infrastructure for profiling code inserted by 'gcc -pg'. 4 * 5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Originally ported from the -rt patch by: 9 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> 10 * 11 * Based on code in the latency_tracer, that is: 12 * 13 * Copyright (C) 2004-2006 Ingo Molnar 14 * Copyright (C) 2004 Nadia Yvette Chambers 15 */ 16 17#include <linux/stop_machine.h> 18#include <linux/clocksource.h> 19#include <linux/sched/task.h> 20#include <linux/kallsyms.h> 21#include <linux/security.h> 22#include <linux/seq_file.h> 23#include <linux/tracefs.h> 24#include <linux/hardirq.h> 25#include <linux/kthread.h> 26#include <linux/uaccess.h> 27#include <linux/bsearch.h> 28#include <linux/module.h> 29#include <linux/ftrace.h> 30#include <linux/sysctl.h> 31#include <linux/slab.h> 32#include <linux/ctype.h> 33#include <linux/sort.h> 34#include <linux/list.h> 35#include <linux/hash.h> 36#include <linux/rcupdate.h> 37#include <linux/kprobes.h> 38 39#include <trace/events/sched.h> 40 41#include <asm/sections.h> 42#include <asm/setup.h> 43 44#include "ftrace_internal.h" 45#include "trace_output.h" 46#include "trace_stat.h" 47 48/* Flags that do not get reset */ 49#define FTRACE_NOCLEAR_FLAGS (FTRACE_FL_DISABLED | FTRACE_FL_TOUCHED | \ 50 FTRACE_FL_MODIFIED) 51 52#define FTRACE_INVALID_FUNCTION "__ftrace_invalid_address__" 53 54#define FTRACE_WARN_ON(cond) \ 55 ({ \ 56 int ___r = cond; \ 57 if (WARN_ON(___r)) \ 58 ftrace_kill(); \ 59 ___r; \ 60 }) 61 62#define FTRACE_WARN_ON_ONCE(cond) \ 63 ({ \ 64 int ___r = cond; \ 65 if (WARN_ON_ONCE(___r)) \ 66 ftrace_kill(); \ 67 ___r; \ 68 }) 69 70/* hash bits for specific function selection */ 71#define FTRACE_HASH_DEFAULT_BITS 10 72#define FTRACE_HASH_MAX_BITS 12 73 74#ifdef CONFIG_DYNAMIC_FTRACE 75#define INIT_OPS_HASH(opsname) \ 76 .func_hash = &opsname.local_hash, \ 77 .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \ 78 .subop_list = LIST_HEAD_INIT(opsname.subop_list), 79#else 80#define INIT_OPS_HASH(opsname) 81#endif 82 83enum { 84 FTRACE_MODIFY_ENABLE_FL = (1 << 0), 85 FTRACE_MODIFY_MAY_SLEEP_FL = (1 << 1), 86}; 87 88struct ftrace_ops ftrace_list_end __read_mostly = { 89 .func = ftrace_stub, 90 .flags = FTRACE_OPS_FL_STUB, 91 INIT_OPS_HASH(ftrace_list_end) 92}; 93 94/* ftrace_enabled is a method to turn ftrace on or off */ 95int ftrace_enabled __read_mostly; 96static int __maybe_unused last_ftrace_enabled; 97 98/* Current function tracing op */ 99struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; 100/* What to set function_trace_op to */ 101static struct ftrace_ops *set_function_trace_op; 102 103bool ftrace_pids_enabled(struct ftrace_ops *ops) 104{ 105 struct trace_array *tr; 106 107 if (!(ops->flags & FTRACE_OPS_FL_PID) || !ops->private) 108 return false; 109 110 tr = ops->private; 111 112 return tr->function_pids != NULL || tr->function_no_pids != NULL; 113} 114 115static void ftrace_update_trampoline(struct ftrace_ops *ops); 116 117/* 118 * ftrace_disabled is set when an anomaly is discovered. 119 * ftrace_disabled is much stronger than ftrace_enabled. 120 */ 121static int ftrace_disabled __read_mostly; 122 123DEFINE_MUTEX(ftrace_lock); 124 125struct ftrace_ops __rcu *ftrace_ops_list __read_mostly = (struct ftrace_ops __rcu *)&ftrace_list_end; 126ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 127struct ftrace_ops global_ops; 128 129/* Defined by vmlinux.lds.h see the comment above arch_ftrace_ops_list_func for details */ 130void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 131 struct ftrace_ops *op, struct ftrace_regs *fregs); 132 133#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS 134/* 135 * Stub used to invoke the list ops without requiring a separate trampoline. 136 */ 137const struct ftrace_ops ftrace_list_ops = { 138 .func = ftrace_ops_list_func, 139 .flags = FTRACE_OPS_FL_STUB, 140}; 141 142static void ftrace_ops_nop_func(unsigned long ip, unsigned long parent_ip, 143 struct ftrace_ops *op, 144 struct ftrace_regs *fregs) 145{ 146 /* do nothing */ 147} 148 149/* 150 * Stub used when a call site is disabled. May be called transiently by threads 151 * which have made it into ftrace_caller but haven't yet recovered the ops at 152 * the point the call site is disabled. 153 */ 154const struct ftrace_ops ftrace_nop_ops = { 155 .func = ftrace_ops_nop_func, 156 .flags = FTRACE_OPS_FL_STUB, 157}; 158#endif 159 160static inline void ftrace_ops_init(struct ftrace_ops *ops) 161{ 162#ifdef CONFIG_DYNAMIC_FTRACE 163 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { 164 mutex_init(&ops->local_hash.regex_lock); 165 INIT_LIST_HEAD(&ops->subop_list); 166 ops->func_hash = &ops->local_hash; 167 ops->flags |= FTRACE_OPS_FL_INITIALIZED; 168 } 169#endif 170} 171 172/* Call this function for when a callback filters on set_ftrace_pid */ 173static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip, 174 struct ftrace_ops *op, struct ftrace_regs *fregs) 175{ 176 struct trace_array *tr = op->private; 177 int pid; 178 179 if (tr) { 180 pid = this_cpu_read(tr->array_buffer.data->ftrace_ignore_pid); 181 if (pid == FTRACE_PID_IGNORE) 182 return; 183 if (pid != FTRACE_PID_TRACE && 184 pid != current->pid) 185 return; 186 } 187 188 op->saved_func(ip, parent_ip, op, fregs); 189} 190 191void ftrace_sync_ipi(void *data) 192{ 193 /* Probably not needed, but do it anyway */ 194 smp_rmb(); 195} 196 197static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) 198{ 199 /* 200 * If this is a dynamic or RCU ops, or we force list func, 201 * then it needs to call the list anyway. 202 */ 203 if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_RCU) || 204 FTRACE_FORCE_LIST_FUNC) 205 return ftrace_ops_list_func; 206 207 return ftrace_ops_get_func(ops); 208} 209 210static void update_ftrace_function(void) 211{ 212 ftrace_func_t func; 213 214 /* 215 * Prepare the ftrace_ops that the arch callback will use. 216 * If there's only one ftrace_ops registered, the ftrace_ops_list 217 * will point to the ops we want. 218 */ 219 set_function_trace_op = rcu_dereference_protected(ftrace_ops_list, 220 lockdep_is_held(&ftrace_lock)); 221 222 /* If there's no ftrace_ops registered, just call the stub function */ 223 if (set_function_trace_op == &ftrace_list_end) { 224 func = ftrace_stub; 225 226 /* 227 * If we are at the end of the list and this ops is 228 * recursion safe and not dynamic and the arch supports passing ops, 229 * then have the mcount trampoline call the function directly. 230 */ 231 } else if (rcu_dereference_protected(ftrace_ops_list->next, 232 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 233 func = ftrace_ops_get_list_func(ftrace_ops_list); 234 235 } else { 236 /* Just use the default ftrace_ops */ 237 set_function_trace_op = &ftrace_list_end; 238 func = ftrace_ops_list_func; 239 } 240 241 /* If there's no change, then do nothing more here */ 242 if (ftrace_trace_function == func) 243 return; 244 245 /* 246 * If we are using the list function, it doesn't care 247 * about the function_trace_ops. 248 */ 249 if (func == ftrace_ops_list_func) { 250 ftrace_trace_function = func; 251 /* 252 * Don't even bother setting function_trace_ops, 253 * it would be racy to do so anyway. 254 */ 255 return; 256 } 257 258#ifndef CONFIG_DYNAMIC_FTRACE 259 /* 260 * For static tracing, we need to be a bit more careful. 261 * The function change takes affect immediately. Thus, 262 * we need to coordinate the setting of the function_trace_ops 263 * with the setting of the ftrace_trace_function. 264 * 265 * Set the function to the list ops, which will call the 266 * function we want, albeit indirectly, but it handles the 267 * ftrace_ops and doesn't depend on function_trace_op. 268 */ 269 ftrace_trace_function = ftrace_ops_list_func; 270 /* 271 * Make sure all CPUs see this. Yes this is slow, but static 272 * tracing is slow and nasty to have enabled. 273 */ 274 synchronize_rcu_tasks_rude(); 275 /* Now all cpus are using the list ops. */ 276 function_trace_op = set_function_trace_op; 277 /* Make sure the function_trace_op is visible on all CPUs */ 278 smp_wmb(); 279 /* Nasty way to force a rmb on all cpus */ 280 smp_call_function(ftrace_sync_ipi, NULL, 1); 281 /* OK, we are all set to update the ftrace_trace_function now! */ 282#endif /* !CONFIG_DYNAMIC_FTRACE */ 283 284 ftrace_trace_function = func; 285} 286 287static void add_ftrace_ops(struct ftrace_ops __rcu **list, 288 struct ftrace_ops *ops) 289{ 290 rcu_assign_pointer(ops->next, *list); 291 292 /* 293 * We are entering ops into the list but another 294 * CPU might be walking that list. We need to make sure 295 * the ops->next pointer is valid before another CPU sees 296 * the ops pointer included into the list. 297 */ 298 rcu_assign_pointer(*list, ops); 299} 300 301static int remove_ftrace_ops(struct ftrace_ops __rcu **list, 302 struct ftrace_ops *ops) 303{ 304 struct ftrace_ops **p; 305 306 /* 307 * If we are removing the last function, then simply point 308 * to the ftrace_stub. 309 */ 310 if (rcu_dereference_protected(*list, 311 lockdep_is_held(&ftrace_lock)) == ops && 312 rcu_dereference_protected(ops->next, 313 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 314 rcu_assign_pointer(*list, &ftrace_list_end); 315 return 0; 316 } 317 318 for (p = list; *p != &ftrace_list_end; p = &(*p)->next) 319 if (*p == ops) 320 break; 321 322 if (*p != ops) 323 return -1; 324 325 *p = (*p)->next; 326 return 0; 327} 328 329static void ftrace_update_trampoline(struct ftrace_ops *ops); 330 331int __register_ftrace_function(struct ftrace_ops *ops) 332{ 333 if (ops->flags & FTRACE_OPS_FL_DELETED) 334 return -EINVAL; 335 336 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) 337 return -EBUSY; 338 339#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS 340 /* 341 * If the ftrace_ops specifies SAVE_REGS, then it only can be used 342 * if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set. 343 * Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant. 344 */ 345 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS && 346 !(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)) 347 return -EINVAL; 348 349 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED) 350 ops->flags |= FTRACE_OPS_FL_SAVE_REGS; 351#endif 352 if (!ftrace_enabled && (ops->flags & FTRACE_OPS_FL_PERMANENT)) 353 return -EBUSY; 354 355 if (!is_kernel_core_data((unsigned long)ops)) 356 ops->flags |= FTRACE_OPS_FL_DYNAMIC; 357 358 add_ftrace_ops(&ftrace_ops_list, ops); 359 360 /* Always save the function, and reset at unregistering */ 361 ops->saved_func = ops->func; 362 363 if (ftrace_pids_enabled(ops)) 364 ops->func = ftrace_pid_func; 365 366 ftrace_update_trampoline(ops); 367 368 if (ftrace_enabled) 369 update_ftrace_function(); 370 371 return 0; 372} 373 374int __unregister_ftrace_function(struct ftrace_ops *ops) 375{ 376 int ret; 377 378 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED))) 379 return -EBUSY; 380 381 ret = remove_ftrace_ops(&ftrace_ops_list, ops); 382 383 if (ret < 0) 384 return ret; 385 386 if (ftrace_enabled) 387 update_ftrace_function(); 388 389 ops->func = ops->saved_func; 390 391 return 0; 392} 393 394static void ftrace_update_pid_func(void) 395{ 396 struct ftrace_ops *op; 397 398 /* Only do something if we are tracing something */ 399 if (ftrace_trace_function == ftrace_stub) 400 return; 401 402 do_for_each_ftrace_op(op, ftrace_ops_list) { 403 if (op->flags & FTRACE_OPS_FL_PID) { 404 op->func = ftrace_pids_enabled(op) ? 405 ftrace_pid_func : op->saved_func; 406 ftrace_update_trampoline(op); 407 } 408 } while_for_each_ftrace_op(op); 409 410 fgraph_update_pid_func(); 411 412 update_ftrace_function(); 413} 414 415#ifdef CONFIG_FUNCTION_PROFILER 416struct ftrace_profile { 417 struct hlist_node node; 418 unsigned long ip; 419 unsigned long counter; 420#ifdef CONFIG_FUNCTION_GRAPH_TRACER 421 unsigned long long time; 422 unsigned long long time_squared; 423#endif 424}; 425 426struct ftrace_profile_page { 427 struct ftrace_profile_page *next; 428 unsigned long index; 429 struct ftrace_profile records[]; 430}; 431 432struct ftrace_profile_stat { 433 atomic_t disabled; 434 struct hlist_head *hash; 435 struct ftrace_profile_page *pages; 436 struct ftrace_profile_page *start; 437 struct tracer_stat stat; 438}; 439 440#define PROFILE_RECORDS_SIZE \ 441 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) 442 443#define PROFILES_PER_PAGE \ 444 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) 445 446static int ftrace_profile_enabled __read_mostly; 447 448/* ftrace_profile_lock - synchronize the enable and disable of the profiler */ 449static DEFINE_MUTEX(ftrace_profile_lock); 450 451static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); 452 453#define FTRACE_PROFILE_HASH_BITS 10 454#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS) 455 456static void * 457function_stat_next(void *v, int idx) 458{ 459 struct ftrace_profile *rec = v; 460 struct ftrace_profile_page *pg; 461 462 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); 463 464 again: 465 if (idx != 0) 466 rec++; 467 468 if ((void *)rec >= (void *)&pg->records[pg->index]) { 469 pg = pg->next; 470 if (!pg) 471 return NULL; 472 rec = &pg->records[0]; 473 if (!rec->counter) 474 goto again; 475 } 476 477 return rec; 478} 479 480static void *function_stat_start(struct tracer_stat *trace) 481{ 482 struct ftrace_profile_stat *stat = 483 container_of(trace, struct ftrace_profile_stat, stat); 484 485 if (!stat || !stat->start) 486 return NULL; 487 488 return function_stat_next(&stat->start->records[0], 0); 489} 490 491#ifdef CONFIG_FUNCTION_GRAPH_TRACER 492/* function graph compares on total time */ 493static int function_stat_cmp(const void *p1, const void *p2) 494{ 495 const struct ftrace_profile *a = p1; 496 const struct ftrace_profile *b = p2; 497 498 if (a->time < b->time) 499 return -1; 500 if (a->time > b->time) 501 return 1; 502 else 503 return 0; 504} 505#else 506/* not function graph compares against hits */ 507static int function_stat_cmp(const void *p1, const void *p2) 508{ 509 const struct ftrace_profile *a = p1; 510 const struct ftrace_profile *b = p2; 511 512 if (a->counter < b->counter) 513 return -1; 514 if (a->counter > b->counter) 515 return 1; 516 else 517 return 0; 518} 519#endif 520 521static int function_stat_headers(struct seq_file *m) 522{ 523#ifdef CONFIG_FUNCTION_GRAPH_TRACER 524 seq_puts(m, " Function " 525 "Hit Time Avg s^2\n" 526 " -------- " 527 "--- ---- --- ---\n"); 528#else 529 seq_puts(m, " Function Hit\n" 530 " -------- ---\n"); 531#endif 532 return 0; 533} 534 535static int function_stat_show(struct seq_file *m, void *v) 536{ 537 struct trace_array *tr = trace_get_global_array(); 538 struct ftrace_profile *rec = v; 539 const char *refsymbol = NULL; 540 char str[KSYM_SYMBOL_LEN]; 541#ifdef CONFIG_FUNCTION_GRAPH_TRACER 542 static struct trace_seq s; 543 unsigned long long avg; 544 unsigned long long stddev; 545 unsigned long long stddev_denom; 546#endif 547 guard(mutex)(&ftrace_profile_lock); 548 549 /* we raced with function_profile_reset() */ 550 if (unlikely(rec->counter == 0)) 551 return -EBUSY; 552 553#ifdef CONFIG_FUNCTION_GRAPH_TRACER 554 avg = div64_ul(rec->time, rec->counter); 555 if (tracing_thresh && (avg < tracing_thresh)) 556 return 0; 557#endif 558 559 if (tr->trace_flags & TRACE_ITER(PROF_TEXT_OFFSET)) { 560 unsigned long offset; 561 562 if (core_kernel_text(rec->ip)) { 563 refsymbol = "_text"; 564 offset = rec->ip - (unsigned long)_text; 565 } else { 566 struct module *mod; 567 568 guard(rcu)(); 569 mod = __module_text_address(rec->ip); 570 if (mod) { 571 refsymbol = mod->name; 572 /* Calculate offset from module's text entry address. */ 573 offset = rec->ip - (unsigned long)mod->mem[MOD_TEXT].base; 574 } 575 } 576 if (refsymbol) 577 snprintf(str, sizeof(str), " %s+%#lx", refsymbol, offset); 578 } 579 if (!refsymbol) 580 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 581 582 seq_printf(m, " %-30.30s %10lu", str, rec->counter); 583 584#ifdef CONFIG_FUNCTION_GRAPH_TRACER 585 seq_puts(m, " "); 586 587 /* 588 * Variance formula: 589 * s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2) 590 * Maybe Welford's method is better here? 591 * Divide only by 1000 for ns^2 -> us^2 conversion. 592 * trace_print_graph_duration will divide by 1000 again. 593 */ 594 stddev = 0; 595 stddev_denom = rec->counter * (rec->counter - 1) * 1000; 596 if (stddev_denom) { 597 stddev = rec->counter * rec->time_squared - 598 rec->time * rec->time; 599 stddev = div64_ul(stddev, stddev_denom); 600 } 601 602 trace_seq_init(&s); 603 trace_print_graph_duration(rec->time, &s); 604 trace_seq_puts(&s, " "); 605 trace_print_graph_duration(avg, &s); 606 trace_seq_puts(&s, " "); 607 trace_print_graph_duration(stddev, &s); 608 trace_print_seq(m, &s); 609#endif 610 seq_putc(m, '\n'); 611 612 return 0; 613} 614 615static void ftrace_profile_reset(struct ftrace_profile_stat *stat) 616{ 617 struct ftrace_profile_page *pg; 618 619 pg = stat->pages = stat->start; 620 621 while (pg) { 622 memset(pg->records, 0, PROFILE_RECORDS_SIZE); 623 pg->index = 0; 624 pg = pg->next; 625 } 626 627 memset(stat->hash, 0, 628 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); 629} 630 631static int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) 632{ 633 struct ftrace_profile_page *pg; 634 int functions; 635 int pages; 636 int i; 637 638 /* If we already allocated, do nothing */ 639 if (stat->pages) 640 return 0; 641 642 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); 643 if (!stat->pages) 644 return -ENOMEM; 645 646#ifdef CONFIG_DYNAMIC_FTRACE 647 functions = ftrace_update_tot_cnt; 648#else 649 /* 650 * We do not know the number of functions that exist because 651 * dynamic tracing is what counts them. With past experience 652 * we have around 20K functions. That should be more than enough. 653 * It is highly unlikely we will execute every function in 654 * the kernel. 655 */ 656 functions = 20000; 657#endif 658 659 pg = stat->start = stat->pages; 660 661 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); 662 663 for (i = 1; i < pages; i++) { 664 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 665 if (!pg->next) 666 goto out_free; 667 pg = pg->next; 668 } 669 670 return 0; 671 672 out_free: 673 pg = stat->start; 674 while (pg) { 675 unsigned long tmp = (unsigned long)pg; 676 677 pg = pg->next; 678 free_page(tmp); 679 } 680 681 stat->pages = NULL; 682 stat->start = NULL; 683 684 return -ENOMEM; 685} 686 687static int ftrace_profile_init_cpu(int cpu) 688{ 689 struct ftrace_profile_stat *stat; 690 int size; 691 692 stat = &per_cpu(ftrace_profile_stats, cpu); 693 694 if (stat->hash) { 695 /* If the profile is already created, simply reset it */ 696 ftrace_profile_reset(stat); 697 return 0; 698 } 699 700 /* 701 * We are profiling all functions, but usually only a few thousand 702 * functions are hit. We'll make a hash of 1024 items. 703 */ 704 size = FTRACE_PROFILE_HASH_SIZE; 705 706 stat->hash = kcalloc(size, sizeof(struct hlist_head), GFP_KERNEL); 707 708 if (!stat->hash) 709 return -ENOMEM; 710 711 /* Preallocate the function profiling pages */ 712 if (ftrace_profile_pages_init(stat) < 0) { 713 kfree(stat->hash); 714 stat->hash = NULL; 715 return -ENOMEM; 716 } 717 718 return 0; 719} 720 721static int ftrace_profile_init(void) 722{ 723 int cpu; 724 int ret = 0; 725 726 for_each_possible_cpu(cpu) { 727 ret = ftrace_profile_init_cpu(cpu); 728 if (ret) 729 break; 730 } 731 732 return ret; 733} 734 735/* interrupts must be disabled */ 736static struct ftrace_profile * 737ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) 738{ 739 struct ftrace_profile *rec; 740 struct hlist_head *hhd; 741 unsigned long key; 742 743 key = hash_long(ip, FTRACE_PROFILE_HASH_BITS); 744 hhd = &stat->hash[key]; 745 746 if (hlist_empty(hhd)) 747 return NULL; 748 749 hlist_for_each_entry_rcu_notrace(rec, hhd, node) { 750 if (rec->ip == ip) 751 return rec; 752 } 753 754 return NULL; 755} 756 757static void ftrace_add_profile(struct ftrace_profile_stat *stat, 758 struct ftrace_profile *rec) 759{ 760 unsigned long key; 761 762 key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS); 763 hlist_add_head_rcu(&rec->node, &stat->hash[key]); 764} 765 766/* 767 * The memory is already allocated, this simply finds a new record to use. 768 */ 769static struct ftrace_profile * 770ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) 771{ 772 struct ftrace_profile *rec = NULL; 773 774 /* prevent recursion (from NMIs) */ 775 if (atomic_inc_return(&stat->disabled) != 1) 776 goto out; 777 778 /* 779 * Try to find the function again since an NMI 780 * could have added it 781 */ 782 rec = ftrace_find_profiled_func(stat, ip); 783 if (rec) 784 goto out; 785 786 if (stat->pages->index == PROFILES_PER_PAGE) { 787 if (!stat->pages->next) 788 goto out; 789 stat->pages = stat->pages->next; 790 } 791 792 rec = &stat->pages->records[stat->pages->index++]; 793 rec->ip = ip; 794 ftrace_add_profile(stat, rec); 795 796 out: 797 atomic_dec(&stat->disabled); 798 799 return rec; 800} 801 802static void 803function_profile_call(unsigned long ip, unsigned long parent_ip, 804 struct ftrace_ops *ops, struct ftrace_regs *fregs) 805{ 806 struct ftrace_profile_stat *stat; 807 struct ftrace_profile *rec; 808 809 if (!ftrace_profile_enabled) 810 return; 811 812 guard(preempt_notrace)(); 813 814 stat = this_cpu_ptr(&ftrace_profile_stats); 815 if (!stat->hash || !ftrace_profile_enabled) 816 return; 817 818 rec = ftrace_find_profiled_func(stat, ip); 819 if (!rec) { 820 rec = ftrace_profile_alloc(stat, ip); 821 if (!rec) 822 return; 823 } 824 825 rec->counter++; 826} 827 828#ifdef CONFIG_FUNCTION_GRAPH_TRACER 829static bool fgraph_graph_time = true; 830 831void ftrace_graph_graph_time_control(bool enable) 832{ 833 fgraph_graph_time = enable; 834} 835 836struct profile_fgraph_data { 837 unsigned long long calltime; 838 unsigned long long subtime; 839 unsigned long long sleeptime; 840}; 841 842static int profile_graph_entry(struct ftrace_graph_ent *trace, 843 struct fgraph_ops *gops, 844 struct ftrace_regs *fregs) 845{ 846 struct profile_fgraph_data *profile_data; 847 848 function_profile_call(trace->func, 0, NULL, NULL); 849 850 /* If function graph is shutting down, ret_stack can be NULL */ 851 if (!current->ret_stack) 852 return 0; 853 854 profile_data = fgraph_reserve_data(gops->idx, sizeof(*profile_data)); 855 if (!profile_data) 856 return 0; 857 858 profile_data->subtime = 0; 859 profile_data->sleeptime = current->ftrace_sleeptime; 860 profile_data->calltime = trace_clock_local(); 861 862 return 1; 863} 864 865bool fprofile_no_sleep_time; 866 867static void profile_graph_return(struct ftrace_graph_ret *trace, 868 struct fgraph_ops *gops, 869 struct ftrace_regs *fregs) 870{ 871 struct profile_fgraph_data *profile_data; 872 struct ftrace_profile_stat *stat; 873 unsigned long long calltime; 874 unsigned long long rettime = trace_clock_local(); 875 struct ftrace_profile *rec; 876 int size; 877 878 guard(preempt_notrace)(); 879 880 stat = this_cpu_ptr(&ftrace_profile_stats); 881 if (!stat->hash || !ftrace_profile_enabled) 882 return; 883 884 profile_data = fgraph_retrieve_data(gops->idx, &size); 885 886 /* If the calltime was zero'd ignore it */ 887 if (!profile_data || !profile_data->calltime) 888 return; 889 890 calltime = rettime - profile_data->calltime; 891 892 if (fprofile_no_sleep_time) { 893 if (current->ftrace_sleeptime) 894 calltime -= current->ftrace_sleeptime - profile_data->sleeptime; 895 } 896 897 if (!fgraph_graph_time) { 898 struct profile_fgraph_data *parent_data; 899 900 /* Append this call time to the parent time to subtract */ 901 parent_data = fgraph_retrieve_parent_data(gops->idx, &size, 1); 902 if (parent_data) 903 parent_data->subtime += calltime; 904 905 if (profile_data->subtime && profile_data->subtime < calltime) 906 calltime -= profile_data->subtime; 907 else 908 calltime = 0; 909 } 910 911 rec = ftrace_find_profiled_func(stat, trace->func); 912 if (rec) { 913 rec->time += calltime; 914 rec->time_squared += calltime * calltime; 915 } 916} 917 918static struct fgraph_ops fprofiler_ops = { 919 .entryfunc = &profile_graph_entry, 920 .retfunc = &profile_graph_return, 921}; 922 923static int register_ftrace_profiler(void) 924{ 925 ftrace_ops_set_global_filter(&fprofiler_ops.ops); 926 return register_ftrace_graph(&fprofiler_ops); 927} 928 929static void unregister_ftrace_profiler(void) 930{ 931 unregister_ftrace_graph(&fprofiler_ops); 932} 933#else 934static struct ftrace_ops ftrace_profile_ops __read_mostly = { 935 .func = function_profile_call, 936}; 937 938static int register_ftrace_profiler(void) 939{ 940 ftrace_ops_set_global_filter(&ftrace_profile_ops); 941 return register_ftrace_function(&ftrace_profile_ops); 942} 943 944static void unregister_ftrace_profiler(void) 945{ 946 unregister_ftrace_function(&ftrace_profile_ops); 947} 948#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 949 950static ssize_t 951ftrace_profile_write(struct file *filp, const char __user *ubuf, 952 size_t cnt, loff_t *ppos) 953{ 954 unsigned long val; 955 int ret; 956 957 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 958 if (ret) 959 return ret; 960 961 val = !!val; 962 963 guard(mutex)(&ftrace_profile_lock); 964 if (ftrace_profile_enabled ^ val) { 965 if (val) { 966 ret = ftrace_profile_init(); 967 if (ret < 0) 968 return ret; 969 970 ret = register_ftrace_profiler(); 971 if (ret < 0) 972 return ret; 973 ftrace_profile_enabled = 1; 974 } else { 975 ftrace_profile_enabled = 0; 976 /* 977 * unregister_ftrace_profiler calls stop_machine 978 * so this acts like an synchronize_rcu. 979 */ 980 unregister_ftrace_profiler(); 981 } 982 } 983 984 *ppos += cnt; 985 986 return cnt; 987} 988 989static ssize_t 990ftrace_profile_read(struct file *filp, char __user *ubuf, 991 size_t cnt, loff_t *ppos) 992{ 993 char buf[64]; /* big enough to hold a number */ 994 int r; 995 996 r = sprintf(buf, "%u\n", ftrace_profile_enabled); 997 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 998} 999 1000static const struct file_operations ftrace_profile_fops = { 1001 .open = tracing_open_generic, 1002 .read = ftrace_profile_read, 1003 .write = ftrace_profile_write, 1004 .llseek = default_llseek, 1005}; 1006 1007/* used to initialize the real stat files */ 1008static struct tracer_stat function_stats __initdata = { 1009 .name = "functions", 1010 .stat_start = function_stat_start, 1011 .stat_next = function_stat_next, 1012 .stat_cmp = function_stat_cmp, 1013 .stat_headers = function_stat_headers, 1014 .stat_show = function_stat_show 1015}; 1016 1017static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 1018{ 1019 struct ftrace_profile_stat *stat; 1020 char *name; 1021 int ret; 1022 int cpu; 1023 1024 for_each_possible_cpu(cpu) { 1025 stat = &per_cpu(ftrace_profile_stats, cpu); 1026 1027 name = kasprintf(GFP_KERNEL, "function%d", cpu); 1028 if (!name) { 1029 /* 1030 * The files created are permanent, if something happens 1031 * we still do not free memory. 1032 */ 1033 WARN(1, 1034 "Could not allocate stat file for cpu %d\n", 1035 cpu); 1036 return; 1037 } 1038 stat->stat = function_stats; 1039 stat->stat.name = name; 1040 ret = register_stat_tracer(&stat->stat); 1041 if (ret) { 1042 WARN(1, 1043 "Could not register function stat for cpu %d\n", 1044 cpu); 1045 kfree(name); 1046 return; 1047 } 1048 } 1049 1050 trace_create_file("function_profile_enabled", 1051 TRACE_MODE_WRITE, d_tracer, NULL, 1052 &ftrace_profile_fops); 1053} 1054 1055#else /* CONFIG_FUNCTION_PROFILER */ 1056static __init void ftrace_profile_tracefs(struct dentry *d_tracer) 1057{ 1058} 1059#endif /* CONFIG_FUNCTION_PROFILER */ 1060 1061#ifdef CONFIG_DYNAMIC_FTRACE 1062 1063static struct ftrace_ops *removed_ops; 1064 1065/* 1066 * Set when doing a global update, like enabling all recs or disabling them. 1067 * It is not set when just updating a single ftrace_ops. 1068 */ 1069static bool update_all_ops; 1070 1071struct ftrace_func_probe { 1072 struct ftrace_probe_ops *probe_ops; 1073 struct ftrace_ops ops; 1074 struct trace_array *tr; 1075 struct list_head list; 1076 void *data; 1077 int ref; 1078}; 1079 1080/* 1081 * We make these constant because no one should touch them, 1082 * but they are used as the default "empty hash", to avoid allocating 1083 * it all the time. These are in a read only section such that if 1084 * anyone does try to modify it, it will cause an exception. 1085 */ 1086static const struct hlist_head empty_buckets[1]; 1087static const struct ftrace_hash empty_hash = { 1088 .buckets = (struct hlist_head *)empty_buckets, 1089}; 1090#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash) 1091 1092struct ftrace_ops global_ops = { 1093 .func = ftrace_stub, 1094 .local_hash.notrace_hash = EMPTY_HASH, 1095 .local_hash.filter_hash = EMPTY_HASH, 1096 INIT_OPS_HASH(global_ops) 1097 .flags = FTRACE_OPS_FL_INITIALIZED | 1098 FTRACE_OPS_FL_PID, 1099}; 1100 1101/* 1102 * Used by the stack unwinder to know about dynamic ftrace trampolines. 1103 */ 1104struct ftrace_ops *ftrace_ops_trampoline(unsigned long addr) 1105{ 1106 struct ftrace_ops *op = NULL; 1107 1108 /* 1109 * Some of the ops may be dynamically allocated, 1110 * they are freed after a synchronize_rcu(). 1111 */ 1112 preempt_disable_notrace(); 1113 1114 do_for_each_ftrace_op(op, ftrace_ops_list) { 1115 /* 1116 * This is to check for dynamically allocated trampolines. 1117 * Trampolines that are in kernel text will have 1118 * core_kernel_text() return true. 1119 */ 1120 if (op->trampoline && op->trampoline_size) 1121 if (addr >= op->trampoline && 1122 addr < op->trampoline + op->trampoline_size) { 1123 preempt_enable_notrace(); 1124 return op; 1125 } 1126 } while_for_each_ftrace_op(op); 1127 preempt_enable_notrace(); 1128 1129 return NULL; 1130} 1131 1132/* 1133 * This is used by __kernel_text_address() to return true if the 1134 * address is on a dynamically allocated trampoline that would 1135 * not return true for either core_kernel_text() or 1136 * is_module_text_address(). 1137 */ 1138bool is_ftrace_trampoline(unsigned long addr) 1139{ 1140 return ftrace_ops_trampoline(addr) != NULL; 1141} 1142 1143struct ftrace_page { 1144 struct ftrace_page *next; 1145 struct dyn_ftrace *records; 1146 int index; 1147 int order; 1148}; 1149 1150#define ENTRY_SIZE sizeof(struct dyn_ftrace) 1151 1152static struct ftrace_page *ftrace_pages_start; 1153static struct ftrace_page *ftrace_pages; 1154 1155static __always_inline unsigned long 1156ftrace_hash_key(struct ftrace_hash *hash, unsigned long ip) 1157{ 1158 if (hash->size_bits > 0) 1159 return hash_long(ip, hash->size_bits); 1160 1161 return 0; 1162} 1163 1164/* Only use this function if ftrace_hash_empty() has already been tested */ 1165static __always_inline struct ftrace_func_entry * 1166__ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) 1167{ 1168 unsigned long key; 1169 struct ftrace_func_entry *entry; 1170 struct hlist_head *hhd; 1171 1172 key = ftrace_hash_key(hash, ip); 1173 hhd = &hash->buckets[key]; 1174 1175 hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) { 1176 if (entry->ip == ip) 1177 return entry; 1178 } 1179 return NULL; 1180} 1181 1182/** 1183 * ftrace_lookup_ip - Test to see if an ip exists in an ftrace_hash 1184 * @hash: The hash to look at 1185 * @ip: The instruction pointer to test 1186 * 1187 * Search a given @hash to see if a given instruction pointer (@ip) 1188 * exists in it. 1189 * 1190 * Returns: the entry that holds the @ip if found. NULL otherwise. 1191 */ 1192struct ftrace_func_entry * 1193ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip) 1194{ 1195 if (ftrace_hash_empty(hash)) 1196 return NULL; 1197 1198 return __ftrace_lookup_ip(hash, ip); 1199} 1200 1201static void __add_hash_entry(struct ftrace_hash *hash, 1202 struct ftrace_func_entry *entry) 1203{ 1204 struct hlist_head *hhd; 1205 unsigned long key; 1206 1207 key = ftrace_hash_key(hash, entry->ip); 1208 hhd = &hash->buckets[key]; 1209 hlist_add_head(&entry->hlist, hhd); 1210 hash->count++; 1211} 1212 1213static struct ftrace_func_entry * 1214add_hash_entry(struct ftrace_hash *hash, unsigned long ip) 1215{ 1216 struct ftrace_func_entry *entry; 1217 1218 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1219 if (!entry) 1220 return NULL; 1221 1222 entry->ip = ip; 1223 __add_hash_entry(hash, entry); 1224 1225 return entry; 1226} 1227 1228static void 1229free_hash_entry(struct ftrace_hash *hash, 1230 struct ftrace_func_entry *entry) 1231{ 1232 hlist_del(&entry->hlist); 1233 kfree(entry); 1234 hash->count--; 1235} 1236 1237static void 1238remove_hash_entry(struct ftrace_hash *hash, 1239 struct ftrace_func_entry *entry) 1240{ 1241 hlist_del_rcu(&entry->hlist); 1242 hash->count--; 1243} 1244 1245static void ftrace_hash_clear(struct ftrace_hash *hash) 1246{ 1247 struct hlist_head *hhd; 1248 struct hlist_node *tn; 1249 struct ftrace_func_entry *entry; 1250 int size = 1 << hash->size_bits; 1251 int i; 1252 1253 if (!hash->count) 1254 return; 1255 1256 for (i = 0; i < size; i++) { 1257 hhd = &hash->buckets[i]; 1258 hlist_for_each_entry_safe(entry, tn, hhd, hlist) 1259 free_hash_entry(hash, entry); 1260 } 1261 FTRACE_WARN_ON(hash->count); 1262} 1263 1264static void free_ftrace_mod(struct ftrace_mod_load *ftrace_mod) 1265{ 1266 list_del(&ftrace_mod->list); 1267 kfree(ftrace_mod->module); 1268 kfree(ftrace_mod->func); 1269 kfree(ftrace_mod); 1270} 1271 1272static void clear_ftrace_mod_list(struct list_head *head) 1273{ 1274 struct ftrace_mod_load *p, *n; 1275 1276 /* stack tracer isn't supported yet */ 1277 if (!head) 1278 return; 1279 1280 mutex_lock(&ftrace_lock); 1281 list_for_each_entry_safe(p, n, head, list) 1282 free_ftrace_mod(p); 1283 mutex_unlock(&ftrace_lock); 1284} 1285 1286static void free_ftrace_hash(struct ftrace_hash *hash) 1287{ 1288 if (!hash || hash == EMPTY_HASH) 1289 return; 1290 ftrace_hash_clear(hash); 1291 kfree(hash->buckets); 1292 kfree(hash); 1293} 1294 1295static void __free_ftrace_hash_rcu(struct rcu_head *rcu) 1296{ 1297 struct ftrace_hash *hash; 1298 1299 hash = container_of(rcu, struct ftrace_hash, rcu); 1300 free_ftrace_hash(hash); 1301} 1302 1303static void free_ftrace_hash_rcu(struct ftrace_hash *hash) 1304{ 1305 if (!hash || hash == EMPTY_HASH) 1306 return; 1307 call_rcu(&hash->rcu, __free_ftrace_hash_rcu); 1308} 1309 1310/** 1311 * ftrace_free_filter - remove all filters for an ftrace_ops 1312 * @ops: the ops to remove the filters from 1313 */ 1314void ftrace_free_filter(struct ftrace_ops *ops) 1315{ 1316 ftrace_ops_init(ops); 1317 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED)) 1318 return; 1319 free_ftrace_hash(ops->func_hash->filter_hash); 1320 free_ftrace_hash(ops->func_hash->notrace_hash); 1321 ops->func_hash->filter_hash = EMPTY_HASH; 1322 ops->func_hash->notrace_hash = EMPTY_HASH; 1323} 1324EXPORT_SYMBOL_GPL(ftrace_free_filter); 1325 1326static struct ftrace_hash *alloc_ftrace_hash(int size_bits) 1327{ 1328 struct ftrace_hash *hash; 1329 int size; 1330 1331 hash = kzalloc(sizeof(*hash), GFP_KERNEL); 1332 if (!hash) 1333 return NULL; 1334 1335 size = 1 << size_bits; 1336 hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL); 1337 1338 if (!hash->buckets) { 1339 kfree(hash); 1340 return NULL; 1341 } 1342 1343 hash->size_bits = size_bits; 1344 1345 return hash; 1346} 1347 1348/* Used to save filters on functions for modules not loaded yet */ 1349static int ftrace_add_mod(struct trace_array *tr, 1350 const char *func, const char *module, 1351 int enable) 1352{ 1353 struct ftrace_mod_load *ftrace_mod; 1354 struct list_head *mod_head = enable ? &tr->mod_trace : &tr->mod_notrace; 1355 1356 ftrace_mod = kzalloc(sizeof(*ftrace_mod), GFP_KERNEL); 1357 if (!ftrace_mod) 1358 return -ENOMEM; 1359 1360 INIT_LIST_HEAD(&ftrace_mod->list); 1361 ftrace_mod->func = kstrdup(func, GFP_KERNEL); 1362 ftrace_mod->module = kstrdup(module, GFP_KERNEL); 1363 ftrace_mod->enable = enable; 1364 1365 if (!ftrace_mod->func || !ftrace_mod->module) 1366 goto out_free; 1367 1368 list_add(&ftrace_mod->list, mod_head); 1369 1370 return 0; 1371 1372 out_free: 1373 free_ftrace_mod(ftrace_mod); 1374 1375 return -ENOMEM; 1376} 1377 1378static struct ftrace_hash * 1379alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash) 1380{ 1381 struct ftrace_func_entry *entry; 1382 struct ftrace_hash *new_hash; 1383 int size; 1384 int i; 1385 1386 new_hash = alloc_ftrace_hash(size_bits); 1387 if (!new_hash) 1388 return NULL; 1389 1390 if (hash) 1391 new_hash->flags = hash->flags; 1392 1393 /* Empty hash? */ 1394 if (ftrace_hash_empty(hash)) 1395 return new_hash; 1396 1397 size = 1 << hash->size_bits; 1398 for (i = 0; i < size; i++) { 1399 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 1400 if (add_hash_entry(new_hash, entry->ip) == NULL) 1401 goto free_hash; 1402 } 1403 } 1404 1405 FTRACE_WARN_ON(new_hash->count != hash->count); 1406 1407 return new_hash; 1408 1409 free_hash: 1410 free_ftrace_hash(new_hash); 1411 return NULL; 1412} 1413 1414static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops); 1415static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops); 1416 1417static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 1418 struct ftrace_hash *new_hash); 1419 1420/* 1421 * Allocate a new hash and remove entries from @src and move them to the new hash. 1422 * On success, the @src hash will be empty and should be freed. 1423 */ 1424static struct ftrace_hash *__move_hash(struct ftrace_hash *src, int size) 1425{ 1426 struct ftrace_func_entry *entry; 1427 struct ftrace_hash *new_hash; 1428 struct hlist_head *hhd; 1429 struct hlist_node *tn; 1430 int bits = 0; 1431 int i; 1432 1433 /* 1434 * Use around half the size (max bit of it), but 1435 * a minimum of 2 is fine (as size of 0 or 1 both give 1 for bits). 1436 */ 1437 bits = fls(size / 2); 1438 1439 /* Don't allocate too much */ 1440 if (bits > FTRACE_HASH_MAX_BITS) 1441 bits = FTRACE_HASH_MAX_BITS; 1442 1443 new_hash = alloc_ftrace_hash(bits); 1444 if (!new_hash) 1445 return NULL; 1446 1447 new_hash->flags = src->flags; 1448 1449 size = 1 << src->size_bits; 1450 for (i = 0; i < size; i++) { 1451 hhd = &src->buckets[i]; 1452 hlist_for_each_entry_safe(entry, tn, hhd, hlist) { 1453 remove_hash_entry(src, entry); 1454 __add_hash_entry(new_hash, entry); 1455 } 1456 } 1457 return new_hash; 1458} 1459 1460/* Move the @src entries to a newly allocated hash */ 1461static struct ftrace_hash * 1462__ftrace_hash_move(struct ftrace_hash *src) 1463{ 1464 int size = src->count; 1465 1466 /* 1467 * If the new source is empty, just return the empty_hash. 1468 */ 1469 if (ftrace_hash_empty(src)) 1470 return EMPTY_HASH; 1471 1472 return __move_hash(src, size); 1473} 1474 1475/** 1476 * ftrace_hash_move - move a new hash to a filter and do updates 1477 * @ops: The ops with the hash that @dst points to 1478 * @enable: True if for the filter hash, false for the notrace hash 1479 * @dst: Points to the @ops hash that should be updated 1480 * @src: The hash to update @dst with 1481 * 1482 * This is called when an ftrace_ops hash is being updated and the 1483 * the kernel needs to reflect this. Note, this only updates the kernel 1484 * function callbacks if the @ops is enabled (not to be confused with 1485 * @enable above). If the @ops is enabled, its hash determines what 1486 * callbacks get called. This function gets called when the @ops hash 1487 * is updated and it requires new callbacks. 1488 * 1489 * On success the elements of @src is moved to @dst, and @dst is updated 1490 * properly, as well as the functions determined by the @ops hashes 1491 * are now calling the @ops callback function. 1492 * 1493 * Regardless of return type, @src should be freed with free_ftrace_hash(). 1494 */ 1495static int 1496ftrace_hash_move(struct ftrace_ops *ops, int enable, 1497 struct ftrace_hash **dst, struct ftrace_hash *src) 1498{ 1499 struct ftrace_hash *new_hash; 1500 int ret; 1501 1502 /* Reject setting notrace hash on IPMODIFY ftrace_ops */ 1503 if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable) 1504 return -EINVAL; 1505 1506 new_hash = __ftrace_hash_move(src); 1507 if (!new_hash) 1508 return -ENOMEM; 1509 1510 /* Make sure this can be applied if it is IPMODIFY ftrace_ops */ 1511 if (enable) { 1512 /* IPMODIFY should be updated only when filter_hash updating */ 1513 ret = ftrace_hash_ipmodify_update(ops, new_hash); 1514 if (ret < 0) { 1515 free_ftrace_hash(new_hash); 1516 return ret; 1517 } 1518 } 1519 1520 /* 1521 * Remove the current set, update the hash and add 1522 * them back. 1523 */ 1524 ftrace_hash_rec_disable_modify(ops); 1525 1526 rcu_assign_pointer(*dst, new_hash); 1527 1528 ftrace_hash_rec_enable_modify(ops); 1529 1530 return 0; 1531} 1532 1533static bool hash_contains_ip(unsigned long ip, 1534 struct ftrace_ops_hash *hash) 1535{ 1536 /* 1537 * The function record is a match if it exists in the filter 1538 * hash and not in the notrace hash. Note, an empty hash is 1539 * considered a match for the filter hash, but an empty 1540 * notrace hash is considered not in the notrace hash. 1541 */ 1542 return (ftrace_hash_empty(hash->filter_hash) || 1543 __ftrace_lookup_ip(hash->filter_hash, ip)) && 1544 (ftrace_hash_empty(hash->notrace_hash) || 1545 !__ftrace_lookup_ip(hash->notrace_hash, ip)); 1546} 1547 1548/* 1549 * Test the hashes for this ops to see if we want to call 1550 * the ops->func or not. 1551 * 1552 * It's a match if the ip is in the ops->filter_hash or 1553 * the filter_hash does not exist or is empty, 1554 * AND 1555 * the ip is not in the ops->notrace_hash. 1556 * 1557 * This needs to be called with preemption disabled as 1558 * the hashes are freed with call_rcu(). 1559 */ 1560int 1561ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs) 1562{ 1563 struct ftrace_ops_hash hash; 1564 int ret; 1565 1566#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS 1567 /* 1568 * There's a small race when adding ops that the ftrace handler 1569 * that wants regs, may be called without them. We can not 1570 * allow that handler to be called if regs is NULL. 1571 */ 1572 if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS)) 1573 return 0; 1574#endif 1575 1576 rcu_assign_pointer(hash.filter_hash, ops->func_hash->filter_hash); 1577 rcu_assign_pointer(hash.notrace_hash, ops->func_hash->notrace_hash); 1578 1579 if (hash_contains_ip(ip, &hash)) 1580 ret = 1; 1581 else 1582 ret = 0; 1583 1584 return ret; 1585} 1586 1587/* 1588 * This is a double for. Do not use 'break' to break out of the loop, 1589 * you must use a goto. 1590 */ 1591#define do_for_each_ftrace_rec(pg, rec) \ 1592 for (pg = ftrace_pages_start; pg; pg = pg->next) { \ 1593 int _____i; \ 1594 for (_____i = 0; _____i < pg->index; _____i++) { \ 1595 rec = &pg->records[_____i]; 1596 1597#define while_for_each_ftrace_rec() \ 1598 } \ 1599 } 1600 1601 1602static int ftrace_cmp_recs(const void *a, const void *b) 1603{ 1604 const struct dyn_ftrace *key = a; 1605 const struct dyn_ftrace *rec = b; 1606 1607 if (key->flags < rec->ip) 1608 return -1; 1609 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE) 1610 return 1; 1611 return 0; 1612} 1613 1614static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end) 1615{ 1616 struct ftrace_page *pg; 1617 struct dyn_ftrace *rec = NULL; 1618 struct dyn_ftrace key; 1619 1620 key.ip = start; 1621 key.flags = end; /* overload flags, as it is unsigned long */ 1622 1623 for (pg = ftrace_pages_start; pg; pg = pg->next) { 1624 if (pg->index == 0 || 1625 end < pg->records[0].ip || 1626 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) 1627 continue; 1628 rec = bsearch(&key, pg->records, pg->index, 1629 sizeof(struct dyn_ftrace), 1630 ftrace_cmp_recs); 1631 if (rec) 1632 break; 1633 } 1634 return rec; 1635} 1636 1637/** 1638 * ftrace_location_range - return the first address of a traced location 1639 * if it touches the given ip range 1640 * @start: start of range to search. 1641 * @end: end of range to search (inclusive). @end points to the last byte 1642 * to check. 1643 * 1644 * Returns: rec->ip if the related ftrace location is a least partly within 1645 * the given address range. That is, the first address of the instruction 1646 * that is either a NOP or call to the function tracer. It checks the ftrace 1647 * internal tables to determine if the address belongs or not. 1648 */ 1649unsigned long ftrace_location_range(unsigned long start, unsigned long end) 1650{ 1651 struct dyn_ftrace *rec; 1652 unsigned long ip = 0; 1653 1654 rcu_read_lock(); 1655 rec = lookup_rec(start, end); 1656 if (rec) 1657 ip = rec->ip; 1658 rcu_read_unlock(); 1659 1660 return ip; 1661} 1662 1663/** 1664 * ftrace_location - return the ftrace location 1665 * @ip: the instruction pointer to check 1666 * 1667 * Returns: 1668 * * If @ip matches the ftrace location, return @ip. 1669 * * If @ip matches sym+0, return sym's ftrace location. 1670 * * Otherwise, return 0. 1671 */ 1672unsigned long ftrace_location(unsigned long ip) 1673{ 1674 unsigned long loc; 1675 unsigned long offset; 1676 unsigned long size; 1677 1678 loc = ftrace_location_range(ip, ip); 1679 if (!loc) { 1680 if (!kallsyms_lookup_size_offset(ip, &size, &offset)) 1681 return 0; 1682 1683 /* map sym+0 to __fentry__ */ 1684 if (!offset) 1685 loc = ftrace_location_range(ip, ip + size - 1); 1686 } 1687 return loc; 1688} 1689 1690/** 1691 * ftrace_text_reserved - return true if range contains an ftrace location 1692 * @start: start of range to search 1693 * @end: end of range to search (inclusive). @end points to the last byte to check. 1694 * 1695 * Returns: 1 if @start and @end contains a ftrace location. 1696 * That is, the instruction that is either a NOP or call to 1697 * the function tracer. It checks the ftrace internal tables to 1698 * determine if the address belongs or not. 1699 */ 1700int ftrace_text_reserved(const void *start, const void *end) 1701{ 1702 unsigned long ret; 1703 1704 ret = ftrace_location_range((unsigned long)start, 1705 (unsigned long)end); 1706 1707 return (int)!!ret; 1708} 1709 1710/* Test if ops registered to this rec needs regs */ 1711static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec) 1712{ 1713 struct ftrace_ops *ops; 1714 bool keep_regs = false; 1715 1716 for (ops = ftrace_ops_list; 1717 ops != &ftrace_list_end; ops = ops->next) { 1718 /* pass rec in as regs to have non-NULL val */ 1719 if (ftrace_ops_test(ops, rec->ip, rec)) { 1720 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1721 keep_regs = true; 1722 break; 1723 } 1724 } 1725 } 1726 1727 return keep_regs; 1728} 1729 1730static struct ftrace_ops * 1731ftrace_find_tramp_ops_any(struct dyn_ftrace *rec); 1732static struct ftrace_ops * 1733ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude); 1734static struct ftrace_ops * 1735ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, struct ftrace_ops *ops); 1736 1737static bool skip_record(struct dyn_ftrace *rec) 1738{ 1739 /* 1740 * At boot up, weak functions are set to disable. Function tracing 1741 * can be enabled before they are, and they still need to be disabled now. 1742 * If the record is disabled, still continue if it is marked as already 1743 * enabled (this is needed to keep the accounting working). 1744 */ 1745 return rec->flags & FTRACE_FL_DISABLED && 1746 !(rec->flags & FTRACE_FL_ENABLED); 1747} 1748 1749/* 1750 * This is the main engine to the ftrace updates to the dyn_ftrace records. 1751 * 1752 * It will iterate through all the available ftrace functions 1753 * (the ones that ftrace can have callbacks to) and set the flags 1754 * in the associated dyn_ftrace records. 1755 * 1756 * @inc: If true, the functions associated to @ops are added to 1757 * the dyn_ftrace records, otherwise they are removed. 1758 */ 1759static bool __ftrace_hash_rec_update(struct ftrace_ops *ops, 1760 bool inc) 1761{ 1762 struct ftrace_hash *hash; 1763 struct ftrace_hash *notrace_hash; 1764 struct ftrace_page *pg; 1765 struct dyn_ftrace *rec; 1766 bool update = false; 1767 int count = 0; 1768 int all = false; 1769 1770 /* Only update if the ops has been registered */ 1771 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 1772 return false; 1773 1774 /* 1775 * If the count is zero, we update all records. 1776 * Otherwise we just update the items in the hash. 1777 */ 1778 hash = ops->func_hash->filter_hash; 1779 notrace_hash = ops->func_hash->notrace_hash; 1780 if (ftrace_hash_empty(hash)) 1781 all = true; 1782 1783 do_for_each_ftrace_rec(pg, rec) { 1784 int in_notrace_hash = 0; 1785 int in_hash = 0; 1786 int match = 0; 1787 1788 if (skip_record(rec)) 1789 continue; 1790 1791 if (all) { 1792 /* 1793 * Only the filter_hash affects all records. 1794 * Update if the record is not in the notrace hash. 1795 */ 1796 if (!notrace_hash || !ftrace_lookup_ip(notrace_hash, rec->ip)) 1797 match = 1; 1798 } else { 1799 in_hash = !!ftrace_lookup_ip(hash, rec->ip); 1800 in_notrace_hash = !!ftrace_lookup_ip(notrace_hash, rec->ip); 1801 1802 /* 1803 * We want to match all functions that are in the hash but 1804 * not in the other hash. 1805 */ 1806 if (in_hash && !in_notrace_hash) 1807 match = 1; 1808 } 1809 if (!match) 1810 continue; 1811 1812 if (inc) { 1813 rec->flags++; 1814 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX)) 1815 return false; 1816 1817 if (ops->flags & FTRACE_OPS_FL_DIRECT) 1818 rec->flags |= FTRACE_FL_DIRECT; 1819 1820 /* 1821 * If there's only a single callback registered to a 1822 * function, and the ops has a trampoline registered 1823 * for it, then we can call it directly. 1824 */ 1825 if (ftrace_rec_count(rec) == 1 && ops->trampoline) 1826 rec->flags |= FTRACE_FL_TRAMP; 1827 else 1828 /* 1829 * If we are adding another function callback 1830 * to this function, and the previous had a 1831 * custom trampoline in use, then we need to go 1832 * back to the default trampoline. 1833 */ 1834 rec->flags &= ~FTRACE_FL_TRAMP; 1835 1836 /* 1837 * If any ops wants regs saved for this function 1838 * then all ops will get saved regs. 1839 */ 1840 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 1841 rec->flags |= FTRACE_FL_REGS; 1842 } else { 1843 if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0)) 1844 return false; 1845 rec->flags--; 1846 1847 /* 1848 * Only the internal direct_ops should have the 1849 * DIRECT flag set. Thus, if it is removing a 1850 * function, then that function should no longer 1851 * be direct. 1852 */ 1853 if (ops->flags & FTRACE_OPS_FL_DIRECT) 1854 rec->flags &= ~FTRACE_FL_DIRECT; 1855 1856 /* 1857 * If the rec had REGS enabled and the ops that is 1858 * being removed had REGS set, then see if there is 1859 * still any ops for this record that wants regs. 1860 * If not, we can stop recording them. 1861 */ 1862 if (ftrace_rec_count(rec) > 0 && 1863 rec->flags & FTRACE_FL_REGS && 1864 ops->flags & FTRACE_OPS_FL_SAVE_REGS) { 1865 if (!test_rec_ops_needs_regs(rec)) 1866 rec->flags &= ~FTRACE_FL_REGS; 1867 } 1868 1869 /* 1870 * The TRAMP needs to be set only if rec count 1871 * is decremented to one, and the ops that is 1872 * left has a trampoline. As TRAMP can only be 1873 * enabled if there is only a single ops attached 1874 * to it. 1875 */ 1876 if (ftrace_rec_count(rec) == 1 && 1877 ftrace_find_tramp_ops_any_other(rec, ops)) 1878 rec->flags |= FTRACE_FL_TRAMP; 1879 else 1880 rec->flags &= ~FTRACE_FL_TRAMP; 1881 1882 /* 1883 * flags will be cleared in ftrace_check_record() 1884 * if rec count is zero. 1885 */ 1886 } 1887 1888 /* 1889 * If the rec has a single associated ops, and ops->func can be 1890 * called directly, allow the call site to call via the ops. 1891 */ 1892 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) && 1893 ftrace_rec_count(rec) == 1 && 1894 ftrace_ops_get_func(ops) == ops->func) 1895 rec->flags |= FTRACE_FL_CALL_OPS; 1896 else 1897 rec->flags &= ~FTRACE_FL_CALL_OPS; 1898 1899 count++; 1900 1901 /* Must match FTRACE_UPDATE_CALLS in ftrace_modify_all_code() */ 1902 update |= ftrace_test_record(rec, true) != FTRACE_UPDATE_IGNORE; 1903 1904 /* Shortcut, if we handled all records, we are done. */ 1905 if (!all && count == hash->count) 1906 return update; 1907 } while_for_each_ftrace_rec(); 1908 1909 return update; 1910} 1911 1912/* 1913 * This is called when an ops is removed from tracing. It will decrement 1914 * the counters of the dyn_ftrace records for all the functions that 1915 * the @ops attached to. 1916 */ 1917static bool ftrace_hash_rec_disable(struct ftrace_ops *ops) 1918{ 1919 return __ftrace_hash_rec_update(ops, false); 1920} 1921 1922/* 1923 * This is called when an ops is added to tracing. It will increment 1924 * the counters of the dyn_ftrace records for all the functions that 1925 * the @ops attached to. 1926 */ 1927static bool ftrace_hash_rec_enable(struct ftrace_ops *ops) 1928{ 1929 return __ftrace_hash_rec_update(ops, true); 1930} 1931 1932/* 1933 * This function will update what functions @ops traces when its filter 1934 * changes. 1935 * 1936 * The @inc states if the @ops callbacks are going to be added or removed. 1937 * When one of the @ops hashes are updated to a "new_hash" the dyn_ftrace 1938 * records are update via: 1939 * 1940 * ftrace_hash_rec_disable_modify(ops); 1941 * ops->hash = new_hash 1942 * ftrace_hash_rec_enable_modify(ops); 1943 * 1944 * Where the @ops is removed from all the records it is tracing using 1945 * its old hash. The @ops hash is updated to the new hash, and then 1946 * the @ops is added back to the records so that it is tracing all 1947 * the new functions. 1948 */ 1949static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops, bool inc) 1950{ 1951 struct ftrace_ops *op; 1952 1953 __ftrace_hash_rec_update(ops, inc); 1954 1955 if (ops->func_hash != &global_ops.local_hash) 1956 return; 1957 1958 /* 1959 * If the ops shares the global_ops hash, then we need to update 1960 * all ops that are enabled and use this hash. 1961 */ 1962 do_for_each_ftrace_op(op, ftrace_ops_list) { 1963 /* Already done */ 1964 if (op == ops) 1965 continue; 1966 if (op->func_hash == &global_ops.local_hash) 1967 __ftrace_hash_rec_update(op, inc); 1968 } while_for_each_ftrace_op(op); 1969} 1970 1971static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops) 1972{ 1973 ftrace_hash_rec_update_modify(ops, false); 1974} 1975 1976static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops) 1977{ 1978 ftrace_hash_rec_update_modify(ops, true); 1979} 1980 1981/* 1982 * Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK 1983 * or no-needed to update, -EBUSY if it detects a conflict of the flag 1984 * on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs. 1985 * Note that old_hash and new_hash has below meanings 1986 * - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected) 1987 * - If the hash is EMPTY_HASH, it hits nothing 1988 * - Anything else hits the recs which match the hash entries. 1989 * 1990 * DIRECT ops does not have IPMODIFY flag, but we still need to check it 1991 * against functions with FTRACE_FL_IPMODIFY. If there is any overlap, call 1992 * ops_func(SHARE_IPMODIFY_SELF) to make sure current ops can share with 1993 * IPMODIFY. If ops_func(SHARE_IPMODIFY_SELF) returns non-zero, propagate 1994 * the return value to the caller and eventually to the owner of the DIRECT 1995 * ops. 1996 */ 1997static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops, 1998 struct ftrace_hash *old_hash, 1999 struct ftrace_hash *new_hash, 2000 bool update_target) 2001{ 2002 struct ftrace_page *pg; 2003 struct dyn_ftrace *rec, *end = NULL; 2004 int in_old, in_new; 2005 bool is_ipmodify, is_direct; 2006 2007 /* Only update if the ops has been registered */ 2008 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 2009 return 0; 2010 2011 is_ipmodify = ops->flags & FTRACE_OPS_FL_IPMODIFY; 2012 is_direct = ops->flags & FTRACE_OPS_FL_DIRECT; 2013 2014 /* neither IPMODIFY nor DIRECT, skip */ 2015 if (!is_ipmodify && !is_direct) 2016 return 0; 2017 2018 if (WARN_ON_ONCE(is_ipmodify && is_direct)) 2019 return 0; 2020 2021 /* 2022 * Since the IPMODIFY and DIRECT are very address sensitive 2023 * actions, we do not allow ftrace_ops to set all functions to new 2024 * hash. 2025 */ 2026 if (!new_hash || !old_hash) 2027 return -EINVAL; 2028 2029 /* Update rec->flags */ 2030 do_for_each_ftrace_rec(pg, rec) { 2031 2032 if (rec->flags & FTRACE_FL_DISABLED) 2033 continue; 2034 2035 /* 2036 * Unless we are updating the target of a direct function, 2037 * we only need to update differences of filter_hash 2038 */ 2039 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 2040 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 2041 if (!update_target && (in_old == in_new)) 2042 continue; 2043 2044 if (in_new) { 2045 if (rec->flags & FTRACE_FL_IPMODIFY) { 2046 int ret; 2047 2048 /* Cannot have two ipmodify on same rec */ 2049 if (is_ipmodify) 2050 goto rollback; 2051 2052 /* 2053 * If this is called by __modify_ftrace_direct() 2054 * then it is only changing where the direct 2055 * pointer is jumping to, and the record already 2056 * points to a direct trampoline. If it isn't, 2057 * then it is a bug to update ipmodify on a direct 2058 * caller. 2059 */ 2060 FTRACE_WARN_ON(!update_target && 2061 (rec->flags & FTRACE_FL_DIRECT)); 2062 2063 /* 2064 * Another ops with IPMODIFY is already 2065 * attached. We are now attaching a direct 2066 * ops. Run SHARE_IPMODIFY_SELF, to check 2067 * whether sharing is supported. 2068 */ 2069 if (!ops->ops_func) 2070 return -EBUSY; 2071 ret = ops->ops_func(ops, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_SELF); 2072 if (ret) 2073 return ret; 2074 } else if (is_ipmodify) { 2075 rec->flags |= FTRACE_FL_IPMODIFY; 2076 } 2077 } else if (is_ipmodify) { 2078 rec->flags &= ~FTRACE_FL_IPMODIFY; 2079 } 2080 } while_for_each_ftrace_rec(); 2081 2082 return 0; 2083 2084rollback: 2085 end = rec; 2086 2087 /* Roll back what we did above */ 2088 do_for_each_ftrace_rec(pg, rec) { 2089 2090 if (rec->flags & FTRACE_FL_DISABLED) 2091 continue; 2092 2093 if (rec == end) 2094 return -EBUSY; 2095 2096 in_old = !!ftrace_lookup_ip(old_hash, rec->ip); 2097 in_new = !!ftrace_lookup_ip(new_hash, rec->ip); 2098 if (in_old == in_new) 2099 continue; 2100 2101 if (in_new) 2102 rec->flags &= ~FTRACE_FL_IPMODIFY; 2103 else 2104 rec->flags |= FTRACE_FL_IPMODIFY; 2105 } while_for_each_ftrace_rec(); 2106 2107 return -EBUSY; 2108} 2109 2110static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops) 2111{ 2112 struct ftrace_hash *hash = ops->func_hash->filter_hash; 2113 2114 if (ftrace_hash_empty(hash)) 2115 hash = NULL; 2116 2117 return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash, false); 2118} 2119 2120/* Disabling always succeeds */ 2121static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops) 2122{ 2123 struct ftrace_hash *hash = ops->func_hash->filter_hash; 2124 2125 if (ftrace_hash_empty(hash)) 2126 hash = NULL; 2127 2128 __ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH, false); 2129} 2130 2131static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops, 2132 struct ftrace_hash *new_hash) 2133{ 2134 struct ftrace_hash *old_hash = ops->func_hash->filter_hash; 2135 2136 if (ftrace_hash_empty(old_hash)) 2137 old_hash = NULL; 2138 2139 if (ftrace_hash_empty(new_hash)) 2140 new_hash = NULL; 2141 2142 return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash, false); 2143} 2144 2145static void print_ip_ins(const char *fmt, const unsigned char *p) 2146{ 2147 char ins[MCOUNT_INSN_SIZE]; 2148 2149 if (copy_from_kernel_nofault(ins, p, MCOUNT_INSN_SIZE)) { 2150 printk(KERN_CONT "%s[FAULT] %px\n", fmt, p); 2151 return; 2152 } 2153 2154 printk(KERN_CONT "%s", fmt); 2155 pr_cont("%*phC", MCOUNT_INSN_SIZE, ins); 2156} 2157 2158enum ftrace_bug_type ftrace_bug_type; 2159const void *ftrace_expected; 2160 2161static void print_bug_type(void) 2162{ 2163 switch (ftrace_bug_type) { 2164 case FTRACE_BUG_UNKNOWN: 2165 break; 2166 case FTRACE_BUG_INIT: 2167 pr_info("Initializing ftrace call sites\n"); 2168 break; 2169 case FTRACE_BUG_NOP: 2170 pr_info("Setting ftrace call site to NOP\n"); 2171 break; 2172 case FTRACE_BUG_CALL: 2173 pr_info("Setting ftrace call site to call ftrace function\n"); 2174 break; 2175 case FTRACE_BUG_UPDATE: 2176 pr_info("Updating ftrace call site to call a different ftrace function\n"); 2177 break; 2178 } 2179} 2180 2181/** 2182 * ftrace_bug - report and shutdown function tracer 2183 * @failed: The failed type (EFAULT, EINVAL, EPERM) 2184 * @rec: The record that failed 2185 * 2186 * The arch code that enables or disables the function tracing 2187 * can call ftrace_bug() when it has detected a problem in 2188 * modifying the code. @failed should be one of either: 2189 * EFAULT - if the problem happens on reading the @ip address 2190 * EINVAL - if what is read at @ip is not what was expected 2191 * EPERM - if the problem happens on writing to the @ip address 2192 */ 2193void ftrace_bug(int failed, struct dyn_ftrace *rec) 2194{ 2195 unsigned long ip = rec ? rec->ip : 0; 2196 2197 pr_info("------------[ ftrace bug ]------------\n"); 2198 2199 switch (failed) { 2200 case -EFAULT: 2201 pr_info("ftrace faulted on modifying "); 2202 print_ip_sym(KERN_INFO, ip); 2203 break; 2204 case -EINVAL: 2205 pr_info("ftrace failed to modify "); 2206 print_ip_sym(KERN_INFO, ip); 2207 print_ip_ins(" actual: ", (unsigned char *)ip); 2208 pr_cont("\n"); 2209 if (ftrace_expected) { 2210 print_ip_ins(" expected: ", ftrace_expected); 2211 pr_cont("\n"); 2212 } 2213 break; 2214 case -EPERM: 2215 pr_info("ftrace faulted on writing "); 2216 print_ip_sym(KERN_INFO, ip); 2217 break; 2218 default: 2219 pr_info("ftrace faulted on unknown error "); 2220 print_ip_sym(KERN_INFO, ip); 2221 } 2222 print_bug_type(); 2223 if (rec) { 2224 struct ftrace_ops *ops = NULL; 2225 2226 pr_info("ftrace record flags: %lx\n", rec->flags); 2227 pr_cont(" (%ld)%s%s", ftrace_rec_count(rec), 2228 rec->flags & FTRACE_FL_REGS ? " R" : " ", 2229 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " "); 2230 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2231 ops = ftrace_find_tramp_ops_any(rec); 2232 if (ops) { 2233 do { 2234 pr_cont("\ttramp: %pS (%pS)", 2235 (void *)ops->trampoline, 2236 (void *)ops->func); 2237 ops = ftrace_find_tramp_ops_next(rec, ops); 2238 } while (ops); 2239 } else 2240 pr_cont("\ttramp: ERROR!"); 2241 2242 } 2243 ip = ftrace_get_addr_curr(rec); 2244 pr_cont("\n expected tramp: %lx\n", ip); 2245 } 2246 2247 FTRACE_WARN_ON_ONCE(1); 2248} 2249 2250static int ftrace_check_record(struct dyn_ftrace *rec, bool enable, bool update) 2251{ 2252 unsigned long flag = 0UL; 2253 2254 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2255 2256 if (skip_record(rec)) 2257 return FTRACE_UPDATE_IGNORE; 2258 2259 /* 2260 * If we are updating calls: 2261 * 2262 * If the record has a ref count, then we need to enable it 2263 * because someone is using it. 2264 * 2265 * Otherwise we make sure its disabled. 2266 * 2267 * If we are disabling calls, then disable all records that 2268 * are enabled. 2269 */ 2270 if (enable && ftrace_rec_count(rec)) 2271 flag = FTRACE_FL_ENABLED; 2272 2273 /* 2274 * If enabling and the REGS flag does not match the REGS_EN, or 2275 * the TRAMP flag doesn't match the TRAMP_EN, then do not ignore 2276 * this record. Set flags to fail the compare against ENABLED. 2277 * Same for direct calls. 2278 */ 2279 if (flag) { 2280 if (!(rec->flags & FTRACE_FL_REGS) != 2281 !(rec->flags & FTRACE_FL_REGS_EN)) 2282 flag |= FTRACE_FL_REGS; 2283 2284 if (!(rec->flags & FTRACE_FL_TRAMP) != 2285 !(rec->flags & FTRACE_FL_TRAMP_EN)) 2286 flag |= FTRACE_FL_TRAMP; 2287 2288 /* 2289 * Direct calls are special, as count matters. 2290 * We must test the record for direct, if the 2291 * DIRECT and DIRECT_EN do not match, but only 2292 * if the count is 1. That's because, if the 2293 * count is something other than one, we do not 2294 * want the direct enabled (it will be done via the 2295 * direct helper). But if DIRECT_EN is set, and 2296 * the count is not one, we need to clear it. 2297 * 2298 */ 2299 if (ftrace_rec_count(rec) == 1) { 2300 if (!(rec->flags & FTRACE_FL_DIRECT) != 2301 !(rec->flags & FTRACE_FL_DIRECT_EN)) 2302 flag |= FTRACE_FL_DIRECT; 2303 } else if (rec->flags & FTRACE_FL_DIRECT_EN) { 2304 flag |= FTRACE_FL_DIRECT; 2305 } 2306 2307 /* 2308 * Ops calls are special, as count matters. 2309 * As with direct calls, they must only be enabled when count 2310 * is one, otherwise they'll be handled via the list ops. 2311 */ 2312 if (ftrace_rec_count(rec) == 1) { 2313 if (!(rec->flags & FTRACE_FL_CALL_OPS) != 2314 !(rec->flags & FTRACE_FL_CALL_OPS_EN)) 2315 flag |= FTRACE_FL_CALL_OPS; 2316 } else if (rec->flags & FTRACE_FL_CALL_OPS_EN) { 2317 flag |= FTRACE_FL_CALL_OPS; 2318 } 2319 } 2320 2321 /* If the state of this record hasn't changed, then do nothing */ 2322 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 2323 return FTRACE_UPDATE_IGNORE; 2324 2325 if (flag) { 2326 /* Save off if rec is being enabled (for return value) */ 2327 flag ^= rec->flags & FTRACE_FL_ENABLED; 2328 2329 if (update) { 2330 rec->flags |= FTRACE_FL_ENABLED | FTRACE_FL_TOUCHED; 2331 if (flag & FTRACE_FL_REGS) { 2332 if (rec->flags & FTRACE_FL_REGS) 2333 rec->flags |= FTRACE_FL_REGS_EN; 2334 else 2335 rec->flags &= ~FTRACE_FL_REGS_EN; 2336 } 2337 if (flag & FTRACE_FL_TRAMP) { 2338 if (rec->flags & FTRACE_FL_TRAMP) 2339 rec->flags |= FTRACE_FL_TRAMP_EN; 2340 else 2341 rec->flags &= ~FTRACE_FL_TRAMP_EN; 2342 } 2343 2344 /* Keep track of anything that modifies the function */ 2345 if (rec->flags & (FTRACE_FL_DIRECT | FTRACE_FL_IPMODIFY)) 2346 rec->flags |= FTRACE_FL_MODIFIED; 2347 2348 if (flag & FTRACE_FL_DIRECT) { 2349 /* 2350 * If there's only one user (direct_ops helper) 2351 * then we can call the direct function 2352 * directly (no ftrace trampoline). 2353 */ 2354 if (ftrace_rec_count(rec) == 1) { 2355 if (rec->flags & FTRACE_FL_DIRECT) 2356 rec->flags |= FTRACE_FL_DIRECT_EN; 2357 else 2358 rec->flags &= ~FTRACE_FL_DIRECT_EN; 2359 } else { 2360 /* 2361 * Can only call directly if there's 2362 * only one callback to the function. 2363 */ 2364 rec->flags &= ~FTRACE_FL_DIRECT_EN; 2365 } 2366 } 2367 2368 if (flag & FTRACE_FL_CALL_OPS) { 2369 if (ftrace_rec_count(rec) == 1) { 2370 if (rec->flags & FTRACE_FL_CALL_OPS) 2371 rec->flags |= FTRACE_FL_CALL_OPS_EN; 2372 else 2373 rec->flags &= ~FTRACE_FL_CALL_OPS_EN; 2374 } else { 2375 /* 2376 * Can only call directly if there's 2377 * only one set of associated ops. 2378 */ 2379 rec->flags &= ~FTRACE_FL_CALL_OPS_EN; 2380 } 2381 } 2382 } 2383 2384 /* 2385 * If this record is being updated from a nop, then 2386 * return UPDATE_MAKE_CALL. 2387 * Otherwise, 2388 * return UPDATE_MODIFY_CALL to tell the caller to convert 2389 * from the save regs, to a non-save regs function or 2390 * vice versa, or from a trampoline call. 2391 */ 2392 if (flag & FTRACE_FL_ENABLED) { 2393 ftrace_bug_type = FTRACE_BUG_CALL; 2394 return FTRACE_UPDATE_MAKE_CALL; 2395 } 2396 2397 ftrace_bug_type = FTRACE_BUG_UPDATE; 2398 return FTRACE_UPDATE_MODIFY_CALL; 2399 } 2400 2401 if (update) { 2402 /* If there's no more users, clear all flags */ 2403 if (!ftrace_rec_count(rec)) 2404 rec->flags &= FTRACE_NOCLEAR_FLAGS; 2405 else 2406 /* 2407 * Just disable the record, but keep the ops TRAMP 2408 * and REGS states. The _EN flags must be disabled though. 2409 */ 2410 rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN | 2411 FTRACE_FL_REGS_EN | FTRACE_FL_DIRECT_EN | 2412 FTRACE_FL_CALL_OPS_EN); 2413 } 2414 2415 ftrace_bug_type = FTRACE_BUG_NOP; 2416 return FTRACE_UPDATE_MAKE_NOP; 2417} 2418 2419/** 2420 * ftrace_update_record - set a record that now is tracing or not 2421 * @rec: the record to update 2422 * @enable: set to true if the record is tracing, false to force disable 2423 * 2424 * The records that represent all functions that can be traced need 2425 * to be updated when tracing has been enabled. 2426 */ 2427int ftrace_update_record(struct dyn_ftrace *rec, bool enable) 2428{ 2429 return ftrace_check_record(rec, enable, true); 2430} 2431 2432/** 2433 * ftrace_test_record - check if the record has been enabled or not 2434 * @rec: the record to test 2435 * @enable: set to true to check if enabled, false if it is disabled 2436 * 2437 * The arch code may need to test if a record is already set to 2438 * tracing to determine how to modify the function code that it 2439 * represents. 2440 */ 2441int ftrace_test_record(struct dyn_ftrace *rec, bool enable) 2442{ 2443 return ftrace_check_record(rec, enable, false); 2444} 2445 2446static struct ftrace_ops * 2447ftrace_find_tramp_ops_any(struct dyn_ftrace *rec) 2448{ 2449 struct ftrace_ops *op; 2450 unsigned long ip = rec->ip; 2451 2452 do_for_each_ftrace_op(op, ftrace_ops_list) { 2453 2454 if (!op->trampoline) 2455 continue; 2456 2457 if (hash_contains_ip(ip, op->func_hash)) 2458 return op; 2459 } while_for_each_ftrace_op(op); 2460 2461 return NULL; 2462} 2463 2464static struct ftrace_ops * 2465ftrace_find_tramp_ops_any_other(struct dyn_ftrace *rec, struct ftrace_ops *op_exclude) 2466{ 2467 struct ftrace_ops *op; 2468 unsigned long ip = rec->ip; 2469 2470 do_for_each_ftrace_op(op, ftrace_ops_list) { 2471 2472 if (op == op_exclude || !op->trampoline) 2473 continue; 2474 2475 if (hash_contains_ip(ip, op->func_hash)) 2476 return op; 2477 } while_for_each_ftrace_op(op); 2478 2479 return NULL; 2480} 2481 2482static struct ftrace_ops * 2483ftrace_find_tramp_ops_next(struct dyn_ftrace *rec, 2484 struct ftrace_ops *op) 2485{ 2486 unsigned long ip = rec->ip; 2487 2488 while_for_each_ftrace_op(op) { 2489 2490 if (!op->trampoline) 2491 continue; 2492 2493 if (hash_contains_ip(ip, op->func_hash)) 2494 return op; 2495 } 2496 2497 return NULL; 2498} 2499 2500static struct ftrace_ops * 2501ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec) 2502{ 2503 struct ftrace_ops *op; 2504 unsigned long ip = rec->ip; 2505 2506 /* 2507 * Need to check removed ops first. 2508 * If they are being removed, and this rec has a tramp, 2509 * and this rec is in the ops list, then it would be the 2510 * one with the tramp. 2511 */ 2512 if (removed_ops) { 2513 if (hash_contains_ip(ip, &removed_ops->old_hash)) 2514 return removed_ops; 2515 } 2516 2517 /* 2518 * Need to find the current trampoline for a rec. 2519 * Now, a trampoline is only attached to a rec if there 2520 * was a single 'ops' attached to it. But this can be called 2521 * when we are adding another op to the rec or removing the 2522 * current one. Thus, if the op is being added, we can 2523 * ignore it because it hasn't attached itself to the rec 2524 * yet. 2525 * 2526 * If an ops is being modified (hooking to different functions) 2527 * then we don't care about the new functions that are being 2528 * added, just the old ones (that are probably being removed). 2529 * 2530 * If we are adding an ops to a function that already is using 2531 * a trampoline, it needs to be removed (trampolines are only 2532 * for single ops connected), then an ops that is not being 2533 * modified also needs to be checked. 2534 */ 2535 do_for_each_ftrace_op(op, ftrace_ops_list) { 2536 2537 if (!op->trampoline) 2538 continue; 2539 2540 /* 2541 * If the ops is being added, it hasn't gotten to 2542 * the point to be removed from this tree yet. 2543 */ 2544 if (op->flags & FTRACE_OPS_FL_ADDING) 2545 continue; 2546 2547 2548 /* 2549 * If the ops is being modified and is in the old 2550 * hash, then it is probably being removed from this 2551 * function. 2552 */ 2553 if ((op->flags & FTRACE_OPS_FL_MODIFYING) && 2554 hash_contains_ip(ip, &op->old_hash)) 2555 return op; 2556 /* 2557 * If the ops is not being added or modified, and it's 2558 * in its normal filter hash, then this must be the one 2559 * we want! 2560 */ 2561 if (!(op->flags & FTRACE_OPS_FL_MODIFYING) && 2562 hash_contains_ip(ip, op->func_hash)) 2563 return op; 2564 2565 } while_for_each_ftrace_op(op); 2566 2567 return NULL; 2568} 2569 2570static struct ftrace_ops * 2571ftrace_find_tramp_ops_new(struct dyn_ftrace *rec) 2572{ 2573 struct ftrace_ops *op; 2574 unsigned long ip = rec->ip; 2575 2576 do_for_each_ftrace_op(op, ftrace_ops_list) { 2577 /* pass rec in as regs to have non-NULL val */ 2578 if (hash_contains_ip(ip, op->func_hash)) 2579 return op; 2580 } while_for_each_ftrace_op(op); 2581 2582 return NULL; 2583} 2584 2585struct ftrace_ops * 2586ftrace_find_unique_ops(struct dyn_ftrace *rec) 2587{ 2588 struct ftrace_ops *op, *found = NULL; 2589 unsigned long ip = rec->ip; 2590 2591 do_for_each_ftrace_op(op, ftrace_ops_list) { 2592 2593 if (hash_contains_ip(ip, op->func_hash)) { 2594 if (found) 2595 return NULL; 2596 found = op; 2597 } 2598 2599 } while_for_each_ftrace_op(op); 2600 2601 return found; 2602} 2603 2604#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS 2605/* Protected by rcu_tasks for reading, and direct_mutex for writing */ 2606static struct ftrace_hash __rcu *direct_functions = EMPTY_HASH; 2607static DEFINE_MUTEX(direct_mutex); 2608 2609/* 2610 * Search the direct_functions hash to see if the given instruction pointer 2611 * has a direct caller attached to it. 2612 */ 2613unsigned long ftrace_find_rec_direct(unsigned long ip) 2614{ 2615 struct ftrace_func_entry *entry; 2616 2617 entry = __ftrace_lookup_ip(direct_functions, ip); 2618 if (!entry) 2619 return 0; 2620 2621 return entry->direct; 2622} 2623 2624static void call_direct_funcs(unsigned long ip, unsigned long pip, 2625 struct ftrace_ops *ops, struct ftrace_regs *fregs) 2626{ 2627 unsigned long addr = READ_ONCE(ops->direct_call); 2628 2629 if (!addr) 2630 return; 2631 2632 arch_ftrace_set_direct_caller(fregs, addr); 2633} 2634#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ 2635 2636/** 2637 * ftrace_get_addr_new - Get the call address to set to 2638 * @rec: The ftrace record descriptor 2639 * 2640 * If the record has the FTRACE_FL_REGS set, that means that it 2641 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS 2642 * is not set, then it wants to convert to the normal callback. 2643 * 2644 * Returns: the address of the trampoline to set to 2645 */ 2646unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec) 2647{ 2648 struct ftrace_ops *ops; 2649 unsigned long addr; 2650 2651 if ((rec->flags & FTRACE_FL_DIRECT) && 2652 (ftrace_rec_count(rec) == 1)) { 2653 addr = ftrace_find_rec_direct(rec->ip); 2654 if (addr) 2655 return addr; 2656 WARN_ON_ONCE(1); 2657 } 2658 2659 /* Trampolines take precedence over regs */ 2660 if (rec->flags & FTRACE_FL_TRAMP) { 2661 ops = ftrace_find_tramp_ops_new(rec); 2662 if (FTRACE_WARN_ON(!ops || !ops->trampoline)) { 2663 pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n", 2664 (void *)rec->ip, (void *)rec->ip, rec->flags); 2665 /* Ftrace is shutting down, return anything */ 2666 return (unsigned long)FTRACE_ADDR; 2667 } 2668 return ops->trampoline; 2669 } 2670 2671 if (rec->flags & FTRACE_FL_REGS) 2672 return (unsigned long)FTRACE_REGS_ADDR; 2673 else 2674 return (unsigned long)FTRACE_ADDR; 2675} 2676 2677/** 2678 * ftrace_get_addr_curr - Get the call address that is already there 2679 * @rec: The ftrace record descriptor 2680 * 2681 * The FTRACE_FL_REGS_EN is set when the record already points to 2682 * a function that saves all the regs. Basically the '_EN' version 2683 * represents the current state of the function. 2684 * 2685 * Returns: the address of the trampoline that is currently being called 2686 */ 2687unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec) 2688{ 2689 struct ftrace_ops *ops; 2690 unsigned long addr; 2691 2692 /* Direct calls take precedence over trampolines */ 2693 if (rec->flags & FTRACE_FL_DIRECT_EN) { 2694 addr = ftrace_find_rec_direct(rec->ip); 2695 if (addr) 2696 return addr; 2697 WARN_ON_ONCE(1); 2698 } 2699 2700 /* Trampolines take precedence over regs */ 2701 if (rec->flags & FTRACE_FL_TRAMP_EN) { 2702 ops = ftrace_find_tramp_ops_curr(rec); 2703 if (FTRACE_WARN_ON(!ops)) { 2704 pr_warn("Bad trampoline accounting at: %p (%pS)\n", 2705 (void *)rec->ip, (void *)rec->ip); 2706 /* Ftrace is shutting down, return anything */ 2707 return (unsigned long)FTRACE_ADDR; 2708 } 2709 return ops->trampoline; 2710 } 2711 2712 if (rec->flags & FTRACE_FL_REGS_EN) 2713 return (unsigned long)FTRACE_REGS_ADDR; 2714 else 2715 return (unsigned long)FTRACE_ADDR; 2716} 2717 2718static int 2719__ftrace_replace_code(struct dyn_ftrace *rec, bool enable) 2720{ 2721 unsigned long ftrace_old_addr; 2722 unsigned long ftrace_addr; 2723 int ret; 2724 2725 ftrace_addr = ftrace_get_addr_new(rec); 2726 2727 /* This needs to be done before we call ftrace_update_record */ 2728 ftrace_old_addr = ftrace_get_addr_curr(rec); 2729 2730 ret = ftrace_update_record(rec, enable); 2731 2732 ftrace_bug_type = FTRACE_BUG_UNKNOWN; 2733 2734 switch (ret) { 2735 case FTRACE_UPDATE_IGNORE: 2736 return 0; 2737 2738 case FTRACE_UPDATE_MAKE_CALL: 2739 ftrace_bug_type = FTRACE_BUG_CALL; 2740 return ftrace_make_call(rec, ftrace_addr); 2741 2742 case FTRACE_UPDATE_MAKE_NOP: 2743 ftrace_bug_type = FTRACE_BUG_NOP; 2744 return ftrace_make_nop(NULL, rec, ftrace_old_addr); 2745 2746 case FTRACE_UPDATE_MODIFY_CALL: 2747 ftrace_bug_type = FTRACE_BUG_UPDATE; 2748 return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr); 2749 } 2750 2751 return -1; /* unknown ftrace bug */ 2752} 2753 2754void __weak ftrace_replace_code(int mod_flags) 2755{ 2756 struct dyn_ftrace *rec; 2757 struct ftrace_page *pg; 2758 bool enable = mod_flags & FTRACE_MODIFY_ENABLE_FL; 2759 int schedulable = mod_flags & FTRACE_MODIFY_MAY_SLEEP_FL; 2760 int failed; 2761 2762 if (unlikely(ftrace_disabled)) 2763 return; 2764 2765 do_for_each_ftrace_rec(pg, rec) { 2766 2767 if (skip_record(rec)) 2768 continue; 2769 2770 failed = __ftrace_replace_code(rec, enable); 2771 if (failed) { 2772 ftrace_bug(failed, rec); 2773 /* Stop processing */ 2774 return; 2775 } 2776 if (schedulable) 2777 cond_resched(); 2778 } while_for_each_ftrace_rec(); 2779} 2780 2781struct ftrace_rec_iter { 2782 struct ftrace_page *pg; 2783 int index; 2784}; 2785 2786/** 2787 * ftrace_rec_iter_start - start up iterating over traced functions 2788 * 2789 * Returns: an iterator handle that is used to iterate over all 2790 * the records that represent address locations where functions 2791 * are traced. 2792 * 2793 * May return NULL if no records are available. 2794 */ 2795struct ftrace_rec_iter *ftrace_rec_iter_start(void) 2796{ 2797 /* 2798 * We only use a single iterator. 2799 * Protected by the ftrace_lock mutex. 2800 */ 2801 static struct ftrace_rec_iter ftrace_rec_iter; 2802 struct ftrace_rec_iter *iter = &ftrace_rec_iter; 2803 2804 iter->pg = ftrace_pages_start; 2805 iter->index = 0; 2806 2807 /* Could have empty pages */ 2808 while (iter->pg && !iter->pg->index) 2809 iter->pg = iter->pg->next; 2810 2811 if (!iter->pg) 2812 return NULL; 2813 2814 return iter; 2815} 2816 2817/** 2818 * ftrace_rec_iter_next - get the next record to process. 2819 * @iter: The handle to the iterator. 2820 * 2821 * Returns: the next iterator after the given iterator @iter. 2822 */ 2823struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter) 2824{ 2825 iter->index++; 2826 2827 if (iter->index >= iter->pg->index) { 2828 iter->pg = iter->pg->next; 2829 iter->index = 0; 2830 2831 /* Could have empty pages */ 2832 while (iter->pg && !iter->pg->index) 2833 iter->pg = iter->pg->next; 2834 } 2835 2836 if (!iter->pg) 2837 return NULL; 2838 2839 return iter; 2840} 2841 2842/** 2843 * ftrace_rec_iter_record - get the record at the iterator location 2844 * @iter: The current iterator location 2845 * 2846 * Returns: the record that the current @iter is at. 2847 */ 2848struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter) 2849{ 2850 return &iter->pg->records[iter->index]; 2851} 2852 2853static int 2854ftrace_nop_initialize(struct module *mod, struct dyn_ftrace *rec) 2855{ 2856 int ret; 2857 2858 if (unlikely(ftrace_disabled)) 2859 return 0; 2860 2861 ret = ftrace_init_nop(mod, rec); 2862 if (ret) { 2863 ftrace_bug_type = FTRACE_BUG_INIT; 2864 ftrace_bug(ret, rec); 2865 return 0; 2866 } 2867 return 1; 2868} 2869 2870/* 2871 * archs can override this function if they must do something 2872 * before the modifying code is performed. 2873 */ 2874void __weak ftrace_arch_code_modify_prepare(void) 2875{ 2876} 2877 2878/* 2879 * archs can override this function if they must do something 2880 * after the modifying code is performed. 2881 */ 2882void __weak ftrace_arch_code_modify_post_process(void) 2883{ 2884} 2885 2886static int update_ftrace_func(ftrace_func_t func) 2887{ 2888 static ftrace_func_t save_func; 2889 2890 /* Avoid updating if it hasn't changed */ 2891 if (func == save_func) 2892 return 0; 2893 2894 save_func = func; 2895 2896 return ftrace_update_ftrace_func(func); 2897} 2898 2899void ftrace_modify_all_code(int command) 2900{ 2901 int update = command & FTRACE_UPDATE_TRACE_FUNC; 2902 int mod_flags = 0; 2903 int err = 0; 2904 2905 if (command & FTRACE_MAY_SLEEP) 2906 mod_flags = FTRACE_MODIFY_MAY_SLEEP_FL; 2907 2908 /* 2909 * If the ftrace_caller calls a ftrace_ops func directly, 2910 * we need to make sure that it only traces functions it 2911 * expects to trace. When doing the switch of functions, 2912 * we need to update to the ftrace_ops_list_func first 2913 * before the transition between old and new calls are set, 2914 * as the ftrace_ops_list_func will check the ops hashes 2915 * to make sure the ops are having the right functions 2916 * traced. 2917 */ 2918 if (update) { 2919 err = update_ftrace_func(ftrace_ops_list_func); 2920 if (FTRACE_WARN_ON(err)) 2921 return; 2922 } 2923 2924 if (command & FTRACE_UPDATE_CALLS) 2925 ftrace_replace_code(mod_flags | FTRACE_MODIFY_ENABLE_FL); 2926 else if (command & FTRACE_DISABLE_CALLS) 2927 ftrace_replace_code(mod_flags); 2928 2929 if (update && ftrace_trace_function != ftrace_ops_list_func) { 2930 function_trace_op = set_function_trace_op; 2931 smp_wmb(); 2932 /* If irqs are disabled, we are in stop machine */ 2933 if (!irqs_disabled()) 2934 smp_call_function(ftrace_sync_ipi, NULL, 1); 2935 err = update_ftrace_func(ftrace_trace_function); 2936 if (FTRACE_WARN_ON(err)) 2937 return; 2938 } 2939 2940 if (command & FTRACE_START_FUNC_RET) 2941 err = ftrace_enable_ftrace_graph_caller(); 2942 else if (command & FTRACE_STOP_FUNC_RET) 2943 err = ftrace_disable_ftrace_graph_caller(); 2944 FTRACE_WARN_ON(err); 2945} 2946 2947static int __ftrace_modify_code(void *data) 2948{ 2949 int *command = data; 2950 2951 ftrace_modify_all_code(*command); 2952 2953 return 0; 2954} 2955 2956/** 2957 * ftrace_run_stop_machine - go back to the stop machine method 2958 * @command: The command to tell ftrace what to do 2959 * 2960 * If an arch needs to fall back to the stop machine method, the 2961 * it can call this function. 2962 */ 2963void ftrace_run_stop_machine(int command) 2964{ 2965 stop_machine(__ftrace_modify_code, &command, NULL); 2966} 2967 2968/** 2969 * arch_ftrace_update_code - modify the code to trace or not trace 2970 * @command: The command that needs to be done 2971 * 2972 * Archs can override this function if it does not need to 2973 * run stop_machine() to modify code. 2974 */ 2975void __weak arch_ftrace_update_code(int command) 2976{ 2977 ftrace_run_stop_machine(command); 2978} 2979 2980static void ftrace_run_update_code(int command) 2981{ 2982 ftrace_arch_code_modify_prepare(); 2983 2984 /* 2985 * By default we use stop_machine() to modify the code. 2986 * But archs can do what ever they want as long as it 2987 * is safe. The stop_machine() is the safest, but also 2988 * produces the most overhead. 2989 */ 2990 arch_ftrace_update_code(command); 2991 2992 ftrace_arch_code_modify_post_process(); 2993} 2994 2995static void ftrace_run_modify_code(struct ftrace_ops *ops, int command, 2996 struct ftrace_ops_hash *old_hash) 2997{ 2998 ops->flags |= FTRACE_OPS_FL_MODIFYING; 2999 ops->old_hash.filter_hash = old_hash->filter_hash; 3000 ops->old_hash.notrace_hash = old_hash->notrace_hash; 3001 ftrace_run_update_code(command); 3002 ops->old_hash.filter_hash = NULL; 3003 ops->old_hash.notrace_hash = NULL; 3004 ops->flags &= ~FTRACE_OPS_FL_MODIFYING; 3005} 3006 3007static ftrace_func_t saved_ftrace_func; 3008static int ftrace_start_up; 3009 3010void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops) 3011{ 3012} 3013 3014/* List of trace_ops that have allocated trampolines */ 3015static LIST_HEAD(ftrace_ops_trampoline_list); 3016 3017static void ftrace_add_trampoline_to_kallsyms(struct ftrace_ops *ops) 3018{ 3019 lockdep_assert_held(&ftrace_lock); 3020 list_add_rcu(&ops->list, &ftrace_ops_trampoline_list); 3021} 3022 3023static void ftrace_remove_trampoline_from_kallsyms(struct ftrace_ops *ops) 3024{ 3025 lockdep_assert_held(&ftrace_lock); 3026 list_del_rcu(&ops->list); 3027 synchronize_rcu(); 3028} 3029 3030/* 3031 * "__builtin__ftrace" is used as a module name in /proc/kallsyms for symbols 3032 * for pages allocated for ftrace purposes, even though "__builtin__ftrace" is 3033 * not a module. 3034 */ 3035#define FTRACE_TRAMPOLINE_MOD "__builtin__ftrace" 3036#define FTRACE_TRAMPOLINE_SYM "ftrace_trampoline" 3037 3038static void ftrace_trampoline_free(struct ftrace_ops *ops) 3039{ 3040 if (ops && (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP) && 3041 ops->trampoline) { 3042 /* 3043 * Record the text poke event before the ksymbol unregister 3044 * event. 3045 */ 3046 perf_event_text_poke((void *)ops->trampoline, 3047 (void *)ops->trampoline, 3048 ops->trampoline_size, NULL, 0); 3049 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, 3050 ops->trampoline, ops->trampoline_size, 3051 true, FTRACE_TRAMPOLINE_SYM); 3052 /* Remove from kallsyms after the perf events */ 3053 ftrace_remove_trampoline_from_kallsyms(ops); 3054 } 3055 3056 arch_ftrace_trampoline_free(ops); 3057} 3058 3059static void ftrace_startup_enable(int command) 3060{ 3061 if (saved_ftrace_func != ftrace_trace_function) { 3062 saved_ftrace_func = ftrace_trace_function; 3063 command |= FTRACE_UPDATE_TRACE_FUNC; 3064 } 3065 3066 if (!command || !ftrace_enabled) 3067 return; 3068 3069 ftrace_run_update_code(command); 3070} 3071 3072static void ftrace_startup_all(int command) 3073{ 3074 update_all_ops = true; 3075 ftrace_startup_enable(command); 3076 update_all_ops = false; 3077} 3078 3079int ftrace_startup(struct ftrace_ops *ops, int command) 3080{ 3081 int ret; 3082 3083 if (unlikely(ftrace_disabled)) 3084 return -ENODEV; 3085 3086 ret = __register_ftrace_function(ops); 3087 if (ret) 3088 return ret; 3089 3090 ftrace_start_up++; 3091 3092 /* 3093 * Note that ftrace probes uses this to start up 3094 * and modify functions it will probe. But we still 3095 * set the ADDING flag for modification, as probes 3096 * do not have trampolines. If they add them in the 3097 * future, then the probes will need to distinguish 3098 * between adding and updating probes. 3099 */ 3100 ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING; 3101 3102 ret = ftrace_hash_ipmodify_enable(ops); 3103 if (ret < 0) { 3104 /* Rollback registration process */ 3105 __unregister_ftrace_function(ops); 3106 ftrace_start_up--; 3107 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 3108 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) 3109 ftrace_trampoline_free(ops); 3110 return ret; 3111 } 3112 3113 if (ftrace_hash_rec_enable(ops)) 3114 command |= FTRACE_UPDATE_CALLS; 3115 3116 ftrace_startup_enable(command); 3117 3118 /* 3119 * If ftrace is in an undefined state, we just remove ops from list 3120 * to prevent the NULL pointer, instead of totally rolling it back and 3121 * free trampoline, because those actions could cause further damage. 3122 */ 3123 if (unlikely(ftrace_disabled)) { 3124 __unregister_ftrace_function(ops); 3125 return -ENODEV; 3126 } 3127 3128 ops->flags &= ~FTRACE_OPS_FL_ADDING; 3129 3130 return 0; 3131} 3132 3133int ftrace_shutdown(struct ftrace_ops *ops, int command) 3134{ 3135 int ret; 3136 3137 if (unlikely(ftrace_disabled)) 3138 return -ENODEV; 3139 3140 ret = __unregister_ftrace_function(ops); 3141 if (ret) 3142 return ret; 3143 3144 ftrace_start_up--; 3145 /* 3146 * Just warn in case of unbalance, no need to kill ftrace, it's not 3147 * critical but the ftrace_call callers may be never nopped again after 3148 * further ftrace uses. 3149 */ 3150 WARN_ON_ONCE(ftrace_start_up < 0); 3151 3152 /* Disabling ipmodify never fails */ 3153 ftrace_hash_ipmodify_disable(ops); 3154 3155 if (ftrace_hash_rec_disable(ops)) 3156 command |= FTRACE_UPDATE_CALLS; 3157 3158 ops->flags &= ~FTRACE_OPS_FL_ENABLED; 3159 3160 if (saved_ftrace_func != ftrace_trace_function) { 3161 saved_ftrace_func = ftrace_trace_function; 3162 command |= FTRACE_UPDATE_TRACE_FUNC; 3163 } 3164 3165 if (!command || !ftrace_enabled) 3166 goto out; 3167 3168 /* 3169 * If the ops uses a trampoline, then it needs to be 3170 * tested first on update. 3171 */ 3172 ops->flags |= FTRACE_OPS_FL_REMOVING; 3173 removed_ops = ops; 3174 3175 /* The trampoline logic checks the old hashes */ 3176 ops->old_hash.filter_hash = ops->func_hash->filter_hash; 3177 ops->old_hash.notrace_hash = ops->func_hash->notrace_hash; 3178 3179 ftrace_run_update_code(command); 3180 3181 /* 3182 * If there's no more ops registered with ftrace, run a 3183 * sanity check to make sure all rec flags are cleared. 3184 */ 3185 if (rcu_dereference_protected(ftrace_ops_list, 3186 lockdep_is_held(&ftrace_lock)) == &ftrace_list_end) { 3187 struct ftrace_page *pg; 3188 struct dyn_ftrace *rec; 3189 3190 do_for_each_ftrace_rec(pg, rec) { 3191 if (FTRACE_WARN_ON_ONCE(rec->flags & ~FTRACE_NOCLEAR_FLAGS)) 3192 pr_warn(" %pS flags:%lx\n", 3193 (void *)rec->ip, rec->flags); 3194 } while_for_each_ftrace_rec(); 3195 } 3196 3197 ops->old_hash.filter_hash = NULL; 3198 ops->old_hash.notrace_hash = NULL; 3199 3200 removed_ops = NULL; 3201 ops->flags &= ~FTRACE_OPS_FL_REMOVING; 3202 3203out: 3204 /* 3205 * Dynamic ops may be freed, we must make sure that all 3206 * callers are done before leaving this function. 3207 */ 3208 if (ops->flags & FTRACE_OPS_FL_DYNAMIC) { 3209 /* 3210 * We need to do a hard force of sched synchronization. 3211 * This is because we use preempt_disable() to do RCU, but 3212 * the function tracers can be called where RCU is not watching 3213 * (like before user_exit()). We can not rely on the RCU 3214 * infrastructure to do the synchronization, thus we must do it 3215 * ourselves. 3216 */ 3217 synchronize_rcu_tasks_rude(); 3218 3219 /* 3220 * When the kernel is preemptive, tasks can be preempted 3221 * while on a ftrace trampoline. Just scheduling a task on 3222 * a CPU is not good enough to flush them. Calling 3223 * synchronize_rcu_tasks() will wait for those tasks to 3224 * execute and either schedule voluntarily or enter user space. 3225 */ 3226 synchronize_rcu_tasks(); 3227 3228 ftrace_trampoline_free(ops); 3229 } 3230 3231 return 0; 3232} 3233 3234/* Simply make a copy of @src and return it */ 3235static struct ftrace_hash *copy_hash(struct ftrace_hash *src) 3236{ 3237 if (ftrace_hash_empty(src)) 3238 return EMPTY_HASH; 3239 3240 return alloc_and_copy_ftrace_hash(src->size_bits, src); 3241} 3242 3243/* 3244 * Append @new_hash entries to @hash: 3245 * 3246 * If @hash is the EMPTY_HASH then it traces all functions and nothing 3247 * needs to be done. 3248 * 3249 * If @new_hash is the EMPTY_HASH, then make *hash the EMPTY_HASH so 3250 * that it traces everything. 3251 * 3252 * Otherwise, go through all of @new_hash and add anything that @hash 3253 * doesn't already have, to @hash. 3254 * 3255 * The filter_hash updates uses just the append_hash() function 3256 * and the notrace_hash does not. 3257 */ 3258static int append_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash, 3259 int size_bits) 3260{ 3261 struct ftrace_func_entry *entry; 3262 int size; 3263 int i; 3264 3265 if (*hash) { 3266 /* An empty hash does everything */ 3267 if (ftrace_hash_empty(*hash)) 3268 return 0; 3269 } else { 3270 *hash = alloc_ftrace_hash(size_bits); 3271 if (!*hash) 3272 return -ENOMEM; 3273 } 3274 3275 /* If new_hash has everything make hash have everything */ 3276 if (ftrace_hash_empty(new_hash)) { 3277 free_ftrace_hash(*hash); 3278 *hash = EMPTY_HASH; 3279 return 0; 3280 } 3281 3282 size = 1 << new_hash->size_bits; 3283 for (i = 0; i < size; i++) { 3284 hlist_for_each_entry(entry, &new_hash->buckets[i], hlist) { 3285 /* Only add if not already in hash */ 3286 if (!__ftrace_lookup_ip(*hash, entry->ip) && 3287 add_hash_entry(*hash, entry->ip) == NULL) 3288 return -ENOMEM; 3289 } 3290 } 3291 return 0; 3292} 3293 3294/* 3295 * Remove functions from @hash that are in @notrace_hash 3296 */ 3297static void remove_hash(struct ftrace_hash *hash, struct ftrace_hash *notrace_hash) 3298{ 3299 struct ftrace_func_entry *entry; 3300 struct hlist_node *tmp; 3301 int size; 3302 int i; 3303 3304 /* If the notrace hash is empty, there's nothing to do */ 3305 if (ftrace_hash_empty(notrace_hash)) 3306 return; 3307 3308 size = 1 << hash->size_bits; 3309 for (i = 0; i < size; i++) { 3310 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) { 3311 if (!__ftrace_lookup_ip(notrace_hash, entry->ip)) 3312 continue; 3313 remove_hash_entry(hash, entry); 3314 kfree(entry); 3315 } 3316 } 3317} 3318 3319/* 3320 * Add to @hash only those that are in both @new_hash1 and @new_hash2 3321 * 3322 * The notrace_hash updates uses just the intersect_hash() function 3323 * and the filter_hash does not. 3324 */ 3325static int intersect_hash(struct ftrace_hash **hash, struct ftrace_hash *new_hash1, 3326 struct ftrace_hash *new_hash2) 3327{ 3328 struct ftrace_func_entry *entry; 3329 int size; 3330 int i; 3331 3332 /* 3333 * If new_hash1 or new_hash2 is the EMPTY_HASH then make the hash 3334 * empty as well as empty for notrace means none are notraced. 3335 */ 3336 if (ftrace_hash_empty(new_hash1) || ftrace_hash_empty(new_hash2)) { 3337 free_ftrace_hash(*hash); 3338 *hash = EMPTY_HASH; 3339 return 0; 3340 } 3341 3342 size = 1 << new_hash1->size_bits; 3343 for (i = 0; i < size; i++) { 3344 hlist_for_each_entry(entry, &new_hash1->buckets[i], hlist) { 3345 /* Only add if in both @new_hash1 and @new_hash2 */ 3346 if (__ftrace_lookup_ip(new_hash2, entry->ip) && 3347 add_hash_entry(*hash, entry->ip) == NULL) 3348 return -ENOMEM; 3349 } 3350 } 3351 /* If nothing intersects, make it the empty set */ 3352 if (ftrace_hash_empty(*hash)) { 3353 free_ftrace_hash(*hash); 3354 *hash = EMPTY_HASH; 3355 } 3356 return 0; 3357} 3358 3359static bool ops_equal(struct ftrace_hash *A, struct ftrace_hash *B) 3360{ 3361 struct ftrace_func_entry *entry; 3362 int size; 3363 int i; 3364 3365 if (ftrace_hash_empty(A)) 3366 return ftrace_hash_empty(B); 3367 3368 if (ftrace_hash_empty(B)) 3369 return ftrace_hash_empty(A); 3370 3371 if (A->count != B->count) 3372 return false; 3373 3374 size = 1 << A->size_bits; 3375 for (i = 0; i < size; i++) { 3376 hlist_for_each_entry(entry, &A->buckets[i], hlist) { 3377 if (!__ftrace_lookup_ip(B, entry->ip)) 3378 return false; 3379 } 3380 } 3381 3382 return true; 3383} 3384 3385static void ftrace_ops_update_code(struct ftrace_ops *ops, 3386 struct ftrace_ops_hash *old_hash); 3387 3388static int __ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, 3389 struct ftrace_hash **orig_hash, 3390 struct ftrace_hash *hash, 3391 int enable) 3392{ 3393 struct ftrace_ops_hash old_hash_ops; 3394 struct ftrace_hash *old_hash; 3395 int ret; 3396 3397 old_hash = *orig_hash; 3398 old_hash_ops.filter_hash = ops->func_hash->filter_hash; 3399 old_hash_ops.notrace_hash = ops->func_hash->notrace_hash; 3400 ret = ftrace_hash_move(ops, enable, orig_hash, hash); 3401 if (!ret) { 3402 ftrace_ops_update_code(ops, &old_hash_ops); 3403 free_ftrace_hash_rcu(old_hash); 3404 } 3405 return ret; 3406} 3407 3408static int ftrace_update_ops(struct ftrace_ops *ops, struct ftrace_hash *filter_hash, 3409 struct ftrace_hash *notrace_hash) 3410{ 3411 int ret; 3412 3413 if (!ops_equal(filter_hash, ops->func_hash->filter_hash)) { 3414 ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->filter_hash, 3415 filter_hash, 1); 3416 if (ret < 0) 3417 return ret; 3418 } 3419 3420 if (!ops_equal(notrace_hash, ops->func_hash->notrace_hash)) { 3421 ret = __ftrace_hash_move_and_update_ops(ops, &ops->func_hash->notrace_hash, 3422 notrace_hash, 0); 3423 if (ret < 0) 3424 return ret; 3425 } 3426 3427 return 0; 3428} 3429 3430static int add_first_hash(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash, 3431 struct ftrace_ops_hash *func_hash) 3432{ 3433 /* If the filter hash is not empty, simply remove the nohash from it */ 3434 if (!ftrace_hash_empty(func_hash->filter_hash)) { 3435 *filter_hash = copy_hash(func_hash->filter_hash); 3436 if (!*filter_hash) 3437 return -ENOMEM; 3438 remove_hash(*filter_hash, func_hash->notrace_hash); 3439 *notrace_hash = EMPTY_HASH; 3440 3441 } else { 3442 *notrace_hash = copy_hash(func_hash->notrace_hash); 3443 if (!*notrace_hash) 3444 return -ENOMEM; 3445 *filter_hash = EMPTY_HASH; 3446 } 3447 return 0; 3448} 3449 3450static int add_next_hash(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash, 3451 struct ftrace_ops_hash *ops_hash, struct ftrace_ops_hash *subops_hash) 3452{ 3453 int size_bits; 3454 int ret; 3455 3456 /* If the subops trace all functions so must the main ops */ 3457 if (ftrace_hash_empty(ops_hash->filter_hash) || 3458 ftrace_hash_empty(subops_hash->filter_hash)) { 3459 *filter_hash = EMPTY_HASH; 3460 } else { 3461 /* 3462 * The main ops filter hash is not empty, so its 3463 * notrace_hash had better be, as the notrace hash 3464 * is only used for empty main filter hashes. 3465 */ 3466 WARN_ON_ONCE(!ftrace_hash_empty(ops_hash->notrace_hash)); 3467 3468 size_bits = max(ops_hash->filter_hash->size_bits, 3469 subops_hash->filter_hash->size_bits); 3470 3471 /* Copy the subops hash */ 3472 *filter_hash = alloc_and_copy_ftrace_hash(size_bits, subops_hash->filter_hash); 3473 if (!*filter_hash) 3474 return -ENOMEM; 3475 /* Remove any notrace functions from the copy */ 3476 remove_hash(*filter_hash, subops_hash->notrace_hash); 3477 3478 ret = append_hash(filter_hash, ops_hash->filter_hash, 3479 size_bits); 3480 if (ret < 0) { 3481 free_ftrace_hash(*filter_hash); 3482 *filter_hash = EMPTY_HASH; 3483 return ret; 3484 } 3485 } 3486 3487 /* 3488 * Only process notrace hashes if the main filter hash is empty 3489 * (tracing all functions), otherwise the filter hash will just 3490 * remove the notrace hash functions, and the notrace hash is 3491 * not needed. 3492 */ 3493 if (ftrace_hash_empty(*filter_hash)) { 3494 /* 3495 * Intersect the notrace functions. That is, if two 3496 * subops are not tracing a set of functions, the 3497 * main ops will only not trace the functions that are 3498 * in both subops, but has to trace the functions that 3499 * are only notrace in one of the subops, for the other 3500 * subops to be able to trace them. 3501 */ 3502 size_bits = max(ops_hash->notrace_hash->size_bits, 3503 subops_hash->notrace_hash->size_bits); 3504 *notrace_hash = alloc_ftrace_hash(size_bits); 3505 if (!*notrace_hash) 3506 return -ENOMEM; 3507 3508 ret = intersect_hash(notrace_hash, ops_hash->notrace_hash, 3509 subops_hash->notrace_hash); 3510 if (ret < 0) { 3511 free_ftrace_hash(*notrace_hash); 3512 *notrace_hash = EMPTY_HASH; 3513 return ret; 3514 } 3515 } 3516 return 0; 3517} 3518 3519/** 3520 * ftrace_startup_subops - enable tracing for subops of an ops 3521 * @ops: Manager ops (used to pick all the functions of its subops) 3522 * @subops: A new ops to add to @ops 3523 * @command: Extra commands to use to enable tracing 3524 * 3525 * The @ops is a manager @ops that has the filter that includes all the functions 3526 * that its list of subops are tracing. Adding a new @subops will add the 3527 * functions of @subops to @ops. 3528 */ 3529int ftrace_startup_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command) 3530{ 3531 struct ftrace_hash *filter_hash = EMPTY_HASH; 3532 struct ftrace_hash *notrace_hash = EMPTY_HASH; 3533 struct ftrace_hash *save_filter_hash; 3534 struct ftrace_hash *save_notrace_hash; 3535 int ret; 3536 3537 if (unlikely(ftrace_disabled)) 3538 return -ENODEV; 3539 3540 ftrace_ops_init(ops); 3541 ftrace_ops_init(subops); 3542 3543 if (WARN_ON_ONCE(subops->flags & FTRACE_OPS_FL_ENABLED)) 3544 return -EBUSY; 3545 3546 /* Make everything canonical (Just in case!) */ 3547 if (!ops->func_hash->filter_hash) 3548 ops->func_hash->filter_hash = EMPTY_HASH; 3549 if (!ops->func_hash->notrace_hash) 3550 ops->func_hash->notrace_hash = EMPTY_HASH; 3551 if (!subops->func_hash->filter_hash) 3552 subops->func_hash->filter_hash = EMPTY_HASH; 3553 if (!subops->func_hash->notrace_hash) 3554 subops->func_hash->notrace_hash = EMPTY_HASH; 3555 3556 /* For the first subops to ops just enable it normally */ 3557 if (list_empty(&ops->subop_list)) { 3558 3559 /* The ops was empty, should have empty hashes */ 3560 WARN_ON_ONCE(!ftrace_hash_empty(ops->func_hash->filter_hash)); 3561 WARN_ON_ONCE(!ftrace_hash_empty(ops->func_hash->notrace_hash)); 3562 3563 ret = add_first_hash(&filter_hash, ¬race_hash, subops->func_hash); 3564 if (ret < 0) 3565 return ret; 3566 3567 save_filter_hash = ops->func_hash->filter_hash; 3568 save_notrace_hash = ops->func_hash->notrace_hash; 3569 3570 ops->func_hash->filter_hash = filter_hash; 3571 ops->func_hash->notrace_hash = notrace_hash; 3572 list_add(&subops->list, &ops->subop_list); 3573 ret = ftrace_startup(ops, command); 3574 if (ret < 0) { 3575 list_del(&subops->list); 3576 ops->func_hash->filter_hash = save_filter_hash; 3577 ops->func_hash->notrace_hash = save_notrace_hash; 3578 free_ftrace_hash(filter_hash); 3579 free_ftrace_hash(notrace_hash); 3580 } else { 3581 free_ftrace_hash(save_filter_hash); 3582 free_ftrace_hash(save_notrace_hash); 3583 subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP; 3584 subops->managed = ops; 3585 } 3586 return ret; 3587 } 3588 3589 /* 3590 * Here there's already something attached. Here are the rules: 3591 * If the new subops and main ops filter hashes are not empty: 3592 * o Make a copy of the subops filter hash 3593 * o Remove all functions in the nohash from it. 3594 * o Add in the main hash filter functions 3595 * o Remove any of these functions from the main notrace hash 3596 */ 3597 3598 ret = add_next_hash(&filter_hash, ¬race_hash, ops->func_hash, subops->func_hash); 3599 if (ret < 0) 3600 return ret; 3601 3602 list_add(&subops->list, &ops->subop_list); 3603 3604 ret = ftrace_update_ops(ops, filter_hash, notrace_hash); 3605 free_ftrace_hash(filter_hash); 3606 free_ftrace_hash(notrace_hash); 3607 if (ret < 0) { 3608 list_del(&subops->list); 3609 } else { 3610 subops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP; 3611 subops->managed = ops; 3612 } 3613 return ret; 3614} 3615 3616static int rebuild_hashes(struct ftrace_hash **filter_hash, struct ftrace_hash **notrace_hash, 3617 struct ftrace_ops *ops) 3618{ 3619 struct ftrace_ops_hash temp_hash; 3620 struct ftrace_ops *subops; 3621 bool first = true; 3622 int ret; 3623 3624 temp_hash.filter_hash = EMPTY_HASH; 3625 temp_hash.notrace_hash = EMPTY_HASH; 3626 3627 list_for_each_entry(subops, &ops->subop_list, list) { 3628 *filter_hash = EMPTY_HASH; 3629 *notrace_hash = EMPTY_HASH; 3630 3631 if (first) { 3632 ret = add_first_hash(filter_hash, notrace_hash, subops->func_hash); 3633 if (ret < 0) 3634 return ret; 3635 first = false; 3636 } else { 3637 ret = add_next_hash(filter_hash, notrace_hash, 3638 &temp_hash, subops->func_hash); 3639 if (ret < 0) { 3640 free_ftrace_hash(temp_hash.filter_hash); 3641 free_ftrace_hash(temp_hash.notrace_hash); 3642 return ret; 3643 } 3644 } 3645 3646 free_ftrace_hash(temp_hash.filter_hash); 3647 free_ftrace_hash(temp_hash.notrace_hash); 3648 3649 temp_hash.filter_hash = *filter_hash; 3650 temp_hash.notrace_hash = *notrace_hash; 3651 } 3652 return 0; 3653} 3654 3655/** 3656 * ftrace_shutdown_subops - Remove a subops from a manager ops 3657 * @ops: A manager ops to remove @subops from 3658 * @subops: The subops to remove from @ops 3659 * @command: Any extra command flags to add to modifying the text 3660 * 3661 * Removes the functions being traced by the @subops from @ops. Note, it 3662 * will not affect functions that are being traced by other subops that 3663 * still exist in @ops. 3664 * 3665 * If the last subops is removed from @ops, then @ops is shutdown normally. 3666 */ 3667int ftrace_shutdown_subops(struct ftrace_ops *ops, struct ftrace_ops *subops, int command) 3668{ 3669 struct ftrace_hash *filter_hash = EMPTY_HASH; 3670 struct ftrace_hash *notrace_hash = EMPTY_HASH; 3671 int ret; 3672 3673 if (unlikely(ftrace_disabled)) 3674 return -ENODEV; 3675 3676 if (WARN_ON_ONCE(!(subops->flags & FTRACE_OPS_FL_ENABLED))) 3677 return -EINVAL; 3678 3679 list_del(&subops->list); 3680 3681 if (list_empty(&ops->subop_list)) { 3682 /* Last one, just disable the current ops */ 3683 3684 ret = ftrace_shutdown(ops, command); 3685 if (ret < 0) { 3686 list_add(&subops->list, &ops->subop_list); 3687 return ret; 3688 } 3689 3690 subops->flags &= ~FTRACE_OPS_FL_ENABLED; 3691 3692 free_ftrace_hash(ops->func_hash->filter_hash); 3693 free_ftrace_hash(ops->func_hash->notrace_hash); 3694 ops->func_hash->filter_hash = EMPTY_HASH; 3695 ops->func_hash->notrace_hash = EMPTY_HASH; 3696 subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP); 3697 subops->managed = NULL; 3698 3699 return 0; 3700 } 3701 3702 /* Rebuild the hashes without subops */ 3703 ret = rebuild_hashes(&filter_hash, ¬race_hash, ops); 3704 if (ret < 0) 3705 return ret; 3706 3707 ret = ftrace_update_ops(ops, filter_hash, notrace_hash); 3708 if (ret < 0) { 3709 list_add(&subops->list, &ops->subop_list); 3710 } else { 3711 subops->flags &= ~(FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_SUBOP); 3712 subops->managed = NULL; 3713 } 3714 free_ftrace_hash(filter_hash); 3715 free_ftrace_hash(notrace_hash); 3716 return ret; 3717} 3718 3719static int ftrace_hash_move_and_update_subops(struct ftrace_ops *subops, 3720 struct ftrace_hash **orig_subhash, 3721 struct ftrace_hash *hash) 3722{ 3723 struct ftrace_ops *ops = subops->managed; 3724 struct ftrace_hash *notrace_hash; 3725 struct ftrace_hash *filter_hash; 3726 struct ftrace_hash *save_hash; 3727 struct ftrace_hash *new_hash; 3728 int ret; 3729 3730 /* Manager ops can not be subops (yet) */ 3731 if (WARN_ON_ONCE(!ops || ops->flags & FTRACE_OPS_FL_SUBOP)) 3732 return -EINVAL; 3733 3734 /* Move the new hash over to the subops hash */ 3735 save_hash = *orig_subhash; 3736 *orig_subhash = __ftrace_hash_move(hash); 3737 if (!*orig_subhash) { 3738 *orig_subhash = save_hash; 3739 return -ENOMEM; 3740 } 3741 3742 ret = rebuild_hashes(&filter_hash, ¬race_hash, ops); 3743 if (!ret) { 3744 ret = ftrace_update_ops(ops, filter_hash, notrace_hash); 3745 free_ftrace_hash(filter_hash); 3746 free_ftrace_hash(notrace_hash); 3747 } 3748 3749 if (ret) { 3750 /* Put back the original hash */ 3751 new_hash = *orig_subhash; 3752 *orig_subhash = save_hash; 3753 free_ftrace_hash_rcu(new_hash); 3754 } else { 3755 free_ftrace_hash_rcu(save_hash); 3756 } 3757 return ret; 3758} 3759 3760 3761u64 ftrace_update_time; 3762u64 ftrace_total_mod_time; 3763unsigned long ftrace_update_tot_cnt; 3764unsigned long ftrace_number_of_pages; 3765unsigned long ftrace_number_of_groups; 3766 3767static inline int ops_traces_mod(struct ftrace_ops *ops) 3768{ 3769 /* 3770 * Filter_hash being empty will default to trace module. 3771 * But notrace hash requires a test of individual module functions. 3772 */ 3773 return ftrace_hash_empty(ops->func_hash->filter_hash) && 3774 ftrace_hash_empty(ops->func_hash->notrace_hash); 3775} 3776 3777static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs) 3778{ 3779 bool init_nop = ftrace_need_init_nop(); 3780 struct ftrace_page *pg; 3781 struct dyn_ftrace *p; 3782 u64 start, stop, update_time; 3783 unsigned long update_cnt = 0; 3784 unsigned long rec_flags = 0; 3785 int i; 3786 3787 start = ftrace_now(raw_smp_processor_id()); 3788 3789 /* 3790 * When a module is loaded, this function is called to convert 3791 * the calls to mcount in its text to nops, and also to create 3792 * an entry in the ftrace data. Now, if ftrace is activated 3793 * after this call, but before the module sets its text to 3794 * read-only, the modification of enabling ftrace can fail if 3795 * the read-only is done while ftrace is converting the calls. 3796 * To prevent this, the module's records are set as disabled 3797 * and will be enabled after the call to set the module's text 3798 * to read-only. 3799 */ 3800 if (mod) 3801 rec_flags |= FTRACE_FL_DISABLED; 3802 3803 for (pg = new_pgs; pg; pg = pg->next) { 3804 3805 for (i = 0; i < pg->index; i++) { 3806 3807 /* If something went wrong, bail without enabling anything */ 3808 if (unlikely(ftrace_disabled)) 3809 return -1; 3810 3811 p = &pg->records[i]; 3812 p->flags = rec_flags; 3813 3814 /* 3815 * Do the initial record conversion from mcount jump 3816 * to the NOP instructions. 3817 */ 3818 if (init_nop && !ftrace_nop_initialize(mod, p)) 3819 break; 3820 3821 update_cnt++; 3822 } 3823 } 3824 3825 stop = ftrace_now(raw_smp_processor_id()); 3826 update_time = stop - start; 3827 if (mod) 3828 ftrace_total_mod_time += update_time; 3829 else 3830 ftrace_update_time = update_time; 3831 ftrace_update_tot_cnt += update_cnt; 3832 3833 return 0; 3834} 3835 3836static int ftrace_allocate_records(struct ftrace_page *pg, int count, 3837 unsigned long *num_pages) 3838{ 3839 int order; 3840 int pages; 3841 int cnt; 3842 3843 if (WARN_ON(!count)) 3844 return -EINVAL; 3845 3846 /* We want to fill as much as possible, with no empty pages */ 3847 pages = DIV_ROUND_UP(count * ENTRY_SIZE, PAGE_SIZE); 3848 order = fls(pages) - 1; 3849 3850 again: 3851 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 3852 3853 if (!pg->records) { 3854 /* if we can't allocate this size, try something smaller */ 3855 if (!order) 3856 return -ENOMEM; 3857 order--; 3858 goto again; 3859 } 3860 3861 ftrace_number_of_pages += 1 << order; 3862 *num_pages += 1 << order; 3863 ftrace_number_of_groups++; 3864 3865 cnt = (PAGE_SIZE << order) / ENTRY_SIZE; 3866 pg->order = order; 3867 3868 if (cnt > count) 3869 cnt = count; 3870 3871 return cnt; 3872} 3873 3874static void ftrace_free_pages(struct ftrace_page *pages) 3875{ 3876 struct ftrace_page *pg = pages; 3877 3878 while (pg) { 3879 if (pg->records) { 3880 free_pages((unsigned long)pg->records, pg->order); 3881 ftrace_number_of_pages -= 1 << pg->order; 3882 } 3883 pages = pg->next; 3884 kfree(pg); 3885 pg = pages; 3886 ftrace_number_of_groups--; 3887 } 3888} 3889 3890static struct ftrace_page * 3891ftrace_allocate_pages(unsigned long num_to_init, unsigned long *num_pages) 3892{ 3893 struct ftrace_page *start_pg; 3894 struct ftrace_page *pg; 3895 int cnt; 3896 3897 *num_pages = 0; 3898 3899 if (!num_to_init) 3900 return NULL; 3901 3902 start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL); 3903 if (!pg) 3904 return NULL; 3905 3906 /* 3907 * Try to allocate as much as possible in one continues 3908 * location that fills in all of the space. We want to 3909 * waste as little space as possible. 3910 */ 3911 for (;;) { 3912 cnt = ftrace_allocate_records(pg, num_to_init, num_pages); 3913 if (cnt < 0) 3914 goto free_pages; 3915 3916 num_to_init -= cnt; 3917 if (!num_to_init) 3918 break; 3919 3920 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); 3921 if (!pg->next) 3922 goto free_pages; 3923 3924 pg = pg->next; 3925 } 3926 3927 return start_pg; 3928 3929 free_pages: 3930 ftrace_free_pages(start_pg); 3931 pr_info("ftrace: FAILED to allocate memory for functions\n"); 3932 return NULL; 3933} 3934 3935#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 3936 3937struct ftrace_iterator { 3938 loff_t pos; 3939 loff_t func_pos; 3940 loff_t mod_pos; 3941 struct ftrace_page *pg; 3942 struct dyn_ftrace *func; 3943 struct ftrace_func_probe *probe; 3944 struct ftrace_func_entry *probe_entry; 3945 struct trace_parser parser; 3946 struct ftrace_hash *hash; 3947 struct ftrace_ops *ops; 3948 struct trace_array *tr; 3949 struct list_head *mod_list; 3950 int pidx; 3951 int idx; 3952 unsigned flags; 3953}; 3954 3955static void * 3956t_probe_next(struct seq_file *m, loff_t *pos) 3957{ 3958 struct ftrace_iterator *iter = m->private; 3959 struct trace_array *tr = iter->ops->private; 3960 struct list_head *func_probes; 3961 struct ftrace_hash *hash; 3962 struct list_head *next; 3963 struct hlist_node *hnd = NULL; 3964 struct hlist_head *hhd; 3965 int size; 3966 3967 (*pos)++; 3968 iter->pos = *pos; 3969 3970 if (!tr) 3971 return NULL; 3972 3973 func_probes = &tr->func_probes; 3974 if (list_empty(func_probes)) 3975 return NULL; 3976 3977 if (!iter->probe) { 3978 next = func_probes->next; 3979 iter->probe = list_entry(next, struct ftrace_func_probe, list); 3980 } 3981 3982 if (iter->probe_entry) 3983 hnd = &iter->probe_entry->hlist; 3984 3985 hash = iter->probe->ops.func_hash->filter_hash; 3986 3987 /* 3988 * A probe being registered may temporarily have an empty hash 3989 * and it's at the end of the func_probes list. 3990 */ 3991 if (!hash || hash == EMPTY_HASH) 3992 return NULL; 3993 3994 size = 1 << hash->size_bits; 3995 3996 retry: 3997 if (iter->pidx >= size) { 3998 if (iter->probe->list.next == func_probes) 3999 return NULL; 4000 next = iter->probe->list.next; 4001 iter->probe = list_entry(next, struct ftrace_func_probe, list); 4002 hash = iter->probe->ops.func_hash->filter_hash; 4003 size = 1 << hash->size_bits; 4004 iter->pidx = 0; 4005 } 4006 4007 hhd = &hash->buckets[iter->pidx]; 4008 4009 if (hlist_empty(hhd)) { 4010 iter->pidx++; 4011 hnd = NULL; 4012 goto retry; 4013 } 4014 4015 if (!hnd) 4016 hnd = hhd->first; 4017 else { 4018 hnd = hnd->next; 4019 if (!hnd) { 4020 iter->pidx++; 4021 goto retry; 4022 } 4023 } 4024 4025 if (WARN_ON_ONCE(!hnd)) 4026 return NULL; 4027 4028 iter->probe_entry = hlist_entry(hnd, struct ftrace_func_entry, hlist); 4029 4030 return iter; 4031} 4032 4033static void *t_probe_start(struct seq_file *m, loff_t *pos) 4034{ 4035 struct ftrace_iterator *iter = m->private; 4036 void *p = NULL; 4037 loff_t l; 4038 4039 if (!(iter->flags & FTRACE_ITER_DO_PROBES)) 4040 return NULL; 4041 4042 if (iter->mod_pos > *pos) 4043 return NULL; 4044 4045 iter->probe = NULL; 4046 iter->probe_entry = NULL; 4047 iter->pidx = 0; 4048 for (l = 0; l <= (*pos - iter->mod_pos); ) { 4049 p = t_probe_next(m, &l); 4050 if (!p) 4051 break; 4052 } 4053 if (!p) 4054 return NULL; 4055 4056 /* Only set this if we have an item */ 4057 iter->flags |= FTRACE_ITER_PROBE; 4058 4059 return iter; 4060} 4061 4062static int 4063t_probe_show(struct seq_file *m, struct ftrace_iterator *iter) 4064{ 4065 struct ftrace_func_entry *probe_entry; 4066 struct ftrace_probe_ops *probe_ops; 4067 struct ftrace_func_probe *probe; 4068 4069 probe = iter->probe; 4070 probe_entry = iter->probe_entry; 4071 4072 if (WARN_ON_ONCE(!probe || !probe_entry)) 4073 return -EIO; 4074 4075 probe_ops = probe->probe_ops; 4076 4077 if (probe_ops->print) 4078 return probe_ops->print(m, probe_entry->ip, probe_ops, probe->data); 4079 4080 seq_printf(m, "%ps:%ps\n", (void *)probe_entry->ip, 4081 (void *)probe_ops->func); 4082 4083 return 0; 4084} 4085 4086static void * 4087t_mod_next(struct seq_file *m, loff_t *pos) 4088{ 4089 struct ftrace_iterator *iter = m->private; 4090 struct trace_array *tr = iter->tr; 4091 4092 (*pos)++; 4093 iter->pos = *pos; 4094 4095 iter->mod_list = iter->mod_list->next; 4096 4097 if (iter->mod_list == &tr->mod_trace || 4098 iter->mod_list == &tr->mod_notrace) { 4099 iter->flags &= ~FTRACE_ITER_MOD; 4100 return NULL; 4101 } 4102 4103 iter->mod_pos = *pos; 4104 4105 return iter; 4106} 4107 4108static void *t_mod_start(struct seq_file *m, loff_t *pos) 4109{ 4110 struct ftrace_iterator *iter = m->private; 4111 void *p = NULL; 4112 loff_t l; 4113 4114 if (iter->func_pos > *pos) 4115 return NULL; 4116 4117 iter->mod_pos = iter->func_pos; 4118 4119 /* probes are only available if tr is set */ 4120 if (!iter->tr) 4121 return NULL; 4122 4123 for (l = 0; l <= (*pos - iter->func_pos); ) { 4124 p = t_mod_next(m, &l); 4125 if (!p) 4126 break; 4127 } 4128 if (!p) { 4129 iter->flags &= ~FTRACE_ITER_MOD; 4130 return t_probe_start(m, pos); 4131 } 4132 4133 /* Only set this if we have an item */ 4134 iter->flags |= FTRACE_ITER_MOD; 4135 4136 return iter; 4137} 4138 4139static int 4140t_mod_show(struct seq_file *m, struct ftrace_iterator *iter) 4141{ 4142 struct ftrace_mod_load *ftrace_mod; 4143 struct trace_array *tr = iter->tr; 4144 4145 if (WARN_ON_ONCE(!iter->mod_list) || 4146 iter->mod_list == &tr->mod_trace || 4147 iter->mod_list == &tr->mod_notrace) 4148 return -EIO; 4149 4150 ftrace_mod = list_entry(iter->mod_list, struct ftrace_mod_load, list); 4151 4152 if (ftrace_mod->func) 4153 seq_printf(m, "%s", ftrace_mod->func); 4154 else 4155 seq_putc(m, '*'); 4156 4157 seq_printf(m, ":mod:%s\n", ftrace_mod->module); 4158 4159 return 0; 4160} 4161 4162static void * 4163t_func_next(struct seq_file *m, loff_t *pos) 4164{ 4165 struct ftrace_iterator *iter = m->private; 4166 struct dyn_ftrace *rec = NULL; 4167 4168 (*pos)++; 4169 4170 retry: 4171 if (iter->idx >= iter->pg->index) { 4172 if (iter->pg->next) { 4173 iter->pg = iter->pg->next; 4174 iter->idx = 0; 4175 goto retry; 4176 } 4177 } else { 4178 rec = &iter->pg->records[iter->idx++]; 4179 if (((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && 4180 !ftrace_lookup_ip(iter->hash, rec->ip)) || 4181 4182 ((iter->flags & FTRACE_ITER_ENABLED) && 4183 !(rec->flags & FTRACE_FL_ENABLED)) || 4184 4185 ((iter->flags & FTRACE_ITER_TOUCHED) && 4186 !(rec->flags & FTRACE_FL_TOUCHED))) { 4187 4188 rec = NULL; 4189 goto retry; 4190 } 4191 } 4192 4193 if (!rec) 4194 return NULL; 4195 4196 iter->pos = iter->func_pos = *pos; 4197 iter->func = rec; 4198 4199 return iter; 4200} 4201 4202static void * 4203t_next(struct seq_file *m, void *v, loff_t *pos) 4204{ 4205 struct ftrace_iterator *iter = m->private; 4206 loff_t l = *pos; /* t_probe_start() must use original pos */ 4207 void *ret; 4208 4209 if (unlikely(ftrace_disabled)) 4210 return NULL; 4211 4212 if (iter->flags & FTRACE_ITER_PROBE) 4213 return t_probe_next(m, pos); 4214 4215 if (iter->flags & FTRACE_ITER_MOD) 4216 return t_mod_next(m, pos); 4217 4218 if (iter->flags & FTRACE_ITER_PRINTALL) { 4219 /* next must increment pos, and t_probe_start does not */ 4220 (*pos)++; 4221 return t_mod_start(m, &l); 4222 } 4223 4224 ret = t_func_next(m, pos); 4225 4226 if (!ret) 4227 return t_mod_start(m, &l); 4228 4229 return ret; 4230} 4231 4232static void reset_iter_read(struct ftrace_iterator *iter) 4233{ 4234 iter->pos = 0; 4235 iter->func_pos = 0; 4236 iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_PROBE | FTRACE_ITER_MOD); 4237} 4238 4239static void *t_start(struct seq_file *m, loff_t *pos) 4240{ 4241 struct ftrace_iterator *iter = m->private; 4242 void *p = NULL; 4243 loff_t l; 4244 4245 mutex_lock(&ftrace_lock); 4246 4247 if (unlikely(ftrace_disabled)) 4248 return NULL; 4249 4250 /* 4251 * If an lseek was done, then reset and start from beginning. 4252 */ 4253 if (*pos < iter->pos) 4254 reset_iter_read(iter); 4255 4256 /* 4257 * For set_ftrace_filter reading, if we have the filter 4258 * off, we can short cut and just print out that all 4259 * functions are enabled. 4260 */ 4261 if ((iter->flags & (FTRACE_ITER_FILTER | FTRACE_ITER_NOTRACE)) && 4262 ftrace_hash_empty(iter->hash)) { 4263 iter->func_pos = 1; /* Account for the message */ 4264 if (*pos > 0) 4265 return t_mod_start(m, pos); 4266 iter->flags |= FTRACE_ITER_PRINTALL; 4267 /* reset in case of seek/pread */ 4268 iter->flags &= ~FTRACE_ITER_PROBE; 4269 return iter; 4270 } 4271 4272 if (iter->flags & FTRACE_ITER_MOD) 4273 return t_mod_start(m, pos); 4274 4275 /* 4276 * Unfortunately, we need to restart at ftrace_pages_start 4277 * every time we let go of the ftrace_mutex. This is because 4278 * those pointers can change without the lock. 4279 */ 4280 iter->pg = ftrace_pages_start; 4281 iter->idx = 0; 4282 for (l = 0; l <= *pos; ) { 4283 p = t_func_next(m, &l); 4284 if (!p) 4285 break; 4286 } 4287 4288 if (!p) 4289 return t_mod_start(m, pos); 4290 4291 return iter; 4292} 4293 4294static void t_stop(struct seq_file *m, void *p) 4295{ 4296 mutex_unlock(&ftrace_lock); 4297} 4298 4299void * __weak 4300arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec) 4301{ 4302 return NULL; 4303} 4304 4305static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops, 4306 struct dyn_ftrace *rec) 4307{ 4308 void *ptr; 4309 4310 ptr = arch_ftrace_trampoline_func(ops, rec); 4311 if (ptr) 4312 seq_printf(m, " ->%pS", ptr); 4313} 4314 4315#ifdef FTRACE_MCOUNT_MAX_OFFSET 4316/* 4317 * Weak functions can still have an mcount/fentry that is saved in 4318 * the __mcount_loc section. These can be detected by having a 4319 * symbol offset of greater than FTRACE_MCOUNT_MAX_OFFSET, as the 4320 * symbol found by kallsyms is not the function that the mcount/fentry 4321 * is part of. The offset is much greater in these cases. 4322 * 4323 * Test the record to make sure that the ip points to a valid kallsyms 4324 * and if not, mark it disabled. 4325 */ 4326static int test_for_valid_rec(struct dyn_ftrace *rec) 4327{ 4328 char str[KSYM_SYMBOL_LEN]; 4329 unsigned long offset; 4330 const char *ret; 4331 4332 ret = kallsyms_lookup(rec->ip, NULL, &offset, NULL, str); 4333 4334 /* Weak functions can cause invalid addresses */ 4335 if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) { 4336 rec->flags |= FTRACE_FL_DISABLED; 4337 return 0; 4338 } 4339 return 1; 4340} 4341 4342static struct workqueue_struct *ftrace_check_wq __initdata; 4343static struct work_struct ftrace_check_work __initdata; 4344 4345/* 4346 * Scan all the mcount/fentry entries to make sure they are valid. 4347 */ 4348static __init void ftrace_check_work_func(struct work_struct *work) 4349{ 4350 struct ftrace_page *pg; 4351 struct dyn_ftrace *rec; 4352 4353 mutex_lock(&ftrace_lock); 4354 do_for_each_ftrace_rec(pg, rec) { 4355 test_for_valid_rec(rec); 4356 } while_for_each_ftrace_rec(); 4357 mutex_unlock(&ftrace_lock); 4358} 4359 4360static int __init ftrace_check_for_weak_functions(void) 4361{ 4362 INIT_WORK(&ftrace_check_work, ftrace_check_work_func); 4363 4364 ftrace_check_wq = alloc_workqueue("ftrace_check_wq", WQ_UNBOUND, 0); 4365 4366 queue_work(ftrace_check_wq, &ftrace_check_work); 4367 return 0; 4368} 4369 4370static int __init ftrace_check_sync(void) 4371{ 4372 /* Make sure the ftrace_check updates are finished */ 4373 if (ftrace_check_wq) 4374 destroy_workqueue(ftrace_check_wq); 4375 return 0; 4376} 4377 4378late_initcall_sync(ftrace_check_sync); 4379subsys_initcall(ftrace_check_for_weak_functions); 4380 4381static int print_rec(struct seq_file *m, unsigned long ip) 4382{ 4383 unsigned long offset; 4384 char str[KSYM_SYMBOL_LEN]; 4385 char *modname; 4386 const char *ret; 4387 4388 ret = kallsyms_lookup(ip, NULL, &offset, &modname, str); 4389 /* Weak functions can cause invalid addresses */ 4390 if (!ret || offset > FTRACE_MCOUNT_MAX_OFFSET) { 4391 snprintf(str, KSYM_SYMBOL_LEN, "%s_%ld", 4392 FTRACE_INVALID_FUNCTION, offset); 4393 ret = NULL; 4394 } 4395 4396 seq_puts(m, str); 4397 if (modname) 4398 seq_printf(m, " [%s]", modname); 4399 return ret == NULL ? -1 : 0; 4400} 4401#else 4402static inline int test_for_valid_rec(struct dyn_ftrace *rec) 4403{ 4404 return 1; 4405} 4406 4407static inline int print_rec(struct seq_file *m, unsigned long ip) 4408{ 4409 seq_printf(m, "%ps", (void *)ip); 4410 return 0; 4411} 4412#endif 4413 4414static void print_subops(struct seq_file *m, struct ftrace_ops *ops, struct dyn_ftrace *rec) 4415{ 4416 struct ftrace_ops *subops; 4417 bool first = true; 4418 4419 list_for_each_entry(subops, &ops->subop_list, list) { 4420 if (!((subops->flags & FTRACE_OPS_FL_ENABLED) && 4421 hash_contains_ip(rec->ip, subops->func_hash))) 4422 continue; 4423 if (first) { 4424 seq_printf(m, "\tsubops:"); 4425 first = false; 4426 } 4427#ifdef CONFIG_FUNCTION_GRAPH_TRACER 4428 if (subops->flags & FTRACE_OPS_FL_GRAPH) { 4429 struct fgraph_ops *gops; 4430 4431 gops = container_of(subops, struct fgraph_ops, ops); 4432 seq_printf(m, " {ent:%pS ret:%pS}", 4433 (void *)gops->entryfunc, 4434 (void *)gops->retfunc); 4435 continue; 4436 } 4437#endif 4438 if (subops->trampoline) { 4439 seq_printf(m, " {%pS (%pS)}", 4440 (void *)subops->trampoline, 4441 (void *)subops->func); 4442 add_trampoline_func(m, subops, rec); 4443 } else { 4444 seq_printf(m, " {%pS}", 4445 (void *)subops->func); 4446 } 4447 } 4448} 4449 4450static int t_show(struct seq_file *m, void *v) 4451{ 4452 struct ftrace_iterator *iter = m->private; 4453 struct dyn_ftrace *rec; 4454 4455 if (iter->flags & FTRACE_ITER_PROBE) 4456 return t_probe_show(m, iter); 4457 4458 if (iter->flags & FTRACE_ITER_MOD) 4459 return t_mod_show(m, iter); 4460 4461 if (iter->flags & FTRACE_ITER_PRINTALL) { 4462 if (iter->flags & FTRACE_ITER_NOTRACE) 4463 seq_puts(m, "#### no functions disabled ####\n"); 4464 else 4465 seq_puts(m, "#### all functions enabled ####\n"); 4466 return 0; 4467 } 4468 4469 rec = iter->func; 4470 4471 if (!rec) 4472 return 0; 4473 4474 if (iter->flags & FTRACE_ITER_ADDRS) 4475 seq_printf(m, "%lx ", rec->ip); 4476 4477 if (print_rec(m, rec->ip)) { 4478 /* This should only happen when a rec is disabled */ 4479 WARN_ON_ONCE(!(rec->flags & FTRACE_FL_DISABLED)); 4480 seq_putc(m, '\n'); 4481 return 0; 4482 } 4483 4484 if (iter->flags & (FTRACE_ITER_ENABLED | FTRACE_ITER_TOUCHED)) { 4485 struct ftrace_ops *ops; 4486 4487 seq_printf(m, " (%ld)%s%s%s%s%s", 4488 ftrace_rec_count(rec), 4489 rec->flags & FTRACE_FL_REGS ? " R" : " ", 4490 rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ", 4491 rec->flags & FTRACE_FL_DIRECT ? " D" : " ", 4492 rec->flags & FTRACE_FL_CALL_OPS ? " O" : " ", 4493 rec->flags & FTRACE_FL_MODIFIED ? " M " : " "); 4494 if (rec->flags & FTRACE_FL_TRAMP_EN) { 4495 ops = ftrace_find_tramp_ops_any(rec); 4496 if (ops) { 4497 do { 4498 seq_printf(m, "\ttramp: %pS (%pS)", 4499 (void *)ops->trampoline, 4500 (void *)ops->func); 4501 add_trampoline_func(m, ops, rec); 4502 print_subops(m, ops, rec); 4503 ops = ftrace_find_tramp_ops_next(rec, ops); 4504 } while (ops); 4505 } else 4506 seq_puts(m, "\ttramp: ERROR!"); 4507 } else { 4508 add_trampoline_func(m, NULL, rec); 4509 } 4510 if (rec->flags & FTRACE_FL_CALL_OPS_EN) { 4511 ops = ftrace_find_unique_ops(rec); 4512 if (ops) { 4513 seq_printf(m, "\tops: %pS (%pS)", 4514 ops, ops->func); 4515 print_subops(m, ops, rec); 4516 } else { 4517 seq_puts(m, "\tops: ERROR!"); 4518 } 4519 } 4520 if (rec->flags & FTRACE_FL_DIRECT) { 4521 unsigned long direct; 4522 4523 direct = ftrace_find_rec_direct(rec->ip); 4524 if (direct) { 4525 seq_printf(m, "\n\tdirect%s-->%pS", 4526 ftrace_is_jmp(direct) ? "(jmp)" : "", 4527 (void *)ftrace_jmp_get(direct)); 4528 } 4529 } 4530 } 4531 4532 seq_putc(m, '\n'); 4533 4534 return 0; 4535} 4536 4537static const struct seq_operations show_ftrace_seq_ops = { 4538 .start = t_start, 4539 .next = t_next, 4540 .stop = t_stop, 4541 .show = t_show, 4542}; 4543 4544static int 4545ftrace_avail_open(struct inode *inode, struct file *file) 4546{ 4547 struct ftrace_iterator *iter; 4548 int ret; 4549 4550 ret = security_locked_down(LOCKDOWN_TRACEFS); 4551 if (ret) 4552 return ret; 4553 4554 if (unlikely(ftrace_disabled)) 4555 return -ENODEV; 4556 4557 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 4558 if (!iter) 4559 return -ENOMEM; 4560 4561 iter->pg = ftrace_pages_start; 4562 iter->ops = &global_ops; 4563 4564 return 0; 4565} 4566 4567static int 4568ftrace_enabled_open(struct inode *inode, struct file *file) 4569{ 4570 struct ftrace_iterator *iter; 4571 4572 /* 4573 * This shows us what functions are currently being 4574 * traced and by what. Not sure if we want lockdown 4575 * to hide such critical information for an admin. 4576 * Although, perhaps it can show information we don't 4577 * want people to see, but if something is tracing 4578 * something, we probably want to know about it. 4579 */ 4580 4581 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 4582 if (!iter) 4583 return -ENOMEM; 4584 4585 iter->pg = ftrace_pages_start; 4586 iter->flags = FTRACE_ITER_ENABLED; 4587 iter->ops = &global_ops; 4588 4589 return 0; 4590} 4591 4592static int 4593ftrace_touched_open(struct inode *inode, struct file *file) 4594{ 4595 struct ftrace_iterator *iter; 4596 4597 /* 4598 * This shows us what functions have ever been enabled 4599 * (traced, direct, patched, etc). Not sure if we want lockdown 4600 * to hide such critical information for an admin. 4601 * Although, perhaps it can show information we don't 4602 * want people to see, but if something had traced 4603 * something, we probably want to know about it. 4604 */ 4605 4606 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 4607 if (!iter) 4608 return -ENOMEM; 4609 4610 iter->pg = ftrace_pages_start; 4611 iter->flags = FTRACE_ITER_TOUCHED; 4612 iter->ops = &global_ops; 4613 4614 return 0; 4615} 4616 4617static int 4618ftrace_avail_addrs_open(struct inode *inode, struct file *file) 4619{ 4620 struct ftrace_iterator *iter; 4621 int ret; 4622 4623 ret = security_locked_down(LOCKDOWN_TRACEFS); 4624 if (ret) 4625 return ret; 4626 4627 if (unlikely(ftrace_disabled)) 4628 return -ENODEV; 4629 4630 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter)); 4631 if (!iter) 4632 return -ENOMEM; 4633 4634 iter->pg = ftrace_pages_start; 4635 iter->flags = FTRACE_ITER_ADDRS; 4636 iter->ops = &global_ops; 4637 4638 return 0; 4639} 4640 4641/** 4642 * ftrace_regex_open - initialize function tracer filter files 4643 * @ops: The ftrace_ops that hold the hash filters 4644 * @flag: The type of filter to process 4645 * @inode: The inode, usually passed in to your open routine 4646 * @file: The file, usually passed in to your open routine 4647 * 4648 * ftrace_regex_open() initializes the filter files for the 4649 * @ops. Depending on @flag it may process the filter hash or 4650 * the notrace hash of @ops. With this called from the open 4651 * routine, you can use ftrace_filter_write() for the write 4652 * routine if @flag has FTRACE_ITER_FILTER set, or 4653 * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set. 4654 * tracing_lseek() should be used as the lseek routine, and 4655 * release must call ftrace_regex_release(). 4656 * 4657 * Returns: 0 on success or a negative errno value on failure 4658 */ 4659int 4660ftrace_regex_open(struct ftrace_ops *ops, int flag, 4661 struct inode *inode, struct file *file) 4662{ 4663 struct ftrace_iterator *iter; 4664 struct ftrace_hash *hash; 4665 struct list_head *mod_head; 4666 struct trace_array *tr = ops->private; 4667 int ret = -ENOMEM; 4668 4669 ftrace_ops_init(ops); 4670 4671 if (unlikely(ftrace_disabled)) 4672 return -ENODEV; 4673 4674 if (tracing_check_open_get_tr(tr)) 4675 return -ENODEV; 4676 4677 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 4678 if (!iter) 4679 goto out; 4680 4681 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) 4682 goto out; 4683 4684 iter->ops = ops; 4685 iter->flags = flag; 4686 iter->tr = tr; 4687 4688 mutex_lock(&ops->func_hash->regex_lock); 4689 4690 if (flag & FTRACE_ITER_NOTRACE) { 4691 hash = ops->func_hash->notrace_hash; 4692 mod_head = tr ? &tr->mod_notrace : NULL; 4693 } else { 4694 hash = ops->func_hash->filter_hash; 4695 mod_head = tr ? &tr->mod_trace : NULL; 4696 } 4697 4698 iter->mod_list = mod_head; 4699 4700 if (file->f_mode & FMODE_WRITE) { 4701 const int size_bits = FTRACE_HASH_DEFAULT_BITS; 4702 4703 if (file->f_flags & O_TRUNC) { 4704 iter->hash = alloc_ftrace_hash(size_bits); 4705 clear_ftrace_mod_list(mod_head); 4706 } else { 4707 iter->hash = alloc_and_copy_ftrace_hash(size_bits, hash); 4708 } 4709 } else { 4710 if (hash) 4711 iter->hash = alloc_and_copy_ftrace_hash(hash->size_bits, hash); 4712 else 4713 iter->hash = EMPTY_HASH; 4714 } 4715 4716 if (!iter->hash) { 4717 trace_parser_put(&iter->parser); 4718 goto out_unlock; 4719 } 4720 4721 ret = 0; 4722 4723 if (file->f_mode & FMODE_READ) { 4724 iter->pg = ftrace_pages_start; 4725 4726 ret = seq_open(file, &show_ftrace_seq_ops); 4727 if (!ret) { 4728 struct seq_file *m = file->private_data; 4729 m->private = iter; 4730 } else { 4731 /* Failed */ 4732 free_ftrace_hash(iter->hash); 4733 trace_parser_put(&iter->parser); 4734 } 4735 } else 4736 file->private_data = iter; 4737 4738 out_unlock: 4739 mutex_unlock(&ops->func_hash->regex_lock); 4740 4741 out: 4742 if (ret) { 4743 kfree(iter); 4744 if (tr) 4745 trace_array_put(tr); 4746 } 4747 4748 return ret; 4749} 4750 4751static int 4752ftrace_filter_open(struct inode *inode, struct file *file) 4753{ 4754 struct ftrace_ops *ops = inode->i_private; 4755 4756 /* Checks for tracefs lockdown */ 4757 return ftrace_regex_open(ops, 4758 FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, 4759 inode, file); 4760} 4761 4762static int 4763ftrace_notrace_open(struct inode *inode, struct file *file) 4764{ 4765 struct ftrace_ops *ops = inode->i_private; 4766 4767 /* Checks for tracefs lockdown */ 4768 return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, 4769 inode, file); 4770} 4771 4772/* Type for quick search ftrace basic regexes (globs) from filter_parse_regex */ 4773struct ftrace_glob { 4774 char *search; 4775 unsigned len; 4776 int type; 4777}; 4778 4779/* 4780 * If symbols in an architecture don't correspond exactly to the user-visible 4781 * name of what they represent, it is possible to define this function to 4782 * perform the necessary adjustments. 4783*/ 4784char * __weak arch_ftrace_match_adjust(char *str, const char *search) 4785{ 4786 return str; 4787} 4788 4789static int ftrace_match(char *str, struct ftrace_glob *g) 4790{ 4791 int matched = 0; 4792 int slen; 4793 4794 str = arch_ftrace_match_adjust(str, g->search); 4795 4796 switch (g->type) { 4797 case MATCH_FULL: 4798 if (strcmp(str, g->search) == 0) 4799 matched = 1; 4800 break; 4801 case MATCH_FRONT_ONLY: 4802 if (strncmp(str, g->search, g->len) == 0) 4803 matched = 1; 4804 break; 4805 case MATCH_MIDDLE_ONLY: 4806 if (strstr(str, g->search)) 4807 matched = 1; 4808 break; 4809 case MATCH_END_ONLY: 4810 slen = strlen(str); 4811 if (slen >= g->len && 4812 memcmp(str + slen - g->len, g->search, g->len) == 0) 4813 matched = 1; 4814 break; 4815 case MATCH_GLOB: 4816 if (glob_match(g->search, str)) 4817 matched = 1; 4818 break; 4819 } 4820 4821 return matched; 4822} 4823 4824static int 4825enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int clear_filter) 4826{ 4827 struct ftrace_func_entry *entry; 4828 int ret = 0; 4829 4830 entry = ftrace_lookup_ip(hash, rec->ip); 4831 if (clear_filter) { 4832 /* Do nothing if it doesn't exist */ 4833 if (!entry) 4834 return 0; 4835 4836 free_hash_entry(hash, entry); 4837 } else { 4838 /* Do nothing if it exists */ 4839 if (entry) 4840 return 0; 4841 if (add_hash_entry(hash, rec->ip) == NULL) 4842 ret = -ENOMEM; 4843 } 4844 return ret; 4845} 4846 4847static int 4848add_rec_by_index(struct ftrace_hash *hash, struct ftrace_glob *func_g, 4849 int clear_filter) 4850{ 4851 long index; 4852 struct ftrace_page *pg; 4853 struct dyn_ftrace *rec; 4854 4855 /* The index starts at 1 */ 4856 if (kstrtoul(func_g->search, 0, &index) || --index < 0) 4857 return 0; 4858 4859 do_for_each_ftrace_rec(pg, rec) { 4860 if (pg->index <= index) { 4861 index -= pg->index; 4862 /* this is a double loop, break goes to the next page */ 4863 break; 4864 } 4865 rec = &pg->records[index]; 4866 enter_record(hash, rec, clear_filter); 4867 return 1; 4868 } while_for_each_ftrace_rec(); 4869 return 0; 4870} 4871 4872#ifdef FTRACE_MCOUNT_MAX_OFFSET 4873static int lookup_ip(unsigned long ip, char **modname, char *str) 4874{ 4875 unsigned long offset; 4876 4877 kallsyms_lookup(ip, NULL, &offset, modname, str); 4878 if (offset > FTRACE_MCOUNT_MAX_OFFSET) 4879 return -1; 4880 return 0; 4881} 4882#else 4883static int lookup_ip(unsigned long ip, char **modname, char *str) 4884{ 4885 kallsyms_lookup(ip, NULL, NULL, modname, str); 4886 return 0; 4887} 4888#endif 4889 4890static int 4891ftrace_match_record(struct dyn_ftrace *rec, struct ftrace_glob *func_g, 4892 struct ftrace_glob *mod_g, int exclude_mod) 4893{ 4894 char str[KSYM_SYMBOL_LEN]; 4895 char *modname; 4896 4897 if (lookup_ip(rec->ip, &modname, str)) { 4898 /* This should only happen when a rec is disabled */ 4899 WARN_ON_ONCE(system_state == SYSTEM_RUNNING && 4900 !(rec->flags & FTRACE_FL_DISABLED)); 4901 return 0; 4902 } 4903 4904 if (mod_g) { 4905 int mod_matches = (modname) ? ftrace_match(modname, mod_g) : 0; 4906 4907 /* blank module name to match all modules */ 4908 if (!mod_g->len) { 4909 /* blank module globbing: modname xor exclude_mod */ 4910 if (!exclude_mod != !modname) 4911 goto func_match; 4912 return 0; 4913 } 4914 4915 /* 4916 * exclude_mod is set to trace everything but the given 4917 * module. If it is set and the module matches, then 4918 * return 0. If it is not set, and the module doesn't match 4919 * also return 0. Otherwise, check the function to see if 4920 * that matches. 4921 */ 4922 if (!mod_matches == !exclude_mod) 4923 return 0; 4924func_match: 4925 /* blank search means to match all funcs in the mod */ 4926 if (!func_g->len) 4927 return 1; 4928 } 4929 4930 return ftrace_match(str, func_g); 4931} 4932 4933static int 4934match_records(struct ftrace_hash *hash, char *func, int len, char *mod) 4935{ 4936 struct ftrace_page *pg; 4937 struct dyn_ftrace *rec; 4938 struct ftrace_glob func_g = { .type = MATCH_FULL }; 4939 struct ftrace_glob mod_g = { .type = MATCH_FULL }; 4940 struct ftrace_glob *mod_match = (mod) ? &mod_g : NULL; 4941 int exclude_mod = 0; 4942 int found = 0; 4943 int ret; 4944 int clear_filter = 0; 4945 4946 if (func) { 4947 func_g.type = filter_parse_regex(func, len, &func_g.search, 4948 &clear_filter); 4949 func_g.len = strlen(func_g.search); 4950 } 4951 4952 if (mod) { 4953 mod_g.type = filter_parse_regex(mod, strlen(mod), 4954 &mod_g.search, &exclude_mod); 4955 mod_g.len = strlen(mod_g.search); 4956 } 4957 4958 guard(mutex)(&ftrace_lock); 4959 4960 if (unlikely(ftrace_disabled)) 4961 return 0; 4962 4963 if (func_g.type == MATCH_INDEX) 4964 return add_rec_by_index(hash, &func_g, clear_filter); 4965 4966 do_for_each_ftrace_rec(pg, rec) { 4967 4968 if (rec->flags & FTRACE_FL_DISABLED) 4969 continue; 4970 4971 if (ftrace_match_record(rec, &func_g, mod_match, exclude_mod)) { 4972 ret = enter_record(hash, rec, clear_filter); 4973 if (ret < 0) 4974 return ret; 4975 found = 1; 4976 } 4977 cond_resched(); 4978 } while_for_each_ftrace_rec(); 4979 4980 return found; 4981} 4982 4983static int 4984ftrace_match_records(struct ftrace_hash *hash, char *buff, int len) 4985{ 4986 return match_records(hash, buff, len, NULL); 4987} 4988 4989static void ftrace_ops_update_code(struct ftrace_ops *ops, 4990 struct ftrace_ops_hash *old_hash) 4991{ 4992 struct ftrace_ops *op; 4993 4994 if (!ftrace_enabled) 4995 return; 4996 4997 if (ops->flags & FTRACE_OPS_FL_ENABLED) { 4998 ftrace_run_modify_code(ops, FTRACE_UPDATE_CALLS, old_hash); 4999 return; 5000 } 5001 5002 /* 5003 * If this is the shared global_ops filter, then we need to 5004 * check if there is another ops that shares it, is enabled. 5005 * If so, we still need to run the modify code. 5006 */ 5007 if (ops->func_hash != &global_ops.local_hash) 5008 return; 5009 5010 do_for_each_ftrace_op(op, ftrace_ops_list) { 5011 if (op->func_hash == &global_ops.local_hash && 5012 op->flags & FTRACE_OPS_FL_ENABLED) { 5013 ftrace_run_modify_code(op, FTRACE_UPDATE_CALLS, old_hash); 5014 /* Only need to do this once */ 5015 return; 5016 } 5017 } while_for_each_ftrace_op(op); 5018} 5019 5020static int ftrace_hash_move_and_update_ops(struct ftrace_ops *ops, 5021 struct ftrace_hash **orig_hash, 5022 struct ftrace_hash *hash, 5023 int enable) 5024{ 5025 if (ops->flags & FTRACE_OPS_FL_SUBOP) 5026 return ftrace_hash_move_and_update_subops(ops, orig_hash, hash); 5027 5028 /* 5029 * If this ops is not enabled, it could be sharing its filters 5030 * with a subop. If that's the case, update the subop instead of 5031 * this ops. Shared filters are only allowed to have one ops set 5032 * at a time, and if we update the ops that is not enabled, 5033 * it will not affect subops that share it. 5034 */ 5035 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) { 5036 struct ftrace_ops *op; 5037 5038 /* Check if any other manager subops maps to this hash */ 5039 do_for_each_ftrace_op(op, ftrace_ops_list) { 5040 struct ftrace_ops *subops; 5041 5042 list_for_each_entry(subops, &op->subop_list, list) { 5043 if ((subops->flags & FTRACE_OPS_FL_ENABLED) && 5044 subops->func_hash == ops->func_hash) { 5045 return ftrace_hash_move_and_update_subops(subops, orig_hash, hash); 5046 } 5047 } 5048 } while_for_each_ftrace_op(op); 5049 } 5050 5051 return __ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable); 5052} 5053 5054static int cache_mod(struct trace_array *tr, 5055 const char *func, char *module, int enable) 5056{ 5057 struct ftrace_mod_load *ftrace_mod, *n; 5058 struct list_head *head = enable ? &tr->mod_trace : &tr->mod_notrace; 5059 5060 guard(mutex)(&ftrace_lock); 5061 5062 /* We do not cache inverse filters */ 5063 if (func[0] == '!') { 5064 int ret = -EINVAL; 5065 5066 func++; 5067 5068 /* Look to remove this hash */ 5069 list_for_each_entry_safe(ftrace_mod, n, head, list) { 5070 if (strcmp(ftrace_mod->module, module) != 0) 5071 continue; 5072 5073 /* no func matches all */ 5074 if (strcmp(func, "*") == 0 || 5075 (ftrace_mod->func && 5076 strcmp(ftrace_mod->func, func) == 0)) { 5077 ret = 0; 5078 free_ftrace_mod(ftrace_mod); 5079 continue; 5080 } 5081 } 5082 return ret; 5083 } 5084 5085 /* We only care about modules that have not been loaded yet */ 5086 if (module_exists(module)) 5087 return -EINVAL; 5088 5089 /* Save this string off, and execute it when the module is loaded */ 5090 return ftrace_add_mod(tr, func, module, enable); 5091} 5092 5093#ifdef CONFIG_MODULES 5094static void process_mod_list(struct list_head *head, struct ftrace_ops *ops, 5095 char *mod, bool enable) 5096{ 5097 struct ftrace_mod_load *ftrace_mod, *n; 5098 struct ftrace_hash **orig_hash, *new_hash; 5099 LIST_HEAD(process_mods); 5100 char *func; 5101 5102 mutex_lock(&ops->func_hash->regex_lock); 5103 5104 if (enable) 5105 orig_hash = &ops->func_hash->filter_hash; 5106 else 5107 orig_hash = &ops->func_hash->notrace_hash; 5108 5109 new_hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, 5110 *orig_hash); 5111 if (!new_hash) 5112 goto out; /* warn? */ 5113 5114 mutex_lock(&ftrace_lock); 5115 5116 list_for_each_entry_safe(ftrace_mod, n, head, list) { 5117 5118 if (strcmp(ftrace_mod->module, mod) != 0) 5119 continue; 5120 5121 if (ftrace_mod->func) 5122 func = kstrdup(ftrace_mod->func, GFP_KERNEL); 5123 else 5124 func = kstrdup("*", GFP_KERNEL); 5125 5126 if (!func) /* warn? */ 5127 continue; 5128 5129 list_move(&ftrace_mod->list, &process_mods); 5130 5131 /* Use the newly allocated func, as it may be "*" */ 5132 kfree(ftrace_mod->func); 5133 ftrace_mod->func = func; 5134 } 5135 5136 mutex_unlock(&ftrace_lock); 5137 5138 list_for_each_entry_safe(ftrace_mod, n, &process_mods, list) { 5139 5140 func = ftrace_mod->func; 5141 5142 /* Grabs ftrace_lock, which is why we have this extra step */ 5143 match_records(new_hash, func, strlen(func), mod); 5144 free_ftrace_mod(ftrace_mod); 5145 } 5146 5147 if (enable && list_empty(head)) 5148 new_hash->flags &= ~FTRACE_HASH_FL_MOD; 5149 5150 mutex_lock(&ftrace_lock); 5151 5152 ftrace_hash_move_and_update_ops(ops, orig_hash, 5153 new_hash, enable); 5154 mutex_unlock(&ftrace_lock); 5155 5156 out: 5157 mutex_unlock(&ops->func_hash->regex_lock); 5158 5159 free_ftrace_hash(new_hash); 5160} 5161 5162static void process_cached_mods(const char *mod_name) 5163{ 5164 struct trace_array *tr; 5165 char *mod; 5166 5167 mod = kstrdup(mod_name, GFP_KERNEL); 5168 if (!mod) 5169 return; 5170 5171 mutex_lock(&trace_types_lock); 5172 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 5173 if (!list_empty(&tr->mod_trace)) 5174 process_mod_list(&tr->mod_trace, tr->ops, mod, true); 5175 if (!list_empty(&tr->mod_notrace)) 5176 process_mod_list(&tr->mod_notrace, tr->ops, mod, false); 5177 } 5178 mutex_unlock(&trace_types_lock); 5179 5180 kfree(mod); 5181} 5182#endif 5183 5184/* 5185 * We register the module command as a template to show others how 5186 * to register the a command as well. 5187 */ 5188 5189static int 5190ftrace_mod_callback(struct trace_array *tr, struct ftrace_hash *hash, 5191 char *func_orig, char *cmd, char *module, int enable) 5192{ 5193 char *func; 5194 int ret; 5195 5196 if (!tr) 5197 return -ENODEV; 5198 5199 /* match_records() modifies func, and we need the original */ 5200 func = kstrdup(func_orig, GFP_KERNEL); 5201 if (!func) 5202 return -ENOMEM; 5203 5204 /* 5205 * cmd == 'mod' because we only registered this func 5206 * for the 'mod' ftrace_func_command. 5207 * But if you register one func with multiple commands, 5208 * you can tell which command was used by the cmd 5209 * parameter. 5210 */ 5211 ret = match_records(hash, func, strlen(func), module); 5212 kfree(func); 5213 5214 if (!ret) 5215 return cache_mod(tr, func_orig, module, enable); 5216 if (ret < 0) 5217 return ret; 5218 return 0; 5219} 5220 5221static struct ftrace_func_command ftrace_mod_cmd = { 5222 .name = "mod", 5223 .func = ftrace_mod_callback, 5224}; 5225 5226static int __init ftrace_mod_cmd_init(void) 5227{ 5228 return register_ftrace_command(&ftrace_mod_cmd); 5229} 5230core_initcall(ftrace_mod_cmd_init); 5231 5232static void function_trace_probe_call(unsigned long ip, unsigned long parent_ip, 5233 struct ftrace_ops *op, struct ftrace_regs *fregs) 5234{ 5235 struct ftrace_probe_ops *probe_ops; 5236 struct ftrace_func_probe *probe; 5237 5238 probe = container_of(op, struct ftrace_func_probe, ops); 5239 probe_ops = probe->probe_ops; 5240 5241 /* 5242 * Disable preemption for these calls to prevent a RCU grace 5243 * period. This syncs the hash iteration and freeing of items 5244 * on the hash. rcu_read_lock is too dangerous here. 5245 */ 5246 preempt_disable_notrace(); 5247 probe_ops->func(ip, parent_ip, probe->tr, probe_ops, probe->data); 5248 preempt_enable_notrace(); 5249} 5250 5251struct ftrace_func_map { 5252 struct ftrace_func_entry entry; 5253 void *data; 5254}; 5255 5256/* 5257 * Note, ftrace_func_mapper is freed by free_ftrace_hash(&mapper->hash). 5258 * The hash field must be the first field. 5259 */ 5260struct ftrace_func_mapper { 5261 struct ftrace_hash hash; /* Must be first! */ 5262}; 5263 5264/** 5265 * allocate_ftrace_func_mapper - allocate a new ftrace_func_mapper 5266 * 5267 * Returns: a ftrace_func_mapper descriptor that can be used to map ips to data. 5268 */ 5269struct ftrace_func_mapper *allocate_ftrace_func_mapper(void) 5270{ 5271 struct ftrace_hash *hash; 5272 5273 /* 5274 * The mapper is simply a ftrace_hash, but since the entries 5275 * in the hash are not ftrace_func_entry type, we define it 5276 * as a separate structure. 5277 */ 5278 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 5279 return (struct ftrace_func_mapper *)hash; 5280} 5281 5282/** 5283 * ftrace_func_mapper_find_ip - Find some data mapped to an ip 5284 * @mapper: The mapper that has the ip maps 5285 * @ip: the instruction pointer to find the data for 5286 * 5287 * Returns: the data mapped to @ip if found otherwise NULL. The return 5288 * is actually the address of the mapper data pointer. The address is 5289 * returned for use cases where the data is no bigger than a long, and 5290 * the user can use the data pointer as its data instead of having to 5291 * allocate more memory for the reference. 5292 */ 5293void **ftrace_func_mapper_find_ip(struct ftrace_func_mapper *mapper, 5294 unsigned long ip) 5295{ 5296 struct ftrace_func_entry *entry; 5297 struct ftrace_func_map *map; 5298 5299 entry = ftrace_lookup_ip(&mapper->hash, ip); 5300 if (!entry) 5301 return NULL; 5302 5303 map = (struct ftrace_func_map *)entry; 5304 return &map->data; 5305} 5306 5307/** 5308 * ftrace_func_mapper_add_ip - Map some data to an ip 5309 * @mapper: The mapper that has the ip maps 5310 * @ip: The instruction pointer address to map @data to 5311 * @data: The data to map to @ip 5312 * 5313 * Returns: 0 on success otherwise an error. 5314 */ 5315int ftrace_func_mapper_add_ip(struct ftrace_func_mapper *mapper, 5316 unsigned long ip, void *data) 5317{ 5318 struct ftrace_func_entry *entry; 5319 struct ftrace_func_map *map; 5320 5321 entry = ftrace_lookup_ip(&mapper->hash, ip); 5322 if (entry) 5323 return -EBUSY; 5324 5325 map = kmalloc(sizeof(*map), GFP_KERNEL); 5326 if (!map) 5327 return -ENOMEM; 5328 5329 map->entry.ip = ip; 5330 map->data = data; 5331 5332 __add_hash_entry(&mapper->hash, &map->entry); 5333 5334 return 0; 5335} 5336 5337/** 5338 * ftrace_func_mapper_remove_ip - Remove an ip from the mapping 5339 * @mapper: The mapper that has the ip maps 5340 * @ip: The instruction pointer address to remove the data from 5341 * 5342 * Returns: the data if it is found, otherwise NULL. 5343 * Note, if the data pointer is used as the data itself, (see 5344 * ftrace_func_mapper_find_ip(), then the return value may be meaningless, 5345 * if the data pointer was set to zero. 5346 */ 5347void *ftrace_func_mapper_remove_ip(struct ftrace_func_mapper *mapper, 5348 unsigned long ip) 5349{ 5350 struct ftrace_func_entry *entry; 5351 struct ftrace_func_map *map; 5352 void *data; 5353 5354 entry = ftrace_lookup_ip(&mapper->hash, ip); 5355 if (!entry) 5356 return NULL; 5357 5358 map = (struct ftrace_func_map *)entry; 5359 data = map->data; 5360 5361 remove_hash_entry(&mapper->hash, entry); 5362 kfree(entry); 5363 5364 return data; 5365} 5366 5367/** 5368 * free_ftrace_func_mapper - free a mapping of ips and data 5369 * @mapper: The mapper that has the ip maps 5370 * @free_func: A function to be called on each data item. 5371 * 5372 * This is used to free the function mapper. The @free_func is optional 5373 * and can be used if the data needs to be freed as well. 5374 */ 5375void free_ftrace_func_mapper(struct ftrace_func_mapper *mapper, 5376 ftrace_mapper_func free_func) 5377{ 5378 struct ftrace_func_entry *entry; 5379 struct ftrace_func_map *map; 5380 struct hlist_head *hhd; 5381 int size, i; 5382 5383 if (!mapper) 5384 return; 5385 5386 if (free_func && mapper->hash.count) { 5387 size = 1 << mapper->hash.size_bits; 5388 for (i = 0; i < size; i++) { 5389 hhd = &mapper->hash.buckets[i]; 5390 hlist_for_each_entry(entry, hhd, hlist) { 5391 map = (struct ftrace_func_map *)entry; 5392 free_func(map); 5393 } 5394 } 5395 } 5396 /* This also frees the mapper itself */ 5397 free_ftrace_hash(&mapper->hash); 5398} 5399 5400static void release_probe(struct ftrace_func_probe *probe) 5401{ 5402 struct ftrace_probe_ops *probe_ops; 5403 5404 guard(mutex)(&ftrace_lock); 5405 5406 WARN_ON(probe->ref <= 0); 5407 5408 /* Subtract the ref that was used to protect this instance */ 5409 probe->ref--; 5410 5411 if (!probe->ref) { 5412 probe_ops = probe->probe_ops; 5413 /* 5414 * Sending zero as ip tells probe_ops to free 5415 * the probe->data itself 5416 */ 5417 if (probe_ops->free) 5418 probe_ops->free(probe_ops, probe->tr, 0, probe->data); 5419 list_del(&probe->list); 5420 kfree(probe); 5421 } 5422} 5423 5424static void acquire_probe_locked(struct ftrace_func_probe *probe) 5425{ 5426 /* 5427 * Add one ref to keep it from being freed when releasing the 5428 * ftrace_lock mutex. 5429 */ 5430 probe->ref++; 5431} 5432 5433int 5434register_ftrace_function_probe(char *glob, struct trace_array *tr, 5435 struct ftrace_probe_ops *probe_ops, 5436 void *data) 5437{ 5438 struct ftrace_func_probe *probe = NULL, *iter; 5439 struct ftrace_func_entry *entry; 5440 struct ftrace_hash **orig_hash; 5441 struct ftrace_hash *old_hash; 5442 struct ftrace_hash *hash; 5443 int count = 0; 5444 int size; 5445 int ret; 5446 int i; 5447 5448 if (WARN_ON(!tr)) 5449 return -EINVAL; 5450 5451 /* We do not support '!' for function probes */ 5452 if (WARN_ON(glob[0] == '!')) 5453 return -EINVAL; 5454 5455 5456 mutex_lock(&ftrace_lock); 5457 /* Check if the probe_ops is already registered */ 5458 list_for_each_entry(iter, &tr->func_probes, list) { 5459 if (iter->probe_ops == probe_ops) { 5460 probe = iter; 5461 break; 5462 } 5463 } 5464 if (!probe) { 5465 probe = kzalloc(sizeof(*probe), GFP_KERNEL); 5466 if (!probe) { 5467 mutex_unlock(&ftrace_lock); 5468 return -ENOMEM; 5469 } 5470 probe->probe_ops = probe_ops; 5471 probe->ops.func = function_trace_probe_call; 5472 probe->tr = tr; 5473 ftrace_ops_init(&probe->ops); 5474 list_add(&probe->list, &tr->func_probes); 5475 } 5476 5477 acquire_probe_locked(probe); 5478 5479 mutex_unlock(&ftrace_lock); 5480 5481 /* 5482 * Note, there's a small window here that the func_hash->filter_hash 5483 * may be NULL or empty. Need to be careful when reading the loop. 5484 */ 5485 mutex_lock(&probe->ops.func_hash->regex_lock); 5486 5487 orig_hash = &probe->ops.func_hash->filter_hash; 5488 old_hash = *orig_hash; 5489 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); 5490 5491 if (!hash) { 5492 ret = -ENOMEM; 5493 goto out; 5494 } 5495 5496 ret = ftrace_match_records(hash, glob, strlen(glob)); 5497 5498 /* Nothing found? */ 5499 if (!ret) 5500 ret = -EINVAL; 5501 5502 if (ret < 0) 5503 goto out; 5504 5505 size = 1 << hash->size_bits; 5506 for (i = 0; i < size; i++) { 5507 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 5508 if (ftrace_lookup_ip(old_hash, entry->ip)) 5509 continue; 5510 /* 5511 * The caller might want to do something special 5512 * for each function we find. We call the callback 5513 * to give the caller an opportunity to do so. 5514 */ 5515 if (probe_ops->init) { 5516 ret = probe_ops->init(probe_ops, tr, 5517 entry->ip, data, 5518 &probe->data); 5519 if (ret < 0) { 5520 if (probe_ops->free && count) 5521 probe_ops->free(probe_ops, tr, 5522 0, probe->data); 5523 probe->data = NULL; 5524 goto out; 5525 } 5526 } 5527 count++; 5528 } 5529 } 5530 5531 mutex_lock(&ftrace_lock); 5532 5533 if (!count) { 5534 /* Nothing was added? */ 5535 ret = -EINVAL; 5536 goto out_unlock; 5537 } 5538 5539 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, 5540 hash, 1); 5541 if (ret < 0) 5542 goto err_unlock; 5543 5544 /* One ref for each new function traced */ 5545 probe->ref += count; 5546 5547 if (!(probe->ops.flags & FTRACE_OPS_FL_ENABLED)) 5548 ret = ftrace_startup(&probe->ops, 0); 5549 5550 out_unlock: 5551 mutex_unlock(&ftrace_lock); 5552 5553 if (!ret) 5554 ret = count; 5555 out: 5556 mutex_unlock(&probe->ops.func_hash->regex_lock); 5557 free_ftrace_hash(hash); 5558 5559 release_probe(probe); 5560 5561 return ret; 5562 5563 err_unlock: 5564 if (!probe_ops->free || !count) 5565 goto out_unlock; 5566 5567 /* Failed to do the move, need to call the free functions */ 5568 for (i = 0; i < size; i++) { 5569 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 5570 if (ftrace_lookup_ip(old_hash, entry->ip)) 5571 continue; 5572 probe_ops->free(probe_ops, tr, entry->ip, probe->data); 5573 } 5574 } 5575 goto out_unlock; 5576} 5577 5578int 5579unregister_ftrace_function_probe_func(char *glob, struct trace_array *tr, 5580 struct ftrace_probe_ops *probe_ops) 5581{ 5582 struct ftrace_func_probe *probe = NULL, *iter; 5583 struct ftrace_ops_hash old_hash_ops; 5584 struct ftrace_func_entry *entry; 5585 struct ftrace_glob func_g; 5586 struct ftrace_hash **orig_hash; 5587 struct ftrace_hash *old_hash; 5588 struct ftrace_hash *hash = NULL; 5589 struct hlist_node *tmp; 5590 struct hlist_head hhd; 5591 char str[KSYM_SYMBOL_LEN]; 5592 int count = 0; 5593 int i, ret = -ENODEV; 5594 int size; 5595 5596 if (!glob || !strlen(glob) || !strcmp(glob, "*")) 5597 func_g.search = NULL; 5598 else { 5599 int not; 5600 5601 func_g.type = filter_parse_regex(glob, strlen(glob), 5602 &func_g.search, ¬); 5603 func_g.len = strlen(func_g.search); 5604 5605 /* we do not support '!' for function probes */ 5606 if (WARN_ON(not)) 5607 return -EINVAL; 5608 } 5609 5610 mutex_lock(&ftrace_lock); 5611 /* Check if the probe_ops is already registered */ 5612 list_for_each_entry(iter, &tr->func_probes, list) { 5613 if (iter->probe_ops == probe_ops) { 5614 probe = iter; 5615 break; 5616 } 5617 } 5618 if (!probe) 5619 goto err_unlock_ftrace; 5620 5621 ret = -EINVAL; 5622 if (!(probe->ops.flags & FTRACE_OPS_FL_INITIALIZED)) 5623 goto err_unlock_ftrace; 5624 5625 acquire_probe_locked(probe); 5626 5627 mutex_unlock(&ftrace_lock); 5628 5629 mutex_lock(&probe->ops.func_hash->regex_lock); 5630 5631 orig_hash = &probe->ops.func_hash->filter_hash; 5632 old_hash = *orig_hash; 5633 5634 if (ftrace_hash_empty(old_hash)) 5635 goto out_unlock; 5636 5637 old_hash_ops.filter_hash = old_hash; 5638 /* Probes only have filters */ 5639 old_hash_ops.notrace_hash = NULL; 5640 5641 ret = -ENOMEM; 5642 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, old_hash); 5643 if (!hash) 5644 goto out_unlock; 5645 5646 INIT_HLIST_HEAD(&hhd); 5647 5648 size = 1 << hash->size_bits; 5649 for (i = 0; i < size; i++) { 5650 hlist_for_each_entry_safe(entry, tmp, &hash->buckets[i], hlist) { 5651 5652 if (func_g.search) { 5653 kallsyms_lookup(entry->ip, NULL, NULL, 5654 NULL, str); 5655 if (!ftrace_match(str, &func_g)) 5656 continue; 5657 } 5658 count++; 5659 remove_hash_entry(hash, entry); 5660 hlist_add_head(&entry->hlist, &hhd); 5661 } 5662 } 5663 5664 /* Nothing found? */ 5665 if (!count) { 5666 ret = -EINVAL; 5667 goto out_unlock; 5668 } 5669 5670 mutex_lock(&ftrace_lock); 5671 5672 WARN_ON(probe->ref < count); 5673 5674 probe->ref -= count; 5675 5676 if (ftrace_hash_empty(hash)) 5677 ftrace_shutdown(&probe->ops, 0); 5678 5679 ret = ftrace_hash_move_and_update_ops(&probe->ops, orig_hash, 5680 hash, 1); 5681 5682 /* still need to update the function call sites */ 5683 if (ftrace_enabled && !ftrace_hash_empty(hash)) 5684 ftrace_run_modify_code(&probe->ops, FTRACE_UPDATE_CALLS, 5685 &old_hash_ops); 5686 synchronize_rcu(); 5687 5688 hlist_for_each_entry_safe(entry, tmp, &hhd, hlist) { 5689 hlist_del(&entry->hlist); 5690 if (probe_ops->free) 5691 probe_ops->free(probe_ops, tr, entry->ip, probe->data); 5692 kfree(entry); 5693 } 5694 mutex_unlock(&ftrace_lock); 5695 5696 out_unlock: 5697 mutex_unlock(&probe->ops.func_hash->regex_lock); 5698 free_ftrace_hash(hash); 5699 5700 release_probe(probe); 5701 5702 return ret; 5703 5704 err_unlock_ftrace: 5705 mutex_unlock(&ftrace_lock); 5706 return ret; 5707} 5708 5709void clear_ftrace_function_probes(struct trace_array *tr) 5710{ 5711 struct ftrace_func_probe *probe, *n; 5712 5713 list_for_each_entry_safe(probe, n, &tr->func_probes, list) 5714 unregister_ftrace_function_probe_func(NULL, tr, probe->probe_ops); 5715} 5716 5717static LIST_HEAD(ftrace_commands); 5718static DEFINE_MUTEX(ftrace_cmd_mutex); 5719 5720/* 5721 * Currently we only register ftrace commands from __init, so mark this 5722 * __init too. 5723 */ 5724__init int register_ftrace_command(struct ftrace_func_command *cmd) 5725{ 5726 struct ftrace_func_command *p; 5727 5728 guard(mutex)(&ftrace_cmd_mutex); 5729 list_for_each_entry(p, &ftrace_commands, list) { 5730 if (strcmp(cmd->name, p->name) == 0) 5731 return -EBUSY; 5732 } 5733 list_add(&cmd->list, &ftrace_commands); 5734 5735 return 0; 5736} 5737 5738/* 5739 * Currently we only unregister ftrace commands from __init, so mark 5740 * this __init too. 5741 */ 5742__init int unregister_ftrace_command(struct ftrace_func_command *cmd) 5743{ 5744 struct ftrace_func_command *p, *n; 5745 5746 guard(mutex)(&ftrace_cmd_mutex); 5747 5748 list_for_each_entry_safe(p, n, &ftrace_commands, list) { 5749 if (strcmp(cmd->name, p->name) == 0) { 5750 list_del_init(&p->list); 5751 return 0; 5752 } 5753 } 5754 5755 return -ENODEV; 5756} 5757 5758static int ftrace_process_regex(struct ftrace_iterator *iter, 5759 char *buff, int len, int enable) 5760{ 5761 struct ftrace_hash *hash = iter->hash; 5762 struct trace_array *tr = iter->ops->private; 5763 char *func, *command, *next = buff; 5764 struct ftrace_func_command *p; 5765 int ret; 5766 5767 func = strsep(&next, ":"); 5768 5769 if (!next) { 5770 ret = ftrace_match_records(hash, func, len); 5771 if (!ret) 5772 ret = -EINVAL; 5773 if (ret < 0) 5774 return ret; 5775 return 0; 5776 } 5777 5778 /* command found */ 5779 5780 command = strsep(&next, ":"); 5781 5782 guard(mutex)(&ftrace_cmd_mutex); 5783 5784 list_for_each_entry(p, &ftrace_commands, list) { 5785 if (strcmp(p->name, command) == 0) 5786 return p->func(tr, hash, func, command, next, enable); 5787 } 5788 5789 return -EINVAL; 5790} 5791 5792static ssize_t 5793ftrace_regex_write(struct file *file, const char __user *ubuf, 5794 size_t cnt, loff_t *ppos, int enable) 5795{ 5796 struct ftrace_iterator *iter; 5797 struct trace_parser *parser; 5798 ssize_t ret, read; 5799 5800 if (!cnt) 5801 return 0; 5802 5803 if (file->f_mode & FMODE_READ) { 5804 struct seq_file *m = file->private_data; 5805 iter = m->private; 5806 } else 5807 iter = file->private_data; 5808 5809 if (unlikely(ftrace_disabled)) 5810 return -ENODEV; 5811 5812 /* iter->hash is a local copy, so we don't need regex_lock */ 5813 5814 parser = &iter->parser; 5815 read = trace_get_user(parser, ubuf, cnt, ppos); 5816 5817 if (read >= 0 && trace_parser_loaded(parser) && 5818 !trace_parser_cont(parser)) { 5819 ret = ftrace_process_regex(iter, parser->buffer, 5820 parser->idx, enable); 5821 trace_parser_clear(parser); 5822 if (ret < 0) 5823 return ret; 5824 } 5825 5826 return read; 5827} 5828 5829ssize_t 5830ftrace_filter_write(struct file *file, const char __user *ubuf, 5831 size_t cnt, loff_t *ppos) 5832{ 5833 return ftrace_regex_write(file, ubuf, cnt, ppos, 1); 5834} 5835 5836ssize_t 5837ftrace_notrace_write(struct file *file, const char __user *ubuf, 5838 size_t cnt, loff_t *ppos) 5839{ 5840 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 5841} 5842 5843static int 5844__ftrace_match_addr(struct ftrace_hash *hash, unsigned long ip, int remove) 5845{ 5846 struct ftrace_func_entry *entry; 5847 5848 ip = ftrace_location(ip); 5849 if (!ip) 5850 return -EINVAL; 5851 5852 if (remove) { 5853 entry = ftrace_lookup_ip(hash, ip); 5854 if (!entry) 5855 return -ENOENT; 5856 free_hash_entry(hash, entry); 5857 return 0; 5858 } else if (__ftrace_lookup_ip(hash, ip) != NULL) { 5859 /* Already exists */ 5860 return 0; 5861 } 5862 5863 entry = add_hash_entry(hash, ip); 5864 return entry ? 0 : -ENOMEM; 5865} 5866 5867static int 5868ftrace_match_addr(struct ftrace_hash *hash, unsigned long *ips, 5869 unsigned int cnt, int remove) 5870{ 5871 unsigned int i; 5872 int err; 5873 5874 for (i = 0; i < cnt; i++) { 5875 err = __ftrace_match_addr(hash, ips[i], remove); 5876 if (err) { 5877 /* 5878 * This expects the @hash is a temporary hash and if this 5879 * fails the caller must free the @hash. 5880 */ 5881 return err; 5882 } 5883 } 5884 return 0; 5885} 5886 5887static int 5888ftrace_set_hash(struct ftrace_ops *ops, unsigned char *buf, int len, 5889 unsigned long *ips, unsigned int cnt, 5890 int remove, int reset, int enable, char *mod) 5891{ 5892 struct ftrace_hash **orig_hash; 5893 struct ftrace_hash *hash; 5894 int ret; 5895 5896 if (unlikely(ftrace_disabled)) 5897 return -ENODEV; 5898 5899 mutex_lock(&ops->func_hash->regex_lock); 5900 5901 if (enable) 5902 orig_hash = &ops->func_hash->filter_hash; 5903 else 5904 orig_hash = &ops->func_hash->notrace_hash; 5905 5906 if (reset) 5907 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 5908 else 5909 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash); 5910 5911 if (!hash) { 5912 ret = -ENOMEM; 5913 goto out_regex_unlock; 5914 } 5915 5916 if (buf && !match_records(hash, buf, len, mod)) { 5917 /* If this was for a module and nothing was enabled, flag it */ 5918 if (mod) 5919 (*orig_hash)->flags |= FTRACE_HASH_FL_MOD; 5920 5921 /* 5922 * Even if it is a mod, return error to let caller know 5923 * nothing was added 5924 */ 5925 ret = -EINVAL; 5926 goto out_regex_unlock; 5927 } 5928 if (ips) { 5929 ret = ftrace_match_addr(hash, ips, cnt, remove); 5930 if (ret < 0) 5931 goto out_regex_unlock; 5932 } 5933 5934 mutex_lock(&ftrace_lock); 5935 ret = ftrace_hash_move_and_update_ops(ops, orig_hash, hash, enable); 5936 mutex_unlock(&ftrace_lock); 5937 5938 out_regex_unlock: 5939 mutex_unlock(&ops->func_hash->regex_lock); 5940 5941 free_ftrace_hash(hash); 5942 return ret; 5943} 5944 5945static int 5946ftrace_set_addr(struct ftrace_ops *ops, unsigned long *ips, unsigned int cnt, 5947 int remove, int reset, int enable) 5948{ 5949 return ftrace_set_hash(ops, NULL, 0, ips, cnt, remove, reset, enable, NULL); 5950} 5951 5952#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS 5953 5954static int register_ftrace_function_nolock(struct ftrace_ops *ops); 5955 5956/* 5957 * If there are multiple ftrace_ops, use SAVE_REGS by default, so that direct 5958 * call will be jumped from ftrace_regs_caller. Only if the architecture does 5959 * not support ftrace_regs_caller but direct_call, use SAVE_ARGS so that it 5960 * jumps from ftrace_caller for multiple ftrace_ops. 5961 */ 5962#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_REGS 5963#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_ARGS) 5964#else 5965#define MULTI_FLAGS (FTRACE_OPS_FL_DIRECT | FTRACE_OPS_FL_SAVE_REGS) 5966#endif 5967 5968static int check_direct_multi(struct ftrace_ops *ops) 5969{ 5970 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) 5971 return -EINVAL; 5972 if ((ops->flags & MULTI_FLAGS) != MULTI_FLAGS) 5973 return -EINVAL; 5974 return 0; 5975} 5976 5977static void remove_direct_functions_hash(struct ftrace_hash *hash, unsigned long addr) 5978{ 5979 struct ftrace_func_entry *entry, *del; 5980 int size, i; 5981 5982 size = 1 << hash->size_bits; 5983 for (i = 0; i < size; i++) { 5984 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 5985 del = __ftrace_lookup_ip(direct_functions, entry->ip); 5986 if (del && ftrace_jmp_get(del->direct) == 5987 ftrace_jmp_get(addr)) { 5988 remove_hash_entry(direct_functions, del); 5989 kfree(del); 5990 } 5991 } 5992 } 5993} 5994 5995static void register_ftrace_direct_cb(struct rcu_head *rhp) 5996{ 5997 struct ftrace_hash *fhp = container_of(rhp, struct ftrace_hash, rcu); 5998 5999 free_ftrace_hash(fhp); 6000} 6001 6002static void reset_direct(struct ftrace_ops *ops, unsigned long addr) 6003{ 6004 struct ftrace_hash *hash = ops->func_hash->filter_hash; 6005 6006 remove_direct_functions_hash(hash, addr); 6007 6008 /* cleanup for possible another register call */ 6009 ops->func = NULL; 6010 ops->trampoline = 0; 6011} 6012 6013/** 6014 * register_ftrace_direct - Call a custom trampoline directly 6015 * for multiple functions registered in @ops 6016 * @ops: The address of the struct ftrace_ops object 6017 * @addr: The address of the trampoline to call at @ops functions 6018 * 6019 * This is used to connect a direct calls to @addr from the nop locations 6020 * of the functions registered in @ops (with by ftrace_set_filter_ip 6021 * function). 6022 * 6023 * The location that it calls (@addr) must be able to handle a direct call, 6024 * and save the parameters of the function being traced, and restore them 6025 * (or inject new ones if needed), before returning. 6026 * 6027 * Returns: 6028 * 0 on success 6029 * -EINVAL - The @ops object was already registered with this call or 6030 * when there are no functions in @ops object. 6031 * -EBUSY - Another direct function is already attached (there can be only one) 6032 * -ENODEV - @ip does not point to a ftrace nop location (or not supported) 6033 * -ENOMEM - There was an allocation failure. 6034 */ 6035int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) 6036{ 6037 struct ftrace_hash *hash, *new_hash = NULL, *free_hash = NULL; 6038 struct ftrace_func_entry *entry, *new; 6039 int err = -EBUSY, size, i; 6040 6041 if (ops->func || ops->trampoline) 6042 return -EINVAL; 6043 if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) 6044 return -EINVAL; 6045 if (ops->flags & FTRACE_OPS_FL_ENABLED) 6046 return -EINVAL; 6047 6048 hash = ops->func_hash->filter_hash; 6049 if (ftrace_hash_empty(hash)) 6050 return -EINVAL; 6051 6052 mutex_lock(&direct_mutex); 6053 6054 /* Make sure requested entries are not already registered.. */ 6055 size = 1 << hash->size_bits; 6056 for (i = 0; i < size; i++) { 6057 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 6058 if (ftrace_find_rec_direct(entry->ip)) 6059 goto out_unlock; 6060 } 6061 } 6062 6063 err = -ENOMEM; 6064 6065 /* Make a copy hash to place the new and the old entries in */ 6066 size = hash->count + direct_functions->count; 6067 size = fls(size); 6068 if (size > FTRACE_HASH_MAX_BITS) 6069 size = FTRACE_HASH_MAX_BITS; 6070 new_hash = alloc_ftrace_hash(size); 6071 if (!new_hash) 6072 goto out_unlock; 6073 6074 /* Now copy over the existing direct entries */ 6075 size = 1 << direct_functions->size_bits; 6076 for (i = 0; i < size; i++) { 6077 hlist_for_each_entry(entry, &direct_functions->buckets[i], hlist) { 6078 new = add_hash_entry(new_hash, entry->ip); 6079 if (!new) 6080 goto out_unlock; 6081 new->direct = entry->direct; 6082 } 6083 } 6084 6085 /* ... and add the new entries */ 6086 size = 1 << hash->size_bits; 6087 for (i = 0; i < size; i++) { 6088 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 6089 new = add_hash_entry(new_hash, entry->ip); 6090 if (!new) 6091 goto out_unlock; 6092 /* Update both the copy and the hash entry */ 6093 new->direct = addr; 6094 entry->direct = addr; 6095 } 6096 } 6097 6098 free_hash = direct_functions; 6099 rcu_assign_pointer(direct_functions, new_hash); 6100 new_hash = NULL; 6101 6102 ops->func = call_direct_funcs; 6103 ops->flags |= MULTI_FLAGS; 6104 ops->trampoline = FTRACE_REGS_ADDR; 6105 ops->direct_call = addr; 6106 6107 err = register_ftrace_function_nolock(ops); 6108 if (err) 6109 reset_direct(ops, addr); 6110 6111 out_unlock: 6112 mutex_unlock(&direct_mutex); 6113 6114 if (free_hash && free_hash != EMPTY_HASH) 6115 call_rcu_tasks(&free_hash->rcu, register_ftrace_direct_cb); 6116 6117 if (new_hash) 6118 free_ftrace_hash(new_hash); 6119 6120 return err; 6121} 6122EXPORT_SYMBOL_GPL(register_ftrace_direct); 6123 6124/** 6125 * unregister_ftrace_direct - Remove calls to custom trampoline 6126 * previously registered by register_ftrace_direct for @ops object. 6127 * @ops: The address of the struct ftrace_ops object 6128 * @addr: The address of the direct function that is called by the @ops functions 6129 * @free_filters: Set to true to remove all filters for the ftrace_ops, false otherwise 6130 * 6131 * This is used to remove a direct calls to @addr from the nop locations 6132 * of the functions registered in @ops (with by ftrace_set_filter_ip 6133 * function). 6134 * 6135 * Returns: 6136 * 0 on success 6137 * -EINVAL - The @ops object was not properly registered. 6138 */ 6139int unregister_ftrace_direct(struct ftrace_ops *ops, unsigned long addr, 6140 bool free_filters) 6141{ 6142 int err; 6143 6144 if (check_direct_multi(ops)) 6145 return -EINVAL; 6146 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 6147 return -EINVAL; 6148 6149 mutex_lock(&direct_mutex); 6150 err = unregister_ftrace_function(ops); 6151 reset_direct(ops, addr); 6152 mutex_unlock(&direct_mutex); 6153 6154 if (free_filters) 6155 ftrace_free_filter(ops); 6156 return err; 6157} 6158EXPORT_SYMBOL_GPL(unregister_ftrace_direct); 6159 6160static int 6161__modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) 6162{ 6163 struct ftrace_hash *hash = ops->func_hash->filter_hash; 6164 struct ftrace_func_entry *entry, *iter; 6165 static struct ftrace_ops tmp_ops = { 6166 .func = ftrace_stub, 6167 .flags = FTRACE_OPS_FL_STUB, 6168 }; 6169 int i, size; 6170 int err; 6171 6172 lockdep_assert_held_once(&direct_mutex); 6173 6174 /* Enable the tmp_ops to have the same functions as the direct ops */ 6175 ftrace_ops_init(&tmp_ops); 6176 tmp_ops.func_hash = ops->func_hash; 6177 tmp_ops.direct_call = addr; 6178 6179 err = register_ftrace_function_nolock(&tmp_ops); 6180 if (err) 6181 return err; 6182 6183 /* 6184 * Call __ftrace_hash_update_ipmodify() here, so that we can call 6185 * ops->ops_func for the ops. This is needed because the above 6186 * register_ftrace_function_nolock() worked on tmp_ops. 6187 */ 6188 err = __ftrace_hash_update_ipmodify(ops, hash, hash, true); 6189 if (err) 6190 goto out; 6191 6192 /* 6193 * Now the ftrace_ops_list_func() is called to do the direct callers. 6194 * We can safely change the direct functions attached to each entry. 6195 */ 6196 mutex_lock(&ftrace_lock); 6197 6198 size = 1 << hash->size_bits; 6199 for (i = 0; i < size; i++) { 6200 hlist_for_each_entry(iter, &hash->buckets[i], hlist) { 6201 entry = __ftrace_lookup_ip(direct_functions, iter->ip); 6202 if (!entry) 6203 continue; 6204 entry->direct = addr; 6205 } 6206 } 6207 /* Prevent store tearing if a trampoline concurrently accesses the value */ 6208 WRITE_ONCE(ops->direct_call, addr); 6209 6210 mutex_unlock(&ftrace_lock); 6211 6212out: 6213 /* Removing the tmp_ops will add the updated direct callers to the functions */ 6214 unregister_ftrace_function(&tmp_ops); 6215 6216 return err; 6217} 6218 6219/** 6220 * modify_ftrace_direct_nolock - Modify an existing direct 'multi' call 6221 * to call something else 6222 * @ops: The address of the struct ftrace_ops object 6223 * @addr: The address of the new trampoline to call at @ops functions 6224 * 6225 * This is used to unregister currently registered direct caller and 6226 * register new one @addr on functions registered in @ops object. 6227 * 6228 * Note there's window between ftrace_shutdown and ftrace_startup calls 6229 * where there will be no callbacks called. 6230 * 6231 * Caller should already have direct_mutex locked, so we don't lock 6232 * direct_mutex here. 6233 * 6234 * Returns: zero on success. Non zero on error, which includes: 6235 * -EINVAL - The @ops object was not properly registered. 6236 */ 6237int modify_ftrace_direct_nolock(struct ftrace_ops *ops, unsigned long addr) 6238{ 6239 if (check_direct_multi(ops)) 6240 return -EINVAL; 6241 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 6242 return -EINVAL; 6243 6244 return __modify_ftrace_direct(ops, addr); 6245} 6246EXPORT_SYMBOL_GPL(modify_ftrace_direct_nolock); 6247 6248/** 6249 * modify_ftrace_direct - Modify an existing direct 'multi' call 6250 * to call something else 6251 * @ops: The address of the struct ftrace_ops object 6252 * @addr: The address of the new trampoline to call at @ops functions 6253 * 6254 * This is used to unregister currently registered direct caller and 6255 * register new one @addr on functions registered in @ops object. 6256 * 6257 * Note there's window between ftrace_shutdown and ftrace_startup calls 6258 * where there will be no callbacks called. 6259 * 6260 * Returns: zero on success. Non zero on error, which includes: 6261 * -EINVAL - The @ops object was not properly registered. 6262 */ 6263int modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr) 6264{ 6265 int err; 6266 6267 if (check_direct_multi(ops)) 6268 return -EINVAL; 6269 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 6270 return -EINVAL; 6271 6272 mutex_lock(&direct_mutex); 6273 err = __modify_ftrace_direct(ops, addr); 6274 mutex_unlock(&direct_mutex); 6275 return err; 6276} 6277EXPORT_SYMBOL_GPL(modify_ftrace_direct); 6278#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ 6279 6280/** 6281 * ftrace_set_filter_ip - set a function to filter on in ftrace by address 6282 * @ops: the ops to set the filter with 6283 * @ip: the address to add to or remove from the filter. 6284 * @remove: non zero to remove the ip from the filter 6285 * @reset: non zero to reset all filters before applying this filter. 6286 * 6287 * Filters denote which functions should be enabled when tracing is enabled 6288 * If @ip is NULL, it fails to update filter. 6289 * 6290 * This can allocate memory which must be freed before @ops can be freed, 6291 * either by removing each filtered addr or by using 6292 * ftrace_free_filter(@ops). 6293 */ 6294int ftrace_set_filter_ip(struct ftrace_ops *ops, unsigned long ip, 6295 int remove, int reset) 6296{ 6297 ftrace_ops_init(ops); 6298 return ftrace_set_addr(ops, &ip, 1, remove, reset, 1); 6299} 6300EXPORT_SYMBOL_GPL(ftrace_set_filter_ip); 6301 6302/** 6303 * ftrace_set_filter_ips - set functions to filter on in ftrace by addresses 6304 * @ops: the ops to set the filter with 6305 * @ips: the array of addresses to add to or remove from the filter. 6306 * @cnt: the number of addresses in @ips 6307 * @remove: non zero to remove ips from the filter 6308 * @reset: non zero to reset all filters before applying this filter. 6309 * 6310 * Filters denote which functions should be enabled when tracing is enabled 6311 * If @ips array or any ip specified within is NULL , it fails to update filter. 6312 * 6313 * This can allocate memory which must be freed before @ops can be freed, 6314 * either by removing each filtered addr or by using 6315 * ftrace_free_filter(@ops). 6316*/ 6317int ftrace_set_filter_ips(struct ftrace_ops *ops, unsigned long *ips, 6318 unsigned int cnt, int remove, int reset) 6319{ 6320 ftrace_ops_init(ops); 6321 return ftrace_set_addr(ops, ips, cnt, remove, reset, 1); 6322} 6323EXPORT_SYMBOL_GPL(ftrace_set_filter_ips); 6324 6325/** 6326 * ftrace_ops_set_global_filter - setup ops to use global filters 6327 * @ops: the ops which will use the global filters 6328 * 6329 * ftrace users who need global function trace filtering should call this. 6330 * It can set the global filter only if ops were not initialized before. 6331 */ 6332void ftrace_ops_set_global_filter(struct ftrace_ops *ops) 6333{ 6334 if (ops->flags & FTRACE_OPS_FL_INITIALIZED) 6335 return; 6336 6337 ftrace_ops_init(ops); 6338 ops->func_hash = &global_ops.local_hash; 6339} 6340EXPORT_SYMBOL_GPL(ftrace_ops_set_global_filter); 6341 6342static int 6343ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len, 6344 int reset, int enable) 6345{ 6346 char *mod = NULL, *func, *command, *next = buf; 6347 char *tmp __free(kfree) = NULL; 6348 struct trace_array *tr = ops->private; 6349 int ret; 6350 6351 func = strsep(&next, ":"); 6352 6353 /* This can also handle :mod: parsing */ 6354 if (next) { 6355 if (!tr) 6356 return -EINVAL; 6357 6358 command = strsep(&next, ":"); 6359 if (strcmp(command, "mod") != 0) 6360 return -EINVAL; 6361 6362 mod = next; 6363 len = command - func; 6364 /* Save the original func as ftrace_set_hash() can modify it */ 6365 tmp = kstrdup(func, GFP_KERNEL); 6366 } 6367 6368 ret = ftrace_set_hash(ops, func, len, NULL, 0, 0, reset, enable, mod); 6369 6370 if (tr && mod && ret < 0) { 6371 /* Did tmp fail to allocate? */ 6372 if (!tmp) 6373 return -ENOMEM; 6374 ret = cache_mod(tr, tmp, mod, enable); 6375 } 6376 6377 return ret; 6378} 6379 6380/** 6381 * ftrace_set_filter - set a function to filter on in ftrace 6382 * @ops: the ops to set the filter with 6383 * @buf: the string that holds the function filter text. 6384 * @len: the length of the string. 6385 * @reset: non-zero to reset all filters before applying this filter. 6386 * 6387 * Filters denote which functions should be enabled when tracing is enabled. 6388 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 6389 * 6390 * This can allocate memory which must be freed before @ops can be freed, 6391 * either by removing each filtered addr or by using 6392 * ftrace_free_filter(@ops). 6393 */ 6394int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf, 6395 int len, int reset) 6396{ 6397 ftrace_ops_init(ops); 6398 return ftrace_set_regex(ops, buf, len, reset, 1); 6399} 6400EXPORT_SYMBOL_GPL(ftrace_set_filter); 6401 6402/** 6403 * ftrace_set_notrace - set a function to not trace in ftrace 6404 * @ops: the ops to set the notrace filter with 6405 * @buf: the string that holds the function notrace text. 6406 * @len: the length of the string. 6407 * @reset: non-zero to reset all filters before applying this filter. 6408 * 6409 * Notrace Filters denote which functions should not be enabled when tracing 6410 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 6411 * for tracing. 6412 * 6413 * This can allocate memory which must be freed before @ops can be freed, 6414 * either by removing each filtered addr or by using 6415 * ftrace_free_filter(@ops). 6416 */ 6417int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, 6418 int len, int reset) 6419{ 6420 ftrace_ops_init(ops); 6421 return ftrace_set_regex(ops, buf, len, reset, 0); 6422} 6423EXPORT_SYMBOL_GPL(ftrace_set_notrace); 6424/** 6425 * ftrace_set_global_filter - set a function to filter on with global tracers 6426 * @buf: the string that holds the function filter text. 6427 * @len: the length of the string. 6428 * @reset: non-zero to reset all filters before applying this filter. 6429 * 6430 * Filters denote which functions should be enabled when tracing is enabled. 6431 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 6432 */ 6433void ftrace_set_global_filter(unsigned char *buf, int len, int reset) 6434{ 6435 ftrace_set_regex(&global_ops, buf, len, reset, 1); 6436} 6437EXPORT_SYMBOL_GPL(ftrace_set_global_filter); 6438 6439/** 6440 * ftrace_set_global_notrace - set a function to not trace with global tracers 6441 * @buf: the string that holds the function notrace text. 6442 * @len: the length of the string. 6443 * @reset: non-zero to reset all filters before applying this filter. 6444 * 6445 * Notrace Filters denote which functions should not be enabled when tracing 6446 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 6447 * for tracing. 6448 */ 6449void ftrace_set_global_notrace(unsigned char *buf, int len, int reset) 6450{ 6451 ftrace_set_regex(&global_ops, buf, len, reset, 0); 6452} 6453EXPORT_SYMBOL_GPL(ftrace_set_global_notrace); 6454 6455/* 6456 * command line interface to allow users to set filters on boot up. 6457 */ 6458#define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE 6459static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 6460static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; 6461 6462/* Used by function selftest to not test if filter is set */ 6463bool ftrace_filter_param __initdata; 6464 6465static int __init set_ftrace_notrace(char *str) 6466{ 6467 ftrace_filter_param = true; 6468 strscpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); 6469 return 1; 6470} 6471__setup("ftrace_notrace=", set_ftrace_notrace); 6472 6473static int __init set_ftrace_filter(char *str) 6474{ 6475 ftrace_filter_param = true; 6476 strscpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); 6477 return 1; 6478} 6479__setup("ftrace_filter=", set_ftrace_filter); 6480 6481#ifdef CONFIG_FUNCTION_GRAPH_TRACER 6482static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; 6483static char ftrace_graph_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 6484static int ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer); 6485 6486static int __init set_graph_function(char *str) 6487{ 6488 strscpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); 6489 return 1; 6490} 6491__setup("ftrace_graph_filter=", set_graph_function); 6492 6493static int __init set_graph_notrace_function(char *str) 6494{ 6495 strscpy(ftrace_graph_notrace_buf, str, FTRACE_FILTER_SIZE); 6496 return 1; 6497} 6498__setup("ftrace_graph_notrace=", set_graph_notrace_function); 6499 6500static int __init set_graph_max_depth_function(char *str) 6501{ 6502 if (!str || kstrtouint(str, 0, &fgraph_max_depth)) 6503 return 0; 6504 return 1; 6505} 6506__setup("ftrace_graph_max_depth=", set_graph_max_depth_function); 6507 6508static void __init set_ftrace_early_graph(char *buf, int enable) 6509{ 6510 int ret; 6511 char *func; 6512 struct ftrace_hash *hash; 6513 6514 hash = alloc_ftrace_hash(FTRACE_HASH_DEFAULT_BITS); 6515 if (MEM_FAIL(!hash, "Failed to allocate hash\n")) 6516 return; 6517 6518 while (buf) { 6519 func = strsep(&buf, ","); 6520 /* we allow only one expression at a time */ 6521 ret = ftrace_graph_set_hash(hash, func); 6522 if (ret) 6523 printk(KERN_DEBUG "ftrace: function %s not " 6524 "traceable\n", func); 6525 } 6526 6527 if (enable) 6528 ftrace_graph_hash = hash; 6529 else 6530 ftrace_graph_notrace_hash = hash; 6531} 6532#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 6533 6534void __init 6535ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable) 6536{ 6537 char *func; 6538 6539 ftrace_ops_init(ops); 6540 6541 /* The trace_array is needed for caching module function filters */ 6542 if (!ops->private) { 6543 struct trace_array *tr = trace_get_global_array(); 6544 6545 ops->private = tr; 6546 ftrace_init_trace_array(tr); 6547 } 6548 6549 while (buf) { 6550 func = strsep(&buf, ","); 6551 ftrace_set_regex(ops, func, strlen(func), 0, enable); 6552 } 6553} 6554 6555static void __init set_ftrace_early_filters(void) 6556{ 6557 if (ftrace_filter_buf[0]) 6558 ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1); 6559 if (ftrace_notrace_buf[0]) 6560 ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0); 6561#ifdef CONFIG_FUNCTION_GRAPH_TRACER 6562 if (ftrace_graph_buf[0]) 6563 set_ftrace_early_graph(ftrace_graph_buf, 1); 6564 if (ftrace_graph_notrace_buf[0]) 6565 set_ftrace_early_graph(ftrace_graph_notrace_buf, 0); 6566#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 6567} 6568 6569int ftrace_regex_release(struct inode *inode, struct file *file) 6570{ 6571 struct seq_file *m = (struct seq_file *)file->private_data; 6572 struct ftrace_iterator *iter; 6573 struct ftrace_hash **orig_hash; 6574 struct trace_parser *parser; 6575 int filter_hash; 6576 6577 if (file->f_mode & FMODE_READ) { 6578 iter = m->private; 6579 seq_release(inode, file); 6580 } else 6581 iter = file->private_data; 6582 6583 parser = &iter->parser; 6584 if (trace_parser_loaded(parser)) { 6585 int enable = !(iter->flags & FTRACE_ITER_NOTRACE); 6586 6587 ftrace_process_regex(iter, parser->buffer, 6588 parser->idx, enable); 6589 } 6590 6591 trace_parser_put(parser); 6592 6593 mutex_lock(&iter->ops->func_hash->regex_lock); 6594 6595 if (file->f_mode & FMODE_WRITE) { 6596 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER); 6597 6598 if (filter_hash) { 6599 orig_hash = &iter->ops->func_hash->filter_hash; 6600 if (iter->tr) { 6601 if (list_empty(&iter->tr->mod_trace)) 6602 iter->hash->flags &= ~FTRACE_HASH_FL_MOD; 6603 else 6604 iter->hash->flags |= FTRACE_HASH_FL_MOD; 6605 } 6606 } else 6607 orig_hash = &iter->ops->func_hash->notrace_hash; 6608 6609 mutex_lock(&ftrace_lock); 6610 ftrace_hash_move_and_update_ops(iter->ops, orig_hash, 6611 iter->hash, filter_hash); 6612 mutex_unlock(&ftrace_lock); 6613 } 6614 6615 mutex_unlock(&iter->ops->func_hash->regex_lock); 6616 free_ftrace_hash(iter->hash); 6617 if (iter->tr) 6618 trace_array_put(iter->tr); 6619 kfree(iter); 6620 6621 return 0; 6622} 6623 6624static const struct file_operations ftrace_avail_fops = { 6625 .open = ftrace_avail_open, 6626 .read = seq_read, 6627 .llseek = seq_lseek, 6628 .release = seq_release_private, 6629}; 6630 6631static const struct file_operations ftrace_enabled_fops = { 6632 .open = ftrace_enabled_open, 6633 .read = seq_read, 6634 .llseek = seq_lseek, 6635 .release = seq_release_private, 6636}; 6637 6638static const struct file_operations ftrace_touched_fops = { 6639 .open = ftrace_touched_open, 6640 .read = seq_read, 6641 .llseek = seq_lseek, 6642 .release = seq_release_private, 6643}; 6644 6645static const struct file_operations ftrace_avail_addrs_fops = { 6646 .open = ftrace_avail_addrs_open, 6647 .read = seq_read, 6648 .llseek = seq_lseek, 6649 .release = seq_release_private, 6650}; 6651 6652static const struct file_operations ftrace_filter_fops = { 6653 .open = ftrace_filter_open, 6654 .read = seq_read, 6655 .write = ftrace_filter_write, 6656 .llseek = tracing_lseek, 6657 .release = ftrace_regex_release, 6658}; 6659 6660static const struct file_operations ftrace_notrace_fops = { 6661 .open = ftrace_notrace_open, 6662 .read = seq_read, 6663 .write = ftrace_notrace_write, 6664 .llseek = tracing_lseek, 6665 .release = ftrace_regex_release, 6666}; 6667 6668#ifdef CONFIG_FUNCTION_GRAPH_TRACER 6669 6670static DEFINE_MUTEX(graph_lock); 6671 6672struct ftrace_hash __rcu *ftrace_graph_hash = EMPTY_HASH; 6673struct ftrace_hash __rcu *ftrace_graph_notrace_hash = EMPTY_HASH; 6674 6675enum graph_filter_type { 6676 GRAPH_FILTER_NOTRACE = 0, 6677 GRAPH_FILTER_FUNCTION, 6678}; 6679 6680#define FTRACE_GRAPH_EMPTY ((void *)1) 6681 6682struct ftrace_graph_data { 6683 struct ftrace_hash *hash; 6684 struct ftrace_func_entry *entry; 6685 int idx; /* for hash table iteration */ 6686 enum graph_filter_type type; 6687 struct ftrace_hash *new_hash; 6688 const struct seq_operations *seq_ops; 6689 struct trace_parser parser; 6690}; 6691 6692static void * 6693__g_next(struct seq_file *m, loff_t *pos) 6694{ 6695 struct ftrace_graph_data *fgd = m->private; 6696 struct ftrace_func_entry *entry = fgd->entry; 6697 struct hlist_head *head; 6698 int i, idx = fgd->idx; 6699 6700 if (*pos >= fgd->hash->count) 6701 return NULL; 6702 6703 if (entry) { 6704 hlist_for_each_entry_continue(entry, hlist) { 6705 fgd->entry = entry; 6706 return entry; 6707 } 6708 6709 idx++; 6710 } 6711 6712 for (i = idx; i < 1 << fgd->hash->size_bits; i++) { 6713 head = &fgd->hash->buckets[i]; 6714 hlist_for_each_entry(entry, head, hlist) { 6715 fgd->entry = entry; 6716 fgd->idx = i; 6717 return entry; 6718 } 6719 } 6720 return NULL; 6721} 6722 6723static void * 6724g_next(struct seq_file *m, void *v, loff_t *pos) 6725{ 6726 (*pos)++; 6727 return __g_next(m, pos); 6728} 6729 6730static void *g_start(struct seq_file *m, loff_t *pos) 6731{ 6732 struct ftrace_graph_data *fgd = m->private; 6733 6734 mutex_lock(&graph_lock); 6735 6736 if (fgd->type == GRAPH_FILTER_FUNCTION) 6737 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, 6738 lockdep_is_held(&graph_lock)); 6739 else 6740 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 6741 lockdep_is_held(&graph_lock)); 6742 6743 /* Nothing, tell g_show to print all functions are enabled */ 6744 if (ftrace_hash_empty(fgd->hash) && !*pos) 6745 return FTRACE_GRAPH_EMPTY; 6746 6747 fgd->idx = 0; 6748 fgd->entry = NULL; 6749 return __g_next(m, pos); 6750} 6751 6752static void g_stop(struct seq_file *m, void *p) 6753{ 6754 mutex_unlock(&graph_lock); 6755} 6756 6757static int g_show(struct seq_file *m, void *v) 6758{ 6759 struct ftrace_func_entry *entry = v; 6760 6761 if (!entry) 6762 return 0; 6763 6764 if (entry == FTRACE_GRAPH_EMPTY) { 6765 struct ftrace_graph_data *fgd = m->private; 6766 6767 if (fgd->type == GRAPH_FILTER_FUNCTION) 6768 seq_puts(m, "#### all functions enabled ####\n"); 6769 else 6770 seq_puts(m, "#### no functions disabled ####\n"); 6771 return 0; 6772 } 6773 6774 seq_printf(m, "%ps\n", (void *)entry->ip); 6775 6776 return 0; 6777} 6778 6779static const struct seq_operations ftrace_graph_seq_ops = { 6780 .start = g_start, 6781 .next = g_next, 6782 .stop = g_stop, 6783 .show = g_show, 6784}; 6785 6786static int 6787__ftrace_graph_open(struct inode *inode, struct file *file, 6788 struct ftrace_graph_data *fgd) 6789{ 6790 int ret; 6791 struct ftrace_hash *new_hash = NULL; 6792 6793 ret = security_locked_down(LOCKDOWN_TRACEFS); 6794 if (ret) 6795 return ret; 6796 6797 if (file->f_mode & FMODE_WRITE) { 6798 const int size_bits = FTRACE_HASH_DEFAULT_BITS; 6799 6800 if (trace_parser_get_init(&fgd->parser, FTRACE_BUFF_MAX)) 6801 return -ENOMEM; 6802 6803 if (file->f_flags & O_TRUNC) 6804 new_hash = alloc_ftrace_hash(size_bits); 6805 else 6806 new_hash = alloc_and_copy_ftrace_hash(size_bits, 6807 fgd->hash); 6808 if (!new_hash) { 6809 ret = -ENOMEM; 6810 goto out; 6811 } 6812 } 6813 6814 if (file->f_mode & FMODE_READ) { 6815 ret = seq_open(file, &ftrace_graph_seq_ops); 6816 if (!ret) { 6817 struct seq_file *m = file->private_data; 6818 m->private = fgd; 6819 } else { 6820 /* Failed */ 6821 free_ftrace_hash(new_hash); 6822 new_hash = NULL; 6823 } 6824 } else 6825 file->private_data = fgd; 6826 6827out: 6828 if (ret < 0 && file->f_mode & FMODE_WRITE) 6829 trace_parser_put(&fgd->parser); 6830 6831 fgd->new_hash = new_hash; 6832 6833 /* 6834 * All uses of fgd->hash must be taken with the graph_lock 6835 * held. The graph_lock is going to be released, so force 6836 * fgd->hash to be reinitialized when it is taken again. 6837 */ 6838 fgd->hash = NULL; 6839 6840 return ret; 6841} 6842 6843static int 6844ftrace_graph_open(struct inode *inode, struct file *file) 6845{ 6846 struct ftrace_graph_data *fgd; 6847 int ret; 6848 6849 if (unlikely(ftrace_disabled)) 6850 return -ENODEV; 6851 6852 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 6853 if (fgd == NULL) 6854 return -ENOMEM; 6855 6856 mutex_lock(&graph_lock); 6857 6858 fgd->hash = rcu_dereference_protected(ftrace_graph_hash, 6859 lockdep_is_held(&graph_lock)); 6860 fgd->type = GRAPH_FILTER_FUNCTION; 6861 fgd->seq_ops = &ftrace_graph_seq_ops; 6862 6863 ret = __ftrace_graph_open(inode, file, fgd); 6864 if (ret < 0) 6865 kfree(fgd); 6866 6867 mutex_unlock(&graph_lock); 6868 return ret; 6869} 6870 6871static int 6872ftrace_graph_notrace_open(struct inode *inode, struct file *file) 6873{ 6874 struct ftrace_graph_data *fgd; 6875 int ret; 6876 6877 if (unlikely(ftrace_disabled)) 6878 return -ENODEV; 6879 6880 fgd = kmalloc(sizeof(*fgd), GFP_KERNEL); 6881 if (fgd == NULL) 6882 return -ENOMEM; 6883 6884 mutex_lock(&graph_lock); 6885 6886 fgd->hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 6887 lockdep_is_held(&graph_lock)); 6888 fgd->type = GRAPH_FILTER_NOTRACE; 6889 fgd->seq_ops = &ftrace_graph_seq_ops; 6890 6891 ret = __ftrace_graph_open(inode, file, fgd); 6892 if (ret < 0) 6893 kfree(fgd); 6894 6895 mutex_unlock(&graph_lock); 6896 return ret; 6897} 6898 6899static int 6900ftrace_graph_release(struct inode *inode, struct file *file) 6901{ 6902 struct ftrace_graph_data *fgd; 6903 struct ftrace_hash *old_hash, *new_hash; 6904 struct trace_parser *parser; 6905 int ret = 0; 6906 6907 if (file->f_mode & FMODE_READ) { 6908 struct seq_file *m = file->private_data; 6909 6910 fgd = m->private; 6911 seq_release(inode, file); 6912 } else { 6913 fgd = file->private_data; 6914 } 6915 6916 6917 if (file->f_mode & FMODE_WRITE) { 6918 6919 parser = &fgd->parser; 6920 6921 if (trace_parser_loaded((parser))) { 6922 ret = ftrace_graph_set_hash(fgd->new_hash, 6923 parser->buffer); 6924 } 6925 6926 trace_parser_put(parser); 6927 6928 new_hash = __ftrace_hash_move(fgd->new_hash); 6929 if (!new_hash) { 6930 ret = -ENOMEM; 6931 goto out; 6932 } 6933 6934 mutex_lock(&graph_lock); 6935 6936 if (fgd->type == GRAPH_FILTER_FUNCTION) { 6937 old_hash = rcu_dereference_protected(ftrace_graph_hash, 6938 lockdep_is_held(&graph_lock)); 6939 rcu_assign_pointer(ftrace_graph_hash, new_hash); 6940 } else { 6941 old_hash = rcu_dereference_protected(ftrace_graph_notrace_hash, 6942 lockdep_is_held(&graph_lock)); 6943 rcu_assign_pointer(ftrace_graph_notrace_hash, new_hash); 6944 } 6945 6946 mutex_unlock(&graph_lock); 6947 6948 /* 6949 * We need to do a hard force of sched synchronization. 6950 * This is because we use preempt_disable() to do RCU, but 6951 * the function tracers can be called where RCU is not watching 6952 * (like before user_exit()). We can not rely on the RCU 6953 * infrastructure to do the synchronization, thus we must do it 6954 * ourselves. 6955 */ 6956 if (old_hash != EMPTY_HASH) 6957 synchronize_rcu_tasks_rude(); 6958 6959 free_ftrace_hash(old_hash); 6960 } 6961 6962 out: 6963 free_ftrace_hash(fgd->new_hash); 6964 kfree(fgd); 6965 6966 return ret; 6967} 6968 6969static int 6970ftrace_graph_set_hash(struct ftrace_hash *hash, char *buffer) 6971{ 6972 struct ftrace_glob func_g; 6973 struct dyn_ftrace *rec; 6974 struct ftrace_page *pg; 6975 struct ftrace_func_entry *entry; 6976 int fail = 1; 6977 int not; 6978 6979 /* decode regex */ 6980 func_g.type = filter_parse_regex(buffer, strlen(buffer), 6981 &func_g.search, ¬); 6982 6983 func_g.len = strlen(func_g.search); 6984 6985 guard(mutex)(&ftrace_lock); 6986 6987 if (unlikely(ftrace_disabled)) 6988 return -ENODEV; 6989 6990 do_for_each_ftrace_rec(pg, rec) { 6991 6992 if (rec->flags & FTRACE_FL_DISABLED) 6993 continue; 6994 6995 if (ftrace_match_record(rec, &func_g, NULL, 0)) { 6996 entry = ftrace_lookup_ip(hash, rec->ip); 6997 6998 if (!not) { 6999 fail = 0; 7000 7001 if (entry) 7002 continue; 7003 if (add_hash_entry(hash, rec->ip) == NULL) 7004 return 0; 7005 } else { 7006 if (entry) { 7007 free_hash_entry(hash, entry); 7008 fail = 0; 7009 } 7010 } 7011 } 7012 cond_resched(); 7013 } while_for_each_ftrace_rec(); 7014 7015 return fail ? -EINVAL : 0; 7016} 7017 7018static ssize_t 7019ftrace_graph_write(struct file *file, const char __user *ubuf, 7020 size_t cnt, loff_t *ppos) 7021{ 7022 ssize_t read, ret = 0; 7023 struct ftrace_graph_data *fgd = file->private_data; 7024 struct trace_parser *parser; 7025 7026 if (!cnt) 7027 return 0; 7028 7029 /* Read mode uses seq functions */ 7030 if (file->f_mode & FMODE_READ) { 7031 struct seq_file *m = file->private_data; 7032 fgd = m->private; 7033 } 7034 7035 parser = &fgd->parser; 7036 7037 read = trace_get_user(parser, ubuf, cnt, ppos); 7038 7039 if (read >= 0 && trace_parser_loaded(parser) && 7040 !trace_parser_cont(parser)) { 7041 7042 ret = ftrace_graph_set_hash(fgd->new_hash, 7043 parser->buffer); 7044 trace_parser_clear(parser); 7045 } 7046 7047 if (!ret) 7048 ret = read; 7049 7050 return ret; 7051} 7052 7053static const struct file_operations ftrace_graph_fops = { 7054 .open = ftrace_graph_open, 7055 .read = seq_read, 7056 .write = ftrace_graph_write, 7057 .llseek = tracing_lseek, 7058 .release = ftrace_graph_release, 7059}; 7060 7061static const struct file_operations ftrace_graph_notrace_fops = { 7062 .open = ftrace_graph_notrace_open, 7063 .read = seq_read, 7064 .write = ftrace_graph_write, 7065 .llseek = tracing_lseek, 7066 .release = ftrace_graph_release, 7067}; 7068#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 7069 7070void ftrace_create_filter_files(struct ftrace_ops *ops, 7071 struct dentry *parent) 7072{ 7073 7074 trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent, 7075 ops, &ftrace_filter_fops); 7076 7077 trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent, 7078 ops, &ftrace_notrace_fops); 7079} 7080 7081/* 7082 * The name "destroy_filter_files" is really a misnomer. Although 7083 * in the future, it may actually delete the files, but this is 7084 * really intended to make sure the ops passed in are disabled 7085 * and that when this function returns, the caller is free to 7086 * free the ops. 7087 * 7088 * The "destroy" name is only to match the "create" name that this 7089 * should be paired with. 7090 */ 7091void ftrace_destroy_filter_files(struct ftrace_ops *ops) 7092{ 7093 mutex_lock(&ftrace_lock); 7094 if (ops->flags & FTRACE_OPS_FL_ENABLED) 7095 ftrace_shutdown(ops, 0); 7096 ops->flags |= FTRACE_OPS_FL_DELETED; 7097 ftrace_free_filter(ops); 7098 mutex_unlock(&ftrace_lock); 7099} 7100 7101static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) 7102{ 7103 7104 trace_create_file("available_filter_functions", TRACE_MODE_READ, 7105 d_tracer, NULL, &ftrace_avail_fops); 7106 7107 trace_create_file("available_filter_functions_addrs", TRACE_MODE_READ, 7108 d_tracer, NULL, &ftrace_avail_addrs_fops); 7109 7110 trace_create_file("enabled_functions", TRACE_MODE_READ, 7111 d_tracer, NULL, &ftrace_enabled_fops); 7112 7113 trace_create_file("touched_functions", TRACE_MODE_READ, 7114 d_tracer, NULL, &ftrace_touched_fops); 7115 7116 ftrace_create_filter_files(&global_ops, d_tracer); 7117 7118#ifdef CONFIG_FUNCTION_GRAPH_TRACER 7119 trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer, 7120 NULL, 7121 &ftrace_graph_fops); 7122 trace_create_file("set_graph_notrace", TRACE_MODE_WRITE, d_tracer, 7123 NULL, 7124 &ftrace_graph_notrace_fops); 7125#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 7126 7127 return 0; 7128} 7129 7130static int ftrace_cmp_ips(const void *a, const void *b) 7131{ 7132 const unsigned long *ipa = a; 7133 const unsigned long *ipb = b; 7134 7135 if (*ipa > *ipb) 7136 return 1; 7137 if (*ipa < *ipb) 7138 return -1; 7139 return 0; 7140} 7141 7142#ifdef CONFIG_FTRACE_SORT_STARTUP_TEST 7143static void test_is_sorted(unsigned long *start, unsigned long count) 7144{ 7145 int i; 7146 7147 for (i = 1; i < count; i++) { 7148 if (WARN(start[i - 1] > start[i], 7149 "[%d] %pS at %lx is not sorted with %pS at %lx\n", i, 7150 (void *)start[i - 1], start[i - 1], 7151 (void *)start[i], start[i])) 7152 break; 7153 } 7154 if (i == count) 7155 pr_info("ftrace section at %px sorted properly\n", start); 7156} 7157#else 7158static void test_is_sorted(unsigned long *start, unsigned long count) 7159{ 7160} 7161#endif 7162 7163static int ftrace_process_locs(struct module *mod, 7164 unsigned long *start, 7165 unsigned long *end) 7166{ 7167 struct ftrace_page *pg_unuse = NULL; 7168 struct ftrace_page *start_pg; 7169 struct ftrace_page *pg; 7170 struct dyn_ftrace *rec; 7171 unsigned long skipped = 0; 7172 unsigned long count; 7173 unsigned long *p; 7174 unsigned long addr; 7175 unsigned long flags = 0; /* Shut up gcc */ 7176 unsigned long pages; 7177 int ret = -ENOMEM; 7178 7179 count = end - start; 7180 7181 if (!count) 7182 return 0; 7183 7184 /* 7185 * Sorting mcount in vmlinux at build time depend on 7186 * CONFIG_BUILDTIME_MCOUNT_SORT, while mcount loc in 7187 * modules can not be sorted at build time. 7188 */ 7189 if (!IS_ENABLED(CONFIG_BUILDTIME_MCOUNT_SORT) || mod) { 7190 sort(start, count, sizeof(*start), 7191 ftrace_cmp_ips, NULL); 7192 } else { 7193 test_is_sorted(start, count); 7194 } 7195 7196 start_pg = ftrace_allocate_pages(count, &pages); 7197 if (!start_pg) 7198 return -ENOMEM; 7199 7200 mutex_lock(&ftrace_lock); 7201 7202 /* 7203 * Core and each module needs their own pages, as 7204 * modules will free them when they are removed. 7205 * Force a new page to be allocated for modules. 7206 */ 7207 if (!mod) { 7208 WARN_ON(ftrace_pages || ftrace_pages_start); 7209 /* First initialization */ 7210 ftrace_pages = ftrace_pages_start = start_pg; 7211 } else { 7212 if (!ftrace_pages) 7213 goto out; 7214 7215 if (WARN_ON(ftrace_pages->next)) { 7216 /* Hmm, we have free pages? */ 7217 while (ftrace_pages->next) 7218 ftrace_pages = ftrace_pages->next; 7219 } 7220 7221 ftrace_pages->next = start_pg; 7222 } 7223 7224 p = start; 7225 pg = start_pg; 7226 while (p < end) { 7227 unsigned long end_offset; 7228 7229 addr = *p++; 7230 7231 /* 7232 * Some architecture linkers will pad between 7233 * the different mcount_loc sections of different 7234 * object files to satisfy alignments. 7235 * Skip any NULL pointers. 7236 */ 7237 if (!addr) { 7238 skipped++; 7239 continue; 7240 } 7241 7242 /* 7243 * If this is core kernel, make sure the address is in core 7244 * or inittext, as weak functions get zeroed and KASLR can 7245 * move them to something other than zero. It just will not 7246 * move it to an area where kernel text is. 7247 */ 7248 if (!mod && !(is_kernel_text(addr) || is_kernel_inittext(addr))) { 7249 skipped++; 7250 continue; 7251 } 7252 7253 addr = ftrace_call_adjust(addr); 7254 7255 end_offset = (pg->index+1) * sizeof(pg->records[0]); 7256 if (end_offset > PAGE_SIZE << pg->order) { 7257 /* We should have allocated enough */ 7258 if (WARN_ON(!pg->next)) 7259 break; 7260 pg = pg->next; 7261 } 7262 7263 rec = &pg->records[pg->index++]; 7264 rec->ip = addr; 7265 } 7266 7267 if (pg->next) { 7268 pg_unuse = pg->next; 7269 pg->next = NULL; 7270 } 7271 7272 /* Assign the last page to ftrace_pages */ 7273 ftrace_pages = pg; 7274 7275 /* 7276 * We only need to disable interrupts on start up 7277 * because we are modifying code that an interrupt 7278 * may execute, and the modification is not atomic. 7279 * But for modules, nothing runs the code we modify 7280 * until we are finished with it, and there's no 7281 * reason to cause large interrupt latencies while we do it. 7282 */ 7283 if (!mod) 7284 local_irq_save(flags); 7285 ftrace_update_code(mod, start_pg); 7286 if (!mod) 7287 local_irq_restore(flags); 7288 ret = 0; 7289 out: 7290 mutex_unlock(&ftrace_lock); 7291 7292 /* We should have used all pages unless we skipped some */ 7293 if (pg_unuse) { 7294 unsigned long pg_remaining, remaining = 0; 7295 long skip; 7296 7297 /* Count the number of entries unused and compare it to skipped. */ 7298 pg_remaining = (PAGE_SIZE << pg->order) / ENTRY_SIZE - pg->index; 7299 7300 if (!WARN(skipped < pg_remaining, "Extra allocated pages for ftrace")) { 7301 7302 skip = skipped - pg_remaining; 7303 7304 for (pg = pg_unuse; pg && skip > 0; pg = pg->next) { 7305 remaining += 1 << pg->order; 7306 skip -= (PAGE_SIZE << pg->order) / ENTRY_SIZE; 7307 } 7308 7309 pages -= remaining; 7310 7311 /* 7312 * Check to see if the number of pages remaining would 7313 * just fit the number of entries skipped. 7314 */ 7315 WARN(pg || skip > 0, "Extra allocated pages for ftrace: %lu with %lu skipped", 7316 remaining, skipped); 7317 } 7318 /* Need to synchronize with ftrace_location_range() */ 7319 synchronize_rcu(); 7320 ftrace_free_pages(pg_unuse); 7321 } 7322 7323 if (!mod) { 7324 count -= skipped; 7325 pr_info("ftrace: allocating %ld entries in %ld pages\n", 7326 count, pages); 7327 } 7328 7329 return ret; 7330} 7331 7332struct ftrace_mod_func { 7333 struct list_head list; 7334 char *name; 7335 unsigned long ip; 7336 unsigned int size; 7337}; 7338 7339struct ftrace_mod_map { 7340 struct rcu_head rcu; 7341 struct list_head list; 7342 struct module *mod; 7343 unsigned long start_addr; 7344 unsigned long end_addr; 7345 struct list_head funcs; 7346 unsigned int num_funcs; 7347}; 7348 7349static int ftrace_get_trampoline_kallsym(unsigned int symnum, 7350 unsigned long *value, char *type, 7351 char *name, char *module_name, 7352 int *exported) 7353{ 7354 struct ftrace_ops *op; 7355 7356 list_for_each_entry_rcu(op, &ftrace_ops_trampoline_list, list) { 7357 if (!op->trampoline || symnum--) 7358 continue; 7359 *value = op->trampoline; 7360 *type = 't'; 7361 strscpy(name, FTRACE_TRAMPOLINE_SYM, KSYM_NAME_LEN); 7362 strscpy(module_name, FTRACE_TRAMPOLINE_MOD, MODULE_NAME_LEN); 7363 *exported = 0; 7364 return 0; 7365 } 7366 7367 return -ERANGE; 7368} 7369 7370#if defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) || defined(CONFIG_MODULES) 7371/* 7372 * Check if the current ops references the given ip. 7373 * 7374 * If the ops traces all functions, then it was already accounted for. 7375 * If the ops does not trace the current record function, skip it. 7376 * If the ops ignores the function via notrace filter, skip it. 7377 */ 7378static bool 7379ops_references_ip(struct ftrace_ops *ops, unsigned long ip) 7380{ 7381 /* If ops isn't enabled, ignore it */ 7382 if (!(ops->flags & FTRACE_OPS_FL_ENABLED)) 7383 return false; 7384 7385 /* If ops traces all then it includes this function */ 7386 if (ops_traces_mod(ops)) 7387 return true; 7388 7389 /* The function must be in the filter */ 7390 if (!ftrace_hash_empty(ops->func_hash->filter_hash) && 7391 !__ftrace_lookup_ip(ops->func_hash->filter_hash, ip)) 7392 return false; 7393 7394 /* If in notrace hash, we ignore it too */ 7395 if (ftrace_lookup_ip(ops->func_hash->notrace_hash, ip)) 7396 return false; 7397 7398 return true; 7399} 7400#endif 7401 7402#ifdef CONFIG_MODULES 7403 7404#define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next) 7405 7406static LIST_HEAD(ftrace_mod_maps); 7407 7408static int referenced_filters(struct dyn_ftrace *rec) 7409{ 7410 struct ftrace_ops *ops; 7411 int cnt = 0; 7412 7413 for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) { 7414 if (ops_references_ip(ops, rec->ip)) { 7415 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_DIRECT)) 7416 continue; 7417 if (WARN_ON_ONCE(ops->flags & FTRACE_OPS_FL_IPMODIFY)) 7418 continue; 7419 cnt++; 7420 if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) 7421 rec->flags |= FTRACE_FL_REGS; 7422 if (cnt == 1 && ops->trampoline) 7423 rec->flags |= FTRACE_FL_TRAMP; 7424 else 7425 rec->flags &= ~FTRACE_FL_TRAMP; 7426 } 7427 } 7428 7429 return cnt; 7430} 7431 7432static void 7433clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash) 7434{ 7435 struct ftrace_func_entry *entry; 7436 struct dyn_ftrace *rec; 7437 int i; 7438 7439 if (ftrace_hash_empty(hash)) 7440 return; 7441 7442 for (i = 0; i < pg->index; i++) { 7443 rec = &pg->records[i]; 7444 entry = __ftrace_lookup_ip(hash, rec->ip); 7445 /* 7446 * Do not allow this rec to match again. 7447 * Yeah, it may waste some memory, but will be removed 7448 * if/when the hash is modified again. 7449 */ 7450 if (entry) 7451 entry->ip = 0; 7452 } 7453} 7454 7455/* Clear any records from hashes */ 7456static void clear_mod_from_hashes(struct ftrace_page *pg) 7457{ 7458 struct trace_array *tr; 7459 7460 mutex_lock(&trace_types_lock); 7461 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 7462 if (!tr->ops || !tr->ops->func_hash) 7463 continue; 7464 mutex_lock(&tr->ops->func_hash->regex_lock); 7465 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash); 7466 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash); 7467 mutex_unlock(&tr->ops->func_hash->regex_lock); 7468 } 7469 mutex_unlock(&trace_types_lock); 7470} 7471 7472static void ftrace_free_mod_map(struct rcu_head *rcu) 7473{ 7474 struct ftrace_mod_map *mod_map = container_of(rcu, struct ftrace_mod_map, rcu); 7475 struct ftrace_mod_func *mod_func; 7476 struct ftrace_mod_func *n; 7477 7478 /* All the contents of mod_map are now not visible to readers */ 7479 list_for_each_entry_safe(mod_func, n, &mod_map->funcs, list) { 7480 kfree(mod_func->name); 7481 list_del(&mod_func->list); 7482 kfree(mod_func); 7483 } 7484 7485 kfree(mod_map); 7486} 7487 7488void ftrace_release_mod(struct module *mod) 7489{ 7490 struct ftrace_mod_map *mod_map; 7491 struct ftrace_mod_map *n; 7492 struct dyn_ftrace *rec; 7493 struct ftrace_page **last_pg; 7494 struct ftrace_page *tmp_page = NULL; 7495 struct ftrace_page *pg; 7496 7497 mutex_lock(&ftrace_lock); 7498 7499 /* 7500 * To avoid the UAF problem after the module is unloaded, the 7501 * 'mod_map' resource needs to be released unconditionally. 7502 */ 7503 list_for_each_entry_safe(mod_map, n, &ftrace_mod_maps, list) { 7504 if (mod_map->mod == mod) { 7505 list_del_rcu(&mod_map->list); 7506 call_rcu(&mod_map->rcu, ftrace_free_mod_map); 7507 break; 7508 } 7509 } 7510 7511 if (ftrace_disabled) 7512 goto out_unlock; 7513 7514 /* 7515 * Each module has its own ftrace_pages, remove 7516 * them from the list. 7517 */ 7518 last_pg = &ftrace_pages_start; 7519 for (pg = ftrace_pages_start; pg; pg = *last_pg) { 7520 rec = &pg->records[0]; 7521 if (within_module(rec->ip, mod)) { 7522 /* 7523 * As core pages are first, the first 7524 * page should never be a module page. 7525 */ 7526 if (WARN_ON(pg == ftrace_pages_start)) 7527 goto out_unlock; 7528 7529 /* Check if we are deleting the last page */ 7530 if (pg == ftrace_pages) 7531 ftrace_pages = next_to_ftrace_page(last_pg); 7532 7533 ftrace_update_tot_cnt -= pg->index; 7534 *last_pg = pg->next; 7535 7536 pg->next = tmp_page; 7537 tmp_page = pg; 7538 } else 7539 last_pg = &pg->next; 7540 } 7541 out_unlock: 7542 mutex_unlock(&ftrace_lock); 7543 7544 /* Need to synchronize with ftrace_location_range() */ 7545 if (tmp_page) 7546 synchronize_rcu(); 7547 for (pg = tmp_page; pg; pg = tmp_page) { 7548 7549 /* Needs to be called outside of ftrace_lock */ 7550 clear_mod_from_hashes(pg); 7551 7552 if (pg->records) { 7553 free_pages((unsigned long)pg->records, pg->order); 7554 ftrace_number_of_pages -= 1 << pg->order; 7555 } 7556 tmp_page = pg->next; 7557 kfree(pg); 7558 ftrace_number_of_groups--; 7559 } 7560} 7561 7562void ftrace_module_enable(struct module *mod) 7563{ 7564 struct dyn_ftrace *rec; 7565 struct ftrace_page *pg; 7566 7567 mutex_lock(&ftrace_lock); 7568 7569 if (ftrace_disabled) 7570 goto out_unlock; 7571 7572 /* 7573 * If the tracing is enabled, go ahead and enable the record. 7574 * 7575 * The reason not to enable the record immediately is the 7576 * inherent check of ftrace_make_nop/ftrace_make_call for 7577 * correct previous instructions. Making first the NOP 7578 * conversion puts the module to the correct state, thus 7579 * passing the ftrace_make_call check. 7580 * 7581 * We also delay this to after the module code already set the 7582 * text to read-only, as we now need to set it back to read-write 7583 * so that we can modify the text. 7584 */ 7585 if (ftrace_start_up) 7586 ftrace_arch_code_modify_prepare(); 7587 7588 do_for_each_ftrace_rec(pg, rec) { 7589 int cnt; 7590 /* 7591 * do_for_each_ftrace_rec() is a double loop. 7592 * module text shares the pg. If a record is 7593 * not part of this module, then skip this pg, 7594 * which the "break" will do. 7595 */ 7596 if (!within_module(rec->ip, mod)) 7597 break; 7598 7599 cond_resched(); 7600 7601 /* Weak functions should still be ignored */ 7602 if (!test_for_valid_rec(rec)) { 7603 /* Clear all other flags. Should not be enabled anyway */ 7604 rec->flags = FTRACE_FL_DISABLED; 7605 continue; 7606 } 7607 7608 cnt = 0; 7609 7610 /* 7611 * When adding a module, we need to check if tracers are 7612 * currently enabled and if they are, and can trace this record, 7613 * we need to enable the module functions as well as update the 7614 * reference counts for those function records. 7615 */ 7616 if (ftrace_start_up) 7617 cnt += referenced_filters(rec); 7618 7619 rec->flags &= ~FTRACE_FL_DISABLED; 7620 rec->flags += cnt; 7621 7622 if (ftrace_start_up && cnt) { 7623 int failed = __ftrace_replace_code(rec, 1); 7624 if (failed) { 7625 ftrace_bug(failed, rec); 7626 goto out_loop; 7627 } 7628 } 7629 7630 } while_for_each_ftrace_rec(); 7631 7632 out_loop: 7633 if (ftrace_start_up) 7634 ftrace_arch_code_modify_post_process(); 7635 7636 out_unlock: 7637 mutex_unlock(&ftrace_lock); 7638 7639 process_cached_mods(mod->name); 7640} 7641 7642void ftrace_module_init(struct module *mod) 7643{ 7644 int ret; 7645 7646 if (ftrace_disabled || !mod->num_ftrace_callsites) 7647 return; 7648 7649 ret = ftrace_process_locs(mod, mod->ftrace_callsites, 7650 mod->ftrace_callsites + mod->num_ftrace_callsites); 7651 if (ret) 7652 pr_warn("ftrace: failed to allocate entries for module '%s' functions\n", 7653 mod->name); 7654} 7655 7656static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map, 7657 struct dyn_ftrace *rec) 7658{ 7659 struct ftrace_mod_func *mod_func; 7660 unsigned long symsize; 7661 unsigned long offset; 7662 char str[KSYM_SYMBOL_LEN]; 7663 char *modname; 7664 const char *ret; 7665 7666 ret = kallsyms_lookup(rec->ip, &symsize, &offset, &modname, str); 7667 if (!ret) 7668 return; 7669 7670 mod_func = kmalloc(sizeof(*mod_func), GFP_KERNEL); 7671 if (!mod_func) 7672 return; 7673 7674 mod_func->name = kstrdup(str, GFP_KERNEL); 7675 if (!mod_func->name) { 7676 kfree(mod_func); 7677 return; 7678 } 7679 7680 mod_func->ip = rec->ip - offset; 7681 mod_func->size = symsize; 7682 7683 mod_map->num_funcs++; 7684 7685 list_add_rcu(&mod_func->list, &mod_map->funcs); 7686} 7687 7688static struct ftrace_mod_map * 7689allocate_ftrace_mod_map(struct module *mod, 7690 unsigned long start, unsigned long end) 7691{ 7692 struct ftrace_mod_map *mod_map; 7693 7694 if (ftrace_disabled) 7695 return NULL; 7696 7697 mod_map = kmalloc(sizeof(*mod_map), GFP_KERNEL); 7698 if (!mod_map) 7699 return NULL; 7700 7701 mod_map->mod = mod; 7702 mod_map->start_addr = start; 7703 mod_map->end_addr = end; 7704 mod_map->num_funcs = 0; 7705 7706 INIT_LIST_HEAD_RCU(&mod_map->funcs); 7707 7708 list_add_rcu(&mod_map->list, &ftrace_mod_maps); 7709 7710 return mod_map; 7711} 7712 7713static int 7714ftrace_func_address_lookup(struct ftrace_mod_map *mod_map, 7715 unsigned long addr, unsigned long *size, 7716 unsigned long *off, char *sym) 7717{ 7718 struct ftrace_mod_func *found_func = NULL; 7719 struct ftrace_mod_func *mod_func; 7720 7721 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { 7722 if (addr >= mod_func->ip && 7723 addr < mod_func->ip + mod_func->size) { 7724 found_func = mod_func; 7725 break; 7726 } 7727 } 7728 7729 if (found_func) { 7730 if (size) 7731 *size = found_func->size; 7732 if (off) 7733 *off = addr - found_func->ip; 7734 return strscpy(sym, found_func->name, KSYM_NAME_LEN); 7735 } 7736 7737 return 0; 7738} 7739 7740int 7741ftrace_mod_address_lookup(unsigned long addr, unsigned long *size, 7742 unsigned long *off, char **modname, 7743 const unsigned char **modbuildid, char *sym) 7744{ 7745 struct ftrace_mod_map *mod_map; 7746 int ret = 0; 7747 7748 /* mod_map is freed via call_rcu() */ 7749 preempt_disable(); 7750 list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) { 7751 ret = ftrace_func_address_lookup(mod_map, addr, size, off, sym); 7752 if (ret) { 7753 if (modname) 7754 *modname = mod_map->mod->name; 7755 if (modbuildid) 7756 *modbuildid = module_buildid(mod_map->mod); 7757 break; 7758 } 7759 } 7760 preempt_enable(); 7761 7762 return ret; 7763} 7764 7765int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value, 7766 char *type, char *name, 7767 char *module_name, int *exported) 7768{ 7769 struct ftrace_mod_map *mod_map; 7770 struct ftrace_mod_func *mod_func; 7771 int ret; 7772 7773 preempt_disable(); 7774 list_for_each_entry_rcu(mod_map, &ftrace_mod_maps, list) { 7775 7776 if (symnum >= mod_map->num_funcs) { 7777 symnum -= mod_map->num_funcs; 7778 continue; 7779 } 7780 7781 list_for_each_entry_rcu(mod_func, &mod_map->funcs, list) { 7782 if (symnum > 1) { 7783 symnum--; 7784 continue; 7785 } 7786 7787 *value = mod_func->ip; 7788 *type = 'T'; 7789 strscpy(name, mod_func->name, KSYM_NAME_LEN); 7790 strscpy(module_name, mod_map->mod->name, MODULE_NAME_LEN); 7791 *exported = 1; 7792 preempt_enable(); 7793 return 0; 7794 } 7795 WARN_ON(1); 7796 break; 7797 } 7798 ret = ftrace_get_trampoline_kallsym(symnum, value, type, name, 7799 module_name, exported); 7800 preempt_enable(); 7801 return ret; 7802} 7803 7804#else 7805static void save_ftrace_mod_rec(struct ftrace_mod_map *mod_map, 7806 struct dyn_ftrace *rec) { } 7807static inline struct ftrace_mod_map * 7808allocate_ftrace_mod_map(struct module *mod, 7809 unsigned long start, unsigned long end) 7810{ 7811 return NULL; 7812} 7813int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *value, 7814 char *type, char *name, char *module_name, 7815 int *exported) 7816{ 7817 int ret; 7818 7819 preempt_disable(); 7820 ret = ftrace_get_trampoline_kallsym(symnum, value, type, name, 7821 module_name, exported); 7822 preempt_enable(); 7823 return ret; 7824} 7825#endif /* CONFIG_MODULES */ 7826 7827struct ftrace_init_func { 7828 struct list_head list; 7829 unsigned long ip; 7830}; 7831 7832/* Clear any init ips from hashes */ 7833static void 7834clear_func_from_hash(struct ftrace_init_func *func, struct ftrace_hash *hash) 7835{ 7836 struct ftrace_func_entry *entry; 7837 7838 entry = ftrace_lookup_ip(hash, func->ip); 7839 /* 7840 * Do not allow this rec to match again. 7841 * Yeah, it may waste some memory, but will be removed 7842 * if/when the hash is modified again. 7843 */ 7844 if (entry) 7845 entry->ip = 0; 7846} 7847 7848static void 7849clear_func_from_hashes(struct ftrace_init_func *func) 7850{ 7851 struct trace_array *tr; 7852 7853 mutex_lock(&trace_types_lock); 7854 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 7855 if (!tr->ops || !tr->ops->func_hash) 7856 continue; 7857 mutex_lock(&tr->ops->func_hash->regex_lock); 7858 clear_func_from_hash(func, tr->ops->func_hash->filter_hash); 7859 clear_func_from_hash(func, tr->ops->func_hash->notrace_hash); 7860 mutex_unlock(&tr->ops->func_hash->regex_lock); 7861 } 7862 mutex_unlock(&trace_types_lock); 7863} 7864 7865static void add_to_clear_hash_list(struct list_head *clear_list, 7866 struct dyn_ftrace *rec) 7867{ 7868 struct ftrace_init_func *func; 7869 7870 func = kmalloc(sizeof(*func), GFP_KERNEL); 7871 if (!func) { 7872 MEM_FAIL(1, "alloc failure, ftrace filter could be stale\n"); 7873 return; 7874 } 7875 7876 func->ip = rec->ip; 7877 list_add(&func->list, clear_list); 7878} 7879 7880void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr) 7881{ 7882 unsigned long start = (unsigned long)(start_ptr); 7883 unsigned long end = (unsigned long)(end_ptr); 7884 struct ftrace_page **last_pg = &ftrace_pages_start; 7885 struct ftrace_page *tmp_page = NULL; 7886 struct ftrace_page *pg; 7887 struct dyn_ftrace *rec; 7888 struct dyn_ftrace key; 7889 struct ftrace_mod_map *mod_map = NULL; 7890 struct ftrace_init_func *func, *func_next; 7891 LIST_HEAD(clear_hash); 7892 7893 key.ip = start; 7894 key.flags = end; /* overload flags, as it is unsigned long */ 7895 7896 mutex_lock(&ftrace_lock); 7897 7898 /* 7899 * If we are freeing module init memory, then check if 7900 * any tracer is active. If so, we need to save a mapping of 7901 * the module functions being freed with the address. 7902 */ 7903 if (mod && ftrace_ops_list != &ftrace_list_end) 7904 mod_map = allocate_ftrace_mod_map(mod, start, end); 7905 7906 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) { 7907 if (end < pg->records[0].ip || 7908 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) 7909 continue; 7910 again: 7911 rec = bsearch(&key, pg->records, pg->index, 7912 sizeof(struct dyn_ftrace), 7913 ftrace_cmp_recs); 7914 if (!rec) 7915 continue; 7916 7917 /* rec will be cleared from hashes after ftrace_lock unlock */ 7918 add_to_clear_hash_list(&clear_hash, rec); 7919 7920 if (mod_map) 7921 save_ftrace_mod_rec(mod_map, rec); 7922 7923 pg->index--; 7924 ftrace_update_tot_cnt--; 7925 if (!pg->index) { 7926 *last_pg = pg->next; 7927 pg->next = tmp_page; 7928 tmp_page = pg; 7929 pg = container_of(last_pg, struct ftrace_page, next); 7930 if (!(*last_pg)) 7931 ftrace_pages = pg; 7932 continue; 7933 } 7934 memmove(rec, rec + 1, 7935 (pg->index - (rec - pg->records)) * sizeof(*rec)); 7936 /* More than one function may be in this block */ 7937 goto again; 7938 } 7939 mutex_unlock(&ftrace_lock); 7940 7941 list_for_each_entry_safe(func, func_next, &clear_hash, list) { 7942 clear_func_from_hashes(func); 7943 kfree(func); 7944 } 7945 /* Need to synchronize with ftrace_location_range() */ 7946 if (tmp_page) { 7947 synchronize_rcu(); 7948 ftrace_free_pages(tmp_page); 7949 } 7950} 7951 7952void __init ftrace_free_init_mem(void) 7953{ 7954 void *start = (void *)(&__init_begin); 7955 void *end = (void *)(&__init_end); 7956 7957 ftrace_boot_snapshot(); 7958 7959 ftrace_free_mem(NULL, start, end); 7960} 7961 7962int __init __weak ftrace_dyn_arch_init(void) 7963{ 7964 return 0; 7965} 7966 7967void __init ftrace_init(void) 7968{ 7969 extern unsigned long __start_mcount_loc[]; 7970 extern unsigned long __stop_mcount_loc[]; 7971 unsigned long count, flags; 7972 int ret; 7973 7974 local_irq_save(flags); 7975 ret = ftrace_dyn_arch_init(); 7976 local_irq_restore(flags); 7977 if (ret) 7978 goto failed; 7979 7980 count = __stop_mcount_loc - __start_mcount_loc; 7981 if (!count) { 7982 pr_info("ftrace: No functions to be traced?\n"); 7983 goto failed; 7984 } 7985 7986 ret = ftrace_process_locs(NULL, 7987 __start_mcount_loc, 7988 __stop_mcount_loc); 7989 if (ret) { 7990 pr_warn("ftrace: failed to allocate entries for functions\n"); 7991 goto failed; 7992 } 7993 7994 pr_info("ftrace: allocated %ld pages with %ld groups\n", 7995 ftrace_number_of_pages, ftrace_number_of_groups); 7996 7997 last_ftrace_enabled = ftrace_enabled = 1; 7998 7999 set_ftrace_early_filters(); 8000 8001 return; 8002 failed: 8003 ftrace_disabled = 1; 8004} 8005 8006/* Do nothing if arch does not support this */ 8007void __weak arch_ftrace_update_trampoline(struct ftrace_ops *ops) 8008{ 8009} 8010 8011static void ftrace_update_trampoline(struct ftrace_ops *ops) 8012{ 8013 unsigned long trampoline = ops->trampoline; 8014 8015 arch_ftrace_update_trampoline(ops); 8016 if (ops->trampoline && ops->trampoline != trampoline && 8017 (ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP)) { 8018 /* Add to kallsyms before the perf events */ 8019 ftrace_add_trampoline_to_kallsyms(ops); 8020 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, 8021 ops->trampoline, ops->trampoline_size, false, 8022 FTRACE_TRAMPOLINE_SYM); 8023 /* 8024 * Record the perf text poke event after the ksymbol register 8025 * event. 8026 */ 8027 perf_event_text_poke((void *)ops->trampoline, NULL, 0, 8028 (void *)ops->trampoline, 8029 ops->trampoline_size); 8030 } 8031} 8032 8033void ftrace_init_trace_array(struct trace_array *tr) 8034{ 8035 if (tr->flags & TRACE_ARRAY_FL_MOD_INIT) 8036 return; 8037 8038 INIT_LIST_HEAD(&tr->func_probes); 8039 INIT_LIST_HEAD(&tr->mod_trace); 8040 INIT_LIST_HEAD(&tr->mod_notrace); 8041 8042 tr->flags |= TRACE_ARRAY_FL_MOD_INIT; 8043} 8044#else 8045 8046struct ftrace_ops global_ops = { 8047 .func = ftrace_stub, 8048 .flags = FTRACE_OPS_FL_INITIALIZED | 8049 FTRACE_OPS_FL_PID, 8050}; 8051 8052static int __init ftrace_nodyn_init(void) 8053{ 8054 ftrace_enabled = 1; 8055 return 0; 8056} 8057core_initcall(ftrace_nodyn_init); 8058 8059static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } 8060static inline void ftrace_startup_all(int command) { } 8061 8062static void ftrace_update_trampoline(struct ftrace_ops *ops) 8063{ 8064} 8065 8066#endif /* CONFIG_DYNAMIC_FTRACE */ 8067 8068__init void ftrace_init_global_array_ops(struct trace_array *tr) 8069{ 8070 tr->ops = &global_ops; 8071 if (!global_ops.private) 8072 global_ops.private = tr; 8073 ftrace_init_trace_array(tr); 8074 init_array_fgraph_ops(tr, tr->ops); 8075} 8076 8077void ftrace_init_array_ops(struct trace_array *tr, ftrace_func_t func) 8078{ 8079 /* If we filter on pids, update to use the pid function */ 8080 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) { 8081 if (WARN_ON(tr->ops->func != ftrace_stub)) 8082 printk("ftrace ops had %pS for function\n", 8083 tr->ops->func); 8084 } 8085 tr->ops->func = func; 8086 tr->ops->private = tr; 8087} 8088 8089void ftrace_reset_array_ops(struct trace_array *tr) 8090{ 8091 tr->ops->func = ftrace_stub; 8092} 8093 8094static nokprobe_inline void 8095__ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 8096 struct ftrace_ops *ignored, struct ftrace_regs *fregs) 8097{ 8098 struct pt_regs *regs = ftrace_get_regs(fregs); 8099 struct ftrace_ops *op; 8100 int bit; 8101 8102 /* 8103 * The ftrace_test_and_set_recursion() will disable preemption, 8104 * which is required since some of the ops may be dynamically 8105 * allocated, they must be freed after a synchronize_rcu(). 8106 */ 8107 bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START); 8108 if (bit < 0) 8109 return; 8110 8111 do_for_each_ftrace_op(op, ftrace_ops_list) { 8112 /* Stub functions don't need to be called nor tested */ 8113 if (op->flags & FTRACE_OPS_FL_STUB) 8114 continue; 8115 /* 8116 * Check the following for each ops before calling their func: 8117 * if RCU flag is set, then rcu_is_watching() must be true 8118 * Otherwise test if the ip matches the ops filter 8119 * 8120 * If any of the above fails then the op->func() is not executed. 8121 */ 8122 if ((!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) && 8123 ftrace_ops_test(op, ip, regs)) { 8124 if (FTRACE_WARN_ON(!op->func)) { 8125 pr_warn("op=%p %pS\n", op, op); 8126 goto out; 8127 } 8128 op->func(ip, parent_ip, op, fregs); 8129 } 8130 } while_for_each_ftrace_op(op); 8131out: 8132 trace_clear_recursion(bit); 8133} 8134 8135/* 8136 * Some archs only support passing ip and parent_ip. Even though 8137 * the list function ignores the op parameter, we do not want any 8138 * C side effects, where a function is called without the caller 8139 * sending a third parameter. 8140 * Archs are to support both the regs and ftrace_ops at the same time. 8141 * If they support ftrace_ops, it is assumed they support regs. 8142 * If call backs want to use regs, they must either check for regs 8143 * being NULL, or CONFIG_DYNAMIC_FTRACE_WITH_REGS. 8144 * Note, CONFIG_DYNAMIC_FTRACE_WITH_REGS expects a full regs to be saved. 8145 * An architecture can pass partial regs with ftrace_ops and still 8146 * set the ARCH_SUPPORTS_FTRACE_OPS. 8147 * 8148 * In vmlinux.lds.h, ftrace_ops_list_func() is defined to be 8149 * arch_ftrace_ops_list_func. 8150 */ 8151#if ARCH_SUPPORTS_FTRACE_OPS 8152void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, 8153 struct ftrace_ops *op, struct ftrace_regs *fregs) 8154{ 8155 kmsan_unpoison_memory(fregs, ftrace_regs_size()); 8156 __ftrace_ops_list_func(ip, parent_ip, NULL, fregs); 8157} 8158#else 8159void arch_ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip) 8160{ 8161 __ftrace_ops_list_func(ip, parent_ip, NULL, NULL); 8162} 8163#endif 8164NOKPROBE_SYMBOL(arch_ftrace_ops_list_func); 8165 8166/* 8167 * If there's only one function registered but it does not support 8168 * recursion, needs RCU protection, then this function will be called 8169 * by the mcount trampoline. 8170 */ 8171static void ftrace_ops_assist_func(unsigned long ip, unsigned long parent_ip, 8172 struct ftrace_ops *op, struct ftrace_regs *fregs) 8173{ 8174 int bit; 8175 8176 bit = trace_test_and_set_recursion(ip, parent_ip, TRACE_LIST_START); 8177 if (bit < 0) 8178 return; 8179 8180 if (!(op->flags & FTRACE_OPS_FL_RCU) || rcu_is_watching()) 8181 op->func(ip, parent_ip, op, fregs); 8182 8183 trace_clear_recursion(bit); 8184} 8185NOKPROBE_SYMBOL(ftrace_ops_assist_func); 8186 8187/** 8188 * ftrace_ops_get_func - get the function a trampoline should call 8189 * @ops: the ops to get the function for 8190 * 8191 * Normally the mcount trampoline will call the ops->func, but there 8192 * are times that it should not. For example, if the ops does not 8193 * have its own recursion protection, then it should call the 8194 * ftrace_ops_assist_func() instead. 8195 * 8196 * Returns: the function that the trampoline should call for @ops. 8197 */ 8198ftrace_func_t ftrace_ops_get_func(struct ftrace_ops *ops) 8199{ 8200 /* 8201 * If the function does not handle recursion or needs to be RCU safe, 8202 * then we need to call the assist handler. 8203 */ 8204 if (ops->flags & (FTRACE_OPS_FL_RECURSION | 8205 FTRACE_OPS_FL_RCU)) 8206 return ftrace_ops_assist_func; 8207 8208 return ops->func; 8209} 8210 8211static void 8212ftrace_filter_pid_sched_switch_probe(void *data, bool preempt, 8213 struct task_struct *prev, 8214 struct task_struct *next, 8215 unsigned int prev_state) 8216{ 8217 struct trace_array *tr = data; 8218 struct trace_pid_list *pid_list; 8219 struct trace_pid_list *no_pid_list; 8220 8221 pid_list = rcu_dereference_sched(tr->function_pids); 8222 no_pid_list = rcu_dereference_sched(tr->function_no_pids); 8223 8224 if (trace_ignore_this_task(pid_list, no_pid_list, next)) 8225 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, 8226 FTRACE_PID_IGNORE); 8227 else 8228 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, 8229 next->pid); 8230} 8231 8232static void 8233ftrace_pid_follow_sched_process_fork(void *data, 8234 struct task_struct *self, 8235 struct task_struct *task) 8236{ 8237 struct trace_pid_list *pid_list; 8238 struct trace_array *tr = data; 8239 8240 pid_list = rcu_dereference_sched(tr->function_pids); 8241 trace_filter_add_remove_task(pid_list, self, task); 8242 8243 pid_list = rcu_dereference_sched(tr->function_no_pids); 8244 trace_filter_add_remove_task(pid_list, self, task); 8245} 8246 8247static void 8248ftrace_pid_follow_sched_process_exit(void *data, struct task_struct *task) 8249{ 8250 struct trace_pid_list *pid_list; 8251 struct trace_array *tr = data; 8252 8253 pid_list = rcu_dereference_sched(tr->function_pids); 8254 trace_filter_add_remove_task(pid_list, NULL, task); 8255 8256 pid_list = rcu_dereference_sched(tr->function_no_pids); 8257 trace_filter_add_remove_task(pid_list, NULL, task); 8258} 8259 8260void ftrace_pid_follow_fork(struct trace_array *tr, bool enable) 8261{ 8262 if (enable) { 8263 register_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork, 8264 tr); 8265 register_trace_sched_process_free(ftrace_pid_follow_sched_process_exit, 8266 tr); 8267 } else { 8268 unregister_trace_sched_process_fork(ftrace_pid_follow_sched_process_fork, 8269 tr); 8270 unregister_trace_sched_process_free(ftrace_pid_follow_sched_process_exit, 8271 tr); 8272 } 8273} 8274 8275static void clear_ftrace_pids(struct trace_array *tr, int type) 8276{ 8277 struct trace_pid_list *pid_list; 8278 struct trace_pid_list *no_pid_list; 8279 int cpu; 8280 8281 pid_list = rcu_dereference_protected(tr->function_pids, 8282 lockdep_is_held(&ftrace_lock)); 8283 no_pid_list = rcu_dereference_protected(tr->function_no_pids, 8284 lockdep_is_held(&ftrace_lock)); 8285 8286 /* Make sure there's something to do */ 8287 if (!pid_type_enabled(type, pid_list, no_pid_list)) 8288 return; 8289 8290 /* See if the pids still need to be checked after this */ 8291 if (!still_need_pid_events(type, pid_list, no_pid_list)) { 8292 unregister_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr); 8293 for_each_possible_cpu(cpu) 8294 per_cpu_ptr(tr->array_buffer.data, cpu)->ftrace_ignore_pid = FTRACE_PID_TRACE; 8295 } 8296 8297 if (type & TRACE_PIDS) 8298 rcu_assign_pointer(tr->function_pids, NULL); 8299 8300 if (type & TRACE_NO_PIDS) 8301 rcu_assign_pointer(tr->function_no_pids, NULL); 8302 8303 /* Wait till all users are no longer using pid filtering */ 8304 synchronize_rcu(); 8305 8306 if ((type & TRACE_PIDS) && pid_list) 8307 trace_pid_list_free(pid_list); 8308 8309 if ((type & TRACE_NO_PIDS) && no_pid_list) 8310 trace_pid_list_free(no_pid_list); 8311} 8312 8313void ftrace_clear_pids(struct trace_array *tr) 8314{ 8315 mutex_lock(&ftrace_lock); 8316 8317 clear_ftrace_pids(tr, TRACE_PIDS | TRACE_NO_PIDS); 8318 8319 mutex_unlock(&ftrace_lock); 8320} 8321 8322static void ftrace_pid_reset(struct trace_array *tr, int type) 8323{ 8324 mutex_lock(&ftrace_lock); 8325 clear_ftrace_pids(tr, type); 8326 8327 ftrace_update_pid_func(); 8328 ftrace_startup_all(0); 8329 8330 mutex_unlock(&ftrace_lock); 8331} 8332 8333/* Greater than any max PID */ 8334#define FTRACE_NO_PIDS (void *)(PID_MAX_LIMIT + 1) 8335 8336static void *fpid_start(struct seq_file *m, loff_t *pos) 8337 __acquires(RCU) 8338{ 8339 struct trace_pid_list *pid_list; 8340 struct trace_array *tr = m->private; 8341 8342 mutex_lock(&ftrace_lock); 8343 rcu_read_lock_sched(); 8344 8345 pid_list = rcu_dereference_sched(tr->function_pids); 8346 8347 if (!pid_list) 8348 return !(*pos) ? FTRACE_NO_PIDS : NULL; 8349 8350 return trace_pid_start(pid_list, pos); 8351} 8352 8353static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) 8354{ 8355 struct trace_array *tr = m->private; 8356 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_pids); 8357 8358 if (v == FTRACE_NO_PIDS) { 8359 (*pos)++; 8360 return NULL; 8361 } 8362 return trace_pid_next(pid_list, v, pos); 8363} 8364 8365static void fpid_stop(struct seq_file *m, void *p) 8366 __releases(RCU) 8367{ 8368 rcu_read_unlock_sched(); 8369 mutex_unlock(&ftrace_lock); 8370} 8371 8372static int fpid_show(struct seq_file *m, void *v) 8373{ 8374 if (v == FTRACE_NO_PIDS) { 8375 seq_puts(m, "no pid\n"); 8376 return 0; 8377 } 8378 8379 return trace_pid_show(m, v); 8380} 8381 8382static const struct seq_operations ftrace_pid_sops = { 8383 .start = fpid_start, 8384 .next = fpid_next, 8385 .stop = fpid_stop, 8386 .show = fpid_show, 8387}; 8388 8389static void *fnpid_start(struct seq_file *m, loff_t *pos) 8390 __acquires(RCU) 8391{ 8392 struct trace_pid_list *pid_list; 8393 struct trace_array *tr = m->private; 8394 8395 mutex_lock(&ftrace_lock); 8396 rcu_read_lock_sched(); 8397 8398 pid_list = rcu_dereference_sched(tr->function_no_pids); 8399 8400 if (!pid_list) 8401 return !(*pos) ? FTRACE_NO_PIDS : NULL; 8402 8403 return trace_pid_start(pid_list, pos); 8404} 8405 8406static void *fnpid_next(struct seq_file *m, void *v, loff_t *pos) 8407{ 8408 struct trace_array *tr = m->private; 8409 struct trace_pid_list *pid_list = rcu_dereference_sched(tr->function_no_pids); 8410 8411 if (v == FTRACE_NO_PIDS) { 8412 (*pos)++; 8413 return NULL; 8414 } 8415 return trace_pid_next(pid_list, v, pos); 8416} 8417 8418static const struct seq_operations ftrace_no_pid_sops = { 8419 .start = fnpid_start, 8420 .next = fnpid_next, 8421 .stop = fpid_stop, 8422 .show = fpid_show, 8423}; 8424 8425static int pid_open(struct inode *inode, struct file *file, int type) 8426{ 8427 const struct seq_operations *seq_ops; 8428 struct trace_array *tr = inode->i_private; 8429 struct seq_file *m; 8430 int ret = 0; 8431 8432 ret = tracing_check_open_get_tr(tr); 8433 if (ret) 8434 return ret; 8435 8436 if ((file->f_mode & FMODE_WRITE) && 8437 (file->f_flags & O_TRUNC)) 8438 ftrace_pid_reset(tr, type); 8439 8440 switch (type) { 8441 case TRACE_PIDS: 8442 seq_ops = &ftrace_pid_sops; 8443 break; 8444 case TRACE_NO_PIDS: 8445 seq_ops = &ftrace_no_pid_sops; 8446 break; 8447 default: 8448 trace_array_put(tr); 8449 WARN_ON_ONCE(1); 8450 return -EINVAL; 8451 } 8452 8453 ret = seq_open(file, seq_ops); 8454 if (ret < 0) { 8455 trace_array_put(tr); 8456 } else { 8457 m = file->private_data; 8458 /* copy tr over to seq ops */ 8459 m->private = tr; 8460 } 8461 8462 return ret; 8463} 8464 8465static int 8466ftrace_pid_open(struct inode *inode, struct file *file) 8467{ 8468 return pid_open(inode, file, TRACE_PIDS); 8469} 8470 8471static int 8472ftrace_no_pid_open(struct inode *inode, struct file *file) 8473{ 8474 return pid_open(inode, file, TRACE_NO_PIDS); 8475} 8476 8477static void ignore_task_cpu(void *data) 8478{ 8479 struct trace_array *tr = data; 8480 struct trace_pid_list *pid_list; 8481 struct trace_pid_list *no_pid_list; 8482 8483 /* 8484 * This function is called by on_each_cpu() while the 8485 * event_mutex is held. 8486 */ 8487 pid_list = rcu_dereference_protected(tr->function_pids, 8488 mutex_is_locked(&ftrace_lock)); 8489 no_pid_list = rcu_dereference_protected(tr->function_no_pids, 8490 mutex_is_locked(&ftrace_lock)); 8491 8492 if (trace_ignore_this_task(pid_list, no_pid_list, current)) 8493 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, 8494 FTRACE_PID_IGNORE); 8495 else 8496 this_cpu_write(tr->array_buffer.data->ftrace_ignore_pid, 8497 current->pid); 8498} 8499 8500static ssize_t 8501pid_write(struct file *filp, const char __user *ubuf, 8502 size_t cnt, loff_t *ppos, int type) 8503{ 8504 struct seq_file *m = filp->private_data; 8505 struct trace_array *tr = m->private; 8506 struct trace_pid_list *filtered_pids; 8507 struct trace_pid_list *other_pids; 8508 struct trace_pid_list *pid_list; 8509 ssize_t ret; 8510 8511 if (!cnt) 8512 return 0; 8513 8514 guard(mutex)(&ftrace_lock); 8515 8516 switch (type) { 8517 case TRACE_PIDS: 8518 filtered_pids = rcu_dereference_protected(tr->function_pids, 8519 lockdep_is_held(&ftrace_lock)); 8520 other_pids = rcu_dereference_protected(tr->function_no_pids, 8521 lockdep_is_held(&ftrace_lock)); 8522 break; 8523 case TRACE_NO_PIDS: 8524 filtered_pids = rcu_dereference_protected(tr->function_no_pids, 8525 lockdep_is_held(&ftrace_lock)); 8526 other_pids = rcu_dereference_protected(tr->function_pids, 8527 lockdep_is_held(&ftrace_lock)); 8528 break; 8529 default: 8530 WARN_ON_ONCE(1); 8531 return -EINVAL; 8532 } 8533 8534 ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt); 8535 if (ret < 0) 8536 return ret; 8537 8538 switch (type) { 8539 case TRACE_PIDS: 8540 rcu_assign_pointer(tr->function_pids, pid_list); 8541 break; 8542 case TRACE_NO_PIDS: 8543 rcu_assign_pointer(tr->function_no_pids, pid_list); 8544 break; 8545 } 8546 8547 8548 if (filtered_pids) { 8549 synchronize_rcu(); 8550 trace_pid_list_free(filtered_pids); 8551 } else if (pid_list && !other_pids) { 8552 /* Register a probe to set whether to ignore the tracing of a task */ 8553 register_trace_sched_switch(ftrace_filter_pid_sched_switch_probe, tr); 8554 } 8555 8556 /* 8557 * Ignoring of pids is done at task switch. But we have to 8558 * check for those tasks that are currently running. 8559 * Always do this in case a pid was appended or removed. 8560 */ 8561 on_each_cpu(ignore_task_cpu, tr, 1); 8562 8563 ftrace_update_pid_func(); 8564 ftrace_startup_all(0); 8565 8566 *ppos += ret; 8567 8568 return ret; 8569} 8570 8571static ssize_t 8572ftrace_pid_write(struct file *filp, const char __user *ubuf, 8573 size_t cnt, loff_t *ppos) 8574{ 8575 return pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS); 8576} 8577 8578static ssize_t 8579ftrace_no_pid_write(struct file *filp, const char __user *ubuf, 8580 size_t cnt, loff_t *ppos) 8581{ 8582 return pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS); 8583} 8584 8585static int 8586ftrace_pid_release(struct inode *inode, struct file *file) 8587{ 8588 struct trace_array *tr = inode->i_private; 8589 8590 trace_array_put(tr); 8591 8592 return seq_release(inode, file); 8593} 8594 8595static const struct file_operations ftrace_pid_fops = { 8596 .open = ftrace_pid_open, 8597 .write = ftrace_pid_write, 8598 .read = seq_read, 8599 .llseek = tracing_lseek, 8600 .release = ftrace_pid_release, 8601}; 8602 8603static const struct file_operations ftrace_no_pid_fops = { 8604 .open = ftrace_no_pid_open, 8605 .write = ftrace_no_pid_write, 8606 .read = seq_read, 8607 .llseek = tracing_lseek, 8608 .release = ftrace_pid_release, 8609}; 8610 8611void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer) 8612{ 8613 trace_create_file("set_ftrace_pid", TRACE_MODE_WRITE, d_tracer, 8614 tr, &ftrace_pid_fops); 8615 trace_create_file("set_ftrace_notrace_pid", TRACE_MODE_WRITE, 8616 d_tracer, tr, &ftrace_no_pid_fops); 8617} 8618 8619void __init ftrace_init_tracefs_toplevel(struct trace_array *tr, 8620 struct dentry *d_tracer) 8621{ 8622 /* Only the top level directory has the dyn_tracefs and profile */ 8623 WARN_ON(!(tr->flags & TRACE_ARRAY_FL_GLOBAL)); 8624 8625 ftrace_init_dyn_tracefs(d_tracer); 8626 ftrace_profile_tracefs(d_tracer); 8627} 8628 8629/** 8630 * ftrace_kill - kill ftrace 8631 * 8632 * This function should be used by panic code. It stops ftrace 8633 * but in a not so nice way. If you need to simply kill ftrace 8634 * from a non-atomic section, use ftrace_kill. 8635 */ 8636void ftrace_kill(void) 8637{ 8638 ftrace_disabled = 1; 8639 ftrace_enabled = 0; 8640 ftrace_trace_function = ftrace_stub; 8641 kprobe_ftrace_kill(); 8642} 8643 8644/** 8645 * ftrace_is_dead - Test if ftrace is dead or not. 8646 * 8647 * Returns: 1 if ftrace is "dead", zero otherwise. 8648 */ 8649int ftrace_is_dead(void) 8650{ 8651 return ftrace_disabled; 8652} 8653 8654#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS 8655/* 8656 * When registering ftrace_ops with IPMODIFY, it is necessary to make sure 8657 * it doesn't conflict with any direct ftrace_ops. If there is existing 8658 * direct ftrace_ops on a kernel function being patched, call 8659 * FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER on it to enable sharing. 8660 * 8661 * @ops: ftrace_ops being registered. 8662 * 8663 * Returns: 8664 * 0 on success; 8665 * Negative on failure. 8666 */ 8667static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) 8668{ 8669 struct ftrace_func_entry *entry; 8670 struct ftrace_hash *hash; 8671 struct ftrace_ops *op; 8672 int size, i, ret; 8673 8674 lockdep_assert_held_once(&direct_mutex); 8675 8676 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) 8677 return 0; 8678 8679 hash = ops->func_hash->filter_hash; 8680 size = 1 << hash->size_bits; 8681 for (i = 0; i < size; i++) { 8682 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 8683 unsigned long ip = entry->ip; 8684 bool found_op = false; 8685 8686 mutex_lock(&ftrace_lock); 8687 do_for_each_ftrace_op(op, ftrace_ops_list) { 8688 if (!(op->flags & FTRACE_OPS_FL_DIRECT)) 8689 continue; 8690 if (ops_references_ip(op, ip)) { 8691 found_op = true; 8692 break; 8693 } 8694 } while_for_each_ftrace_op(op); 8695 mutex_unlock(&ftrace_lock); 8696 8697 if (found_op) { 8698 if (!op->ops_func) 8699 return -EBUSY; 8700 8701 ret = op->ops_func(op, FTRACE_OPS_CMD_ENABLE_SHARE_IPMODIFY_PEER); 8702 if (ret) 8703 return ret; 8704 } 8705 } 8706 } 8707 8708 return 0; 8709} 8710 8711/* 8712 * Similar to prepare_direct_functions_for_ipmodify, clean up after ops 8713 * with IPMODIFY is unregistered. The cleanup is optional for most DIRECT 8714 * ops. 8715 */ 8716static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) 8717{ 8718 struct ftrace_func_entry *entry; 8719 struct ftrace_hash *hash; 8720 struct ftrace_ops *op; 8721 int size, i; 8722 8723 if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY)) 8724 return; 8725 8726 mutex_lock(&direct_mutex); 8727 8728 hash = ops->func_hash->filter_hash; 8729 size = 1 << hash->size_bits; 8730 for (i = 0; i < size; i++) { 8731 hlist_for_each_entry(entry, &hash->buckets[i], hlist) { 8732 unsigned long ip = entry->ip; 8733 bool found_op = false; 8734 8735 mutex_lock(&ftrace_lock); 8736 do_for_each_ftrace_op(op, ftrace_ops_list) { 8737 if (!(op->flags & FTRACE_OPS_FL_DIRECT)) 8738 continue; 8739 if (ops_references_ip(op, ip)) { 8740 found_op = true; 8741 break; 8742 } 8743 } while_for_each_ftrace_op(op); 8744 mutex_unlock(&ftrace_lock); 8745 8746 /* The cleanup is optional, ignore any errors */ 8747 if (found_op && op->ops_func) 8748 op->ops_func(op, FTRACE_OPS_CMD_DISABLE_SHARE_IPMODIFY_PEER); 8749 } 8750 } 8751 mutex_unlock(&direct_mutex); 8752} 8753 8754#define lock_direct_mutex() mutex_lock(&direct_mutex) 8755#define unlock_direct_mutex() mutex_unlock(&direct_mutex) 8756 8757#else /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ 8758 8759static int prepare_direct_functions_for_ipmodify(struct ftrace_ops *ops) 8760{ 8761 return 0; 8762} 8763 8764static void cleanup_direct_functions_after_ipmodify(struct ftrace_ops *ops) 8765{ 8766} 8767 8768#define lock_direct_mutex() do { } while (0) 8769#define unlock_direct_mutex() do { } while (0) 8770 8771#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */ 8772 8773/* 8774 * Similar to register_ftrace_function, except we don't lock direct_mutex. 8775 */ 8776static int register_ftrace_function_nolock(struct ftrace_ops *ops) 8777{ 8778 int ret; 8779 8780 ftrace_ops_init(ops); 8781 8782 mutex_lock(&ftrace_lock); 8783 8784 ret = ftrace_startup(ops, 0); 8785 8786 mutex_unlock(&ftrace_lock); 8787 8788 return ret; 8789} 8790 8791/** 8792 * register_ftrace_function - register a function for profiling 8793 * @ops: ops structure that holds the function for profiling. 8794 * 8795 * Register a function to be called by all functions in the 8796 * kernel. 8797 * 8798 * Note: @ops->func and all the functions it calls must be labeled 8799 * with "notrace", otherwise it will go into a 8800 * recursive loop. 8801 */ 8802int register_ftrace_function(struct ftrace_ops *ops) 8803{ 8804 int ret; 8805 8806 lock_direct_mutex(); 8807 ret = prepare_direct_functions_for_ipmodify(ops); 8808 if (ret < 0) 8809 goto out_unlock; 8810 8811 ret = register_ftrace_function_nolock(ops); 8812 8813out_unlock: 8814 unlock_direct_mutex(); 8815 return ret; 8816} 8817EXPORT_SYMBOL_GPL(register_ftrace_function); 8818 8819/** 8820 * unregister_ftrace_function - unregister a function for profiling. 8821 * @ops: ops structure that holds the function to unregister 8822 * 8823 * Unregister a function that was added to be called by ftrace profiling. 8824 */ 8825int unregister_ftrace_function(struct ftrace_ops *ops) 8826{ 8827 int ret; 8828 8829 mutex_lock(&ftrace_lock); 8830 ret = ftrace_shutdown(ops, 0); 8831 mutex_unlock(&ftrace_lock); 8832 8833 cleanup_direct_functions_after_ipmodify(ops); 8834 return ret; 8835} 8836EXPORT_SYMBOL_GPL(unregister_ftrace_function); 8837 8838static int symbols_cmp(const void *a, const void *b) 8839{ 8840 const char **str_a = (const char **) a; 8841 const char **str_b = (const char **) b; 8842 8843 return strcmp(*str_a, *str_b); 8844} 8845 8846struct kallsyms_data { 8847 unsigned long *addrs; 8848 const char **syms; 8849 size_t cnt; 8850 size_t found; 8851}; 8852 8853/* This function gets called for all kernel and module symbols 8854 * and returns 1 in case we resolved all the requested symbols, 8855 * 0 otherwise. 8856 */ 8857static int kallsyms_callback(void *data, const char *name, unsigned long addr) 8858{ 8859 struct kallsyms_data *args = data; 8860 const char **sym; 8861 int idx; 8862 8863 sym = bsearch(&name, args->syms, args->cnt, sizeof(*args->syms), symbols_cmp); 8864 if (!sym) 8865 return 0; 8866 8867 idx = sym - args->syms; 8868 if (args->addrs[idx]) 8869 return 0; 8870 8871 if (!ftrace_location(addr)) 8872 return 0; 8873 8874 args->addrs[idx] = addr; 8875 args->found++; 8876 return args->found == args->cnt ? 1 : 0; 8877} 8878 8879/** 8880 * ftrace_lookup_symbols - Lookup addresses for array of symbols 8881 * 8882 * @sorted_syms: array of symbols pointers symbols to resolve, 8883 * must be alphabetically sorted 8884 * @cnt: number of symbols/addresses in @syms/@addrs arrays 8885 * @addrs: array for storing resulting addresses 8886 * 8887 * This function looks up addresses for array of symbols provided in 8888 * @syms array (must be alphabetically sorted) and stores them in 8889 * @addrs array, which needs to be big enough to store at least @cnt 8890 * addresses. 8891 * 8892 * Returns: 0 if all provided symbols are found, -ESRCH otherwise. 8893 */ 8894int ftrace_lookup_symbols(const char **sorted_syms, size_t cnt, unsigned long *addrs) 8895{ 8896 struct kallsyms_data args; 8897 int found_all; 8898 8899 memset(addrs, 0, sizeof(*addrs) * cnt); 8900 args.addrs = addrs; 8901 args.syms = sorted_syms; 8902 args.cnt = cnt; 8903 args.found = 0; 8904 8905 found_all = kallsyms_on_each_symbol(kallsyms_callback, &args); 8906 if (found_all) 8907 return 0; 8908 found_all = module_kallsyms_on_each_symbol(NULL, kallsyms_callback, &args); 8909 return found_all ? 0 : -ESRCH; 8910} 8911 8912#ifdef CONFIG_SYSCTL 8913 8914#ifdef CONFIG_DYNAMIC_FTRACE 8915static void ftrace_startup_sysctl(void) 8916{ 8917 int command; 8918 8919 if (unlikely(ftrace_disabled)) 8920 return; 8921 8922 /* Force update next time */ 8923 saved_ftrace_func = NULL; 8924 /* ftrace_start_up is true if we want ftrace running */ 8925 if (ftrace_start_up) { 8926 command = FTRACE_UPDATE_CALLS; 8927 if (ftrace_graph_active) 8928 command |= FTRACE_START_FUNC_RET; 8929 ftrace_startup_enable(command); 8930 } 8931} 8932 8933static void ftrace_shutdown_sysctl(void) 8934{ 8935 int command; 8936 8937 if (unlikely(ftrace_disabled)) 8938 return; 8939 8940 /* ftrace_start_up is true if ftrace is running */ 8941 if (ftrace_start_up) { 8942 command = FTRACE_DISABLE_CALLS; 8943 if (ftrace_graph_active) 8944 command |= FTRACE_STOP_FUNC_RET; 8945 ftrace_run_update_code(command); 8946 } 8947} 8948#else 8949# define ftrace_startup_sysctl() do { } while (0) 8950# define ftrace_shutdown_sysctl() do { } while (0) 8951#endif /* CONFIG_DYNAMIC_FTRACE */ 8952 8953static bool is_permanent_ops_registered(void) 8954{ 8955 struct ftrace_ops *op; 8956 8957 do_for_each_ftrace_op(op, ftrace_ops_list) { 8958 if (op->flags & FTRACE_OPS_FL_PERMANENT) 8959 return true; 8960 } while_for_each_ftrace_op(op); 8961 8962 return false; 8963} 8964 8965static int 8966ftrace_enable_sysctl(const struct ctl_table *table, int write, 8967 void *buffer, size_t *lenp, loff_t *ppos) 8968{ 8969 int ret; 8970 8971 guard(mutex)(&ftrace_lock); 8972 8973 if (unlikely(ftrace_disabled)) 8974 return -ENODEV; 8975 8976 ret = proc_dointvec(table, write, buffer, lenp, ppos); 8977 8978 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 8979 return ret; 8980 8981 if (ftrace_enabled) { 8982 8983 /* we are starting ftrace again */ 8984 if (rcu_dereference_protected(ftrace_ops_list, 8985 lockdep_is_held(&ftrace_lock)) != &ftrace_list_end) 8986 update_ftrace_function(); 8987 8988 ftrace_startup_sysctl(); 8989 8990 } else { 8991 if (is_permanent_ops_registered()) { 8992 ftrace_enabled = true; 8993 return -EBUSY; 8994 } 8995 8996 /* stopping ftrace calls (just send to ftrace_stub) */ 8997 ftrace_trace_function = ftrace_stub; 8998 8999 ftrace_shutdown_sysctl(); 9000 } 9001 9002 last_ftrace_enabled = !!ftrace_enabled; 9003 return 0; 9004} 9005 9006static const struct ctl_table ftrace_sysctls[] = { 9007 { 9008 .procname = "ftrace_enabled", 9009 .data = &ftrace_enabled, 9010 .maxlen = sizeof(int), 9011 .mode = 0644, 9012 .proc_handler = ftrace_enable_sysctl, 9013 }, 9014}; 9015 9016static int __init ftrace_sysctl_init(void) 9017{ 9018 register_sysctl_init("kernel", ftrace_sysctls); 9019 return 0; 9020} 9021late_initcall(ftrace_sysctl_init); 9022#endif
~/
Read-only file preview.