FILE
trace.c
File preview with syntax highlighting.
linux-6.19.14/kernel/trace/trace.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * ring buffer based function tracer 4 * 5 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com> 6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 7 * 8 * Originally taken from the RT patch by: 9 * Arnaldo Carvalho de Melo <acme@redhat.com> 10 * 11 * Based on code from the latency_tracer, that is: 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 Nadia Yvette Chambers 14 */ 15#include <linux/ring_buffer.h> 16#include <linux/utsname.h> 17#include <linux/stacktrace.h> 18#include <linux/writeback.h> 19#include <linux/kallsyms.h> 20#include <linux/security.h> 21#include <linux/seq_file.h> 22#include <linux/irqflags.h> 23#include <linux/syscalls.h> 24#include <linux/debugfs.h> 25#include <linux/tracefs.h> 26#include <linux/pagemap.h> 27#include <linux/hardirq.h> 28#include <linux/linkage.h> 29#include <linux/uaccess.h> 30#include <linux/cleanup.h> 31#include <linux/vmalloc.h> 32#include <linux/ftrace.h> 33#include <linux/module.h> 34#include <linux/percpu.h> 35#include <linux/splice.h> 36#include <linux/kdebug.h> 37#include <linux/string.h> 38#include <linux/mount.h> 39#include <linux/rwsem.h> 40#include <linux/slab.h> 41#include <linux/ctype.h> 42#include <linux/init.h> 43#include <linux/panic_notifier.h> 44#include <linux/poll.h> 45#include <linux/nmi.h> 46#include <linux/fs.h> 47#include <linux/trace.h> 48#include <linux/sched/clock.h> 49#include <linux/sched/rt.h> 50#include <linux/fsnotify.h> 51#include <linux/irq_work.h> 52#include <linux/workqueue.h> 53#include <linux/sort.h> 54#include <linux/io.h> /* vmap_page_range() */ 55#include <linux/fs_context.h> 56 57#include <asm/setup.h> /* COMMAND_LINE_SIZE */ 58 59#include "trace.h" 60#include "trace_output.h" 61 62#ifdef CONFIG_FTRACE_STARTUP_TEST 63/* 64 * We need to change this state when a selftest is running. 65 * A selftest will lurk into the ring-buffer to count the 66 * entries inserted during the selftest although some concurrent 67 * insertions into the ring-buffer such as trace_printk could occurred 68 * at the same time, giving false positive or negative results. 69 */ 70static bool __read_mostly tracing_selftest_running; 71 72/* 73 * If boot-time tracing including tracers/events via kernel cmdline 74 * is running, we do not want to run SELFTEST. 75 */ 76bool __read_mostly tracing_selftest_disabled; 77 78void __init disable_tracing_selftest(const char *reason) 79{ 80 if (!tracing_selftest_disabled) { 81 tracing_selftest_disabled = true; 82 pr_info("Ftrace startup test is disabled due to %s\n", reason); 83 } 84} 85#else 86#define tracing_selftest_running 0 87#define tracing_selftest_disabled 0 88#endif 89 90/* Pipe tracepoints to printk */ 91static struct trace_iterator *tracepoint_print_iter; 92int tracepoint_printk; 93static bool tracepoint_printk_stop_on_boot __initdata; 94static bool traceoff_after_boot __initdata; 95static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key); 96 97/* Store tracers and their flags per instance */ 98struct tracers { 99 struct list_head list; 100 struct tracer *tracer; 101 struct tracer_flags *flags; 102}; 103 104/* 105 * To prevent the comm cache from being overwritten when no 106 * tracing is active, only save the comm when a trace event 107 * occurred. 108 */ 109DEFINE_PER_CPU(bool, trace_taskinfo_save); 110 111/* 112 * Kill all tracing for good (never come back). 113 * It is initialized to 1 but will turn to zero if the initialization 114 * of the tracer is successful. But that is the only place that sets 115 * this back to zero. 116 */ 117static int tracing_disabled = 1; 118 119cpumask_var_t __read_mostly tracing_buffer_mask; 120 121#define MAX_TRACER_SIZE 100 122/* 123 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 124 * 125 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 126 * is set, then ftrace_dump is called. This will output the contents 127 * of the ftrace buffers to the console. This is very useful for 128 * capturing traces that lead to crashes and outputting it to a 129 * serial console. 130 * 131 * It is default off, but you can enable it with either specifying 132 * "ftrace_dump_on_oops" in the kernel command line, or setting 133 * /proc/sys/kernel/ftrace_dump_on_oops 134 * Set 1 if you want to dump buffers of all CPUs 135 * Set 2 if you want to dump the buffer of the CPU that triggered oops 136 * Set instance name if you want to dump the specific trace instance 137 * Multiple instance dump is also supported, and instances are separated 138 * by commas. 139 */ 140/* Set to string format zero to disable by default */ 141static char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0"; 142 143/* When set, tracing will stop when a WARN*() is hit */ 144static int __disable_trace_on_warning; 145 146int tracepoint_printk_sysctl(const struct ctl_table *table, int write, 147 void *buffer, size_t *lenp, loff_t *ppos); 148static const struct ctl_table trace_sysctl_table[] = { 149 { 150 .procname = "ftrace_dump_on_oops", 151 .data = &ftrace_dump_on_oops, 152 .maxlen = MAX_TRACER_SIZE, 153 .mode = 0644, 154 .proc_handler = proc_dostring, 155 }, 156 { 157 .procname = "traceoff_on_warning", 158 .data = &__disable_trace_on_warning, 159 .maxlen = sizeof(__disable_trace_on_warning), 160 .mode = 0644, 161 .proc_handler = proc_dointvec, 162 }, 163 { 164 .procname = "tracepoint_printk", 165 .data = &tracepoint_printk, 166 .maxlen = sizeof(tracepoint_printk), 167 .mode = 0644, 168 .proc_handler = tracepoint_printk_sysctl, 169 }, 170}; 171 172static int __init init_trace_sysctls(void) 173{ 174 register_sysctl_init("kernel", trace_sysctl_table); 175 return 0; 176} 177subsys_initcall(init_trace_sysctls); 178 179#ifdef CONFIG_TRACE_EVAL_MAP_FILE 180/* Map of enums to their values, for "eval_map" file */ 181struct trace_eval_map_head { 182 struct module *mod; 183 unsigned long length; 184}; 185 186union trace_eval_map_item; 187 188struct trace_eval_map_tail { 189 /* 190 * "end" is first and points to NULL as it must be different 191 * than "mod" or "eval_string" 192 */ 193 union trace_eval_map_item *next; 194 const char *end; /* points to NULL */ 195}; 196 197static DEFINE_MUTEX(trace_eval_mutex); 198 199/* 200 * The trace_eval_maps are saved in an array with two extra elements, 201 * one at the beginning, and one at the end. The beginning item contains 202 * the count of the saved maps (head.length), and the module they 203 * belong to if not built in (head.mod). The ending item contains a 204 * pointer to the next array of saved eval_map items. 205 */ 206union trace_eval_map_item { 207 struct trace_eval_map map; 208 struct trace_eval_map_head head; 209 struct trace_eval_map_tail tail; 210}; 211 212static union trace_eval_map_item *trace_eval_maps; 213#endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 214 215int tracing_set_tracer(struct trace_array *tr, const char *buf); 216static void ftrace_trace_userstack(struct trace_array *tr, 217 struct trace_buffer *buffer, 218 unsigned int trace_ctx); 219 220static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 221static char *default_bootup_tracer; 222 223static bool allocate_snapshot; 224static bool snapshot_at_boot; 225 226static char boot_instance_info[COMMAND_LINE_SIZE] __initdata; 227static int boot_instance_index; 228 229static char boot_snapshot_info[COMMAND_LINE_SIZE] __initdata; 230static int boot_snapshot_index; 231 232static int __init set_cmdline_ftrace(char *str) 233{ 234 strscpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 235 default_bootup_tracer = bootup_tracer_buf; 236 /* We are using ftrace early, expand it */ 237 trace_set_ring_buffer_expanded(NULL); 238 return 1; 239} 240__setup("ftrace=", set_cmdline_ftrace); 241 242int ftrace_dump_on_oops_enabled(void) 243{ 244 if (!strcmp("0", ftrace_dump_on_oops)) 245 return 0; 246 else 247 return 1; 248} 249 250static int __init set_ftrace_dump_on_oops(char *str) 251{ 252 if (!*str) { 253 strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE); 254 return 1; 255 } 256 257 if (*str == ',') { 258 strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE); 259 strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1); 260 return 1; 261 } 262 263 if (*str++ == '=') { 264 strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE); 265 return 1; 266 } 267 268 return 0; 269} 270__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 271 272static int __init stop_trace_on_warning(char *str) 273{ 274 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 275 __disable_trace_on_warning = 1; 276 return 1; 277} 278__setup("traceoff_on_warning", stop_trace_on_warning); 279 280static int __init boot_alloc_snapshot(char *str) 281{ 282 char *slot = boot_snapshot_info + boot_snapshot_index; 283 int left = sizeof(boot_snapshot_info) - boot_snapshot_index; 284 int ret; 285 286 if (str[0] == '=') { 287 str++; 288 if (strlen(str) >= left) 289 return -1; 290 291 ret = snprintf(slot, left, "%s\t", str); 292 boot_snapshot_index += ret; 293 } else { 294 allocate_snapshot = true; 295 /* We also need the main ring buffer expanded */ 296 trace_set_ring_buffer_expanded(NULL); 297 } 298 return 1; 299} 300__setup("alloc_snapshot", boot_alloc_snapshot); 301 302 303static int __init boot_snapshot(char *str) 304{ 305 snapshot_at_boot = true; 306 boot_alloc_snapshot(str); 307 return 1; 308} 309__setup("ftrace_boot_snapshot", boot_snapshot); 310 311 312static int __init boot_instance(char *str) 313{ 314 char *slot = boot_instance_info + boot_instance_index; 315 int left = sizeof(boot_instance_info) - boot_instance_index; 316 int ret; 317 318 if (strlen(str) >= left) 319 return -1; 320 321 ret = snprintf(slot, left, "%s\t", str); 322 boot_instance_index += ret; 323 324 return 1; 325} 326__setup("trace_instance=", boot_instance); 327 328 329static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata; 330 331static int __init set_trace_boot_options(char *str) 332{ 333 strscpy(trace_boot_options_buf, str, MAX_TRACER_SIZE); 334 return 1; 335} 336__setup("trace_options=", set_trace_boot_options); 337 338static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata; 339static char *trace_boot_clock __initdata; 340 341static int __init set_trace_boot_clock(char *str) 342{ 343 strscpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE); 344 trace_boot_clock = trace_boot_clock_buf; 345 return 1; 346} 347__setup("trace_clock=", set_trace_boot_clock); 348 349static int __init set_tracepoint_printk(char *str) 350{ 351 /* Ignore the "tp_printk_stop_on_boot" param */ 352 if (*str == '_') 353 return 0; 354 355 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0)) 356 tracepoint_printk = 1; 357 return 1; 358} 359__setup("tp_printk", set_tracepoint_printk); 360 361static int __init set_tracepoint_printk_stop(char *str) 362{ 363 tracepoint_printk_stop_on_boot = true; 364 return 1; 365} 366__setup("tp_printk_stop_on_boot", set_tracepoint_printk_stop); 367 368static int __init set_traceoff_after_boot(char *str) 369{ 370 traceoff_after_boot = true; 371 return 1; 372} 373__setup("traceoff_after_boot", set_traceoff_after_boot); 374 375unsigned long long ns2usecs(u64 nsec) 376{ 377 nsec += 500; 378 do_div(nsec, 1000); 379 return nsec; 380} 381 382static void 383trace_process_export(struct trace_export *export, 384 struct ring_buffer_event *event, int flag) 385{ 386 struct trace_entry *entry; 387 unsigned int size = 0; 388 389 if (export->flags & flag) { 390 entry = ring_buffer_event_data(event); 391 size = ring_buffer_event_length(event); 392 export->write(export, entry, size); 393 } 394} 395 396static DEFINE_MUTEX(ftrace_export_lock); 397 398static struct trace_export __rcu *ftrace_exports_list __read_mostly; 399 400static DEFINE_STATIC_KEY_FALSE(trace_function_exports_enabled); 401static DEFINE_STATIC_KEY_FALSE(trace_event_exports_enabled); 402static DEFINE_STATIC_KEY_FALSE(trace_marker_exports_enabled); 403 404static inline void ftrace_exports_enable(struct trace_export *export) 405{ 406 if (export->flags & TRACE_EXPORT_FUNCTION) 407 static_branch_inc(&trace_function_exports_enabled); 408 409 if (export->flags & TRACE_EXPORT_EVENT) 410 static_branch_inc(&trace_event_exports_enabled); 411 412 if (export->flags & TRACE_EXPORT_MARKER) 413 static_branch_inc(&trace_marker_exports_enabled); 414} 415 416static inline void ftrace_exports_disable(struct trace_export *export) 417{ 418 if (export->flags & TRACE_EXPORT_FUNCTION) 419 static_branch_dec(&trace_function_exports_enabled); 420 421 if (export->flags & TRACE_EXPORT_EVENT) 422 static_branch_dec(&trace_event_exports_enabled); 423 424 if (export->flags & TRACE_EXPORT_MARKER) 425 static_branch_dec(&trace_marker_exports_enabled); 426} 427 428static void ftrace_exports(struct ring_buffer_event *event, int flag) 429{ 430 struct trace_export *export; 431 432 guard(preempt_notrace)(); 433 434 export = rcu_dereference_raw_check(ftrace_exports_list); 435 while (export) { 436 trace_process_export(export, event, flag); 437 export = rcu_dereference_raw_check(export->next); 438 } 439} 440 441static inline void 442add_trace_export(struct trace_export **list, struct trace_export *export) 443{ 444 rcu_assign_pointer(export->next, *list); 445 /* 446 * We are entering export into the list but another 447 * CPU might be walking that list. We need to make sure 448 * the export->next pointer is valid before another CPU sees 449 * the export pointer included into the list. 450 */ 451 rcu_assign_pointer(*list, export); 452} 453 454static inline int 455rm_trace_export(struct trace_export **list, struct trace_export *export) 456{ 457 struct trace_export **p; 458 459 for (p = list; *p != NULL; p = &(*p)->next) 460 if (*p == export) 461 break; 462 463 if (*p != export) 464 return -1; 465 466 rcu_assign_pointer(*p, (*p)->next); 467 468 return 0; 469} 470 471static inline void 472add_ftrace_export(struct trace_export **list, struct trace_export *export) 473{ 474 ftrace_exports_enable(export); 475 476 add_trace_export(list, export); 477} 478 479static inline int 480rm_ftrace_export(struct trace_export **list, struct trace_export *export) 481{ 482 int ret; 483 484 ret = rm_trace_export(list, export); 485 ftrace_exports_disable(export); 486 487 return ret; 488} 489 490int register_ftrace_export(struct trace_export *export) 491{ 492 if (WARN_ON_ONCE(!export->write)) 493 return -1; 494 495 guard(mutex)(&ftrace_export_lock); 496 497 add_ftrace_export(&ftrace_exports_list, export); 498 499 return 0; 500} 501EXPORT_SYMBOL_GPL(register_ftrace_export); 502 503int unregister_ftrace_export(struct trace_export *export) 504{ 505 guard(mutex)(&ftrace_export_lock); 506 return rm_ftrace_export(&ftrace_exports_list, export); 507} 508EXPORT_SYMBOL_GPL(unregister_ftrace_export); 509 510/* trace_flags holds trace_options default values */ 511#define TRACE_DEFAULT_FLAGS \ 512 (FUNCTION_DEFAULT_FLAGS | FPROFILE_DEFAULT_FLAGS | \ 513 TRACE_ITER(PRINT_PARENT) | TRACE_ITER(PRINTK) | \ 514 TRACE_ITER(ANNOTATE) | TRACE_ITER(CONTEXT_INFO) | \ 515 TRACE_ITER(RECORD_CMD) | TRACE_ITER(OVERWRITE) | \ 516 TRACE_ITER(IRQ_INFO) | TRACE_ITER(MARKERS) | \ 517 TRACE_ITER(HASH_PTR) | TRACE_ITER(TRACE_PRINTK) | \ 518 TRACE_ITER(COPY_MARKER)) 519 520/* trace_options that are only supported by global_trace */ 521#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER(PRINTK) | \ 522 TRACE_ITER(PRINTK_MSGONLY) | TRACE_ITER(RECORD_CMD) | \ 523 TRACE_ITER(PROF_TEXT_OFFSET) | FPROFILE_DEFAULT_FLAGS) 524 525/* trace_flags that are default zero for instances */ 526#define ZEROED_TRACE_FLAGS \ 527 (TRACE_ITER(EVENT_FORK) | TRACE_ITER(FUNC_FORK) | TRACE_ITER(TRACE_PRINTK) | \ 528 TRACE_ITER(COPY_MARKER)) 529 530/* 531 * The global_trace is the descriptor that holds the top-level tracing 532 * buffers for the live tracing. 533 */ 534static struct trace_array global_trace = { 535 .trace_flags = TRACE_DEFAULT_FLAGS, 536}; 537 538static struct trace_array *printk_trace = &global_trace; 539 540/* List of trace_arrays interested in the top level trace_marker */ 541static LIST_HEAD(marker_copies); 542 543static __always_inline bool printk_binsafe(struct trace_array *tr) 544{ 545 /* 546 * The binary format of traceprintk can cause a crash if used 547 * by a buffer from another boot. Force the use of the 548 * non binary version of trace_printk if the trace_printk 549 * buffer is a boot mapped ring buffer. 550 */ 551 return !(tr->flags & TRACE_ARRAY_FL_BOOT); 552} 553 554static void update_printk_trace(struct trace_array *tr) 555{ 556 if (printk_trace == tr) 557 return; 558 559 printk_trace->trace_flags &= ~TRACE_ITER(TRACE_PRINTK); 560 printk_trace = tr; 561 tr->trace_flags |= TRACE_ITER(TRACE_PRINTK); 562} 563 564/* Returns true if the status of tr changed */ 565static bool update_marker_trace(struct trace_array *tr, int enabled) 566{ 567 lockdep_assert_held(&event_mutex); 568 569 if (enabled) { 570 if (tr->trace_flags & TRACE_ITER(COPY_MARKER)) 571 return false; 572 573 list_add_rcu(&tr->marker_list, &marker_copies); 574 tr->trace_flags |= TRACE_ITER(COPY_MARKER); 575 return true; 576 } 577 578 if (!(tr->trace_flags & TRACE_ITER(COPY_MARKER))) 579 return false; 580 581 list_del_rcu(&tr->marker_list); 582 tr->trace_flags &= ~TRACE_ITER(COPY_MARKER); 583 return true; 584} 585 586void trace_set_ring_buffer_expanded(struct trace_array *tr) 587{ 588 if (!tr) 589 tr = &global_trace; 590 tr->ring_buffer_expanded = true; 591} 592 593LIST_HEAD(ftrace_trace_arrays); 594 595int trace_array_get(struct trace_array *this_tr) 596{ 597 struct trace_array *tr; 598 599 guard(mutex)(&trace_types_lock); 600 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 601 if (tr == this_tr) { 602 tr->ref++; 603 return 0; 604 } 605 } 606 607 return -ENODEV; 608} 609 610static void __trace_array_put(struct trace_array *this_tr) 611{ 612 WARN_ON(!this_tr->ref); 613 this_tr->ref--; 614} 615 616/** 617 * trace_array_put - Decrement the reference counter for this trace array. 618 * @this_tr : pointer to the trace array 619 * 620 * NOTE: Use this when we no longer need the trace array returned by 621 * trace_array_get_by_name(). This ensures the trace array can be later 622 * destroyed. 623 * 624 */ 625void trace_array_put(struct trace_array *this_tr) 626{ 627 if (!this_tr) 628 return; 629 630 guard(mutex)(&trace_types_lock); 631 __trace_array_put(this_tr); 632} 633EXPORT_SYMBOL_GPL(trace_array_put); 634 635int tracing_check_open_get_tr(struct trace_array *tr) 636{ 637 int ret; 638 639 ret = security_locked_down(LOCKDOWN_TRACEFS); 640 if (ret) 641 return ret; 642 643 if (tracing_disabled) 644 return -ENODEV; 645 646 if (tr && trace_array_get(tr) < 0) 647 return -ENODEV; 648 649 return 0; 650} 651 652/** 653 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list 654 * @filtered_pids: The list of pids to check 655 * @search_pid: The PID to find in @filtered_pids 656 * 657 * Returns true if @search_pid is found in @filtered_pids, and false otherwise. 658 */ 659bool 660trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid) 661{ 662 return trace_pid_list_is_set(filtered_pids, search_pid); 663} 664 665/** 666 * trace_ignore_this_task - should a task be ignored for tracing 667 * @filtered_pids: The list of pids to check 668 * @filtered_no_pids: The list of pids not to be traced 669 * @task: The task that should be ignored if not filtered 670 * 671 * Checks if @task should be traced or not from @filtered_pids. 672 * Returns true if @task should *NOT* be traced. 673 * Returns false if @task should be traced. 674 */ 675bool 676trace_ignore_this_task(struct trace_pid_list *filtered_pids, 677 struct trace_pid_list *filtered_no_pids, 678 struct task_struct *task) 679{ 680 /* 681 * If filtered_no_pids is not empty, and the task's pid is listed 682 * in filtered_no_pids, then return true. 683 * Otherwise, if filtered_pids is empty, that means we can 684 * trace all tasks. If it has content, then only trace pids 685 * within filtered_pids. 686 */ 687 688 return (filtered_pids && 689 !trace_find_filtered_pid(filtered_pids, task->pid)) || 690 (filtered_no_pids && 691 trace_find_filtered_pid(filtered_no_pids, task->pid)); 692} 693 694/** 695 * trace_filter_add_remove_task - Add or remove a task from a pid_list 696 * @pid_list: The list to modify 697 * @self: The current task for fork or NULL for exit 698 * @task: The task to add or remove 699 * 700 * If adding a task, if @self is defined, the task is only added if @self 701 * is also included in @pid_list. This happens on fork and tasks should 702 * only be added when the parent is listed. If @self is NULL, then the 703 * @task pid will be removed from the list, which would happen on exit 704 * of a task. 705 */ 706void trace_filter_add_remove_task(struct trace_pid_list *pid_list, 707 struct task_struct *self, 708 struct task_struct *task) 709{ 710 if (!pid_list) 711 return; 712 713 /* For forks, we only add if the forking task is listed */ 714 if (self) { 715 if (!trace_find_filtered_pid(pid_list, self->pid)) 716 return; 717 } 718 719 /* "self" is set for forks, and NULL for exits */ 720 if (self) 721 trace_pid_list_set(pid_list, task->pid); 722 else 723 trace_pid_list_clear(pid_list, task->pid); 724} 725 726/** 727 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list 728 * @pid_list: The pid list to show 729 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed) 730 * @pos: The position of the file 731 * 732 * This is used by the seq_file "next" operation to iterate the pids 733 * listed in a trace_pid_list structure. 734 * 735 * Returns the pid+1 as we want to display pid of zero, but NULL would 736 * stop the iteration. 737 */ 738void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos) 739{ 740 long pid = (unsigned long)v; 741 unsigned int next; 742 743 (*pos)++; 744 745 /* pid already is +1 of the actual previous bit */ 746 if (trace_pid_list_next(pid_list, pid, &next) < 0) 747 return NULL; 748 749 pid = next; 750 751 /* Return pid + 1 to allow zero to be represented */ 752 return (void *)(pid + 1); 753} 754 755/** 756 * trace_pid_start - Used for seq_file to start reading pid lists 757 * @pid_list: The pid list to show 758 * @pos: The position of the file 759 * 760 * This is used by seq_file "start" operation to start the iteration 761 * of listing pids. 762 * 763 * Returns the pid+1 as we want to display pid of zero, but NULL would 764 * stop the iteration. 765 */ 766void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos) 767{ 768 unsigned long pid; 769 unsigned int first; 770 loff_t l = 0; 771 772 if (trace_pid_list_first(pid_list, &first) < 0) 773 return NULL; 774 775 pid = first; 776 777 /* Return pid + 1 so that zero can be the exit value */ 778 for (pid++; pid && l < *pos; 779 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l)) 780 ; 781 return (void *)pid; 782} 783 784/** 785 * trace_pid_show - show the current pid in seq_file processing 786 * @m: The seq_file structure to write into 787 * @v: A void pointer of the pid (+1) value to display 788 * 789 * Can be directly used by seq_file operations to display the current 790 * pid value. 791 */ 792int trace_pid_show(struct seq_file *m, void *v) 793{ 794 unsigned long pid = (unsigned long)v - 1; 795 796 seq_printf(m, "%lu\n", pid); 797 return 0; 798} 799 800/* 128 should be much more than enough */ 801#define PID_BUF_SIZE 127 802 803int trace_pid_write(struct trace_pid_list *filtered_pids, 804 struct trace_pid_list **new_pid_list, 805 const char __user *ubuf, size_t cnt) 806{ 807 struct trace_pid_list *pid_list; 808 struct trace_parser parser; 809 unsigned long val; 810 int nr_pids = 0; 811 ssize_t read = 0; 812 ssize_t ret; 813 loff_t pos; 814 pid_t pid; 815 816 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1)) 817 return -ENOMEM; 818 819 /* 820 * Always recreate a new array. The write is an all or nothing 821 * operation. Always create a new array when adding new pids by 822 * the user. If the operation fails, then the current list is 823 * not modified. 824 */ 825 pid_list = trace_pid_list_alloc(); 826 if (!pid_list) { 827 trace_parser_put(&parser); 828 return -ENOMEM; 829 } 830 831 if (filtered_pids) { 832 /* copy the current bits to the new max */ 833 ret = trace_pid_list_first(filtered_pids, &pid); 834 while (!ret) { 835 ret = trace_pid_list_set(pid_list, pid); 836 if (ret < 0) 837 goto out; 838 839 ret = trace_pid_list_next(filtered_pids, pid + 1, &pid); 840 nr_pids++; 841 } 842 } 843 844 ret = 0; 845 while (cnt > 0) { 846 847 pos = 0; 848 849 ret = trace_get_user(&parser, ubuf, cnt, &pos); 850 if (ret < 0) 851 break; 852 853 read += ret; 854 ubuf += ret; 855 cnt -= ret; 856 857 if (!trace_parser_loaded(&parser)) 858 break; 859 860 ret = -EINVAL; 861 if (kstrtoul(parser.buffer, 0, &val)) 862 break; 863 864 pid = (pid_t)val; 865 866 if (trace_pid_list_set(pid_list, pid) < 0) { 867 ret = -1; 868 break; 869 } 870 nr_pids++; 871 872 trace_parser_clear(&parser); 873 ret = 0; 874 } 875 out: 876 trace_parser_put(&parser); 877 878 if (ret < 0) { 879 trace_pid_list_free(pid_list); 880 return ret; 881 } 882 883 if (!nr_pids) { 884 /* Cleared the list of pids */ 885 trace_pid_list_free(pid_list); 886 pid_list = NULL; 887 } 888 889 *new_pid_list = pid_list; 890 891 return read; 892} 893 894static u64 buffer_ftrace_now(struct array_buffer *buf, int cpu) 895{ 896 u64 ts; 897 898 /* Early boot up does not have a buffer yet */ 899 if (!buf->buffer) 900 return trace_clock_local(); 901 902 ts = ring_buffer_time_stamp(buf->buffer); 903 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts); 904 905 return ts; 906} 907 908u64 ftrace_now(int cpu) 909{ 910 return buffer_ftrace_now(&global_trace.array_buffer, cpu); 911} 912 913/** 914 * tracing_is_enabled - Show if global_trace has been enabled 915 * 916 * Shows if the global trace has been enabled or not. It uses the 917 * mirror flag "buffer_disabled" to be used in fast paths such as for 918 * the irqsoff tracer. But it may be inaccurate due to races. If you 919 * need to know the accurate state, use tracing_is_on() which is a little 920 * slower, but accurate. 921 */ 922int tracing_is_enabled(void) 923{ 924 /* 925 * For quick access (irqsoff uses this in fast path), just 926 * return the mirror variable of the state of the ring buffer. 927 * It's a little racy, but we don't really care. 928 */ 929 return !global_trace.buffer_disabled; 930} 931 932/* 933 * trace_buf_size is the size in bytes that is allocated 934 * for a buffer. Note, the number of bytes is always rounded 935 * to page size. 936 * 937 * This number is purposely set to a low number of 16384. 938 * If the dump on oops happens, it will be much appreciated 939 * to not have to wait for all that output. Anyway this can be 940 * boot time and run time configurable. 941 */ 942#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 943 944static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 945 946/* trace_types holds a link list of available tracers. */ 947static struct tracer *trace_types __read_mostly; 948 949/* 950 * trace_types_lock is used to protect the trace_types list. 951 */ 952DEFINE_MUTEX(trace_types_lock); 953 954/* 955 * serialize the access of the ring buffer 956 * 957 * ring buffer serializes readers, but it is low level protection. 958 * The validity of the events (which returns by ring_buffer_peek() ..etc) 959 * are not protected by ring buffer. 960 * 961 * The content of events may become garbage if we allow other process consumes 962 * these events concurrently: 963 * A) the page of the consumed events may become a normal page 964 * (not reader page) in ring buffer, and this page will be rewritten 965 * by events producer. 966 * B) The page of the consumed events may become a page for splice_read, 967 * and this page will be returned to system. 968 * 969 * These primitives allow multi process access to different cpu ring buffer 970 * concurrently. 971 * 972 * These primitives don't distinguish read-only and read-consume access. 973 * Multi read-only access are also serialized. 974 */ 975 976#ifdef CONFIG_SMP 977static DECLARE_RWSEM(all_cpu_access_lock); 978static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 979 980static inline void trace_access_lock(int cpu) 981{ 982 if (cpu == RING_BUFFER_ALL_CPUS) { 983 /* gain it for accessing the whole ring buffer. */ 984 down_write(&all_cpu_access_lock); 985 } else { 986 /* gain it for accessing a cpu ring buffer. */ 987 988 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */ 989 down_read(&all_cpu_access_lock); 990 991 /* Secondly block other access to this @cpu ring buffer. */ 992 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 993 } 994} 995 996static inline void trace_access_unlock(int cpu) 997{ 998 if (cpu == RING_BUFFER_ALL_CPUS) { 999 up_write(&all_cpu_access_lock); 1000 } else { 1001 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 1002 up_read(&all_cpu_access_lock); 1003 } 1004} 1005 1006static inline void trace_access_lock_init(void) 1007{ 1008 int cpu; 1009 1010 for_each_possible_cpu(cpu) 1011 mutex_init(&per_cpu(cpu_access_lock, cpu)); 1012} 1013 1014#else 1015 1016static DEFINE_MUTEX(access_lock); 1017 1018static inline void trace_access_lock(int cpu) 1019{ 1020 (void)cpu; 1021 mutex_lock(&access_lock); 1022} 1023 1024static inline void trace_access_unlock(int cpu) 1025{ 1026 (void)cpu; 1027 mutex_unlock(&access_lock); 1028} 1029 1030static inline void trace_access_lock_init(void) 1031{ 1032} 1033 1034#endif 1035 1036#ifdef CONFIG_STACKTRACE 1037static void __ftrace_trace_stack(struct trace_array *tr, 1038 struct trace_buffer *buffer, 1039 unsigned int trace_ctx, 1040 int skip, struct pt_regs *regs); 1041static inline void ftrace_trace_stack(struct trace_array *tr, 1042 struct trace_buffer *buffer, 1043 unsigned int trace_ctx, 1044 int skip, struct pt_regs *regs); 1045 1046#else 1047static inline void __ftrace_trace_stack(struct trace_array *tr, 1048 struct trace_buffer *buffer, 1049 unsigned int trace_ctx, 1050 int skip, struct pt_regs *regs) 1051{ 1052} 1053static inline void ftrace_trace_stack(struct trace_array *tr, 1054 struct trace_buffer *buffer, 1055 unsigned long trace_ctx, 1056 int skip, struct pt_regs *regs) 1057{ 1058} 1059 1060#endif 1061 1062static __always_inline void 1063trace_event_setup(struct ring_buffer_event *event, 1064 int type, unsigned int trace_ctx) 1065{ 1066 struct trace_entry *ent = ring_buffer_event_data(event); 1067 1068 tracing_generic_entry_update(ent, type, trace_ctx); 1069} 1070 1071static __always_inline struct ring_buffer_event * 1072__trace_buffer_lock_reserve(struct trace_buffer *buffer, 1073 int type, 1074 unsigned long len, 1075 unsigned int trace_ctx) 1076{ 1077 struct ring_buffer_event *event; 1078 1079 event = ring_buffer_lock_reserve(buffer, len); 1080 if (event != NULL) 1081 trace_event_setup(event, type, trace_ctx); 1082 1083 return event; 1084} 1085 1086void tracer_tracing_on(struct trace_array *tr) 1087{ 1088 if (tr->array_buffer.buffer) 1089 ring_buffer_record_on(tr->array_buffer.buffer); 1090 /* 1091 * This flag is looked at when buffers haven't been allocated 1092 * yet, or by some tracers (like irqsoff), that just want to 1093 * know if the ring buffer has been disabled, but it can handle 1094 * races of where it gets disabled but we still do a record. 1095 * As the check is in the fast path of the tracers, it is more 1096 * important to be fast than accurate. 1097 */ 1098 tr->buffer_disabled = 0; 1099} 1100 1101/** 1102 * tracing_on - enable tracing buffers 1103 * 1104 * This function enables tracing buffers that may have been 1105 * disabled with tracing_off. 1106 */ 1107void tracing_on(void) 1108{ 1109 tracer_tracing_on(&global_trace); 1110} 1111EXPORT_SYMBOL_GPL(tracing_on); 1112 1113 1114static __always_inline void 1115__buffer_unlock_commit(struct trace_buffer *buffer, struct ring_buffer_event *event) 1116{ 1117 __this_cpu_write(trace_taskinfo_save, true); 1118 1119 /* If this is the temp buffer, we need to commit fully */ 1120 if (this_cpu_read(trace_buffered_event) == event) { 1121 /* Length is in event->array[0] */ 1122 ring_buffer_write(buffer, event->array[0], &event->array[1]); 1123 /* Release the temp buffer */ 1124 this_cpu_dec(trace_buffered_event_cnt); 1125 /* ring_buffer_unlock_commit() enables preemption */ 1126 preempt_enable_notrace(); 1127 } else 1128 ring_buffer_unlock_commit(buffer); 1129} 1130 1131int __trace_array_puts(struct trace_array *tr, unsigned long ip, 1132 const char *str, int size) 1133{ 1134 struct ring_buffer_event *event; 1135 struct trace_buffer *buffer; 1136 struct print_entry *entry; 1137 unsigned int trace_ctx; 1138 int alloc; 1139 1140 if (!(tr->trace_flags & TRACE_ITER(PRINTK))) 1141 return 0; 1142 1143 if (unlikely(tracing_selftest_running && tr == &global_trace)) 1144 return 0; 1145 1146 if (unlikely(tracing_disabled)) 1147 return 0; 1148 1149 alloc = sizeof(*entry) + size + 2; /* possible \n added */ 1150 1151 trace_ctx = tracing_gen_ctx(); 1152 buffer = tr->array_buffer.buffer; 1153 guard(ring_buffer_nest)(buffer); 1154 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 1155 trace_ctx); 1156 if (!event) 1157 return 0; 1158 1159 entry = ring_buffer_event_data(event); 1160 entry->ip = ip; 1161 1162 memcpy(&entry->buf, str, size); 1163 1164 /* Add a newline if necessary */ 1165 if (entry->buf[size - 1] != '\n') { 1166 entry->buf[size] = '\n'; 1167 entry->buf[size + 1] = '\0'; 1168 } else 1169 entry->buf[size] = '\0'; 1170 1171 __buffer_unlock_commit(buffer, event); 1172 ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL); 1173 return size; 1174} 1175EXPORT_SYMBOL_GPL(__trace_array_puts); 1176 1177/** 1178 * __trace_puts - write a constant string into the trace buffer. 1179 * @ip: The address of the caller 1180 * @str: The constant string to write 1181 * @size: The size of the string. 1182 */ 1183int __trace_puts(unsigned long ip, const char *str, int size) 1184{ 1185 return __trace_array_puts(printk_trace, ip, str, size); 1186} 1187EXPORT_SYMBOL_GPL(__trace_puts); 1188 1189/** 1190 * __trace_bputs - write the pointer to a constant string into trace buffer 1191 * @ip: The address of the caller 1192 * @str: The constant string to write to the buffer to 1193 */ 1194int __trace_bputs(unsigned long ip, const char *str) 1195{ 1196 struct trace_array *tr = READ_ONCE(printk_trace); 1197 struct ring_buffer_event *event; 1198 struct trace_buffer *buffer; 1199 struct bputs_entry *entry; 1200 unsigned int trace_ctx; 1201 int size = sizeof(struct bputs_entry); 1202 1203 if (!printk_binsafe(tr)) 1204 return __trace_puts(ip, str, strlen(str)); 1205 1206 if (!(tr->trace_flags & TRACE_ITER(PRINTK))) 1207 return 0; 1208 1209 if (unlikely(tracing_selftest_running || tracing_disabled)) 1210 return 0; 1211 1212 trace_ctx = tracing_gen_ctx(); 1213 buffer = tr->array_buffer.buffer; 1214 1215 guard(ring_buffer_nest)(buffer); 1216 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, 1217 trace_ctx); 1218 if (!event) 1219 return 0; 1220 1221 entry = ring_buffer_event_data(event); 1222 entry->ip = ip; 1223 entry->str = str; 1224 1225 __buffer_unlock_commit(buffer, event); 1226 ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL); 1227 1228 return 1; 1229} 1230EXPORT_SYMBOL_GPL(__trace_bputs); 1231 1232#ifdef CONFIG_TRACER_SNAPSHOT 1233static void tracing_snapshot_instance_cond(struct trace_array *tr, 1234 void *cond_data) 1235{ 1236 struct tracer *tracer = tr->current_trace; 1237 unsigned long flags; 1238 1239 if (in_nmi()) { 1240 trace_array_puts(tr, "*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); 1241 trace_array_puts(tr, "*** snapshot is being ignored ***\n"); 1242 return; 1243 } 1244 1245 if (!tr->allocated_snapshot) { 1246 trace_array_puts(tr, "*** SNAPSHOT NOT ALLOCATED ***\n"); 1247 trace_array_puts(tr, "*** stopping trace here! ***\n"); 1248 tracer_tracing_off(tr); 1249 return; 1250 } 1251 1252 /* Note, snapshot can not be used when the tracer uses it */ 1253 if (tracer->use_max_tr) { 1254 trace_array_puts(tr, "*** LATENCY TRACER ACTIVE ***\n"); 1255 trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); 1256 return; 1257 } 1258 1259 if (tr->mapped) { 1260 trace_array_puts(tr, "*** BUFFER MEMORY MAPPED ***\n"); 1261 trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); 1262 return; 1263 } 1264 1265 local_irq_save(flags); 1266 update_max_tr(tr, current, smp_processor_id(), cond_data); 1267 local_irq_restore(flags); 1268} 1269 1270void tracing_snapshot_instance(struct trace_array *tr) 1271{ 1272 tracing_snapshot_instance_cond(tr, NULL); 1273} 1274 1275/** 1276 * tracing_snapshot - take a snapshot of the current buffer. 1277 * 1278 * This causes a swap between the snapshot buffer and the current live 1279 * tracing buffer. You can use this to take snapshots of the live 1280 * trace when some condition is triggered, but continue to trace. 1281 * 1282 * Note, make sure to allocate the snapshot with either 1283 * a tracing_snapshot_alloc(), or by doing it manually 1284 * with: echo 1 > /sys/kernel/tracing/snapshot 1285 * 1286 * If the snapshot buffer is not allocated, it will stop tracing. 1287 * Basically making a permanent snapshot. 1288 */ 1289void tracing_snapshot(void) 1290{ 1291 struct trace_array *tr = &global_trace; 1292 1293 tracing_snapshot_instance(tr); 1294} 1295EXPORT_SYMBOL_GPL(tracing_snapshot); 1296 1297/** 1298 * tracing_snapshot_cond - conditionally take a snapshot of the current buffer. 1299 * @tr: The tracing instance to snapshot 1300 * @cond_data: The data to be tested conditionally, and possibly saved 1301 * 1302 * This is the same as tracing_snapshot() except that the snapshot is 1303 * conditional - the snapshot will only happen if the 1304 * cond_snapshot.update() implementation receiving the cond_data 1305 * returns true, which means that the trace array's cond_snapshot 1306 * update() operation used the cond_data to determine whether the 1307 * snapshot should be taken, and if it was, presumably saved it along 1308 * with the snapshot. 1309 */ 1310void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) 1311{ 1312 tracing_snapshot_instance_cond(tr, cond_data); 1313} 1314EXPORT_SYMBOL_GPL(tracing_snapshot_cond); 1315 1316/** 1317 * tracing_cond_snapshot_data - get the user data associated with a snapshot 1318 * @tr: The tracing instance 1319 * 1320 * When the user enables a conditional snapshot using 1321 * tracing_snapshot_cond_enable(), the user-defined cond_data is saved 1322 * with the snapshot. This accessor is used to retrieve it. 1323 * 1324 * Should not be called from cond_snapshot.update(), since it takes 1325 * the tr->max_lock lock, which the code calling 1326 * cond_snapshot.update() has already done. 1327 * 1328 * Returns the cond_data associated with the trace array's snapshot. 1329 */ 1330void *tracing_cond_snapshot_data(struct trace_array *tr) 1331{ 1332 void *cond_data = NULL; 1333 1334 local_irq_disable(); 1335 arch_spin_lock(&tr->max_lock); 1336 1337 if (tr->cond_snapshot) 1338 cond_data = tr->cond_snapshot->cond_data; 1339 1340 arch_spin_unlock(&tr->max_lock); 1341 local_irq_enable(); 1342 1343 return cond_data; 1344} 1345EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); 1346 1347static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, 1348 struct array_buffer *size_buf, int cpu_id); 1349static void set_buffer_entries(struct array_buffer *buf, unsigned long val); 1350 1351int tracing_alloc_snapshot_instance(struct trace_array *tr) 1352{ 1353 int order; 1354 int ret; 1355 1356 if (!tr->allocated_snapshot) { 1357 1358 /* Make the snapshot buffer have the same order as main buffer */ 1359 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 1360 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); 1361 if (ret < 0) 1362 return ret; 1363 1364 /* allocate spare buffer */ 1365 ret = resize_buffer_duplicate_size(&tr->max_buffer, 1366 &tr->array_buffer, RING_BUFFER_ALL_CPUS); 1367 if (ret < 0) 1368 return ret; 1369 1370 tr->allocated_snapshot = true; 1371 } 1372 1373 return 0; 1374} 1375 1376static void free_snapshot(struct trace_array *tr) 1377{ 1378 /* 1379 * We don't free the ring buffer. instead, resize it because 1380 * The max_tr ring buffer has some state (e.g. ring->clock) and 1381 * we want preserve it. 1382 */ 1383 ring_buffer_subbuf_order_set(tr->max_buffer.buffer, 0); 1384 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); 1385 set_buffer_entries(&tr->max_buffer, 1); 1386 tracing_reset_online_cpus(&tr->max_buffer); 1387 tr->allocated_snapshot = false; 1388} 1389 1390static int tracing_arm_snapshot_locked(struct trace_array *tr) 1391{ 1392 int ret; 1393 1394 lockdep_assert_held(&trace_types_lock); 1395 1396 spin_lock(&tr->snapshot_trigger_lock); 1397 if (tr->snapshot == UINT_MAX || tr->mapped) { 1398 spin_unlock(&tr->snapshot_trigger_lock); 1399 return -EBUSY; 1400 } 1401 1402 tr->snapshot++; 1403 spin_unlock(&tr->snapshot_trigger_lock); 1404 1405 ret = tracing_alloc_snapshot_instance(tr); 1406 if (ret) { 1407 spin_lock(&tr->snapshot_trigger_lock); 1408 tr->snapshot--; 1409 spin_unlock(&tr->snapshot_trigger_lock); 1410 } 1411 1412 return ret; 1413} 1414 1415int tracing_arm_snapshot(struct trace_array *tr) 1416{ 1417 guard(mutex)(&trace_types_lock); 1418 return tracing_arm_snapshot_locked(tr); 1419} 1420 1421void tracing_disarm_snapshot(struct trace_array *tr) 1422{ 1423 spin_lock(&tr->snapshot_trigger_lock); 1424 if (!WARN_ON(!tr->snapshot)) 1425 tr->snapshot--; 1426 spin_unlock(&tr->snapshot_trigger_lock); 1427} 1428 1429/** 1430 * tracing_alloc_snapshot - allocate snapshot buffer. 1431 * 1432 * This only allocates the snapshot buffer if it isn't already 1433 * allocated - it doesn't also take a snapshot. 1434 * 1435 * This is meant to be used in cases where the snapshot buffer needs 1436 * to be set up for events that can't sleep but need to be able to 1437 * trigger a snapshot. 1438 */ 1439int tracing_alloc_snapshot(void) 1440{ 1441 struct trace_array *tr = &global_trace; 1442 int ret; 1443 1444 ret = tracing_alloc_snapshot_instance(tr); 1445 WARN_ON(ret < 0); 1446 1447 return ret; 1448} 1449EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1450 1451/** 1452 * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer. 1453 * 1454 * This is similar to tracing_snapshot(), but it will allocate the 1455 * snapshot buffer if it isn't already allocated. Use this only 1456 * where it is safe to sleep, as the allocation may sleep. 1457 * 1458 * This causes a swap between the snapshot buffer and the current live 1459 * tracing buffer. You can use this to take snapshots of the live 1460 * trace when some condition is triggered, but continue to trace. 1461 */ 1462void tracing_snapshot_alloc(void) 1463{ 1464 int ret; 1465 1466 ret = tracing_alloc_snapshot(); 1467 if (ret < 0) 1468 return; 1469 1470 tracing_snapshot(); 1471} 1472EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1473 1474/** 1475 * tracing_snapshot_cond_enable - enable conditional snapshot for an instance 1476 * @tr: The tracing instance 1477 * @cond_data: User data to associate with the snapshot 1478 * @update: Implementation of the cond_snapshot update function 1479 * 1480 * Check whether the conditional snapshot for the given instance has 1481 * already been enabled, or if the current tracer is already using a 1482 * snapshot; if so, return -EBUSY, else create a cond_snapshot and 1483 * save the cond_data and update function inside. 1484 * 1485 * Returns 0 if successful, error otherwise. 1486 */ 1487int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, 1488 cond_update_fn_t update) 1489{ 1490 struct cond_snapshot *cond_snapshot __free(kfree) = 1491 kzalloc(sizeof(*cond_snapshot), GFP_KERNEL); 1492 int ret; 1493 1494 if (!cond_snapshot) 1495 return -ENOMEM; 1496 1497 cond_snapshot->cond_data = cond_data; 1498 cond_snapshot->update = update; 1499 1500 guard(mutex)(&trace_types_lock); 1501 1502 if (tr->current_trace->use_max_tr) 1503 return -EBUSY; 1504 1505 /* 1506 * The cond_snapshot can only change to NULL without the 1507 * trace_types_lock. We don't care if we race with it going 1508 * to NULL, but we want to make sure that it's not set to 1509 * something other than NULL when we get here, which we can 1510 * do safely with only holding the trace_types_lock and not 1511 * having to take the max_lock. 1512 */ 1513 if (tr->cond_snapshot) 1514 return -EBUSY; 1515 1516 ret = tracing_arm_snapshot_locked(tr); 1517 if (ret) 1518 return ret; 1519 1520 local_irq_disable(); 1521 arch_spin_lock(&tr->max_lock); 1522 tr->cond_snapshot = no_free_ptr(cond_snapshot); 1523 arch_spin_unlock(&tr->max_lock); 1524 local_irq_enable(); 1525 1526 return 0; 1527} 1528EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); 1529 1530/** 1531 * tracing_snapshot_cond_disable - disable conditional snapshot for an instance 1532 * @tr: The tracing instance 1533 * 1534 * Check whether the conditional snapshot for the given instance is 1535 * enabled; if so, free the cond_snapshot associated with it, 1536 * otherwise return -EINVAL. 1537 * 1538 * Returns 0 if successful, error otherwise. 1539 */ 1540int tracing_snapshot_cond_disable(struct trace_array *tr) 1541{ 1542 int ret = 0; 1543 1544 local_irq_disable(); 1545 arch_spin_lock(&tr->max_lock); 1546 1547 if (!tr->cond_snapshot) 1548 ret = -EINVAL; 1549 else { 1550 kfree(tr->cond_snapshot); 1551 tr->cond_snapshot = NULL; 1552 } 1553 1554 arch_spin_unlock(&tr->max_lock); 1555 local_irq_enable(); 1556 1557 tracing_disarm_snapshot(tr); 1558 1559 return ret; 1560} 1561EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); 1562#else 1563void tracing_snapshot(void) 1564{ 1565 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used"); 1566} 1567EXPORT_SYMBOL_GPL(tracing_snapshot); 1568void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) 1569{ 1570 WARN_ONCE(1, "Snapshot feature not enabled, but internal conditional snapshot used"); 1571} 1572EXPORT_SYMBOL_GPL(tracing_snapshot_cond); 1573int tracing_alloc_snapshot(void) 1574{ 1575 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used"); 1576 return -ENODEV; 1577} 1578EXPORT_SYMBOL_GPL(tracing_alloc_snapshot); 1579void tracing_snapshot_alloc(void) 1580{ 1581 /* Give warning */ 1582 tracing_snapshot(); 1583} 1584EXPORT_SYMBOL_GPL(tracing_snapshot_alloc); 1585void *tracing_cond_snapshot_data(struct trace_array *tr) 1586{ 1587 return NULL; 1588} 1589EXPORT_SYMBOL_GPL(tracing_cond_snapshot_data); 1590int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update) 1591{ 1592 return -ENODEV; 1593} 1594EXPORT_SYMBOL_GPL(tracing_snapshot_cond_enable); 1595int tracing_snapshot_cond_disable(struct trace_array *tr) 1596{ 1597 return false; 1598} 1599EXPORT_SYMBOL_GPL(tracing_snapshot_cond_disable); 1600#define free_snapshot(tr) do { } while (0) 1601#define tracing_arm_snapshot_locked(tr) ({ -EBUSY; }) 1602#endif /* CONFIG_TRACER_SNAPSHOT */ 1603 1604void tracer_tracing_off(struct trace_array *tr) 1605{ 1606 if (tr->array_buffer.buffer) 1607 ring_buffer_record_off(tr->array_buffer.buffer); 1608 /* 1609 * This flag is looked at when buffers haven't been allocated 1610 * yet, or by some tracers (like irqsoff), that just want to 1611 * know if the ring buffer has been disabled, but it can handle 1612 * races of where it gets disabled but we still do a record. 1613 * As the check is in the fast path of the tracers, it is more 1614 * important to be fast than accurate. 1615 */ 1616 tr->buffer_disabled = 1; 1617} 1618 1619/** 1620 * tracer_tracing_disable() - temporary disable the buffer from write 1621 * @tr: The trace array to disable its buffer for 1622 * 1623 * Expects trace_tracing_enable() to re-enable tracing. 1624 * The difference between this and tracer_tracing_off() is that this 1625 * is a counter and can nest, whereas, tracer_tracing_off() can 1626 * be called multiple times and a single trace_tracing_on() will 1627 * enable it. 1628 */ 1629void tracer_tracing_disable(struct trace_array *tr) 1630{ 1631 if (WARN_ON_ONCE(!tr->array_buffer.buffer)) 1632 return; 1633 1634 ring_buffer_record_disable(tr->array_buffer.buffer); 1635} 1636 1637/** 1638 * tracer_tracing_enable() - counter part of tracer_tracing_disable() 1639 * @tr: The trace array that had tracer_tracincg_disable() called on it 1640 * 1641 * This is called after tracer_tracing_disable() has been called on @tr, 1642 * when it's safe to re-enable tracing. 1643 */ 1644void tracer_tracing_enable(struct trace_array *tr) 1645{ 1646 if (WARN_ON_ONCE(!tr->array_buffer.buffer)) 1647 return; 1648 1649 ring_buffer_record_enable(tr->array_buffer.buffer); 1650} 1651 1652/** 1653 * tracing_off - turn off tracing buffers 1654 * 1655 * This function stops the tracing buffers from recording data. 1656 * It does not disable any overhead the tracers themselves may 1657 * be causing. This function simply causes all recording to 1658 * the ring buffers to fail. 1659 */ 1660void tracing_off(void) 1661{ 1662 tracer_tracing_off(&global_trace); 1663} 1664EXPORT_SYMBOL_GPL(tracing_off); 1665 1666void disable_trace_on_warning(void) 1667{ 1668 if (__disable_trace_on_warning) { 1669 trace_array_printk_buf(global_trace.array_buffer.buffer, _THIS_IP_, 1670 "Disabling tracing due to warning\n"); 1671 tracing_off(); 1672 } 1673} 1674 1675/** 1676 * tracer_tracing_is_on - show real state of ring buffer enabled 1677 * @tr : the trace array to know if ring buffer is enabled 1678 * 1679 * Shows real state of the ring buffer if it is enabled or not. 1680 */ 1681bool tracer_tracing_is_on(struct trace_array *tr) 1682{ 1683 if (tr->array_buffer.buffer) 1684 return ring_buffer_record_is_set_on(tr->array_buffer.buffer); 1685 return !tr->buffer_disabled; 1686} 1687 1688/** 1689 * tracing_is_on - show state of ring buffers enabled 1690 */ 1691int tracing_is_on(void) 1692{ 1693 return tracer_tracing_is_on(&global_trace); 1694} 1695EXPORT_SYMBOL_GPL(tracing_is_on); 1696 1697static int __init set_buf_size(char *str) 1698{ 1699 unsigned long buf_size; 1700 1701 if (!str) 1702 return 0; 1703 buf_size = memparse(str, &str); 1704 /* 1705 * nr_entries can not be zero and the startup 1706 * tests require some buffer space. Therefore 1707 * ensure we have at least 4096 bytes of buffer. 1708 */ 1709 trace_buf_size = max(4096UL, buf_size); 1710 return 1; 1711} 1712__setup("trace_buf_size=", set_buf_size); 1713 1714static int __init set_tracing_thresh(char *str) 1715{ 1716 unsigned long threshold; 1717 int ret; 1718 1719 if (!str) 1720 return 0; 1721 ret = kstrtoul(str, 0, &threshold); 1722 if (ret < 0) 1723 return 0; 1724 tracing_thresh = threshold * 1000; 1725 return 1; 1726} 1727__setup("tracing_thresh=", set_tracing_thresh); 1728 1729unsigned long nsecs_to_usecs(unsigned long nsecs) 1730{ 1731 return nsecs / 1000; 1732} 1733 1734/* 1735 * TRACE_FLAGS is defined as a tuple matching bit masks with strings. 1736 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that 1737 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list 1738 * of strings in the order that the evals (enum) were defined. 1739 */ 1740#undef C 1741#define C(a, b) b 1742 1743/* These must match the bit positions in trace_iterator_flags */ 1744static const char *trace_options[] = { 1745 TRACE_FLAGS 1746 NULL 1747}; 1748 1749static struct { 1750 u64 (*func)(void); 1751 const char *name; 1752 int in_ns; /* is this clock in nanoseconds? */ 1753} trace_clocks[] = { 1754 { trace_clock_local, "local", 1 }, 1755 { trace_clock_global, "global", 1 }, 1756 { trace_clock_counter, "counter", 0 }, 1757 { trace_clock_jiffies, "uptime", 0 }, 1758 { trace_clock, "perf", 1 }, 1759 { ktime_get_mono_fast_ns, "mono", 1 }, 1760 { ktime_get_raw_fast_ns, "mono_raw", 1 }, 1761 { ktime_get_boot_fast_ns, "boot", 1 }, 1762 { ktime_get_tai_fast_ns, "tai", 1 }, 1763 ARCH_TRACE_CLOCKS 1764}; 1765 1766bool trace_clock_in_ns(struct trace_array *tr) 1767{ 1768 if (trace_clocks[tr->clock_id].in_ns) 1769 return true; 1770 1771 return false; 1772} 1773 1774/* 1775 * trace_parser_get_init - gets the buffer for trace parser 1776 */ 1777int trace_parser_get_init(struct trace_parser *parser, int size) 1778{ 1779 memset(parser, 0, sizeof(*parser)); 1780 1781 parser->buffer = kmalloc(size, GFP_KERNEL); 1782 if (!parser->buffer) 1783 return 1; 1784 1785 parser->size = size; 1786 return 0; 1787} 1788 1789/* 1790 * trace_parser_put - frees the buffer for trace parser 1791 */ 1792void trace_parser_put(struct trace_parser *parser) 1793{ 1794 kfree(parser->buffer); 1795 parser->buffer = NULL; 1796} 1797 1798/* 1799 * trace_get_user - reads the user input string separated by space 1800 * (matched by isspace(ch)) 1801 * 1802 * For each string found the 'struct trace_parser' is updated, 1803 * and the function returns. 1804 * 1805 * Returns number of bytes read. 1806 * 1807 * See kernel/trace/trace.h for 'struct trace_parser' details. 1808 */ 1809int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 1810 size_t cnt, loff_t *ppos) 1811{ 1812 char ch; 1813 size_t read = 0; 1814 ssize_t ret; 1815 1816 if (!*ppos) 1817 trace_parser_clear(parser); 1818 1819 ret = get_user(ch, ubuf++); 1820 if (ret) 1821 goto fail; 1822 1823 read++; 1824 cnt--; 1825 1826 /* 1827 * The parser is not finished with the last write, 1828 * continue reading the user input without skipping spaces. 1829 */ 1830 if (!parser->cont) { 1831 /* skip white space */ 1832 while (cnt && isspace(ch)) { 1833 ret = get_user(ch, ubuf++); 1834 if (ret) 1835 goto fail; 1836 read++; 1837 cnt--; 1838 } 1839 1840 parser->idx = 0; 1841 1842 /* only spaces were written */ 1843 if (isspace(ch) || !ch) { 1844 *ppos += read; 1845 return read; 1846 } 1847 } 1848 1849 /* read the non-space input */ 1850 while (cnt && !isspace(ch) && ch) { 1851 if (parser->idx < parser->size - 1) 1852 parser->buffer[parser->idx++] = ch; 1853 else { 1854 ret = -EINVAL; 1855 goto fail; 1856 } 1857 1858 ret = get_user(ch, ubuf++); 1859 if (ret) 1860 goto fail; 1861 read++; 1862 cnt--; 1863 } 1864 1865 /* We either got finished input or we have to wait for another call. */ 1866 if (isspace(ch) || !ch) { 1867 parser->buffer[parser->idx] = 0; 1868 parser->cont = false; 1869 } else if (parser->idx < parser->size - 1) { 1870 parser->cont = true; 1871 parser->buffer[parser->idx++] = ch; 1872 /* Make sure the parsed string always terminates with '\0'. */ 1873 parser->buffer[parser->idx] = 0; 1874 } else { 1875 ret = -EINVAL; 1876 goto fail; 1877 } 1878 1879 *ppos += read; 1880 return read; 1881fail: 1882 trace_parser_fail(parser); 1883 return ret; 1884} 1885 1886/* TODO add a seq_buf_to_buffer() */ 1887static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 1888{ 1889 int len; 1890 1891 if (trace_seq_used(s) <= s->readpos) 1892 return -EBUSY; 1893 1894 len = trace_seq_used(s) - s->readpos; 1895 if (cnt > len) 1896 cnt = len; 1897 memcpy(buf, s->buffer + s->readpos, cnt); 1898 1899 s->readpos += cnt; 1900 return cnt; 1901} 1902 1903unsigned long __read_mostly tracing_thresh; 1904 1905#ifdef CONFIG_TRACER_MAX_TRACE 1906static const struct file_operations tracing_max_lat_fops; 1907 1908#ifdef LATENCY_FS_NOTIFY 1909 1910static struct workqueue_struct *fsnotify_wq; 1911 1912static void latency_fsnotify_workfn(struct work_struct *work) 1913{ 1914 struct trace_array *tr = container_of(work, struct trace_array, 1915 fsnotify_work); 1916 fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY); 1917} 1918 1919static void latency_fsnotify_workfn_irq(struct irq_work *iwork) 1920{ 1921 struct trace_array *tr = container_of(iwork, struct trace_array, 1922 fsnotify_irqwork); 1923 queue_work(fsnotify_wq, &tr->fsnotify_work); 1924} 1925 1926static void trace_create_maxlat_file(struct trace_array *tr, 1927 struct dentry *d_tracer) 1928{ 1929 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); 1930 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); 1931 tr->d_max_latency = trace_create_file("tracing_max_latency", 1932 TRACE_MODE_WRITE, 1933 d_tracer, tr, 1934 &tracing_max_lat_fops); 1935} 1936 1937__init static int latency_fsnotify_init(void) 1938{ 1939 fsnotify_wq = alloc_workqueue("tr_max_lat_wq", 1940 WQ_UNBOUND | WQ_HIGHPRI, 0); 1941 if (!fsnotify_wq) { 1942 pr_err("Unable to allocate tr_max_lat_wq\n"); 1943 return -ENOMEM; 1944 } 1945 return 0; 1946} 1947 1948late_initcall_sync(latency_fsnotify_init); 1949 1950void latency_fsnotify(struct trace_array *tr) 1951{ 1952 if (!fsnotify_wq) 1953 return; 1954 /* 1955 * We cannot call queue_work(&tr->fsnotify_work) from here because it's 1956 * possible that we are called from __schedule() or do_idle(), which 1957 * could cause a deadlock. 1958 */ 1959 irq_work_queue(&tr->fsnotify_irqwork); 1960} 1961 1962#else /* !LATENCY_FS_NOTIFY */ 1963 1964#define trace_create_maxlat_file(tr, d_tracer) \ 1965 trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \ 1966 d_tracer, tr, &tracing_max_lat_fops) 1967 1968#endif 1969 1970/* 1971 * Copy the new maximum trace into the separate maximum-trace 1972 * structure. (this way the maximum trace is permanently saved, 1973 * for later retrieval via /sys/kernel/tracing/tracing_max_latency) 1974 */ 1975static void 1976__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 1977{ 1978 struct array_buffer *trace_buf = &tr->array_buffer; 1979 struct array_buffer *max_buf = &tr->max_buffer; 1980 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu); 1981 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu); 1982 1983 max_buf->cpu = cpu; 1984 max_buf->time_start = data->preempt_timestamp; 1985 1986 max_data->saved_latency = tr->max_latency; 1987 max_data->critical_start = data->critical_start; 1988 max_data->critical_end = data->critical_end; 1989 1990 strscpy(max_data->comm, tsk->comm); 1991 max_data->pid = tsk->pid; 1992 /* 1993 * If tsk == current, then use current_uid(), as that does not use 1994 * RCU. The irq tracer can be called out of RCU scope. 1995 */ 1996 if (tsk == current) 1997 max_data->uid = current_uid(); 1998 else 1999 max_data->uid = task_uid(tsk); 2000 2001 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 2002 max_data->policy = tsk->policy; 2003 max_data->rt_priority = tsk->rt_priority; 2004 2005 /* record this tasks comm */ 2006 tracing_record_cmdline(tsk); 2007 latency_fsnotify(tr); 2008} 2009 2010/** 2011 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 2012 * @tr: tracer 2013 * @tsk: the task with the latency 2014 * @cpu: The cpu that initiated the trace. 2015 * @cond_data: User data associated with a conditional snapshot 2016 * 2017 * Flip the buffers between the @tr and the max_tr and record information 2018 * about which task was the cause of this latency. 2019 */ 2020void 2021update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu, 2022 void *cond_data) 2023{ 2024 if (tr->stop_count) 2025 return; 2026 2027 WARN_ON_ONCE(!irqs_disabled()); 2028 2029 if (!tr->allocated_snapshot) { 2030 /* Only the nop tracer should hit this when disabling */ 2031 WARN_ON_ONCE(tr->current_trace != &nop_trace); 2032 return; 2033 } 2034 2035 arch_spin_lock(&tr->max_lock); 2036 2037 /* Inherit the recordable setting from array_buffer */ 2038 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer)) 2039 ring_buffer_record_on(tr->max_buffer.buffer); 2040 else 2041 ring_buffer_record_off(tr->max_buffer.buffer); 2042 2043#ifdef CONFIG_TRACER_SNAPSHOT 2044 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) { 2045 arch_spin_unlock(&tr->max_lock); 2046 return; 2047 } 2048#endif 2049 swap(tr->array_buffer.buffer, tr->max_buffer.buffer); 2050 2051 __update_max_tr(tr, tsk, cpu); 2052 2053 arch_spin_unlock(&tr->max_lock); 2054 2055 /* Any waiters on the old snapshot buffer need to wake up */ 2056 ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS); 2057} 2058 2059/** 2060 * update_max_tr_single - only copy one trace over, and reset the rest 2061 * @tr: tracer 2062 * @tsk: task with the latency 2063 * @cpu: the cpu of the buffer to copy. 2064 * 2065 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 2066 */ 2067void 2068update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 2069{ 2070 int ret; 2071 2072 if (tr->stop_count) 2073 return; 2074 2075 WARN_ON_ONCE(!irqs_disabled()); 2076 if (!tr->allocated_snapshot) { 2077 /* Only the nop tracer should hit this when disabling */ 2078 WARN_ON_ONCE(tr->current_trace != &nop_trace); 2079 return; 2080 } 2081 2082 arch_spin_lock(&tr->max_lock); 2083 2084 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu); 2085 2086 if (ret == -EBUSY) { 2087 /* 2088 * We failed to swap the buffer due to a commit taking 2089 * place on this CPU. We fail to record, but we reset 2090 * the max trace buffer (no one writes directly to it) 2091 * and flag that it failed. 2092 * Another reason is resize is in progress. 2093 */ 2094 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, 2095 "Failed to swap buffers due to commit or resize in progress\n"); 2096 } 2097 2098 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 2099 2100 __update_max_tr(tr, tsk, cpu); 2101 arch_spin_unlock(&tr->max_lock); 2102} 2103 2104#endif /* CONFIG_TRACER_MAX_TRACE */ 2105 2106struct pipe_wait { 2107 struct trace_iterator *iter; 2108 int wait_index; 2109}; 2110 2111static bool wait_pipe_cond(void *data) 2112{ 2113 struct pipe_wait *pwait = data; 2114 struct trace_iterator *iter = pwait->iter; 2115 2116 if (atomic_read_acquire(&iter->wait_index) != pwait->wait_index) 2117 return true; 2118 2119 return iter->closed; 2120} 2121 2122static int wait_on_pipe(struct trace_iterator *iter, int full) 2123{ 2124 struct pipe_wait pwait; 2125 int ret; 2126 2127 /* Iterators are static, they should be filled or empty */ 2128 if (trace_buffer_iter(iter, iter->cpu_file)) 2129 return 0; 2130 2131 pwait.wait_index = atomic_read_acquire(&iter->wait_index); 2132 pwait.iter = iter; 2133 2134 ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full, 2135 wait_pipe_cond, &pwait); 2136 2137#ifdef CONFIG_TRACER_MAX_TRACE 2138 /* 2139 * Make sure this is still the snapshot buffer, as if a snapshot were 2140 * to happen, this would now be the main buffer. 2141 */ 2142 if (iter->snapshot) 2143 iter->array_buffer = &iter->tr->max_buffer; 2144#endif 2145 return ret; 2146} 2147 2148#ifdef CONFIG_FTRACE_STARTUP_TEST 2149static bool selftests_can_run; 2150 2151struct trace_selftests { 2152 struct list_head list; 2153 struct tracer *type; 2154}; 2155 2156static LIST_HEAD(postponed_selftests); 2157 2158static int save_selftest(struct tracer *type) 2159{ 2160 struct trace_selftests *selftest; 2161 2162 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL); 2163 if (!selftest) 2164 return -ENOMEM; 2165 2166 selftest->type = type; 2167 list_add(&selftest->list, &postponed_selftests); 2168 return 0; 2169} 2170 2171static int run_tracer_selftest(struct tracer *type) 2172{ 2173 struct trace_array *tr = &global_trace; 2174 struct tracer_flags *saved_flags = tr->current_trace_flags; 2175 struct tracer *saved_tracer = tr->current_trace; 2176 int ret; 2177 2178 if (!type->selftest || tracing_selftest_disabled) 2179 return 0; 2180 2181 /* 2182 * If a tracer registers early in boot up (before scheduling is 2183 * initialized and such), then do not run its selftests yet. 2184 * Instead, run it a little later in the boot process. 2185 */ 2186 if (!selftests_can_run) 2187 return save_selftest(type); 2188 2189 if (!tracing_is_on()) { 2190 pr_warn("Selftest for tracer %s skipped due to tracing disabled\n", 2191 type->name); 2192 return 0; 2193 } 2194 2195 /* 2196 * Run a selftest on this tracer. 2197 * Here we reset the trace buffer, and set the current 2198 * tracer to be this tracer. The tracer can then run some 2199 * internal tracing to verify that everything is in order. 2200 * If we fail, we do not register this tracer. 2201 */ 2202 tracing_reset_online_cpus(&tr->array_buffer); 2203 2204 tr->current_trace = type; 2205 tr->current_trace_flags = type->flags ? : type->default_flags; 2206 2207#ifdef CONFIG_TRACER_MAX_TRACE 2208 if (type->use_max_tr) { 2209 /* If we expanded the buffers, make sure the max is expanded too */ 2210 if (tr->ring_buffer_expanded) 2211 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, 2212 RING_BUFFER_ALL_CPUS); 2213 tr->allocated_snapshot = true; 2214 } 2215#endif 2216 2217 /* the test is responsible for initializing and enabling */ 2218 pr_info("Testing tracer %s: ", type->name); 2219 ret = type->selftest(type, tr); 2220 /* the test is responsible for resetting too */ 2221 tr->current_trace = saved_tracer; 2222 tr->current_trace_flags = saved_flags; 2223 if (ret) { 2224 printk(KERN_CONT "FAILED!\n"); 2225 /* Add the warning after printing 'FAILED' */ 2226 WARN_ON(1); 2227 return -1; 2228 } 2229 /* Only reset on passing, to avoid touching corrupted buffers */ 2230 tracing_reset_online_cpus(&tr->array_buffer); 2231 2232#ifdef CONFIG_TRACER_MAX_TRACE 2233 if (type->use_max_tr) { 2234 tr->allocated_snapshot = false; 2235 2236 /* Shrink the max buffer again */ 2237 if (tr->ring_buffer_expanded) 2238 ring_buffer_resize(tr->max_buffer.buffer, 1, 2239 RING_BUFFER_ALL_CPUS); 2240 } 2241#endif 2242 2243 printk(KERN_CONT "PASSED\n"); 2244 return 0; 2245} 2246 2247static int do_run_tracer_selftest(struct tracer *type) 2248{ 2249 int ret; 2250 2251 /* 2252 * Tests can take a long time, especially if they are run one after the 2253 * other, as does happen during bootup when all the tracers are 2254 * registered. This could cause the soft lockup watchdog to trigger. 2255 */ 2256 cond_resched(); 2257 2258 tracing_selftest_running = true; 2259 ret = run_tracer_selftest(type); 2260 tracing_selftest_running = false; 2261 2262 return ret; 2263} 2264 2265static __init int init_trace_selftests(void) 2266{ 2267 struct trace_selftests *p, *n; 2268 struct tracer *t, **last; 2269 int ret; 2270 2271 selftests_can_run = true; 2272 2273 guard(mutex)(&trace_types_lock); 2274 2275 if (list_empty(&postponed_selftests)) 2276 return 0; 2277 2278 pr_info("Running postponed tracer tests:\n"); 2279 2280 tracing_selftest_running = true; 2281 list_for_each_entry_safe(p, n, &postponed_selftests, list) { 2282 /* This loop can take minutes when sanitizers are enabled, so 2283 * lets make sure we allow RCU processing. 2284 */ 2285 cond_resched(); 2286 ret = run_tracer_selftest(p->type); 2287 /* If the test fails, then warn and remove from available_tracers */ 2288 if (ret < 0) { 2289 WARN(1, "tracer: %s failed selftest, disabling\n", 2290 p->type->name); 2291 last = &trace_types; 2292 for (t = trace_types; t; t = t->next) { 2293 if (t == p->type) { 2294 *last = t->next; 2295 break; 2296 } 2297 last = &t->next; 2298 } 2299 } 2300 list_del(&p->list); 2301 kfree(p); 2302 } 2303 tracing_selftest_running = false; 2304 2305 return 0; 2306} 2307core_initcall(init_trace_selftests); 2308#else 2309static inline int do_run_tracer_selftest(struct tracer *type) 2310{ 2311 return 0; 2312} 2313#endif /* CONFIG_FTRACE_STARTUP_TEST */ 2314 2315static int add_tracer(struct trace_array *tr, struct tracer *t); 2316 2317static void __init apply_trace_boot_options(void); 2318 2319static void free_tracers(struct trace_array *tr) 2320{ 2321 struct tracers *t, *n; 2322 2323 lockdep_assert_held(&trace_types_lock); 2324 2325 list_for_each_entry_safe(t, n, &tr->tracers, list) { 2326 list_del(&t->list); 2327 kfree(t->flags); 2328 kfree(t); 2329 } 2330} 2331 2332/** 2333 * register_tracer - register a tracer with the ftrace system. 2334 * @type: the plugin for the tracer 2335 * 2336 * Register a new plugin tracer. 2337 */ 2338int __init register_tracer(struct tracer *type) 2339{ 2340 struct trace_array *tr; 2341 struct tracer *t; 2342 int ret = 0; 2343 2344 if (!type->name) { 2345 pr_info("Tracer must have a name\n"); 2346 return -1; 2347 } 2348 2349 if (strlen(type->name) >= MAX_TRACER_SIZE) { 2350 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 2351 return -1; 2352 } 2353 2354 if (security_locked_down(LOCKDOWN_TRACEFS)) { 2355 pr_warn("Can not register tracer %s due to lockdown\n", 2356 type->name); 2357 return -EPERM; 2358 } 2359 2360 mutex_lock(&trace_types_lock); 2361 2362 for (t = trace_types; t; t = t->next) { 2363 if (strcmp(type->name, t->name) == 0) { 2364 /* already found */ 2365 pr_info("Tracer %s already registered\n", 2366 type->name); 2367 ret = -1; 2368 goto out; 2369 } 2370 } 2371 2372 /* store the tracer for __set_tracer_option */ 2373 if (type->flags) 2374 type->flags->trace = type; 2375 2376 ret = do_run_tracer_selftest(type); 2377 if (ret < 0) 2378 goto out; 2379 2380 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 2381 ret = add_tracer(tr, type); 2382 if (ret < 0) { 2383 /* The tracer will still exist but without options */ 2384 pr_warn("Failed to create tracer options for %s\n", type->name); 2385 break; 2386 } 2387 } 2388 2389 type->next = trace_types; 2390 trace_types = type; 2391 2392 out: 2393 mutex_unlock(&trace_types_lock); 2394 2395 if (ret || !default_bootup_tracer) 2396 return ret; 2397 2398 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 2399 return 0; 2400 2401 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 2402 /* Do we want this tracer to start on bootup? */ 2403 WARN_ON(tracing_set_tracer(&global_trace, type->name) < 0); 2404 default_bootup_tracer = NULL; 2405 2406 apply_trace_boot_options(); 2407 2408 /* disable other selftests, since this will break it. */ 2409 disable_tracing_selftest("running a tracer"); 2410 2411 return 0; 2412} 2413 2414static void tracing_reset_cpu(struct array_buffer *buf, int cpu) 2415{ 2416 struct trace_buffer *buffer = buf->buffer; 2417 2418 if (!buffer) 2419 return; 2420 2421 ring_buffer_record_disable(buffer); 2422 2423 /* Make sure all commits have finished */ 2424 synchronize_rcu(); 2425 ring_buffer_reset_cpu(buffer, cpu); 2426 2427 ring_buffer_record_enable(buffer); 2428} 2429 2430void tracing_reset_online_cpus(struct array_buffer *buf) 2431{ 2432 struct trace_buffer *buffer = buf->buffer; 2433 2434 if (!buffer) 2435 return; 2436 2437 ring_buffer_record_disable(buffer); 2438 2439 /* Make sure all commits have finished */ 2440 synchronize_rcu(); 2441 2442 buf->time_start = buffer_ftrace_now(buf, buf->cpu); 2443 2444 ring_buffer_reset_online_cpus(buffer); 2445 2446 ring_buffer_record_enable(buffer); 2447} 2448 2449static void tracing_reset_all_cpus(struct array_buffer *buf) 2450{ 2451 struct trace_buffer *buffer = buf->buffer; 2452 2453 if (!buffer) 2454 return; 2455 2456 ring_buffer_record_disable(buffer); 2457 2458 /* Make sure all commits have finished */ 2459 synchronize_rcu(); 2460 2461 buf->time_start = buffer_ftrace_now(buf, buf->cpu); 2462 2463 ring_buffer_reset(buffer); 2464 2465 ring_buffer_record_enable(buffer); 2466} 2467 2468/* Must have trace_types_lock held */ 2469void tracing_reset_all_online_cpus_unlocked(void) 2470{ 2471 struct trace_array *tr; 2472 2473 lockdep_assert_held(&trace_types_lock); 2474 2475 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 2476 if (!tr->clear_trace) 2477 continue; 2478 tr->clear_trace = false; 2479 tracing_reset_online_cpus(&tr->array_buffer); 2480#ifdef CONFIG_TRACER_MAX_TRACE 2481 tracing_reset_online_cpus(&tr->max_buffer); 2482#endif 2483 } 2484} 2485 2486void tracing_reset_all_online_cpus(void) 2487{ 2488 guard(mutex)(&trace_types_lock); 2489 tracing_reset_all_online_cpus_unlocked(); 2490} 2491 2492int is_tracing_stopped(void) 2493{ 2494 return global_trace.stop_count; 2495} 2496 2497static void tracing_start_tr(struct trace_array *tr) 2498{ 2499 struct trace_buffer *buffer; 2500 2501 if (tracing_disabled) 2502 return; 2503 2504 guard(raw_spinlock_irqsave)(&tr->start_lock); 2505 if (--tr->stop_count) { 2506 if (WARN_ON_ONCE(tr->stop_count < 0)) { 2507 /* Someone screwed up their debugging */ 2508 tr->stop_count = 0; 2509 } 2510 return; 2511 } 2512 2513 /* Prevent the buffers from switching */ 2514 arch_spin_lock(&tr->max_lock); 2515 2516 buffer = tr->array_buffer.buffer; 2517 if (buffer) 2518 ring_buffer_record_enable(buffer); 2519 2520#ifdef CONFIG_TRACER_MAX_TRACE 2521 buffer = tr->max_buffer.buffer; 2522 if (buffer) 2523 ring_buffer_record_enable(buffer); 2524#endif 2525 2526 arch_spin_unlock(&tr->max_lock); 2527} 2528 2529/** 2530 * tracing_start - quick start of the tracer 2531 * 2532 * If tracing is enabled but was stopped by tracing_stop, 2533 * this will start the tracer back up. 2534 */ 2535void tracing_start(void) 2536 2537{ 2538 return tracing_start_tr(&global_trace); 2539} 2540 2541static void tracing_stop_tr(struct trace_array *tr) 2542{ 2543 struct trace_buffer *buffer; 2544 2545 guard(raw_spinlock_irqsave)(&tr->start_lock); 2546 if (tr->stop_count++) 2547 return; 2548 2549 /* Prevent the buffers from switching */ 2550 arch_spin_lock(&tr->max_lock); 2551 2552 buffer = tr->array_buffer.buffer; 2553 if (buffer) 2554 ring_buffer_record_disable(buffer); 2555 2556#ifdef CONFIG_TRACER_MAX_TRACE 2557 buffer = tr->max_buffer.buffer; 2558 if (buffer) 2559 ring_buffer_record_disable(buffer); 2560#endif 2561 2562 arch_spin_unlock(&tr->max_lock); 2563} 2564 2565/** 2566 * tracing_stop - quick stop of the tracer 2567 * 2568 * Light weight way to stop tracing. Use in conjunction with 2569 * tracing_start. 2570 */ 2571void tracing_stop(void) 2572{ 2573 return tracing_stop_tr(&global_trace); 2574} 2575 2576/* 2577 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq 2578 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function 2579 * simplifies those functions and keeps them in sync. 2580 */ 2581enum print_line_t trace_handle_return(struct trace_seq *s) 2582{ 2583 return trace_seq_has_overflowed(s) ? 2584 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED; 2585} 2586EXPORT_SYMBOL_GPL(trace_handle_return); 2587 2588static unsigned short migration_disable_value(void) 2589{ 2590#if defined(CONFIG_SMP) 2591 return current->migration_disabled; 2592#else 2593 return 0; 2594#endif 2595} 2596 2597unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status) 2598{ 2599 unsigned int trace_flags = irqs_status; 2600 unsigned int pc; 2601 2602 pc = preempt_count(); 2603 2604 if (pc & NMI_MASK) 2605 trace_flags |= TRACE_FLAG_NMI; 2606 if (pc & HARDIRQ_MASK) 2607 trace_flags |= TRACE_FLAG_HARDIRQ; 2608 if (in_serving_softirq()) 2609 trace_flags |= TRACE_FLAG_SOFTIRQ; 2610 if (softirq_count() >> (SOFTIRQ_SHIFT + 1)) 2611 trace_flags |= TRACE_FLAG_BH_OFF; 2612 2613 if (tif_need_resched()) 2614 trace_flags |= TRACE_FLAG_NEED_RESCHED; 2615 if (test_preempt_need_resched()) 2616 trace_flags |= TRACE_FLAG_PREEMPT_RESCHED; 2617 if (IS_ENABLED(CONFIG_ARCH_HAS_PREEMPT_LAZY) && tif_test_bit(TIF_NEED_RESCHED_LAZY)) 2618 trace_flags |= TRACE_FLAG_NEED_RESCHED_LAZY; 2619 return (trace_flags << 16) | (min_t(unsigned int, pc & 0xff, 0xf)) | 2620 (min_t(unsigned int, migration_disable_value(), 0xf)) << 4; 2621} 2622 2623struct ring_buffer_event * 2624trace_buffer_lock_reserve(struct trace_buffer *buffer, 2625 int type, 2626 unsigned long len, 2627 unsigned int trace_ctx) 2628{ 2629 return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx); 2630} 2631 2632DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); 2633DEFINE_PER_CPU(int, trace_buffered_event_cnt); 2634static int trace_buffered_event_ref; 2635 2636/** 2637 * trace_buffered_event_enable - enable buffering events 2638 * 2639 * When events are being filtered, it is quicker to use a temporary 2640 * buffer to write the event data into if there's a likely chance 2641 * that it will not be committed. The discard of the ring buffer 2642 * is not as fast as committing, and is much slower than copying 2643 * a commit. 2644 * 2645 * When an event is to be filtered, allocate per cpu buffers to 2646 * write the event data into, and if the event is filtered and discarded 2647 * it is simply dropped, otherwise, the entire data is to be committed 2648 * in one shot. 2649 */ 2650void trace_buffered_event_enable(void) 2651{ 2652 struct ring_buffer_event *event; 2653 struct page *page; 2654 int cpu; 2655 2656 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2657 2658 if (trace_buffered_event_ref++) 2659 return; 2660 2661 for_each_tracing_cpu(cpu) { 2662 page = alloc_pages_node(cpu_to_node(cpu), 2663 GFP_KERNEL | __GFP_NORETRY, 0); 2664 /* This is just an optimization and can handle failures */ 2665 if (!page) { 2666 pr_err("Failed to allocate event buffer\n"); 2667 break; 2668 } 2669 2670 event = page_address(page); 2671 memset(event, 0, sizeof(*event)); 2672 2673 per_cpu(trace_buffered_event, cpu) = event; 2674 2675 scoped_guard(preempt,) { 2676 if (cpu == smp_processor_id() && 2677 __this_cpu_read(trace_buffered_event) != 2678 per_cpu(trace_buffered_event, cpu)) 2679 WARN_ON_ONCE(1); 2680 } 2681 } 2682} 2683 2684static void enable_trace_buffered_event(void *data) 2685{ 2686 this_cpu_dec(trace_buffered_event_cnt); 2687} 2688 2689static void disable_trace_buffered_event(void *data) 2690{ 2691 this_cpu_inc(trace_buffered_event_cnt); 2692} 2693 2694/** 2695 * trace_buffered_event_disable - disable buffering events 2696 * 2697 * When a filter is removed, it is faster to not use the buffered 2698 * events, and to commit directly into the ring buffer. Free up 2699 * the temp buffers when there are no more users. This requires 2700 * special synchronization with current events. 2701 */ 2702void trace_buffered_event_disable(void) 2703{ 2704 int cpu; 2705 2706 WARN_ON_ONCE(!mutex_is_locked(&event_mutex)); 2707 2708 if (WARN_ON_ONCE(!trace_buffered_event_ref)) 2709 return; 2710 2711 if (--trace_buffered_event_ref) 2712 return; 2713 2714 /* For each CPU, set the buffer as used. */ 2715 on_each_cpu_mask(tracing_buffer_mask, disable_trace_buffered_event, 2716 NULL, true); 2717 2718 /* Wait for all current users to finish */ 2719 synchronize_rcu(); 2720 2721 for_each_tracing_cpu(cpu) { 2722 free_page((unsigned long)per_cpu(trace_buffered_event, cpu)); 2723 per_cpu(trace_buffered_event, cpu) = NULL; 2724 } 2725 2726 /* 2727 * Wait for all CPUs that potentially started checking if they can use 2728 * their event buffer only after the previous synchronize_rcu() call and 2729 * they still read a valid pointer from trace_buffered_event. It must be 2730 * ensured they don't see cleared trace_buffered_event_cnt else they 2731 * could wrongly decide to use the pointed-to buffer which is now freed. 2732 */ 2733 synchronize_rcu(); 2734 2735 /* For each CPU, relinquish the buffer */ 2736 on_each_cpu_mask(tracing_buffer_mask, enable_trace_buffered_event, NULL, 2737 true); 2738} 2739 2740static struct trace_buffer *temp_buffer; 2741 2742struct ring_buffer_event * 2743trace_event_buffer_lock_reserve(struct trace_buffer **current_rb, 2744 struct trace_event_file *trace_file, 2745 int type, unsigned long len, 2746 unsigned int trace_ctx) 2747{ 2748 struct ring_buffer_event *entry; 2749 struct trace_array *tr = trace_file->tr; 2750 int val; 2751 2752 *current_rb = tr->array_buffer.buffer; 2753 2754 if (!tr->no_filter_buffering_ref && 2755 (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) { 2756 preempt_disable_notrace(); 2757 /* 2758 * Filtering is on, so try to use the per cpu buffer first. 2759 * This buffer will simulate a ring_buffer_event, 2760 * where the type_len is zero and the array[0] will 2761 * hold the full length. 2762 * (see include/linux/ring-buffer.h for details on 2763 * how the ring_buffer_event is structured). 2764 * 2765 * Using a temp buffer during filtering and copying it 2766 * on a matched filter is quicker than writing directly 2767 * into the ring buffer and then discarding it when 2768 * it doesn't match. That is because the discard 2769 * requires several atomic operations to get right. 2770 * Copying on match and doing nothing on a failed match 2771 * is still quicker than no copy on match, but having 2772 * to discard out of the ring buffer on a failed match. 2773 */ 2774 if ((entry = __this_cpu_read(trace_buffered_event))) { 2775 int max_len = PAGE_SIZE - struct_size(entry, array, 1); 2776 2777 val = this_cpu_inc_return(trace_buffered_event_cnt); 2778 2779 /* 2780 * Preemption is disabled, but interrupts and NMIs 2781 * can still come in now. If that happens after 2782 * the above increment, then it will have to go 2783 * back to the old method of allocating the event 2784 * on the ring buffer, and if the filter fails, it 2785 * will have to call ring_buffer_discard_commit() 2786 * to remove it. 2787 * 2788 * Need to also check the unlikely case that the 2789 * length is bigger than the temp buffer size. 2790 * If that happens, then the reserve is pretty much 2791 * guaranteed to fail, as the ring buffer currently 2792 * only allows events less than a page. But that may 2793 * change in the future, so let the ring buffer reserve 2794 * handle the failure in that case. 2795 */ 2796 if (val == 1 && likely(len <= max_len)) { 2797 trace_event_setup(entry, type, trace_ctx); 2798 entry->array[0] = len; 2799 /* Return with preemption disabled */ 2800 return entry; 2801 } 2802 this_cpu_dec(trace_buffered_event_cnt); 2803 } 2804 /* __trace_buffer_lock_reserve() disables preemption */ 2805 preempt_enable_notrace(); 2806 } 2807 2808 entry = __trace_buffer_lock_reserve(*current_rb, type, len, 2809 trace_ctx); 2810 /* 2811 * If tracing is off, but we have triggers enabled 2812 * we still need to look at the event data. Use the temp_buffer 2813 * to store the trace event for the trigger to use. It's recursive 2814 * safe and will not be recorded anywhere. 2815 */ 2816 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { 2817 *current_rb = temp_buffer; 2818 entry = __trace_buffer_lock_reserve(*current_rb, type, len, 2819 trace_ctx); 2820 } 2821 return entry; 2822} 2823EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve); 2824 2825static DEFINE_RAW_SPINLOCK(tracepoint_iter_lock); 2826static DEFINE_MUTEX(tracepoint_printk_mutex); 2827 2828static void output_printk(struct trace_event_buffer *fbuffer) 2829{ 2830 struct trace_event_call *event_call; 2831 struct trace_event_file *file; 2832 struct trace_event *event; 2833 unsigned long flags; 2834 struct trace_iterator *iter = tracepoint_print_iter; 2835 2836 /* We should never get here if iter is NULL */ 2837 if (WARN_ON_ONCE(!iter)) 2838 return; 2839 2840 event_call = fbuffer->trace_file->event_call; 2841 if (!event_call || !event_call->event.funcs || 2842 !event_call->event.funcs->trace) 2843 return; 2844 2845 file = fbuffer->trace_file; 2846 if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) || 2847 (unlikely(file->flags & EVENT_FILE_FL_FILTERED) && 2848 !filter_match_preds(file->filter, fbuffer->entry))) 2849 return; 2850 2851 event = &fbuffer->trace_file->event_call->event; 2852 2853 raw_spin_lock_irqsave(&tracepoint_iter_lock, flags); 2854 trace_seq_init(&iter->seq); 2855 iter->ent = fbuffer->entry; 2856 event_call->event.funcs->trace(iter, 0, event); 2857 trace_seq_putc(&iter->seq, 0); 2858 printk("%s", iter->seq.buffer); 2859 2860 raw_spin_unlock_irqrestore(&tracepoint_iter_lock, flags); 2861} 2862 2863int tracepoint_printk_sysctl(const struct ctl_table *table, int write, 2864 void *buffer, size_t *lenp, 2865 loff_t *ppos) 2866{ 2867 int save_tracepoint_printk; 2868 int ret; 2869 2870 guard(mutex)(&tracepoint_printk_mutex); 2871 save_tracepoint_printk = tracepoint_printk; 2872 2873 ret = proc_dointvec(table, write, buffer, lenp, ppos); 2874 2875 /* 2876 * This will force exiting early, as tracepoint_printk 2877 * is always zero when tracepoint_printk_iter is not allocated 2878 */ 2879 if (!tracepoint_print_iter) 2880 tracepoint_printk = 0; 2881 2882 if (save_tracepoint_printk == tracepoint_printk) 2883 return ret; 2884 2885 if (tracepoint_printk) 2886 static_key_enable(&tracepoint_printk_key.key); 2887 else 2888 static_key_disable(&tracepoint_printk_key.key); 2889 2890 return ret; 2891} 2892 2893void trace_event_buffer_commit(struct trace_event_buffer *fbuffer) 2894{ 2895 enum event_trigger_type tt = ETT_NONE; 2896 struct trace_event_file *file = fbuffer->trace_file; 2897 2898 if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event, 2899 fbuffer->entry, &tt)) 2900 goto discard; 2901 2902 if (static_key_false(&tracepoint_printk_key.key)) 2903 output_printk(fbuffer); 2904 2905 if (static_branch_unlikely(&trace_event_exports_enabled)) 2906 ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT); 2907 2908 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer, 2909 fbuffer->event, fbuffer->trace_ctx, fbuffer->regs); 2910 2911discard: 2912 if (tt) 2913 event_triggers_post_call(file, tt); 2914 2915} 2916EXPORT_SYMBOL_GPL(trace_event_buffer_commit); 2917 2918/* 2919 * Skip 3: 2920 * 2921 * trace_buffer_unlock_commit_regs() 2922 * trace_event_buffer_commit() 2923 * trace_event_raw_event_xxx() 2924 */ 2925# define STACK_SKIP 3 2926 2927void trace_buffer_unlock_commit_regs(struct trace_array *tr, 2928 struct trace_buffer *buffer, 2929 struct ring_buffer_event *event, 2930 unsigned int trace_ctx, 2931 struct pt_regs *regs) 2932{ 2933 __buffer_unlock_commit(buffer, event); 2934 2935 /* 2936 * If regs is not set, then skip the necessary functions. 2937 * Note, we can still get here via blktrace, wakeup tracer 2938 * and mmiotrace, but that's ok if they lose a function or 2939 * two. They are not that meaningful. 2940 */ 2941 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs); 2942 ftrace_trace_userstack(tr, buffer, trace_ctx); 2943} 2944 2945/* 2946 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack. 2947 */ 2948void 2949trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer, 2950 struct ring_buffer_event *event) 2951{ 2952 __buffer_unlock_commit(buffer, event); 2953} 2954 2955void 2956trace_function(struct trace_array *tr, unsigned long ip, unsigned long 2957 parent_ip, unsigned int trace_ctx, struct ftrace_regs *fregs) 2958{ 2959 struct trace_buffer *buffer = tr->array_buffer.buffer; 2960 struct ring_buffer_event *event; 2961 struct ftrace_entry *entry; 2962 int size = sizeof(*entry); 2963 2964 size += FTRACE_REGS_MAX_ARGS * !!fregs * sizeof(long); 2965 2966 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, size, 2967 trace_ctx); 2968 if (!event) 2969 return; 2970 entry = ring_buffer_event_data(event); 2971 entry->ip = ip; 2972 entry->parent_ip = parent_ip; 2973 2974#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 2975 if (fregs) { 2976 for (int i = 0; i < FTRACE_REGS_MAX_ARGS; i++) 2977 entry->args[i] = ftrace_regs_get_argument(fregs, i); 2978 } 2979#endif 2980 2981 if (static_branch_unlikely(&trace_function_exports_enabled)) 2982 ftrace_exports(event, TRACE_EXPORT_FUNCTION); 2983 __buffer_unlock_commit(buffer, event); 2984} 2985 2986#ifdef CONFIG_STACKTRACE 2987 2988/* Allow 4 levels of nesting: normal, softirq, irq, NMI */ 2989#define FTRACE_KSTACK_NESTING 4 2990 2991#define FTRACE_KSTACK_ENTRIES (SZ_4K / FTRACE_KSTACK_NESTING) 2992 2993struct ftrace_stack { 2994 unsigned long calls[FTRACE_KSTACK_ENTRIES]; 2995}; 2996 2997 2998struct ftrace_stacks { 2999 struct ftrace_stack stacks[FTRACE_KSTACK_NESTING]; 3000}; 3001 3002static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks); 3003static DEFINE_PER_CPU(int, ftrace_stack_reserve); 3004 3005static void __ftrace_trace_stack(struct trace_array *tr, 3006 struct trace_buffer *buffer, 3007 unsigned int trace_ctx, 3008 int skip, struct pt_regs *regs) 3009{ 3010 struct ring_buffer_event *event; 3011 unsigned int size, nr_entries; 3012 struct ftrace_stack *fstack; 3013 struct stack_entry *entry; 3014 int stackidx; 3015 int bit; 3016 3017 bit = trace_test_and_set_recursion(_THIS_IP_, _RET_IP_, TRACE_EVENT_START); 3018 if (bit < 0) 3019 return; 3020 3021 /* 3022 * Add one, for this function and the call to save_stack_trace() 3023 * If regs is set, then these functions will not be in the way. 3024 */ 3025#ifndef CONFIG_UNWINDER_ORC 3026 if (!regs) 3027 skip++; 3028#endif 3029 3030 guard(preempt_notrace)(); 3031 3032 stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1; 3033 3034 /* This should never happen. If it does, yell once and skip */ 3035 if (WARN_ON_ONCE(stackidx >= FTRACE_KSTACK_NESTING)) 3036 goto out; 3037 3038 /* 3039 * The above __this_cpu_inc_return() is 'atomic' cpu local. An 3040 * interrupt will either see the value pre increment or post 3041 * increment. If the interrupt happens pre increment it will have 3042 * restored the counter when it returns. We just need a barrier to 3043 * keep gcc from moving things around. 3044 */ 3045 barrier(); 3046 3047 fstack = this_cpu_ptr(ftrace_stacks.stacks) + stackidx; 3048 size = ARRAY_SIZE(fstack->calls); 3049 3050 if (regs) { 3051 nr_entries = stack_trace_save_regs(regs, fstack->calls, 3052 size, skip); 3053 } else { 3054 nr_entries = stack_trace_save(fstack->calls, size, skip); 3055 } 3056 3057#ifdef CONFIG_DYNAMIC_FTRACE 3058 /* Mark entry of stack trace as trampoline code */ 3059 if (tr->ops && tr->ops->trampoline) { 3060 unsigned long tramp_start = tr->ops->trampoline; 3061 unsigned long tramp_end = tramp_start + tr->ops->trampoline_size; 3062 unsigned long *calls = fstack->calls; 3063 3064 for (int i = 0; i < nr_entries; i++) { 3065 if (calls[i] >= tramp_start && calls[i] < tramp_end) 3066 calls[i] = FTRACE_TRAMPOLINE_MARKER; 3067 } 3068 } 3069#endif 3070 3071 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK, 3072 struct_size(entry, caller, nr_entries), 3073 trace_ctx); 3074 if (!event) 3075 goto out; 3076 entry = ring_buffer_event_data(event); 3077 3078 entry->size = nr_entries; 3079 memcpy(&entry->caller, fstack->calls, 3080 flex_array_size(entry, caller, nr_entries)); 3081 3082 __buffer_unlock_commit(buffer, event); 3083 3084 out: 3085 /* Again, don't let gcc optimize things here */ 3086 barrier(); 3087 __this_cpu_dec(ftrace_stack_reserve); 3088 trace_clear_recursion(bit); 3089} 3090 3091static inline void ftrace_trace_stack(struct trace_array *tr, 3092 struct trace_buffer *buffer, 3093 unsigned int trace_ctx, 3094 int skip, struct pt_regs *regs) 3095{ 3096 if (!(tr->trace_flags & TRACE_ITER(STACKTRACE))) 3097 return; 3098 3099 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, regs); 3100} 3101 3102void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, 3103 int skip) 3104{ 3105 struct trace_buffer *buffer = tr->array_buffer.buffer; 3106 3107 if (rcu_is_watching()) { 3108 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL); 3109 return; 3110 } 3111 3112 if (WARN_ON_ONCE(IS_ENABLED(CONFIG_GENERIC_ENTRY))) 3113 return; 3114 3115 /* 3116 * When an NMI triggers, RCU is enabled via ct_nmi_enter(), 3117 * but if the above rcu_is_watching() failed, then the NMI 3118 * triggered someplace critical, and ct_irq_enter() should 3119 * not be called from NMI. 3120 */ 3121 if (unlikely(in_nmi())) 3122 return; 3123 3124 ct_irq_enter_irqson(); 3125 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL); 3126 ct_irq_exit_irqson(); 3127} 3128 3129/** 3130 * trace_dump_stack - record a stack back trace in the trace buffer 3131 * @skip: Number of functions to skip (helper handlers) 3132 */ 3133void trace_dump_stack(int skip) 3134{ 3135 if (tracing_disabled || tracing_selftest_running) 3136 return; 3137 3138#ifndef CONFIG_UNWINDER_ORC 3139 /* Skip 1 to skip this function. */ 3140 skip++; 3141#endif 3142 __ftrace_trace_stack(printk_trace, printk_trace->array_buffer.buffer, 3143 tracing_gen_ctx(), skip, NULL); 3144} 3145EXPORT_SYMBOL_GPL(trace_dump_stack); 3146 3147#ifdef CONFIG_USER_STACKTRACE_SUPPORT 3148static DEFINE_PER_CPU(int, user_stack_count); 3149 3150static void 3151ftrace_trace_userstack(struct trace_array *tr, 3152 struct trace_buffer *buffer, unsigned int trace_ctx) 3153{ 3154 struct ring_buffer_event *event; 3155 struct userstack_entry *entry; 3156 3157 if (!(tr->trace_flags & TRACE_ITER(USERSTACKTRACE))) 3158 return; 3159 3160 /* 3161 * NMIs can not handle page faults, even with fix ups. 3162 * The save user stack can (and often does) fault. 3163 */ 3164 if (unlikely(in_nmi())) 3165 return; 3166 3167 /* 3168 * prevent recursion, since the user stack tracing may 3169 * trigger other kernel events. 3170 */ 3171 guard(preempt)(); 3172 if (__this_cpu_read(user_stack_count)) 3173 return; 3174 3175 __this_cpu_inc(user_stack_count); 3176 3177 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 3178 sizeof(*entry), trace_ctx); 3179 if (!event) 3180 goto out_drop_count; 3181 entry = ring_buffer_event_data(event); 3182 3183 entry->tgid = current->tgid; 3184 memset(&entry->caller, 0, sizeof(entry->caller)); 3185 3186 stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES); 3187 __buffer_unlock_commit(buffer, event); 3188 3189 out_drop_count: 3190 __this_cpu_dec(user_stack_count); 3191} 3192#else /* CONFIG_USER_STACKTRACE_SUPPORT */ 3193static void ftrace_trace_userstack(struct trace_array *tr, 3194 struct trace_buffer *buffer, 3195 unsigned int trace_ctx) 3196{ 3197} 3198#endif /* !CONFIG_USER_STACKTRACE_SUPPORT */ 3199 3200#endif /* CONFIG_STACKTRACE */ 3201 3202static inline void 3203func_repeats_set_delta_ts(struct func_repeats_entry *entry, 3204 unsigned long long delta) 3205{ 3206 entry->bottom_delta_ts = delta & U32_MAX; 3207 entry->top_delta_ts = (delta >> 32); 3208} 3209 3210void trace_last_func_repeats(struct trace_array *tr, 3211 struct trace_func_repeats *last_info, 3212 unsigned int trace_ctx) 3213{ 3214 struct trace_buffer *buffer = tr->array_buffer.buffer; 3215 struct func_repeats_entry *entry; 3216 struct ring_buffer_event *event; 3217 u64 delta; 3218 3219 event = __trace_buffer_lock_reserve(buffer, TRACE_FUNC_REPEATS, 3220 sizeof(*entry), trace_ctx); 3221 if (!event) 3222 return; 3223 3224 delta = ring_buffer_event_time_stamp(buffer, event) - 3225 last_info->ts_last_call; 3226 3227 entry = ring_buffer_event_data(event); 3228 entry->ip = last_info->ip; 3229 entry->parent_ip = last_info->parent_ip; 3230 entry->count = last_info->count; 3231 func_repeats_set_delta_ts(entry, delta); 3232 3233 __buffer_unlock_commit(buffer, event); 3234} 3235 3236/* created for use with alloc_percpu */ 3237struct trace_buffer_struct { 3238 int nesting; 3239 char buffer[4][TRACE_BUF_SIZE]; 3240}; 3241 3242static struct trace_buffer_struct __percpu *trace_percpu_buffer; 3243 3244/* 3245 * This allows for lockless recording. If we're nested too deeply, then 3246 * this returns NULL. 3247 */ 3248static char *get_trace_buf(void) 3249{ 3250 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer); 3251 3252 if (!trace_percpu_buffer || buffer->nesting >= 4) 3253 return NULL; 3254 3255 buffer->nesting++; 3256 3257 /* Interrupts must see nesting incremented before we use the buffer */ 3258 barrier(); 3259 return &buffer->buffer[buffer->nesting - 1][0]; 3260} 3261 3262static void put_trace_buf(void) 3263{ 3264 /* Don't let the decrement of nesting leak before this */ 3265 barrier(); 3266 this_cpu_dec(trace_percpu_buffer->nesting); 3267} 3268 3269static int alloc_percpu_trace_buffer(void) 3270{ 3271 struct trace_buffer_struct __percpu *buffers; 3272 3273 if (trace_percpu_buffer) 3274 return 0; 3275 3276 buffers = alloc_percpu(struct trace_buffer_struct); 3277 if (MEM_FAIL(!buffers, "Could not allocate percpu trace_printk buffer")) 3278 return -ENOMEM; 3279 3280 trace_percpu_buffer = buffers; 3281 return 0; 3282} 3283 3284static int buffers_allocated; 3285 3286void trace_printk_init_buffers(void) 3287{ 3288 if (buffers_allocated) 3289 return; 3290 3291 if (alloc_percpu_trace_buffer()) 3292 return; 3293 3294 /* trace_printk() is for debug use only. Don't use it in production. */ 3295 3296 pr_warn("\n"); 3297 pr_warn("**********************************************************\n"); 3298 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 3299 pr_warn("** **\n"); 3300 pr_warn("** trace_printk() being used. Allocating extra memory. **\n"); 3301 pr_warn("** **\n"); 3302 pr_warn("** This means that this is a DEBUG kernel and it is **\n"); 3303 pr_warn("** unsafe for production use. **\n"); 3304 pr_warn("** **\n"); 3305 pr_warn("** If you see this message and you are not debugging **\n"); 3306 pr_warn("** the kernel, report this immediately to your vendor! **\n"); 3307 pr_warn("** **\n"); 3308 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n"); 3309 pr_warn("**********************************************************\n"); 3310 3311 /* Expand the buffers to set size */ 3312 tracing_update_buffers(&global_trace); 3313 3314 buffers_allocated = 1; 3315 3316 /* 3317 * trace_printk_init_buffers() can be called by modules. 3318 * If that happens, then we need to start cmdline recording 3319 * directly here. If the global_trace.buffer is already 3320 * allocated here, then this was called by module code. 3321 */ 3322 if (global_trace.array_buffer.buffer) 3323 tracing_start_cmdline_record(); 3324} 3325EXPORT_SYMBOL_GPL(trace_printk_init_buffers); 3326 3327void trace_printk_start_comm(void) 3328{ 3329 /* Start tracing comms if trace printk is set */ 3330 if (!buffers_allocated) 3331 return; 3332 tracing_start_cmdline_record(); 3333} 3334 3335static void trace_printk_start_stop_comm(int enabled) 3336{ 3337 if (!buffers_allocated) 3338 return; 3339 3340 if (enabled) 3341 tracing_start_cmdline_record(); 3342 else 3343 tracing_stop_cmdline_record(); 3344} 3345 3346/** 3347 * trace_vbprintk - write binary msg to tracing buffer 3348 * @ip: The address of the caller 3349 * @fmt: The string format to write to the buffer 3350 * @args: Arguments for @fmt 3351 */ 3352int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 3353{ 3354 struct ring_buffer_event *event; 3355 struct trace_buffer *buffer; 3356 struct trace_array *tr = READ_ONCE(printk_trace); 3357 struct bprint_entry *entry; 3358 unsigned int trace_ctx; 3359 char *tbuffer; 3360 int len = 0, size; 3361 3362 if (!printk_binsafe(tr)) 3363 return trace_vprintk(ip, fmt, args); 3364 3365 if (unlikely(tracing_selftest_running || tracing_disabled)) 3366 return 0; 3367 3368 /* Don't pollute graph traces with trace_vprintk internals */ 3369 pause_graph_tracing(); 3370 3371 trace_ctx = tracing_gen_ctx(); 3372 guard(preempt_notrace)(); 3373 3374 tbuffer = get_trace_buf(); 3375 if (!tbuffer) { 3376 len = 0; 3377 goto out_nobuffer; 3378 } 3379 3380 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args); 3381 3382 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) 3383 goto out_put; 3384 3385 size = sizeof(*entry) + sizeof(u32) * len; 3386 buffer = tr->array_buffer.buffer; 3387 scoped_guard(ring_buffer_nest, buffer) { 3388 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 3389 trace_ctx); 3390 if (!event) 3391 goto out_put; 3392 entry = ring_buffer_event_data(event); 3393 entry->ip = ip; 3394 entry->fmt = fmt; 3395 3396 memcpy(entry->buf, tbuffer, sizeof(u32) * len); 3397 __buffer_unlock_commit(buffer, event); 3398 ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL); 3399 } 3400out_put: 3401 put_trace_buf(); 3402 3403out_nobuffer: 3404 unpause_graph_tracing(); 3405 3406 return len; 3407} 3408EXPORT_SYMBOL_GPL(trace_vbprintk); 3409 3410static __printf(3, 0) 3411int __trace_array_vprintk(struct trace_buffer *buffer, 3412 unsigned long ip, const char *fmt, va_list args) 3413{ 3414 struct ring_buffer_event *event; 3415 int len = 0, size; 3416 struct print_entry *entry; 3417 unsigned int trace_ctx; 3418 char *tbuffer; 3419 3420 if (tracing_disabled) 3421 return 0; 3422 3423 /* Don't pollute graph traces with trace_vprintk internals */ 3424 pause_graph_tracing(); 3425 3426 trace_ctx = tracing_gen_ctx(); 3427 guard(preempt_notrace)(); 3428 3429 3430 tbuffer = get_trace_buf(); 3431 if (!tbuffer) { 3432 len = 0; 3433 goto out_nobuffer; 3434 } 3435 3436 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args); 3437 3438 size = sizeof(*entry) + len + 1; 3439 scoped_guard(ring_buffer_nest, buffer) { 3440 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 3441 trace_ctx); 3442 if (!event) 3443 goto out; 3444 entry = ring_buffer_event_data(event); 3445 entry->ip = ip; 3446 3447 memcpy(&entry->buf, tbuffer, len + 1); 3448 __buffer_unlock_commit(buffer, event); 3449 ftrace_trace_stack(printk_trace, buffer, trace_ctx, 6, NULL); 3450 } 3451out: 3452 put_trace_buf(); 3453 3454out_nobuffer: 3455 unpause_graph_tracing(); 3456 3457 return len; 3458} 3459 3460int trace_array_vprintk(struct trace_array *tr, 3461 unsigned long ip, const char *fmt, va_list args) 3462{ 3463 if (tracing_selftest_running && tr == &global_trace) 3464 return 0; 3465 3466 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args); 3467} 3468 3469/** 3470 * trace_array_printk - Print a message to a specific instance 3471 * @tr: The instance trace_array descriptor 3472 * @ip: The instruction pointer that this is called from. 3473 * @fmt: The format to print (printf format) 3474 * 3475 * If a subsystem sets up its own instance, they have the right to 3476 * printk strings into their tracing instance buffer using this 3477 * function. Note, this function will not write into the top level 3478 * buffer (use trace_printk() for that), as writing into the top level 3479 * buffer should only have events that can be individually disabled. 3480 * trace_printk() is only used for debugging a kernel, and should not 3481 * be ever incorporated in normal use. 3482 * 3483 * trace_array_printk() can be used, as it will not add noise to the 3484 * top level tracing buffer. 3485 * 3486 * Note, trace_array_init_printk() must be called on @tr before this 3487 * can be used. 3488 */ 3489int trace_array_printk(struct trace_array *tr, 3490 unsigned long ip, const char *fmt, ...) 3491{ 3492 int ret; 3493 va_list ap; 3494 3495 if (!tr) 3496 return -ENOENT; 3497 3498 /* This is only allowed for created instances */ 3499 if (tr == &global_trace) 3500 return 0; 3501 3502 if (!(tr->trace_flags & TRACE_ITER(PRINTK))) 3503 return 0; 3504 3505 va_start(ap, fmt); 3506 ret = trace_array_vprintk(tr, ip, fmt, ap); 3507 va_end(ap); 3508 return ret; 3509} 3510EXPORT_SYMBOL_GPL(trace_array_printk); 3511 3512/** 3513 * trace_array_init_printk - Initialize buffers for trace_array_printk() 3514 * @tr: The trace array to initialize the buffers for 3515 * 3516 * As trace_array_printk() only writes into instances, they are OK to 3517 * have in the kernel (unlike trace_printk()). This needs to be called 3518 * before trace_array_printk() can be used on a trace_array. 3519 */ 3520int trace_array_init_printk(struct trace_array *tr) 3521{ 3522 if (!tr) 3523 return -ENOENT; 3524 3525 /* This is only allowed for created instances */ 3526 if (tr == &global_trace) 3527 return -EINVAL; 3528 3529 return alloc_percpu_trace_buffer(); 3530} 3531EXPORT_SYMBOL_GPL(trace_array_init_printk); 3532 3533int trace_array_printk_buf(struct trace_buffer *buffer, 3534 unsigned long ip, const char *fmt, ...) 3535{ 3536 int ret; 3537 va_list ap; 3538 3539 if (!(printk_trace->trace_flags & TRACE_ITER(PRINTK))) 3540 return 0; 3541 3542 va_start(ap, fmt); 3543 ret = __trace_array_vprintk(buffer, ip, fmt, ap); 3544 va_end(ap); 3545 return ret; 3546} 3547 3548int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 3549{ 3550 return trace_array_vprintk(printk_trace, ip, fmt, args); 3551} 3552EXPORT_SYMBOL_GPL(trace_vprintk); 3553 3554static void trace_iterator_increment(struct trace_iterator *iter) 3555{ 3556 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu); 3557 3558 iter->idx++; 3559 if (buf_iter) 3560 ring_buffer_iter_advance(buf_iter); 3561} 3562 3563static struct trace_entry * 3564peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 3565 unsigned long *lost_events) 3566{ 3567 struct ring_buffer_event *event; 3568 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu); 3569 3570 if (buf_iter) { 3571 event = ring_buffer_iter_peek(buf_iter, ts); 3572 if (lost_events) 3573 *lost_events = ring_buffer_iter_dropped(buf_iter) ? 3574 (unsigned long)-1 : 0; 3575 } else { 3576 event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts, 3577 lost_events); 3578 } 3579 3580 if (event) { 3581 iter->ent_size = ring_buffer_event_length(event); 3582 return ring_buffer_event_data(event); 3583 } 3584 iter->ent_size = 0; 3585 return NULL; 3586} 3587 3588static struct trace_entry * 3589__find_next_entry(struct trace_iterator *iter, int *ent_cpu, 3590 unsigned long *missing_events, u64 *ent_ts) 3591{ 3592 struct trace_buffer *buffer = iter->array_buffer->buffer; 3593 struct trace_entry *ent, *next = NULL; 3594 unsigned long lost_events = 0, next_lost = 0; 3595 int cpu_file = iter->cpu_file; 3596 u64 next_ts = 0, ts; 3597 int next_cpu = -1; 3598 int next_size = 0; 3599 int cpu; 3600 3601 /* 3602 * If we are in a per_cpu trace file, don't bother by iterating over 3603 * all cpu and peek directly. 3604 */ 3605 if (cpu_file > RING_BUFFER_ALL_CPUS) { 3606 if (ring_buffer_empty_cpu(buffer, cpu_file)) 3607 return NULL; 3608 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 3609 if (ent_cpu) 3610 *ent_cpu = cpu_file; 3611 3612 return ent; 3613 } 3614 3615 for_each_tracing_cpu(cpu) { 3616 3617 if (ring_buffer_empty_cpu(buffer, cpu)) 3618 continue; 3619 3620 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 3621 3622 /* 3623 * Pick the entry with the smallest timestamp: 3624 */ 3625 if (ent && (!next || ts < next_ts)) { 3626 next = ent; 3627 next_cpu = cpu; 3628 next_ts = ts; 3629 next_lost = lost_events; 3630 next_size = iter->ent_size; 3631 } 3632 } 3633 3634 iter->ent_size = next_size; 3635 3636 if (ent_cpu) 3637 *ent_cpu = next_cpu; 3638 3639 if (ent_ts) 3640 *ent_ts = next_ts; 3641 3642 if (missing_events) 3643 *missing_events = next_lost; 3644 3645 return next; 3646} 3647 3648#define STATIC_FMT_BUF_SIZE 128 3649static char static_fmt_buf[STATIC_FMT_BUF_SIZE]; 3650 3651char *trace_iter_expand_format(struct trace_iterator *iter) 3652{ 3653 char *tmp; 3654 3655 /* 3656 * iter->tr is NULL when used with tp_printk, which makes 3657 * this get called where it is not safe to call krealloc(). 3658 */ 3659 if (!iter->tr || iter->fmt == static_fmt_buf) 3660 return NULL; 3661 3662 tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE, 3663 GFP_KERNEL); 3664 if (tmp) { 3665 iter->fmt_size += STATIC_FMT_BUF_SIZE; 3666 iter->fmt = tmp; 3667 } 3668 3669 return tmp; 3670} 3671 3672/* Returns true if the string is safe to dereference from an event */ 3673static bool trace_safe_str(struct trace_iterator *iter, const char *str) 3674{ 3675 unsigned long addr = (unsigned long)str; 3676 struct trace_event *trace_event; 3677 struct trace_event_call *event; 3678 3679 /* OK if part of the event data */ 3680 if ((addr >= (unsigned long)iter->ent) && 3681 (addr < (unsigned long)iter->ent + iter->ent_size)) 3682 return true; 3683 3684 /* OK if part of the temp seq buffer */ 3685 if ((addr >= (unsigned long)iter->tmp_seq.buffer) && 3686 (addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE)) 3687 return true; 3688 3689 /* Core rodata can not be freed */ 3690 if (is_kernel_rodata(addr)) 3691 return true; 3692 3693 if (trace_is_tracepoint_string(str)) 3694 return true; 3695 3696 /* 3697 * Now this could be a module event, referencing core module 3698 * data, which is OK. 3699 */ 3700 if (!iter->ent) 3701 return false; 3702 3703 trace_event = ftrace_find_event(iter->ent->type); 3704 if (!trace_event) 3705 return false; 3706 3707 event = container_of(trace_event, struct trace_event_call, event); 3708 if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module) 3709 return false; 3710 3711 /* Would rather have rodata, but this will suffice */ 3712 if (within_module_core(addr, event->module)) 3713 return true; 3714 3715 return false; 3716} 3717 3718/** 3719 * ignore_event - Check dereferenced fields while writing to the seq buffer 3720 * @iter: The iterator that holds the seq buffer and the event being printed 3721 * 3722 * At boot up, test_event_printk() will flag any event that dereferences 3723 * a string with "%s" that does exist in the ring buffer. It may still 3724 * be valid, as the string may point to a static string in the kernel 3725 * rodata that never gets freed. But if the string pointer is pointing 3726 * to something that was allocated, there's a chance that it can be freed 3727 * by the time the user reads the trace. This would cause a bad memory 3728 * access by the kernel and possibly crash the system. 3729 * 3730 * This function will check if the event has any fields flagged as needing 3731 * to be checked at runtime and perform those checks. 3732 * 3733 * If it is found that a field is unsafe, it will write into the @iter->seq 3734 * a message stating what was found to be unsafe. 3735 * 3736 * @return: true if the event is unsafe and should be ignored, 3737 * false otherwise. 3738 */ 3739bool ignore_event(struct trace_iterator *iter) 3740{ 3741 struct ftrace_event_field *field; 3742 struct trace_event *trace_event; 3743 struct trace_event_call *event; 3744 struct list_head *head; 3745 struct trace_seq *seq; 3746 const void *ptr; 3747 3748 trace_event = ftrace_find_event(iter->ent->type); 3749 3750 seq = &iter->seq; 3751 3752 if (!trace_event) { 3753 trace_seq_printf(seq, "EVENT ID %d NOT FOUND?\n", iter->ent->type); 3754 return true; 3755 } 3756 3757 event = container_of(trace_event, struct trace_event_call, event); 3758 if (!(event->flags & TRACE_EVENT_FL_TEST_STR)) 3759 return false; 3760 3761 head = trace_get_fields(event); 3762 if (!head) { 3763 trace_seq_printf(seq, "FIELDS FOR EVENT '%s' NOT FOUND?\n", 3764 trace_event_name(event)); 3765 return true; 3766 } 3767 3768 /* Offsets are from the iter->ent that points to the raw event */ 3769 ptr = iter->ent; 3770 3771 list_for_each_entry(field, head, link) { 3772 const char *str; 3773 bool good; 3774 3775 if (!field->needs_test) 3776 continue; 3777 3778 str = *(const char **)(ptr + field->offset); 3779 3780 good = trace_safe_str(iter, str); 3781 3782 /* 3783 * If you hit this warning, it is likely that the 3784 * trace event in question used %s on a string that 3785 * was saved at the time of the event, but may not be 3786 * around when the trace is read. Use __string(), 3787 * __assign_str() and __get_str() helpers in the TRACE_EVENT() 3788 * instead. See samples/trace_events/trace-events-sample.h 3789 * for reference. 3790 */ 3791 if (WARN_ONCE(!good, "event '%s' has unsafe pointer field '%s'", 3792 trace_event_name(event), field->name)) { 3793 trace_seq_printf(seq, "EVENT %s: HAS UNSAFE POINTER FIELD '%s'\n", 3794 trace_event_name(event), field->name); 3795 return true; 3796 } 3797 } 3798 return false; 3799} 3800 3801const char *trace_event_format(struct trace_iterator *iter, const char *fmt) 3802{ 3803 const char *p, *new_fmt; 3804 char *q; 3805 3806 if (WARN_ON_ONCE(!fmt)) 3807 return fmt; 3808 3809 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER(HASH_PTR)) 3810 return fmt; 3811 3812 p = fmt; 3813 new_fmt = q = iter->fmt; 3814 while (*p) { 3815 if (unlikely(q - new_fmt + 3 > iter->fmt_size)) { 3816 if (!trace_iter_expand_format(iter)) 3817 return fmt; 3818 3819 q += iter->fmt - new_fmt; 3820 new_fmt = iter->fmt; 3821 } 3822 3823 *q++ = *p++; 3824 3825 /* Replace %p with %px */ 3826 if (p[-1] == '%') { 3827 if (p[0] == '%') { 3828 *q++ = *p++; 3829 } else if (p[0] == 'p' && !isalnum(p[1])) { 3830 *q++ = *p++; 3831 *q++ = 'x'; 3832 } 3833 } 3834 } 3835 *q = '\0'; 3836 3837 return new_fmt; 3838} 3839 3840#define STATIC_TEMP_BUF_SIZE 128 3841static char static_temp_buf[STATIC_TEMP_BUF_SIZE] __aligned(4); 3842 3843/* Find the next real entry, without updating the iterator itself */ 3844struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 3845 int *ent_cpu, u64 *ent_ts) 3846{ 3847 /* __find_next_entry will reset ent_size */ 3848 int ent_size = iter->ent_size; 3849 struct trace_entry *entry; 3850 3851 /* 3852 * If called from ftrace_dump(), then the iter->temp buffer 3853 * will be the static_temp_buf and not created from kmalloc. 3854 * If the entry size is greater than the buffer, we can 3855 * not save it. Just return NULL in that case. This is only 3856 * used to add markers when two consecutive events' time 3857 * stamps have a large delta. See trace_print_lat_context() 3858 */ 3859 if (iter->temp == static_temp_buf && 3860 STATIC_TEMP_BUF_SIZE < ent_size) 3861 return NULL; 3862 3863 /* 3864 * The __find_next_entry() may call peek_next_entry(), which may 3865 * call ring_buffer_peek() that may make the contents of iter->ent 3866 * undefined. Need to copy iter->ent now. 3867 */ 3868 if (iter->ent && iter->ent != iter->temp) { 3869 if ((!iter->temp || iter->temp_size < iter->ent_size) && 3870 !WARN_ON_ONCE(iter->temp == static_temp_buf)) { 3871 void *temp; 3872 temp = kmalloc(iter->ent_size, GFP_KERNEL); 3873 if (!temp) 3874 return NULL; 3875 kfree(iter->temp); 3876 iter->temp = temp; 3877 iter->temp_size = iter->ent_size; 3878 } 3879 memcpy(iter->temp, iter->ent, iter->ent_size); 3880 iter->ent = iter->temp; 3881 } 3882 entry = __find_next_entry(iter, ent_cpu, NULL, ent_ts); 3883 /* Put back the original ent_size */ 3884 iter->ent_size = ent_size; 3885 3886 return entry; 3887} 3888 3889/* Find the next real entry, and increment the iterator to the next entry */ 3890void *trace_find_next_entry_inc(struct trace_iterator *iter) 3891{ 3892 iter->ent = __find_next_entry(iter, &iter->cpu, 3893 &iter->lost_events, &iter->ts); 3894 3895 if (iter->ent) 3896 trace_iterator_increment(iter); 3897 3898 return iter->ent ? iter : NULL; 3899} 3900 3901static void trace_consume(struct trace_iterator *iter) 3902{ 3903 ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts, 3904 &iter->lost_events); 3905} 3906 3907static void *s_next(struct seq_file *m, void *v, loff_t *pos) 3908{ 3909 struct trace_iterator *iter = m->private; 3910 int i = (int)*pos; 3911 void *ent; 3912 3913 WARN_ON_ONCE(iter->leftover); 3914 3915 (*pos)++; 3916 3917 /* can't go backwards */ 3918 if (iter->idx > i) 3919 return NULL; 3920 3921 if (iter->idx < 0) 3922 ent = trace_find_next_entry_inc(iter); 3923 else 3924 ent = iter; 3925 3926 while (ent && iter->idx < i) 3927 ent = trace_find_next_entry_inc(iter); 3928 3929 iter->pos = *pos; 3930 3931 return ent; 3932} 3933 3934void tracing_iter_reset(struct trace_iterator *iter, int cpu) 3935{ 3936 struct ring_buffer_iter *buf_iter; 3937 unsigned long entries = 0; 3938 u64 ts; 3939 3940 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0; 3941 3942 buf_iter = trace_buffer_iter(iter, cpu); 3943 if (!buf_iter) 3944 return; 3945 3946 ring_buffer_iter_reset(buf_iter); 3947 3948 /* 3949 * We could have the case with the max latency tracers 3950 * that a reset never took place on a cpu. This is evident 3951 * by the timestamp being before the start of the buffer. 3952 */ 3953 while (ring_buffer_iter_peek(buf_iter, &ts)) { 3954 if (ts >= iter->array_buffer->time_start) 3955 break; 3956 entries++; 3957 ring_buffer_iter_advance(buf_iter); 3958 /* This could be a big loop */ 3959 cond_resched(); 3960 } 3961 3962 per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries; 3963} 3964 3965/* 3966 * The current tracer is copied to avoid a global locking 3967 * all around. 3968 */ 3969static void *s_start(struct seq_file *m, loff_t *pos) 3970{ 3971 struct trace_iterator *iter = m->private; 3972 struct trace_array *tr = iter->tr; 3973 int cpu_file = iter->cpu_file; 3974 void *p = NULL; 3975 loff_t l = 0; 3976 int cpu; 3977 3978 mutex_lock(&trace_types_lock); 3979 if (unlikely(tr->current_trace != iter->trace)) { 3980 /* Close iter->trace before switching to the new current tracer */ 3981 if (iter->trace->close) 3982 iter->trace->close(iter); 3983 iter->trace = tr->current_trace; 3984 /* Reopen the new current tracer */ 3985 if (iter->trace->open) 3986 iter->trace->open(iter); 3987 } 3988 mutex_unlock(&trace_types_lock); 3989 3990#ifdef CONFIG_TRACER_MAX_TRACE 3991 if (iter->snapshot && iter->trace->use_max_tr) 3992 return ERR_PTR(-EBUSY); 3993#endif 3994 3995 if (*pos != iter->pos) { 3996 iter->ent = NULL; 3997 iter->cpu = 0; 3998 iter->idx = -1; 3999 4000 if (cpu_file == RING_BUFFER_ALL_CPUS) { 4001 for_each_tracing_cpu(cpu) 4002 tracing_iter_reset(iter, cpu); 4003 } else 4004 tracing_iter_reset(iter, cpu_file); 4005 4006 iter->leftover = 0; 4007 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 4008 ; 4009 4010 } else { 4011 /* 4012 * If we overflowed the seq_file before, then we want 4013 * to just reuse the trace_seq buffer again. 4014 */ 4015 if (iter->leftover) 4016 p = iter; 4017 else { 4018 l = *pos - 1; 4019 p = s_next(m, p, &l); 4020 } 4021 } 4022 4023 trace_event_read_lock(); 4024 trace_access_lock(cpu_file); 4025 return p; 4026} 4027 4028static void s_stop(struct seq_file *m, void *p) 4029{ 4030 struct trace_iterator *iter = m->private; 4031 4032#ifdef CONFIG_TRACER_MAX_TRACE 4033 if (iter->snapshot && iter->trace->use_max_tr) 4034 return; 4035#endif 4036 4037 trace_access_unlock(iter->cpu_file); 4038 trace_event_read_unlock(); 4039} 4040 4041static void 4042get_total_entries_cpu(struct array_buffer *buf, unsigned long *total, 4043 unsigned long *entries, int cpu) 4044{ 4045 unsigned long count; 4046 4047 count = ring_buffer_entries_cpu(buf->buffer, cpu); 4048 /* 4049 * If this buffer has skipped entries, then we hold all 4050 * entries for the trace and we need to ignore the 4051 * ones before the time stamp. 4052 */ 4053 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) { 4054 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries; 4055 /* total is the same as the entries */ 4056 *total = count; 4057 } else 4058 *total = count + 4059 ring_buffer_overrun_cpu(buf->buffer, cpu); 4060 *entries = count; 4061} 4062 4063static void 4064get_total_entries(struct array_buffer *buf, 4065 unsigned long *total, unsigned long *entries) 4066{ 4067 unsigned long t, e; 4068 int cpu; 4069 4070 *total = 0; 4071 *entries = 0; 4072 4073 for_each_tracing_cpu(cpu) { 4074 get_total_entries_cpu(buf, &t, &e, cpu); 4075 *total += t; 4076 *entries += e; 4077 } 4078} 4079 4080unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu) 4081{ 4082 unsigned long total, entries; 4083 4084 if (!tr) 4085 tr = &global_trace; 4086 4087 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu); 4088 4089 return entries; 4090} 4091 4092unsigned long trace_total_entries(struct trace_array *tr) 4093{ 4094 unsigned long total, entries; 4095 4096 if (!tr) 4097 tr = &global_trace; 4098 4099 get_total_entries(&tr->array_buffer, &total, &entries); 4100 4101 return entries; 4102} 4103 4104static void print_lat_help_header(struct seq_file *m) 4105{ 4106 seq_puts(m, "# _------=> CPU# \n" 4107 "# / _-----=> irqs-off/BH-disabled\n" 4108 "# | / _----=> need-resched \n" 4109 "# || / _---=> hardirq/softirq \n" 4110 "# ||| / _--=> preempt-depth \n" 4111 "# |||| / _-=> migrate-disable \n" 4112 "# ||||| / delay \n" 4113 "# cmd pid |||||| time | caller \n" 4114 "# \\ / |||||| \\ | / \n"); 4115} 4116 4117static void print_event_info(struct array_buffer *buf, struct seq_file *m) 4118{ 4119 unsigned long total; 4120 unsigned long entries; 4121 4122 get_total_entries(buf, &total, &entries); 4123 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n", 4124 entries, total, num_online_cpus()); 4125 seq_puts(m, "#\n"); 4126} 4127 4128static void print_func_help_header(struct array_buffer *buf, struct seq_file *m, 4129 unsigned int flags) 4130{ 4131 bool tgid = flags & TRACE_ITER(RECORD_TGID); 4132 4133 print_event_info(buf, m); 4134 4135 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? " TGID " : ""); 4136 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : ""); 4137} 4138 4139static void print_func_help_header_irq(struct array_buffer *buf, struct seq_file *m, 4140 unsigned int flags) 4141{ 4142 bool tgid = flags & TRACE_ITER(RECORD_TGID); 4143 static const char space[] = " "; 4144 int prec = tgid ? 12 : 2; 4145 4146 print_event_info(buf, m); 4147 4148 seq_printf(m, "# %.*s _-----=> irqs-off/BH-disabled\n", prec, space); 4149 seq_printf(m, "# %.*s / _----=> need-resched\n", prec, space); 4150 seq_printf(m, "# %.*s| / _---=> hardirq/softirq\n", prec, space); 4151 seq_printf(m, "# %.*s|| / _--=> preempt-depth\n", prec, space); 4152 seq_printf(m, "# %.*s||| / _-=> migrate-disable\n", prec, space); 4153 seq_printf(m, "# %.*s|||| / delay\n", prec, space); 4154 seq_printf(m, "# TASK-PID %.*s CPU# ||||| TIMESTAMP FUNCTION\n", prec, " TGID "); 4155 seq_printf(m, "# | | %.*s | ||||| | |\n", prec, " | "); 4156} 4157 4158void 4159print_trace_header(struct seq_file *m, struct trace_iterator *iter) 4160{ 4161 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK); 4162 struct array_buffer *buf = iter->array_buffer; 4163 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu); 4164 struct tracer *type = iter->trace; 4165 unsigned long entries; 4166 unsigned long total; 4167 const char *name = type->name; 4168 4169 get_total_entries(buf, &total, &entries); 4170 4171 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 4172 name, init_utsname()->release); 4173 seq_puts(m, "# -----------------------------------" 4174 "---------------------------------\n"); 4175 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 4176 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 4177 nsecs_to_usecs(data->saved_latency), 4178 entries, 4179 total, 4180 buf->cpu, 4181 preempt_model_str(), 4182 /* These are reserved for later use */ 4183 0, 0, 0, 0); 4184#ifdef CONFIG_SMP 4185 seq_printf(m, " #P:%d)\n", num_online_cpus()); 4186#else 4187 seq_puts(m, ")\n"); 4188#endif 4189 seq_puts(m, "# -----------------\n"); 4190 seq_printf(m, "# | task: %.16s-%d " 4191 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 4192 data->comm, data->pid, 4193 from_kuid_munged(seq_user_ns(m), data->uid), data->nice, 4194 data->policy, data->rt_priority); 4195 seq_puts(m, "# -----------------\n"); 4196 4197 if (data->critical_start) { 4198 seq_puts(m, "# => started at: "); 4199 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 4200 trace_print_seq(m, &iter->seq); 4201 seq_puts(m, "\n# => ended at: "); 4202 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 4203 trace_print_seq(m, &iter->seq); 4204 seq_puts(m, "\n#\n"); 4205 } 4206 4207 seq_puts(m, "#\n"); 4208} 4209 4210static void test_cpu_buff_start(struct trace_iterator *iter) 4211{ 4212 struct trace_seq *s = &iter->seq; 4213 struct trace_array *tr = iter->tr; 4214 4215 if (!(tr->trace_flags & TRACE_ITER(ANNOTATE))) 4216 return; 4217 4218 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 4219 return; 4220 4221 if (cpumask_available(iter->started) && 4222 cpumask_test_cpu(iter->cpu, iter->started)) 4223 return; 4224 4225 if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries) 4226 return; 4227 4228 if (cpumask_available(iter->started)) 4229 cpumask_set_cpu(iter->cpu, iter->started); 4230 4231 /* Don't print started cpu buffer for the first entry of the trace */ 4232 if (iter->idx > 1) 4233 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 4234 iter->cpu); 4235} 4236 4237#ifdef CONFIG_FTRACE_SYSCALLS 4238static bool is_syscall_event(struct trace_event *event) 4239{ 4240 return (event->funcs == &enter_syscall_print_funcs) || 4241 (event->funcs == &exit_syscall_print_funcs); 4242 4243} 4244#define syscall_buf_size CONFIG_TRACE_SYSCALL_BUF_SIZE_DEFAULT 4245#else 4246static inline bool is_syscall_event(struct trace_event *event) 4247{ 4248 return false; 4249} 4250#define syscall_buf_size 0 4251#endif /* CONFIG_FTRACE_SYSCALLS */ 4252 4253static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 4254{ 4255 struct trace_array *tr = iter->tr; 4256 struct trace_seq *s = &iter->seq; 4257 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); 4258 struct trace_entry *entry; 4259 struct trace_event *event; 4260 4261 entry = iter->ent; 4262 4263 test_cpu_buff_start(iter); 4264 4265 event = ftrace_find_event(entry->type); 4266 4267 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) { 4268 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 4269 trace_print_lat_context(iter); 4270 else 4271 trace_print_context(iter); 4272 } 4273 4274 if (trace_seq_has_overflowed(s)) 4275 return TRACE_TYPE_PARTIAL_LINE; 4276 4277 if (event) { 4278 if (tr->trace_flags & TRACE_ITER(FIELDS)) 4279 return print_event_fields(iter, event); 4280 /* 4281 * For TRACE_EVENT() events, the print_fmt is not 4282 * safe to use if the array has delta offsets 4283 * Force printing via the fields. 4284 */ 4285 if ((tr->text_delta)) { 4286 /* ftrace and system call events are still OK */ 4287 if ((event->type > __TRACE_LAST_TYPE) && 4288 !is_syscall_event(event)) 4289 return print_event_fields(iter, event); 4290 } 4291 return event->funcs->trace(iter, sym_flags, event); 4292 } 4293 4294 trace_seq_printf(s, "Unknown type %d\n", entry->type); 4295 4296 return trace_handle_return(s); 4297} 4298 4299static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 4300{ 4301 struct trace_array *tr = iter->tr; 4302 struct trace_seq *s = &iter->seq; 4303 struct trace_entry *entry; 4304 struct trace_event *event; 4305 4306 entry = iter->ent; 4307 4308 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) 4309 trace_seq_printf(s, "%d %d %llu ", 4310 entry->pid, iter->cpu, iter->ts); 4311 4312 if (trace_seq_has_overflowed(s)) 4313 return TRACE_TYPE_PARTIAL_LINE; 4314 4315 event = ftrace_find_event(entry->type); 4316 if (event) 4317 return event->funcs->raw(iter, 0, event); 4318 4319 trace_seq_printf(s, "%d ?\n", entry->type); 4320 4321 return trace_handle_return(s); 4322} 4323 4324static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 4325{ 4326 struct trace_array *tr = iter->tr; 4327 struct trace_seq *s = &iter->seq; 4328 unsigned char newline = '\n'; 4329 struct trace_entry *entry; 4330 struct trace_event *event; 4331 4332 entry = iter->ent; 4333 4334 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) { 4335 SEQ_PUT_HEX_FIELD(s, entry->pid); 4336 SEQ_PUT_HEX_FIELD(s, iter->cpu); 4337 SEQ_PUT_HEX_FIELD(s, iter->ts); 4338 if (trace_seq_has_overflowed(s)) 4339 return TRACE_TYPE_PARTIAL_LINE; 4340 } 4341 4342 event = ftrace_find_event(entry->type); 4343 if (event) { 4344 enum print_line_t ret = event->funcs->hex(iter, 0, event); 4345 if (ret != TRACE_TYPE_HANDLED) 4346 return ret; 4347 } 4348 4349 SEQ_PUT_FIELD(s, newline); 4350 4351 return trace_handle_return(s); 4352} 4353 4354static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 4355{ 4356 struct trace_array *tr = iter->tr; 4357 struct trace_seq *s = &iter->seq; 4358 struct trace_entry *entry; 4359 struct trace_event *event; 4360 4361 entry = iter->ent; 4362 4363 if (tr->trace_flags & TRACE_ITER(CONTEXT_INFO)) { 4364 SEQ_PUT_FIELD(s, entry->pid); 4365 SEQ_PUT_FIELD(s, iter->cpu); 4366 SEQ_PUT_FIELD(s, iter->ts); 4367 if (trace_seq_has_overflowed(s)) 4368 return TRACE_TYPE_PARTIAL_LINE; 4369 } 4370 4371 event = ftrace_find_event(entry->type); 4372 return event ? event->funcs->binary(iter, 0, event) : 4373 TRACE_TYPE_HANDLED; 4374} 4375 4376int trace_empty(struct trace_iterator *iter) 4377{ 4378 struct ring_buffer_iter *buf_iter; 4379 int cpu; 4380 4381 /* If we are looking at one CPU buffer, only check that one */ 4382 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) { 4383 cpu = iter->cpu_file; 4384 buf_iter = trace_buffer_iter(iter, cpu); 4385 if (buf_iter) { 4386 if (!ring_buffer_iter_empty(buf_iter)) 4387 return 0; 4388 } else { 4389 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) 4390 return 0; 4391 } 4392 return 1; 4393 } 4394 4395 for_each_tracing_cpu(cpu) { 4396 buf_iter = trace_buffer_iter(iter, cpu); 4397 if (buf_iter) { 4398 if (!ring_buffer_iter_empty(buf_iter)) 4399 return 0; 4400 } else { 4401 if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu)) 4402 return 0; 4403 } 4404 } 4405 4406 return 1; 4407} 4408 4409/* Called with trace_event_read_lock() held. */ 4410enum print_line_t print_trace_line(struct trace_iterator *iter) 4411{ 4412 struct trace_array *tr = iter->tr; 4413 unsigned long trace_flags = tr->trace_flags; 4414 enum print_line_t ret; 4415 4416 if (iter->lost_events) { 4417 if (iter->lost_events == (unsigned long)-1) 4418 trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n", 4419 iter->cpu); 4420 else 4421 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 4422 iter->cpu, iter->lost_events); 4423 if (trace_seq_has_overflowed(&iter->seq)) 4424 return TRACE_TYPE_PARTIAL_LINE; 4425 } 4426 4427 if (iter->trace && iter->trace->print_line) { 4428 ret = iter->trace->print_line(iter); 4429 if (ret != TRACE_TYPE_UNHANDLED) 4430 return ret; 4431 } 4432 4433 if (iter->ent->type == TRACE_BPUTS && 4434 trace_flags & TRACE_ITER(PRINTK) && 4435 trace_flags & TRACE_ITER(PRINTK_MSGONLY)) 4436 return trace_print_bputs_msg_only(iter); 4437 4438 if (iter->ent->type == TRACE_BPRINT && 4439 trace_flags & TRACE_ITER(PRINTK) && 4440 trace_flags & TRACE_ITER(PRINTK_MSGONLY)) 4441 return trace_print_bprintk_msg_only(iter); 4442 4443 if (iter->ent->type == TRACE_PRINT && 4444 trace_flags & TRACE_ITER(PRINTK) && 4445 trace_flags & TRACE_ITER(PRINTK_MSGONLY)) 4446 return trace_print_printk_msg_only(iter); 4447 4448 if (trace_flags & TRACE_ITER(BIN)) 4449 return print_bin_fmt(iter); 4450 4451 if (trace_flags & TRACE_ITER(HEX)) 4452 return print_hex_fmt(iter); 4453 4454 if (trace_flags & TRACE_ITER(RAW)) 4455 return print_raw_fmt(iter); 4456 4457 return print_trace_fmt(iter); 4458} 4459 4460void trace_latency_header(struct seq_file *m) 4461{ 4462 struct trace_iterator *iter = m->private; 4463 struct trace_array *tr = iter->tr; 4464 4465 /* print nothing if the buffers are empty */ 4466 if (trace_empty(iter)) 4467 return; 4468 4469 if (iter->iter_flags & TRACE_FILE_LAT_FMT) 4470 print_trace_header(m, iter); 4471 4472 if (!(tr->trace_flags & TRACE_ITER(VERBOSE))) 4473 print_lat_help_header(m); 4474} 4475 4476void trace_default_header(struct seq_file *m) 4477{ 4478 struct trace_iterator *iter = m->private; 4479 struct trace_array *tr = iter->tr; 4480 unsigned long trace_flags = tr->trace_flags; 4481 4482 if (!(trace_flags & TRACE_ITER(CONTEXT_INFO))) 4483 return; 4484 4485 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 4486 /* print nothing if the buffers are empty */ 4487 if (trace_empty(iter)) 4488 return; 4489 print_trace_header(m, iter); 4490 if (!(trace_flags & TRACE_ITER(VERBOSE))) 4491 print_lat_help_header(m); 4492 } else { 4493 if (!(trace_flags & TRACE_ITER(VERBOSE))) { 4494 if (trace_flags & TRACE_ITER(IRQ_INFO)) 4495 print_func_help_header_irq(iter->array_buffer, 4496 m, trace_flags); 4497 else 4498 print_func_help_header(iter->array_buffer, m, 4499 trace_flags); 4500 } 4501 } 4502} 4503 4504static void test_ftrace_alive(struct seq_file *m) 4505{ 4506 if (!ftrace_is_dead()) 4507 return; 4508 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n" 4509 "# MAY BE MISSING FUNCTION EVENTS\n"); 4510} 4511 4512#ifdef CONFIG_TRACER_MAX_TRACE 4513static void show_snapshot_main_help(struct seq_file *m) 4514{ 4515 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n" 4516 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 4517 "# Takes a snapshot of the main buffer.\n" 4518 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n" 4519 "# (Doesn't have to be '2' works with any number that\n" 4520 "# is not a '0' or '1')\n"); 4521} 4522 4523static void show_snapshot_percpu_help(struct seq_file *m) 4524{ 4525 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n"); 4526#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP 4527 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n" 4528 "# Takes a snapshot of the main buffer for this cpu.\n"); 4529#else 4530 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n" 4531 "# Must use main snapshot file to allocate.\n"); 4532#endif 4533 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n" 4534 "# (Doesn't have to be '2' works with any number that\n" 4535 "# is not a '0' or '1')\n"); 4536} 4537 4538static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) 4539{ 4540 if (iter->tr->allocated_snapshot) 4541 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n"); 4542 else 4543 seq_puts(m, "#\n# * Snapshot is freed *\n#\n"); 4544 4545 seq_puts(m, "# Snapshot commands:\n"); 4546 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 4547 show_snapshot_main_help(m); 4548 else 4549 show_snapshot_percpu_help(m); 4550} 4551#else 4552/* Should never be called */ 4553static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { } 4554#endif 4555 4556static int s_show(struct seq_file *m, void *v) 4557{ 4558 struct trace_iterator *iter = v; 4559 int ret; 4560 4561 if (iter->ent == NULL) { 4562 if (iter->tr) { 4563 seq_printf(m, "# tracer: %s\n", iter->trace->name); 4564 seq_puts(m, "#\n"); 4565 test_ftrace_alive(m); 4566 } 4567 if (iter->snapshot && trace_empty(iter)) 4568 print_snapshot_help(m, iter); 4569 else if (iter->trace && iter->trace->print_header) 4570 iter->trace->print_header(m); 4571 else 4572 trace_default_header(m); 4573 4574 } else if (iter->leftover) { 4575 /* 4576 * If we filled the seq_file buffer earlier, we 4577 * want to just show it now. 4578 */ 4579 ret = trace_print_seq(m, &iter->seq); 4580 4581 /* ret should this time be zero, but you never know */ 4582 iter->leftover = ret; 4583 4584 } else { 4585 ret = print_trace_line(iter); 4586 if (ret == TRACE_TYPE_PARTIAL_LINE) { 4587 iter->seq.full = 0; 4588 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n"); 4589 } 4590 ret = trace_print_seq(m, &iter->seq); 4591 /* 4592 * If we overflow the seq_file buffer, then it will 4593 * ask us for this data again at start up. 4594 * Use that instead. 4595 * ret is 0 if seq_file write succeeded. 4596 * -1 otherwise. 4597 */ 4598 iter->leftover = ret; 4599 } 4600 4601 return 0; 4602} 4603 4604/* 4605 * Should be used after trace_array_get(), trace_types_lock 4606 * ensures that i_cdev was already initialized. 4607 */ 4608static inline int tracing_get_cpu(struct inode *inode) 4609{ 4610 if (inode->i_cdev) /* See trace_create_cpu_file() */ 4611 return (long)inode->i_cdev - 1; 4612 return RING_BUFFER_ALL_CPUS; 4613} 4614 4615static const struct seq_operations tracer_seq_ops = { 4616 .start = s_start, 4617 .next = s_next, 4618 .stop = s_stop, 4619 .show = s_show, 4620}; 4621 4622/* 4623 * Note, as iter itself can be allocated and freed in different 4624 * ways, this function is only used to free its content, and not 4625 * the iterator itself. The only requirement to all the allocations 4626 * is that it must zero all fields (kzalloc), as freeing works with 4627 * ethier allocated content or NULL. 4628 */ 4629static void free_trace_iter_content(struct trace_iterator *iter) 4630{ 4631 /* The fmt is either NULL, allocated or points to static_fmt_buf */ 4632 if (iter->fmt != static_fmt_buf) 4633 kfree(iter->fmt); 4634 4635 kfree(iter->temp); 4636 kfree(iter->buffer_iter); 4637 mutex_destroy(&iter->mutex); 4638 free_cpumask_var(iter->started); 4639} 4640 4641static struct trace_iterator * 4642__tracing_open(struct inode *inode, struct file *file, bool snapshot) 4643{ 4644 struct trace_array *tr = inode->i_private; 4645 struct trace_iterator *iter; 4646 int cpu; 4647 4648 if (tracing_disabled) 4649 return ERR_PTR(-ENODEV); 4650 4651 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter)); 4652 if (!iter) 4653 return ERR_PTR(-ENOMEM); 4654 4655 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter), 4656 GFP_KERNEL); 4657 if (!iter->buffer_iter) 4658 goto release; 4659 4660 /* 4661 * trace_find_next_entry() may need to save off iter->ent. 4662 * It will place it into the iter->temp buffer. As most 4663 * events are less than 128, allocate a buffer of that size. 4664 * If one is greater, then trace_find_next_entry() will 4665 * allocate a new buffer to adjust for the bigger iter->ent. 4666 * It's not critical if it fails to get allocated here. 4667 */ 4668 iter->temp = kmalloc(128, GFP_KERNEL); 4669 if (iter->temp) 4670 iter->temp_size = 128; 4671 4672 /* 4673 * trace_event_printf() may need to modify given format 4674 * string to replace %p with %px so that it shows real address 4675 * instead of hash value. However, that is only for the event 4676 * tracing, other tracer may not need. Defer the allocation 4677 * until it is needed. 4678 */ 4679 iter->fmt = NULL; 4680 iter->fmt_size = 0; 4681 4682 mutex_lock(&trace_types_lock); 4683 iter->trace = tr->current_trace; 4684 4685 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 4686 goto fail; 4687 4688 iter->tr = tr; 4689 4690#ifdef CONFIG_TRACER_MAX_TRACE 4691 /* Currently only the top directory has a snapshot */ 4692 if (tr->current_trace->print_max || snapshot) 4693 iter->array_buffer = &tr->max_buffer; 4694 else 4695#endif 4696 iter->array_buffer = &tr->array_buffer; 4697 iter->snapshot = snapshot; 4698 iter->pos = -1; 4699 iter->cpu_file = tracing_get_cpu(inode); 4700 mutex_init(&iter->mutex); 4701 4702 /* Notify the tracer early; before we stop tracing. */ 4703 if (iter->trace->open) 4704 iter->trace->open(iter); 4705 4706 /* Annotate start of buffers if we had overruns */ 4707 if (ring_buffer_overruns(iter->array_buffer->buffer)) 4708 iter->iter_flags |= TRACE_FILE_ANNOTATE; 4709 4710 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 4711 if (trace_clocks[tr->clock_id].in_ns) 4712 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 4713 4714 /* 4715 * If pause-on-trace is enabled, then stop the trace while 4716 * dumping, unless this is the "snapshot" file 4717 */ 4718 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER(PAUSE_ON_TRACE))) { 4719 iter->iter_flags |= TRACE_FILE_PAUSE; 4720 tracing_stop_tr(tr); 4721 } 4722 4723 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 4724 for_each_tracing_cpu(cpu) { 4725 iter->buffer_iter[cpu] = 4726 ring_buffer_read_start(iter->array_buffer->buffer, 4727 cpu, GFP_KERNEL); 4728 tracing_iter_reset(iter, cpu); 4729 } 4730 } else { 4731 cpu = iter->cpu_file; 4732 iter->buffer_iter[cpu] = 4733 ring_buffer_read_start(iter->array_buffer->buffer, 4734 cpu, GFP_KERNEL); 4735 tracing_iter_reset(iter, cpu); 4736 } 4737 4738 mutex_unlock(&trace_types_lock); 4739 4740 return iter; 4741 4742 fail: 4743 mutex_unlock(&trace_types_lock); 4744 free_trace_iter_content(iter); 4745release: 4746 seq_release_private(inode, file); 4747 return ERR_PTR(-ENOMEM); 4748} 4749 4750int tracing_open_generic(struct inode *inode, struct file *filp) 4751{ 4752 int ret; 4753 4754 ret = tracing_check_open_get_tr(NULL); 4755 if (ret) 4756 return ret; 4757 4758 filp->private_data = inode->i_private; 4759 return 0; 4760} 4761 4762bool tracing_is_disabled(void) 4763{ 4764 return (tracing_disabled) ? true: false; 4765} 4766 4767/* 4768 * Open and update trace_array ref count. 4769 * Must have the current trace_array passed to it. 4770 */ 4771int tracing_open_generic_tr(struct inode *inode, struct file *filp) 4772{ 4773 struct trace_array *tr = inode->i_private; 4774 int ret; 4775 4776 ret = tracing_check_open_get_tr(tr); 4777 if (ret) 4778 return ret; 4779 4780 filp->private_data = inode->i_private; 4781 4782 return 0; 4783} 4784 4785/* 4786 * The private pointer of the inode is the trace_event_file. 4787 * Update the tr ref count associated to it. 4788 */ 4789int tracing_open_file_tr(struct inode *inode, struct file *filp) 4790{ 4791 struct trace_event_file *file = inode->i_private; 4792 int ret; 4793 4794 ret = tracing_check_open_get_tr(file->tr); 4795 if (ret) 4796 return ret; 4797 4798 guard(mutex)(&event_mutex); 4799 4800 /* Fail if the file is marked for removal */ 4801 if (file->flags & EVENT_FILE_FL_FREED) { 4802 trace_array_put(file->tr); 4803 return -ENODEV; 4804 } else { 4805 event_file_get(file); 4806 } 4807 4808 filp->private_data = inode->i_private; 4809 4810 return 0; 4811} 4812 4813int tracing_release_file_tr(struct inode *inode, struct file *filp) 4814{ 4815 struct trace_event_file *file = inode->i_private; 4816 4817 trace_array_put(file->tr); 4818 event_file_put(file); 4819 4820 return 0; 4821} 4822 4823int tracing_single_release_file_tr(struct inode *inode, struct file *filp) 4824{ 4825 tracing_release_file_tr(inode, filp); 4826 return single_release(inode, filp); 4827} 4828 4829static int tracing_release(struct inode *inode, struct file *file) 4830{ 4831 struct trace_array *tr = inode->i_private; 4832 struct seq_file *m = file->private_data; 4833 struct trace_iterator *iter; 4834 int cpu; 4835 4836 if (!(file->f_mode & FMODE_READ)) { 4837 trace_array_put(tr); 4838 return 0; 4839 } 4840 4841 /* Writes do not use seq_file */ 4842 iter = m->private; 4843 mutex_lock(&trace_types_lock); 4844 4845 for_each_tracing_cpu(cpu) { 4846 if (iter->buffer_iter[cpu]) 4847 ring_buffer_read_finish(iter->buffer_iter[cpu]); 4848 } 4849 4850 if (iter->trace && iter->trace->close) 4851 iter->trace->close(iter); 4852 4853 if (iter->iter_flags & TRACE_FILE_PAUSE) 4854 /* reenable tracing if it was previously enabled */ 4855 tracing_start_tr(tr); 4856 4857 __trace_array_put(tr); 4858 4859 mutex_unlock(&trace_types_lock); 4860 4861 free_trace_iter_content(iter); 4862 seq_release_private(inode, file); 4863 4864 return 0; 4865} 4866 4867int tracing_release_generic_tr(struct inode *inode, struct file *file) 4868{ 4869 struct trace_array *tr = inode->i_private; 4870 4871 trace_array_put(tr); 4872 return 0; 4873} 4874 4875static int tracing_single_release_tr(struct inode *inode, struct file *file) 4876{ 4877 struct trace_array *tr = inode->i_private; 4878 4879 trace_array_put(tr); 4880 4881 return single_release(inode, file); 4882} 4883 4884static bool update_last_data_if_empty(struct trace_array *tr); 4885 4886static int tracing_open(struct inode *inode, struct file *file) 4887{ 4888 struct trace_array *tr = inode->i_private; 4889 struct trace_iterator *iter; 4890 int ret; 4891 4892 ret = tracing_check_open_get_tr(tr); 4893 if (ret) 4894 return ret; 4895 4896 /* If this file was open for write, then erase contents */ 4897 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) { 4898 int cpu = tracing_get_cpu(inode); 4899 struct array_buffer *trace_buf = &tr->array_buffer; 4900 4901#ifdef CONFIG_TRACER_MAX_TRACE 4902 if (tr->current_trace->print_max) 4903 trace_buf = &tr->max_buffer; 4904#endif 4905 4906 if (cpu == RING_BUFFER_ALL_CPUS) 4907 tracing_reset_online_cpus(trace_buf); 4908 else 4909 tracing_reset_cpu(trace_buf, cpu); 4910 4911 update_last_data_if_empty(tr); 4912 } 4913 4914 if (file->f_mode & FMODE_READ) { 4915 iter = __tracing_open(inode, file, false); 4916 if (IS_ERR(iter)) 4917 ret = PTR_ERR(iter); 4918 else if (tr->trace_flags & TRACE_ITER(LATENCY_FMT)) 4919 iter->iter_flags |= TRACE_FILE_LAT_FMT; 4920 } 4921 4922 if (ret < 0) 4923 trace_array_put(tr); 4924 4925 return ret; 4926} 4927 4928/* 4929 * Some tracers are not suitable for instance buffers. 4930 * A tracer is always available for the global array (toplevel) 4931 * or if it explicitly states that it is. 4932 */ 4933static bool 4934trace_ok_for_array(struct tracer *t, struct trace_array *tr) 4935{ 4936#ifdef CONFIG_TRACER_SNAPSHOT 4937 /* arrays with mapped buffer range do not have snapshots */ 4938 if (tr->range_addr_start && t->use_max_tr) 4939 return false; 4940#endif 4941 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; 4942} 4943 4944/* Find the next tracer that this trace array may use */ 4945static struct tracer * 4946get_tracer_for_array(struct trace_array *tr, struct tracer *t) 4947{ 4948 while (t && !trace_ok_for_array(t, tr)) 4949 t = t->next; 4950 4951 return t; 4952} 4953 4954static void * 4955t_next(struct seq_file *m, void *v, loff_t *pos) 4956{ 4957 struct trace_array *tr = m->private; 4958 struct tracer *t = v; 4959 4960 (*pos)++; 4961 4962 if (t) 4963 t = get_tracer_for_array(tr, t->next); 4964 4965 return t; 4966} 4967 4968static void *t_start(struct seq_file *m, loff_t *pos) 4969{ 4970 struct trace_array *tr = m->private; 4971 struct tracer *t; 4972 loff_t l = 0; 4973 4974 mutex_lock(&trace_types_lock); 4975 4976 t = get_tracer_for_array(tr, trace_types); 4977 for (; t && l < *pos; t = t_next(m, t, &l)) 4978 ; 4979 4980 return t; 4981} 4982 4983static void t_stop(struct seq_file *m, void *p) 4984{ 4985 mutex_unlock(&trace_types_lock); 4986} 4987 4988static int t_show(struct seq_file *m, void *v) 4989{ 4990 struct tracer *t = v; 4991 4992 if (!t) 4993 return 0; 4994 4995 seq_puts(m, t->name); 4996 if (t->next) 4997 seq_putc(m, ' '); 4998 else 4999 seq_putc(m, '\n'); 5000 5001 return 0; 5002} 5003 5004static const struct seq_operations show_traces_seq_ops = { 5005 .start = t_start, 5006 .next = t_next, 5007 .stop = t_stop, 5008 .show = t_show, 5009}; 5010 5011static int show_traces_open(struct inode *inode, struct file *file) 5012{ 5013 struct trace_array *tr = inode->i_private; 5014 struct seq_file *m; 5015 int ret; 5016 5017 ret = tracing_check_open_get_tr(tr); 5018 if (ret) 5019 return ret; 5020 5021 ret = seq_open(file, &show_traces_seq_ops); 5022 if (ret) { 5023 trace_array_put(tr); 5024 return ret; 5025 } 5026 5027 m = file->private_data; 5028 m->private = tr; 5029 5030 return 0; 5031} 5032 5033static int tracing_seq_release(struct inode *inode, struct file *file) 5034{ 5035 struct trace_array *tr = inode->i_private; 5036 5037 trace_array_put(tr); 5038 return seq_release(inode, file); 5039} 5040 5041static ssize_t 5042tracing_write_stub(struct file *filp, const char __user *ubuf, 5043 size_t count, loff_t *ppos) 5044{ 5045 return count; 5046} 5047 5048loff_t tracing_lseek(struct file *file, loff_t offset, int whence) 5049{ 5050 int ret; 5051 5052 if (file->f_mode & FMODE_READ) 5053 ret = seq_lseek(file, offset, whence); 5054 else 5055 file->f_pos = ret = 0; 5056 5057 return ret; 5058} 5059 5060static const struct file_operations tracing_fops = { 5061 .open = tracing_open, 5062 .read = seq_read, 5063 .read_iter = seq_read_iter, 5064 .splice_read = copy_splice_read, 5065 .write = tracing_write_stub, 5066 .llseek = tracing_lseek, 5067 .release = tracing_release, 5068}; 5069 5070static const struct file_operations show_traces_fops = { 5071 .open = show_traces_open, 5072 .read = seq_read, 5073 .llseek = seq_lseek, 5074 .release = tracing_seq_release, 5075}; 5076 5077static ssize_t 5078tracing_cpumask_read(struct file *filp, char __user *ubuf, 5079 size_t count, loff_t *ppos) 5080{ 5081 struct trace_array *tr = file_inode(filp)->i_private; 5082 char *mask_str __free(kfree) = NULL; 5083 int len; 5084 5085 len = snprintf(NULL, 0, "%*pb\n", 5086 cpumask_pr_args(tr->tracing_cpumask)) + 1; 5087 mask_str = kmalloc(len, GFP_KERNEL); 5088 if (!mask_str) 5089 return -ENOMEM; 5090 5091 len = snprintf(mask_str, len, "%*pb\n", 5092 cpumask_pr_args(tr->tracing_cpumask)); 5093 if (len >= count) 5094 return -EINVAL; 5095 5096 return simple_read_from_buffer(ubuf, count, ppos, mask_str, len); 5097} 5098 5099int tracing_set_cpumask(struct trace_array *tr, 5100 cpumask_var_t tracing_cpumask_new) 5101{ 5102 int cpu; 5103 5104 if (!tr) 5105 return -EINVAL; 5106 5107 local_irq_disable(); 5108 arch_spin_lock(&tr->max_lock); 5109 for_each_tracing_cpu(cpu) { 5110 /* 5111 * Increase/decrease the disabled counter if we are 5112 * about to flip a bit in the cpumask: 5113 */ 5114 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && 5115 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 5116 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu); 5117#ifdef CONFIG_TRACER_MAX_TRACE 5118 ring_buffer_record_disable_cpu(tr->max_buffer.buffer, cpu); 5119#endif 5120 } 5121 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && 5122 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 5123 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu); 5124#ifdef CONFIG_TRACER_MAX_TRACE 5125 ring_buffer_record_enable_cpu(tr->max_buffer.buffer, cpu); 5126#endif 5127 } 5128 } 5129 arch_spin_unlock(&tr->max_lock); 5130 local_irq_enable(); 5131 5132 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); 5133 5134 return 0; 5135} 5136 5137static ssize_t 5138tracing_cpumask_write(struct file *filp, const char __user *ubuf, 5139 size_t count, loff_t *ppos) 5140{ 5141 struct trace_array *tr = file_inode(filp)->i_private; 5142 cpumask_var_t tracing_cpumask_new; 5143 int err; 5144 5145 if (count == 0 || count > KMALLOC_MAX_SIZE) 5146 return -EINVAL; 5147 5148 if (!zalloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 5149 return -ENOMEM; 5150 5151 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 5152 if (err) 5153 goto err_free; 5154 5155 err = tracing_set_cpumask(tr, tracing_cpumask_new); 5156 if (err) 5157 goto err_free; 5158 5159 free_cpumask_var(tracing_cpumask_new); 5160 5161 return count; 5162 5163err_free: 5164 free_cpumask_var(tracing_cpumask_new); 5165 5166 return err; 5167} 5168 5169static const struct file_operations tracing_cpumask_fops = { 5170 .open = tracing_open_generic_tr, 5171 .read = tracing_cpumask_read, 5172 .write = tracing_cpumask_write, 5173 .release = tracing_release_generic_tr, 5174 .llseek = generic_file_llseek, 5175}; 5176 5177static int tracing_trace_options_show(struct seq_file *m, void *v) 5178{ 5179 struct tracer_opt *trace_opts; 5180 struct trace_array *tr = m->private; 5181 struct tracer_flags *flags; 5182 u32 tracer_flags; 5183 int i; 5184 5185 guard(mutex)(&trace_types_lock); 5186 5187 for (i = 0; trace_options[i]; i++) { 5188 if (tr->trace_flags & (1ULL << i)) 5189 seq_printf(m, "%s\n", trace_options[i]); 5190 else 5191 seq_printf(m, "no%s\n", trace_options[i]); 5192 } 5193 5194 flags = tr->current_trace_flags; 5195 if (!flags || !flags->opts) 5196 return 0; 5197 5198 tracer_flags = flags->val; 5199 trace_opts = flags->opts; 5200 5201 for (i = 0; trace_opts[i].name; i++) { 5202 if (tracer_flags & trace_opts[i].bit) 5203 seq_printf(m, "%s\n", trace_opts[i].name); 5204 else 5205 seq_printf(m, "no%s\n", trace_opts[i].name); 5206 } 5207 5208 return 0; 5209} 5210 5211static int __set_tracer_option(struct trace_array *tr, 5212 struct tracer_flags *tracer_flags, 5213 struct tracer_opt *opts, int neg) 5214{ 5215 struct tracer *trace = tracer_flags->trace; 5216 int ret = 0; 5217 5218 if (trace->set_flag) 5219 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); 5220 if (ret) 5221 return ret; 5222 5223 if (neg) 5224 tracer_flags->val &= ~opts->bit; 5225 else 5226 tracer_flags->val |= opts->bit; 5227 return 0; 5228} 5229 5230/* Try to assign a tracer specific option */ 5231static int set_tracer_option(struct trace_array *tr, char *cmp, int neg) 5232{ 5233 struct tracer_flags *tracer_flags = tr->current_trace_flags; 5234 struct tracer_opt *opts = NULL; 5235 int i; 5236 5237 if (!tracer_flags || !tracer_flags->opts) 5238 return 0; 5239 5240 for (i = 0; tracer_flags->opts[i].name; i++) { 5241 opts = &tracer_flags->opts[i]; 5242 5243 if (strcmp(cmp, opts->name) == 0) 5244 return __set_tracer_option(tr, tracer_flags, opts, neg); 5245 } 5246 5247 return -EINVAL; 5248} 5249 5250/* Some tracers require overwrite to stay enabled */ 5251int trace_keep_overwrite(struct tracer *tracer, u64 mask, int set) 5252{ 5253 if (tracer->enabled && (mask & TRACE_ITER(OVERWRITE)) && !set) 5254 return -1; 5255 5256 return 0; 5257} 5258 5259int set_tracer_flag(struct trace_array *tr, u64 mask, int enabled) 5260{ 5261 switch (mask) { 5262 case TRACE_ITER(RECORD_TGID): 5263 case TRACE_ITER(RECORD_CMD): 5264 case TRACE_ITER(TRACE_PRINTK): 5265 case TRACE_ITER(COPY_MARKER): 5266 lockdep_assert_held(&event_mutex); 5267 } 5268 5269 /* do nothing if flag is already set */ 5270 if (!!(tr->trace_flags & mask) == !!enabled) 5271 return 0; 5272 5273 /* Give the tracer a chance to approve the change */ 5274 if (tr->current_trace->flag_changed) 5275 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) 5276 return -EINVAL; 5277 5278 switch (mask) { 5279 case TRACE_ITER(TRACE_PRINTK): 5280 if (enabled) { 5281 update_printk_trace(tr); 5282 } else { 5283 /* 5284 * The global_trace cannot clear this. 5285 * It's flag only gets cleared if another instance sets it. 5286 */ 5287 if (printk_trace == &global_trace) 5288 return -EINVAL; 5289 /* 5290 * An instance must always have it set. 5291 * by default, that's the global_trace instance. 5292 */ 5293 if (printk_trace == tr) 5294 update_printk_trace(&global_trace); 5295 } 5296 break; 5297 5298 case TRACE_ITER(COPY_MARKER): 5299 update_marker_trace(tr, enabled); 5300 /* update_marker_trace updates the tr->trace_flags */ 5301 return 0; 5302 } 5303 5304 if (enabled) 5305 tr->trace_flags |= mask; 5306 else 5307 tr->trace_flags &= ~mask; 5308 5309 switch (mask) { 5310 case TRACE_ITER(RECORD_CMD): 5311 trace_event_enable_cmd_record(enabled); 5312 break; 5313 5314 case TRACE_ITER(RECORD_TGID): 5315 5316 if (trace_alloc_tgid_map() < 0) { 5317 tr->trace_flags &= ~TRACE_ITER(RECORD_TGID); 5318 return -ENOMEM; 5319 } 5320 5321 trace_event_enable_tgid_record(enabled); 5322 break; 5323 5324 case TRACE_ITER(EVENT_FORK): 5325 trace_event_follow_fork(tr, enabled); 5326 break; 5327 5328 case TRACE_ITER(FUNC_FORK): 5329 ftrace_pid_follow_fork(tr, enabled); 5330 break; 5331 5332 case TRACE_ITER(OVERWRITE): 5333 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled); 5334#ifdef CONFIG_TRACER_MAX_TRACE 5335 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); 5336#endif 5337 break; 5338 5339 case TRACE_ITER(PRINTK): 5340 trace_printk_start_stop_comm(enabled); 5341 trace_printk_control(enabled); 5342 break; 5343 5344#if defined(CONFIG_FUNCTION_PROFILER) && defined(CONFIG_FUNCTION_GRAPH_TRACER) 5345 case TRACE_GRAPH_GRAPH_TIME: 5346 ftrace_graph_graph_time_control(enabled); 5347 break; 5348#endif 5349 } 5350 5351 return 0; 5352} 5353 5354int trace_set_options(struct trace_array *tr, char *option) 5355{ 5356 char *cmp; 5357 int neg = 0; 5358 int ret; 5359 size_t orig_len = strlen(option); 5360 int len; 5361 5362 cmp = strstrip(option); 5363 5364 len = str_has_prefix(cmp, "no"); 5365 if (len) 5366 neg = 1; 5367 5368 cmp += len; 5369 5370 mutex_lock(&event_mutex); 5371 mutex_lock(&trace_types_lock); 5372 5373 ret = match_string(trace_options, -1, cmp); 5374 /* If no option could be set, test the specific tracer options */ 5375 if (ret < 0) 5376 ret = set_tracer_option(tr, cmp, neg); 5377 else 5378 ret = set_tracer_flag(tr, 1ULL << ret, !neg); 5379 5380 mutex_unlock(&trace_types_lock); 5381 mutex_unlock(&event_mutex); 5382 5383 /* 5384 * If the first trailing whitespace is replaced with '\0' by strstrip, 5385 * turn it back into a space. 5386 */ 5387 if (orig_len > strlen(option)) 5388 option[strlen(option)] = ' '; 5389 5390 return ret; 5391} 5392 5393static void __init apply_trace_boot_options(void) 5394{ 5395 char *buf = trace_boot_options_buf; 5396 char *option; 5397 5398 while (true) { 5399 option = strsep(&buf, ","); 5400 5401 if (!option) 5402 break; 5403 5404 if (*option) 5405 trace_set_options(&global_trace, option); 5406 5407 /* Put back the comma to allow this to be called again */ 5408 if (buf) 5409 *(buf - 1) = ','; 5410 } 5411} 5412 5413static ssize_t 5414tracing_trace_options_write(struct file *filp, const char __user *ubuf, 5415 size_t cnt, loff_t *ppos) 5416{ 5417 struct seq_file *m = filp->private_data; 5418 struct trace_array *tr = m->private; 5419 char buf[64]; 5420 int ret; 5421 5422 if (cnt >= sizeof(buf)) 5423 return -EINVAL; 5424 5425 if (copy_from_user(buf, ubuf, cnt)) 5426 return -EFAULT; 5427 5428 buf[cnt] = 0; 5429 5430 ret = trace_set_options(tr, buf); 5431 if (ret < 0) 5432 return ret; 5433 5434 *ppos += cnt; 5435 5436 return cnt; 5437} 5438 5439static int tracing_trace_options_open(struct inode *inode, struct file *file) 5440{ 5441 struct trace_array *tr = inode->i_private; 5442 int ret; 5443 5444 ret = tracing_check_open_get_tr(tr); 5445 if (ret) 5446 return ret; 5447 5448 ret = single_open(file, tracing_trace_options_show, inode->i_private); 5449 if (ret < 0) 5450 trace_array_put(tr); 5451 5452 return ret; 5453} 5454 5455static const struct file_operations tracing_iter_fops = { 5456 .open = tracing_trace_options_open, 5457 .read = seq_read, 5458 .llseek = seq_lseek, 5459 .release = tracing_single_release_tr, 5460 .write = tracing_trace_options_write, 5461}; 5462 5463static const char readme_msg[] = 5464 "tracing mini-HOWTO:\n\n" 5465 "By default tracefs removes all OTH file permission bits.\n" 5466 "When mounting tracefs an optional group id can be specified\n" 5467 "which adds the group to every directory and file in tracefs:\n\n" 5468 "\t e.g. mount -t tracefs [-o [gid=<gid>]] nodev /sys/kernel/tracing\n\n" 5469 "# echo 0 > tracing_on : quick way to disable tracing\n" 5470 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n" 5471 " Important files:\n" 5472 " trace\t\t\t- The static contents of the buffer\n" 5473 "\t\t\t To clear the buffer write into this file: echo > trace\n" 5474 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n" 5475 " current_tracer\t- function and latency tracers\n" 5476 " available_tracers\t- list of configured tracers for current_tracer\n" 5477 " error_log\t- error log for failed commands (that support it)\n" 5478 " buffer_size_kb\t- view and modify size of per cpu buffer\n" 5479 " buffer_total_size_kb - view total size of all cpu buffers\n\n" 5480 " trace_clock\t\t- change the clock used to order events\n" 5481 " local: Per cpu clock but may not be synced across CPUs\n" 5482 " global: Synced across CPUs but slows tracing down.\n" 5483 " counter: Not a clock, but just an increment\n" 5484 " uptime: Jiffy counter from time of boot\n" 5485 " perf: Same clock that perf events use\n" 5486#ifdef CONFIG_X86_64 5487 " x86-tsc: TSC cycle counter\n" 5488#endif 5489 "\n timestamp_mode\t- view the mode used to timestamp events\n" 5490 " delta: Delta difference against a buffer-wide timestamp\n" 5491 " absolute: Absolute (standalone) timestamp\n" 5492 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n" 5493 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n" 5494 " tracing_cpumask\t- Limit which CPUs to trace\n" 5495 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n" 5496 "\t\t\t Remove sub-buffer with rmdir\n" 5497 " trace_options\t\t- Set format or modify how tracing happens\n" 5498 "\t\t\t Disable an option by prefixing 'no' to the\n" 5499 "\t\t\t option name\n" 5500 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n" 5501#ifdef CONFIG_DYNAMIC_FTRACE 5502 "\n available_filter_functions - list of functions that can be filtered on\n" 5503 " set_ftrace_filter\t- echo function name in here to only trace these\n" 5504 "\t\t\t functions\n" 5505 "\t accepts: func_full_name or glob-matching-pattern\n" 5506 "\t modules: Can select a group via module\n" 5507 "\t Format: :mod:<module-name>\n" 5508 "\t example: echo :mod:ext3 > set_ftrace_filter\n" 5509 "\t triggers: a command to perform when function is hit\n" 5510 "\t Format: <function>:<trigger>[:count]\n" 5511 "\t trigger: traceon, traceoff\n" 5512 "\t\t enable_event:<system>:<event>\n" 5513 "\t\t disable_event:<system>:<event>\n" 5514#ifdef CONFIG_STACKTRACE 5515 "\t\t stacktrace\n" 5516#endif 5517#ifdef CONFIG_TRACER_SNAPSHOT 5518 "\t\t snapshot\n" 5519#endif 5520 "\t\t dump\n" 5521 "\t\t cpudump\n" 5522 "\t example: echo do_fault:traceoff > set_ftrace_filter\n" 5523 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n" 5524 "\t The first one will disable tracing every time do_fault is hit\n" 5525 "\t The second will disable tracing at most 3 times when do_trap is hit\n" 5526 "\t The first time do trap is hit and it disables tracing, the\n" 5527 "\t counter will decrement to 2. If tracing is already disabled,\n" 5528 "\t the counter will not decrement. It only decrements when the\n" 5529 "\t trigger did work\n" 5530 "\t To remove trigger without count:\n" 5531 "\t echo '!<function>:<trigger> > set_ftrace_filter\n" 5532 "\t To remove trigger with a count:\n" 5533 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n" 5534 " set_ftrace_notrace\t- echo function name in here to never trace.\n" 5535 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n" 5536 "\t modules: Can select a group via module command :mod:\n" 5537 "\t Does not accept triggers\n" 5538#endif /* CONFIG_DYNAMIC_FTRACE */ 5539#ifdef CONFIG_FUNCTION_TRACER 5540 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n" 5541 "\t\t (function)\n" 5542 " set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n" 5543 "\t\t (function)\n" 5544#endif 5545#ifdef CONFIG_FUNCTION_GRAPH_TRACER 5546 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n" 5547 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n" 5548 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n" 5549#endif 5550#ifdef CONFIG_TRACER_SNAPSHOT 5551 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n" 5552 "\t\t\t snapshot buffer. Read the contents for more\n" 5553 "\t\t\t information\n" 5554#endif 5555#ifdef CONFIG_STACK_TRACER 5556 " stack_trace\t\t- Shows the max stack trace when active\n" 5557 " stack_max_size\t- Shows current max stack size that was traced\n" 5558 "\t\t\t Write into this file to reset the max size (trigger a\n" 5559 "\t\t\t new trace)\n" 5560#ifdef CONFIG_DYNAMIC_FTRACE 5561 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n" 5562 "\t\t\t traces\n" 5563#endif 5564#endif /* CONFIG_STACK_TRACER */ 5565#ifdef CONFIG_DYNAMIC_EVENTS 5566 " dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n" 5567 "\t\t\t Write into this file to define/undefine new trace events.\n" 5568#endif 5569#ifdef CONFIG_KPROBE_EVENTS 5570 " kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n" 5571 "\t\t\t Write into this file to define/undefine new trace events.\n" 5572#endif 5573#ifdef CONFIG_UPROBE_EVENTS 5574 " uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n" 5575 "\t\t\t Write into this file to define/undefine new trace events.\n" 5576#endif 5577#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) || \ 5578 defined(CONFIG_FPROBE_EVENTS) 5579 "\t accepts: event-definitions (one definition per line)\n" 5580#if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) 5581 "\t Format: p[:[<group>/][<event>]] <place> [<args>]\n" 5582 "\t r[maxactive][:[<group>/][<event>]] <place> [<args>]\n" 5583#endif 5584#ifdef CONFIG_FPROBE_EVENTS 5585 "\t f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n" 5586 "\t t[:[<group>/][<event>]] <tracepoint> [<args>]\n" 5587#endif 5588#ifdef CONFIG_HIST_TRIGGERS 5589 "\t s:[synthetic/]<event> <field> [<field>]\n" 5590#endif 5591 "\t e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n" 5592 "\t -:[<group>/][<event>]\n" 5593#ifdef CONFIG_KPROBE_EVENTS 5594 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n" 5595 "place (kretprobe): [<module>:]<symbol>[+<offset>]%return|<memaddr>\n" 5596#endif 5597#ifdef CONFIG_UPROBE_EVENTS 5598 " place (uprobe): <path>:<offset>[%return][(ref_ctr_offset)]\n" 5599#endif 5600 "\t args: <name>=fetcharg[:type]\n" 5601 "\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n" 5602#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API 5603 "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n" 5604#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS 5605 "\t <argname>[->field[->field|.field...]],\n" 5606#endif 5607#else 5608 "\t $stack<index>, $stack, $retval, $comm,\n" 5609#endif 5610 "\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n" 5611 "\t kernel return probes support: $retval, $arg<N>, $comm\n" 5612 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n" 5613 "\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n" 5614 "\t symstr, %pd/%pD, <type>\\[<array-size>\\]\n" 5615#ifdef CONFIG_HIST_TRIGGERS 5616 "\t field: <stype> <name>;\n" 5617 "\t stype: u8/u16/u32/u64, s8/s16/s32/s64, pid_t,\n" 5618 "\t [unsigned] char/int/long\n" 5619#endif 5620 "\t efield: For event probes ('e' types), the field is on of the fields\n" 5621 "\t of the <attached-group>/<attached-event>.\n" 5622#endif 5623 " set_event\t\t- Enables events by name written into it\n" 5624 "\t\t\t Can enable module events via: :mod:<module>\n" 5625 " events/\t\t- Directory containing all trace event subsystems:\n" 5626 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n" 5627 " events/<system>/\t- Directory containing all trace events for <system>:\n" 5628 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n" 5629 "\t\t\t events\n" 5630 " filter\t\t- If set, only events passing filter are traced\n" 5631 " events/<system>/<event>/\t- Directory containing control files for\n" 5632 "\t\t\t <event>:\n" 5633 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n" 5634 " filter\t\t- If set, only events passing filter are traced\n" 5635 " trigger\t\t- If set, a command to perform when event is hit\n" 5636 "\t Format: <trigger>[:count][if <filter>]\n" 5637 "\t trigger: traceon, traceoff\n" 5638 "\t enable_event:<system>:<event>\n" 5639 "\t disable_event:<system>:<event>\n" 5640#ifdef CONFIG_HIST_TRIGGERS 5641 "\t enable_hist:<system>:<event>\n" 5642 "\t disable_hist:<system>:<event>\n" 5643#endif 5644#ifdef CONFIG_STACKTRACE 5645 "\t\t stacktrace\n" 5646#endif 5647#ifdef CONFIG_TRACER_SNAPSHOT 5648 "\t\t snapshot\n" 5649#endif 5650#ifdef CONFIG_HIST_TRIGGERS 5651 "\t\t hist (see below)\n" 5652#endif 5653 "\t example: echo traceoff > events/block/block_unplug/trigger\n" 5654 "\t echo traceoff:3 > events/block/block_unplug/trigger\n" 5655 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n" 5656 "\t events/block/block_unplug/trigger\n" 5657 "\t The first disables tracing every time block_unplug is hit.\n" 5658 "\t The second disables tracing the first 3 times block_unplug is hit.\n" 5659 "\t The third enables the kmalloc event the first 3 times block_unplug\n" 5660 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n" 5661 "\t Like function triggers, the counter is only decremented if it\n" 5662 "\t enabled or disabled tracing.\n" 5663 "\t To remove a trigger without a count:\n" 5664 "\t echo '!<trigger> > <system>/<event>/trigger\n" 5665 "\t To remove a trigger with a count:\n" 5666 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n" 5667 "\t Filters can be ignored when removing a trigger.\n" 5668#ifdef CONFIG_HIST_TRIGGERS 5669 " hist trigger\t- If set, event hits are aggregated into a hash table\n" 5670 "\t Format: hist:keys=<field1[,field2,...]>\n" 5671 "\t [:<var1>=<field|var_ref|numeric_literal>[,<var2>=...]]\n" 5672 "\t [:values=<field1[,field2,...]>]\n" 5673 "\t [:sort=<field1[,field2,...]>]\n" 5674 "\t [:size=#entries]\n" 5675 "\t [:pause][:continue][:clear]\n" 5676 "\t [:name=histname1]\n" 5677 "\t [:nohitcount]\n" 5678 "\t [:<handler>.<action>]\n" 5679 "\t [if <filter>]\n\n" 5680 "\t Note, special fields can be used as well:\n" 5681 "\t common_timestamp - to record current timestamp\n" 5682 "\t common_cpu - to record the CPU the event happened on\n" 5683 "\n" 5684 "\t A hist trigger variable can be:\n" 5685 "\t - a reference to a field e.g. x=current_timestamp,\n" 5686 "\t - a reference to another variable e.g. y=$x,\n" 5687 "\t - a numeric literal: e.g. ms_per_sec=1000,\n" 5688 "\t - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n" 5689 "\n" 5690 "\t hist trigger arithmetic expressions support addition(+), subtraction(-),\n" 5691 "\t multiplication(*) and division(/) operators. An operand can be either a\n" 5692 "\t variable reference, field or numeric literal.\n" 5693 "\n" 5694 "\t When a matching event is hit, an entry is added to a hash\n" 5695 "\t table using the key(s) and value(s) named, and the value of a\n" 5696 "\t sum called 'hitcount' is incremented. Keys and values\n" 5697 "\t correspond to fields in the event's format description. Keys\n" 5698 "\t can be any field, or the special string 'common_stacktrace'.\n" 5699 "\t Compound keys consisting of up to two fields can be specified\n" 5700 "\t by the 'keys' keyword. Values must correspond to numeric\n" 5701 "\t fields. Sort keys consisting of up to two fields can be\n" 5702 "\t specified using the 'sort' keyword. The sort direction can\n" 5703 "\t be modified by appending '.descending' or '.ascending' to a\n" 5704 "\t sort field. The 'size' parameter can be used to specify more\n" 5705 "\t or fewer than the default 2048 entries for the hashtable size.\n" 5706 "\t If a hist trigger is given a name using the 'name' parameter,\n" 5707 "\t its histogram data will be shared with other triggers of the\n" 5708 "\t same name, and trigger hits will update this common data.\n\n" 5709 "\t Reading the 'hist' file for the event will dump the hash\n" 5710 "\t table in its entirety to stdout. If there are multiple hist\n" 5711 "\t triggers attached to an event, there will be a table for each\n" 5712 "\t trigger in the output. The table displayed for a named\n" 5713 "\t trigger will be the same as any other instance having the\n" 5714 "\t same name. The default format used to display a given field\n" 5715 "\t can be modified by appending any of the following modifiers\n" 5716 "\t to the field name, as applicable:\n\n" 5717 "\t .hex display a number as a hex value\n" 5718 "\t .sym display an address as a symbol\n" 5719 "\t .sym-offset display an address as a symbol and offset\n" 5720 "\t .execname display a common_pid as a program name\n" 5721 "\t .syscall display a syscall id as a syscall name\n" 5722 "\t .log2 display log2 value rather than raw number\n" 5723 "\t .buckets=size display values in groups of size rather than raw number\n" 5724 "\t .usecs display a common_timestamp in microseconds\n" 5725 "\t .percent display a number of percentage value\n" 5726 "\t .graph display a bar-graph of a value\n\n" 5727 "\t The 'pause' parameter can be used to pause an existing hist\n" 5728 "\t trigger or to start a hist trigger but not log any events\n" 5729 "\t until told to do so. 'continue' can be used to start or\n" 5730 "\t restart a paused hist trigger.\n\n" 5731 "\t The 'clear' parameter will clear the contents of a running\n" 5732 "\t hist trigger and leave its current paused/active state\n" 5733 "\t unchanged.\n\n" 5734 "\t The 'nohitcount' (or NOHC) parameter will suppress display of\n" 5735 "\t raw hitcount in the histogram.\n\n" 5736 "\t The enable_hist and disable_hist triggers can be used to\n" 5737 "\t have one event conditionally start and stop another event's\n" 5738 "\t already-attached hist trigger. The syntax is analogous to\n" 5739 "\t the enable_event and disable_event triggers.\n\n" 5740 "\t Hist trigger handlers and actions are executed whenever a\n" 5741 "\t a histogram entry is added or updated. They take the form:\n\n" 5742 "\t <handler>.<action>\n\n" 5743 "\t The available handlers are:\n\n" 5744 "\t onmatch(matching.event) - invoke on addition or update\n" 5745 "\t onmax(var) - invoke if var exceeds current max\n" 5746 "\t onchange(var) - invoke action if var changes\n\n" 5747 "\t The available actions are:\n\n" 5748 "\t trace(<synthetic_event>,param list) - generate synthetic event\n" 5749 "\t save(field,...) - save current event fields\n" 5750#ifdef CONFIG_TRACER_SNAPSHOT 5751 "\t snapshot() - snapshot the trace buffer\n\n" 5752#endif 5753#ifdef CONFIG_SYNTH_EVENTS 5754 " events/synthetic_events\t- Create/append/remove/show synthetic events\n" 5755 "\t Write into this file to define/undefine new synthetic events.\n" 5756 "\t example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n" 5757#endif 5758#endif 5759; 5760 5761static ssize_t 5762tracing_readme_read(struct file *filp, char __user *ubuf, 5763 size_t cnt, loff_t *ppos) 5764{ 5765 return simple_read_from_buffer(ubuf, cnt, ppos, 5766 readme_msg, strlen(readme_msg)); 5767} 5768 5769static const struct file_operations tracing_readme_fops = { 5770 .open = tracing_open_generic, 5771 .read = tracing_readme_read, 5772 .llseek = generic_file_llseek, 5773}; 5774 5775#ifdef CONFIG_TRACE_EVAL_MAP_FILE 5776static union trace_eval_map_item * 5777update_eval_map(union trace_eval_map_item *ptr) 5778{ 5779 if (!ptr->map.eval_string) { 5780 if (ptr->tail.next) { 5781 ptr = ptr->tail.next; 5782 /* Set ptr to the next real item (skip head) */ 5783 ptr++; 5784 } else 5785 return NULL; 5786 } 5787 return ptr; 5788} 5789 5790static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos) 5791{ 5792 union trace_eval_map_item *ptr = v; 5793 5794 /* 5795 * Paranoid! If ptr points to end, we don't want to increment past it. 5796 * This really should never happen. 5797 */ 5798 (*pos)++; 5799 ptr = update_eval_map(ptr); 5800 if (WARN_ON_ONCE(!ptr)) 5801 return NULL; 5802 5803 ptr++; 5804 ptr = update_eval_map(ptr); 5805 5806 return ptr; 5807} 5808 5809static void *eval_map_start(struct seq_file *m, loff_t *pos) 5810{ 5811 union trace_eval_map_item *v; 5812 loff_t l = 0; 5813 5814 mutex_lock(&trace_eval_mutex); 5815 5816 v = trace_eval_maps; 5817 if (v) 5818 v++; 5819 5820 while (v && l < *pos) { 5821 v = eval_map_next(m, v, &l); 5822 } 5823 5824 return v; 5825} 5826 5827static void eval_map_stop(struct seq_file *m, void *v) 5828{ 5829 mutex_unlock(&trace_eval_mutex); 5830} 5831 5832static int eval_map_show(struct seq_file *m, void *v) 5833{ 5834 union trace_eval_map_item *ptr = v; 5835 5836 seq_printf(m, "%s %ld (%s)\n", 5837 ptr->map.eval_string, ptr->map.eval_value, 5838 ptr->map.system); 5839 5840 return 0; 5841} 5842 5843static const struct seq_operations tracing_eval_map_seq_ops = { 5844 .start = eval_map_start, 5845 .next = eval_map_next, 5846 .stop = eval_map_stop, 5847 .show = eval_map_show, 5848}; 5849 5850static int tracing_eval_map_open(struct inode *inode, struct file *filp) 5851{ 5852 int ret; 5853 5854 ret = tracing_check_open_get_tr(NULL); 5855 if (ret) 5856 return ret; 5857 5858 return seq_open(filp, &tracing_eval_map_seq_ops); 5859} 5860 5861static const struct file_operations tracing_eval_map_fops = { 5862 .open = tracing_eval_map_open, 5863 .read = seq_read, 5864 .llseek = seq_lseek, 5865 .release = seq_release, 5866}; 5867 5868static inline union trace_eval_map_item * 5869trace_eval_jmp_to_tail(union trace_eval_map_item *ptr) 5870{ 5871 /* Return tail of array given the head */ 5872 return ptr + ptr->head.length + 1; 5873} 5874 5875static void 5876trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start, 5877 int len) 5878{ 5879 struct trace_eval_map **stop; 5880 struct trace_eval_map **map; 5881 union trace_eval_map_item *map_array; 5882 union trace_eval_map_item *ptr; 5883 5884 stop = start + len; 5885 5886 /* 5887 * The trace_eval_maps contains the map plus a head and tail item, 5888 * where the head holds the module and length of array, and the 5889 * tail holds a pointer to the next list. 5890 */ 5891 map_array = kmalloc_array(len + 2, sizeof(*map_array), GFP_KERNEL); 5892 if (!map_array) { 5893 pr_warn("Unable to allocate trace eval mapping\n"); 5894 return; 5895 } 5896 5897 guard(mutex)(&trace_eval_mutex); 5898 5899 if (!trace_eval_maps) 5900 trace_eval_maps = map_array; 5901 else { 5902 ptr = trace_eval_maps; 5903 for (;;) { 5904 ptr = trace_eval_jmp_to_tail(ptr); 5905 if (!ptr->tail.next) 5906 break; 5907 ptr = ptr->tail.next; 5908 5909 } 5910 ptr->tail.next = map_array; 5911 } 5912 map_array->head.mod = mod; 5913 map_array->head.length = len; 5914 map_array++; 5915 5916 for (map = start; (unsigned long)map < (unsigned long)stop; map++) { 5917 map_array->map = **map; 5918 map_array++; 5919 } 5920 memset(map_array, 0, sizeof(*map_array)); 5921} 5922 5923static void trace_create_eval_file(struct dentry *d_tracer) 5924{ 5925 trace_create_file("eval_map", TRACE_MODE_READ, d_tracer, 5926 NULL, &tracing_eval_map_fops); 5927} 5928 5929#else /* CONFIG_TRACE_EVAL_MAP_FILE */ 5930static inline void trace_create_eval_file(struct dentry *d_tracer) { } 5931static inline void trace_insert_eval_map_file(struct module *mod, 5932 struct trace_eval_map **start, int len) { } 5933#endif /* !CONFIG_TRACE_EVAL_MAP_FILE */ 5934 5935static void 5936trace_event_update_with_eval_map(struct module *mod, 5937 struct trace_eval_map **start, 5938 int len) 5939{ 5940 struct trace_eval_map **map; 5941 5942 /* Always run sanitizer only if btf_type_tag attr exists. */ 5943 if (len <= 0) { 5944 if (!(IS_ENABLED(CONFIG_DEBUG_INFO_BTF) && 5945 IS_ENABLED(CONFIG_PAHOLE_HAS_BTF_TAG) && 5946 __has_attribute(btf_type_tag))) 5947 return; 5948 } 5949 5950 map = start; 5951 5952 trace_event_update_all(map, len); 5953 5954 if (len <= 0) 5955 return; 5956 5957 trace_insert_eval_map_file(mod, start, len); 5958} 5959 5960static ssize_t 5961tracing_set_trace_read(struct file *filp, char __user *ubuf, 5962 size_t cnt, loff_t *ppos) 5963{ 5964 struct trace_array *tr = filp->private_data; 5965 char buf[MAX_TRACER_SIZE+2]; 5966 int r; 5967 5968 scoped_guard(mutex, &trace_types_lock) { 5969 r = sprintf(buf, "%s\n", tr->current_trace->name); 5970 } 5971 5972 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 5973} 5974 5975int tracer_init(struct tracer *t, struct trace_array *tr) 5976{ 5977 tracing_reset_online_cpus(&tr->array_buffer); 5978 update_last_data_if_empty(tr); 5979 return t->init(tr); 5980} 5981 5982static void set_buffer_entries(struct array_buffer *buf, unsigned long val) 5983{ 5984 int cpu; 5985 5986 for_each_tracing_cpu(cpu) 5987 per_cpu_ptr(buf->data, cpu)->entries = val; 5988} 5989 5990static void update_buffer_entries(struct array_buffer *buf, int cpu) 5991{ 5992 if (cpu == RING_BUFFER_ALL_CPUS) { 5993 set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0)); 5994 } else { 5995 per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu); 5996 } 5997} 5998 5999#ifdef CONFIG_TRACER_MAX_TRACE 6000/* resize @tr's buffer to the size of @size_tr's entries */ 6001static int resize_buffer_duplicate_size(struct array_buffer *trace_buf, 6002 struct array_buffer *size_buf, int cpu_id) 6003{ 6004 int cpu, ret = 0; 6005 6006 if (cpu_id == RING_BUFFER_ALL_CPUS) { 6007 for_each_tracing_cpu(cpu) { 6008 ret = ring_buffer_resize(trace_buf->buffer, 6009 per_cpu_ptr(size_buf->data, cpu)->entries, cpu); 6010 if (ret < 0) 6011 break; 6012 per_cpu_ptr(trace_buf->data, cpu)->entries = 6013 per_cpu_ptr(size_buf->data, cpu)->entries; 6014 } 6015 } else { 6016 ret = ring_buffer_resize(trace_buf->buffer, 6017 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id); 6018 if (ret == 0) 6019 per_cpu_ptr(trace_buf->data, cpu_id)->entries = 6020 per_cpu_ptr(size_buf->data, cpu_id)->entries; 6021 } 6022 6023 return ret; 6024} 6025#endif /* CONFIG_TRACER_MAX_TRACE */ 6026 6027static int __tracing_resize_ring_buffer(struct trace_array *tr, 6028 unsigned long size, int cpu) 6029{ 6030 int ret; 6031 6032 /* 6033 * If kernel or user changes the size of the ring buffer 6034 * we use the size that was given, and we can forget about 6035 * expanding it later. 6036 */ 6037 trace_set_ring_buffer_expanded(tr); 6038 6039 /* May be called before buffers are initialized */ 6040 if (!tr->array_buffer.buffer) 6041 return 0; 6042 6043 /* Do not allow tracing while resizing ring buffer */ 6044 tracing_stop_tr(tr); 6045 6046 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu); 6047 if (ret < 0) 6048 goto out_start; 6049 6050#ifdef CONFIG_TRACER_MAX_TRACE 6051 if (!tr->allocated_snapshot) 6052 goto out; 6053 6054 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); 6055 if (ret < 0) { 6056 int r = resize_buffer_duplicate_size(&tr->array_buffer, 6057 &tr->array_buffer, cpu); 6058 if (r < 0) { 6059 /* 6060 * AARGH! We are left with different 6061 * size max buffer!!!! 6062 * The max buffer is our "snapshot" buffer. 6063 * When a tracer needs a snapshot (one of the 6064 * latency tracers), it swaps the max buffer 6065 * with the saved snap shot. We succeeded to 6066 * update the size of the main buffer, but failed to 6067 * update the size of the max buffer. But when we tried 6068 * to reset the main buffer to the original size, we 6069 * failed there too. This is very unlikely to 6070 * happen, but if it does, warn and kill all 6071 * tracing. 6072 */ 6073 WARN_ON(1); 6074 tracing_disabled = 1; 6075 } 6076 goto out_start; 6077 } 6078 6079 update_buffer_entries(&tr->max_buffer, cpu); 6080 6081 out: 6082#endif /* CONFIG_TRACER_MAX_TRACE */ 6083 6084 update_buffer_entries(&tr->array_buffer, cpu); 6085 out_start: 6086 tracing_start_tr(tr); 6087 return ret; 6088} 6089 6090ssize_t tracing_resize_ring_buffer(struct trace_array *tr, 6091 unsigned long size, int cpu_id) 6092{ 6093 guard(mutex)(&trace_types_lock); 6094 6095 if (cpu_id != RING_BUFFER_ALL_CPUS) { 6096 /* make sure, this cpu is enabled in the mask */ 6097 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) 6098 return -EINVAL; 6099 } 6100 6101 return __tracing_resize_ring_buffer(tr, size, cpu_id); 6102} 6103 6104struct trace_mod_entry { 6105 unsigned long mod_addr; 6106 char mod_name[MODULE_NAME_LEN]; 6107}; 6108 6109struct trace_scratch { 6110 unsigned int clock_id; 6111 unsigned long text_addr; 6112 unsigned long nr_entries; 6113 struct trace_mod_entry entries[]; 6114}; 6115 6116static DEFINE_MUTEX(scratch_mutex); 6117 6118static int cmp_mod_entry(const void *key, const void *pivot) 6119{ 6120 unsigned long addr = (unsigned long)key; 6121 const struct trace_mod_entry *ent = pivot; 6122 6123 if (addr < ent[0].mod_addr) 6124 return -1; 6125 6126 return addr >= ent[1].mod_addr; 6127} 6128 6129/** 6130 * trace_adjust_address() - Adjust prev boot address to current address. 6131 * @tr: Persistent ring buffer's trace_array. 6132 * @addr: Address in @tr which is adjusted. 6133 */ 6134unsigned long trace_adjust_address(struct trace_array *tr, unsigned long addr) 6135{ 6136 struct trace_module_delta *module_delta; 6137 struct trace_scratch *tscratch; 6138 struct trace_mod_entry *entry; 6139 unsigned long raddr; 6140 int idx = 0, nr_entries; 6141 6142 /* If we don't have last boot delta, return the address */ 6143 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) 6144 return addr; 6145 6146 /* tr->module_delta must be protected by rcu. */ 6147 guard(rcu)(); 6148 tscratch = tr->scratch; 6149 /* if there is no tscrach, module_delta must be NULL. */ 6150 module_delta = READ_ONCE(tr->module_delta); 6151 if (!module_delta || !tscratch->nr_entries || 6152 tscratch->entries[0].mod_addr > addr) { 6153 raddr = addr + tr->text_delta; 6154 return __is_kernel(raddr) || is_kernel_core_data(raddr) || 6155 is_kernel_rodata(raddr) ? raddr : addr; 6156 } 6157 6158 /* Note that entries must be sorted. */ 6159 nr_entries = tscratch->nr_entries; 6160 if (nr_entries == 1 || 6161 tscratch->entries[nr_entries - 1].mod_addr < addr) 6162 idx = nr_entries - 1; 6163 else { 6164 entry = __inline_bsearch((void *)addr, 6165 tscratch->entries, 6166 nr_entries - 1, 6167 sizeof(tscratch->entries[0]), 6168 cmp_mod_entry); 6169 if (entry) 6170 idx = entry - tscratch->entries; 6171 } 6172 6173 return addr + module_delta->delta[idx]; 6174} 6175 6176#ifdef CONFIG_MODULES 6177static int save_mod(struct module *mod, void *data) 6178{ 6179 struct trace_array *tr = data; 6180 struct trace_scratch *tscratch; 6181 struct trace_mod_entry *entry; 6182 unsigned int size; 6183 6184 tscratch = tr->scratch; 6185 if (!tscratch) 6186 return -1; 6187 size = tr->scratch_size; 6188 6189 if (struct_size(tscratch, entries, tscratch->nr_entries + 1) > size) 6190 return -1; 6191 6192 entry = &tscratch->entries[tscratch->nr_entries]; 6193 6194 tscratch->nr_entries++; 6195 6196 entry->mod_addr = (unsigned long)mod->mem[MOD_TEXT].base; 6197 strscpy(entry->mod_name, mod->name); 6198 6199 return 0; 6200} 6201#else 6202static int save_mod(struct module *mod, void *data) 6203{ 6204 return 0; 6205} 6206#endif 6207 6208static void update_last_data(struct trace_array *tr) 6209{ 6210 struct trace_module_delta *module_delta; 6211 struct trace_scratch *tscratch; 6212 6213 if (!(tr->flags & TRACE_ARRAY_FL_BOOT)) 6214 return; 6215 6216 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) 6217 return; 6218 6219 /* Only if the buffer has previous boot data clear and update it. */ 6220 tr->flags &= ~TRACE_ARRAY_FL_LAST_BOOT; 6221 6222 /* Reset the module list and reload them */ 6223 if (tr->scratch) { 6224 struct trace_scratch *tscratch = tr->scratch; 6225 6226 tscratch->clock_id = tr->clock_id; 6227 memset(tscratch->entries, 0, 6228 flex_array_size(tscratch, entries, tscratch->nr_entries)); 6229 tscratch->nr_entries = 0; 6230 6231 guard(mutex)(&scratch_mutex); 6232 module_for_each_mod(save_mod, tr); 6233 } 6234 6235 /* 6236 * Need to clear all CPU buffers as there cannot be events 6237 * from the previous boot mixed with events with this boot 6238 * as that will cause a confusing trace. Need to clear all 6239 * CPU buffers, even for those that may currently be offline. 6240 */ 6241 tracing_reset_all_cpus(&tr->array_buffer); 6242 6243 /* Using current data now */ 6244 tr->text_delta = 0; 6245 6246 if (!tr->scratch) 6247 return; 6248 6249 tscratch = tr->scratch; 6250 module_delta = READ_ONCE(tr->module_delta); 6251 WRITE_ONCE(tr->module_delta, NULL); 6252 kfree_rcu(module_delta, rcu); 6253 6254 /* Set the persistent ring buffer meta data to this address */ 6255 tscratch->text_addr = (unsigned long)_text; 6256} 6257 6258/** 6259 * tracing_update_buffers - used by tracing facility to expand ring buffers 6260 * @tr: The tracing instance 6261 * 6262 * To save on memory when the tracing is never used on a system with it 6263 * configured in. The ring buffers are set to a minimum size. But once 6264 * a user starts to use the tracing facility, then they need to grow 6265 * to their default size. 6266 * 6267 * This function is to be called when a tracer is about to be used. 6268 */ 6269int tracing_update_buffers(struct trace_array *tr) 6270{ 6271 int ret = 0; 6272 6273 guard(mutex)(&trace_types_lock); 6274 6275 update_last_data(tr); 6276 6277 if (!tr->ring_buffer_expanded) 6278 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 6279 RING_BUFFER_ALL_CPUS); 6280 return ret; 6281} 6282 6283/* 6284 * Used to clear out the tracer before deletion of an instance. 6285 * Must have trace_types_lock held. 6286 */ 6287static void tracing_set_nop(struct trace_array *tr) 6288{ 6289 if (tr->current_trace == &nop_trace) 6290 return; 6291 6292 tr->current_trace->enabled--; 6293 6294 if (tr->current_trace->reset) 6295 tr->current_trace->reset(tr); 6296 6297 tr->current_trace = &nop_trace; 6298 tr->current_trace_flags = nop_trace.flags; 6299} 6300 6301static bool tracer_options_updated; 6302 6303int tracing_set_tracer(struct trace_array *tr, const char *buf) 6304{ 6305 struct tracer *trace = NULL; 6306 struct tracers *t; 6307#ifdef CONFIG_TRACER_MAX_TRACE 6308 bool had_max_tr; 6309#endif 6310 int ret; 6311 6312 guard(mutex)(&trace_types_lock); 6313 6314 update_last_data(tr); 6315 6316 if (!tr->ring_buffer_expanded) { 6317 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, 6318 RING_BUFFER_ALL_CPUS); 6319 if (ret < 0) 6320 return ret; 6321 ret = 0; 6322 } 6323 6324 list_for_each_entry(t, &tr->tracers, list) { 6325 if (strcmp(t->tracer->name, buf) == 0) { 6326 trace = t->tracer; 6327 break; 6328 } 6329 } 6330 if (!trace) 6331 return -EINVAL; 6332 6333 if (trace == tr->current_trace) 6334 return 0; 6335 6336#ifdef CONFIG_TRACER_SNAPSHOT 6337 if (trace->use_max_tr) { 6338 local_irq_disable(); 6339 arch_spin_lock(&tr->max_lock); 6340 ret = tr->cond_snapshot ? -EBUSY : 0; 6341 arch_spin_unlock(&tr->max_lock); 6342 local_irq_enable(); 6343 if (ret) 6344 return ret; 6345 } 6346#endif 6347 /* Some tracers won't work on kernel command line */ 6348 if (system_state < SYSTEM_RUNNING && trace->noboot) { 6349 pr_warn("Tracer '%s' is not allowed on command line, ignored\n", 6350 trace->name); 6351 return -EINVAL; 6352 } 6353 6354 /* Some tracers are only allowed for the top level buffer */ 6355 if (!trace_ok_for_array(trace, tr)) 6356 return -EINVAL; 6357 6358 /* If trace pipe files are being read, we can't change the tracer */ 6359 if (tr->trace_ref) 6360 return -EBUSY; 6361 6362 trace_branch_disable(); 6363 6364 tr->current_trace->enabled--; 6365 6366 if (tr->current_trace->reset) 6367 tr->current_trace->reset(tr); 6368 6369#ifdef CONFIG_TRACER_MAX_TRACE 6370 had_max_tr = tr->current_trace->use_max_tr; 6371 6372 /* Current trace needs to be nop_trace before synchronize_rcu */ 6373 tr->current_trace = &nop_trace; 6374 tr->current_trace_flags = nop_trace.flags; 6375 6376 if (had_max_tr && !trace->use_max_tr) { 6377 /* 6378 * We need to make sure that the update_max_tr sees that 6379 * current_trace changed to nop_trace to keep it from 6380 * swapping the buffers after we resize it. 6381 * The update_max_tr is called from interrupts disabled 6382 * so a synchronized_sched() is sufficient. 6383 */ 6384 synchronize_rcu(); 6385 free_snapshot(tr); 6386 tracing_disarm_snapshot(tr); 6387 } 6388 6389 if (!had_max_tr && trace->use_max_tr) { 6390 ret = tracing_arm_snapshot_locked(tr); 6391 if (ret) 6392 return ret; 6393 } 6394#else 6395 tr->current_trace = &nop_trace; 6396#endif 6397 6398 tr->current_trace_flags = t->flags ? : t->tracer->flags; 6399 6400 if (trace->init) { 6401 ret = tracer_init(trace, tr); 6402 if (ret) { 6403#ifdef CONFIG_TRACER_MAX_TRACE 6404 if (trace->use_max_tr) 6405 tracing_disarm_snapshot(tr); 6406#endif 6407 tr->current_trace_flags = nop_trace.flags; 6408 return ret; 6409 } 6410 } 6411 6412 tr->current_trace = trace; 6413 tr->current_trace->enabled++; 6414 trace_branch_enable(tr); 6415 6416 return 0; 6417} 6418 6419static ssize_t 6420tracing_set_trace_write(struct file *filp, const char __user *ubuf, 6421 size_t cnt, loff_t *ppos) 6422{ 6423 struct trace_array *tr = filp->private_data; 6424 char buf[MAX_TRACER_SIZE+1]; 6425 char *name; 6426 size_t ret; 6427 int err; 6428 6429 ret = cnt; 6430 6431 if (cnt > MAX_TRACER_SIZE) 6432 cnt = MAX_TRACER_SIZE; 6433 6434 if (copy_from_user(buf, ubuf, cnt)) 6435 return -EFAULT; 6436 6437 buf[cnt] = 0; 6438 6439 name = strim(buf); 6440 6441 err = tracing_set_tracer(tr, name); 6442 if (err) 6443 return err; 6444 6445 *ppos += ret; 6446 6447 return ret; 6448} 6449 6450static ssize_t 6451tracing_nsecs_read(unsigned long *ptr, char __user *ubuf, 6452 size_t cnt, loff_t *ppos) 6453{ 6454 char buf[64]; 6455 int r; 6456 6457 r = snprintf(buf, sizeof(buf), "%ld\n", 6458 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 6459 if (r > sizeof(buf)) 6460 r = sizeof(buf); 6461 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6462} 6463 6464static ssize_t 6465tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf, 6466 size_t cnt, loff_t *ppos) 6467{ 6468 unsigned long val; 6469 int ret; 6470 6471 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6472 if (ret) 6473 return ret; 6474 6475 *ptr = val * 1000; 6476 6477 return cnt; 6478} 6479 6480static ssize_t 6481tracing_thresh_read(struct file *filp, char __user *ubuf, 6482 size_t cnt, loff_t *ppos) 6483{ 6484 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos); 6485} 6486 6487static ssize_t 6488tracing_thresh_write(struct file *filp, const char __user *ubuf, 6489 size_t cnt, loff_t *ppos) 6490{ 6491 struct trace_array *tr = filp->private_data; 6492 int ret; 6493 6494 guard(mutex)(&trace_types_lock); 6495 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos); 6496 if (ret < 0) 6497 return ret; 6498 6499 if (tr->current_trace->update_thresh) { 6500 ret = tr->current_trace->update_thresh(tr); 6501 if (ret < 0) 6502 return ret; 6503 } 6504 6505 return cnt; 6506} 6507 6508#ifdef CONFIG_TRACER_MAX_TRACE 6509 6510static ssize_t 6511tracing_max_lat_read(struct file *filp, char __user *ubuf, 6512 size_t cnt, loff_t *ppos) 6513{ 6514 struct trace_array *tr = filp->private_data; 6515 6516 return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos); 6517} 6518 6519static ssize_t 6520tracing_max_lat_write(struct file *filp, const char __user *ubuf, 6521 size_t cnt, loff_t *ppos) 6522{ 6523 struct trace_array *tr = filp->private_data; 6524 6525 return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos); 6526} 6527 6528#endif 6529 6530static int open_pipe_on_cpu(struct trace_array *tr, int cpu) 6531{ 6532 if (cpu == RING_BUFFER_ALL_CPUS) { 6533 if (cpumask_empty(tr->pipe_cpumask)) { 6534 cpumask_setall(tr->pipe_cpumask); 6535 return 0; 6536 } 6537 } else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) { 6538 cpumask_set_cpu(cpu, tr->pipe_cpumask); 6539 return 0; 6540 } 6541 return -EBUSY; 6542} 6543 6544static void close_pipe_on_cpu(struct trace_array *tr, int cpu) 6545{ 6546 if (cpu == RING_BUFFER_ALL_CPUS) { 6547 WARN_ON(!cpumask_full(tr->pipe_cpumask)); 6548 cpumask_clear(tr->pipe_cpumask); 6549 } else { 6550 WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask)); 6551 cpumask_clear_cpu(cpu, tr->pipe_cpumask); 6552 } 6553} 6554 6555static int tracing_open_pipe(struct inode *inode, struct file *filp) 6556{ 6557 struct trace_array *tr = inode->i_private; 6558 struct trace_iterator *iter; 6559 int cpu; 6560 int ret; 6561 6562 ret = tracing_check_open_get_tr(tr); 6563 if (ret) 6564 return ret; 6565 6566 guard(mutex)(&trace_types_lock); 6567 cpu = tracing_get_cpu(inode); 6568 ret = open_pipe_on_cpu(tr, cpu); 6569 if (ret) 6570 goto fail_pipe_on_cpu; 6571 6572 /* create a buffer to store the information to pass to userspace */ 6573 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 6574 if (!iter) { 6575 ret = -ENOMEM; 6576 goto fail_alloc_iter; 6577 } 6578 6579 trace_seq_init(&iter->seq); 6580 iter->trace = tr->current_trace; 6581 6582 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 6583 ret = -ENOMEM; 6584 goto fail; 6585 } 6586 6587 /* trace pipe does not show start of buffer */ 6588 cpumask_setall(iter->started); 6589 6590 if (tr->trace_flags & TRACE_ITER(LATENCY_FMT)) 6591 iter->iter_flags |= TRACE_FILE_LAT_FMT; 6592 6593 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 6594 if (trace_clocks[tr->clock_id].in_ns) 6595 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 6596 6597 iter->tr = tr; 6598 iter->array_buffer = &tr->array_buffer; 6599 iter->cpu_file = cpu; 6600 mutex_init(&iter->mutex); 6601 filp->private_data = iter; 6602 6603 if (iter->trace->pipe_open) 6604 iter->trace->pipe_open(iter); 6605 6606 nonseekable_open(inode, filp); 6607 6608 tr->trace_ref++; 6609 6610 return ret; 6611 6612fail: 6613 kfree(iter); 6614fail_alloc_iter: 6615 close_pipe_on_cpu(tr, cpu); 6616fail_pipe_on_cpu: 6617 __trace_array_put(tr); 6618 return ret; 6619} 6620 6621static int tracing_release_pipe(struct inode *inode, struct file *file) 6622{ 6623 struct trace_iterator *iter = file->private_data; 6624 struct trace_array *tr = inode->i_private; 6625 6626 scoped_guard(mutex, &trace_types_lock) { 6627 tr->trace_ref--; 6628 6629 if (iter->trace->pipe_close) 6630 iter->trace->pipe_close(iter); 6631 close_pipe_on_cpu(tr, iter->cpu_file); 6632 } 6633 6634 free_trace_iter_content(iter); 6635 kfree(iter); 6636 6637 trace_array_put(tr); 6638 6639 return 0; 6640} 6641 6642static __poll_t 6643trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table) 6644{ 6645 struct trace_array *tr = iter->tr; 6646 6647 /* Iterators are static, they should be filled or empty */ 6648 if (trace_buffer_iter(iter, iter->cpu_file)) 6649 return EPOLLIN | EPOLLRDNORM; 6650 6651 if (tr->trace_flags & TRACE_ITER(BLOCK)) 6652 /* 6653 * Always select as readable when in blocking mode 6654 */ 6655 return EPOLLIN | EPOLLRDNORM; 6656 else 6657 return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file, 6658 filp, poll_table, iter->tr->buffer_percent); 6659} 6660 6661static __poll_t 6662tracing_poll_pipe(struct file *filp, poll_table *poll_table) 6663{ 6664 struct trace_iterator *iter = filp->private_data; 6665 6666 return trace_poll(iter, filp, poll_table); 6667} 6668 6669/* Must be called with iter->mutex held. */ 6670static int tracing_wait_pipe(struct file *filp) 6671{ 6672 struct trace_iterator *iter = filp->private_data; 6673 int ret; 6674 6675 while (trace_empty(iter)) { 6676 6677 if ((filp->f_flags & O_NONBLOCK)) { 6678 return -EAGAIN; 6679 } 6680 6681 /* 6682 * We block until we read something and tracing is disabled. 6683 * We still block if tracing is disabled, but we have never 6684 * read anything. This allows a user to cat this file, and 6685 * then enable tracing. But after we have read something, 6686 * we give an EOF when tracing is again disabled. 6687 * 6688 * iter->pos will be 0 if we haven't read anything. 6689 */ 6690 if (!tracer_tracing_is_on(iter->tr) && iter->pos) 6691 break; 6692 6693 mutex_unlock(&iter->mutex); 6694 6695 ret = wait_on_pipe(iter, 0); 6696 6697 mutex_lock(&iter->mutex); 6698 6699 if (ret) 6700 return ret; 6701 } 6702 6703 return 1; 6704} 6705 6706static bool update_last_data_if_empty(struct trace_array *tr) 6707{ 6708 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) 6709 return false; 6710 6711 if (!ring_buffer_empty(tr->array_buffer.buffer)) 6712 return false; 6713 6714 /* 6715 * If the buffer contains the last boot data and all per-cpu 6716 * buffers are empty, reset it from the kernel side. 6717 */ 6718 update_last_data(tr); 6719 return true; 6720} 6721 6722/* 6723 * Consumer reader. 6724 */ 6725static ssize_t 6726tracing_read_pipe(struct file *filp, char __user *ubuf, 6727 size_t cnt, loff_t *ppos) 6728{ 6729 struct trace_iterator *iter = filp->private_data; 6730 ssize_t sret; 6731 6732 /* 6733 * Avoid more than one consumer on a single file descriptor 6734 * This is just a matter of traces coherency, the ring buffer itself 6735 * is protected. 6736 */ 6737 guard(mutex)(&iter->mutex); 6738 6739 /* return any leftover data */ 6740 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 6741 if (sret != -EBUSY) 6742 return sret; 6743 6744 trace_seq_init(&iter->seq); 6745 6746 if (iter->trace->read) { 6747 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 6748 if (sret) 6749 return sret; 6750 } 6751 6752waitagain: 6753 if (update_last_data_if_empty(iter->tr)) 6754 return 0; 6755 6756 sret = tracing_wait_pipe(filp); 6757 if (sret <= 0) 6758 return sret; 6759 6760 /* stop when tracing is finished */ 6761 if (trace_empty(iter)) 6762 return 0; 6763 6764 if (cnt >= TRACE_SEQ_BUFFER_SIZE) 6765 cnt = TRACE_SEQ_BUFFER_SIZE - 1; 6766 6767 /* reset all but tr, trace, and overruns */ 6768 trace_iterator_reset(iter); 6769 cpumask_clear(iter->started); 6770 trace_seq_init(&iter->seq); 6771 6772 trace_event_read_lock(); 6773 trace_access_lock(iter->cpu_file); 6774 while (trace_find_next_entry_inc(iter) != NULL) { 6775 enum print_line_t ret; 6776 int save_len = iter->seq.seq.len; 6777 6778 ret = print_trace_line(iter); 6779 if (ret == TRACE_TYPE_PARTIAL_LINE) { 6780 /* 6781 * If one print_trace_line() fills entire trace_seq in one shot, 6782 * trace_seq_to_user() will returns -EBUSY because save_len == 0, 6783 * In this case, we need to consume it, otherwise, loop will peek 6784 * this event next time, resulting in an infinite loop. 6785 */ 6786 if (save_len == 0) { 6787 iter->seq.full = 0; 6788 trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n"); 6789 trace_consume(iter); 6790 break; 6791 } 6792 6793 /* In other cases, don't print partial lines */ 6794 iter->seq.seq.len = save_len; 6795 break; 6796 } 6797 if (ret != TRACE_TYPE_NO_CONSUME) 6798 trace_consume(iter); 6799 6800 if (trace_seq_used(&iter->seq) >= cnt) 6801 break; 6802 6803 /* 6804 * Setting the full flag means we reached the trace_seq buffer 6805 * size and we should leave by partial output condition above. 6806 * One of the trace_seq_* functions is not used properly. 6807 */ 6808 WARN_ONCE(iter->seq.full, "full flag set for trace type %d", 6809 iter->ent->type); 6810 } 6811 trace_access_unlock(iter->cpu_file); 6812 trace_event_read_unlock(); 6813 6814 /* Now copy what we have to the user */ 6815 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 6816 if (iter->seq.readpos >= trace_seq_used(&iter->seq)) 6817 trace_seq_init(&iter->seq); 6818 6819 /* 6820 * If there was nothing to send to user, in spite of consuming trace 6821 * entries, go back to wait for more entries. 6822 */ 6823 if (sret == -EBUSY) 6824 goto waitagain; 6825 6826 return sret; 6827} 6828 6829static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 6830 unsigned int idx) 6831{ 6832 __free_page(spd->pages[idx]); 6833} 6834 6835static size_t 6836tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 6837{ 6838 size_t count; 6839 int save_len; 6840 int ret; 6841 6842 /* Seq buffer is page-sized, exactly what we need. */ 6843 for (;;) { 6844 save_len = iter->seq.seq.len; 6845 ret = print_trace_line(iter); 6846 6847 if (trace_seq_has_overflowed(&iter->seq)) { 6848 iter->seq.seq.len = save_len; 6849 break; 6850 } 6851 6852 /* 6853 * This should not be hit, because it should only 6854 * be set if the iter->seq overflowed. But check it 6855 * anyway to be safe. 6856 */ 6857 if (ret == TRACE_TYPE_PARTIAL_LINE) { 6858 iter->seq.seq.len = save_len; 6859 break; 6860 } 6861 6862 count = trace_seq_used(&iter->seq) - save_len; 6863 if (rem < count) { 6864 rem = 0; 6865 iter->seq.seq.len = save_len; 6866 break; 6867 } 6868 6869 if (ret != TRACE_TYPE_NO_CONSUME) 6870 trace_consume(iter); 6871 rem -= count; 6872 if (!trace_find_next_entry_inc(iter)) { 6873 rem = 0; 6874 iter->ent = NULL; 6875 break; 6876 } 6877 } 6878 6879 return rem; 6880} 6881 6882static ssize_t tracing_splice_read_pipe(struct file *filp, 6883 loff_t *ppos, 6884 struct pipe_inode_info *pipe, 6885 size_t len, 6886 unsigned int flags) 6887{ 6888 struct page *pages_def[PIPE_DEF_BUFFERS]; 6889 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 6890 struct trace_iterator *iter = filp->private_data; 6891 struct splice_pipe_desc spd = { 6892 .pages = pages_def, 6893 .partial = partial_def, 6894 .nr_pages = 0, /* This gets updated below. */ 6895 .nr_pages_max = PIPE_DEF_BUFFERS, 6896 .ops = &default_pipe_buf_ops, 6897 .spd_release = tracing_spd_release_pipe, 6898 }; 6899 ssize_t ret; 6900 size_t rem; 6901 unsigned int i; 6902 6903 if (splice_grow_spd(pipe, &spd)) 6904 return -ENOMEM; 6905 6906 mutex_lock(&iter->mutex); 6907 6908 if (iter->trace->splice_read) { 6909 ret = iter->trace->splice_read(iter, filp, 6910 ppos, pipe, len, flags); 6911 if (ret) 6912 goto out_err; 6913 } 6914 6915 ret = tracing_wait_pipe(filp); 6916 if (ret <= 0) 6917 goto out_err; 6918 6919 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 6920 ret = -EFAULT; 6921 goto out_err; 6922 } 6923 6924 trace_event_read_lock(); 6925 trace_access_lock(iter->cpu_file); 6926 6927 /* Fill as many pages as possible. */ 6928 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) { 6929 spd.pages[i] = alloc_page(GFP_KERNEL); 6930 if (!spd.pages[i]) 6931 break; 6932 6933 rem = tracing_fill_pipe_page(rem, iter); 6934 6935 /* Copy the data into the page, so we can start over. */ 6936 ret = trace_seq_to_buffer(&iter->seq, 6937 page_address(spd.pages[i]), 6938 min((size_t)trace_seq_used(&iter->seq), 6939 (size_t)PAGE_SIZE)); 6940 if (ret < 0) { 6941 __free_page(spd.pages[i]); 6942 break; 6943 } 6944 spd.partial[i].offset = 0; 6945 spd.partial[i].len = ret; 6946 6947 trace_seq_init(&iter->seq); 6948 } 6949 6950 trace_access_unlock(iter->cpu_file); 6951 trace_event_read_unlock(); 6952 mutex_unlock(&iter->mutex); 6953 6954 spd.nr_pages = i; 6955 6956 if (i) 6957 ret = splice_to_pipe(pipe, &spd); 6958 else 6959 ret = 0; 6960out: 6961 splice_shrink_spd(&spd); 6962 return ret; 6963 6964out_err: 6965 mutex_unlock(&iter->mutex); 6966 goto out; 6967} 6968 6969static ssize_t 6970tracing_syscall_buf_read(struct file *filp, char __user *ubuf, 6971 size_t cnt, loff_t *ppos) 6972{ 6973 struct inode *inode = file_inode(filp); 6974 struct trace_array *tr = inode->i_private; 6975 char buf[64]; 6976 int r; 6977 6978 r = snprintf(buf, 64, "%d\n", tr->syscall_buf_sz); 6979 6980 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 6981} 6982 6983static ssize_t 6984tracing_syscall_buf_write(struct file *filp, const char __user *ubuf, 6985 size_t cnt, loff_t *ppos) 6986{ 6987 struct inode *inode = file_inode(filp); 6988 struct trace_array *tr = inode->i_private; 6989 unsigned long val; 6990 int ret; 6991 6992 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 6993 if (ret) 6994 return ret; 6995 6996 if (val > SYSCALL_FAULT_USER_MAX) 6997 val = SYSCALL_FAULT_USER_MAX; 6998 6999 tr->syscall_buf_sz = val; 7000 7001 *ppos += cnt; 7002 7003 return cnt; 7004} 7005 7006static ssize_t 7007tracing_entries_read(struct file *filp, char __user *ubuf, 7008 size_t cnt, loff_t *ppos) 7009{ 7010 struct inode *inode = file_inode(filp); 7011 struct trace_array *tr = inode->i_private; 7012 int cpu = tracing_get_cpu(inode); 7013 char buf[64]; 7014 int r = 0; 7015 ssize_t ret; 7016 7017 mutex_lock(&trace_types_lock); 7018 7019 if (cpu == RING_BUFFER_ALL_CPUS) { 7020 int cpu, buf_size_same; 7021 unsigned long size; 7022 7023 size = 0; 7024 buf_size_same = 1; 7025 /* check if all cpu sizes are same */ 7026 for_each_tracing_cpu(cpu) { 7027 /* fill in the size from first enabled cpu */ 7028 if (size == 0) 7029 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries; 7030 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) { 7031 buf_size_same = 0; 7032 break; 7033 } 7034 } 7035 7036 if (buf_size_same) { 7037 if (!tr->ring_buffer_expanded) 7038 r = sprintf(buf, "%lu (expanded: %lu)\n", 7039 size >> 10, 7040 trace_buf_size >> 10); 7041 else 7042 r = sprintf(buf, "%lu\n", size >> 10); 7043 } else 7044 r = sprintf(buf, "X\n"); 7045 } else 7046 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10); 7047 7048 mutex_unlock(&trace_types_lock); 7049 7050 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 7051 return ret; 7052} 7053 7054static ssize_t 7055tracing_entries_write(struct file *filp, const char __user *ubuf, 7056 size_t cnt, loff_t *ppos) 7057{ 7058 struct inode *inode = file_inode(filp); 7059 struct trace_array *tr = inode->i_private; 7060 unsigned long val; 7061 int ret; 7062 7063 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7064 if (ret) 7065 return ret; 7066 7067 /* must have at least 1 entry */ 7068 if (!val) 7069 return -EINVAL; 7070 7071 /* value is in KB */ 7072 val <<= 10; 7073 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode)); 7074 if (ret < 0) 7075 return ret; 7076 7077 *ppos += cnt; 7078 7079 return cnt; 7080} 7081 7082static ssize_t 7083tracing_total_entries_read(struct file *filp, char __user *ubuf, 7084 size_t cnt, loff_t *ppos) 7085{ 7086 struct trace_array *tr = filp->private_data; 7087 char buf[64]; 7088 int r, cpu; 7089 unsigned long size = 0, expanded_size = 0; 7090 7091 mutex_lock(&trace_types_lock); 7092 for_each_tracing_cpu(cpu) { 7093 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10; 7094 if (!tr->ring_buffer_expanded) 7095 expanded_size += trace_buf_size >> 10; 7096 } 7097 if (tr->ring_buffer_expanded) 7098 r = sprintf(buf, "%lu\n", size); 7099 else 7100 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size); 7101 mutex_unlock(&trace_types_lock); 7102 7103 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 7104} 7105 7106#define LAST_BOOT_HEADER ((void *)1) 7107 7108static void *l_next(struct seq_file *m, void *v, loff_t *pos) 7109{ 7110 struct trace_array *tr = m->private; 7111 struct trace_scratch *tscratch = tr->scratch; 7112 unsigned int index = *pos; 7113 7114 (*pos)++; 7115 7116 if (*pos == 1) 7117 return LAST_BOOT_HEADER; 7118 7119 /* Only show offsets of the last boot data */ 7120 if (!tscratch || !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) 7121 return NULL; 7122 7123 /* *pos 0 is for the header, 1 is for the first module */ 7124 index--; 7125 7126 if (index >= tscratch->nr_entries) 7127 return NULL; 7128 7129 return &tscratch->entries[index]; 7130} 7131 7132static void *l_start(struct seq_file *m, loff_t *pos) 7133{ 7134 mutex_lock(&scratch_mutex); 7135 7136 return l_next(m, NULL, pos); 7137} 7138 7139static void l_stop(struct seq_file *m, void *p) 7140{ 7141 mutex_unlock(&scratch_mutex); 7142} 7143 7144static void show_last_boot_header(struct seq_file *m, struct trace_array *tr) 7145{ 7146 struct trace_scratch *tscratch = tr->scratch; 7147 7148 /* 7149 * Do not leak KASLR address. This only shows the KASLR address of 7150 * the last boot. When the ring buffer is started, the LAST_BOOT 7151 * flag gets cleared, and this should only report "current". 7152 * Otherwise it shows the KASLR address from the previous boot which 7153 * should not be the same as the current boot. 7154 */ 7155 if (tscratch && (tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) 7156 seq_printf(m, "%lx\t[kernel]\n", tscratch->text_addr); 7157 else 7158 seq_puts(m, "# Current\n"); 7159} 7160 7161static int l_show(struct seq_file *m, void *v) 7162{ 7163 struct trace_array *tr = m->private; 7164 struct trace_mod_entry *entry = v; 7165 7166 if (v == LAST_BOOT_HEADER) { 7167 show_last_boot_header(m, tr); 7168 return 0; 7169 } 7170 7171 seq_printf(m, "%lx\t%s\n", entry->mod_addr, entry->mod_name); 7172 return 0; 7173} 7174 7175static const struct seq_operations last_boot_seq_ops = { 7176 .start = l_start, 7177 .next = l_next, 7178 .stop = l_stop, 7179 .show = l_show, 7180}; 7181 7182static int tracing_last_boot_open(struct inode *inode, struct file *file) 7183{ 7184 struct trace_array *tr = inode->i_private; 7185 struct seq_file *m; 7186 int ret; 7187 7188 ret = tracing_check_open_get_tr(tr); 7189 if (ret) 7190 return ret; 7191 7192 ret = seq_open(file, &last_boot_seq_ops); 7193 if (ret) { 7194 trace_array_put(tr); 7195 return ret; 7196 } 7197 7198 m = file->private_data; 7199 m->private = tr; 7200 7201 return 0; 7202} 7203 7204static int tracing_buffer_meta_open(struct inode *inode, struct file *filp) 7205{ 7206 struct trace_array *tr = inode->i_private; 7207 int cpu = tracing_get_cpu(inode); 7208 int ret; 7209 7210 ret = tracing_check_open_get_tr(tr); 7211 if (ret) 7212 return ret; 7213 7214 ret = ring_buffer_meta_seq_init(filp, tr->array_buffer.buffer, cpu); 7215 if (ret < 0) 7216 __trace_array_put(tr); 7217 return ret; 7218} 7219 7220static ssize_t 7221tracing_free_buffer_write(struct file *filp, const char __user *ubuf, 7222 size_t cnt, loff_t *ppos) 7223{ 7224 /* 7225 * There is no need to read what the user has written, this function 7226 * is just to make sure that there is no error when "echo" is used 7227 */ 7228 7229 *ppos += cnt; 7230 7231 return cnt; 7232} 7233 7234static int 7235tracing_free_buffer_release(struct inode *inode, struct file *filp) 7236{ 7237 struct trace_array *tr = inode->i_private; 7238 7239 /* disable tracing ? */ 7240 if (tr->trace_flags & TRACE_ITER(STOP_ON_FREE)) 7241 tracer_tracing_off(tr); 7242 /* resize the ring buffer to 0 */ 7243 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS); 7244 7245 trace_array_put(tr); 7246 7247 return 0; 7248} 7249 7250#define TRACE_MARKER_MAX_SIZE 4096 7251 7252static ssize_t write_marker_to_buffer(struct trace_array *tr, const char *buf, 7253 size_t cnt, unsigned long ip) 7254{ 7255 struct ring_buffer_event *event; 7256 enum event_trigger_type tt = ETT_NONE; 7257 struct trace_buffer *buffer; 7258 struct print_entry *entry; 7259 int meta_size; 7260 ssize_t written; 7261 size_t size; 7262 7263 meta_size = sizeof(*entry) + 2; /* add '\0' and possible '\n' */ 7264 again: 7265 size = cnt + meta_size; 7266 7267 buffer = tr->array_buffer.buffer; 7268 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 7269 tracing_gen_ctx()); 7270 if (unlikely(!event)) { 7271 /* 7272 * If the size was greater than what was allowed, then 7273 * make it smaller and try again. 7274 */ 7275 if (size > ring_buffer_max_event_size(buffer)) { 7276 cnt = ring_buffer_max_event_size(buffer) - meta_size; 7277 /* The above should only happen once */ 7278 if (WARN_ON_ONCE(cnt + meta_size == size)) 7279 return -EBADF; 7280 goto again; 7281 } 7282 7283 /* Ring buffer disabled, return as if not open for write */ 7284 return -EBADF; 7285 } 7286 7287 entry = ring_buffer_event_data(event); 7288 entry->ip = ip; 7289 memcpy(&entry->buf, buf, cnt); 7290 written = cnt; 7291 7292 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) { 7293 /* do not add \n before testing triggers, but add \0 */ 7294 entry->buf[cnt] = '\0'; 7295 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event); 7296 } 7297 7298 if (entry->buf[cnt - 1] != '\n') { 7299 entry->buf[cnt] = '\n'; 7300 entry->buf[cnt + 1] = '\0'; 7301 } else 7302 entry->buf[cnt] = '\0'; 7303 7304 if (static_branch_unlikely(&trace_marker_exports_enabled)) 7305 ftrace_exports(event, TRACE_EXPORT_MARKER); 7306 __buffer_unlock_commit(buffer, event); 7307 7308 if (tt) 7309 event_triggers_post_call(tr->trace_marker_file, tt); 7310 7311 return written; 7312} 7313 7314struct trace_user_buf { 7315 char *buf; 7316}; 7317 7318static DEFINE_MUTEX(trace_user_buffer_mutex); 7319static struct trace_user_buf_info *trace_user_buffer; 7320 7321/** 7322 * trace_user_fault_destroy - free up allocated memory of a trace user buffer 7323 * @tinfo: The descriptor to free up 7324 * 7325 * Frees any data allocated in the trace info dsecriptor. 7326 */ 7327void trace_user_fault_destroy(struct trace_user_buf_info *tinfo) 7328{ 7329 char *buf; 7330 int cpu; 7331 7332 if (!tinfo || !tinfo->tbuf) 7333 return; 7334 7335 for_each_possible_cpu(cpu) { 7336 buf = per_cpu_ptr(tinfo->tbuf, cpu)->buf; 7337 kfree(buf); 7338 } 7339 free_percpu(tinfo->tbuf); 7340} 7341 7342static int user_fault_buffer_enable(struct trace_user_buf_info *tinfo, size_t size) 7343{ 7344 char *buf; 7345 int cpu; 7346 7347 lockdep_assert_held(&trace_user_buffer_mutex); 7348 7349 tinfo->tbuf = alloc_percpu(struct trace_user_buf); 7350 if (!tinfo->tbuf) 7351 return -ENOMEM; 7352 7353 tinfo->ref = 1; 7354 tinfo->size = size; 7355 7356 /* Clear each buffer in case of error */ 7357 for_each_possible_cpu(cpu) { 7358 per_cpu_ptr(tinfo->tbuf, cpu)->buf = NULL; 7359 } 7360 7361 for_each_possible_cpu(cpu) { 7362 buf = kmalloc_node(size, GFP_KERNEL, 7363 cpu_to_node(cpu)); 7364 if (!buf) 7365 return -ENOMEM; 7366 per_cpu_ptr(tinfo->tbuf, cpu)->buf = buf; 7367 } 7368 7369 return 0; 7370} 7371 7372/* For internal use. Free and reinitialize */ 7373static void user_buffer_free(struct trace_user_buf_info **tinfo) 7374{ 7375 lockdep_assert_held(&trace_user_buffer_mutex); 7376 7377 trace_user_fault_destroy(*tinfo); 7378 kfree(*tinfo); 7379 *tinfo = NULL; 7380} 7381 7382/* For internal use. Initialize and allocate */ 7383static int user_buffer_init(struct trace_user_buf_info **tinfo, size_t size) 7384{ 7385 bool alloc = false; 7386 int ret; 7387 7388 lockdep_assert_held(&trace_user_buffer_mutex); 7389 7390 if (!*tinfo) { 7391 alloc = true; 7392 *tinfo = kzalloc(sizeof(**tinfo), GFP_KERNEL); 7393 if (!*tinfo) 7394 return -ENOMEM; 7395 } 7396 7397 ret = user_fault_buffer_enable(*tinfo, size); 7398 if (ret < 0 && alloc) 7399 user_buffer_free(tinfo); 7400 7401 return ret; 7402} 7403 7404/* For internal use, derefrence and free if necessary */ 7405static void user_buffer_put(struct trace_user_buf_info **tinfo) 7406{ 7407 guard(mutex)(&trace_user_buffer_mutex); 7408 7409 if (WARN_ON_ONCE(!*tinfo || !(*tinfo)->ref)) 7410 return; 7411 7412 if (--(*tinfo)->ref) 7413 return; 7414 7415 user_buffer_free(tinfo); 7416} 7417 7418/** 7419 * trace_user_fault_init - Allocated or reference a per CPU buffer 7420 * @tinfo: A pointer to the trace buffer descriptor 7421 * @size: The size to allocate each per CPU buffer 7422 * 7423 * Create a per CPU buffer that can be used to copy from user space 7424 * in a task context. When calling trace_user_fault_read(), preemption 7425 * must be disabled, and it will enable preemption and copy user 7426 * space data to the buffer. If any schedule switches occur, it will 7427 * retry until it succeeds without a schedule switch knowing the buffer 7428 * is still valid. 7429 * 7430 * Returns 0 on success, negative on failure. 7431 */ 7432int trace_user_fault_init(struct trace_user_buf_info *tinfo, size_t size) 7433{ 7434 int ret; 7435 7436 if (!tinfo) 7437 return -EINVAL; 7438 7439 guard(mutex)(&trace_user_buffer_mutex); 7440 7441 ret = user_buffer_init(&tinfo, size); 7442 if (ret < 0) 7443 trace_user_fault_destroy(tinfo); 7444 7445 return ret; 7446} 7447 7448/** 7449 * trace_user_fault_get - up the ref count for the user buffer 7450 * @tinfo: A pointer to a pointer to the trace buffer descriptor 7451 * 7452 * Ups the ref count of the trace buffer. 7453 * 7454 * Returns the new ref count. 7455 */ 7456int trace_user_fault_get(struct trace_user_buf_info *tinfo) 7457{ 7458 if (!tinfo) 7459 return -1; 7460 7461 guard(mutex)(&trace_user_buffer_mutex); 7462 7463 tinfo->ref++; 7464 return tinfo->ref; 7465} 7466 7467/** 7468 * trace_user_fault_put - dereference a per cpu trace buffer 7469 * @tinfo: The @tinfo that was passed to trace_user_fault_get() 7470 * 7471 * Decrement the ref count of @tinfo. 7472 * 7473 * Returns the new refcount (negative on error). 7474 */ 7475int trace_user_fault_put(struct trace_user_buf_info *tinfo) 7476{ 7477 guard(mutex)(&trace_user_buffer_mutex); 7478 7479 if (WARN_ON_ONCE(!tinfo || !tinfo->ref)) 7480 return -1; 7481 7482 --tinfo->ref; 7483 return tinfo->ref; 7484} 7485 7486/** 7487 * trace_user_fault_read - Read user space into a per CPU buffer 7488 * @tinfo: The @tinfo allocated by trace_user_fault_get() 7489 * @ptr: The user space pointer to read 7490 * @size: The size of user space to read. 7491 * @copy_func: Optional function to use to copy from user space 7492 * @data: Data to pass to copy_func if it was supplied 7493 * 7494 * Preemption must be disabled when this is called, and must not 7495 * be enabled while using the returned buffer. 7496 * This does the copying from user space into a per CPU buffer. 7497 * 7498 * The @size must not be greater than the size passed in to 7499 * trace_user_fault_init(). 7500 * 7501 * If @copy_func is NULL, trace_user_fault_read() will use copy_from_user(), 7502 * otherwise it will call @copy_func. It will call @copy_func with: 7503 * 7504 * buffer: the per CPU buffer of the @tinfo. 7505 * ptr: The pointer @ptr to user space to read 7506 * size: The @size of the ptr to read 7507 * data: The @data parameter 7508 * 7509 * It is expected that @copy_func will return 0 on success and non zero 7510 * if there was a fault. 7511 * 7512 * Returns a pointer to the buffer with the content read from @ptr. 7513 * Preemption must remain disabled while the caller accesses the 7514 * buffer returned by this function. 7515 * Returns NULL if there was a fault, or the size passed in is 7516 * greater than the size passed to trace_user_fault_init(). 7517 */ 7518char *trace_user_fault_read(struct trace_user_buf_info *tinfo, 7519 const char __user *ptr, size_t size, 7520 trace_user_buf_copy copy_func, void *data) 7521{ 7522 int cpu = smp_processor_id(); 7523 char *buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf; 7524 unsigned int cnt; 7525 int trys = 0; 7526 int ret; 7527 7528 lockdep_assert_preemption_disabled(); 7529 7530 /* 7531 * It's up to the caller to not try to copy more than it said 7532 * it would. 7533 */ 7534 if (size > tinfo->size) 7535 return NULL; 7536 7537 /* 7538 * This acts similar to a seqcount. The per CPU context switches are 7539 * recorded, migration is disabled and preemption is enabled. The 7540 * read of the user space memory is copied into the per CPU buffer. 7541 * Preemption is disabled again, and if the per CPU context switches count 7542 * is still the same, it means the buffer has not been corrupted. 7543 * If the count is different, it is assumed the buffer is corrupted 7544 * and reading must be tried again. 7545 */ 7546 7547 do { 7548 /* 7549 * It is possible that something is trying to migrate this 7550 * task. What happens then, is when preemption is enabled, 7551 * the migration thread will preempt this task, try to 7552 * migrate it, fail, then let it run again. That will 7553 * cause this to loop again and never succeed. 7554 * On failures, enabled and disable preemption with 7555 * migration enabled, to allow the migration thread to 7556 * migrate this task. 7557 */ 7558 if (trys) { 7559 preempt_enable_notrace(); 7560 preempt_disable_notrace(); 7561 cpu = smp_processor_id(); 7562 buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf; 7563 } 7564 7565 /* 7566 * If for some reason, copy_from_user() always causes a context 7567 * switch, this would then cause an infinite loop. 7568 * If this task is preempted by another user space task, it 7569 * will cause this task to try again. But just in case something 7570 * changes where the copying from user space causes another task 7571 * to run, prevent this from going into an infinite loop. 7572 * 100 tries should be plenty. 7573 */ 7574 if (WARN_ONCE(trys++ > 100, "Error: Too many tries to read user space")) 7575 return NULL; 7576 7577 /* Read the current CPU context switch counter */ 7578 cnt = nr_context_switches_cpu(cpu); 7579 7580 /* 7581 * Preemption is going to be enabled, but this task must 7582 * remain on this CPU. 7583 */ 7584 migrate_disable(); 7585 7586 /* 7587 * Now preemption is being enabled and another task can come in 7588 * and use the same buffer and corrupt our data. 7589 */ 7590 preempt_enable_notrace(); 7591 7592 /* Make sure preemption is enabled here */ 7593 lockdep_assert_preemption_enabled(); 7594 7595 if (copy_func) { 7596 ret = copy_func(buffer, ptr, size, data); 7597 } else { 7598 ret = __copy_from_user(buffer, ptr, size); 7599 } 7600 7601 preempt_disable_notrace(); 7602 migrate_enable(); 7603 7604 /* if it faulted, no need to test if the buffer was corrupted */ 7605 if (ret) 7606 return NULL; 7607 7608 /* 7609 * Preemption is disabled again, now check the per CPU context 7610 * switch counter. If it doesn't match, then another user space 7611 * process may have schedule in and corrupted our buffer. In that 7612 * case the copying must be retried. 7613 */ 7614 } while (nr_context_switches_cpu(cpu) != cnt); 7615 7616 return buffer; 7617} 7618 7619static ssize_t 7620tracing_mark_write(struct file *filp, const char __user *ubuf, 7621 size_t cnt, loff_t *fpos) 7622{ 7623 struct trace_array *tr = filp->private_data; 7624 ssize_t written = -ENODEV; 7625 unsigned long ip; 7626 char *buf; 7627 7628 if (tracing_disabled) 7629 return -EINVAL; 7630 7631 if (!(tr->trace_flags & TRACE_ITER(MARKERS))) 7632 return -EINVAL; 7633 7634 if ((ssize_t)cnt < 0) 7635 return -EINVAL; 7636 7637 if (cnt > TRACE_MARKER_MAX_SIZE) 7638 cnt = TRACE_MARKER_MAX_SIZE; 7639 7640 /* Must have preemption disabled while having access to the buffer */ 7641 guard(preempt_notrace)(); 7642 7643 buf = trace_user_fault_read(trace_user_buffer, ubuf, cnt, NULL, NULL); 7644 if (!buf) 7645 return -EFAULT; 7646 7647 /* The selftests expect this function to be the IP address */ 7648 ip = _THIS_IP_; 7649 7650 /* The global trace_marker can go to multiple instances */ 7651 if (tr == &global_trace) { 7652 guard(rcu)(); 7653 list_for_each_entry_rcu(tr, &marker_copies, marker_list) { 7654 written = write_marker_to_buffer(tr, buf, cnt, ip); 7655 if (written < 0) 7656 break; 7657 } 7658 } else { 7659 written = write_marker_to_buffer(tr, buf, cnt, ip); 7660 } 7661 7662 return written; 7663} 7664 7665static ssize_t write_raw_marker_to_buffer(struct trace_array *tr, 7666 const char *buf, size_t cnt) 7667{ 7668 struct ring_buffer_event *event; 7669 struct trace_buffer *buffer; 7670 struct raw_data_entry *entry; 7671 ssize_t written; 7672 size_t size; 7673 7674 /* cnt includes both the entry->id and the data behind it. */ 7675 size = struct_offset(entry, id) + cnt; 7676 7677 buffer = tr->array_buffer.buffer; 7678 7679 if (size > ring_buffer_max_event_size(buffer)) 7680 return -EINVAL; 7681 7682 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, 7683 tracing_gen_ctx()); 7684 if (!event) 7685 /* Ring buffer disabled, return as if not open for write */ 7686 return -EBADF; 7687 7688 entry = ring_buffer_event_data(event); 7689 unsafe_memcpy(&entry->id, buf, cnt, 7690 "id and content already reserved on ring buffer" 7691 "'buf' includes the 'id' and the data." 7692 "'entry' was allocated with cnt from 'id'."); 7693 written = cnt; 7694 7695 __buffer_unlock_commit(buffer, event); 7696 7697 return written; 7698} 7699 7700static ssize_t 7701tracing_mark_raw_write(struct file *filp, const char __user *ubuf, 7702 size_t cnt, loff_t *fpos) 7703{ 7704 struct trace_array *tr = filp->private_data; 7705 ssize_t written = -ENODEV; 7706 char *buf; 7707 7708 if (tracing_disabled) 7709 return -EINVAL; 7710 7711 if (!(tr->trace_flags & TRACE_ITER(MARKERS))) 7712 return -EINVAL; 7713 7714 /* The marker must at least have a tag id */ 7715 if (cnt < sizeof(unsigned int)) 7716 return -EINVAL; 7717 7718 /* raw write is all or nothing */ 7719 if (cnt > TRACE_MARKER_MAX_SIZE) 7720 return -EINVAL; 7721 7722 /* Must have preemption disabled while having access to the buffer */ 7723 guard(preempt_notrace)(); 7724 7725 buf = trace_user_fault_read(trace_user_buffer, ubuf, cnt, NULL, NULL); 7726 if (!buf) 7727 return -EFAULT; 7728 7729 /* The global trace_marker_raw can go to multiple instances */ 7730 if (tr == &global_trace) { 7731 guard(rcu)(); 7732 list_for_each_entry_rcu(tr, &marker_copies, marker_list) { 7733 written = write_raw_marker_to_buffer(tr, buf, cnt); 7734 if (written < 0) 7735 break; 7736 } 7737 } else { 7738 written = write_raw_marker_to_buffer(tr, buf, cnt); 7739 } 7740 7741 return written; 7742} 7743 7744static int tracing_mark_open(struct inode *inode, struct file *filp) 7745{ 7746 int ret; 7747 7748 scoped_guard(mutex, &trace_user_buffer_mutex) { 7749 if (!trace_user_buffer) { 7750 ret = user_buffer_init(&trace_user_buffer, TRACE_MARKER_MAX_SIZE); 7751 if (ret < 0) 7752 return ret; 7753 } else { 7754 trace_user_buffer->ref++; 7755 } 7756 } 7757 7758 stream_open(inode, filp); 7759 ret = tracing_open_generic_tr(inode, filp); 7760 if (ret < 0) 7761 user_buffer_put(&trace_user_buffer); 7762 return ret; 7763} 7764 7765static int tracing_mark_release(struct inode *inode, struct file *file) 7766{ 7767 user_buffer_put(&trace_user_buffer); 7768 return tracing_release_generic_tr(inode, file); 7769} 7770 7771static int tracing_clock_show(struct seq_file *m, void *v) 7772{ 7773 struct trace_array *tr = m->private; 7774 int i; 7775 7776 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 7777 seq_printf(m, 7778 "%s%s%s%s", i ? " " : "", 7779 i == tr->clock_id ? "[" : "", trace_clocks[i].name, 7780 i == tr->clock_id ? "]" : ""); 7781 seq_putc(m, '\n'); 7782 7783 return 0; 7784} 7785 7786int tracing_set_clock(struct trace_array *tr, const char *clockstr) 7787{ 7788 int i; 7789 7790 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 7791 if (strcmp(trace_clocks[i].name, clockstr) == 0) 7792 break; 7793 } 7794 if (i == ARRAY_SIZE(trace_clocks)) 7795 return -EINVAL; 7796 7797 guard(mutex)(&trace_types_lock); 7798 7799 tr->clock_id = i; 7800 7801 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func); 7802 7803 /* 7804 * New clock may not be consistent with the previous clock. 7805 * Reset the buffer so that it doesn't have incomparable timestamps. 7806 */ 7807 tracing_reset_online_cpus(&tr->array_buffer); 7808 7809#ifdef CONFIG_TRACER_MAX_TRACE 7810 if (tr->max_buffer.buffer) 7811 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); 7812 tracing_reset_online_cpus(&tr->max_buffer); 7813#endif 7814 update_last_data_if_empty(tr); 7815 7816 if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) { 7817 struct trace_scratch *tscratch = tr->scratch; 7818 7819 tscratch->clock_id = i; 7820 } 7821 7822 return 0; 7823} 7824 7825static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 7826 size_t cnt, loff_t *fpos) 7827{ 7828 struct seq_file *m = filp->private_data; 7829 struct trace_array *tr = m->private; 7830 char buf[64]; 7831 const char *clockstr; 7832 int ret; 7833 7834 if (cnt >= sizeof(buf)) 7835 return -EINVAL; 7836 7837 if (copy_from_user(buf, ubuf, cnt)) 7838 return -EFAULT; 7839 7840 buf[cnt] = 0; 7841 7842 clockstr = strstrip(buf); 7843 7844 ret = tracing_set_clock(tr, clockstr); 7845 if (ret) 7846 return ret; 7847 7848 *fpos += cnt; 7849 7850 return cnt; 7851} 7852 7853static int tracing_clock_open(struct inode *inode, struct file *file) 7854{ 7855 struct trace_array *tr = inode->i_private; 7856 int ret; 7857 7858 ret = tracing_check_open_get_tr(tr); 7859 if (ret) 7860 return ret; 7861 7862 ret = single_open(file, tracing_clock_show, inode->i_private); 7863 if (ret < 0) 7864 trace_array_put(tr); 7865 7866 return ret; 7867} 7868 7869static int tracing_time_stamp_mode_show(struct seq_file *m, void *v) 7870{ 7871 struct trace_array *tr = m->private; 7872 7873 guard(mutex)(&trace_types_lock); 7874 7875 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer)) 7876 seq_puts(m, "delta [absolute]\n"); 7877 else 7878 seq_puts(m, "[delta] absolute\n"); 7879 7880 return 0; 7881} 7882 7883static int tracing_time_stamp_mode_open(struct inode *inode, struct file *file) 7884{ 7885 struct trace_array *tr = inode->i_private; 7886 int ret; 7887 7888 ret = tracing_check_open_get_tr(tr); 7889 if (ret) 7890 return ret; 7891 7892 ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private); 7893 if (ret < 0) 7894 trace_array_put(tr); 7895 7896 return ret; 7897} 7898 7899u64 tracing_event_time_stamp(struct trace_buffer *buffer, struct ring_buffer_event *rbe) 7900{ 7901 if (rbe == this_cpu_read(trace_buffered_event)) 7902 return ring_buffer_time_stamp(buffer); 7903 7904 return ring_buffer_event_time_stamp(buffer, rbe); 7905} 7906 7907/* 7908 * Set or disable using the per CPU trace_buffer_event when possible. 7909 */ 7910int tracing_set_filter_buffering(struct trace_array *tr, bool set) 7911{ 7912 guard(mutex)(&trace_types_lock); 7913 7914 if (set && tr->no_filter_buffering_ref++) 7915 return 0; 7916 7917 if (!set) { 7918 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) 7919 return -EINVAL; 7920 7921 --tr->no_filter_buffering_ref; 7922 } 7923 7924 return 0; 7925} 7926 7927struct ftrace_buffer_info { 7928 struct trace_iterator iter; 7929 void *spare; 7930 unsigned int spare_cpu; 7931 unsigned int spare_size; 7932 unsigned int read; 7933}; 7934 7935#ifdef CONFIG_TRACER_SNAPSHOT 7936static int tracing_snapshot_open(struct inode *inode, struct file *file) 7937{ 7938 struct trace_array *tr = inode->i_private; 7939 struct trace_iterator *iter; 7940 struct seq_file *m; 7941 int ret; 7942 7943 ret = tracing_check_open_get_tr(tr); 7944 if (ret) 7945 return ret; 7946 7947 if (file->f_mode & FMODE_READ) { 7948 iter = __tracing_open(inode, file, true); 7949 if (IS_ERR(iter)) 7950 ret = PTR_ERR(iter); 7951 } else { 7952 /* Writes still need the seq_file to hold the private data */ 7953 ret = -ENOMEM; 7954 m = kzalloc(sizeof(*m), GFP_KERNEL); 7955 if (!m) 7956 goto out; 7957 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 7958 if (!iter) { 7959 kfree(m); 7960 goto out; 7961 } 7962 ret = 0; 7963 7964 iter->tr = tr; 7965 iter->array_buffer = &tr->max_buffer; 7966 iter->cpu_file = tracing_get_cpu(inode); 7967 m->private = iter; 7968 file->private_data = m; 7969 } 7970out: 7971 if (ret < 0) 7972 trace_array_put(tr); 7973 7974 return ret; 7975} 7976 7977static void tracing_swap_cpu_buffer(void *tr) 7978{ 7979 update_max_tr_single((struct trace_array *)tr, current, smp_processor_id()); 7980} 7981 7982static ssize_t 7983tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt, 7984 loff_t *ppos) 7985{ 7986 struct seq_file *m = filp->private_data; 7987 struct trace_iterator *iter = m->private; 7988 struct trace_array *tr = iter->tr; 7989 unsigned long val; 7990 int ret; 7991 7992 ret = tracing_update_buffers(tr); 7993 if (ret < 0) 7994 return ret; 7995 7996 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 7997 if (ret) 7998 return ret; 7999 8000 guard(mutex)(&trace_types_lock); 8001 8002 if (tr->current_trace->use_max_tr) 8003 return -EBUSY; 8004 8005 local_irq_disable(); 8006 arch_spin_lock(&tr->max_lock); 8007 if (tr->cond_snapshot) 8008 ret = -EBUSY; 8009 arch_spin_unlock(&tr->max_lock); 8010 local_irq_enable(); 8011 if (ret) 8012 return ret; 8013 8014 switch (val) { 8015 case 0: 8016 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) 8017 return -EINVAL; 8018 if (tr->allocated_snapshot) 8019 free_snapshot(tr); 8020 break; 8021 case 1: 8022/* Only allow per-cpu swap if the ring buffer supports it */ 8023#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP 8024 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) 8025 return -EINVAL; 8026#endif 8027 if (tr->allocated_snapshot) 8028 ret = resize_buffer_duplicate_size(&tr->max_buffer, 8029 &tr->array_buffer, iter->cpu_file); 8030 8031 ret = tracing_arm_snapshot_locked(tr); 8032 if (ret) 8033 return ret; 8034 8035 /* Now, we're going to swap */ 8036 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { 8037 local_irq_disable(); 8038 update_max_tr(tr, current, smp_processor_id(), NULL); 8039 local_irq_enable(); 8040 } else { 8041 smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer, 8042 (void *)tr, 1); 8043 } 8044 tracing_disarm_snapshot(tr); 8045 break; 8046 default: 8047 if (tr->allocated_snapshot) { 8048 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) 8049 tracing_reset_online_cpus(&tr->max_buffer); 8050 else 8051 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file); 8052 } 8053 break; 8054 } 8055 8056 if (ret >= 0) { 8057 *ppos += cnt; 8058 ret = cnt; 8059 } 8060 8061 return ret; 8062} 8063 8064static int tracing_snapshot_release(struct inode *inode, struct file *file) 8065{ 8066 struct seq_file *m = file->private_data; 8067 int ret; 8068 8069 ret = tracing_release(inode, file); 8070 8071 if (file->f_mode & FMODE_READ) 8072 return ret; 8073 8074 /* If write only, the seq_file is just a stub */ 8075 if (m) 8076 kfree(m->private); 8077 kfree(m); 8078 8079 return 0; 8080} 8081 8082static int tracing_buffers_open(struct inode *inode, struct file *filp); 8083static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf, 8084 size_t count, loff_t *ppos); 8085static int tracing_buffers_release(struct inode *inode, struct file *file); 8086static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos, 8087 struct pipe_inode_info *pipe, size_t len, unsigned int flags); 8088 8089static int snapshot_raw_open(struct inode *inode, struct file *filp) 8090{ 8091 struct ftrace_buffer_info *info; 8092 int ret; 8093 8094 /* The following checks for tracefs lockdown */ 8095 ret = tracing_buffers_open(inode, filp); 8096 if (ret < 0) 8097 return ret; 8098 8099 info = filp->private_data; 8100 8101 if (info->iter.trace->use_max_tr) { 8102 tracing_buffers_release(inode, filp); 8103 return -EBUSY; 8104 } 8105 8106 info->iter.snapshot = true; 8107 info->iter.array_buffer = &info->iter.tr->max_buffer; 8108 8109 return ret; 8110} 8111 8112#endif /* CONFIG_TRACER_SNAPSHOT */ 8113 8114 8115static const struct file_operations tracing_thresh_fops = { 8116 .open = tracing_open_generic, 8117 .read = tracing_thresh_read, 8118 .write = tracing_thresh_write, 8119 .llseek = generic_file_llseek, 8120}; 8121 8122#ifdef CONFIG_TRACER_MAX_TRACE 8123static const struct file_operations tracing_max_lat_fops = { 8124 .open = tracing_open_generic_tr, 8125 .read = tracing_max_lat_read, 8126 .write = tracing_max_lat_write, 8127 .llseek = generic_file_llseek, 8128 .release = tracing_release_generic_tr, 8129}; 8130#endif 8131 8132static const struct file_operations set_tracer_fops = { 8133 .open = tracing_open_generic_tr, 8134 .read = tracing_set_trace_read, 8135 .write = tracing_set_trace_write, 8136 .llseek = generic_file_llseek, 8137 .release = tracing_release_generic_tr, 8138}; 8139 8140static const struct file_operations tracing_pipe_fops = { 8141 .open = tracing_open_pipe, 8142 .poll = tracing_poll_pipe, 8143 .read = tracing_read_pipe, 8144 .splice_read = tracing_splice_read_pipe, 8145 .release = tracing_release_pipe, 8146}; 8147 8148static const struct file_operations tracing_entries_fops = { 8149 .open = tracing_open_generic_tr, 8150 .read = tracing_entries_read, 8151 .write = tracing_entries_write, 8152 .llseek = generic_file_llseek, 8153 .release = tracing_release_generic_tr, 8154}; 8155 8156static const struct file_operations tracing_syscall_buf_fops = { 8157 .open = tracing_open_generic_tr, 8158 .read = tracing_syscall_buf_read, 8159 .write = tracing_syscall_buf_write, 8160 .llseek = generic_file_llseek, 8161 .release = tracing_release_generic_tr, 8162}; 8163 8164static const struct file_operations tracing_buffer_meta_fops = { 8165 .open = tracing_buffer_meta_open, 8166 .read = seq_read, 8167 .llseek = seq_lseek, 8168 .release = tracing_seq_release, 8169}; 8170 8171static const struct file_operations tracing_total_entries_fops = { 8172 .open = tracing_open_generic_tr, 8173 .read = tracing_total_entries_read, 8174 .llseek = generic_file_llseek, 8175 .release = tracing_release_generic_tr, 8176}; 8177 8178static const struct file_operations tracing_free_buffer_fops = { 8179 .open = tracing_open_generic_tr, 8180 .write = tracing_free_buffer_write, 8181 .release = tracing_free_buffer_release, 8182}; 8183 8184static const struct file_operations tracing_mark_fops = { 8185 .open = tracing_mark_open, 8186 .write = tracing_mark_write, 8187 .release = tracing_mark_release, 8188}; 8189 8190static const struct file_operations tracing_mark_raw_fops = { 8191 .open = tracing_mark_open, 8192 .write = tracing_mark_raw_write, 8193 .release = tracing_mark_release, 8194}; 8195 8196static const struct file_operations trace_clock_fops = { 8197 .open = tracing_clock_open, 8198 .read = seq_read, 8199 .llseek = seq_lseek, 8200 .release = tracing_single_release_tr, 8201 .write = tracing_clock_write, 8202}; 8203 8204static const struct file_operations trace_time_stamp_mode_fops = { 8205 .open = tracing_time_stamp_mode_open, 8206 .read = seq_read, 8207 .llseek = seq_lseek, 8208 .release = tracing_single_release_tr, 8209}; 8210 8211static const struct file_operations last_boot_fops = { 8212 .open = tracing_last_boot_open, 8213 .read = seq_read, 8214 .llseek = seq_lseek, 8215 .release = tracing_seq_release, 8216}; 8217 8218#ifdef CONFIG_TRACER_SNAPSHOT 8219static const struct file_operations snapshot_fops = { 8220 .open = tracing_snapshot_open, 8221 .read = seq_read, 8222 .write = tracing_snapshot_write, 8223 .llseek = tracing_lseek, 8224 .release = tracing_snapshot_release, 8225}; 8226 8227static const struct file_operations snapshot_raw_fops = { 8228 .open = snapshot_raw_open, 8229 .read = tracing_buffers_read, 8230 .release = tracing_buffers_release, 8231 .splice_read = tracing_buffers_splice_read, 8232}; 8233 8234#endif /* CONFIG_TRACER_SNAPSHOT */ 8235 8236/* 8237 * trace_min_max_write - Write a u64 value to a trace_min_max_param struct 8238 * @filp: The active open file structure 8239 * @ubuf: The userspace provided buffer to read value into 8240 * @cnt: The maximum number of bytes to read 8241 * @ppos: The current "file" position 8242 * 8243 * This function implements the write interface for a struct trace_min_max_param. 8244 * The filp->private_data must point to a trace_min_max_param structure that 8245 * defines where to write the value, the min and the max acceptable values, 8246 * and a lock to protect the write. 8247 */ 8248static ssize_t 8249trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) 8250{ 8251 struct trace_min_max_param *param = filp->private_data; 8252 u64 val; 8253 int err; 8254 8255 if (!param) 8256 return -EFAULT; 8257 8258 err = kstrtoull_from_user(ubuf, cnt, 10, &val); 8259 if (err) 8260 return err; 8261 8262 if (param->lock) 8263 mutex_lock(param->lock); 8264 8265 if (param->min && val < *param->min) 8266 err = -EINVAL; 8267 8268 if (param->max && val > *param->max) 8269 err = -EINVAL; 8270 8271 if (!err) 8272 *param->val = val; 8273 8274 if (param->lock) 8275 mutex_unlock(param->lock); 8276 8277 if (err) 8278 return err; 8279 8280 return cnt; 8281} 8282 8283/* 8284 * trace_min_max_read - Read a u64 value from a trace_min_max_param struct 8285 * @filp: The active open file structure 8286 * @ubuf: The userspace provided buffer to read value into 8287 * @cnt: The maximum number of bytes to read 8288 * @ppos: The current "file" position 8289 * 8290 * This function implements the read interface for a struct trace_min_max_param. 8291 * The filp->private_data must point to a trace_min_max_param struct with valid 8292 * data. 8293 */ 8294static ssize_t 8295trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) 8296{ 8297 struct trace_min_max_param *param = filp->private_data; 8298 char buf[U64_STR_SIZE]; 8299 int len; 8300 u64 val; 8301 8302 if (!param) 8303 return -EFAULT; 8304 8305 val = *param->val; 8306 8307 if (cnt > sizeof(buf)) 8308 cnt = sizeof(buf); 8309 8310 len = snprintf(buf, sizeof(buf), "%llu\n", val); 8311 8312 return simple_read_from_buffer(ubuf, cnt, ppos, buf, len); 8313} 8314 8315const struct file_operations trace_min_max_fops = { 8316 .open = tracing_open_generic, 8317 .read = trace_min_max_read, 8318 .write = trace_min_max_write, 8319}; 8320 8321#define TRACING_LOG_ERRS_MAX 8 8322#define TRACING_LOG_LOC_MAX 128 8323 8324#define CMD_PREFIX " Command: " 8325 8326struct err_info { 8327 const char **errs; /* ptr to loc-specific array of err strings */ 8328 u8 type; /* index into errs -> specific err string */ 8329 u16 pos; /* caret position */ 8330 u64 ts; 8331}; 8332 8333struct tracing_log_err { 8334 struct list_head list; 8335 struct err_info info; 8336 char loc[TRACING_LOG_LOC_MAX]; /* err location */ 8337 char *cmd; /* what caused err */ 8338}; 8339 8340static DEFINE_MUTEX(tracing_err_log_lock); 8341 8342static struct tracing_log_err *alloc_tracing_log_err(int len) 8343{ 8344 struct tracing_log_err *err; 8345 8346 err = kzalloc(sizeof(*err), GFP_KERNEL); 8347 if (!err) 8348 return ERR_PTR(-ENOMEM); 8349 8350 err->cmd = kzalloc(len, GFP_KERNEL); 8351 if (!err->cmd) { 8352 kfree(err); 8353 return ERR_PTR(-ENOMEM); 8354 } 8355 8356 return err; 8357} 8358 8359static void free_tracing_log_err(struct tracing_log_err *err) 8360{ 8361 kfree(err->cmd); 8362 kfree(err); 8363} 8364 8365static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr, 8366 int len) 8367{ 8368 struct tracing_log_err *err; 8369 char *cmd; 8370 8371 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) { 8372 err = alloc_tracing_log_err(len); 8373 if (PTR_ERR(err) != -ENOMEM) 8374 tr->n_err_log_entries++; 8375 8376 return err; 8377 } 8378 cmd = kzalloc(len, GFP_KERNEL); 8379 if (!cmd) 8380 return ERR_PTR(-ENOMEM); 8381 err = list_first_entry(&tr->err_log, struct tracing_log_err, list); 8382 kfree(err->cmd); 8383 err->cmd = cmd; 8384 list_del(&err->list); 8385 8386 return err; 8387} 8388 8389/** 8390 * err_pos - find the position of a string within a command for error careting 8391 * @cmd: The tracing command that caused the error 8392 * @str: The string to position the caret at within @cmd 8393 * 8394 * Finds the position of the first occurrence of @str within @cmd. The 8395 * return value can be passed to tracing_log_err() for caret placement 8396 * within @cmd. 8397 * 8398 * Returns the index within @cmd of the first occurrence of @str or 0 8399 * if @str was not found. 8400 */ 8401unsigned int err_pos(char *cmd, const char *str) 8402{ 8403 char *found; 8404 8405 if (WARN_ON(!strlen(cmd))) 8406 return 0; 8407 8408 found = strstr(cmd, str); 8409 if (found) 8410 return found - cmd; 8411 8412 return 0; 8413} 8414 8415/** 8416 * tracing_log_err - write an error to the tracing error log 8417 * @tr: The associated trace array for the error (NULL for top level array) 8418 * @loc: A string describing where the error occurred 8419 * @cmd: The tracing command that caused the error 8420 * @errs: The array of loc-specific static error strings 8421 * @type: The index into errs[], which produces the specific static err string 8422 * @pos: The position the caret should be placed in the cmd 8423 * 8424 * Writes an error into tracing/error_log of the form: 8425 * 8426 * <loc>: error: <text> 8427 * Command: <cmd> 8428 * ^ 8429 * 8430 * tracing/error_log is a small log file containing the last 8431 * TRACING_LOG_ERRS_MAX errors (8). Memory for errors isn't allocated 8432 * unless there has been a tracing error, and the error log can be 8433 * cleared and have its memory freed by writing the empty string in 8434 * truncation mode to it i.e. echo > tracing/error_log. 8435 * 8436 * NOTE: the @errs array along with the @type param are used to 8437 * produce a static error string - this string is not copied and saved 8438 * when the error is logged - only a pointer to it is saved. See 8439 * existing callers for examples of how static strings are typically 8440 * defined for use with tracing_log_err(). 8441 */ 8442void tracing_log_err(struct trace_array *tr, 8443 const char *loc, const char *cmd, 8444 const char **errs, u8 type, u16 pos) 8445{ 8446 struct tracing_log_err *err; 8447 int len = 0; 8448 8449 if (!tr) 8450 tr = &global_trace; 8451 8452 len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1; 8453 8454 guard(mutex)(&tracing_err_log_lock); 8455 8456 err = get_tracing_log_err(tr, len); 8457 if (PTR_ERR(err) == -ENOMEM) 8458 return; 8459 8460 snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc); 8461 snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd); 8462 8463 err->info.errs = errs; 8464 err->info.type = type; 8465 err->info.pos = pos; 8466 err->info.ts = local_clock(); 8467 8468 list_add_tail(&err->list, &tr->err_log); 8469} 8470 8471static void clear_tracing_err_log(struct trace_array *tr) 8472{ 8473 struct tracing_log_err *err, *next; 8474 8475 guard(mutex)(&tracing_err_log_lock); 8476 8477 list_for_each_entry_safe(err, next, &tr->err_log, list) { 8478 list_del(&err->list); 8479 free_tracing_log_err(err); 8480 } 8481 8482 tr->n_err_log_entries = 0; 8483} 8484 8485static void *tracing_err_log_seq_start(struct seq_file *m, loff_t *pos) 8486{ 8487 struct trace_array *tr = m->private; 8488 8489 mutex_lock(&tracing_err_log_lock); 8490 8491 return seq_list_start(&tr->err_log, *pos); 8492} 8493 8494static void *tracing_err_log_seq_next(struct seq_file *m, void *v, loff_t *pos) 8495{ 8496 struct trace_array *tr = m->private; 8497 8498 return seq_list_next(v, &tr->err_log, pos); 8499} 8500 8501static void tracing_err_log_seq_stop(struct seq_file *m, void *v) 8502{ 8503 mutex_unlock(&tracing_err_log_lock); 8504} 8505 8506static void tracing_err_log_show_pos(struct seq_file *m, u16 pos) 8507{ 8508 u16 i; 8509 8510 for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++) 8511 seq_putc(m, ' '); 8512 for (i = 0; i < pos; i++) 8513 seq_putc(m, ' '); 8514 seq_puts(m, "^\n"); 8515} 8516 8517static int tracing_err_log_seq_show(struct seq_file *m, void *v) 8518{ 8519 struct tracing_log_err *err = v; 8520 8521 if (err) { 8522 const char *err_text = err->info.errs[err->info.type]; 8523 u64 sec = err->info.ts; 8524 u32 nsec; 8525 8526 nsec = do_div(sec, NSEC_PER_SEC); 8527 seq_printf(m, "[%5llu.%06u] %s%s", sec, nsec / 1000, 8528 err->loc, err_text); 8529 seq_printf(m, "%s", err->cmd); 8530 tracing_err_log_show_pos(m, err->info.pos); 8531 } 8532 8533 return 0; 8534} 8535 8536static const struct seq_operations tracing_err_log_seq_ops = { 8537 .start = tracing_err_log_seq_start, 8538 .next = tracing_err_log_seq_next, 8539 .stop = tracing_err_log_seq_stop, 8540 .show = tracing_err_log_seq_show 8541}; 8542 8543static int tracing_err_log_open(struct inode *inode, struct file *file) 8544{ 8545 struct trace_array *tr = inode->i_private; 8546 int ret = 0; 8547 8548 ret = tracing_check_open_get_tr(tr); 8549 if (ret) 8550 return ret; 8551 8552 /* If this file was opened for write, then erase contents */ 8553 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) 8554 clear_tracing_err_log(tr); 8555 8556 if (file->f_mode & FMODE_READ) { 8557 ret = seq_open(file, &tracing_err_log_seq_ops); 8558 if (!ret) { 8559 struct seq_file *m = file->private_data; 8560 m->private = tr; 8561 } else { 8562 trace_array_put(tr); 8563 } 8564 } 8565 return ret; 8566} 8567 8568static ssize_t tracing_err_log_write(struct file *file, 8569 const char __user *buffer, 8570 size_t count, loff_t *ppos) 8571{ 8572 return count; 8573} 8574 8575static int tracing_err_log_release(struct inode *inode, struct file *file) 8576{ 8577 struct trace_array *tr = inode->i_private; 8578 8579 trace_array_put(tr); 8580 8581 if (file->f_mode & FMODE_READ) 8582 seq_release(inode, file); 8583 8584 return 0; 8585} 8586 8587static const struct file_operations tracing_err_log_fops = { 8588 .open = tracing_err_log_open, 8589 .write = tracing_err_log_write, 8590 .read = seq_read, 8591 .llseek = tracing_lseek, 8592 .release = tracing_err_log_release, 8593}; 8594 8595static int tracing_buffers_open(struct inode *inode, struct file *filp) 8596{ 8597 struct trace_array *tr = inode->i_private; 8598 struct ftrace_buffer_info *info; 8599 int ret; 8600 8601 ret = tracing_check_open_get_tr(tr); 8602 if (ret) 8603 return ret; 8604 8605 info = kvzalloc(sizeof(*info), GFP_KERNEL); 8606 if (!info) { 8607 trace_array_put(tr); 8608 return -ENOMEM; 8609 } 8610 8611 mutex_lock(&trace_types_lock); 8612 8613 info->iter.tr = tr; 8614 info->iter.cpu_file = tracing_get_cpu(inode); 8615 info->iter.trace = tr->current_trace; 8616 info->iter.array_buffer = &tr->array_buffer; 8617 info->spare = NULL; 8618 /* Force reading ring buffer for first read */ 8619 info->read = (unsigned int)-1; 8620 8621 filp->private_data = info; 8622 8623 tr->trace_ref++; 8624 8625 mutex_unlock(&trace_types_lock); 8626 8627 ret = nonseekable_open(inode, filp); 8628 if (ret < 0) 8629 trace_array_put(tr); 8630 8631 return ret; 8632} 8633 8634static __poll_t 8635tracing_buffers_poll(struct file *filp, poll_table *poll_table) 8636{ 8637 struct ftrace_buffer_info *info = filp->private_data; 8638 struct trace_iterator *iter = &info->iter; 8639 8640 return trace_poll(iter, filp, poll_table); 8641} 8642 8643static ssize_t 8644tracing_buffers_read(struct file *filp, char __user *ubuf, 8645 size_t count, loff_t *ppos) 8646{ 8647 struct ftrace_buffer_info *info = filp->private_data; 8648 struct trace_iterator *iter = &info->iter; 8649 void *trace_data; 8650 int page_size; 8651 ssize_t ret = 0; 8652 ssize_t size; 8653 8654 if (!count) 8655 return 0; 8656 8657#ifdef CONFIG_TRACER_MAX_TRACE 8658 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 8659 return -EBUSY; 8660#endif 8661 8662 page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); 8663 8664 /* Make sure the spare matches the current sub buffer size */ 8665 if (info->spare) { 8666 if (page_size != info->spare_size) { 8667 ring_buffer_free_read_page(iter->array_buffer->buffer, 8668 info->spare_cpu, info->spare); 8669 info->spare = NULL; 8670 } 8671 } 8672 8673 if (!info->spare) { 8674 info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer, 8675 iter->cpu_file); 8676 if (IS_ERR(info->spare)) { 8677 ret = PTR_ERR(info->spare); 8678 info->spare = NULL; 8679 } else { 8680 info->spare_cpu = iter->cpu_file; 8681 info->spare_size = page_size; 8682 } 8683 } 8684 if (!info->spare) 8685 return ret; 8686 8687 /* Do we have previous read data to read? */ 8688 if (info->read < page_size) 8689 goto read; 8690 8691 again: 8692 trace_access_lock(iter->cpu_file); 8693 ret = ring_buffer_read_page(iter->array_buffer->buffer, 8694 info->spare, 8695 count, 8696 iter->cpu_file, 0); 8697 trace_access_unlock(iter->cpu_file); 8698 8699 if (ret < 0) { 8700 if (trace_empty(iter) && !iter->closed) { 8701 if (update_last_data_if_empty(iter->tr)) 8702 return 0; 8703 8704 if ((filp->f_flags & O_NONBLOCK)) 8705 return -EAGAIN; 8706 8707 ret = wait_on_pipe(iter, 0); 8708 if (ret) 8709 return ret; 8710 8711 goto again; 8712 } 8713 return 0; 8714 } 8715 8716 info->read = 0; 8717 read: 8718 size = page_size - info->read; 8719 if (size > count) 8720 size = count; 8721 trace_data = ring_buffer_read_page_data(info->spare); 8722 ret = copy_to_user(ubuf, trace_data + info->read, size); 8723 if (ret == size) 8724 return -EFAULT; 8725 8726 size -= ret; 8727 8728 *ppos += size; 8729 info->read += size; 8730 8731 return size; 8732} 8733 8734static int tracing_buffers_flush(struct file *file, fl_owner_t id) 8735{ 8736 struct ftrace_buffer_info *info = file->private_data; 8737 struct trace_iterator *iter = &info->iter; 8738 8739 iter->closed = true; 8740 /* Make sure the waiters see the new wait_index */ 8741 (void)atomic_fetch_inc_release(&iter->wait_index); 8742 8743 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); 8744 8745 return 0; 8746} 8747 8748static int tracing_buffers_release(struct inode *inode, struct file *file) 8749{ 8750 struct ftrace_buffer_info *info = file->private_data; 8751 struct trace_iterator *iter = &info->iter; 8752 8753 guard(mutex)(&trace_types_lock); 8754 8755 iter->tr->trace_ref--; 8756 8757 __trace_array_put(iter->tr); 8758 8759 if (info->spare) 8760 ring_buffer_free_read_page(iter->array_buffer->buffer, 8761 info->spare_cpu, info->spare); 8762 kvfree(info); 8763 8764 return 0; 8765} 8766 8767struct buffer_ref { 8768 struct trace_buffer *buffer; 8769 void *page; 8770 int cpu; 8771 refcount_t refcount; 8772}; 8773 8774static void buffer_ref_release(struct buffer_ref *ref) 8775{ 8776 if (!refcount_dec_and_test(&ref->refcount)) 8777 return; 8778 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page); 8779 kfree(ref); 8780} 8781 8782static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 8783 struct pipe_buffer *buf) 8784{ 8785 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 8786 8787 buffer_ref_release(ref); 8788 buf->private = 0; 8789} 8790 8791static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe, 8792 struct pipe_buffer *buf) 8793{ 8794 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 8795 8796 if (refcount_read(&ref->refcount) > INT_MAX/2) 8797 return false; 8798 8799 refcount_inc(&ref->refcount); 8800 return true; 8801} 8802 8803/* Pipe buffer operations for a buffer. */ 8804static const struct pipe_buf_operations buffer_pipe_buf_ops = { 8805 .release = buffer_pipe_buf_release, 8806 .get = buffer_pipe_buf_get, 8807}; 8808 8809/* 8810 * Callback from splice_to_pipe(), if we need to release some pages 8811 * at the end of the spd in case we error'ed out in filling the pipe. 8812 */ 8813static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 8814{ 8815 struct buffer_ref *ref = 8816 (struct buffer_ref *)spd->partial[i].private; 8817 8818 buffer_ref_release(ref); 8819 spd->partial[i].private = 0; 8820} 8821 8822static ssize_t 8823tracing_buffers_splice_read(struct file *file, loff_t *ppos, 8824 struct pipe_inode_info *pipe, size_t len, 8825 unsigned int flags) 8826{ 8827 struct ftrace_buffer_info *info = file->private_data; 8828 struct trace_iterator *iter = &info->iter; 8829 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 8830 struct page *pages_def[PIPE_DEF_BUFFERS]; 8831 struct splice_pipe_desc spd = { 8832 .pages = pages_def, 8833 .partial = partial_def, 8834 .nr_pages_max = PIPE_DEF_BUFFERS, 8835 .ops = &buffer_pipe_buf_ops, 8836 .spd_release = buffer_spd_release, 8837 }; 8838 struct buffer_ref *ref; 8839 bool woken = false; 8840 int page_size; 8841 int entries, i; 8842 ssize_t ret = 0; 8843 8844#ifdef CONFIG_TRACER_MAX_TRACE 8845 if (iter->snapshot && iter->tr->current_trace->use_max_tr) 8846 return -EBUSY; 8847#endif 8848 8849 page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer); 8850 if (*ppos & (page_size - 1)) 8851 return -EINVAL; 8852 8853 if (len & (page_size - 1)) { 8854 if (len < page_size) 8855 return -EINVAL; 8856 len &= (~(page_size - 1)); 8857 } 8858 8859 if (splice_grow_spd(pipe, &spd)) 8860 return -ENOMEM; 8861 8862 again: 8863 trace_access_lock(iter->cpu_file); 8864 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); 8865 8866 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) { 8867 struct page *page; 8868 int r; 8869 8870 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 8871 if (!ref) { 8872 ret = -ENOMEM; 8873 break; 8874 } 8875 8876 refcount_set(&ref->refcount, 1); 8877 ref->buffer = iter->array_buffer->buffer; 8878 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file); 8879 if (IS_ERR(ref->page)) { 8880 ret = PTR_ERR(ref->page); 8881 ref->page = NULL; 8882 kfree(ref); 8883 break; 8884 } 8885 ref->cpu = iter->cpu_file; 8886 8887 r = ring_buffer_read_page(ref->buffer, ref->page, 8888 len, iter->cpu_file, 1); 8889 if (r < 0) { 8890 ring_buffer_free_read_page(ref->buffer, ref->cpu, 8891 ref->page); 8892 kfree(ref); 8893 break; 8894 } 8895 8896 page = virt_to_page(ring_buffer_read_page_data(ref->page)); 8897 8898 spd.pages[i] = page; 8899 spd.partial[i].len = page_size; 8900 spd.partial[i].offset = 0; 8901 spd.partial[i].private = (unsigned long)ref; 8902 spd.nr_pages++; 8903 *ppos += page_size; 8904 8905 entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file); 8906 } 8907 8908 trace_access_unlock(iter->cpu_file); 8909 spd.nr_pages = i; 8910 8911 /* did we read anything? */ 8912 if (!spd.nr_pages) { 8913 8914 if (ret) 8915 goto out; 8916 8917 if (woken) 8918 goto out; 8919 8920 ret = -EAGAIN; 8921 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) 8922 goto out; 8923 8924 ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent); 8925 if (ret) 8926 goto out; 8927 8928 /* No need to wait after waking up when tracing is off */ 8929 if (!tracer_tracing_is_on(iter->tr)) 8930 goto out; 8931 8932 /* Iterate one more time to collect any new data then exit */ 8933 woken = true; 8934 8935 goto again; 8936 } 8937 8938 ret = splice_to_pipe(pipe, &spd); 8939out: 8940 splice_shrink_spd(&spd); 8941 8942 return ret; 8943} 8944 8945static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 8946{ 8947 struct ftrace_buffer_info *info = file->private_data; 8948 struct trace_iterator *iter = &info->iter; 8949 int err; 8950 8951 if (cmd == TRACE_MMAP_IOCTL_GET_READER) { 8952 if (!(file->f_flags & O_NONBLOCK)) { 8953 err = ring_buffer_wait(iter->array_buffer->buffer, 8954 iter->cpu_file, 8955 iter->tr->buffer_percent, 8956 NULL, NULL); 8957 if (err) 8958 return err; 8959 } 8960 8961 return ring_buffer_map_get_reader(iter->array_buffer->buffer, 8962 iter->cpu_file); 8963 } else if (cmd) { 8964 return -ENOTTY; 8965 } 8966 8967 /* 8968 * An ioctl call with cmd 0 to the ring buffer file will wake up all 8969 * waiters 8970 */ 8971 guard(mutex)(&trace_types_lock); 8972 8973 /* Make sure the waiters see the new wait_index */ 8974 (void)atomic_fetch_inc_release(&iter->wait_index); 8975 8976 ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file); 8977 8978 return 0; 8979} 8980 8981#ifdef CONFIG_TRACER_MAX_TRACE 8982static int get_snapshot_map(struct trace_array *tr) 8983{ 8984 int err = 0; 8985 8986 /* 8987 * Called with mmap_lock held. lockdep would be unhappy if we would now 8988 * take trace_types_lock. Instead use the specific 8989 * snapshot_trigger_lock. 8990 */ 8991 spin_lock(&tr->snapshot_trigger_lock); 8992 8993 if (tr->snapshot || tr->mapped == UINT_MAX) 8994 err = -EBUSY; 8995 else 8996 tr->mapped++; 8997 8998 spin_unlock(&tr->snapshot_trigger_lock); 8999 9000 /* Wait for update_max_tr() to observe iter->tr->mapped */ 9001 if (tr->mapped == 1) 9002 synchronize_rcu(); 9003 9004 return err; 9005 9006} 9007static void put_snapshot_map(struct trace_array *tr) 9008{ 9009 spin_lock(&tr->snapshot_trigger_lock); 9010 if (!WARN_ON(!tr->mapped)) 9011 tr->mapped--; 9012 spin_unlock(&tr->snapshot_trigger_lock); 9013} 9014#else 9015static inline int get_snapshot_map(struct trace_array *tr) { return 0; } 9016static inline void put_snapshot_map(struct trace_array *tr) { } 9017#endif 9018 9019/* 9020 * This is called when a VMA is duplicated (e.g., on fork()) to increment 9021 * the user_mapped counter without remapping pages. 9022 */ 9023static void tracing_buffers_mmap_open(struct vm_area_struct *vma) 9024{ 9025 struct ftrace_buffer_info *info = vma->vm_file->private_data; 9026 struct trace_iterator *iter = &info->iter; 9027 9028 ring_buffer_map_dup(iter->array_buffer->buffer, iter->cpu_file); 9029} 9030 9031static void tracing_buffers_mmap_close(struct vm_area_struct *vma) 9032{ 9033 struct ftrace_buffer_info *info = vma->vm_file->private_data; 9034 struct trace_iterator *iter = &info->iter; 9035 9036 WARN_ON(ring_buffer_unmap(iter->array_buffer->buffer, iter->cpu_file)); 9037 put_snapshot_map(iter->tr); 9038} 9039 9040static int tracing_buffers_may_split(struct vm_area_struct *vma, unsigned long addr) 9041{ 9042 /* 9043 * Trace buffer mappings require the complete buffer including 9044 * the meta page. Partial mappings are not supported. 9045 */ 9046 return -EINVAL; 9047} 9048 9049static const struct vm_operations_struct tracing_buffers_vmops = { 9050 .open = tracing_buffers_mmap_open, 9051 .close = tracing_buffers_mmap_close, 9052 .may_split = tracing_buffers_may_split, 9053}; 9054 9055static int tracing_buffers_mmap(struct file *filp, struct vm_area_struct *vma) 9056{ 9057 struct ftrace_buffer_info *info = filp->private_data; 9058 struct trace_iterator *iter = &info->iter; 9059 int ret = 0; 9060 9061 /* A memmap'ed and backup buffers are not supported for user space mmap */ 9062 if (iter->tr->flags & (TRACE_ARRAY_FL_MEMMAP | TRACE_ARRAY_FL_VMALLOC)) 9063 return -ENODEV; 9064 9065 ret = get_snapshot_map(iter->tr); 9066 if (ret) 9067 return ret; 9068 9069 ret = ring_buffer_map(iter->array_buffer->buffer, iter->cpu_file, vma); 9070 if (ret) 9071 put_snapshot_map(iter->tr); 9072 9073 vma->vm_ops = &tracing_buffers_vmops; 9074 9075 return ret; 9076} 9077 9078static const struct file_operations tracing_buffers_fops = { 9079 .open = tracing_buffers_open, 9080 .read = tracing_buffers_read, 9081 .poll = tracing_buffers_poll, 9082 .release = tracing_buffers_release, 9083 .flush = tracing_buffers_flush, 9084 .splice_read = tracing_buffers_splice_read, 9085 .unlocked_ioctl = tracing_buffers_ioctl, 9086 .mmap = tracing_buffers_mmap, 9087}; 9088 9089static ssize_t 9090tracing_stats_read(struct file *filp, char __user *ubuf, 9091 size_t count, loff_t *ppos) 9092{ 9093 struct inode *inode = file_inode(filp); 9094 struct trace_array *tr = inode->i_private; 9095 struct array_buffer *trace_buf = &tr->array_buffer; 9096 int cpu = tracing_get_cpu(inode); 9097 struct trace_seq *s; 9098 unsigned long cnt; 9099 unsigned long long t; 9100 unsigned long usec_rem; 9101 9102 s = kmalloc(sizeof(*s), GFP_KERNEL); 9103 if (!s) 9104 return -ENOMEM; 9105 9106 trace_seq_init(s); 9107 9108 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu); 9109 trace_seq_printf(s, "entries: %ld\n", cnt); 9110 9111 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu); 9112 trace_seq_printf(s, "overrun: %ld\n", cnt); 9113 9114 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu); 9115 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 9116 9117 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu); 9118 trace_seq_printf(s, "bytes: %ld\n", cnt); 9119 9120 if (trace_clocks[tr->clock_id].in_ns) { 9121 /* local or global for trace_clock */ 9122 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 9123 usec_rem = do_div(t, USEC_PER_SEC); 9124 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", 9125 t, usec_rem); 9126 9127 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer)); 9128 usec_rem = do_div(t, USEC_PER_SEC); 9129 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem); 9130 } else { 9131 /* counter or tsc mode for trace_clock */ 9132 trace_seq_printf(s, "oldest event ts: %llu\n", 9133 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu)); 9134 9135 trace_seq_printf(s, "now ts: %llu\n", 9136 ring_buffer_time_stamp(trace_buf->buffer)); 9137 } 9138 9139 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu); 9140 trace_seq_printf(s, "dropped events: %ld\n", cnt); 9141 9142 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu); 9143 trace_seq_printf(s, "read events: %ld\n", cnt); 9144 9145 count = simple_read_from_buffer(ubuf, count, ppos, 9146 s->buffer, trace_seq_used(s)); 9147 9148 kfree(s); 9149 9150 return count; 9151} 9152 9153static const struct file_operations tracing_stats_fops = { 9154 .open = tracing_open_generic_tr, 9155 .read = tracing_stats_read, 9156 .llseek = generic_file_llseek, 9157 .release = tracing_release_generic_tr, 9158}; 9159 9160#ifdef CONFIG_DYNAMIC_FTRACE 9161 9162static ssize_t 9163tracing_read_dyn_info(struct file *filp, char __user *ubuf, 9164 size_t cnt, loff_t *ppos) 9165{ 9166 ssize_t ret; 9167 char *buf; 9168 int r; 9169 9170 /* 512 should be plenty to hold the amount needed */ 9171#define DYN_INFO_BUF_SIZE 512 9172 9173 buf = kmalloc(DYN_INFO_BUF_SIZE, GFP_KERNEL); 9174 if (!buf) 9175 return -ENOMEM; 9176 9177 r = scnprintf(buf, DYN_INFO_BUF_SIZE, 9178 "%ld pages:%ld groups: %ld\n" 9179 "ftrace boot update time = %llu (ns)\n" 9180 "ftrace module total update time = %llu (ns)\n", 9181 ftrace_update_tot_cnt, 9182 ftrace_number_of_pages, 9183 ftrace_number_of_groups, 9184 ftrace_update_time, 9185 ftrace_total_mod_time); 9186 9187 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 9188 kfree(buf); 9189 return ret; 9190} 9191 9192static const struct file_operations tracing_dyn_info_fops = { 9193 .open = tracing_open_generic, 9194 .read = tracing_read_dyn_info, 9195 .llseek = generic_file_llseek, 9196}; 9197#endif /* CONFIG_DYNAMIC_FTRACE */ 9198 9199#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) 9200static void 9201ftrace_snapshot(unsigned long ip, unsigned long parent_ip, 9202 struct trace_array *tr, struct ftrace_probe_ops *ops, 9203 void *data) 9204{ 9205 tracing_snapshot_instance(tr); 9206} 9207 9208static void 9209ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, 9210 struct trace_array *tr, struct ftrace_probe_ops *ops, 9211 void *data) 9212{ 9213 struct ftrace_func_mapper *mapper = data; 9214 long *count = NULL; 9215 9216 if (mapper) 9217 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 9218 9219 if (count) { 9220 9221 if (*count <= 0) 9222 return; 9223 9224 (*count)--; 9225 } 9226 9227 tracing_snapshot_instance(tr); 9228} 9229 9230static int 9231ftrace_snapshot_print(struct seq_file *m, unsigned long ip, 9232 struct ftrace_probe_ops *ops, void *data) 9233{ 9234 struct ftrace_func_mapper *mapper = data; 9235 long *count = NULL; 9236 9237 seq_printf(m, "%ps:", (void *)ip); 9238 9239 seq_puts(m, "snapshot"); 9240 9241 if (mapper) 9242 count = (long *)ftrace_func_mapper_find_ip(mapper, ip); 9243 9244 if (count) 9245 seq_printf(m, ":count=%ld\n", *count); 9246 else 9247 seq_puts(m, ":unlimited\n"); 9248 9249 return 0; 9250} 9251 9252static int 9253ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, 9254 unsigned long ip, void *init_data, void **data) 9255{ 9256 struct ftrace_func_mapper *mapper = *data; 9257 9258 if (!mapper) { 9259 mapper = allocate_ftrace_func_mapper(); 9260 if (!mapper) 9261 return -ENOMEM; 9262 *data = mapper; 9263 } 9264 9265 return ftrace_func_mapper_add_ip(mapper, ip, init_data); 9266} 9267 9268static void 9269ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, 9270 unsigned long ip, void *data) 9271{ 9272 struct ftrace_func_mapper *mapper = data; 9273 9274 if (!ip) { 9275 if (!mapper) 9276 return; 9277 free_ftrace_func_mapper(mapper, NULL); 9278 return; 9279 } 9280 9281 ftrace_func_mapper_remove_ip(mapper, ip); 9282} 9283 9284static struct ftrace_probe_ops snapshot_probe_ops = { 9285 .func = ftrace_snapshot, 9286 .print = ftrace_snapshot_print, 9287}; 9288 9289static struct ftrace_probe_ops snapshot_count_probe_ops = { 9290 .func = ftrace_count_snapshot, 9291 .print = ftrace_snapshot_print, 9292 .init = ftrace_snapshot_init, 9293 .free = ftrace_snapshot_free, 9294}; 9295 9296static int 9297ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, 9298 char *glob, char *cmd, char *param, int enable) 9299{ 9300 struct ftrace_probe_ops *ops; 9301 void *count = (void *)-1; 9302 char *number; 9303 int ret; 9304 9305 if (!tr) 9306 return -ENODEV; 9307 9308 /* hash funcs only work with set_ftrace_filter */ 9309 if (!enable) 9310 return -EINVAL; 9311 9312 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops; 9313 9314 if (glob[0] == '!') { 9315 ret = unregister_ftrace_function_probe_func(glob+1, tr, ops); 9316 if (!ret) 9317 tracing_disarm_snapshot(tr); 9318 9319 return ret; 9320 } 9321 9322 if (!param) 9323 goto out_reg; 9324 9325 number = strsep(¶m, ":"); 9326 9327 if (!strlen(number)) 9328 goto out_reg; 9329 9330 /* 9331 * We use the callback data field (which is a pointer) 9332 * as our counter. 9333 */ 9334 ret = kstrtoul(number, 0, (unsigned long *)&count); 9335 if (ret) 9336 return ret; 9337 9338 out_reg: 9339 ret = tracing_arm_snapshot(tr); 9340 if (ret < 0) 9341 return ret; 9342 9343 ret = register_ftrace_function_probe(glob, tr, ops, count); 9344 if (ret < 0) 9345 tracing_disarm_snapshot(tr); 9346 9347 return ret < 0 ? ret : 0; 9348} 9349 9350static struct ftrace_func_command ftrace_snapshot_cmd = { 9351 .name = "snapshot", 9352 .func = ftrace_trace_snapshot_callback, 9353}; 9354 9355static __init int register_snapshot_cmd(void) 9356{ 9357 return register_ftrace_command(&ftrace_snapshot_cmd); 9358} 9359#else 9360static inline __init int register_snapshot_cmd(void) { return 0; } 9361#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */ 9362 9363static struct dentry *tracing_get_dentry(struct trace_array *tr) 9364{ 9365 /* Top directory uses NULL as the parent */ 9366 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) 9367 return NULL; 9368 9369 if (WARN_ON(!tr->dir)) 9370 return ERR_PTR(-ENODEV); 9371 9372 /* All sub buffers have a descriptor */ 9373 return tr->dir; 9374} 9375 9376static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu) 9377{ 9378 struct dentry *d_tracer; 9379 9380 if (tr->percpu_dir) 9381 return tr->percpu_dir; 9382 9383 d_tracer = tracing_get_dentry(tr); 9384 if (IS_ERR(d_tracer)) 9385 return NULL; 9386 9387 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); 9388 9389 MEM_FAIL(!tr->percpu_dir, 9390 "Could not create tracefs directory 'per_cpu/%d'\n", cpu); 9391 9392 return tr->percpu_dir; 9393} 9394 9395static struct dentry * 9396trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent, 9397 void *data, long cpu, const struct file_operations *fops) 9398{ 9399 struct dentry *ret = trace_create_file(name, mode, parent, data, fops); 9400 9401 if (ret) /* See tracing_get_cpu() */ 9402 d_inode(ret)->i_cdev = (void *)(cpu + 1); 9403 return ret; 9404} 9405 9406static void 9407tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) 9408{ 9409 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu); 9410 struct dentry *d_cpu; 9411 char cpu_dir[30]; /* 30 characters should be more than enough */ 9412 9413 if (!d_percpu) 9414 return; 9415 9416 snprintf(cpu_dir, 30, "cpu%ld", cpu); 9417 d_cpu = tracefs_create_dir(cpu_dir, d_percpu); 9418 if (!d_cpu) { 9419 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir); 9420 return; 9421 } 9422 9423 /* per cpu trace_pipe */ 9424 trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu, 9425 tr, cpu, &tracing_pipe_fops); 9426 9427 /* per cpu trace */ 9428 trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu, 9429 tr, cpu, &tracing_fops); 9430 9431 trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu, 9432 tr, cpu, &tracing_buffers_fops); 9433 9434 trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu, 9435 tr, cpu, &tracing_stats_fops); 9436 9437 trace_create_cpu_file("buffer_size_kb", TRACE_MODE_WRITE, d_cpu, 9438 tr, cpu, &tracing_entries_fops); 9439 9440 if (tr->range_addr_start) 9441 trace_create_cpu_file("buffer_meta", TRACE_MODE_READ, d_cpu, 9442 tr, cpu, &tracing_buffer_meta_fops); 9443#ifdef CONFIG_TRACER_SNAPSHOT 9444 if (!tr->range_addr_start) { 9445 trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu, 9446 tr, cpu, &snapshot_fops); 9447 9448 trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu, 9449 tr, cpu, &snapshot_raw_fops); 9450 } 9451#endif 9452} 9453 9454#ifdef CONFIG_FTRACE_SELFTEST 9455/* Let selftest have access to static functions in this file */ 9456#include "trace_selftest.c" 9457#endif 9458 9459static ssize_t 9460trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 9461 loff_t *ppos) 9462{ 9463 struct trace_option_dentry *topt = filp->private_data; 9464 char *buf; 9465 9466 if (topt->flags->val & topt->opt->bit) 9467 buf = "1\n"; 9468 else 9469 buf = "0\n"; 9470 9471 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 9472} 9473 9474static ssize_t 9475trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 9476 loff_t *ppos) 9477{ 9478 struct trace_option_dentry *topt = filp->private_data; 9479 unsigned long val; 9480 int ret; 9481 9482 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9483 if (ret) 9484 return ret; 9485 9486 if (val != 0 && val != 1) 9487 return -EINVAL; 9488 9489 if (!!(topt->flags->val & topt->opt->bit) != val) { 9490 guard(mutex)(&trace_types_lock); 9491 ret = __set_tracer_option(topt->tr, topt->flags, 9492 topt->opt, !val); 9493 if (ret) 9494 return ret; 9495 } 9496 9497 *ppos += cnt; 9498 9499 return cnt; 9500} 9501 9502static int tracing_open_options(struct inode *inode, struct file *filp) 9503{ 9504 struct trace_option_dentry *topt = inode->i_private; 9505 int ret; 9506 9507 ret = tracing_check_open_get_tr(topt->tr); 9508 if (ret) 9509 return ret; 9510 9511 filp->private_data = inode->i_private; 9512 return 0; 9513} 9514 9515static int tracing_release_options(struct inode *inode, struct file *file) 9516{ 9517 struct trace_option_dentry *topt = file->private_data; 9518 9519 trace_array_put(topt->tr); 9520 return 0; 9521} 9522 9523static const struct file_operations trace_options_fops = { 9524 .open = tracing_open_options, 9525 .read = trace_options_read, 9526 .write = trace_options_write, 9527 .llseek = generic_file_llseek, 9528 .release = tracing_release_options, 9529}; 9530 9531/* 9532 * In order to pass in both the trace_array descriptor as well as the index 9533 * to the flag that the trace option file represents, the trace_array 9534 * has a character array of trace_flags_index[], which holds the index 9535 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc. 9536 * The address of this character array is passed to the flag option file 9537 * read/write callbacks. 9538 * 9539 * In order to extract both the index and the trace_array descriptor, 9540 * get_tr_index() uses the following algorithm. 9541 * 9542 * idx = *ptr; 9543 * 9544 * As the pointer itself contains the address of the index (remember 9545 * index[1] == 1). 9546 * 9547 * Then to get the trace_array descriptor, by subtracting that index 9548 * from the ptr, we get to the start of the index itself. 9549 * 9550 * ptr - idx == &index[0] 9551 * 9552 * Then a simple container_of() from that pointer gets us to the 9553 * trace_array descriptor. 9554 */ 9555static void get_tr_index(void *data, struct trace_array **ptr, 9556 unsigned int *pindex) 9557{ 9558 *pindex = *(unsigned char *)data; 9559 9560 *ptr = container_of(data - *pindex, struct trace_array, 9561 trace_flags_index); 9562} 9563 9564static ssize_t 9565trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 9566 loff_t *ppos) 9567{ 9568 void *tr_index = filp->private_data; 9569 struct trace_array *tr; 9570 unsigned int index; 9571 char *buf; 9572 9573 get_tr_index(tr_index, &tr, &index); 9574 9575 if (tr->trace_flags & (1ULL << index)) 9576 buf = "1\n"; 9577 else 9578 buf = "0\n"; 9579 9580 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 9581} 9582 9583static ssize_t 9584trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 9585 loff_t *ppos) 9586{ 9587 void *tr_index = filp->private_data; 9588 struct trace_array *tr; 9589 unsigned int index; 9590 unsigned long val; 9591 int ret; 9592 9593 get_tr_index(tr_index, &tr, &index); 9594 9595 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9596 if (ret) 9597 return ret; 9598 9599 if (val != 0 && val != 1) 9600 return -EINVAL; 9601 9602 mutex_lock(&event_mutex); 9603 mutex_lock(&trace_types_lock); 9604 ret = set_tracer_flag(tr, 1ULL << index, val); 9605 mutex_unlock(&trace_types_lock); 9606 mutex_unlock(&event_mutex); 9607 9608 if (ret < 0) 9609 return ret; 9610 9611 *ppos += cnt; 9612 9613 return cnt; 9614} 9615 9616static const struct file_operations trace_options_core_fops = { 9617 .open = tracing_open_generic, 9618 .read = trace_options_core_read, 9619 .write = trace_options_core_write, 9620 .llseek = generic_file_llseek, 9621}; 9622 9623struct dentry *trace_create_file(const char *name, 9624 umode_t mode, 9625 struct dentry *parent, 9626 void *data, 9627 const struct file_operations *fops) 9628{ 9629 struct dentry *ret; 9630 9631 ret = tracefs_create_file(name, mode, parent, data, fops); 9632 if (!ret) 9633 pr_warn("Could not create tracefs '%s' entry\n", name); 9634 9635 return ret; 9636} 9637 9638 9639static struct dentry *trace_options_init_dentry(struct trace_array *tr) 9640{ 9641 struct dentry *d_tracer; 9642 9643 if (tr->options) 9644 return tr->options; 9645 9646 d_tracer = tracing_get_dentry(tr); 9647 if (IS_ERR(d_tracer)) 9648 return NULL; 9649 9650 tr->options = tracefs_create_dir("options", d_tracer); 9651 if (!tr->options) { 9652 pr_warn("Could not create tracefs directory 'options'\n"); 9653 return NULL; 9654 } 9655 9656 return tr->options; 9657} 9658 9659static void 9660create_trace_option_file(struct trace_array *tr, 9661 struct trace_option_dentry *topt, 9662 struct tracer_flags *flags, 9663 struct tracer_opt *opt) 9664{ 9665 struct dentry *t_options; 9666 9667 t_options = trace_options_init_dentry(tr); 9668 if (!t_options) 9669 return; 9670 9671 topt->flags = flags; 9672 topt->opt = opt; 9673 topt->tr = tr; 9674 9675 topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE, 9676 t_options, topt, &trace_options_fops); 9677} 9678 9679static int 9680create_trace_option_files(struct trace_array *tr, struct tracer *tracer, 9681 struct tracer_flags *flags) 9682{ 9683 struct trace_option_dentry *topts; 9684 struct trace_options *tr_topts; 9685 struct tracer_opt *opts; 9686 int cnt; 9687 9688 if (!flags || !flags->opts) 9689 return 0; 9690 9691 opts = flags->opts; 9692 9693 for (cnt = 0; opts[cnt].name; cnt++) 9694 ; 9695 9696 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 9697 if (!topts) 9698 return 0; 9699 9700 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), 9701 GFP_KERNEL); 9702 if (!tr_topts) { 9703 kfree(topts); 9704 return -ENOMEM; 9705 } 9706 9707 tr->topts = tr_topts; 9708 tr->topts[tr->nr_topts].tracer = tracer; 9709 tr->topts[tr->nr_topts].topts = topts; 9710 tr->nr_topts++; 9711 9712 for (cnt = 0; opts[cnt].name; cnt++) { 9713 create_trace_option_file(tr, &topts[cnt], flags, 9714 &opts[cnt]); 9715 MEM_FAIL(topts[cnt].entry == NULL, 9716 "Failed to create trace option: %s", 9717 opts[cnt].name); 9718 } 9719 return 0; 9720} 9721 9722static int get_global_flags_val(struct tracer *tracer) 9723{ 9724 struct tracers *t; 9725 9726 list_for_each_entry(t, &global_trace.tracers, list) { 9727 if (t->tracer != tracer) 9728 continue; 9729 if (!t->flags) 9730 return -1; 9731 return t->flags->val; 9732 } 9733 return -1; 9734} 9735 9736static int add_tracer_options(struct trace_array *tr, struct tracers *t) 9737{ 9738 struct tracer *tracer = t->tracer; 9739 struct tracer_flags *flags = t->flags ?: tracer->flags; 9740 9741 if (!flags) 9742 return 0; 9743 9744 /* Only add tracer options after update_tracer_options finish */ 9745 if (!tracer_options_updated) 9746 return 0; 9747 9748 return create_trace_option_files(tr, tracer, flags); 9749} 9750 9751static int add_tracer(struct trace_array *tr, struct tracer *tracer) 9752{ 9753 struct tracer_flags *flags; 9754 struct tracers *t; 9755 int ret; 9756 9757 /* Only enable if the directory has been created already. */ 9758 if (!tr->dir && !(tr->flags & TRACE_ARRAY_FL_GLOBAL)) 9759 return 0; 9760 9761 /* 9762 * If this is an instance, only create flags for tracers 9763 * the instance may have. 9764 */ 9765 if (!trace_ok_for_array(tracer, tr)) 9766 return 0; 9767 9768 t = kmalloc(sizeof(*t), GFP_KERNEL); 9769 if (!t) 9770 return -ENOMEM; 9771 9772 t->tracer = tracer; 9773 t->flags = NULL; 9774 list_add(&t->list, &tr->tracers); 9775 9776 flags = tracer->flags; 9777 if (!flags) { 9778 if (!tracer->default_flags) 9779 return 0; 9780 9781 /* 9782 * If the tracer defines default flags, it means the flags are 9783 * per trace instance. 9784 */ 9785 flags = kmalloc(sizeof(*flags), GFP_KERNEL); 9786 if (!flags) 9787 return -ENOMEM; 9788 9789 *flags = *tracer->default_flags; 9790 flags->trace = tracer; 9791 9792 t->flags = flags; 9793 9794 /* If this is an instance, inherit the global_trace flags */ 9795 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL)) { 9796 int val = get_global_flags_val(tracer); 9797 if (!WARN_ON_ONCE(val < 0)) 9798 flags->val = val; 9799 } 9800 } 9801 9802 ret = add_tracer_options(tr, t); 9803 if (ret < 0) { 9804 list_del(&t->list); 9805 kfree(t->flags); 9806 kfree(t); 9807 } 9808 9809 return ret; 9810} 9811 9812static struct dentry * 9813create_trace_option_core_file(struct trace_array *tr, 9814 const char *option, long index) 9815{ 9816 struct dentry *t_options; 9817 9818 t_options = trace_options_init_dentry(tr); 9819 if (!t_options) 9820 return NULL; 9821 9822 return trace_create_file(option, TRACE_MODE_WRITE, t_options, 9823 (void *)&tr->trace_flags_index[index], 9824 &trace_options_core_fops); 9825} 9826 9827static void create_trace_options_dir(struct trace_array *tr) 9828{ 9829 struct dentry *t_options; 9830 bool top_level = tr == &global_trace; 9831 int i; 9832 9833 t_options = trace_options_init_dentry(tr); 9834 if (!t_options) 9835 return; 9836 9837 for (i = 0; trace_options[i]; i++) { 9838 if (top_level || 9839 !((1ULL << i) & TOP_LEVEL_TRACE_FLAGS)) { 9840 create_trace_option_core_file(tr, trace_options[i], i); 9841 } 9842 } 9843} 9844 9845static ssize_t 9846rb_simple_read(struct file *filp, char __user *ubuf, 9847 size_t cnt, loff_t *ppos) 9848{ 9849 struct trace_array *tr = filp->private_data; 9850 char buf[64]; 9851 int r; 9852 9853 r = tracer_tracing_is_on(tr); 9854 r = sprintf(buf, "%d\n", r); 9855 9856 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 9857} 9858 9859static ssize_t 9860rb_simple_write(struct file *filp, const char __user *ubuf, 9861 size_t cnt, loff_t *ppos) 9862{ 9863 struct trace_array *tr = filp->private_data; 9864 struct trace_buffer *buffer = tr->array_buffer.buffer; 9865 unsigned long val; 9866 int ret; 9867 9868 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9869 if (ret) 9870 return ret; 9871 9872 if (buffer) { 9873 guard(mutex)(&trace_types_lock); 9874 if (!!val == tracer_tracing_is_on(tr)) { 9875 val = 0; /* do nothing */ 9876 } else if (val) { 9877 tracer_tracing_on(tr); 9878 if (tr->current_trace->start) 9879 tr->current_trace->start(tr); 9880 } else { 9881 tracer_tracing_off(tr); 9882 if (tr->current_trace->stop) 9883 tr->current_trace->stop(tr); 9884 /* Wake up any waiters */ 9885 ring_buffer_wake_waiters(buffer, RING_BUFFER_ALL_CPUS); 9886 } 9887 } 9888 9889 (*ppos)++; 9890 9891 return cnt; 9892} 9893 9894static const struct file_operations rb_simple_fops = { 9895 .open = tracing_open_generic_tr, 9896 .read = rb_simple_read, 9897 .write = rb_simple_write, 9898 .release = tracing_release_generic_tr, 9899 .llseek = default_llseek, 9900}; 9901 9902static ssize_t 9903buffer_percent_read(struct file *filp, char __user *ubuf, 9904 size_t cnt, loff_t *ppos) 9905{ 9906 struct trace_array *tr = filp->private_data; 9907 char buf[64]; 9908 int r; 9909 9910 r = tr->buffer_percent; 9911 r = sprintf(buf, "%d\n", r); 9912 9913 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 9914} 9915 9916static ssize_t 9917buffer_percent_write(struct file *filp, const char __user *ubuf, 9918 size_t cnt, loff_t *ppos) 9919{ 9920 struct trace_array *tr = filp->private_data; 9921 unsigned long val; 9922 int ret; 9923 9924 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9925 if (ret) 9926 return ret; 9927 9928 if (val > 100) 9929 return -EINVAL; 9930 9931 tr->buffer_percent = val; 9932 9933 (*ppos)++; 9934 9935 return cnt; 9936} 9937 9938static const struct file_operations buffer_percent_fops = { 9939 .open = tracing_open_generic_tr, 9940 .read = buffer_percent_read, 9941 .write = buffer_percent_write, 9942 .release = tracing_release_generic_tr, 9943 .llseek = default_llseek, 9944}; 9945 9946static ssize_t 9947buffer_subbuf_size_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) 9948{ 9949 struct trace_array *tr = filp->private_data; 9950 size_t size; 9951 char buf[64]; 9952 int order; 9953 int r; 9954 9955 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 9956 size = (PAGE_SIZE << order) / 1024; 9957 9958 r = sprintf(buf, "%zd\n", size); 9959 9960 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 9961} 9962 9963static ssize_t 9964buffer_subbuf_size_write(struct file *filp, const char __user *ubuf, 9965 size_t cnt, loff_t *ppos) 9966{ 9967 struct trace_array *tr = filp->private_data; 9968 unsigned long val; 9969 int old_order; 9970 int order; 9971 int pages; 9972 int ret; 9973 9974 ret = kstrtoul_from_user(ubuf, cnt, 10, &val); 9975 if (ret) 9976 return ret; 9977 9978 val *= 1024; /* value passed in is in KB */ 9979 9980 pages = DIV_ROUND_UP(val, PAGE_SIZE); 9981 order = fls(pages - 1); 9982 9983 /* limit between 1 and 128 system pages */ 9984 if (order < 0 || order > 7) 9985 return -EINVAL; 9986 9987 /* Do not allow tracing while changing the order of the ring buffer */ 9988 tracing_stop_tr(tr); 9989 9990 old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); 9991 if (old_order == order) 9992 goto out; 9993 9994 ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order); 9995 if (ret) 9996 goto out; 9997 9998#ifdef CONFIG_TRACER_MAX_TRACE 9999 10000 if (!tr->allocated_snapshot) 10001 goto out_max; 10002 10003 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); 10004 if (ret) { 10005 /* Put back the old order */ 10006 cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order); 10007 if (WARN_ON_ONCE(cnt)) { 10008 /* 10009 * AARGH! We are left with different orders! 10010 * The max buffer is our "snapshot" buffer. 10011 * When a tracer needs a snapshot (one of the 10012 * latency tracers), it swaps the max buffer 10013 * with the saved snap shot. We succeeded to 10014 * update the order of the main buffer, but failed to 10015 * update the order of the max buffer. But when we tried 10016 * to reset the main buffer to the original size, we 10017 * failed there too. This is very unlikely to 10018 * happen, but if it does, warn and kill all 10019 * tracing. 10020 */ 10021 tracing_disabled = 1; 10022 } 10023 goto out; 10024 } 10025 out_max: 10026#endif 10027 (*ppos)++; 10028 out: 10029 if (ret) 10030 cnt = ret; 10031 tracing_start_tr(tr); 10032 return cnt; 10033} 10034 10035static const struct file_operations buffer_subbuf_size_fops = { 10036 .open = tracing_open_generic_tr, 10037 .read = buffer_subbuf_size_read, 10038 .write = buffer_subbuf_size_write, 10039 .release = tracing_release_generic_tr, 10040 .llseek = default_llseek, 10041}; 10042 10043static struct dentry *trace_instance_dir; 10044 10045static void 10046init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer); 10047 10048#ifdef CONFIG_MODULES 10049static int make_mod_delta(struct module *mod, void *data) 10050{ 10051 struct trace_module_delta *module_delta; 10052 struct trace_scratch *tscratch; 10053 struct trace_mod_entry *entry; 10054 struct trace_array *tr = data; 10055 int i; 10056 10057 tscratch = tr->scratch; 10058 module_delta = READ_ONCE(tr->module_delta); 10059 for (i = 0; i < tscratch->nr_entries; i++) { 10060 entry = &tscratch->entries[i]; 10061 if (strcmp(mod->name, entry->mod_name)) 10062 continue; 10063 if (mod->state == MODULE_STATE_GOING) 10064 module_delta->delta[i] = 0; 10065 else 10066 module_delta->delta[i] = (unsigned long)mod->mem[MOD_TEXT].base 10067 - entry->mod_addr; 10068 break; 10069 } 10070 return 0; 10071} 10072#else 10073static int make_mod_delta(struct module *mod, void *data) 10074{ 10075 return 0; 10076} 10077#endif 10078 10079static int mod_addr_comp(const void *a, const void *b, const void *data) 10080{ 10081 const struct trace_mod_entry *e1 = a; 10082 const struct trace_mod_entry *e2 = b; 10083 10084 return e1->mod_addr > e2->mod_addr ? 1 : -1; 10085} 10086 10087static void setup_trace_scratch(struct trace_array *tr, 10088 struct trace_scratch *tscratch, unsigned int size) 10089{ 10090 struct trace_module_delta *module_delta; 10091 struct trace_mod_entry *entry; 10092 int i, nr_entries; 10093 10094 if (!tscratch) 10095 return; 10096 10097 tr->scratch = tscratch; 10098 tr->scratch_size = size; 10099 10100 if (tscratch->text_addr) 10101 tr->text_delta = (unsigned long)_text - tscratch->text_addr; 10102 10103 if (struct_size(tscratch, entries, tscratch->nr_entries) > size) 10104 goto reset; 10105 10106 /* Check if each module name is a valid string */ 10107 for (i = 0; i < tscratch->nr_entries; i++) { 10108 int n; 10109 10110 entry = &tscratch->entries[i]; 10111 10112 for (n = 0; n < MODULE_NAME_LEN; n++) { 10113 if (entry->mod_name[n] == '\0') 10114 break; 10115 if (!isprint(entry->mod_name[n])) 10116 goto reset; 10117 } 10118 if (n == MODULE_NAME_LEN) 10119 goto reset; 10120 } 10121 10122 /* Sort the entries so that we can find appropriate module from address. */ 10123 nr_entries = tscratch->nr_entries; 10124 sort_r(tscratch->entries, nr_entries, sizeof(struct trace_mod_entry), 10125 mod_addr_comp, NULL, NULL); 10126 10127 if (IS_ENABLED(CONFIG_MODULES)) { 10128 module_delta = kzalloc(struct_size(module_delta, delta, nr_entries), GFP_KERNEL); 10129 if (!module_delta) { 10130 pr_info("module_delta allocation failed. Not able to decode module address."); 10131 goto reset; 10132 } 10133 init_rcu_head(&module_delta->rcu); 10134 } else 10135 module_delta = NULL; 10136 WRITE_ONCE(tr->module_delta, module_delta); 10137 10138 /* Scan modules to make text delta for modules. */ 10139 module_for_each_mod(make_mod_delta, tr); 10140 10141 /* Set trace_clock as the same of the previous boot. */ 10142 if (tscratch->clock_id != tr->clock_id) { 10143 if (tscratch->clock_id >= ARRAY_SIZE(trace_clocks) || 10144 tracing_set_clock(tr, trace_clocks[tscratch->clock_id].name) < 0) { 10145 pr_info("the previous trace_clock info is not valid."); 10146 goto reset; 10147 } 10148 } 10149 return; 10150 reset: 10151 /* Invalid trace modules */ 10152 memset(tscratch, 0, size); 10153} 10154 10155static int 10156allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, unsigned long size) 10157{ 10158 enum ring_buffer_flags rb_flags; 10159 struct trace_scratch *tscratch; 10160 unsigned int scratch_size = 0; 10161 10162 rb_flags = tr->trace_flags & TRACE_ITER(OVERWRITE) ? RB_FL_OVERWRITE : 0; 10163 10164 buf->tr = tr; 10165 10166 if (tr->range_addr_start && tr->range_addr_size) { 10167 /* Add scratch buffer to handle 128 modules */ 10168 buf->buffer = ring_buffer_alloc_range(size, rb_flags, 0, 10169 tr->range_addr_start, 10170 tr->range_addr_size, 10171 struct_size(tscratch, entries, 128)); 10172 10173 tscratch = ring_buffer_meta_scratch(buf->buffer, &scratch_size); 10174 setup_trace_scratch(tr, tscratch, scratch_size); 10175 10176 /* 10177 * This is basically the same as a mapped buffer, 10178 * with the same restrictions. 10179 */ 10180 tr->mapped++; 10181 } else { 10182 buf->buffer = ring_buffer_alloc(size, rb_flags); 10183 } 10184 if (!buf->buffer) 10185 return -ENOMEM; 10186 10187 buf->data = alloc_percpu(struct trace_array_cpu); 10188 if (!buf->data) { 10189 ring_buffer_free(buf->buffer); 10190 buf->buffer = NULL; 10191 return -ENOMEM; 10192 } 10193 10194 /* Allocate the first page for all buffers */ 10195 set_buffer_entries(&tr->array_buffer, 10196 ring_buffer_size(tr->array_buffer.buffer, 0)); 10197 10198 return 0; 10199} 10200 10201static void free_trace_buffer(struct array_buffer *buf) 10202{ 10203 if (buf->buffer) { 10204 ring_buffer_free(buf->buffer); 10205 buf->buffer = NULL; 10206 free_percpu(buf->data); 10207 buf->data = NULL; 10208 } 10209} 10210 10211static int allocate_trace_buffers(struct trace_array *tr, unsigned long size) 10212{ 10213 int ret; 10214 10215 ret = allocate_trace_buffer(tr, &tr->array_buffer, size); 10216 if (ret) 10217 return ret; 10218 10219#ifdef CONFIG_TRACER_MAX_TRACE 10220 /* Fix mapped buffer trace arrays do not have snapshot buffers */ 10221 if (tr->range_addr_start) 10222 return 0; 10223 10224 ret = allocate_trace_buffer(tr, &tr->max_buffer, 10225 allocate_snapshot ? size : 1); 10226 if (MEM_FAIL(ret, "Failed to allocate trace buffer\n")) { 10227 free_trace_buffer(&tr->array_buffer); 10228 return -ENOMEM; 10229 } 10230 tr->allocated_snapshot = allocate_snapshot; 10231 10232 allocate_snapshot = false; 10233#endif 10234 10235 return 0; 10236} 10237 10238static void free_trace_buffers(struct trace_array *tr) 10239{ 10240 if (!tr) 10241 return; 10242 10243 free_trace_buffer(&tr->array_buffer); 10244 kfree(tr->module_delta); 10245 10246#ifdef CONFIG_TRACER_MAX_TRACE 10247 free_trace_buffer(&tr->max_buffer); 10248#endif 10249} 10250 10251static void init_trace_flags_index(struct trace_array *tr) 10252{ 10253 int i; 10254 10255 /* Used by the trace options files */ 10256 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) 10257 tr->trace_flags_index[i] = i; 10258} 10259 10260static int __update_tracer(struct trace_array *tr) 10261{ 10262 struct tracer *t; 10263 int ret = 0; 10264 10265 for (t = trace_types; t && !ret; t = t->next) 10266 ret = add_tracer(tr, t); 10267 10268 return ret; 10269} 10270 10271static __init int __update_tracer_options(struct trace_array *tr) 10272{ 10273 struct tracers *t; 10274 int ret = 0; 10275 10276 list_for_each_entry(t, &tr->tracers, list) { 10277 ret = add_tracer_options(tr, t); 10278 if (ret < 0) 10279 break; 10280 } 10281 10282 return ret; 10283} 10284 10285static __init void update_tracer_options(void) 10286{ 10287 struct trace_array *tr; 10288 10289 guard(mutex)(&trace_types_lock); 10290 tracer_options_updated = true; 10291 list_for_each_entry(tr, &ftrace_trace_arrays, list) 10292 __update_tracer_options(tr); 10293} 10294 10295/* Must have trace_types_lock held */ 10296struct trace_array *trace_array_find(const char *instance) 10297{ 10298 struct trace_array *tr, *found = NULL; 10299 10300 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 10301 if (tr->name && strcmp(tr->name, instance) == 0) { 10302 found = tr; 10303 break; 10304 } 10305 } 10306 10307 return found; 10308} 10309 10310struct trace_array *trace_array_find_get(const char *instance) 10311{ 10312 struct trace_array *tr; 10313 10314 guard(mutex)(&trace_types_lock); 10315 tr = trace_array_find(instance); 10316 if (tr) 10317 tr->ref++; 10318 10319 return tr; 10320} 10321 10322static int trace_array_create_dir(struct trace_array *tr) 10323{ 10324 int ret; 10325 10326 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir); 10327 if (!tr->dir) 10328 return -EINVAL; 10329 10330 ret = event_trace_add_tracer(tr->dir, tr); 10331 if (ret) { 10332 tracefs_remove(tr->dir); 10333 return ret; 10334 } 10335 10336 init_tracer_tracefs(tr, tr->dir); 10337 ret = __update_tracer(tr); 10338 if (ret) { 10339 event_trace_del_tracer(tr); 10340 tracefs_remove(tr->dir); 10341 return ret; 10342 } 10343 return 0; 10344} 10345 10346static struct trace_array * 10347trace_array_create_systems(const char *name, const char *systems, 10348 unsigned long range_addr_start, 10349 unsigned long range_addr_size) 10350{ 10351 struct trace_array *tr; 10352 int ret; 10353 10354 ret = -ENOMEM; 10355 tr = kzalloc(sizeof(*tr), GFP_KERNEL); 10356 if (!tr) 10357 return ERR_PTR(ret); 10358 10359 tr->name = kstrdup(name, GFP_KERNEL); 10360 if (!tr->name) 10361 goto out_free_tr; 10362 10363 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) 10364 goto out_free_tr; 10365 10366 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL)) 10367 goto out_free_tr; 10368 10369 if (systems) { 10370 tr->system_names = kstrdup_const(systems, GFP_KERNEL); 10371 if (!tr->system_names) 10372 goto out_free_tr; 10373 } 10374 10375 /* Only for boot up memory mapped ring buffers */ 10376 tr->range_addr_start = range_addr_start; 10377 tr->range_addr_size = range_addr_size; 10378 10379 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; 10380 10381 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); 10382 10383 raw_spin_lock_init(&tr->start_lock); 10384 10385 tr->syscall_buf_sz = global_trace.syscall_buf_sz; 10386 10387 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 10388#ifdef CONFIG_TRACER_MAX_TRACE 10389 spin_lock_init(&tr->snapshot_trigger_lock); 10390#endif 10391 tr->current_trace = &nop_trace; 10392 tr->current_trace_flags = nop_trace.flags; 10393 10394 INIT_LIST_HEAD(&tr->systems); 10395 INIT_LIST_HEAD(&tr->events); 10396 INIT_LIST_HEAD(&tr->hist_vars); 10397 INIT_LIST_HEAD(&tr->err_log); 10398 INIT_LIST_HEAD(&tr->tracers); 10399 INIT_LIST_HEAD(&tr->marker_list); 10400 10401#ifdef CONFIG_MODULES 10402 INIT_LIST_HEAD(&tr->mod_events); 10403#endif 10404 10405 if (allocate_trace_buffers(tr, trace_buf_size) < 0) 10406 goto out_free_tr; 10407 10408 /* The ring buffer is defaultly expanded */ 10409 trace_set_ring_buffer_expanded(tr); 10410 10411 if (ftrace_allocate_ftrace_ops(tr) < 0) 10412 goto out_free_tr; 10413 10414 ftrace_init_trace_array(tr); 10415 10416 init_trace_flags_index(tr); 10417 10418 if (trace_instance_dir) { 10419 ret = trace_array_create_dir(tr); 10420 if (ret) 10421 goto out_free_tr; 10422 } else 10423 __trace_early_add_events(tr); 10424 10425 list_add(&tr->list, &ftrace_trace_arrays); 10426 10427 tr->ref++; 10428 10429 return tr; 10430 10431 out_free_tr: 10432 ftrace_free_ftrace_ops(tr); 10433 free_trace_buffers(tr); 10434 free_cpumask_var(tr->pipe_cpumask); 10435 free_cpumask_var(tr->tracing_cpumask); 10436 kfree_const(tr->system_names); 10437 kfree(tr->range_name); 10438 kfree(tr->name); 10439 kfree(tr); 10440 10441 return ERR_PTR(ret); 10442} 10443 10444static struct trace_array *trace_array_create(const char *name) 10445{ 10446 return trace_array_create_systems(name, NULL, 0, 0); 10447} 10448 10449static int instance_mkdir(const char *name) 10450{ 10451 struct trace_array *tr; 10452 int ret; 10453 10454 guard(mutex)(&event_mutex); 10455 guard(mutex)(&trace_types_lock); 10456 10457 ret = -EEXIST; 10458 if (trace_array_find(name)) 10459 return -EEXIST; 10460 10461 tr = trace_array_create(name); 10462 10463 ret = PTR_ERR_OR_ZERO(tr); 10464 10465 return ret; 10466} 10467 10468#ifdef CONFIG_MMU 10469static u64 map_pages(unsigned long start, unsigned long size) 10470{ 10471 unsigned long vmap_start, vmap_end; 10472 struct vm_struct *area; 10473 int ret; 10474 10475 area = get_vm_area(size, VM_IOREMAP); 10476 if (!area) 10477 return 0; 10478 10479 vmap_start = (unsigned long) area->addr; 10480 vmap_end = vmap_start + size; 10481 10482 ret = vmap_page_range(vmap_start, vmap_end, 10483 start, pgprot_nx(PAGE_KERNEL)); 10484 if (ret < 0) { 10485 free_vm_area(area); 10486 return 0; 10487 } 10488 10489 return (u64)vmap_start; 10490} 10491#else 10492static inline u64 map_pages(unsigned long start, unsigned long size) 10493{ 10494 return 0; 10495} 10496#endif 10497 10498/** 10499 * trace_array_get_by_name - Create/Lookup a trace array, given its name. 10500 * @name: The name of the trace array to be looked up/created. 10501 * @systems: A list of systems to create event directories for (NULL for all) 10502 * 10503 * Returns pointer to trace array with given name. 10504 * NULL, if it cannot be created. 10505 * 10506 * NOTE: This function increments the reference counter associated with the 10507 * trace array returned. This makes sure it cannot be freed while in use. 10508 * Use trace_array_put() once the trace array is no longer needed. 10509 * If the trace_array is to be freed, trace_array_destroy() needs to 10510 * be called after the trace_array_put(), or simply let user space delete 10511 * it from the tracefs instances directory. But until the 10512 * trace_array_put() is called, user space can not delete it. 10513 * 10514 */ 10515struct trace_array *trace_array_get_by_name(const char *name, const char *systems) 10516{ 10517 struct trace_array *tr; 10518 10519 guard(mutex)(&event_mutex); 10520 guard(mutex)(&trace_types_lock); 10521 10522 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 10523 if (tr->name && strcmp(tr->name, name) == 0) { 10524 tr->ref++; 10525 return tr; 10526 } 10527 } 10528 10529 tr = trace_array_create_systems(name, systems, 0, 0); 10530 10531 if (IS_ERR(tr)) 10532 tr = NULL; 10533 else 10534 tr->ref++; 10535 10536 return tr; 10537} 10538EXPORT_SYMBOL_GPL(trace_array_get_by_name); 10539 10540static int __remove_instance(struct trace_array *tr) 10541{ 10542 int i; 10543 10544 /* Reference counter for a newly created trace array = 1. */ 10545 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref)) 10546 return -EBUSY; 10547 10548 list_del(&tr->list); 10549 10550 if (printk_trace == tr) 10551 update_printk_trace(&global_trace); 10552 10553 /* Must be done before disabling all the flags */ 10554 if (update_marker_trace(tr, 0)) 10555 synchronize_rcu(); 10556 10557 /* Disable all the flags that were enabled coming in */ 10558 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) { 10559 if ((1ULL << i) & ZEROED_TRACE_FLAGS) 10560 set_tracer_flag(tr, 1ULL << i, 0); 10561 } 10562 10563 tracing_set_nop(tr); 10564 clear_ftrace_function_probes(tr); 10565 event_trace_del_tracer(tr); 10566 ftrace_clear_pids(tr); 10567 ftrace_destroy_function_files(tr); 10568 tracefs_remove(tr->dir); 10569 free_percpu(tr->last_func_repeats); 10570 free_trace_buffers(tr); 10571 clear_tracing_err_log(tr); 10572 free_tracers(tr); 10573 10574 if (tr->range_name) { 10575 reserve_mem_release_by_name(tr->range_name); 10576 kfree(tr->range_name); 10577 } 10578 if (tr->flags & TRACE_ARRAY_FL_VMALLOC) 10579 vfree((void *)tr->range_addr_start); 10580 10581 for (i = 0; i < tr->nr_topts; i++) { 10582 kfree(tr->topts[i].topts); 10583 } 10584 kfree(tr->topts); 10585 10586 free_cpumask_var(tr->pipe_cpumask); 10587 free_cpumask_var(tr->tracing_cpumask); 10588 kfree_const(tr->system_names); 10589 kfree(tr->name); 10590 kfree(tr); 10591 10592 return 0; 10593} 10594 10595int trace_array_destroy(struct trace_array *this_tr) 10596{ 10597 struct trace_array *tr; 10598 10599 if (!this_tr) 10600 return -EINVAL; 10601 10602 guard(mutex)(&event_mutex); 10603 guard(mutex)(&trace_types_lock); 10604 10605 10606 /* Making sure trace array exists before destroying it. */ 10607 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 10608 if (tr == this_tr) 10609 return __remove_instance(tr); 10610 } 10611 10612 return -ENODEV; 10613} 10614EXPORT_SYMBOL_GPL(trace_array_destroy); 10615 10616static int instance_rmdir(const char *name) 10617{ 10618 struct trace_array *tr; 10619 10620 guard(mutex)(&event_mutex); 10621 guard(mutex)(&trace_types_lock); 10622 10623 tr = trace_array_find(name); 10624 if (!tr) 10625 return -ENODEV; 10626 10627 return __remove_instance(tr); 10628} 10629 10630static __init void create_trace_instances(struct dentry *d_tracer) 10631{ 10632 struct trace_array *tr; 10633 10634 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer, 10635 instance_mkdir, 10636 instance_rmdir); 10637 if (MEM_FAIL(!trace_instance_dir, "Failed to create instances directory\n")) 10638 return; 10639 10640 guard(mutex)(&event_mutex); 10641 guard(mutex)(&trace_types_lock); 10642 10643 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 10644 if (!tr->name) 10645 continue; 10646 if (MEM_FAIL(trace_array_create_dir(tr) < 0, 10647 "Failed to create instance directory\n")) 10648 return; 10649 } 10650} 10651 10652static void 10653init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) 10654{ 10655 int cpu; 10656 10657 trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer, 10658 tr, &show_traces_fops); 10659 10660 trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer, 10661 tr, &set_tracer_fops); 10662 10663 trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer, 10664 tr, &tracing_cpumask_fops); 10665 10666 trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer, 10667 tr, &tracing_iter_fops); 10668 10669 trace_create_file("trace", TRACE_MODE_WRITE, d_tracer, 10670 tr, &tracing_fops); 10671 10672 trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer, 10673 tr, &tracing_pipe_fops); 10674 10675 trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer, 10676 tr, &tracing_entries_fops); 10677 10678 trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer, 10679 tr, &tracing_total_entries_fops); 10680 10681 trace_create_file("free_buffer", 0200, d_tracer, 10682 tr, &tracing_free_buffer_fops); 10683 10684 trace_create_file("trace_marker", 0220, d_tracer, 10685 tr, &tracing_mark_fops); 10686 10687 tr->trace_marker_file = __find_event_file(tr, "ftrace", "print"); 10688 10689 trace_create_file("trace_marker_raw", 0220, d_tracer, 10690 tr, &tracing_mark_raw_fops); 10691 10692 trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr, 10693 &trace_clock_fops); 10694 10695 trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer, 10696 tr, &rb_simple_fops); 10697 10698 trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr, 10699 &trace_time_stamp_mode_fops); 10700 10701 tr->buffer_percent = 50; 10702 10703 trace_create_file("buffer_percent", TRACE_MODE_WRITE, d_tracer, 10704 tr, &buffer_percent_fops); 10705 10706 trace_create_file("buffer_subbuf_size_kb", TRACE_MODE_WRITE, d_tracer, 10707 tr, &buffer_subbuf_size_fops); 10708 10709 trace_create_file("syscall_user_buf_size", TRACE_MODE_WRITE, d_tracer, 10710 tr, &tracing_syscall_buf_fops); 10711 10712 create_trace_options_dir(tr); 10713 10714#ifdef CONFIG_TRACER_MAX_TRACE 10715 trace_create_maxlat_file(tr, d_tracer); 10716#endif 10717 10718 if (ftrace_create_function_files(tr, d_tracer)) 10719 MEM_FAIL(1, "Could not allocate function filter files"); 10720 10721 if (tr->range_addr_start) { 10722 trace_create_file("last_boot_info", TRACE_MODE_READ, d_tracer, 10723 tr, &last_boot_fops); 10724#ifdef CONFIG_TRACER_SNAPSHOT 10725 } else { 10726 trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer, 10727 tr, &snapshot_fops); 10728#endif 10729 } 10730 10731 trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer, 10732 tr, &tracing_err_log_fops); 10733 10734 for_each_tracing_cpu(cpu) 10735 tracing_init_tracefs_percpu(tr, cpu); 10736 10737 ftrace_init_tracefs(tr, d_tracer); 10738} 10739 10740#ifdef CONFIG_TRACEFS_AUTOMOUNT_DEPRECATED 10741static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore) 10742{ 10743 struct vfsmount *mnt; 10744 struct file_system_type *type; 10745 struct fs_context *fc; 10746 int ret; 10747 10748 /* 10749 * To maintain backward compatibility for tools that mount 10750 * debugfs to get to the tracing facility, tracefs is automatically 10751 * mounted to the debugfs/tracing directory. 10752 */ 10753 type = get_fs_type("tracefs"); 10754 if (!type) 10755 return NULL; 10756 10757 fc = fs_context_for_submount(type, mntpt); 10758 put_filesystem(type); 10759 if (IS_ERR(fc)) 10760 return ERR_CAST(fc); 10761 10762 pr_warn("NOTICE: Automounting of tracing to debugfs is deprecated and will be removed in 2030\n"); 10763 10764 ret = vfs_parse_fs_string(fc, "source", "tracefs"); 10765 if (!ret) 10766 mnt = fc_mount(fc); 10767 else 10768 mnt = ERR_PTR(ret); 10769 10770 put_fs_context(fc); 10771 return mnt; 10772} 10773#endif 10774 10775/** 10776 * tracing_init_dentry - initialize top level trace array 10777 * 10778 * This is called when creating files or directories in the tracing 10779 * directory. It is called via fs_initcall() by any of the boot up code 10780 * and expects to return the dentry of the top level tracing directory. 10781 */ 10782int tracing_init_dentry(void) 10783{ 10784 struct trace_array *tr = &global_trace; 10785 10786 if (security_locked_down(LOCKDOWN_TRACEFS)) { 10787 pr_warn("Tracing disabled due to lockdown\n"); 10788 return -EPERM; 10789 } 10790 10791 /* The top level trace array uses NULL as parent */ 10792 if (tr->dir) 10793 return 0; 10794 10795 if (WARN_ON(!tracefs_initialized())) 10796 return -ENODEV; 10797 10798#ifdef CONFIG_TRACEFS_AUTOMOUNT_DEPRECATED 10799 /* 10800 * As there may still be users that expect the tracing 10801 * files to exist in debugfs/tracing, we must automount 10802 * the tracefs file system there, so older tools still 10803 * work with the newer kernel. 10804 */ 10805 tr->dir = debugfs_create_automount("tracing", NULL, 10806 trace_automount, NULL); 10807#endif 10808 10809 return 0; 10810} 10811 10812extern struct trace_eval_map *__start_ftrace_eval_maps[]; 10813extern struct trace_eval_map *__stop_ftrace_eval_maps[]; 10814 10815static struct workqueue_struct *eval_map_wq __initdata; 10816static struct work_struct eval_map_work __initdata; 10817static struct work_struct tracerfs_init_work __initdata; 10818 10819static void __init eval_map_work_func(struct work_struct *work) 10820{ 10821 int len; 10822 10823 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps; 10824 trace_event_update_with_eval_map(NULL, __start_ftrace_eval_maps, len); 10825} 10826 10827static int __init trace_eval_init(void) 10828{ 10829 INIT_WORK(&eval_map_work, eval_map_work_func); 10830 10831 eval_map_wq = alloc_workqueue("eval_map_wq", WQ_UNBOUND, 0); 10832 if (!eval_map_wq) { 10833 pr_err("Unable to allocate eval_map_wq\n"); 10834 /* Do work here */ 10835 eval_map_work_func(&eval_map_work); 10836 return -ENOMEM; 10837 } 10838 10839 queue_work(eval_map_wq, &eval_map_work); 10840 return 0; 10841} 10842 10843subsys_initcall(trace_eval_init); 10844 10845static int __init trace_eval_sync(void) 10846{ 10847 /* Make sure the eval map updates are finished */ 10848 if (eval_map_wq) 10849 destroy_workqueue(eval_map_wq); 10850 return 0; 10851} 10852 10853late_initcall_sync(trace_eval_sync); 10854 10855 10856#ifdef CONFIG_MODULES 10857 10858bool module_exists(const char *module) 10859{ 10860 /* All modules have the symbol __this_module */ 10861 static const char this_mod[] = "__this_module"; 10862 char modname[MODULE_NAME_LEN + sizeof(this_mod) + 2]; 10863 unsigned long val; 10864 int n; 10865 10866 n = snprintf(modname, sizeof(modname), "%s:%s", module, this_mod); 10867 10868 if (n > sizeof(modname) - 1) 10869 return false; 10870 10871 val = module_kallsyms_lookup_name(modname); 10872 return val != 0; 10873} 10874 10875static void trace_module_add_evals(struct module *mod) 10876{ 10877 /* 10878 * Modules with bad taint do not have events created, do 10879 * not bother with enums either. 10880 */ 10881 if (trace_module_has_bad_taint(mod)) 10882 return; 10883 10884 /* Even if no trace_evals, this need to sanitize field types. */ 10885 trace_event_update_with_eval_map(mod, mod->trace_evals, mod->num_trace_evals); 10886} 10887 10888#ifdef CONFIG_TRACE_EVAL_MAP_FILE 10889static void trace_module_remove_evals(struct module *mod) 10890{ 10891 union trace_eval_map_item *map; 10892 union trace_eval_map_item **last = &trace_eval_maps; 10893 10894 if (!mod->num_trace_evals) 10895 return; 10896 10897 guard(mutex)(&trace_eval_mutex); 10898 10899 map = trace_eval_maps; 10900 10901 while (map) { 10902 if (map->head.mod == mod) 10903 break; 10904 map = trace_eval_jmp_to_tail(map); 10905 last = &map->tail.next; 10906 map = map->tail.next; 10907 } 10908 if (!map) 10909 return; 10910 10911 *last = trace_eval_jmp_to_tail(map)->tail.next; 10912 kfree(map); 10913} 10914#else 10915static inline void trace_module_remove_evals(struct module *mod) { } 10916#endif /* CONFIG_TRACE_EVAL_MAP_FILE */ 10917 10918static void trace_module_record(struct module *mod, bool add) 10919{ 10920 struct trace_array *tr; 10921 unsigned long flags; 10922 10923 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 10924 flags = tr->flags & (TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT); 10925 /* Update any persistent trace array that has already been started */ 10926 if (flags == TRACE_ARRAY_FL_BOOT && add) { 10927 guard(mutex)(&scratch_mutex); 10928 save_mod(mod, tr); 10929 } else if (flags & TRACE_ARRAY_FL_LAST_BOOT) { 10930 /* Update delta if the module loaded in previous boot */ 10931 make_mod_delta(mod, tr); 10932 } 10933 } 10934} 10935 10936static int trace_module_notify(struct notifier_block *self, 10937 unsigned long val, void *data) 10938{ 10939 struct module *mod = data; 10940 10941 switch (val) { 10942 case MODULE_STATE_COMING: 10943 trace_module_add_evals(mod); 10944 trace_module_record(mod, true); 10945 break; 10946 case MODULE_STATE_GOING: 10947 trace_module_remove_evals(mod); 10948 trace_module_record(mod, false); 10949 break; 10950 } 10951 10952 return NOTIFY_OK; 10953} 10954 10955static struct notifier_block trace_module_nb = { 10956 .notifier_call = trace_module_notify, 10957 .priority = 0, 10958}; 10959#endif /* CONFIG_MODULES */ 10960 10961static __init void tracer_init_tracefs_work_func(struct work_struct *work) 10962{ 10963 10964 event_trace_init(); 10965 10966 init_tracer_tracefs(&global_trace, NULL); 10967 ftrace_init_tracefs_toplevel(&global_trace, NULL); 10968 10969 trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL, 10970 &global_trace, &tracing_thresh_fops); 10971 10972 trace_create_file("README", TRACE_MODE_READ, NULL, 10973 NULL, &tracing_readme_fops); 10974 10975 trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL, 10976 NULL, &tracing_saved_cmdlines_fops); 10977 10978 trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL, 10979 NULL, &tracing_saved_cmdlines_size_fops); 10980 10981 trace_create_file("saved_tgids", TRACE_MODE_READ, NULL, 10982 NULL, &tracing_saved_tgids_fops); 10983 10984 trace_create_eval_file(NULL); 10985 10986#ifdef CONFIG_MODULES 10987 register_module_notifier(&trace_module_nb); 10988#endif 10989 10990#ifdef CONFIG_DYNAMIC_FTRACE 10991 trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL, 10992 NULL, &tracing_dyn_info_fops); 10993#endif 10994 10995 create_trace_instances(NULL); 10996 10997 update_tracer_options(); 10998} 10999 11000static __init int tracer_init_tracefs(void) 11001{ 11002 int ret; 11003 11004 trace_access_lock_init(); 11005 11006 ret = tracing_init_dentry(); 11007 if (ret) 11008 return 0; 11009 11010 if (eval_map_wq) { 11011 INIT_WORK(&tracerfs_init_work, tracer_init_tracefs_work_func); 11012 queue_work(eval_map_wq, &tracerfs_init_work); 11013 } else { 11014 tracer_init_tracefs_work_func(NULL); 11015 } 11016 11017 if (rv_init_interface()) 11018 pr_err("RV: Error while creating the RV interface\n"); 11019 11020 return 0; 11021} 11022 11023fs_initcall(tracer_init_tracefs); 11024 11025static int trace_die_panic_handler(struct notifier_block *self, 11026 unsigned long ev, void *unused); 11027 11028static struct notifier_block trace_panic_notifier = { 11029 .notifier_call = trace_die_panic_handler, 11030 .priority = INT_MAX - 1, 11031}; 11032 11033static struct notifier_block trace_die_notifier = { 11034 .notifier_call = trace_die_panic_handler, 11035 .priority = INT_MAX - 1, 11036}; 11037 11038/* 11039 * The idea is to execute the following die/panic callback early, in order 11040 * to avoid showing irrelevant information in the trace (like other panic 11041 * notifier functions); we are the 2nd to run, after hung_task/rcu_stall 11042 * warnings get disabled (to prevent potential log flooding). 11043 */ 11044static int trace_die_panic_handler(struct notifier_block *self, 11045 unsigned long ev, void *unused) 11046{ 11047 if (!ftrace_dump_on_oops_enabled()) 11048 return NOTIFY_DONE; 11049 11050 /* The die notifier requires DIE_OOPS to trigger */ 11051 if (self == &trace_die_notifier && ev != DIE_OOPS) 11052 return NOTIFY_DONE; 11053 11054 ftrace_dump(DUMP_PARAM); 11055 11056 return NOTIFY_DONE; 11057} 11058 11059/* 11060 * printk is set to max of 1024, we really don't need it that big. 11061 * Nothing should be printing 1000 characters anyway. 11062 */ 11063#define TRACE_MAX_PRINT 1000 11064 11065/* 11066 * Define here KERN_TRACE so that we have one place to modify 11067 * it if we decide to change what log level the ftrace dump 11068 * should be at. 11069 */ 11070#define KERN_TRACE KERN_EMERG 11071 11072void 11073trace_printk_seq(struct trace_seq *s) 11074{ 11075 /* Probably should print a warning here. */ 11076 if (s->seq.len >= TRACE_MAX_PRINT) 11077 s->seq.len = TRACE_MAX_PRINT; 11078 11079 /* 11080 * More paranoid code. Although the buffer size is set to 11081 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just 11082 * an extra layer of protection. 11083 */ 11084 if (WARN_ON_ONCE(s->seq.len >= s->seq.size)) 11085 s->seq.len = s->seq.size - 1; 11086 11087 /* should be zero ended, but we are paranoid. */ 11088 s->buffer[s->seq.len] = 0; 11089 11090 printk(KERN_TRACE "%s", s->buffer); 11091 11092 trace_seq_init(s); 11093} 11094 11095static void trace_init_iter(struct trace_iterator *iter, struct trace_array *tr) 11096{ 11097 iter->tr = tr; 11098 iter->trace = iter->tr->current_trace; 11099 iter->cpu_file = RING_BUFFER_ALL_CPUS; 11100 iter->array_buffer = &tr->array_buffer; 11101 11102 if (iter->trace && iter->trace->open) 11103 iter->trace->open(iter); 11104 11105 /* Annotate start of buffers if we had overruns */ 11106 if (ring_buffer_overruns(iter->array_buffer->buffer)) 11107 iter->iter_flags |= TRACE_FILE_ANNOTATE; 11108 11109 /* Output in nanoseconds only if we are using a clock in nanoseconds. */ 11110 if (trace_clocks[iter->tr->clock_id].in_ns) 11111 iter->iter_flags |= TRACE_FILE_TIME_IN_NS; 11112 11113 /* Can not use kmalloc for iter.temp and iter.fmt */ 11114 iter->temp = static_temp_buf; 11115 iter->temp_size = STATIC_TEMP_BUF_SIZE; 11116 iter->fmt = static_fmt_buf; 11117 iter->fmt_size = STATIC_FMT_BUF_SIZE; 11118} 11119 11120void trace_init_global_iter(struct trace_iterator *iter) 11121{ 11122 trace_init_iter(iter, &global_trace); 11123} 11124 11125static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode) 11126{ 11127 /* use static because iter can be a bit big for the stack */ 11128 static struct trace_iterator iter; 11129 unsigned int old_userobj; 11130 unsigned long flags; 11131 int cnt = 0; 11132 11133 /* 11134 * Always turn off tracing when we dump. 11135 * We don't need to show trace output of what happens 11136 * between multiple crashes. 11137 * 11138 * If the user does a sysrq-z, then they can re-enable 11139 * tracing with echo 1 > tracing_on. 11140 */ 11141 tracer_tracing_off(tr); 11142 11143 local_irq_save(flags); 11144 11145 /* Simulate the iterator */ 11146 trace_init_iter(&iter, tr); 11147 11148 /* While dumping, do not allow the buffer to be enable */ 11149 tracer_tracing_disable(tr); 11150 11151 old_userobj = tr->trace_flags & TRACE_ITER(SYM_USEROBJ); 11152 11153 /* don't look at user memory in panic mode */ 11154 tr->trace_flags &= ~TRACE_ITER(SYM_USEROBJ); 11155 11156 if (dump_mode == DUMP_ORIG) 11157 iter.cpu_file = raw_smp_processor_id(); 11158 else 11159 iter.cpu_file = RING_BUFFER_ALL_CPUS; 11160 11161 if (tr == &global_trace) 11162 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 11163 else 11164 printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name); 11165 11166 /* Did function tracer already get disabled? */ 11167 if (ftrace_is_dead()) { 11168 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n"); 11169 printk("# MAY BE MISSING FUNCTION EVENTS\n"); 11170 } 11171 11172 /* 11173 * We need to stop all tracing on all CPUS to read 11174 * the next buffer. This is a bit expensive, but is 11175 * not done often. We fill all what we can read, 11176 * and then release the locks again. 11177 */ 11178 11179 while (!trace_empty(&iter)) { 11180 11181 if (!cnt) 11182 printk(KERN_TRACE "---------------------------------\n"); 11183 11184 cnt++; 11185 11186 trace_iterator_reset(&iter); 11187 iter.iter_flags |= TRACE_FILE_LAT_FMT; 11188 11189 if (trace_find_next_entry_inc(&iter) != NULL) { 11190 int ret; 11191 11192 ret = print_trace_line(&iter); 11193 if (ret != TRACE_TYPE_NO_CONSUME) 11194 trace_consume(&iter); 11195 11196 trace_printk_seq(&iter.seq); 11197 } 11198 touch_nmi_watchdog(); 11199 } 11200 11201 if (!cnt) 11202 printk(KERN_TRACE " (ftrace buffer empty)\n"); 11203 else 11204 printk(KERN_TRACE "---------------------------------\n"); 11205 11206 tr->trace_flags |= old_userobj; 11207 11208 tracer_tracing_enable(tr); 11209 local_irq_restore(flags); 11210} 11211 11212static void ftrace_dump_by_param(void) 11213{ 11214 bool first_param = true; 11215 char dump_param[MAX_TRACER_SIZE]; 11216 char *buf, *token, *inst_name; 11217 struct trace_array *tr; 11218 11219 strscpy(dump_param, ftrace_dump_on_oops, MAX_TRACER_SIZE); 11220 buf = dump_param; 11221 11222 while ((token = strsep(&buf, ",")) != NULL) { 11223 if (first_param) { 11224 first_param = false; 11225 if (!strcmp("0", token)) 11226 continue; 11227 else if (!strcmp("1", token)) { 11228 ftrace_dump_one(&global_trace, DUMP_ALL); 11229 continue; 11230 } 11231 else if (!strcmp("2", token) || 11232 !strcmp("orig_cpu", token)) { 11233 ftrace_dump_one(&global_trace, DUMP_ORIG); 11234 continue; 11235 } 11236 } 11237 11238 inst_name = strsep(&token, "="); 11239 tr = trace_array_find(inst_name); 11240 if (!tr) { 11241 printk(KERN_TRACE "Instance %s not found\n", inst_name); 11242 continue; 11243 } 11244 11245 if (token && (!strcmp("2", token) || 11246 !strcmp("orig_cpu", token))) 11247 ftrace_dump_one(tr, DUMP_ORIG); 11248 else 11249 ftrace_dump_one(tr, DUMP_ALL); 11250 } 11251} 11252 11253void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 11254{ 11255 static atomic_t dump_running; 11256 11257 /* Only allow one dump user at a time. */ 11258 if (atomic_inc_return(&dump_running) != 1) { 11259 atomic_dec(&dump_running); 11260 return; 11261 } 11262 11263 switch (oops_dump_mode) { 11264 case DUMP_ALL: 11265 ftrace_dump_one(&global_trace, DUMP_ALL); 11266 break; 11267 case DUMP_ORIG: 11268 ftrace_dump_one(&global_trace, DUMP_ORIG); 11269 break; 11270 case DUMP_PARAM: 11271 ftrace_dump_by_param(); 11272 break; 11273 case DUMP_NONE: 11274 break; 11275 default: 11276 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 11277 ftrace_dump_one(&global_trace, DUMP_ALL); 11278 } 11279 11280 atomic_dec(&dump_running); 11281} 11282EXPORT_SYMBOL_GPL(ftrace_dump); 11283 11284#define WRITE_BUFSIZE 4096 11285 11286ssize_t trace_parse_run_command(struct file *file, const char __user *buffer, 11287 size_t count, loff_t *ppos, 11288 int (*createfn)(const char *)) 11289{ 11290 char *kbuf __free(kfree) = NULL; 11291 char *buf, *tmp; 11292 int ret = 0; 11293 size_t done = 0; 11294 size_t size; 11295 11296 kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL); 11297 if (!kbuf) 11298 return -ENOMEM; 11299 11300 while (done < count) { 11301 size = count - done; 11302 11303 if (size >= WRITE_BUFSIZE) 11304 size = WRITE_BUFSIZE - 1; 11305 11306 if (copy_from_user(kbuf, buffer + done, size)) 11307 return -EFAULT; 11308 11309 kbuf[size] = '\0'; 11310 buf = kbuf; 11311 do { 11312 tmp = strchr(buf, '\n'); 11313 if (tmp) { 11314 *tmp = '\0'; 11315 size = tmp - buf + 1; 11316 } else { 11317 size = strlen(buf); 11318 if (done + size < count) { 11319 if (buf != kbuf) 11320 break; 11321 /* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */ 11322 pr_warn("Line length is too long: Should be less than %d\n", 11323 WRITE_BUFSIZE - 2); 11324 return -EINVAL; 11325 } 11326 } 11327 done += size; 11328 11329 /* Remove comments */ 11330 tmp = strchr(buf, '#'); 11331 11332 if (tmp) 11333 *tmp = '\0'; 11334 11335 ret = createfn(buf); 11336 if (ret) 11337 return ret; 11338 buf += size; 11339 11340 } while (done < count); 11341 } 11342 return done; 11343} 11344 11345#ifdef CONFIG_TRACER_MAX_TRACE 11346__init static bool tr_needs_alloc_snapshot(const char *name) 11347{ 11348 char *test; 11349 int len = strlen(name); 11350 bool ret; 11351 11352 if (!boot_snapshot_index) 11353 return false; 11354 11355 if (strncmp(name, boot_snapshot_info, len) == 0 && 11356 boot_snapshot_info[len] == '\t') 11357 return true; 11358 11359 test = kmalloc(strlen(name) + 3, GFP_KERNEL); 11360 if (!test) 11361 return false; 11362 11363 sprintf(test, "\t%s\t", name); 11364 ret = strstr(boot_snapshot_info, test) == NULL; 11365 kfree(test); 11366 return ret; 11367} 11368 11369__init static void do_allocate_snapshot(const char *name) 11370{ 11371 if (!tr_needs_alloc_snapshot(name)) 11372 return; 11373 11374 /* 11375 * When allocate_snapshot is set, the next call to 11376 * allocate_trace_buffers() (called by trace_array_get_by_name()) 11377 * will allocate the snapshot buffer. That will also clear 11378 * this flag. 11379 */ 11380 allocate_snapshot = true; 11381} 11382#else 11383static inline void do_allocate_snapshot(const char *name) { } 11384#endif 11385 11386__init static int backup_instance_area(const char *backup, 11387 unsigned long *addr, phys_addr_t *size) 11388{ 11389 struct trace_array *backup_tr; 11390 void *allocated_vaddr = NULL; 11391 11392 backup_tr = trace_array_get_by_name(backup, NULL); 11393 if (!backup_tr) { 11394 pr_warn("Tracing: Instance %s is not found.\n", backup); 11395 return -ENOENT; 11396 } 11397 11398 if (!(backup_tr->flags & TRACE_ARRAY_FL_BOOT)) { 11399 pr_warn("Tracing: Instance %s is not boot mapped.\n", backup); 11400 trace_array_put(backup_tr); 11401 return -EINVAL; 11402 } 11403 11404 *size = backup_tr->range_addr_size; 11405 11406 allocated_vaddr = vzalloc(*size); 11407 if (!allocated_vaddr) { 11408 pr_warn("Tracing: Failed to allocate memory for copying instance %s (size 0x%lx)\n", 11409 backup, (unsigned long)*size); 11410 trace_array_put(backup_tr); 11411 return -ENOMEM; 11412 } 11413 11414 memcpy(allocated_vaddr, 11415 (void *)backup_tr->range_addr_start, (size_t)*size); 11416 *addr = (unsigned long)allocated_vaddr; 11417 11418 trace_array_put(backup_tr); 11419 return 0; 11420} 11421 11422__init static void enable_instances(void) 11423{ 11424 struct trace_array *tr; 11425 bool memmap_area = false; 11426 char *curr_str; 11427 char *name; 11428 char *str; 11429 char *tok; 11430 11431 /* A tab is always appended */ 11432 boot_instance_info[boot_instance_index - 1] = '\0'; 11433 str = boot_instance_info; 11434 11435 while ((curr_str = strsep(&str, "\t"))) { 11436 phys_addr_t start = 0; 11437 phys_addr_t size = 0; 11438 unsigned long addr = 0; 11439 bool traceprintk = false; 11440 bool traceoff = false; 11441 char *flag_delim; 11442 char *addr_delim; 11443 char *rname __free(kfree) = NULL; 11444 char *backup; 11445 11446 tok = strsep(&curr_str, ","); 11447 11448 name = strsep(&tok, "="); 11449 backup = tok; 11450 11451 flag_delim = strchr(name, '^'); 11452 addr_delim = strchr(name, '@'); 11453 11454 if (addr_delim) 11455 *addr_delim++ = '\0'; 11456 11457 if (flag_delim) 11458 *flag_delim++ = '\0'; 11459 11460 if (backup) { 11461 if (backup_instance_area(backup, &addr, &size) < 0) 11462 continue; 11463 } 11464 11465 if (flag_delim) { 11466 char *flag; 11467 11468 while ((flag = strsep(&flag_delim, "^"))) { 11469 if (strcmp(flag, "traceoff") == 0) { 11470 traceoff = true; 11471 } else if ((strcmp(flag, "printk") == 0) || 11472 (strcmp(flag, "traceprintk") == 0) || 11473 (strcmp(flag, "trace_printk") == 0)) { 11474 traceprintk = true; 11475 } else { 11476 pr_info("Tracing: Invalid instance flag '%s' for %s\n", 11477 flag, name); 11478 } 11479 } 11480 } 11481 11482 tok = addr_delim; 11483 if (tok && isdigit(*tok)) { 11484 start = memparse(tok, &tok); 11485 if (!start) { 11486 pr_warn("Tracing: Invalid boot instance address for %s\n", 11487 name); 11488 continue; 11489 } 11490 if (*tok != ':') { 11491 pr_warn("Tracing: No size specified for instance %s\n", name); 11492 continue; 11493 } 11494 tok++; 11495 size = memparse(tok, &tok); 11496 if (!size) { 11497 pr_warn("Tracing: Invalid boot instance size for %s\n", 11498 name); 11499 continue; 11500 } 11501 memmap_area = true; 11502 } else if (tok) { 11503 if (!reserve_mem_find_by_name(tok, &start, &size)) { 11504 start = 0; 11505 pr_warn("Failed to map boot instance %s to %s\n", name, tok); 11506 continue; 11507 } 11508 rname = kstrdup(tok, GFP_KERNEL); 11509 } 11510 11511 if (start) { 11512 /* Start and size must be page aligned */ 11513 if (start & ~PAGE_MASK) { 11514 pr_warn("Tracing: mapping start addr %pa is not page aligned\n", &start); 11515 continue; 11516 } 11517 if (size & ~PAGE_MASK) { 11518 pr_warn("Tracing: mapping size %pa is not page aligned\n", &size); 11519 continue; 11520 } 11521 11522 if (memmap_area) 11523 addr = map_pages(start, size); 11524 else 11525 addr = (unsigned long)phys_to_virt(start); 11526 if (addr) { 11527 pr_info("Tracing: mapped boot instance %s at physical memory %pa of size 0x%lx\n", 11528 name, &start, (unsigned long)size); 11529 } else { 11530 pr_warn("Tracing: Failed to map boot instance %s\n", name); 11531 continue; 11532 } 11533 } else { 11534 /* Only non mapped buffers have snapshot buffers */ 11535 if (IS_ENABLED(CONFIG_TRACER_MAX_TRACE)) 11536 do_allocate_snapshot(name); 11537 } 11538 11539 tr = trace_array_create_systems(name, NULL, addr, size); 11540 if (IS_ERR(tr)) { 11541 pr_warn("Tracing: Failed to create instance buffer %s\n", curr_str); 11542 continue; 11543 } 11544 11545 if (traceoff) 11546 tracer_tracing_off(tr); 11547 11548 if (traceprintk) 11549 update_printk_trace(tr); 11550 11551 /* 11552 * memmap'd buffers can not be freed. 11553 */ 11554 if (memmap_area) { 11555 tr->flags |= TRACE_ARRAY_FL_MEMMAP; 11556 tr->ref++; 11557 } 11558 11559 /* 11560 * Backup buffers can be freed but need vfree(). 11561 */ 11562 if (backup) 11563 tr->flags |= TRACE_ARRAY_FL_VMALLOC; 11564 11565 if (start || backup) { 11566 tr->flags |= TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT; 11567 tr->range_name = no_free_ptr(rname); 11568 } 11569 11570 while ((tok = strsep(&curr_str, ","))) { 11571 early_enable_events(tr, tok, true); 11572 } 11573 } 11574} 11575 11576__init static int tracer_alloc_buffers(void) 11577{ 11578 unsigned long ring_buf_size; 11579 int ret = -ENOMEM; 11580 11581 11582 if (security_locked_down(LOCKDOWN_TRACEFS)) { 11583 pr_warn("Tracing disabled due to lockdown\n"); 11584 return -EPERM; 11585 } 11586 11587 /* 11588 * Make sure we don't accidentally add more trace options 11589 * than we have bits for. 11590 */ 11591 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE); 11592 11593 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 11594 return -ENOMEM; 11595 11596 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL)) 11597 goto out_free_buffer_mask; 11598 11599 /* Only allocate trace_printk buffers if a trace_printk exists */ 11600 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt) 11601 /* Must be called before global_trace.buffer is allocated */ 11602 trace_printk_init_buffers(); 11603 11604 /* To save memory, keep the ring buffer size to its minimum */ 11605 if (global_trace.ring_buffer_expanded) 11606 ring_buf_size = trace_buf_size; 11607 else 11608 ring_buf_size = 1; 11609 11610 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 11611 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask); 11612 11613 raw_spin_lock_init(&global_trace.start_lock); 11614 11615 /* 11616 * The prepare callbacks allocates some memory for the ring buffer. We 11617 * don't free the buffer if the CPU goes down. If we were to free 11618 * the buffer, then the user would lose any trace that was in the 11619 * buffer. The memory will be removed once the "instance" is removed. 11620 */ 11621 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE, 11622 "trace/RB:prepare", trace_rb_cpu_prepare, 11623 NULL); 11624 if (ret < 0) 11625 goto out_free_cpumask; 11626 /* Used for event triggers */ 11627 ret = -ENOMEM; 11628 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); 11629 if (!temp_buffer) 11630 goto out_rm_hp_state; 11631 11632 if (trace_create_savedcmd() < 0) 11633 goto out_free_temp_buffer; 11634 11635 if (!zalloc_cpumask_var(&global_trace.pipe_cpumask, GFP_KERNEL)) 11636 goto out_free_savedcmd; 11637 11638 /* TODO: make the number of buffers hot pluggable with CPUS */ 11639 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) { 11640 MEM_FAIL(1, "tracer: failed to allocate ring buffer!\n"); 11641 goto out_free_pipe_cpumask; 11642 } 11643 if (global_trace.buffer_disabled) 11644 tracing_off(); 11645 11646 if (trace_boot_clock) { 11647 ret = tracing_set_clock(&global_trace, trace_boot_clock); 11648 if (ret < 0) 11649 pr_warn("Trace clock %s not defined, going back to default\n", 11650 trace_boot_clock); 11651 } 11652 11653 /* 11654 * register_tracer() might reference current_trace, so it 11655 * needs to be set before we register anything. This is 11656 * just a bootstrap of current_trace anyway. 11657 */ 11658 global_trace.current_trace = &nop_trace; 11659 global_trace.current_trace_flags = nop_trace.flags; 11660 11661 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 11662#ifdef CONFIG_TRACER_MAX_TRACE 11663 spin_lock_init(&global_trace.snapshot_trigger_lock); 11664#endif 11665 ftrace_init_global_array_ops(&global_trace); 11666 11667#ifdef CONFIG_MODULES 11668 INIT_LIST_HEAD(&global_trace.mod_events); 11669#endif 11670 11671 init_trace_flags_index(&global_trace); 11672 11673 INIT_LIST_HEAD(&global_trace.tracers); 11674 11675 /* All seems OK, enable tracing */ 11676 tracing_disabled = 0; 11677 11678 atomic_notifier_chain_register(&panic_notifier_list, 11679 &trace_panic_notifier); 11680 11681 register_die_notifier(&trace_die_notifier); 11682 11683 global_trace.flags = TRACE_ARRAY_FL_GLOBAL; 11684 11685 global_trace.syscall_buf_sz = syscall_buf_size; 11686 11687 INIT_LIST_HEAD(&global_trace.systems); 11688 INIT_LIST_HEAD(&global_trace.events); 11689 INIT_LIST_HEAD(&global_trace.hist_vars); 11690 INIT_LIST_HEAD(&global_trace.err_log); 11691 list_add(&global_trace.marker_list, &marker_copies); 11692 list_add(&global_trace.list, &ftrace_trace_arrays); 11693 11694 register_tracer(&nop_trace); 11695 11696 /* Function tracing may start here (via kernel command line) */ 11697 init_function_trace(); 11698 11699 apply_trace_boot_options(); 11700 11701 register_snapshot_cmd(); 11702 11703 return 0; 11704 11705out_free_pipe_cpumask: 11706 free_cpumask_var(global_trace.pipe_cpumask); 11707out_free_savedcmd: 11708 trace_free_saved_cmdlines_buffer(); 11709out_free_temp_buffer: 11710 ring_buffer_free(temp_buffer); 11711out_rm_hp_state: 11712 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE); 11713out_free_cpumask: 11714 free_cpumask_var(global_trace.tracing_cpumask); 11715out_free_buffer_mask: 11716 free_cpumask_var(tracing_buffer_mask); 11717 return ret; 11718} 11719 11720#ifdef CONFIG_FUNCTION_TRACER 11721/* Used to set module cached ftrace filtering at boot up */ 11722struct trace_array *trace_get_global_array(void) 11723{ 11724 return &global_trace; 11725} 11726#endif 11727 11728void __init ftrace_boot_snapshot(void) 11729{ 11730#ifdef CONFIG_TRACER_MAX_TRACE 11731 struct trace_array *tr; 11732 11733 if (!snapshot_at_boot) 11734 return; 11735 11736 list_for_each_entry(tr, &ftrace_trace_arrays, list) { 11737 if (!tr->allocated_snapshot) 11738 continue; 11739 11740 tracing_snapshot_instance(tr); 11741 trace_array_puts(tr, "** Boot snapshot taken **\n"); 11742 } 11743#endif 11744} 11745 11746void __init early_trace_init(void) 11747{ 11748 if (tracepoint_printk) { 11749 tracepoint_print_iter = 11750 kzalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL); 11751 if (MEM_FAIL(!tracepoint_print_iter, 11752 "Failed to allocate trace iterator\n")) 11753 tracepoint_printk = 0; 11754 else 11755 static_key_enable(&tracepoint_printk_key.key); 11756 } 11757 tracer_alloc_buffers(); 11758 11759 init_events(); 11760} 11761 11762void __init trace_init(void) 11763{ 11764 trace_event_init(); 11765 11766 if (boot_instance_index) 11767 enable_instances(); 11768} 11769 11770__init static void clear_boot_tracer(void) 11771{ 11772 /* 11773 * The default tracer at boot buffer is an init section. 11774 * This function is called in lateinit. If we did not 11775 * find the boot tracer, then clear it out, to prevent 11776 * later registration from accessing the buffer that is 11777 * about to be freed. 11778 */ 11779 if (!default_bootup_tracer) 11780 return; 11781 11782 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 11783 default_bootup_tracer); 11784 default_bootup_tracer = NULL; 11785} 11786 11787#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 11788__init static void tracing_set_default_clock(void) 11789{ 11790 /* sched_clock_stable() is determined in late_initcall */ 11791 if (!trace_boot_clock && !sched_clock_stable()) { 11792 if (security_locked_down(LOCKDOWN_TRACEFS)) { 11793 pr_warn("Can not set tracing clock due to lockdown\n"); 11794 return; 11795 } 11796 11797 printk(KERN_WARNING 11798 "Unstable clock detected, switching default tracing clock to \"global\"\n" 11799 "If you want to keep using the local clock, then add:\n" 11800 " \"trace_clock=local\"\n" 11801 "on the kernel command line\n"); 11802 tracing_set_clock(&global_trace, "global"); 11803 } 11804} 11805#else 11806static inline void tracing_set_default_clock(void) { } 11807#endif 11808 11809__init static int late_trace_init(void) 11810{ 11811 if (tracepoint_printk && tracepoint_printk_stop_on_boot) { 11812 static_key_disable(&tracepoint_printk_key.key); 11813 tracepoint_printk = 0; 11814 } 11815 11816 if (traceoff_after_boot) 11817 tracing_off(); 11818 11819 tracing_set_default_clock(); 11820 clear_boot_tracer(); 11821 return 0; 11822} 11823 11824late_initcall_sync(late_trace_init);
~/
Read-only file preview.