trace_events.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * event tracer
   4 *
   5 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
   6 *
   7 *  - Added format output of fields of the trace point.
   8 *    This was based off of work by Tom Zanussi <tzanussi@gmail.com>.
   9 *
  10 */
  11
  12#define pr_fmt(fmt) fmt
  13
  14#include <linux/workqueue.h>
  15#include <linux/security.h>
  16#include <linux/spinlock.h>
  17#include <linux/kthread.h>
  18#include <linux/tracefs.h>
  19#include <linux/uaccess.h>
  20#include <linux/module.h>
  21#include <linux/ctype.h>
  22#include <linux/sort.h>
  23#include <linux/slab.h>
  24#include <linux/delay.h>
  25
  26#include <trace/events/sched.h>
  27#include <trace/syscall.h>
  28
  29#include <asm/setup.h>
  30
  31#include "trace_output.h"
  32
  33#undef TRACE_SYSTEM
  34#define TRACE_SYSTEM "TRACE_SYSTEM"
  35
  36DEFINE_MUTEX(event_mutex);
  37
  38LIST_HEAD(ftrace_events);
  39static LIST_HEAD(ftrace_generic_fields);
  40static LIST_HEAD(ftrace_common_fields);
  41static bool eventdir_initialized;
  42
  43static LIST_HEAD(module_strings);
  44
  45struct module_string {
  46	struct list_head	next;
  47	struct module		*module;
  48	char			*str;
  49};
  50
  51#define GFP_TRACE (GFP_KERNEL | __GFP_ZERO)
  52
  53static struct kmem_cache *field_cachep;
  54static struct kmem_cache *file_cachep;
  55
  56static inline int system_refcount(struct event_subsystem *system)
  57{
  58	return system->ref_count;
  59}
  60
  61static int system_refcount_inc(struct event_subsystem *system)
  62{
  63	return system->ref_count++;
  64}
  65
  66static int system_refcount_dec(struct event_subsystem *system)
  67{
  68	return --system->ref_count;
  69}
  70
  71/* Double loops, do not use break, only goto's work */
  72#define do_for_each_event_file(tr, file)			\
  73	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
  74		list_for_each_entry(file, &tr->events, list)
  75
  76#define do_for_each_event_file_safe(tr, file)			\
  77	list_for_each_entry(tr, &ftrace_trace_arrays, list) {	\
  78		struct trace_event_file *___n;				\
  79		list_for_each_entry_safe(file, ___n, &tr->events, list)
  80
  81#define while_for_each_event_file()		\
  82	}
  83
  84static struct ftrace_event_field *
  85__find_event_field(struct list_head *head, const char *name)
  86{
  87	struct ftrace_event_field *field;
  88
  89	list_for_each_entry(field, head, link) {
  90		if (!strcmp(field->name, name))
  91			return field;
  92	}
  93
  94	return NULL;
  95}
  96
  97struct ftrace_event_field *
  98trace_find_event_field(struct trace_event_call *call, char *name)
  99{
 100	struct ftrace_event_field *field;
 101	struct list_head *head;
 102
 103	head = trace_get_fields(call);
 104	field = __find_event_field(head, name);
 105	if (field)
 106		return field;
 107
 108	field = __find_event_field(&ftrace_generic_fields, name);
 109	if (field)
 110		return field;
 111
 112	return __find_event_field(&ftrace_common_fields, name);
 113}
 114
 115static int __trace_define_field(struct list_head *head, const char *type,
 116				const char *name, int offset, int size,
 117				int is_signed, int filter_type, int len,
 118				int need_test)
 119{
 120	struct ftrace_event_field *field;
 121
 122	field = kmem_cache_alloc(field_cachep, GFP_TRACE);
 123	if (!field)
 124		return -ENOMEM;
 125
 126	field->name = name;
 127	field->type = type;
 128
 129	if (filter_type == FILTER_OTHER)
 130		field->filter_type = filter_assign_type(type);
 131	else
 132		field->filter_type = filter_type;
 133
 134	field->offset = offset;
 135	field->size = size;
 136	field->is_signed = is_signed;
 137	field->needs_test = need_test;
 138	field->len = len;
 139
 140	list_add(&field->link, head);
 141
 142	return 0;
 143}
 144
 145int trace_define_field(struct trace_event_call *call, const char *type,
 146		       const char *name, int offset, int size, int is_signed,
 147		       int filter_type)
 148{
 149	struct list_head *head;
 150
 151	if (WARN_ON(!call->class))
 152		return 0;
 153
 154	head = trace_get_fields(call);
 155	return __trace_define_field(head, type, name, offset, size,
 156				    is_signed, filter_type, 0, 0);
 157}
 158EXPORT_SYMBOL_GPL(trace_define_field);
 159
 160static int trace_define_field_ext(struct trace_event_call *call, const char *type,
 161		       const char *name, int offset, int size, int is_signed,
 162		       int filter_type, int len, int need_test)
 163{
 164	struct list_head *head;
 165
 166	if (WARN_ON(!call->class))
 167		return 0;
 168
 169	head = trace_get_fields(call);
 170	return __trace_define_field(head, type, name, offset, size,
 171				    is_signed, filter_type, len, need_test);
 172}
 173
 174#define __generic_field(type, item, filter_type)			\
 175	ret = __trace_define_field(&ftrace_generic_fields, #type,	\
 176				   #item, 0, 0, is_signed_type(type),	\
 177				   filter_type, 0, 0);			\
 178	if (ret)							\
 179		return ret;
 180
 181#define __common_field(type, item)					\
 182	ret = __trace_define_field(&ftrace_common_fields, #type,	\
 183				   "common_" #item,			\
 184				   offsetof(typeof(ent), item),		\
 185				   sizeof(ent.item),			\
 186				   is_signed_type(type), FILTER_OTHER,	\
 187				   0, 0);				\
 188	if (ret)							\
 189		return ret;
 190
 191static int trace_define_generic_fields(void)
 192{
 193	int ret;
 194
 195	__generic_field(int, CPU, FILTER_CPU);
 196	__generic_field(int, cpu, FILTER_CPU);
 197	__generic_field(int, common_cpu, FILTER_CPU);
 198	__generic_field(char *, COMM, FILTER_COMM);
 199	__generic_field(char *, comm, FILTER_COMM);
 200	__generic_field(char *, stacktrace, FILTER_STACKTRACE);
 201	__generic_field(char *, STACKTRACE, FILTER_STACKTRACE);
 202
 203	return ret;
 204}
 205
 206static int trace_define_common_fields(void)
 207{
 208	int ret;
 209	struct trace_entry ent;
 210
 211	__common_field(unsigned short, type);
 212	__common_field(unsigned char, flags);
 213	/* Holds both preempt_count and migrate_disable */
 214	__common_field(unsigned char, preempt_count);
 215	__common_field(int, pid);
 216
 217	return ret;
 218}
 219
 220static void trace_destroy_fields(struct trace_event_call *call)
 221{
 222	struct ftrace_event_field *field, *next;
 223	struct list_head *head;
 224
 225	head = trace_get_fields(call);
 226	list_for_each_entry_safe(field, next, head, link) {
 227		list_del(&field->link);
 228		kmem_cache_free(field_cachep, field);
 229	}
 230}
 231
 232/*
 233 * run-time version of trace_event_get_offsets_<call>() that returns the last
 234 * accessible offset of trace fields excluding __dynamic_array bytes
 235 */
 236int trace_event_get_offsets(struct trace_event_call *call)
 237{
 238	struct ftrace_event_field *tail;
 239	struct list_head *head;
 240
 241	head = trace_get_fields(call);
 242	/*
 243	 * head->next points to the last field with the largest offset,
 244	 * since it was added last by trace_define_field()
 245	 */
 246	tail = list_first_entry(head, struct ftrace_event_field, link);
 247	return tail->offset + tail->size;
 248}
 249
 250
 251static struct trace_event_fields *find_event_field(const char *fmt,
 252						   struct trace_event_call *call)
 253{
 254	struct trace_event_fields *field = call->class->fields_array;
 255	const char *p = fmt;
 256	int len;
 257
 258	if (!(len = str_has_prefix(fmt, "REC->")))
 259		return NULL;
 260	fmt += len;
 261	for (p = fmt; *p; p++) {
 262		if (!isalnum(*p) && *p != '_')
 263			break;
 264	}
 265	len = p - fmt;
 266
 267	for (; field->type; field++) {
 268		if (strncmp(field->name, fmt, len) || field->name[len])
 269			continue;
 270
 271		return field;
 272	}
 273	return NULL;
 274}
 275
 276/*
 277 * Check if the referenced field is an array and return true,
 278 * as arrays are OK to dereference.
 279 */
 280static bool test_field(const char *fmt, struct trace_event_call *call)
 281{
 282	struct trace_event_fields *field;
 283
 284	field = find_event_field(fmt, call);
 285	if (!field)
 286		return false;
 287
 288	/* This is an array and is OK to dereference. */
 289	return strchr(field->type, '[') != NULL;
 290}
 291
 292/* Look for a string within an argument */
 293static bool find_print_string(const char *arg, const char *str, const char *end)
 294{
 295	const char *r;
 296
 297	r = strstr(arg, str);
 298	return r && r < end;
 299}
 300
 301/* Return true if the argument pointer is safe */
 302static bool process_pointer(const char *fmt, int len, struct trace_event_call *call)
 303{
 304	const char *r, *e, *a;
 305
 306	e = fmt + len;
 307
 308	/* Find the REC-> in the argument */
 309	r = strstr(fmt, "REC->");
 310	if (r && r < e) {
 311		/*
 312		 * Addresses of events on the buffer, or an array on the buffer is
 313		 * OK to dereference. There's ways to fool this, but
 314		 * this is to catch common mistakes, not malicious code.
 315		 */
 316		a = strchr(fmt, '&');
 317		if ((a && (a < r)) || test_field(r, call))
 318			return true;
 319	} else if (find_print_string(fmt, "__get_dynamic_array(", e)) {
 320		return true;
 321	} else if (find_print_string(fmt, "__get_rel_dynamic_array(", e)) {
 322		return true;
 323	} else if (find_print_string(fmt, "__get_dynamic_array_len(", e)) {
 324		return true;
 325	} else if (find_print_string(fmt, "__get_rel_dynamic_array_len(", e)) {
 326		return true;
 327	} else if (find_print_string(fmt, "__get_sockaddr(", e)) {
 328		return true;
 329	} else if (find_print_string(fmt, "__get_rel_sockaddr(", e)) {
 330		return true;
 331	}
 332	return false;
 333}
 334
 335/* Return true if the string is safe */
 336static bool process_string(const char *fmt, int len, struct trace_event_call *call)
 337{
 338	struct trace_event_fields *field;
 339	const char *r, *e, *s;
 340
 341	e = fmt + len;
 342
 343	/*
 344	 * There are several helper functions that return strings.
 345	 * If the argument contains a function, then assume its field is valid.
 346	 * It is considered that the argument has a function if it has:
 347	 *   alphanumeric or '_' before a parenthesis.
 348	 */
 349	s = fmt;
 350	do {
 351		r = strstr(s, "(");
 352		if (!r || r >= e)
 353			break;
 354		for (int i = 1; r - i >= s; i++) {
 355			char ch = *(r - i);
 356			if (isspace(ch))
 357				continue;
 358			if (isalnum(ch) || ch == '_')
 359				return true;
 360			/* Anything else, this isn't a function */
 361			break;
 362		}
 363		/* A function could be wrapped in parenthesis, try the next one */
 364		s = r + 1;
 365	} while (s < e);
 366
 367	/*
 368	 * Check for arrays. If the argument has: foo[REC->val]
 369	 * then it is very likely that foo is an array of strings
 370	 * that are safe to use.
 371	 */
 372	r = strstr(s, "[");
 373	if (r && r < e) {
 374		r = strstr(r, "REC->");
 375		if (r && r < e)
 376			return true;
 377	}
 378
 379	/*
 380	 * If there's any strings in the argument consider this arg OK as it
 381	 * could be: REC->field ? "foo" : "bar" and we don't want to get into
 382	 * verifying that logic here.
 383	 */
 384	if (find_print_string(fmt, "\"", e))
 385		return true;
 386
 387	/* Dereferenced strings are also valid like any other pointer */
 388	if (process_pointer(fmt, len, call))
 389		return true;
 390
 391	/* Make sure the field is found */
 392	field = find_event_field(fmt, call);
 393	if (!field)
 394		return false;
 395
 396	/* Test this field's string before printing the event */
 397	call->flags |= TRACE_EVENT_FL_TEST_STR;
 398	field->needs_test = 1;
 399
 400	return true;
 401}
 402
 403static void handle_dereference_arg(const char *arg_str, u64 string_flags, int len,
 404				   u64 *dereference_flags, int arg,
 405				   struct trace_event_call *call)
 406{
 407	if (string_flags & (1ULL << arg)) {
 408		if (process_string(arg_str, len, call))
 409			*dereference_flags &= ~(1ULL << arg);
 410	} else if (process_pointer(arg_str, len, call))
 411		*dereference_flags &= ~(1ULL << arg);
 412	else
 413		pr_warn("TRACE EVENT ERROR: Bad dereference argument: '%.*s'\n",
 414			len, arg_str);
 415}
 416
 417/*
 418 * Examine the print fmt of the event looking for unsafe dereference
 419 * pointers using %p* that could be recorded in the trace event and
 420 * much later referenced after the pointer was freed. Dereferencing
 421 * pointers are OK, if it is dereferenced into the event itself.
 422 */
 423static void test_event_printk(struct trace_event_call *call)
 424{
 425	u64 dereference_flags = 0;
 426	u64 string_flags = 0;
 427	bool first = true;
 428	const char *fmt;
 429	int parens = 0;
 430	char in_quote = 0;
 431	int start_arg = 0;
 432	int arg = 0;
 433	int i, e;
 434
 435	fmt = call->print_fmt;
 436
 437	if (!fmt)
 438		return;
 439
 440	for (i = 0; fmt[i]; i++) {
 441		switch (fmt[i]) {
 442		case '\\':
 443			i++;
 444			if (!fmt[i])
 445				return;
 446			continue;
 447		case '"':
 448		case '\'':
 449			/*
 450			 * The print fmt starts with a string that
 451			 * is processed first to find %p* usage,
 452			 * then after the first string, the print fmt
 453			 * contains arguments that are used to check
 454			 * if the dereferenced %p* usage is safe.
 455			 */
 456			if (first) {
 457				if (fmt[i] == '\'')
 458					continue;
 459				if (in_quote) {
 460					arg = 0;
 461					first = false;
 462					/*
 463					 * If there was no %p* uses
 464					 * the fmt is OK.
 465					 */
 466					if (!dereference_flags)
 467						return;
 468				}
 469			}
 470			if (in_quote) {
 471				if (in_quote == fmt[i])
 472					in_quote = 0;
 473			} else {
 474				in_quote = fmt[i];
 475			}
 476			continue;
 477		case '%':
 478			if (!first || !in_quote)
 479				continue;
 480			i++;
 481			if (!fmt[i])
 482				return;
 483			switch (fmt[i]) {
 484			case '%':
 485				continue;
 486			case 'p':
 487 do_pointer:
 488				/* Find dereferencing fields */
 489				switch (fmt[i + 1]) {
 490				case 'B': case 'R': case 'r':
 491				case 'b': case 'M': case 'm':
 492				case 'I': case 'i': case 'E':
 493				case 'U': case 'V': case 'N':
 494				case 'a': case 'd': case 'D':
 495				case 'g': case 't': case 'C':
 496				case 'O': case 'f':
 497					if (WARN_ONCE(arg == 63,
 498						      "Too many args for event: %s",
 499						      trace_event_name(call)))
 500						return;
 501					dereference_flags |= 1ULL << arg;
 502				}
 503				break;
 504			default:
 505			{
 506				bool star = false;
 507				int j;
 508
 509				/* Increment arg if %*s exists. */
 510				for (j = 0; fmt[i + j]; j++) {
 511					if (isdigit(fmt[i + j]) ||
 512					    fmt[i + j] == '.')
 513						continue;
 514					if (fmt[i + j] == '*') {
 515						star = true;
 516						/* Handle %*pbl case */
 517						if (!j && fmt[i + 1] == 'p') {
 518							arg++;
 519							i++;
 520							goto do_pointer;
 521						}
 522						continue;
 523					}
 524					if ((fmt[i + j] == 's')) {
 525						if (star)
 526							arg++;
 527						if (WARN_ONCE(arg == 63,
 528							      "Too many args for event: %s",
 529							      trace_event_name(call)))
 530							return;
 531						dereference_flags |= 1ULL << arg;
 532						string_flags |= 1ULL << arg;
 533					}
 534					break;
 535				}
 536				break;
 537			} /* default */
 538
 539			} /* switch */
 540			arg++;
 541			continue;
 542		case '(':
 543			if (in_quote)
 544				continue;
 545			parens++;
 546			continue;
 547		case ')':
 548			if (in_quote)
 549				continue;
 550			parens--;
 551			if (WARN_ONCE(parens < 0,
 552				      "Paren mismatch for event: %s\narg='%s'\n%*s",
 553				      trace_event_name(call),
 554				      fmt + start_arg,
 555				      (i - start_arg) + 5, "^"))
 556				return;
 557			continue;
 558		case ',':
 559			if (in_quote || parens)
 560				continue;
 561			e = i;
 562			i++;
 563			while (isspace(fmt[i]))
 564				i++;
 565
 566			/*
 567			 * If start_arg is zero, then this is the start of the
 568			 * first argument. The processing of the argument happens
 569			 * when the end of the argument is found, as it needs to
 570			 * handle parenthesis and such.
 571			 */
 572			if (!start_arg) {
 573				start_arg = i;
 574				/* Balance out the i++ in the for loop */
 575				i--;
 576				continue;
 577			}
 578
 579			if (dereference_flags & (1ULL << arg)) {
 580				handle_dereference_arg(fmt + start_arg, string_flags,
 581						       e - start_arg,
 582						       &dereference_flags, arg, call);
 583			}
 584
 585			start_arg = i;
 586			arg++;
 587			/* Balance out the i++ in the for loop */
 588			i--;
 589		}
 590	}
 591
 592	if (dereference_flags & (1ULL << arg)) {
 593		handle_dereference_arg(fmt + start_arg, string_flags,
 594				       i - start_arg,
 595				       &dereference_flags, arg, call);
 596	}
 597
 598	/*
 599	 * If you triggered the below warning, the trace event reported
 600	 * uses an unsafe dereference pointer %p*. As the data stored
 601	 * at the trace event time may no longer exist when the trace
 602	 * event is printed, dereferencing to the original source is
 603	 * unsafe. The source of the dereference must be copied into the
 604	 * event itself, and the dereference must access the copy instead.
 605	 */
 606	if (WARN_ON_ONCE(dereference_flags)) {
 607		arg = 1;
 608		while (!(dereference_flags & 1)) {
 609			dereference_flags >>= 1;
 610			arg++;
 611		}
 612		pr_warn("event %s has unsafe dereference of argument %d\n",
 613			trace_event_name(call), arg);
 614		pr_warn("print_fmt: %s\n", fmt);
 615	}
 616}
 617
 618int trace_event_raw_init(struct trace_event_call *call)
 619{
 620	int id;
 621
 622	id = register_trace_event(&call->event);
 623	if (!id)
 624		return -ENODEV;
 625
 626	test_event_printk(call);
 627
 628	return 0;
 629}
 630EXPORT_SYMBOL_GPL(trace_event_raw_init);
 631
 632bool trace_event_ignore_this_pid(struct trace_event_file *trace_file)
 633{
 634	struct trace_array *tr = trace_file->tr;
 635	struct trace_pid_list *no_pid_list;
 636	struct trace_pid_list *pid_list;
 637
 638	pid_list = rcu_dereference_raw(tr->filtered_pids);
 639	no_pid_list = rcu_dereference_raw(tr->filtered_no_pids);
 640
 641	if (!pid_list && !no_pid_list)
 642		return false;
 643
 644	/*
 645	 * This is recorded at every sched_switch for this task.
 646	 * Thus, even if the task migrates the ignore value will be the same.
 647	 */
 648	return this_cpu_read(tr->array_buffer.data->ignore_pid) != 0;
 649}
 650EXPORT_SYMBOL_GPL(trace_event_ignore_this_pid);
 651
 652void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
 653				 struct trace_event_file *trace_file,
 654				 unsigned long len)
 655{
 656	struct trace_event_call *event_call = trace_file->event_call;
 657
 658	if ((trace_file->flags & EVENT_FILE_FL_PID_FILTER) &&
 659	    trace_event_ignore_this_pid(trace_file))
 660		return NULL;
 661
 662	/*
 663	 * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables
 664	 * preemption (adding one to the preempt_count). Since we are
 665	 * interested in the preempt_count at the time the tracepoint was
 666	 * hit, we need to subtract one to offset the increment.
 667	 */
 668	fbuffer->trace_ctx = tracing_gen_ctx_dec();
 669	fbuffer->trace_file = trace_file;
 670
 671	fbuffer->event =
 672		trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file,
 673						event_call->event.type, len,
 674						fbuffer->trace_ctx);
 675	if (!fbuffer->event)
 676		return NULL;
 677
 678	fbuffer->regs = NULL;
 679	fbuffer->entry = ring_buffer_event_data(fbuffer->event);
 680	return fbuffer->entry;
 681}
 682EXPORT_SYMBOL_GPL(trace_event_buffer_reserve);
 683
 684int trace_event_reg(struct trace_event_call *call,
 685		    enum trace_reg type, void *data)
 686{
 687	struct trace_event_file *file = data;
 688
 689	WARN_ON(!(call->flags & TRACE_EVENT_FL_TRACEPOINT));
 690	switch (type) {
 691	case TRACE_REG_REGISTER:
 692		return tracepoint_probe_register(call->tp,
 693						 call->class->probe,
 694						 file);
 695	case TRACE_REG_UNREGISTER:
 696		tracepoint_probe_unregister(call->tp,
 697					    call->class->probe,
 698					    file);
 699		return 0;
 700
 701#ifdef CONFIG_PERF_EVENTS
 702	case TRACE_REG_PERF_REGISTER:
 703		if (!call->class->perf_probe)
 704			return -ENODEV;
 705		return tracepoint_probe_register(call->tp,
 706						 call->class->perf_probe,
 707						 call);
 708	case TRACE_REG_PERF_UNREGISTER:
 709		tracepoint_probe_unregister(call->tp,
 710					    call->class->perf_probe,
 711					    call);
 712		return 0;
 713	case TRACE_REG_PERF_OPEN:
 714	case TRACE_REG_PERF_CLOSE:
 715	case TRACE_REG_PERF_ADD:
 716	case TRACE_REG_PERF_DEL:
 717		return 0;
 718#endif
 719	}
 720	return 0;
 721}
 722EXPORT_SYMBOL_GPL(trace_event_reg);
 723
 724void trace_event_enable_cmd_record(bool enable)
 725{
 726	struct trace_event_file *file;
 727	struct trace_array *tr;
 728
 729	lockdep_assert_held(&event_mutex);
 730
 731	do_for_each_event_file(tr, file) {
 732
 733		if (!(file->flags & EVENT_FILE_FL_ENABLED))
 734			continue;
 735
 736		if (enable) {
 737			tracing_start_cmdline_record();
 738			set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 739		} else {
 740			tracing_stop_cmdline_record();
 741			clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 742		}
 743	} while_for_each_event_file();
 744}
 745
 746void trace_event_enable_tgid_record(bool enable)
 747{
 748	struct trace_event_file *file;
 749	struct trace_array *tr;
 750
 751	lockdep_assert_held(&event_mutex);
 752
 753	do_for_each_event_file(tr, file) {
 754		if (!(file->flags & EVENT_FILE_FL_ENABLED))
 755			continue;
 756
 757		if (enable) {
 758			tracing_start_tgid_record();
 759			set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 760		} else {
 761			tracing_stop_tgid_record();
 762			clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT,
 763				  &file->flags);
 764		}
 765	} while_for_each_event_file();
 766}
 767
 768static int __ftrace_event_enable_disable(struct trace_event_file *file,
 769					 int enable, int soft_disable)
 770{
 771	struct trace_event_call *call = file->event_call;
 772	struct trace_array *tr = file->tr;
 773	bool soft_mode = atomic_read(&file->sm_ref) != 0;
 774	int ret = 0;
 775	int disable;
 776
 777	switch (enable) {
 778	case 0:
 779		/*
 780		 * When soft_disable is set and enable is cleared, the sm_ref
 781		 * reference counter is decremented. If it reaches 0, we want
 782		 * to clear the SOFT_DISABLED flag but leave the event in the
 783		 * state that it was. That is, if the event was enabled and
 784		 * SOFT_DISABLED isn't set, then do nothing. But if SOFT_DISABLED
 785		 * is set we do not want the event to be enabled before we
 786		 * clear the bit.
 787		 *
 788		 * When soft_disable is not set but the soft_mode is,
 789		 * we do nothing. Do not disable the tracepoint, otherwise
 790		 * "soft enable"s (clearing the SOFT_DISABLED bit) won't work.
 791		 */
 792		if (soft_disable) {
 793			if (atomic_dec_return(&file->sm_ref) > 0)
 794				break;
 795			disable = file->flags & EVENT_FILE_FL_SOFT_DISABLED;
 796			soft_mode = false;
 797			/* Disable use of trace_buffered_event */
 798			trace_buffered_event_disable();
 799		} else
 800			disable = !soft_mode;
 801
 802		if (disable && (file->flags & EVENT_FILE_FL_ENABLED)) {
 803			clear_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
 804			if (file->flags & EVENT_FILE_FL_RECORDED_CMD) {
 805				tracing_stop_cmdline_record();
 806				clear_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 807			}
 808
 809			if (file->flags & EVENT_FILE_FL_RECORDED_TGID) {
 810				tracing_stop_tgid_record();
 811				clear_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 812			}
 813
 814			ret = call->class->reg(call, TRACE_REG_UNREGISTER, file);
 815
 816			WARN_ON_ONCE(ret);
 817		}
 818		/* If in soft mode, just set the SOFT_DISABLE_BIT, else clear it */
 819		if (soft_mode)
 820			set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 821		else
 822			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 823		break;
 824	case 1:
 825		/*
 826		 * When soft_disable is set and enable is set, we want to
 827		 * register the tracepoint for the event, but leave the event
 828		 * as is. That means, if the event was already enabled, we do
 829		 * nothing. If the event is disabled, we set SOFT_DISABLED
 830		 * before enabling the event tracepoint, so it still seems
 831		 * to be disabled.
 832		 */
 833		if (!soft_disable)
 834			clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 835		else {
 836			if (atomic_inc_return(&file->sm_ref) > 1)
 837				break;
 838			/* Enable use of trace_buffered_event */
 839			trace_buffered_event_enable();
 840		}
 841
 842		if (!(file->flags & EVENT_FILE_FL_ENABLED)) {
 843			bool cmd = false, tgid = false;
 844
 845			/* Keep the event disabled, when going to soft mode. */
 846			if (soft_disable)
 847				set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags);
 848
 849			if (tr->trace_flags & TRACE_ITER(RECORD_CMD)) {
 850				cmd = true;
 851				tracing_start_cmdline_record();
 852				set_bit(EVENT_FILE_FL_RECORDED_CMD_BIT, &file->flags);
 853			}
 854
 855			if (tr->trace_flags & TRACE_ITER(RECORD_TGID)) {
 856				tgid = true;
 857				tracing_start_tgid_record();
 858				set_bit(EVENT_FILE_FL_RECORDED_TGID_BIT, &file->flags);
 859			}
 860
 861			ret = call->class->reg(call, TRACE_REG_REGISTER, file);
 862			if (ret) {
 863				if (cmd)
 864					tracing_stop_cmdline_record();
 865				if (tgid)
 866					tracing_stop_tgid_record();
 867				pr_info("event trace: Could not enable event "
 868					"%s\n", trace_event_name(call));
 869				break;
 870			}
 871			set_bit(EVENT_FILE_FL_ENABLED_BIT, &file->flags);
 872
 873			/* WAS_ENABLED gets set but never cleared. */
 874			set_bit(EVENT_FILE_FL_WAS_ENABLED_BIT, &file->flags);
 875		}
 876		break;
 877	}
 878
 879	return ret;
 880}
 881
 882int trace_event_enable_disable(struct trace_event_file *file,
 883			       int enable, int soft_disable)
 884{
 885	return __ftrace_event_enable_disable(file, enable, soft_disable);
 886}
 887
 888static int ftrace_event_enable_disable(struct trace_event_file *file,
 889				       int enable)
 890{
 891	return __ftrace_event_enable_disable(file, enable, 0);
 892}
 893
 894#ifdef CONFIG_MODULES
 895struct event_mod_load {
 896	struct list_head	list;
 897	char			*module;
 898	char			*match;
 899	char			*system;
 900	char			*event;
 901};
 902
 903static void free_event_mod(struct event_mod_load *event_mod)
 904{
 905	list_del(&event_mod->list);
 906	kfree(event_mod->module);
 907	kfree(event_mod->match);
 908	kfree(event_mod->system);
 909	kfree(event_mod->event);
 910	kfree(event_mod);
 911}
 912
 913static void clear_mod_events(struct trace_array *tr)
 914{
 915	struct event_mod_load *event_mod, *n;
 916
 917	list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
 918		free_event_mod(event_mod);
 919	}
 920}
 921
 922static int remove_cache_mod(struct trace_array *tr, const char *mod,
 923			    const char *match, const char *system, const char *event)
 924{
 925	struct event_mod_load *event_mod, *n;
 926	int ret = -EINVAL;
 927
 928	list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
 929		if (strcmp(event_mod->module, mod) != 0)
 930			continue;
 931
 932		if (match && strcmp(event_mod->match, match) != 0)
 933			continue;
 934
 935		if (system &&
 936		    (!event_mod->system || strcmp(event_mod->system, system) != 0))
 937			continue;
 938
 939		if (event &&
 940		    (!event_mod->event || strcmp(event_mod->event, event) != 0))
 941			continue;
 942
 943		free_event_mod(event_mod);
 944		ret = 0;
 945	}
 946
 947	return ret;
 948}
 949
 950static int cache_mod(struct trace_array *tr, const char *mod, int set,
 951		     const char *match, const char *system, const char *event)
 952{
 953	struct event_mod_load *event_mod;
 954
 955	/* If the module exists, then this just failed to find an event */
 956	if (module_exists(mod))
 957		return -EINVAL;
 958
 959	/* See if this is to remove a cached filter */
 960	if (!set)
 961		return remove_cache_mod(tr, mod, match, system, event);
 962
 963	event_mod = kzalloc(sizeof(*event_mod), GFP_KERNEL);
 964	if (!event_mod)
 965		return -ENOMEM;
 966
 967	INIT_LIST_HEAD(&event_mod->list);
 968	event_mod->module = kstrdup(mod, GFP_KERNEL);
 969	if (!event_mod->module)
 970		goto out_free;
 971
 972	if (match) {
 973		event_mod->match = kstrdup(match, GFP_KERNEL);
 974		if (!event_mod->match)
 975			goto out_free;
 976	}
 977
 978	if (system) {
 979		event_mod->system = kstrdup(system, GFP_KERNEL);
 980		if (!event_mod->system)
 981			goto out_free;
 982	}
 983
 984	if (event) {
 985		event_mod->event = kstrdup(event, GFP_KERNEL);
 986		if (!event_mod->event)
 987			goto out_free;
 988	}
 989
 990	list_add(&event_mod->list, &tr->mod_events);
 991
 992	return 0;
 993
 994 out_free:
 995	free_event_mod(event_mod);
 996
 997	return -ENOMEM;
 998}
 999#else /* CONFIG_MODULES */
1000static inline void clear_mod_events(struct trace_array *tr) { }
1001static int cache_mod(struct trace_array *tr, const char *mod, int set,
1002		     const char *match, const char *system, const char *event)
1003{
1004	return -EINVAL;
1005}
1006#endif
1007
1008static void ftrace_clear_events(struct trace_array *tr)
1009{
1010	struct trace_event_file *file;
1011
1012	mutex_lock(&event_mutex);
1013	list_for_each_entry(file, &tr->events, list) {
1014		ftrace_event_enable_disable(file, 0);
1015	}
1016	clear_mod_events(tr);
1017	mutex_unlock(&event_mutex);
1018}
1019
1020static void
1021event_filter_pid_sched_process_exit(void *data, struct task_struct *task)
1022{
1023	struct trace_pid_list *pid_list;
1024	struct trace_array *tr = data;
1025
1026	pid_list = rcu_dereference_raw(tr->filtered_pids);
1027	trace_filter_add_remove_task(pid_list, NULL, task);
1028
1029	pid_list = rcu_dereference_raw(tr->filtered_no_pids);
1030	trace_filter_add_remove_task(pid_list, NULL, task);
1031}
1032
1033static void
1034event_filter_pid_sched_process_fork(void *data,
1035				    struct task_struct *self,
1036				    struct task_struct *task)
1037{
1038	struct trace_pid_list *pid_list;
1039	struct trace_array *tr = data;
1040
1041	pid_list = rcu_dereference_sched(tr->filtered_pids);
1042	trace_filter_add_remove_task(pid_list, self, task);
1043
1044	pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1045	trace_filter_add_remove_task(pid_list, self, task);
1046}
1047
1048void trace_event_follow_fork(struct trace_array *tr, bool enable)
1049{
1050	if (enable) {
1051		register_trace_prio_sched_process_fork(event_filter_pid_sched_process_fork,
1052						       tr, INT_MIN);
1053		register_trace_prio_sched_process_free(event_filter_pid_sched_process_exit,
1054						       tr, INT_MAX);
1055	} else {
1056		unregister_trace_sched_process_fork(event_filter_pid_sched_process_fork,
1057						    tr);
1058		unregister_trace_sched_process_free(event_filter_pid_sched_process_exit,
1059						    tr);
1060	}
1061}
1062
1063static void
1064event_filter_pid_sched_switch_probe_pre(void *data, bool preempt,
1065					struct task_struct *prev,
1066					struct task_struct *next,
1067					unsigned int prev_state)
1068{
1069	struct trace_array *tr = data;
1070	struct trace_pid_list *no_pid_list;
1071	struct trace_pid_list *pid_list;
1072	bool ret;
1073
1074	pid_list = rcu_dereference_sched(tr->filtered_pids);
1075	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1076
1077	/*
1078	 * Sched switch is funny, as we only want to ignore it
1079	 * in the notrace case if both prev and next should be ignored.
1080	 */
1081	ret = trace_ignore_this_task(NULL, no_pid_list, prev) &&
1082		trace_ignore_this_task(NULL, no_pid_list, next);
1083
1084	this_cpu_write(tr->array_buffer.data->ignore_pid, ret ||
1085		       (trace_ignore_this_task(pid_list, NULL, prev) &&
1086			trace_ignore_this_task(pid_list, NULL, next)));
1087}
1088
1089static void
1090event_filter_pid_sched_switch_probe_post(void *data, bool preempt,
1091					 struct task_struct *prev,
1092					 struct task_struct *next,
1093					 unsigned int prev_state)
1094{
1095	struct trace_array *tr = data;
1096	struct trace_pid_list *no_pid_list;
1097	struct trace_pid_list *pid_list;
1098
1099	pid_list = rcu_dereference_sched(tr->filtered_pids);
1100	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1101
1102	this_cpu_write(tr->array_buffer.data->ignore_pid,
1103		       trace_ignore_this_task(pid_list, no_pid_list, next));
1104}
1105
1106static void
1107event_filter_pid_sched_wakeup_probe_pre(void *data, struct task_struct *task)
1108{
1109	struct trace_array *tr = data;
1110	struct trace_pid_list *no_pid_list;
1111	struct trace_pid_list *pid_list;
1112
1113	/* Nothing to do if we are already tracing */
1114	if (!this_cpu_read(tr->array_buffer.data->ignore_pid))
1115		return;
1116
1117	pid_list = rcu_dereference_sched(tr->filtered_pids);
1118	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1119
1120	this_cpu_write(tr->array_buffer.data->ignore_pid,
1121		       trace_ignore_this_task(pid_list, no_pid_list, task));
1122}
1123
1124static void
1125event_filter_pid_sched_wakeup_probe_post(void *data, struct task_struct *task)
1126{
1127	struct trace_array *tr = data;
1128	struct trace_pid_list *no_pid_list;
1129	struct trace_pid_list *pid_list;
1130
1131	/* Nothing to do if we are not tracing */
1132	if (this_cpu_read(tr->array_buffer.data->ignore_pid))
1133		return;
1134
1135	pid_list = rcu_dereference_sched(tr->filtered_pids);
1136	no_pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1137
1138	/* Set tracing if current is enabled */
1139	this_cpu_write(tr->array_buffer.data->ignore_pid,
1140		       trace_ignore_this_task(pid_list, no_pid_list, current));
1141}
1142
1143static void unregister_pid_events(struct trace_array *tr)
1144{
1145	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_pre, tr);
1146	unregister_trace_sched_switch(event_filter_pid_sched_switch_probe_post, tr);
1147
1148	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre, tr);
1149	unregister_trace_sched_wakeup(event_filter_pid_sched_wakeup_probe_post, tr);
1150
1151	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre, tr);
1152	unregister_trace_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post, tr);
1153
1154	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_pre, tr);
1155	unregister_trace_sched_waking(event_filter_pid_sched_wakeup_probe_post, tr);
1156}
1157
1158static void __ftrace_clear_event_pids(struct trace_array *tr, int type)
1159{
1160	struct trace_pid_list *pid_list;
1161	struct trace_pid_list *no_pid_list;
1162	struct trace_event_file *file;
1163	int cpu;
1164
1165	pid_list = rcu_dereference_protected(tr->filtered_pids,
1166					     lockdep_is_held(&event_mutex));
1167	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
1168					     lockdep_is_held(&event_mutex));
1169
1170	/* Make sure there's something to do */
1171	if (!pid_type_enabled(type, pid_list, no_pid_list))
1172		return;
1173
1174	if (!still_need_pid_events(type, pid_list, no_pid_list)) {
1175		unregister_pid_events(tr);
1176
1177		list_for_each_entry(file, &tr->events, list) {
1178			clear_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
1179		}
1180
1181		for_each_possible_cpu(cpu)
1182			per_cpu_ptr(tr->array_buffer.data, cpu)->ignore_pid = false;
1183	}
1184
1185	if (type & TRACE_PIDS)
1186		rcu_assign_pointer(tr->filtered_pids, NULL);
1187
1188	if (type & TRACE_NO_PIDS)
1189		rcu_assign_pointer(tr->filtered_no_pids, NULL);
1190
1191	/* Wait till all users are no longer using pid filtering */
1192	tracepoint_synchronize_unregister();
1193
1194	if ((type & TRACE_PIDS) && pid_list)
1195		trace_pid_list_free(pid_list);
1196
1197	if ((type & TRACE_NO_PIDS) && no_pid_list)
1198		trace_pid_list_free(no_pid_list);
1199}
1200
1201static void ftrace_clear_event_pids(struct trace_array *tr, int type)
1202{
1203	mutex_lock(&event_mutex);
1204	__ftrace_clear_event_pids(tr, type);
1205	mutex_unlock(&event_mutex);
1206}
1207
1208static void __put_system(struct event_subsystem *system)
1209{
1210	struct event_filter *filter = system->filter;
1211
1212	WARN_ON_ONCE(system_refcount(system) == 0);
1213	if (system_refcount_dec(system))
1214		return;
1215
1216	list_del(&system->list);
1217
1218	if (filter) {
1219		kfree(filter->filter_string);
1220		kfree(filter);
1221	}
1222	kfree_const(system->name);
1223	kfree(system);
1224}
1225
1226static void __get_system(struct event_subsystem *system)
1227{
1228	WARN_ON_ONCE(system_refcount(system) == 0);
1229	system_refcount_inc(system);
1230}
1231
1232static void __get_system_dir(struct trace_subsystem_dir *dir)
1233{
1234	WARN_ON_ONCE(dir->ref_count == 0);
1235	dir->ref_count++;
1236	__get_system(dir->subsystem);
1237}
1238
1239static void __put_system_dir(struct trace_subsystem_dir *dir)
1240{
1241	WARN_ON_ONCE(dir->ref_count == 0);
1242	/* If the subsystem is about to be freed, the dir must be too */
1243	WARN_ON_ONCE(system_refcount(dir->subsystem) == 1 && dir->ref_count != 1);
1244
1245	__put_system(dir->subsystem);
1246	if (!--dir->ref_count)
1247		kfree(dir);
1248}
1249
1250static void put_system(struct trace_subsystem_dir *dir)
1251{
1252	mutex_lock(&event_mutex);
1253	__put_system_dir(dir);
1254	mutex_unlock(&event_mutex);
1255}
1256
1257static void remove_subsystem(struct trace_subsystem_dir *dir)
1258{
1259	if (!dir)
1260		return;
1261
1262	if (!--dir->nr_events) {
1263		eventfs_remove_dir(dir->ei);
1264		list_del(&dir->list);
1265		__put_system_dir(dir);
1266	}
1267}
1268
1269void event_file_get(struct trace_event_file *file)
1270{
1271	refcount_inc(&file->ref);
1272}
1273
1274void event_file_put(struct trace_event_file *file)
1275{
1276	if (WARN_ON_ONCE(!refcount_read(&file->ref))) {
1277		if (file->flags & EVENT_FILE_FL_FREED)
1278			kmem_cache_free(file_cachep, file);
1279		return;
1280	}
1281
1282	if (refcount_dec_and_test(&file->ref)) {
1283		/* Count should only go to zero when it is freed */
1284		if (WARN_ON_ONCE(!(file->flags & EVENT_FILE_FL_FREED)))
1285			return;
1286		kmem_cache_free(file_cachep, file);
1287	}
1288}
1289
1290static void remove_event_file_dir(struct trace_event_file *file)
1291{
1292	eventfs_remove_dir(file->ei);
1293	list_del(&file->list);
1294	remove_subsystem(file->system);
1295	free_event_filter(file->filter);
1296	file->flags |= EVENT_FILE_FL_FREED;
1297	event_file_put(file);
1298
1299	/* Wake up hist poll waiters to notice the EVENT_FILE_FL_FREED flag. */
1300	hist_poll_wakeup();
1301}
1302
1303/*
1304 * __ftrace_set_clr_event(NULL, NULL, NULL, set) will set/unset all events.
1305 */
1306static int
1307__ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match,
1308			      const char *sub, const char *event, int set,
1309			      const char *mod)
1310{
1311	struct trace_event_file *file;
1312	struct trace_event_call *call;
1313	char *module __free(kfree) = NULL;
1314	const char *name;
1315	int ret = -EINVAL;
1316	int eret = 0;
1317
1318	if (mod) {
1319		char *p;
1320
1321		module = kstrdup(mod, GFP_KERNEL);
1322		if (!module)
1323			return -ENOMEM;
1324
1325		/* Replace all '-' with '_' as that's what modules do */
1326		for (p = strchr(module, '-'); p; p = strchr(p + 1, '-'))
1327			*p = '_';
1328	}
1329
1330	list_for_each_entry(file, &tr->events, list) {
1331
1332		call = file->event_call;
1333
1334		/* If a module is specified, skip events that are not that module */
1335		if (module && (!call->module || strcmp(module_name(call->module), module)))
1336			continue;
1337
1338		name = trace_event_name(call);
1339
1340		if (!name || !call->class || !call->class->reg)
1341			continue;
1342
1343		if (call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
1344			continue;
1345
1346		if (match &&
1347		    strcmp(match, name) != 0 &&
1348		    strcmp(match, call->class->system) != 0)
1349			continue;
1350
1351		if (sub && strcmp(sub, call->class->system) != 0)
1352			continue;
1353
1354		if (event && strcmp(event, name) != 0)
1355			continue;
1356
1357		ret = ftrace_event_enable_disable(file, set);
1358
1359		/*
1360		 * Save the first error and return that. Some events
1361		 * may still have been enabled, but let the user
1362		 * know that something went wrong.
1363		 */
1364		if (ret && !eret)
1365			eret = ret;
1366
1367		ret = eret;
1368	}
1369
1370	/*
1371	 * If this is a module setting and nothing was found,
1372	 * check if the module was loaded. If it wasn't cache it.
1373	 */
1374	if (module && ret == -EINVAL && !eret)
1375		ret = cache_mod(tr, module, set, match, sub, event);
1376
1377	return ret;
1378}
1379
1380static int __ftrace_set_clr_event(struct trace_array *tr, const char *match,
1381				  const char *sub, const char *event, int set,
1382				  const char *mod)
1383{
1384	int ret;
1385
1386	mutex_lock(&event_mutex);
1387	ret = __ftrace_set_clr_event_nolock(tr, match, sub, event, set, mod);
1388	mutex_unlock(&event_mutex);
1389
1390	return ret;
1391}
1392
1393int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set)
1394{
1395	char *event = NULL, *sub = NULL, *match, *mod;
1396	int ret;
1397
1398	if (!tr)
1399		return -ENOENT;
1400
1401	/* Modules events can be appended with :mod:<module> */
1402	mod = strstr(buf, ":mod:");
1403	if (mod) {
1404		*mod = '\0';
1405		/* move to the module name */
1406		mod += 5;
1407	}
1408
1409	/*
1410	 * The buf format can be <subsystem>:<event-name>
1411	 *  *:<event-name> means any event by that name.
1412	 *  :<event-name> is the same.
1413	 *
1414	 *  <subsystem>:* means all events in that subsystem
1415	 *  <subsystem>: means the same.
1416	 *
1417	 *  <name> (no ':') means all events in a subsystem with
1418	 *  the name <name> or any event that matches <name>
1419	 */
1420
1421	match = strsep(&buf, ":");
1422	if (buf) {
1423		sub = match;
1424		event = buf;
1425		match = NULL;
1426
1427		if (!strlen(sub) || strcmp(sub, "*") == 0)
1428			sub = NULL;
1429		if (!strlen(event) || strcmp(event, "*") == 0)
1430			event = NULL;
1431	} else if (mod) {
1432		/* Allow wildcard for no length or star */
1433		if (!strlen(match) || strcmp(match, "*") == 0)
1434			match = NULL;
1435	}
1436
1437	ret = __ftrace_set_clr_event(tr, match, sub, event, set, mod);
1438
1439	/* Put back the colon to allow this to be called again */
1440	if (buf)
1441		*(buf - 1) = ':';
1442
1443	return ret;
1444}
1445
1446/**
1447 * trace_set_clr_event - enable or disable an event
1448 * @system: system name to match (NULL for any system)
1449 * @event: event name to match (NULL for all events, within system)
1450 * @set: 1 to enable, 0 to disable
1451 *
1452 * This is a way for other parts of the kernel to enable or disable
1453 * event recording.
1454 *
1455 * Returns 0 on success, -EINVAL if the parameters do not match any
1456 * registered events.
1457 */
1458int trace_set_clr_event(const char *system, const char *event, int set)
1459{
1460	struct trace_array *tr = top_trace_array();
1461
1462	if (!tr)
1463		return -ENODEV;
1464
1465	return __ftrace_set_clr_event(tr, NULL, system, event, set, NULL);
1466}
1467EXPORT_SYMBOL_GPL(trace_set_clr_event);
1468
1469/**
1470 * trace_array_set_clr_event - enable or disable an event for a trace array.
1471 * @tr: concerned trace array.
1472 * @system: system name to match (NULL for any system)
1473 * @event: event name to match (NULL for all events, within system)
1474 * @enable: true to enable, false to disable
1475 *
1476 * This is a way for other parts of the kernel to enable or disable
1477 * event recording.
1478 *
1479 * Returns 0 on success, -EINVAL if the parameters do not match any
1480 * registered events.
1481 */
1482int trace_array_set_clr_event(struct trace_array *tr, const char *system,
1483		const char *event, bool enable)
1484{
1485	int set;
1486
1487	if (!tr)
1488		return -ENOENT;
1489
1490	set = (enable == true) ? 1 : 0;
1491	return __ftrace_set_clr_event(tr, NULL, system, event, set, NULL);
1492}
1493EXPORT_SYMBOL_GPL(trace_array_set_clr_event);
1494
1495/* 128 should be much more than enough */
1496#define EVENT_BUF_SIZE		127
1497
1498static ssize_t
1499ftrace_event_write(struct file *file, const char __user *ubuf,
1500		   size_t cnt, loff_t *ppos)
1501{
1502	struct trace_parser parser;
1503	struct seq_file *m = file->private_data;
1504	struct trace_array *tr = m->private;
1505	ssize_t read, ret;
1506
1507	if (!cnt)
1508		return 0;
1509
1510	ret = tracing_update_buffers(tr);
1511	if (ret < 0)
1512		return ret;
1513
1514	if (trace_parser_get_init(&parser, EVENT_BUF_SIZE + 1))
1515		return -ENOMEM;
1516
1517	read = trace_get_user(&parser, ubuf, cnt, ppos);
1518
1519	if (read >= 0 && trace_parser_loaded((&parser))) {
1520		int set = 1;
1521
1522		if (*parser.buffer == '!')
1523			set = 0;
1524
1525		ret = ftrace_set_clr_event(tr, parser.buffer + !set, set);
1526		if (ret)
1527			goto out_put;
1528	}
1529
1530	ret = read;
1531
1532 out_put:
1533	trace_parser_put(&parser);
1534
1535	return ret;
1536}
1537
1538static void *
1539t_next(struct seq_file *m, void *v, loff_t *pos)
1540{
1541	struct trace_event_file *file = v;
1542	struct trace_event_call *call;
1543	struct trace_array *tr = m->private;
1544
1545	(*pos)++;
1546
1547	list_for_each_entry_continue(file, &tr->events, list) {
1548		call = file->event_call;
1549		/*
1550		 * The ftrace subsystem is for showing formats only.
1551		 * They can not be enabled or disabled via the event files.
1552		 */
1553		if (call->class && call->class->reg &&
1554		    !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE))
1555			return file;
1556	}
1557
1558	return NULL;
1559}
1560
1561static void *t_start(struct seq_file *m, loff_t *pos)
1562{
1563	struct trace_event_file *file;
1564	struct trace_array *tr = m->private;
1565	loff_t l;
1566
1567	mutex_lock(&event_mutex);
1568
1569	file = list_entry(&tr->events, struct trace_event_file, list);
1570	for (l = 0; l <= *pos; ) {
1571		file = t_next(m, file, &l);
1572		if (!file)
1573			break;
1574	}
1575	return file;
1576}
1577
1578enum set_event_iter_type {
1579	SET_EVENT_FILE,
1580	SET_EVENT_MOD,
1581};
1582
1583struct set_event_iter {
1584	enum set_event_iter_type	type;
1585	union {
1586		struct trace_event_file	*file;
1587		struct event_mod_load	*event_mod;
1588	};
1589};
1590
1591static void *
1592s_next(struct seq_file *m, void *v, loff_t *pos)
1593{
1594	struct set_event_iter *iter = v;
1595	struct trace_event_file *file;
1596	struct trace_array *tr = m->private;
1597
1598	(*pos)++;
1599
1600	if (iter->type == SET_EVENT_FILE) {
1601		file = iter->file;
1602		list_for_each_entry_continue(file, &tr->events, list) {
1603			if (file->flags & EVENT_FILE_FL_ENABLED) {
1604				iter->file = file;
1605				return iter;
1606			}
1607		}
1608#ifdef CONFIG_MODULES
1609		iter->type = SET_EVENT_MOD;
1610		iter->event_mod = list_entry(&tr->mod_events, struct event_mod_load, list);
1611#endif
1612	}
1613
1614#ifdef CONFIG_MODULES
1615	list_for_each_entry_continue(iter->event_mod, &tr->mod_events, list)
1616		return iter;
1617#endif
1618
1619	/*
1620	 * The iter is allocated in s_start() and passed via the 'v'
1621	 * parameter. To stop the iterator, NULL must be returned. But
1622	 * the return value is what the 'v' parameter in s_stop() receives
1623	 * and frees. Free iter here as it will no longer be used.
1624	 */
1625	kfree(iter);
1626	return NULL;
1627}
1628
1629static void *s_start(struct seq_file *m, loff_t *pos)
1630{
1631	struct trace_array *tr = m->private;
1632	struct set_event_iter *iter;
1633	loff_t l;
1634
1635	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1636	mutex_lock(&event_mutex);
1637	if (!iter)
1638		return NULL;
1639
1640	iter->type = SET_EVENT_FILE;
1641	iter->file = list_entry(&tr->events, struct trace_event_file, list);
1642
1643	for (l = 0; l <= *pos; ) {
1644		iter = s_next(m, iter, &l);
1645		if (!iter)
1646			break;
1647	}
1648	return iter;
1649}
1650
1651static int t_show(struct seq_file *m, void *v)
1652{
1653	struct trace_event_file *file = v;
1654	struct trace_event_call *call = file->event_call;
1655
1656	if (strcmp(call->class->system, TRACE_SYSTEM) != 0)
1657		seq_printf(m, "%s:", call->class->system);
1658	seq_printf(m, "%s\n", trace_event_name(call));
1659
1660	return 0;
1661}
1662
1663static void t_stop(struct seq_file *m, void *p)
1664{
1665	mutex_unlock(&event_mutex);
1666}
1667
1668#ifdef CONFIG_MODULES
1669static int s_show(struct seq_file *m, void *v)
1670{
1671	struct set_event_iter *iter = v;
1672	const char *system;
1673	const char *event;
1674
1675	if (iter->type == SET_EVENT_FILE)
1676		return t_show(m, iter->file);
1677
1678	/* When match is set, system and event are not */
1679	if (iter->event_mod->match) {
1680		seq_printf(m, "%s:mod:%s\n", iter->event_mod->match,
1681			   iter->event_mod->module);
1682		return 0;
1683	}
1684
1685	system = iter->event_mod->system ? : "*";
1686	event = iter->event_mod->event ? : "*";
1687
1688	seq_printf(m, "%s:%s:mod:%s\n", system, event, iter->event_mod->module);
1689
1690	return 0;
1691}
1692#else /* CONFIG_MODULES */
1693static int s_show(struct seq_file *m, void *v)
1694{
1695	struct set_event_iter *iter = v;
1696
1697	return t_show(m, iter->file);
1698}
1699#endif
1700
1701static void s_stop(struct seq_file *m, void *v)
1702{
1703	kfree(v);
1704	t_stop(m, NULL);
1705}
1706
1707static void *
1708__next(struct seq_file *m, void *v, loff_t *pos, int type)
1709{
1710	struct trace_array *tr = m->private;
1711	struct trace_pid_list *pid_list;
1712
1713	if (type == TRACE_PIDS)
1714		pid_list = rcu_dereference_sched(tr->filtered_pids);
1715	else
1716		pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1717
1718	return trace_pid_next(pid_list, v, pos);
1719}
1720
1721static void *
1722p_next(struct seq_file *m, void *v, loff_t *pos)
1723{
1724	return __next(m, v, pos, TRACE_PIDS);
1725}
1726
1727static void *
1728np_next(struct seq_file *m, void *v, loff_t *pos)
1729{
1730	return __next(m, v, pos, TRACE_NO_PIDS);
1731}
1732
1733static void *__start(struct seq_file *m, loff_t *pos, int type)
1734	__acquires(RCU)
1735{
1736	struct trace_pid_list *pid_list;
1737	struct trace_array *tr = m->private;
1738
1739	/*
1740	 * Grab the mutex, to keep calls to p_next() having the same
1741	 * tr->filtered_pids as p_start() has.
1742	 * If we just passed the tr->filtered_pids around, then RCU would
1743	 * have been enough, but doing that makes things more complex.
1744	 */
1745	mutex_lock(&event_mutex);
1746	rcu_read_lock_sched();
1747
1748	if (type == TRACE_PIDS)
1749		pid_list = rcu_dereference_sched(tr->filtered_pids);
1750	else
1751		pid_list = rcu_dereference_sched(tr->filtered_no_pids);
1752
1753	if (!pid_list)
1754		return NULL;
1755
1756	return trace_pid_start(pid_list, pos);
1757}
1758
1759static void *p_start(struct seq_file *m, loff_t *pos)
1760	__acquires(RCU)
1761{
1762	return __start(m, pos, TRACE_PIDS);
1763}
1764
1765static void *np_start(struct seq_file *m, loff_t *pos)
1766	__acquires(RCU)
1767{
1768	return __start(m, pos, TRACE_NO_PIDS);
1769}
1770
1771static void p_stop(struct seq_file *m, void *p)
1772	__releases(RCU)
1773{
1774	rcu_read_unlock_sched();
1775	mutex_unlock(&event_mutex);
1776}
1777
1778static ssize_t
1779event_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1780		  loff_t *ppos)
1781{
1782	struct trace_event_file *file;
1783	unsigned long flags;
1784	char buf[4] = "0";
1785
1786	mutex_lock(&event_mutex);
1787	file = event_file_file(filp);
1788	if (likely(file))
1789		flags = file->flags;
1790	mutex_unlock(&event_mutex);
1791
1792	if (!file)
1793		return -ENODEV;
1794
1795	if (flags & EVENT_FILE_FL_ENABLED &&
1796	    !(flags & EVENT_FILE_FL_SOFT_DISABLED))
1797		strcpy(buf, "1");
1798
1799	if (atomic_read(&file->sm_ref) != 0)
1800		strcat(buf, "*");
1801
1802	strcat(buf, "\n");
1803
1804	return simple_read_from_buffer(ubuf, cnt, ppos, buf, strlen(buf));
1805}
1806
1807static ssize_t
1808event_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1809		   loff_t *ppos)
1810{
1811	struct trace_event_file *file;
1812	unsigned long val;
1813	int ret;
1814
1815	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1816	if (ret)
1817		return ret;
1818
1819	guard(mutex)(&event_mutex);
1820
1821	switch (val) {
1822	case 0:
1823	case 1:
1824		file = event_file_file(filp);
1825		if (!file)
1826			return -ENODEV;
1827		ret = tracing_update_buffers(file->tr);
1828		if (ret < 0)
1829			return ret;
1830		ret = ftrace_event_enable_disable(file, val);
1831		if (ret < 0)
1832			return ret;
1833		break;
1834
1835	default:
1836		return -EINVAL;
1837	}
1838
1839	*ppos += cnt;
1840
1841	return cnt;
1842}
1843
1844/*
1845 * Returns:
1846 *   0 : no events exist?
1847 *   1 : all events are disabled
1848 *   2 : all events are enabled
1849 *   3 : some events are enabled and some are enabled
1850 */
1851int trace_events_enabled(struct trace_array *tr, const char *system)
1852{
1853	struct trace_event_call *call;
1854	struct trace_event_file *file;
1855	int set = 0;
1856
1857	guard(mutex)(&event_mutex);
1858
1859	list_for_each_entry(file, &tr->events, list) {
1860		call = file->event_call;
1861		if ((call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
1862		    !trace_event_name(call) || !call->class || !call->class->reg)
1863			continue;
1864
1865		if (system && strcmp(call->class->system, system) != 0)
1866			continue;
1867
1868		/*
1869		 * We need to find out if all the events are set
1870		 * or if all events or cleared, or if we have
1871		 * a mixture.
1872		 */
1873		set |= (1 << !!(file->flags & EVENT_FILE_FL_ENABLED));
1874
1875		/*
1876		 * If we have a mixture, no need to look further.
1877		 */
1878		if (set == 3)
1879			break;
1880	}
1881
1882	return set;
1883}
1884
1885static ssize_t
1886system_enable_read(struct file *filp, char __user *ubuf, size_t cnt,
1887		   loff_t *ppos)
1888{
1889	const char set_to_char[4] = { '?', '0', '1', 'X' };
1890	struct trace_subsystem_dir *dir = filp->private_data;
1891	struct event_subsystem *system = dir->subsystem;
1892	struct trace_array *tr = dir->tr;
1893	char buf[2];
1894	int set;
1895	int ret;
1896
1897	set = trace_events_enabled(tr, system ? system->name : NULL);
1898
1899	buf[0] = set_to_char[set];
1900	buf[1] = '\n';
1901
1902	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
1903
1904	return ret;
1905}
1906
1907static ssize_t
1908system_enable_write(struct file *filp, const char __user *ubuf, size_t cnt,
1909		    loff_t *ppos)
1910{
1911	struct trace_subsystem_dir *dir = filp->private_data;
1912	struct event_subsystem *system = dir->subsystem;
1913	const char *name = NULL;
1914	unsigned long val;
1915	ssize_t ret;
1916
1917	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
1918	if (ret)
1919		return ret;
1920
1921	ret = tracing_update_buffers(dir->tr);
1922	if (ret < 0)
1923		return ret;
1924
1925	if (val != 0 && val != 1)
1926		return -EINVAL;
1927
1928	/*
1929	 * Opening of "enable" adds a ref count to system,
1930	 * so the name is safe to use.
1931	 */
1932	if (system)
1933		name = system->name;
1934
1935	ret = __ftrace_set_clr_event(dir->tr, NULL, name, NULL, val, NULL);
1936	if (ret)
1937		goto out;
1938
1939	ret = cnt;
1940
1941out:
1942	*ppos += cnt;
1943
1944	return ret;
1945}
1946
1947enum {
1948	FORMAT_HEADER		= 1,
1949	FORMAT_FIELD_SEPERATOR	= 2,
1950	FORMAT_PRINTFMT		= 3,
1951};
1952
1953static void *f_next(struct seq_file *m, void *v, loff_t *pos)
1954{
1955	struct trace_event_file *file = event_file_data(m->private);
1956	struct trace_event_call *call = file->event_call;
1957	struct list_head *common_head = &ftrace_common_fields;
1958	struct list_head *head = trace_get_fields(call);
1959	struct list_head *node = v;
1960
1961	(*pos)++;
1962
1963	switch ((unsigned long)v) {
1964	case FORMAT_HEADER:
1965		node = common_head;
1966		break;
1967
1968	case FORMAT_FIELD_SEPERATOR:
1969		node = head;
1970		break;
1971
1972	case FORMAT_PRINTFMT:
1973		/* all done */
1974		return NULL;
1975	}
1976
1977	node = node->prev;
1978	if (node == common_head)
1979		return (void *)FORMAT_FIELD_SEPERATOR;
1980	else if (node == head)
1981		return (void *)FORMAT_PRINTFMT;
1982	else
1983		return node;
1984}
1985
1986static int f_show(struct seq_file *m, void *v)
1987{
1988	struct trace_event_file *file = event_file_data(m->private);
1989	struct trace_event_call *call = file->event_call;
1990	struct ftrace_event_field *field;
1991	const char *array_descriptor;
1992
1993	switch ((unsigned long)v) {
1994	case FORMAT_HEADER:
1995		seq_printf(m, "name: %s\n", trace_event_name(call));
1996		seq_printf(m, "ID: %d\n", call->event.type);
1997		seq_puts(m, "format:\n");
1998		return 0;
1999
2000	case FORMAT_FIELD_SEPERATOR:
2001		seq_putc(m, '\n');
2002		return 0;
2003
2004	case FORMAT_PRINTFMT:
2005		seq_printf(m, "\nprint fmt: %s\n",
2006			   call->print_fmt);
2007		return 0;
2008	}
2009
2010	field = list_entry(v, struct ftrace_event_field, link);
2011	/*
2012	 * Smartly shows the array type(except dynamic array).
2013	 * Normal:
2014	 *	field:TYPE VAR
2015	 * If TYPE := TYPE[LEN], it is shown:
2016	 *	field:TYPE VAR[LEN]
2017	 */
2018	array_descriptor = strchr(field->type, '[');
2019
2020	if (str_has_prefix(field->type, "__data_loc"))
2021		array_descriptor = NULL;
2022
2023	if (!array_descriptor)
2024		seq_printf(m, "\tfield:%s %s;\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2025			   field->type, field->name, field->offset,
2026			   field->size, !!field->is_signed);
2027	else if (field->len)
2028		seq_printf(m, "\tfield:%.*s %s[%d];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2029			   (int)(array_descriptor - field->type),
2030			   field->type, field->name,
2031			   field->len, field->offset,
2032			   field->size, !!field->is_signed);
2033	else
2034		seq_printf(m, "\tfield:%.*s %s[];\toffset:%u;\tsize:%u;\tsigned:%d;\n",
2035				(int)(array_descriptor - field->type),
2036				field->type, field->name,
2037				field->offset, field->size, !!field->is_signed);
2038
2039	return 0;
2040}
2041
2042static void *f_start(struct seq_file *m, loff_t *pos)
2043{
2044	struct trace_event_file *file;
2045	void *p = (void *)FORMAT_HEADER;
2046	loff_t l = 0;
2047
2048	/* ->stop() is called even if ->start() fails */
2049	mutex_lock(&event_mutex);
2050	file = event_file_file(m->private);
2051	if (!file)
2052		return ERR_PTR(-ENODEV);
2053
2054	while (l < *pos && p)
2055		p = f_next(m, p, &l);
2056
2057	return p;
2058}
2059
2060static void f_stop(struct seq_file *m, void *p)
2061{
2062	mutex_unlock(&event_mutex);
2063}
2064
2065static const struct seq_operations trace_format_seq_ops = {
2066	.start		= f_start,
2067	.next		= f_next,
2068	.stop		= f_stop,
2069	.show		= f_show,
2070};
2071
2072static int trace_format_open(struct inode *inode, struct file *file)
2073{
2074	struct seq_file *m;
2075	int ret;
2076
2077	/* Do we want to hide event format files on tracefs lockdown? */
2078
2079	ret = seq_open(file, &trace_format_seq_ops);
2080	if (ret < 0)
2081		return ret;
2082
2083	m = file->private_data;
2084	m->private = file;
2085
2086	return 0;
2087}
2088
2089#ifdef CONFIG_PERF_EVENTS
2090static ssize_t
2091event_id_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2092{
2093	int id = (long)event_file_data(filp);
2094	char buf[32];
2095	int len;
2096
2097	if (unlikely(!id))
2098		return -ENODEV;
2099
2100	len = sprintf(buf, "%d\n", id);
2101
2102	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);
2103}
2104#endif
2105
2106static ssize_t
2107event_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2108		  loff_t *ppos)
2109{
2110	struct trace_event_file *file;
2111	struct trace_seq *s;
2112	int r = -ENODEV;
2113
2114	if (*ppos)
2115		return 0;
2116
2117	s = kmalloc(sizeof(*s), GFP_KERNEL);
2118
2119	if (!s)
2120		return -ENOMEM;
2121
2122	trace_seq_init(s);
2123
2124	mutex_lock(&event_mutex);
2125	file = event_file_file(filp);
2126	if (file)
2127		print_event_filter(file, s);
2128	mutex_unlock(&event_mutex);
2129
2130	if (file)
2131		r = simple_read_from_buffer(ubuf, cnt, ppos,
2132					    s->buffer, trace_seq_used(s));
2133
2134	kfree(s);
2135
2136	return r;
2137}
2138
2139static ssize_t
2140event_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2141		   loff_t *ppos)
2142{
2143	struct trace_event_file *file;
2144	char *buf;
2145	int err = -ENODEV;
2146
2147	if (cnt >= PAGE_SIZE)
2148		return -EINVAL;
2149
2150	buf = memdup_user_nul(ubuf, cnt);
2151	if (IS_ERR(buf))
2152		return PTR_ERR(buf);
2153
2154	mutex_lock(&event_mutex);
2155	file = event_file_file(filp);
2156	if (file) {
2157		if (file->flags & EVENT_FILE_FL_FREED)
2158			err = -ENODEV;
2159		else
2160			err = apply_event_filter(file, buf);
2161	}
2162	mutex_unlock(&event_mutex);
2163
2164	kfree(buf);
2165	if (err < 0)
2166		return err;
2167
2168	*ppos += cnt;
2169
2170	return cnt;
2171}
2172
2173static LIST_HEAD(event_subsystems);
2174
2175static int subsystem_open(struct inode *inode, struct file *filp)
2176{
2177	struct trace_subsystem_dir *dir = NULL, *iter_dir;
2178	struct trace_array *tr = NULL, *iter_tr;
2179	struct event_subsystem *system = NULL;
2180	int ret;
2181
2182	if (tracing_is_disabled())
2183		return -ENODEV;
2184
2185	/* Make sure the system still exists */
2186	mutex_lock(&event_mutex);
2187	mutex_lock(&trace_types_lock);
2188	list_for_each_entry(iter_tr, &ftrace_trace_arrays, list) {
2189		list_for_each_entry(iter_dir, &iter_tr->systems, list) {
2190			if (iter_dir == inode->i_private) {
2191				/* Don't open systems with no events */
2192				tr = iter_tr;
2193				dir = iter_dir;
2194				if (dir->nr_events) {
2195					__get_system_dir(dir);
2196					system = dir->subsystem;
2197				}
2198				goto exit_loop;
2199			}
2200		}
2201	}
2202 exit_loop:
2203	mutex_unlock(&trace_types_lock);
2204	mutex_unlock(&event_mutex);
2205
2206	if (!system)
2207		return -ENODEV;
2208
2209	/* Still need to increment the ref count of the system */
2210	if (trace_array_get(tr) < 0) {
2211		put_system(dir);
2212		return -ENODEV;
2213	}
2214
2215	ret = tracing_open_generic(inode, filp);
2216	if (ret < 0) {
2217		trace_array_put(tr);
2218		put_system(dir);
2219	}
2220
2221	return ret;
2222}
2223
2224static int system_tr_open(struct inode *inode, struct file *filp)
2225{
2226	struct trace_subsystem_dir *dir;
2227	struct trace_array *tr = inode->i_private;
2228	int ret;
2229
2230	/* Make a temporary dir that has no system but points to tr */
2231	dir = kzalloc(sizeof(*dir), GFP_KERNEL);
2232	if (!dir)
2233		return -ENOMEM;
2234
2235	ret = tracing_open_generic_tr(inode, filp);
2236	if (ret < 0) {
2237		kfree(dir);
2238		return ret;
2239	}
2240	dir->tr = tr;
2241	filp->private_data = dir;
2242
2243	return 0;
2244}
2245
2246static int subsystem_release(struct inode *inode, struct file *file)
2247{
2248	struct trace_subsystem_dir *dir = file->private_data;
2249
2250	trace_array_put(dir->tr);
2251
2252	/*
2253	 * If dir->subsystem is NULL, then this is a temporary
2254	 * descriptor that was made for a trace_array to enable
2255	 * all subsystems.
2256	 */
2257	if (dir->subsystem)
2258		put_system(dir);
2259	else
2260		kfree(dir);
2261
2262	return 0;
2263}
2264
2265static ssize_t
2266subsystem_filter_read(struct file *filp, char __user *ubuf, size_t cnt,
2267		      loff_t *ppos)
2268{
2269	struct trace_subsystem_dir *dir = filp->private_data;
2270	struct event_subsystem *system = dir->subsystem;
2271	struct trace_seq *s;
2272	int r;
2273
2274	if (*ppos)
2275		return 0;
2276
2277	s = kmalloc(sizeof(*s), GFP_KERNEL);
2278	if (!s)
2279		return -ENOMEM;
2280
2281	trace_seq_init(s);
2282
2283	print_subsystem_event_filter(system, s);
2284	r = simple_read_from_buffer(ubuf, cnt, ppos,
2285				    s->buffer, trace_seq_used(s));
2286
2287	kfree(s);
2288
2289	return r;
2290}
2291
2292static ssize_t
2293subsystem_filter_write(struct file *filp, const char __user *ubuf, size_t cnt,
2294		       loff_t *ppos)
2295{
2296	struct trace_subsystem_dir *dir = filp->private_data;
2297	char *buf;
2298	int err;
2299
2300	if (cnt >= PAGE_SIZE)
2301		return -EINVAL;
2302
2303	buf = memdup_user_nul(ubuf, cnt);
2304	if (IS_ERR(buf))
2305		return PTR_ERR(buf);
2306
2307	err = apply_subsystem_event_filter(dir, buf);
2308	kfree(buf);
2309	if (err < 0)
2310		return err;
2311
2312	*ppos += cnt;
2313
2314	return cnt;
2315}
2316
2317static ssize_t
2318show_header_page_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2319{
2320	struct trace_array *tr = filp->private_data;
2321	struct trace_seq *s;
2322	int r;
2323
2324	if (*ppos)
2325		return 0;
2326
2327	s = kmalloc(sizeof(*s), GFP_KERNEL);
2328	if (!s)
2329		return -ENOMEM;
2330
2331	trace_seq_init(s);
2332
2333	ring_buffer_print_page_header(tr->array_buffer.buffer, s);
2334	r = simple_read_from_buffer(ubuf, cnt, ppos,
2335				    s->buffer, trace_seq_used(s));
2336
2337	kfree(s);
2338
2339	return r;
2340}
2341
2342static ssize_t
2343show_header_event_file(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)
2344{
2345	struct trace_seq *s;
2346	int r;
2347
2348	if (*ppos)
2349		return 0;
2350
2351	s = kmalloc(sizeof(*s), GFP_KERNEL);
2352	if (!s)
2353		return -ENOMEM;
2354
2355	trace_seq_init(s);
2356
2357	ring_buffer_print_entry_header(s);
2358	r = simple_read_from_buffer(ubuf, cnt, ppos,
2359				    s->buffer, trace_seq_used(s));
2360
2361	kfree(s);
2362
2363	return r;
2364}
2365
2366static void ignore_task_cpu(void *data)
2367{
2368	struct trace_array *tr = data;
2369	struct trace_pid_list *pid_list;
2370	struct trace_pid_list *no_pid_list;
2371
2372	/*
2373	 * This function is called by on_each_cpu() while the
2374	 * event_mutex is held.
2375	 */
2376	pid_list = rcu_dereference_protected(tr->filtered_pids,
2377					     mutex_is_locked(&event_mutex));
2378	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
2379					     mutex_is_locked(&event_mutex));
2380
2381	this_cpu_write(tr->array_buffer.data->ignore_pid,
2382		       trace_ignore_this_task(pid_list, no_pid_list, current));
2383}
2384
2385static void register_pid_events(struct trace_array *tr)
2386{
2387	/*
2388	 * Register a probe that is called before all other probes
2389	 * to set ignore_pid if next or prev do not match.
2390	 * Register a probe this is called after all other probes
2391	 * to only keep ignore_pid set if next pid matches.
2392	 */
2393	register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_pre,
2394					 tr, INT_MAX);
2395	register_trace_prio_sched_switch(event_filter_pid_sched_switch_probe_post,
2396					 tr, 0);
2397
2398	register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_pre,
2399					 tr, INT_MAX);
2400	register_trace_prio_sched_wakeup(event_filter_pid_sched_wakeup_probe_post,
2401					 tr, 0);
2402
2403	register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_pre,
2404					     tr, INT_MAX);
2405	register_trace_prio_sched_wakeup_new(event_filter_pid_sched_wakeup_probe_post,
2406					     tr, 0);
2407
2408	register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_pre,
2409					 tr, INT_MAX);
2410	register_trace_prio_sched_waking(event_filter_pid_sched_wakeup_probe_post,
2411					 tr, 0);
2412}
2413
2414static ssize_t
2415event_pid_write(struct file *filp, const char __user *ubuf,
2416		size_t cnt, loff_t *ppos, int type)
2417{
2418	struct seq_file *m = filp->private_data;
2419	struct trace_array *tr = m->private;
2420	struct trace_pid_list *filtered_pids = NULL;
2421	struct trace_pid_list *other_pids = NULL;
2422	struct trace_pid_list *pid_list;
2423	struct trace_event_file *file;
2424	ssize_t ret;
2425
2426	if (!cnt)
2427		return 0;
2428
2429	ret = tracing_update_buffers(tr);
2430	if (ret < 0)
2431		return ret;
2432
2433	guard(mutex)(&event_mutex);
2434
2435	if (type == TRACE_PIDS) {
2436		filtered_pids = rcu_dereference_protected(tr->filtered_pids,
2437							  lockdep_is_held(&event_mutex));
2438		other_pids = rcu_dereference_protected(tr->filtered_no_pids,
2439							  lockdep_is_held(&event_mutex));
2440	} else {
2441		filtered_pids = rcu_dereference_protected(tr->filtered_no_pids,
2442							  lockdep_is_held(&event_mutex));
2443		other_pids = rcu_dereference_protected(tr->filtered_pids,
2444							  lockdep_is_held(&event_mutex));
2445	}
2446
2447	ret = trace_pid_write(filtered_pids, &pid_list, ubuf, cnt);
2448	if (ret < 0)
2449		return ret;
2450
2451	if (type == TRACE_PIDS)
2452		rcu_assign_pointer(tr->filtered_pids, pid_list);
2453	else
2454		rcu_assign_pointer(tr->filtered_no_pids, pid_list);
2455
2456	list_for_each_entry(file, &tr->events, list) {
2457		set_bit(EVENT_FILE_FL_PID_FILTER_BIT, &file->flags);
2458	}
2459
2460	if (filtered_pids) {
2461		tracepoint_synchronize_unregister();
2462		trace_pid_list_free(filtered_pids);
2463	} else if (pid_list && !other_pids) {
2464		register_pid_events(tr);
2465	}
2466
2467	/*
2468	 * Ignoring of pids is done at task switch. But we have to
2469	 * check for those tasks that are currently running.
2470	 * Always do this in case a pid was appended or removed.
2471	 */
2472	on_each_cpu(ignore_task_cpu, tr, 1);
2473
2474	*ppos += ret;
2475
2476	return ret;
2477}
2478
2479static ssize_t
2480ftrace_event_pid_write(struct file *filp, const char __user *ubuf,
2481		       size_t cnt, loff_t *ppos)
2482{
2483	return event_pid_write(filp, ubuf, cnt, ppos, TRACE_PIDS);
2484}
2485
2486static ssize_t
2487ftrace_event_npid_write(struct file *filp, const char __user *ubuf,
2488			size_t cnt, loff_t *ppos)
2489{
2490	return event_pid_write(filp, ubuf, cnt, ppos, TRACE_NO_PIDS);
2491}
2492
2493static int ftrace_event_avail_open(struct inode *inode, struct file *file);
2494static int ftrace_event_set_open(struct inode *inode, struct file *file);
2495static int ftrace_event_set_pid_open(struct inode *inode, struct file *file);
2496static int ftrace_event_set_npid_open(struct inode *inode, struct file *file);
2497static int ftrace_event_release(struct inode *inode, struct file *file);
2498
2499static const struct seq_operations show_event_seq_ops = {
2500	.start = t_start,
2501	.next = t_next,
2502	.show = t_show,
2503	.stop = t_stop,
2504};
2505
2506static const struct seq_operations show_set_event_seq_ops = {
2507	.start = s_start,
2508	.next = s_next,
2509	.show = s_show,
2510	.stop = s_stop,
2511};
2512
2513static const struct seq_operations show_set_pid_seq_ops = {
2514	.start = p_start,
2515	.next = p_next,
2516	.show = trace_pid_show,
2517	.stop = p_stop,
2518};
2519
2520static const struct seq_operations show_set_no_pid_seq_ops = {
2521	.start = np_start,
2522	.next = np_next,
2523	.show = trace_pid_show,
2524	.stop = p_stop,
2525};
2526
2527static const struct file_operations ftrace_avail_fops = {
2528	.open = ftrace_event_avail_open,
2529	.read = seq_read,
2530	.llseek = seq_lseek,
2531	.release = seq_release,
2532};
2533
2534static const struct file_operations ftrace_set_event_fops = {
2535	.open = ftrace_event_set_open,
2536	.read = seq_read,
2537	.write = ftrace_event_write,
2538	.llseek = seq_lseek,
2539	.release = ftrace_event_release,
2540};
2541
2542static const struct file_operations ftrace_set_event_pid_fops = {
2543	.open = ftrace_event_set_pid_open,
2544	.read = seq_read,
2545	.write = ftrace_event_pid_write,
2546	.llseek = seq_lseek,
2547	.release = ftrace_event_release,
2548};
2549
2550static const struct file_operations ftrace_set_event_notrace_pid_fops = {
2551	.open = ftrace_event_set_npid_open,
2552	.read = seq_read,
2553	.write = ftrace_event_npid_write,
2554	.llseek = seq_lseek,
2555	.release = ftrace_event_release,
2556};
2557
2558static const struct file_operations ftrace_enable_fops = {
2559	.open = tracing_open_file_tr,
2560	.read = event_enable_read,
2561	.write = event_enable_write,
2562	.release = tracing_release_file_tr,
2563	.llseek = default_llseek,
2564};
2565
2566static const struct file_operations ftrace_event_format_fops = {
2567	.open = trace_format_open,
2568	.read = seq_read,
2569	.llseek = seq_lseek,
2570	.release = seq_release,
2571};
2572
2573#ifdef CONFIG_PERF_EVENTS
2574static const struct file_operations ftrace_event_id_fops = {
2575	.read = event_id_read,
2576	.llseek = default_llseek,
2577};
2578#endif
2579
2580static const struct file_operations ftrace_event_filter_fops = {
2581	.open = tracing_open_file_tr,
2582	.read = event_filter_read,
2583	.write = event_filter_write,
2584	.release = tracing_release_file_tr,
2585	.llseek = default_llseek,
2586};
2587
2588static const struct file_operations ftrace_subsystem_filter_fops = {
2589	.open = subsystem_open,
2590	.read = subsystem_filter_read,
2591	.write = subsystem_filter_write,
2592	.llseek = default_llseek,
2593	.release = subsystem_release,
2594};
2595
2596static const struct file_operations ftrace_system_enable_fops = {
2597	.open = subsystem_open,
2598	.read = system_enable_read,
2599	.write = system_enable_write,
2600	.llseek = default_llseek,
2601	.release = subsystem_release,
2602};
2603
2604static const struct file_operations ftrace_tr_enable_fops = {
2605	.open = system_tr_open,
2606	.read = system_enable_read,
2607	.write = system_enable_write,
2608	.llseek = default_llseek,
2609	.release = subsystem_release,
2610};
2611
2612static const struct file_operations ftrace_show_header_page_fops = {
2613	.open = tracing_open_generic_tr,
2614	.read = show_header_page_file,
2615	.llseek = default_llseek,
2616	.release = tracing_release_generic_tr,
2617};
2618
2619static const struct file_operations ftrace_show_header_event_fops = {
2620	.open = tracing_open_generic_tr,
2621	.read = show_header_event_file,
2622	.llseek = default_llseek,
2623	.release = tracing_release_generic_tr,
2624};
2625
2626static int
2627ftrace_event_open(struct inode *inode, struct file *file,
2628		  const struct seq_operations *seq_ops)
2629{
2630	struct seq_file *m;
2631	int ret;
2632
2633	ret = security_locked_down(LOCKDOWN_TRACEFS);
2634	if (ret)
2635		return ret;
2636
2637	ret = seq_open(file, seq_ops);
2638	if (ret < 0)
2639		return ret;
2640	m = file->private_data;
2641	/* copy tr over to seq ops */
2642	m->private = inode->i_private;
2643
2644	return ret;
2645}
2646
2647static int ftrace_event_release(struct inode *inode, struct file *file)
2648{
2649	struct trace_array *tr = inode->i_private;
2650
2651	trace_array_put(tr);
2652
2653	return seq_release(inode, file);
2654}
2655
2656static int
2657ftrace_event_avail_open(struct inode *inode, struct file *file)
2658{
2659	const struct seq_operations *seq_ops = &show_event_seq_ops;
2660
2661	/* Checks for tracefs lockdown */
2662	return ftrace_event_open(inode, file, seq_ops);
2663}
2664
2665static int
2666ftrace_event_set_open(struct inode *inode, struct file *file)
2667{
2668	const struct seq_operations *seq_ops = &show_set_event_seq_ops;
2669	struct trace_array *tr = inode->i_private;
2670	int ret;
2671
2672	ret = tracing_check_open_get_tr(tr);
2673	if (ret)
2674		return ret;
2675
2676	if ((file->f_mode & FMODE_WRITE) &&
2677	    (file->f_flags & O_TRUNC))
2678		ftrace_clear_events(tr);
2679
2680	ret = ftrace_event_open(inode, file, seq_ops);
2681	if (ret < 0)
2682		trace_array_put(tr);
2683	return ret;
2684}
2685
2686static int
2687ftrace_event_set_pid_open(struct inode *inode, struct file *file)
2688{
2689	const struct seq_operations *seq_ops = &show_set_pid_seq_ops;
2690	struct trace_array *tr = inode->i_private;
2691	int ret;
2692
2693	ret = tracing_check_open_get_tr(tr);
2694	if (ret)
2695		return ret;
2696
2697	if ((file->f_mode & FMODE_WRITE) &&
2698	    (file->f_flags & O_TRUNC))
2699		ftrace_clear_event_pids(tr, TRACE_PIDS);
2700
2701	ret = ftrace_event_open(inode, file, seq_ops);
2702	if (ret < 0)
2703		trace_array_put(tr);
2704	return ret;
2705}
2706
2707static int
2708ftrace_event_set_npid_open(struct inode *inode, struct file *file)
2709{
2710	const struct seq_operations *seq_ops = &show_set_no_pid_seq_ops;
2711	struct trace_array *tr = inode->i_private;
2712	int ret;
2713
2714	ret = tracing_check_open_get_tr(tr);
2715	if (ret)
2716		return ret;
2717
2718	if ((file->f_mode & FMODE_WRITE) &&
2719	    (file->f_flags & O_TRUNC))
2720		ftrace_clear_event_pids(tr, TRACE_NO_PIDS);
2721
2722	ret = ftrace_event_open(inode, file, seq_ops);
2723	if (ret < 0)
2724		trace_array_put(tr);
2725	return ret;
2726}
2727
2728static struct event_subsystem *
2729create_new_subsystem(const char *name)
2730{
2731	struct event_subsystem *system;
2732
2733	/* need to create new entry */
2734	system = kmalloc(sizeof(*system), GFP_KERNEL);
2735	if (!system)
2736		return NULL;
2737
2738	system->ref_count = 1;
2739
2740	/* Only allocate if dynamic (kprobes and modules) */
2741	system->name = kstrdup_const(name, GFP_KERNEL);
2742	if (!system->name)
2743		goto out_free;
2744
2745	system->filter = kzalloc(sizeof(struct event_filter), GFP_KERNEL);
2746	if (!system->filter)
2747		goto out_free;
2748
2749	list_add(&system->list, &event_subsystems);
2750
2751	return system;
2752
2753 out_free:
2754	kfree_const(system->name);
2755	kfree(system);
2756	return NULL;
2757}
2758
2759static int system_callback(const char *name, umode_t *mode, void **data,
2760		    const struct file_operations **fops)
2761{
2762	if (strcmp(name, "filter") == 0)
2763		*fops = &ftrace_subsystem_filter_fops;
2764
2765	else if (strcmp(name, "enable") == 0)
2766		*fops = &ftrace_system_enable_fops;
2767
2768	else
2769		return 0;
2770
2771	*mode = TRACE_MODE_WRITE;
2772	return 1;
2773}
2774
2775static struct eventfs_inode *
2776event_subsystem_dir(struct trace_array *tr, const char *name,
2777		    struct trace_event_file *file, struct eventfs_inode *parent)
2778{
2779	struct event_subsystem *system, *iter;
2780	struct trace_subsystem_dir *dir;
2781	struct eventfs_inode *ei;
2782	int nr_entries;
2783	static struct eventfs_entry system_entries[] = {
2784		{
2785			.name		= "filter",
2786			.callback	= system_callback,
2787		},
2788		{
2789			.name		= "enable",
2790			.callback	= system_callback,
2791		}
2792	};
2793
2794	/* First see if we did not already create this dir */
2795	list_for_each_entry(dir, &tr->systems, list) {
2796		system = dir->subsystem;
2797		if (strcmp(system->name, name) == 0) {
2798			dir->nr_events++;
2799			file->system = dir;
2800			return dir->ei;
2801		}
2802	}
2803
2804	/* Now see if the system itself exists. */
2805	system = NULL;
2806	list_for_each_entry(iter, &event_subsystems, list) {
2807		if (strcmp(iter->name, name) == 0) {
2808			system = iter;
2809			break;
2810		}
2811	}
2812
2813	dir = kmalloc(sizeof(*dir), GFP_KERNEL);
2814	if (!dir)
2815		goto out_fail;
2816
2817	if (!system) {
2818		system = create_new_subsystem(name);
2819		if (!system)
2820			goto out_free;
2821	} else
2822		__get_system(system);
2823
2824	/* ftrace only has directories no files */
2825	if (strcmp(name, "ftrace") == 0)
2826		nr_entries = 0;
2827	else
2828		nr_entries = ARRAY_SIZE(system_entries);
2829
2830	ei = eventfs_create_dir(name, parent, system_entries, nr_entries, dir);
2831	if (IS_ERR(ei)) {
2832		pr_warn("Failed to create system directory %s\n", name);
2833		__put_system(system);
2834		goto out_free;
2835	}
2836
2837	dir->ei = ei;
2838	dir->tr = tr;
2839	dir->ref_count = 1;
2840	dir->nr_events = 1;
2841	dir->subsystem = system;
2842	file->system = dir;
2843
2844	list_add(&dir->list, &tr->systems);
2845
2846	return dir->ei;
2847
2848 out_free:
2849	kfree(dir);
2850 out_fail:
2851	/* Only print this message if failed on memory allocation */
2852	if (!dir || !system)
2853		pr_warn("No memory to create event subsystem %s\n", name);
2854	return NULL;
2855}
2856
2857static int
2858event_define_fields(struct trace_event_call *call)
2859{
2860	struct list_head *head;
2861	int ret = 0;
2862
2863	/*
2864	 * Other events may have the same class. Only update
2865	 * the fields if they are not already defined.
2866	 */
2867	head = trace_get_fields(call);
2868	if (list_empty(head)) {
2869		struct trace_event_fields *field = call->class->fields_array;
2870		unsigned int offset = sizeof(struct trace_entry);
2871
2872		for (; field->type; field++) {
2873			if (field->type == TRACE_FUNCTION_TYPE) {
2874				field->define_fields(call);
2875				break;
2876			}
2877
2878			offset = ALIGN(offset, field->align);
2879			ret = trace_define_field_ext(call, field->type, field->name,
2880						 offset, field->size,
2881						 field->is_signed, field->filter_type,
2882						 field->len, field->needs_test);
2883			if (WARN_ON_ONCE(ret)) {
2884				pr_err("error code is %d\n", ret);
2885				break;
2886			}
2887
2888			offset += field->size;
2889		}
2890	}
2891
2892	return ret;
2893}
2894
2895static int event_callback(const char *name, umode_t *mode, void **data,
2896			  const struct file_operations **fops)
2897{
2898	struct trace_event_file *file = *data;
2899	struct trace_event_call *call = file->event_call;
2900
2901	if (strcmp(name, "format") == 0) {
2902		*mode = TRACE_MODE_READ;
2903		*fops = &ftrace_event_format_fops;
2904		return 1;
2905	}
2906
2907	/*
2908	 * Only event directories that can be enabled should have
2909	 * triggers or filters, with the exception of the "print"
2910	 * event that can have a "trigger" file.
2911	 */
2912	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) {
2913		if (call->class->reg && strcmp(name, "enable") == 0) {
2914			*mode = TRACE_MODE_WRITE;
2915			*fops = &ftrace_enable_fops;
2916			return 1;
2917		}
2918
2919		if (strcmp(name, "filter") == 0) {
2920			*mode = TRACE_MODE_WRITE;
2921			*fops = &ftrace_event_filter_fops;
2922			return 1;
2923		}
2924	}
2925
2926	if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE) ||
2927	    strcmp(trace_event_name(call), "print") == 0) {
2928		if (strcmp(name, "trigger") == 0) {
2929			*mode = TRACE_MODE_WRITE;
2930			*fops = &event_trigger_fops;
2931			return 1;
2932		}
2933	}
2934
2935#ifdef CONFIG_PERF_EVENTS
2936	if (call->event.type && call->class->reg &&
2937	    strcmp(name, "id") == 0) {
2938		*mode = TRACE_MODE_READ;
2939		*data = (void *)(long)call->event.type;
2940		*fops = &ftrace_event_id_fops;
2941		return 1;
2942	}
2943#endif
2944
2945#ifdef CONFIG_HIST_TRIGGERS
2946	if (strcmp(name, "hist") == 0) {
2947		*mode = TRACE_MODE_READ;
2948		*fops = &event_hist_fops;
2949		return 1;
2950	}
2951#endif
2952#ifdef CONFIG_HIST_TRIGGERS_DEBUG
2953	if (strcmp(name, "hist_debug") == 0) {
2954		*mode = TRACE_MODE_READ;
2955		*fops = &event_hist_debug_fops;
2956		return 1;
2957	}
2958#endif
2959#ifdef CONFIG_TRACE_EVENT_INJECT
2960	if (call->event.type && call->class->reg &&
2961	    strcmp(name, "inject") == 0) {
2962		*mode = 0200;
2963		*fops = &event_inject_fops;
2964		return 1;
2965	}
2966#endif
2967	return 0;
2968}
2969
2970/* The file is incremented on creation and freeing the enable file decrements it */
2971static void event_release(const char *name, void *data)
2972{
2973	struct trace_event_file *file = data;
2974
2975	event_file_put(file);
2976}
2977
2978static int
2979event_create_dir(struct eventfs_inode *parent, struct trace_event_file *file)
2980{
2981	struct trace_event_call *call = file->event_call;
2982	struct trace_array *tr = file->tr;
2983	struct eventfs_inode *e_events;
2984	struct eventfs_inode *ei;
2985	const char *name;
2986	int nr_entries;
2987	int ret;
2988	static struct eventfs_entry event_entries[] = {
2989		{
2990			.name		= "enable",
2991			.callback	= event_callback,
2992			.release	= event_release,
2993		},
2994		{
2995			.name		= "filter",
2996			.callback	= event_callback,
2997		},
2998		{
2999			.name		= "trigger",
3000			.callback	= event_callback,
3001		},
3002		{
3003			.name		= "format",
3004			.callback	= event_callback,
3005		},
3006#ifdef CONFIG_PERF_EVENTS
3007		{
3008			.name		= "id",
3009			.callback	= event_callback,
3010		},
3011#endif
3012#ifdef CONFIG_HIST_TRIGGERS
3013		{
3014			.name		= "hist",
3015			.callback	= event_callback,
3016		},
3017#endif
3018#ifdef CONFIG_HIST_TRIGGERS_DEBUG
3019		{
3020			.name		= "hist_debug",
3021			.callback	= event_callback,
3022		},
3023#endif
3024#ifdef CONFIG_TRACE_EVENT_INJECT
3025		{
3026			.name		= "inject",
3027			.callback	= event_callback,
3028		},
3029#endif
3030	};
3031
3032	/*
3033	 * If the trace point header did not define TRACE_SYSTEM
3034	 * then the system would be called "TRACE_SYSTEM". This should
3035	 * never happen.
3036	 */
3037	if (WARN_ON_ONCE(strcmp(call->class->system, TRACE_SYSTEM) == 0))
3038		return -ENODEV;
3039
3040	e_events = event_subsystem_dir(tr, call->class->system, file, parent);
3041	if (!e_events)
3042		return -ENOMEM;
3043
3044	nr_entries = ARRAY_SIZE(event_entries);
3045
3046	name = trace_event_name(call);
3047	ei = eventfs_create_dir(name, e_events, event_entries, nr_entries, file);
3048	if (IS_ERR(ei)) {
3049		pr_warn("Could not create tracefs '%s' directory\n", name);
3050		return -1;
3051	}
3052
3053	file->ei = ei;
3054
3055	ret = event_define_fields(call);
3056	if (ret < 0) {
3057		pr_warn("Could not initialize trace point events/%s\n", name);
3058		return ret;
3059	}
3060
3061	/* Gets decremented on freeing of the "enable" file */
3062	event_file_get(file);
3063
3064	return 0;
3065}
3066
3067static void remove_event_from_tracers(struct trace_event_call *call)
3068{
3069	struct trace_event_file *file;
3070	struct trace_array *tr;
3071
3072	do_for_each_event_file_safe(tr, file) {
3073		if (file->event_call != call)
3074			continue;
3075
3076		remove_event_file_dir(file);
3077		/*
3078		 * The do_for_each_event_file_safe() is
3079		 * a double loop. After finding the call for this
3080		 * trace_array, we use break to jump to the next
3081		 * trace_array.
3082		 */
3083		break;
3084	} while_for_each_event_file();
3085}
3086
3087static void event_remove(struct trace_event_call *call)
3088{
3089	struct trace_array *tr;
3090	struct trace_event_file *file;
3091
3092	do_for_each_event_file(tr, file) {
3093		if (file->event_call != call)
3094			continue;
3095
3096		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3097			tr->clear_trace = true;
3098
3099		ftrace_event_enable_disable(file, 0);
3100		/*
3101		 * The do_for_each_event_file() is
3102		 * a double loop. After finding the call for this
3103		 * trace_array, we use break to jump to the next
3104		 * trace_array.
3105		 */
3106		break;
3107	} while_for_each_event_file();
3108
3109	if (call->event.funcs)
3110		__unregister_trace_event(&call->event);
3111	remove_event_from_tracers(call);
3112	list_del(&call->list);
3113}
3114
3115static int event_init(struct trace_event_call *call)
3116{
3117	int ret = 0;
3118	const char *name;
3119
3120	name = trace_event_name(call);
3121	if (WARN_ON(!name))
3122		return -EINVAL;
3123
3124	if (call->class->raw_init) {
3125		ret = call->class->raw_init(call);
3126		if (ret < 0 && ret != -ENOSYS)
3127			pr_warn("Could not initialize trace events/%s\n", name);
3128	}
3129
3130	return ret;
3131}
3132
3133static int
3134__register_event(struct trace_event_call *call, struct module *mod)
3135{
3136	int ret;
3137
3138	ret = event_init(call);
3139	if (ret < 0)
3140		return ret;
3141
3142	down_write(&trace_event_sem);
3143	list_add(&call->list, &ftrace_events);
3144	up_write(&trace_event_sem);
3145
3146	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
3147		atomic_set(&call->refcnt, 0);
3148	else
3149		call->module = mod;
3150
3151	return 0;
3152}
3153
3154static char *eval_replace(char *ptr, struct trace_eval_map *map, int len)
3155{
3156	int rlen;
3157	int elen;
3158
3159	/* Find the length of the eval value as a string */
3160	elen = snprintf(ptr, 0, "%ld", map->eval_value);
3161	/* Make sure there's enough room to replace the string with the value */
3162	if (len < elen)
3163		return NULL;
3164
3165	snprintf(ptr, elen + 1, "%ld", map->eval_value);
3166
3167	/* Get the rest of the string of ptr */
3168	rlen = strlen(ptr + len);
3169	memmove(ptr + elen, ptr + len, rlen);
3170	/* Make sure we end the new string */
3171	ptr[elen + rlen] = 0;
3172
3173	return ptr + elen;
3174}
3175
3176static void update_event_printk(struct trace_event_call *call,
3177				struct trace_eval_map *map)
3178{
3179	char *ptr;
3180	int quote = 0;
3181	int len = strlen(map->eval_string);
3182
3183	for (ptr = call->print_fmt; *ptr; ptr++) {
3184		if (*ptr == '\\') {
3185			ptr++;
3186			/* paranoid */
3187			if (!*ptr)
3188				break;
3189			continue;
3190		}
3191		if (*ptr == '"') {
3192			quote ^= 1;
3193			continue;
3194		}
3195		if (quote)
3196			continue;
3197		if (isdigit(*ptr)) {
3198			/* skip numbers */
3199			do {
3200				ptr++;
3201				/* Check for alpha chars like ULL */
3202			} while (isalnum(*ptr));
3203			if (!*ptr)
3204				break;
3205			/*
3206			 * A number must have some kind of delimiter after
3207			 * it, and we can ignore that too.
3208			 */
3209			continue;
3210		}
3211		if (isalpha(*ptr) || *ptr == '_') {
3212			if (strncmp(map->eval_string, ptr, len) == 0 &&
3213			    !isalnum(ptr[len]) && ptr[len] != '_') {
3214				ptr = eval_replace(ptr, map, len);
3215				/* enum/sizeof string smaller than value */
3216				if (WARN_ON_ONCE(!ptr))
3217					return;
3218				/*
3219				 * No need to decrement here, as eval_replace()
3220				 * returns the pointer to the character passed
3221				 * the eval, and two evals can not be placed
3222				 * back to back without something in between.
3223				 * We can skip that something in between.
3224				 */
3225				continue;
3226			}
3227		skip_more:
3228			do {
3229				ptr++;
3230			} while (isalnum(*ptr) || *ptr == '_');
3231			if (!*ptr)
3232				break;
3233			/*
3234			 * If what comes after this variable is a '.' or
3235			 * '->' then we can continue to ignore that string.
3236			 */
3237			if (*ptr == '.' || (ptr[0] == '-' && ptr[1] == '>')) {
3238				ptr += *ptr == '.' ? 1 : 2;
3239				if (!*ptr)
3240					break;
3241				goto skip_more;
3242			}
3243			/*
3244			 * Once again, we can skip the delimiter that came
3245			 * after the string.
3246			 */
3247			continue;
3248		}
3249	}
3250}
3251
3252static void add_str_to_module(struct module *module, char *str)
3253{
3254	struct module_string *modstr;
3255
3256	modstr = kmalloc(sizeof(*modstr), GFP_KERNEL);
3257
3258	/*
3259	 * If we failed to allocate memory here, then we'll just
3260	 * let the str memory leak when the module is removed.
3261	 * If this fails to allocate, there's worse problems than
3262	 * a leaked string on module removal.
3263	 */
3264	if (WARN_ON_ONCE(!modstr))
3265		return;
3266
3267	modstr->module = module;
3268	modstr->str = str;
3269
3270	list_add(&modstr->next, &module_strings);
3271}
3272
3273#define ATTRIBUTE_STR "__attribute__("
3274#define ATTRIBUTE_STR_LEN (sizeof(ATTRIBUTE_STR) - 1)
3275
3276/* Remove all __attribute__() from @type. Return allocated string or @type. */
3277static char *sanitize_field_type(const char *type)
3278{
3279	char *attr, *tmp, *next, *ret = (char *)type;
3280	int depth;
3281
3282	next = (char *)type;
3283	while ((attr = strstr(next, ATTRIBUTE_STR))) {
3284		/* Retry if "__attribute__(" is a part of another word. */
3285		if (attr != next && !isspace(attr[-1])) {
3286			next = attr + ATTRIBUTE_STR_LEN;
3287			continue;
3288		}
3289
3290		if (ret == type) {
3291			ret = kstrdup(type, GFP_KERNEL);
3292			if (WARN_ON_ONCE(!ret))
3293				return NULL;
3294			attr = ret + (attr - type);
3295		}
3296
3297		/* the ATTRIBUTE_STR already has the first '(' */
3298		depth = 1;
3299		next = attr + ATTRIBUTE_STR_LEN;
3300		do {
3301			tmp = strpbrk(next, "()");
3302			/* There is unbalanced parentheses */
3303			if (WARN_ON_ONCE(!tmp)) {
3304				kfree(ret);
3305				return (char *)type;
3306			}
3307
3308			if (*tmp == '(')
3309				depth++;
3310			else
3311				depth--;
3312			next = tmp + 1;
3313		} while (depth > 0);
3314		next = skip_spaces(next);
3315		strcpy(attr, next);
3316		next = attr;
3317	}
3318	return ret;
3319}
3320
3321static char *find_replacable_eval(const char *type, const char *eval_string,
3322				  int len)
3323{
3324	char *ptr;
3325
3326	if (!eval_string)
3327		return NULL;
3328
3329	ptr = strchr(type, '[');
3330	if (!ptr)
3331		return NULL;
3332	ptr++;
3333
3334	if (!isalpha(*ptr) && *ptr != '_')
3335		return NULL;
3336
3337	if (strncmp(eval_string, ptr, len) != 0)
3338		return NULL;
3339
3340	return ptr;
3341}
3342
3343static void update_event_fields(struct trace_event_call *call,
3344				struct trace_eval_map *map)
3345{
3346	struct ftrace_event_field *field;
3347	const char *eval_string = NULL;
3348	struct list_head *head;
3349	int len = 0;
3350	char *ptr;
3351	char *str;
3352
3353	/* Dynamic events should never have field maps */
3354	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
3355		return;
3356
3357	if (map) {
3358		eval_string = map->eval_string;
3359		len = strlen(map->eval_string);
3360	}
3361
3362	head = trace_get_fields(call);
3363	list_for_each_entry(field, head, link) {
3364		str = sanitize_field_type(field->type);
3365		if (!str)
3366			return;
3367
3368		ptr = find_replacable_eval(str, eval_string, len);
3369		if (ptr) {
3370			if (str == field->type) {
3371				str = kstrdup(field->type, GFP_KERNEL);
3372				if (WARN_ON_ONCE(!str))
3373					return;
3374				ptr = str + (ptr - field->type);
3375			}
3376
3377			ptr = eval_replace(ptr, map, len);
3378			/* enum/sizeof string smaller than value */
3379			if (WARN_ON_ONCE(!ptr)) {
3380				kfree(str);
3381				continue;
3382			}
3383		}
3384
3385		if (str == field->type)
3386			continue;
3387		/*
3388		 * If the event is part of a module, then we need to free the string
3389		 * when the module is removed. Otherwise, it will stay allocated
3390		 * until a reboot.
3391		 */
3392		if (call->module)
3393			add_str_to_module(call->module, str);
3394
3395		field->type = str;
3396		if (field->filter_type == FILTER_OTHER)
3397			field->filter_type = filter_assign_type(field->type);
3398	}
3399}
3400
3401/* Update all events for replacing eval and sanitizing */
3402void trace_event_update_all(struct trace_eval_map **map, int len)
3403{
3404	struct trace_event_call *call, *p;
3405	const char *last_system = NULL;
3406	bool first = false;
3407	bool updated;
3408	int last_i;
3409	int i;
3410
3411	down_write(&trace_event_sem);
3412	list_for_each_entry_safe(call, p, &ftrace_events, list) {
3413		/* events are usually grouped together with systems */
3414		if (!last_system || call->class->system != last_system) {
3415			first = true;
3416			last_i = 0;
3417			last_system = call->class->system;
3418		}
3419
3420		updated = false;
3421		/*
3422		 * Since calls are grouped by systems, the likelihood that the
3423		 * next call in the iteration belongs to the same system as the
3424		 * previous call is high. As an optimization, we skip searching
3425		 * for a map[] that matches the call's system if the last call
3426		 * was from the same system. That's what last_i is for. If the
3427		 * call has the same system as the previous call, then last_i
3428		 * will be the index of the first map[] that has a matching
3429		 * system.
3430		 */
3431		for (i = last_i; i < len; i++) {
3432			if (call->class->system == map[i]->system) {
3433				/* Save the first system if need be */
3434				if (first) {
3435					last_i = i;
3436					first = false;
3437				}
3438				update_event_printk(call, map[i]);
3439				update_event_fields(call, map[i]);
3440				updated = true;
3441			}
3442		}
3443		/* If not updated yet, update field for sanitizing. */
3444		if (!updated)
3445			update_event_fields(call, NULL);
3446		cond_resched();
3447	}
3448	up_write(&trace_event_sem);
3449}
3450
3451static bool event_in_systems(struct trace_event_call *call,
3452			     const char *systems)
3453{
3454	const char *system;
3455	const char *p;
3456
3457	if (!systems)
3458		return true;
3459
3460	system = call->class->system;
3461	p = strstr(systems, system);
3462	if (!p)
3463		return false;
3464
3465	if (p != systems && !isspace(*(p - 1)) && *(p - 1) != ',')
3466		return false;
3467
3468	p += strlen(system);
3469	return !*p || isspace(*p) || *p == ',';
3470}
3471
3472#ifdef CONFIG_HIST_TRIGGERS
3473/*
3474 * Wake up waiter on the hist_poll_wq from irq_work because the hist trigger
3475 * may happen in any context.
3476 */
3477static void hist_poll_event_irq_work(struct irq_work *work)
3478{
3479	wake_up_all(&hist_poll_wq);
3480}
3481
3482DEFINE_IRQ_WORK(hist_poll_work, hist_poll_event_irq_work);
3483DECLARE_WAIT_QUEUE_HEAD(hist_poll_wq);
3484#endif
3485
3486static struct trace_event_file *
3487trace_create_new_event(struct trace_event_call *call,
3488		       struct trace_array *tr)
3489{
3490	struct trace_pid_list *no_pid_list;
3491	struct trace_pid_list *pid_list;
3492	struct trace_event_file *file;
3493	unsigned int first;
3494
3495	if (!event_in_systems(call, tr->system_names))
3496		return NULL;
3497
3498	file = kmem_cache_alloc(file_cachep, GFP_TRACE);
3499	if (!file)
3500		return ERR_PTR(-ENOMEM);
3501
3502	pid_list = rcu_dereference_protected(tr->filtered_pids,
3503					     lockdep_is_held(&event_mutex));
3504	no_pid_list = rcu_dereference_protected(tr->filtered_no_pids,
3505					     lockdep_is_held(&event_mutex));
3506
3507	if (!trace_pid_list_first(pid_list, &first) ||
3508	    !trace_pid_list_first(no_pid_list, &first))
3509		file->flags |= EVENT_FILE_FL_PID_FILTER;
3510
3511	file->event_call = call;
3512	file->tr = tr;
3513	atomic_set(&file->sm_ref, 0);
3514	atomic_set(&file->tm_ref, 0);
3515	INIT_LIST_HEAD(&file->triggers);
3516	list_add(&file->list, &tr->events);
3517	refcount_set(&file->ref, 1);
3518
3519	return file;
3520}
3521
3522#define MAX_BOOT_TRIGGERS 32
3523
3524static struct boot_triggers {
3525	const char		*event;
3526	char			*trigger;
3527} bootup_triggers[MAX_BOOT_TRIGGERS];
3528
3529static char bootup_trigger_buf[COMMAND_LINE_SIZE];
3530static int nr_boot_triggers;
3531
3532static __init int setup_trace_triggers(char *str)
3533{
3534	char *trigger;
3535	char *buf;
3536	int i;
3537
3538	strscpy(bootup_trigger_buf, str, COMMAND_LINE_SIZE);
3539	trace_set_ring_buffer_expanded(NULL);
3540	disable_tracing_selftest("running event triggers");
3541
3542	buf = bootup_trigger_buf;
3543	for (i = 0; i < MAX_BOOT_TRIGGERS; i++) {
3544		trigger = strsep(&buf, ",");
3545		if (!trigger)
3546			break;
3547		bootup_triggers[i].event = strsep(&trigger, ".");
3548		bootup_triggers[i].trigger = trigger;
3549		if (!bootup_triggers[i].trigger)
3550			break;
3551	}
3552
3553	nr_boot_triggers = i;
3554	return 1;
3555}
3556__setup("trace_trigger=", setup_trace_triggers);
3557
3558/* Add an event to a trace directory */
3559static int
3560__trace_add_new_event(struct trace_event_call *call, struct trace_array *tr)
3561{
3562	struct trace_event_file *file;
3563
3564	file = trace_create_new_event(call, tr);
3565	/*
3566	 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3567	 * allocation, or NULL if the event is not part of the tr->system_names.
3568	 * When the event is not part of the tr->system_names, return zero, not
3569	 * an error.
3570	 */
3571	if (!file)
3572		return 0;
3573
3574	if (IS_ERR(file))
3575		return PTR_ERR(file);
3576
3577	if (eventdir_initialized)
3578		return event_create_dir(tr->event_dir, file);
3579	else
3580		return event_define_fields(call);
3581}
3582
3583static void trace_early_triggers(struct trace_event_file *file, const char *name)
3584{
3585	int ret;
3586	int i;
3587
3588	for (i = 0; i < nr_boot_triggers; i++) {
3589		if (strcmp(name, bootup_triggers[i].event))
3590			continue;
3591		mutex_lock(&event_mutex);
3592		ret = trigger_process_regex(file, bootup_triggers[i].trigger);
3593		mutex_unlock(&event_mutex);
3594		if (ret)
3595			pr_err("Failed to register trigger '%s' on event %s\n",
3596			       bootup_triggers[i].trigger,
3597			       bootup_triggers[i].event);
3598	}
3599}
3600
3601/*
3602 * Just create a descriptor for early init. A descriptor is required
3603 * for enabling events at boot. We want to enable events before
3604 * the filesystem is initialized.
3605 */
3606static int
3607__trace_early_add_new_event(struct trace_event_call *call,
3608			    struct trace_array *tr)
3609{
3610	struct trace_event_file *file;
3611	int ret;
3612
3613	file = trace_create_new_event(call, tr);
3614	/*
3615	 * trace_create_new_event() returns ERR_PTR(-ENOMEM) if failed
3616	 * allocation, or NULL if the event is not part of the tr->system_names.
3617	 * When the event is not part of the tr->system_names, return zero, not
3618	 * an error.
3619	 */
3620	if (!file)
3621		return 0;
3622
3623	if (IS_ERR(file))
3624		return PTR_ERR(file);
3625
3626	ret = event_define_fields(call);
3627	if (ret)
3628		return ret;
3629
3630	trace_early_triggers(file, trace_event_name(call));
3631
3632	return 0;
3633}
3634
3635struct ftrace_module_file_ops;
3636static void __add_event_to_tracers(struct trace_event_call *call);
3637
3638/* Add an additional event_call dynamically */
3639int trace_add_event_call(struct trace_event_call *call)
3640{
3641	int ret;
3642	lockdep_assert_held(&event_mutex);
3643
3644	guard(mutex)(&trace_types_lock);
3645
3646	ret = __register_event(call, NULL);
3647	if (ret < 0)
3648		return ret;
3649
3650	__add_event_to_tracers(call);
3651	return ret;
3652}
3653EXPORT_SYMBOL_GPL(trace_add_event_call);
3654
3655/*
3656 * Must be called under locking of trace_types_lock, event_mutex and
3657 * trace_event_sem.
3658 */
3659static void __trace_remove_event_call(struct trace_event_call *call)
3660{
3661	event_remove(call);
3662	trace_destroy_fields(call);
3663}
3664
3665static int probe_remove_event_call(struct trace_event_call *call)
3666{
3667	struct trace_array *tr;
3668	struct trace_event_file *file;
3669
3670#ifdef CONFIG_PERF_EVENTS
3671	if (call->perf_refcount)
3672		return -EBUSY;
3673#endif
3674	do_for_each_event_file(tr, file) {
3675		if (file->event_call != call)
3676			continue;
3677		/*
3678		 * We can't rely on ftrace_event_enable_disable(enable => 0)
3679		 * we are going to do, soft mode can suppress
3680		 * TRACE_REG_UNREGISTER.
3681		 */
3682		if (file->flags & EVENT_FILE_FL_ENABLED)
3683			goto busy;
3684
3685		if (file->flags & EVENT_FILE_FL_WAS_ENABLED)
3686			tr->clear_trace = true;
3687		/*
3688		 * The do_for_each_event_file_safe() is
3689		 * a double loop. After finding the call for this
3690		 * trace_array, we use break to jump to the next
3691		 * trace_array.
3692		 */
3693		break;
3694	} while_for_each_event_file();
3695
3696	__trace_remove_event_call(call);
3697
3698	return 0;
3699 busy:
3700	/* No need to clear the trace now */
3701	list_for_each_entry(tr, &ftrace_trace_arrays, list) {
3702		tr->clear_trace = false;
3703	}
3704	return -EBUSY;
3705}
3706
3707/* Remove an event_call */
3708int trace_remove_event_call(struct trace_event_call *call)
3709{
3710	int ret;
3711
3712	lockdep_assert_held(&event_mutex);
3713
3714	mutex_lock(&trace_types_lock);
3715	down_write(&trace_event_sem);
3716	ret = probe_remove_event_call(call);
3717	up_write(&trace_event_sem);
3718	mutex_unlock(&trace_types_lock);
3719
3720	return ret;
3721}
3722EXPORT_SYMBOL_GPL(trace_remove_event_call);
3723
3724#define for_each_event(event, start, end)			\
3725	for (event = start;					\
3726	     (unsigned long)event < (unsigned long)end;		\
3727	     event++)
3728
3729#ifdef CONFIG_MODULES
3730static void update_mod_cache(struct trace_array *tr, struct module *mod)
3731{
3732	struct event_mod_load *event_mod, *n;
3733
3734	list_for_each_entry_safe(event_mod, n, &tr->mod_events, list) {
3735		if (strcmp(event_mod->module, mod->name) != 0)
3736			continue;
3737
3738		__ftrace_set_clr_event_nolock(tr, event_mod->match,
3739					      event_mod->system,
3740					      event_mod->event, 1, mod->name);
3741		free_event_mod(event_mod);
3742	}
3743}
3744
3745static void update_cache_events(struct module *mod)
3746{
3747	struct trace_array *tr;
3748
3749	list_for_each_entry(tr, &ftrace_trace_arrays, list)
3750		update_mod_cache(tr, mod);
3751}
3752
3753static void trace_module_add_events(struct module *mod)
3754{
3755	struct trace_event_call **call, **start, **end;
3756
3757	if (!mod->num_trace_events)
3758		return;
3759
3760	/* Don't add infrastructure for mods without tracepoints */
3761	if (trace_module_has_bad_taint(mod)) {
3762		pr_err("%s: module has bad taint, not creating trace events\n",
3763		       mod->name);
3764		return;
3765	}
3766
3767	start = mod->trace_events;
3768	end = mod->trace_events + mod->num_trace_events;
3769
3770	for_each_event(call, start, end) {
3771		__register_event(*call, mod);
3772		__add_event_to_tracers(*call);
3773	}
3774
3775	update_cache_events(mod);
3776}
3777
3778static void trace_module_remove_events(struct module *mod)
3779{
3780	struct trace_event_call *call, *p;
3781	struct module_string *modstr, *m;
3782
3783	down_write(&trace_event_sem);
3784	list_for_each_entry_safe(call, p, &ftrace_events, list) {
3785		if ((call->flags & TRACE_EVENT_FL_DYNAMIC) || !call->module)
3786			continue;
3787		if (call->module == mod)
3788			__trace_remove_event_call(call);
3789	}
3790	/* Check for any strings allocated for this module */
3791	list_for_each_entry_safe(modstr, m, &module_strings, next) {
3792		if (modstr->module != mod)
3793			continue;
3794		list_del(&modstr->next);
3795		kfree(modstr->str);
3796		kfree(modstr);
3797	}
3798	up_write(&trace_event_sem);
3799
3800	/*
3801	 * It is safest to reset the ring buffer if the module being unloaded
3802	 * registered any events that were used. The only worry is if
3803	 * a new module gets loaded, and takes on the same id as the events
3804	 * of this module. When printing out the buffer, traced events left
3805	 * over from this module may be passed to the new module events and
3806	 * unexpected results may occur.
3807	 */
3808	tracing_reset_all_online_cpus_unlocked();
3809}
3810
3811static int trace_module_notify(struct notifier_block *self,
3812			       unsigned long val, void *data)
3813{
3814	struct module *mod = data;
3815
3816	mutex_lock(&event_mutex);
3817	mutex_lock(&trace_types_lock);
3818	switch (val) {
3819	case MODULE_STATE_COMING:
3820		trace_module_add_events(mod);
3821		break;
3822	case MODULE_STATE_GOING:
3823		trace_module_remove_events(mod);
3824		break;
3825	}
3826	mutex_unlock(&trace_types_lock);
3827	mutex_unlock(&event_mutex);
3828
3829	return NOTIFY_OK;
3830}
3831
3832static struct notifier_block trace_module_nb = {
3833	.notifier_call = trace_module_notify,
3834	.priority = 1, /* higher than trace.c module notify */
3835};
3836#endif /* CONFIG_MODULES */
3837
3838/* Create a new event directory structure for a trace directory. */
3839static void
3840__trace_add_event_dirs(struct trace_array *tr)
3841{
3842	struct trace_event_call *call;
3843	int ret;
3844
3845	lockdep_assert_held(&trace_event_sem);
3846
3847	list_for_each_entry(call, &ftrace_events, list) {
3848		ret = __trace_add_new_event(call, tr);
3849		if (ret < 0)
3850			pr_warn("Could not create directory for event %s\n",
3851				trace_event_name(call));
3852	}
3853}
3854
3855/* Returns any file that matches the system and event */
3856struct trace_event_file *
3857__find_event_file(struct trace_array *tr, const char *system, const char *event)
3858{
3859	struct trace_event_file *file;
3860	struct trace_event_call *call;
3861	const char *name;
3862
3863	list_for_each_entry(file, &tr->events, list) {
3864
3865		call = file->event_call;
3866		name = trace_event_name(call);
3867
3868		if (!name || !call->class)
3869			continue;
3870
3871		if (strcmp(event, name) == 0 &&
3872		    strcmp(system, call->class->system) == 0)
3873			return file;
3874	}
3875	return NULL;
3876}
3877
3878/* Returns valid trace event files that match system and event */
3879struct trace_event_file *
3880find_event_file(struct trace_array *tr, const char *system, const char *event)
3881{
3882	struct trace_event_file *file;
3883
3884	file = __find_event_file(tr, system, event);
3885	if (!file || !file->event_call->class->reg ||
3886	    file->event_call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)
3887		return NULL;
3888
3889	return file;
3890}
3891
3892/**
3893 * trace_get_event_file - Find and return a trace event file
3894 * @instance: The name of the trace instance containing the event
3895 * @system: The name of the system containing the event
3896 * @event: The name of the event
3897 *
3898 * Return a trace event file given the trace instance name, trace
3899 * system, and trace event name.  If the instance name is NULL, it
3900 * refers to the top-level trace array.
3901 *
3902 * This function will look it up and return it if found, after calling
3903 * trace_array_get() to prevent the instance from going away, and
3904 * increment the event's module refcount to prevent it from being
3905 * removed.
3906 *
3907 * To release the file, call trace_put_event_file(), which will call
3908 * trace_array_put() and decrement the event's module refcount.
3909 *
3910 * Return: The trace event on success, ERR_PTR otherwise.
3911 */
3912struct trace_event_file *trace_get_event_file(const char *instance,
3913					      const char *system,
3914					      const char *event)
3915{
3916	struct trace_array *tr = top_trace_array();
3917	struct trace_event_file *file = NULL;
3918	int ret = -EINVAL;
3919
3920	if (instance) {
3921		tr = trace_array_find_get(instance);
3922		if (!tr)
3923			return ERR_PTR(-ENOENT);
3924	} else {
3925		ret = trace_array_get(tr);
3926		if (ret)
3927			return ERR_PTR(ret);
3928	}
3929
3930	guard(mutex)(&event_mutex);
3931
3932	file = find_event_file(tr, system, event);
3933	if (!file) {
3934		trace_array_put(tr);
3935		return ERR_PTR(-EINVAL);
3936	}
3937
3938	/* Don't let event modules unload while in use */
3939	ret = trace_event_try_get_ref(file->event_call);
3940	if (!ret) {
3941		trace_array_put(tr);
3942		return ERR_PTR(-EBUSY);
3943	}
3944
3945	return file;
3946}
3947EXPORT_SYMBOL_GPL(trace_get_event_file);
3948
3949/**
3950 * trace_put_event_file - Release a file from trace_get_event_file()
3951 * @file: The trace event file
3952 *
3953 * If a file was retrieved using trace_get_event_file(), this should
3954 * be called when it's no longer needed.  It will cancel the previous
3955 * trace_array_get() called by that function, and decrement the
3956 * event's module refcount.
3957 */
3958void trace_put_event_file(struct trace_event_file *file)
3959{
3960	mutex_lock(&event_mutex);
3961	trace_event_put_ref(file->event_call);
3962	mutex_unlock(&event_mutex);
3963
3964	trace_array_put(file->tr);
3965}
3966EXPORT_SYMBOL_GPL(trace_put_event_file);
3967
3968#ifdef CONFIG_DYNAMIC_FTRACE
3969struct event_probe_data {
3970	struct trace_event_file	*file;
3971	unsigned long			count;
3972	int				ref;
3973	bool				enable;
3974};
3975
3976static void update_event_probe(struct event_probe_data *data)
3977{
3978	if (data->enable)
3979		clear_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3980	else
3981		set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &data->file->flags);
3982}
3983
3984static void
3985event_enable_probe(unsigned long ip, unsigned long parent_ip,
3986		   struct trace_array *tr, struct ftrace_probe_ops *ops,
3987		   void *data)
3988{
3989	struct ftrace_func_mapper *mapper = data;
3990	struct event_probe_data *edata;
3991	void **pdata;
3992
3993	pdata = ftrace_func_mapper_find_ip(mapper, ip);
3994	if (!pdata || !*pdata)
3995		return;
3996
3997	edata = *pdata;
3998	update_event_probe(edata);
3999}
4000
4001static void
4002event_enable_count_probe(unsigned long ip, unsigned long parent_ip,
4003			 struct trace_array *tr, struct ftrace_probe_ops *ops,
4004			 void *data)
4005{
4006	struct ftrace_func_mapper *mapper = data;
4007	struct event_probe_data *edata;
4008	void **pdata;
4009
4010	pdata = ftrace_func_mapper_find_ip(mapper, ip);
4011	if (!pdata || !*pdata)
4012		return;
4013
4014	edata = *pdata;
4015
4016	if (!edata->count)
4017		return;
4018
4019	/* Skip if the event is in a state we want to switch to */
4020	if (edata->enable == !(edata->file->flags & EVENT_FILE_FL_SOFT_DISABLED))
4021		return;
4022
4023	if (edata->count != -1)
4024		(edata->count)--;
4025
4026	update_event_probe(edata);
4027}
4028
4029static int
4030event_enable_print(struct seq_file *m, unsigned long ip,
4031		   struct ftrace_probe_ops *ops, void *data)
4032{
4033	struct ftrace_func_mapper *mapper = data;
4034	struct event_probe_data *edata;
4035	void **pdata;
4036
4037	pdata = ftrace_func_mapper_find_ip(mapper, ip);
4038
4039	if (WARN_ON_ONCE(!pdata || !*pdata))
4040		return 0;
4041
4042	edata = *pdata;
4043
4044	seq_printf(m, "%ps:", (void *)ip);
4045
4046	seq_printf(m, "%s:%s:%s",
4047		   edata->enable ? ENABLE_EVENT_STR : DISABLE_EVENT_STR,
4048		   edata->file->event_call->class->system,
4049		   trace_event_name(edata->file->event_call));
4050
4051	if (edata->count == -1)
4052		seq_puts(m, ":unlimited\n");
4053	else
4054		seq_printf(m, ":count=%ld\n", edata->count);
4055
4056	return 0;
4057}
4058
4059static int
4060event_enable_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
4061		  unsigned long ip, void *init_data, void **data)
4062{
4063	struct ftrace_func_mapper *mapper = *data;
4064	struct event_probe_data *edata = init_data;
4065	int ret;
4066
4067	if (!mapper) {
4068		mapper = allocate_ftrace_func_mapper();
4069		if (!mapper)
4070			return -ENODEV;
4071		*data = mapper;
4072	}
4073
4074	ret = ftrace_func_mapper_add_ip(mapper, ip, edata);
4075	if (ret < 0)
4076		return ret;
4077
4078	edata->ref++;
4079
4080	return 0;
4081}
4082
4083static int free_probe_data(void *data)
4084{
4085	struct event_probe_data *edata = data;
4086
4087	edata->ref--;
4088	if (!edata->ref) {
4089		/* Remove soft mode */
4090		__ftrace_event_enable_disable(edata->file, 0, 1);
4091		trace_event_put_ref(edata->file->event_call);
4092		kfree(edata);
4093	}
4094	return 0;
4095}
4096
4097static void
4098event_enable_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
4099		  unsigned long ip, void *data)
4100{
4101	struct ftrace_func_mapper *mapper = data;
4102	struct event_probe_data *edata;
4103
4104	if (!ip) {
4105		if (!mapper)
4106			return;
4107		free_ftrace_func_mapper(mapper, free_probe_data);
4108		return;
4109	}
4110
4111	edata = ftrace_func_mapper_remove_ip(mapper, ip);
4112
4113	if (WARN_ON_ONCE(!edata))
4114		return;
4115
4116	if (WARN_ON_ONCE(edata->ref <= 0))
4117		return;
4118
4119	free_probe_data(edata);
4120}
4121
4122static struct ftrace_probe_ops event_enable_probe_ops = {
4123	.func			= event_enable_probe,
4124	.print			= event_enable_print,
4125	.init			= event_enable_init,
4126	.free			= event_enable_free,
4127};
4128
4129static struct ftrace_probe_ops event_enable_count_probe_ops = {
4130	.func			= event_enable_count_probe,
4131	.print			= event_enable_print,
4132	.init			= event_enable_init,
4133	.free			= event_enable_free,
4134};
4135
4136static struct ftrace_probe_ops event_disable_probe_ops = {
4137	.func			= event_enable_probe,
4138	.print			= event_enable_print,
4139	.init			= event_enable_init,
4140	.free			= event_enable_free,
4141};
4142
4143static struct ftrace_probe_ops event_disable_count_probe_ops = {
4144	.func			= event_enable_count_probe,
4145	.print			= event_enable_print,
4146	.init			= event_enable_init,
4147	.free			= event_enable_free,
4148};
4149
4150static int
4151event_enable_func(struct trace_array *tr, struct ftrace_hash *hash,
4152		  char *glob, char *cmd, char *param, int enabled)
4153{
4154	struct trace_event_file *file;
4155	struct ftrace_probe_ops *ops;
4156	struct event_probe_data *data;
4157	unsigned long count = -1;
4158	const char *system;
4159	const char *event;
4160	char *number;
4161	bool enable;
4162	int ret;
4163
4164	if (!tr)
4165		return -ENODEV;
4166
4167	/* hash funcs only work with set_ftrace_filter */
4168	if (!enabled || !param)
4169		return -EINVAL;
4170
4171	system = strsep(&param, ":");
4172	if (!param)
4173		return -EINVAL;
4174
4175	event = strsep(&param, ":");
4176
4177	guard(mutex)(&event_mutex);
4178
4179	file = find_event_file(tr, system, event);
4180	if (!file)
4181		return -EINVAL;
4182
4183	enable = strcmp(cmd, ENABLE_EVENT_STR) == 0;
4184
4185	if (enable)
4186		ops = param ? &event_enable_count_probe_ops : &event_enable_probe_ops;
4187	else
4188		ops = param ? &event_disable_count_probe_ops : &event_disable_probe_ops;
4189
4190	if (glob[0] == '!')
4191		return unregister_ftrace_function_probe_func(glob+1, tr, ops);
4192
4193	if (param) {
4194		number = strsep(&param, ":");
4195
4196		if (!strlen(number))
4197			return -EINVAL;
4198
4199		/*
4200		 * We use the callback data field (which is a pointer)
4201		 * as our counter.
4202		 */
4203		ret = kstrtoul(number, 0, &count);
4204		if (ret)
4205			return ret;
4206	}
4207
4208	/* Don't let event modules unload while probe registered */
4209	ret = trace_event_try_get_ref(file->event_call);
4210	if (!ret)
4211		return -EBUSY;
4212
4213	ret = __ftrace_event_enable_disable(file, 1, 1);
4214	if (ret < 0)
4215		goto out_put;
4216
4217	ret = -ENOMEM;
4218	data = kzalloc(sizeof(*data), GFP_KERNEL);
4219	if (!data)
4220		goto out_put;
4221
4222	data->enable = enable;
4223	data->count = count;
4224	data->file = file;
4225
4226	ret = register_ftrace_function_probe(glob, tr, ops, data);
4227	/*
4228	 * The above returns on success the # of functions enabled,
4229	 * but if it didn't find any functions it returns zero.
4230	 * Consider no functions a failure too.
4231	 */
4232
4233	/* Just return zero, not the number of enabled functions */
4234	if (ret > 0)
4235		return 0;
4236
4237	kfree(data);
4238
4239	if (!ret)
4240		ret = -ENOENT;
4241
4242	__ftrace_event_enable_disable(file, 0, 1);
4243 out_put:
4244	trace_event_put_ref(file->event_call);
4245	return ret;
4246}
4247
4248static struct ftrace_func_command event_enable_cmd = {
4249	.name			= ENABLE_EVENT_STR,
4250	.func			= event_enable_func,
4251};
4252
4253static struct ftrace_func_command event_disable_cmd = {
4254	.name			= DISABLE_EVENT_STR,
4255	.func			= event_enable_func,
4256};
4257
4258static __init int register_event_cmds(void)
4259{
4260	int ret;
4261
4262	ret = register_ftrace_command(&event_enable_cmd);
4263	if (WARN_ON(ret < 0))
4264		return ret;
4265	ret = register_ftrace_command(&event_disable_cmd);
4266	if (WARN_ON(ret < 0))
4267		unregister_ftrace_command(&event_enable_cmd);
4268	return ret;
4269}
4270#else
4271static inline int register_event_cmds(void) { return 0; }
4272#endif /* CONFIG_DYNAMIC_FTRACE */
4273
4274/*
4275 * The top level array and trace arrays created by boot-time tracing
4276 * have already had its trace_event_file descriptors created in order
4277 * to allow for early events to be recorded.
4278 * This function is called after the tracefs has been initialized,
4279 * and we now have to create the files associated to the events.
4280 */
4281static void __trace_early_add_event_dirs(struct trace_array *tr)
4282{
4283	struct trace_event_file *file;
4284	int ret;
4285
4286
4287	list_for_each_entry(file, &tr->events, list) {
4288		ret = event_create_dir(tr->event_dir, file);
4289		if (ret < 0)
4290			pr_warn("Could not create directory for event %s\n",
4291				trace_event_name(file->event_call));
4292	}
4293}
4294
4295/*
4296 * For early boot up, the top trace array and the trace arrays created
4297 * by boot-time tracing require to have a list of events that can be
4298 * enabled. This must be done before the filesystem is set up in order
4299 * to allow events to be traced early.
4300 */
4301void __trace_early_add_events(struct trace_array *tr)
4302{
4303	struct trace_event_call *call;
4304	int ret;
4305
4306	list_for_each_entry(call, &ftrace_events, list) {
4307		/* Early boot up should not have any modules loaded */
4308		if (!(call->flags & TRACE_EVENT_FL_DYNAMIC) &&
4309		    WARN_ON_ONCE(call->module))
4310			continue;
4311
4312		ret = __trace_early_add_new_event(call, tr);
4313		if (ret < 0)
4314			pr_warn("Could not create early event %s\n",
4315				trace_event_name(call));
4316	}
4317}
4318
4319/* Remove the event directory structure for a trace directory. */
4320static void
4321__trace_remove_event_dirs(struct trace_array *tr)
4322{
4323	struct trace_event_file *file, *next;
4324
4325	list_for_each_entry_safe(file, next, &tr->events, list)
4326		remove_event_file_dir(file);
4327}
4328
4329static void __add_event_to_tracers(struct trace_event_call *call)
4330{
4331	struct trace_array *tr;
4332
4333	list_for_each_entry(tr, &ftrace_trace_arrays, list)
4334		__trace_add_new_event(call, tr);
4335}
4336
4337extern struct trace_event_call *__start_ftrace_events[];
4338extern struct trace_event_call *__stop_ftrace_events[];
4339
4340static char bootup_event_buf[COMMAND_LINE_SIZE] __initdata;
4341
4342static __init int setup_trace_event(char *str)
4343{
4344	if (bootup_event_buf[0] != '\0')
4345		strlcat(bootup_event_buf, ",", COMMAND_LINE_SIZE);
4346
4347	strlcat(bootup_event_buf, str, COMMAND_LINE_SIZE);
4348
4349	trace_set_ring_buffer_expanded(NULL);
4350	disable_tracing_selftest("running event tracing");
4351
4352	return 1;
4353}
4354__setup("trace_event=", setup_trace_event);
4355
4356static int events_callback(const char *name, umode_t *mode, void **data,
4357			   const struct file_operations **fops)
4358{
4359	if (strcmp(name, "enable") == 0) {
4360		*mode = TRACE_MODE_WRITE;
4361		*fops = &ftrace_tr_enable_fops;
4362		return 1;
4363	}
4364
4365	if (strcmp(name, "header_page") == 0) {
4366		*mode = TRACE_MODE_READ;
4367		*fops = &ftrace_show_header_page_fops;
4368
4369	} else if (strcmp(name, "header_event") == 0) {
4370		*mode = TRACE_MODE_READ;
4371		*fops = &ftrace_show_header_event_fops;
4372	} else
4373		return 0;
4374
4375	return 1;
4376}
4377
4378/* Expects to have event_mutex held when called */
4379static int
4380create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)
4381{
4382	struct eventfs_inode *e_events;
4383	struct dentry *entry;
4384	int nr_entries;
4385	static struct eventfs_entry events_entries[] = {
4386		{
4387			.name		= "enable",
4388			.callback	= events_callback,
4389		},
4390		{
4391			.name		= "header_page",
4392			.callback	= events_callback,
4393		},
4394		{
4395			.name		= "header_event",
4396			.callback	= events_callback,
4397		},
4398	};
4399
4400	entry = trace_create_file("set_event", TRACE_MODE_WRITE, parent,
4401				  tr, &ftrace_set_event_fops);
4402	if (!entry)
4403		return -ENOMEM;
4404
4405	nr_entries = ARRAY_SIZE(events_entries);
4406
4407	e_events = eventfs_create_events_dir("events", parent, events_entries,
4408					     nr_entries, tr);
4409	if (IS_ERR(e_events)) {
4410		pr_warn("Could not create tracefs 'events' directory\n");
4411		return -ENOMEM;
4412	}
4413
4414	/* There are not as crucial, just warn if they are not created */
4415
4416	trace_create_file("set_event_pid", TRACE_MODE_WRITE, parent,
4417			  tr, &ftrace_set_event_pid_fops);
4418
4419	trace_create_file("set_event_notrace_pid",
4420			  TRACE_MODE_WRITE, parent, tr,
4421			  &ftrace_set_event_notrace_pid_fops);
4422
4423	tr->event_dir = e_events;
4424
4425	return 0;
4426}
4427
4428/**
4429 * event_trace_add_tracer - add a instance of a trace_array to events
4430 * @parent: The parent dentry to place the files/directories for events in
4431 * @tr: The trace array associated with these events
4432 *
4433 * When a new instance is created, it needs to set up its events
4434 * directory, as well as other files associated with events. It also
4435 * creates the event hierarchy in the @parent/events directory.
4436 *
4437 * Returns 0 on success.
4438 *
4439 * Must be called with event_mutex held.
4440 */
4441int event_trace_add_tracer(struct dentry *parent, struct trace_array *tr)
4442{
4443	int ret;
4444
4445	lockdep_assert_held(&event_mutex);
4446
4447	ret = create_event_toplevel_files(parent, tr);
4448	if (ret)
4449		goto out;
4450
4451	down_write(&trace_event_sem);
4452	/* If tr already has the event list, it is initialized in early boot. */
4453	if (unlikely(!list_empty(&tr->events)))
4454		__trace_early_add_event_dirs(tr);
4455	else
4456		__trace_add_event_dirs(tr);
4457	up_write(&trace_event_sem);
4458
4459 out:
4460	return ret;
4461}
4462
4463/*
4464 * The top trace array already had its file descriptors created.
4465 * Now the files themselves need to be created.
4466 */
4467static __init int
4468early_event_add_tracer(struct dentry *parent, struct trace_array *tr)
4469{
4470	int ret;
4471
4472	guard(mutex)(&event_mutex);
4473
4474	ret = create_event_toplevel_files(parent, tr);
4475	if (ret)
4476		return ret;
4477
4478	down_write(&trace_event_sem);
4479	__trace_early_add_event_dirs(tr);
4480	up_write(&trace_event_sem);
4481
4482	return 0;
4483}
4484
4485/* Must be called with event_mutex held */
4486int event_trace_del_tracer(struct trace_array *tr)
4487{
4488	lockdep_assert_held(&event_mutex);
4489
4490	/* Disable any event triggers and associated soft-disabled events */
4491	clear_event_triggers(tr);
4492
4493	/* Clear the pid list */
4494	__ftrace_clear_event_pids(tr, TRACE_PIDS | TRACE_NO_PIDS);
4495
4496	/* Disable any running events */
4497	__ftrace_set_clr_event_nolock(tr, NULL, NULL, NULL, 0, NULL);
4498
4499	/* Make sure no more events are being executed */
4500	tracepoint_synchronize_unregister();
4501
4502	down_write(&trace_event_sem);
4503	__trace_remove_event_dirs(tr);
4504	eventfs_remove_events_dir(tr->event_dir);
4505	up_write(&trace_event_sem);
4506
4507	tr->event_dir = NULL;
4508
4509	return 0;
4510}
4511
4512static __init int event_trace_memsetup(void)
4513{
4514	field_cachep = KMEM_CACHE(ftrace_event_field, SLAB_PANIC);
4515	file_cachep = KMEM_CACHE(trace_event_file, SLAB_PANIC);
4516	return 0;
4517}
4518
4519/*
4520 * Helper function to enable or disable a comma-separated list of events
4521 * from the bootup buffer.
4522 */
4523static __init void __early_set_events(struct trace_array *tr, char *buf, bool enable)
4524{
4525	char *token;
4526
4527	while ((token = strsep(&buf, ","))) {
4528		if (*token) {
4529			if (enable) {
4530				if (ftrace_set_clr_event(tr, token, 1))
4531					pr_warn("Failed to enable trace event: %s\n", token);
4532			} else {
4533				ftrace_set_clr_event(tr, token, 0);
4534			}
4535		}
4536
4537		/* Put back the comma to allow this to be called again */
4538		if (buf)
4539			*(buf - 1) = ',';
4540	}
4541}
4542
4543/**
4544 * early_enable_events - enable events from the bootup buffer
4545 * @tr: The trace array to enable the events in
4546 * @buf: The buffer containing the comma separated list of events
4547 * @disable_first: If true, disable all events in @buf before enabling them
4548 *
4549 * This function enables events from the bootup buffer. If @disable_first
4550 * is true, it will first disable all events in the buffer before enabling
4551 * them.
4552 *
4553 * For syscall events, which rely on a global refcount to register the
4554 * SYSCALL_WORK_SYSCALL_TRACEPOINT flag (especially for pid 1), we must
4555 * ensure the refcount hits zero before re-enabling them. A simple
4556 * "disable then enable" per-event is not enough if multiple syscalls are
4557 * used, as the refcount will stay above zero. Thus, we need a two-phase
4558 * approach: disable all, then enable all.
4559 */
4560__init void
4561early_enable_events(struct trace_array *tr, char *buf, bool disable_first)
4562{
4563	if (disable_first)
4564		__early_set_events(tr, buf, false);
4565
4566	__early_set_events(tr, buf, true);
4567}
4568
4569static __init int event_trace_enable(void)
4570{
4571	struct trace_array *tr = top_trace_array();
4572	struct trace_event_call **iter, *call;
4573	int ret;
4574
4575	if (!tr)
4576		return -ENODEV;
4577
4578	for_each_event(iter, __start_ftrace_events, __stop_ftrace_events) {
4579
4580		call = *iter;
4581		ret = event_init(call);
4582		if (!ret)
4583			list_add(&call->list, &ftrace_events);
4584	}
4585
4586	register_trigger_cmds();
4587
4588	/*
4589	 * We need the top trace array to have a working set of trace
4590	 * points at early init, before the debug files and directories
4591	 * are created. Create the file entries now, and attach them
4592	 * to the actual file dentries later.
4593	 */
4594	__trace_early_add_events(tr);
4595
4596	early_enable_events(tr, bootup_event_buf, false);
4597
4598	trace_printk_start_comm();
4599
4600	register_event_cmds();
4601
4602
4603	return 0;
4604}
4605
4606/*
4607 * event_trace_enable() is called from trace_event_init() first to
4608 * initialize events and perhaps start any events that are on the
4609 * command line. Unfortunately, there are some events that will not
4610 * start this early, like the system call tracepoints that need
4611 * to set the %SYSCALL_WORK_SYSCALL_TRACEPOINT flag of pid 1. But
4612 * event_trace_enable() is called before pid 1 starts, and this flag
4613 * is never set, making the syscall tracepoint never get reached, but
4614 * the event is enabled regardless (and not doing anything).
4615 */
4616static __init int event_trace_enable_again(void)
4617{
4618	struct trace_array *tr;
4619
4620	tr = top_trace_array();
4621	if (!tr)
4622		return -ENODEV;
4623
4624	early_enable_events(tr, bootup_event_buf, true);
4625
4626	return 0;
4627}
4628
4629early_initcall(event_trace_enable_again);
4630
4631/* Init fields which doesn't related to the tracefs */
4632static __init int event_trace_init_fields(void)
4633{
4634	if (trace_define_generic_fields())
4635		pr_warn("tracing: Failed to allocated generic fields");
4636
4637	if (trace_define_common_fields())
4638		pr_warn("tracing: Failed to allocate common fields");
4639
4640	return 0;
4641}
4642
4643__init int event_trace_init(void)
4644{
4645	struct trace_array *tr;
4646	int ret;
4647
4648	tr = top_trace_array();
4649	if (!tr)
4650		return -ENODEV;
4651
4652	trace_create_file("available_events", TRACE_MODE_READ,
4653			  NULL, tr, &ftrace_avail_fops);
4654
4655	ret = early_event_add_tracer(NULL, tr);
4656	if (ret)
4657		return ret;
4658
4659#ifdef CONFIG_MODULES
4660	ret = register_module_notifier(&trace_module_nb);
4661	if (ret)
4662		pr_warn("Failed to register trace events module notifier\n");
4663#endif
4664
4665	eventdir_initialized = true;
4666
4667	return 0;
4668}
4669
4670void __init trace_event_init(void)
4671{
4672	event_trace_memsetup();
4673	init_ftrace_syscalls();
4674	event_trace_enable();
4675	event_trace_init_fields();
4676}
4677
4678#ifdef CONFIG_EVENT_TRACE_STARTUP_TEST
4679
4680static DEFINE_SPINLOCK(test_spinlock);
4681static DEFINE_SPINLOCK(test_spinlock_irq);
4682static DEFINE_MUTEX(test_mutex);
4683
4684static __init void test_work(struct work_struct *dummy)
4685{
4686	spin_lock(&test_spinlock);
4687	spin_lock_irq(&test_spinlock_irq);
4688	udelay(1);
4689	spin_unlock_irq(&test_spinlock_irq);
4690	spin_unlock(&test_spinlock);
4691
4692	mutex_lock(&test_mutex);
4693	msleep(1);
4694	mutex_unlock(&test_mutex);
4695}
4696
4697static __init int event_test_thread(void *unused)
4698{
4699	void *test_malloc;
4700
4701	test_malloc = kmalloc(1234, GFP_KERNEL);
4702	if (!test_malloc)
4703		pr_info("failed to kmalloc\n");
4704
4705	schedule_on_each_cpu(test_work);
4706
4707	kfree(test_malloc);
4708
4709	set_current_state(TASK_INTERRUPTIBLE);
4710	while (!kthread_should_stop()) {
4711		schedule();
4712		set_current_state(TASK_INTERRUPTIBLE);
4713	}
4714	__set_current_state(TASK_RUNNING);
4715
4716	return 0;
4717}
4718
4719/*
4720 * Do various things that may trigger events.
4721 */
4722static __init void event_test_stuff(void)
4723{
4724	struct task_struct *test_thread;
4725
4726	test_thread = kthread_run(event_test_thread, NULL, "test-events");
4727	msleep(1);
4728	kthread_stop(test_thread);
4729}
4730
4731/*
4732 * For every trace event defined, we will test each trace point separately,
4733 * and then by groups, and finally all trace points.
4734 */
4735static __init void event_trace_self_tests(void)
4736{
4737	struct trace_subsystem_dir *dir;
4738	struct trace_event_file *file;
4739	struct trace_event_call *call;
4740	struct event_subsystem *system;
4741	struct trace_array *tr;
4742	int ret;
4743
4744	tr = top_trace_array();
4745	if (!tr)
4746		return;
4747
4748	pr_info("Running tests on trace events:\n");
4749
4750	list_for_each_entry(file, &tr->events, list) {
4751
4752		call = file->event_call;
4753
4754		/* Only test those that have a probe */
4755		if (!call->class || !call->class->probe)
4756			continue;
4757
4758/*
4759 * Testing syscall events here is pretty useless, but
4760 * we still do it if configured. But this is time consuming.
4761 * What we really need is a user thread to perform the
4762 * syscalls as we test.
4763 */
4764#ifndef CONFIG_EVENT_TRACE_TEST_SYSCALLS
4765		if (call->class->system &&
4766		    strcmp(call->class->system, "syscalls") == 0)
4767			continue;
4768#endif
4769
4770		pr_info("Testing event %s: ", trace_event_name(call));
4771
4772		/*
4773		 * If an event is already enabled, someone is using
4774		 * it and the self test should not be on.
4775		 */
4776		if (file->flags & EVENT_FILE_FL_ENABLED) {
4777			pr_warn("Enabled event during self test!\n");
4778			WARN_ON_ONCE(1);
4779			continue;
4780		}
4781
4782		ftrace_event_enable_disable(file, 1);
4783		event_test_stuff();
4784		ftrace_event_enable_disable(file, 0);
4785
4786		pr_cont("OK\n");
4787	}
4788
4789	/* Now test at the sub system level */
4790
4791	pr_info("Running tests on trace event systems:\n");
4792
4793	list_for_each_entry(dir, &tr->systems, list) {
4794
4795		system = dir->subsystem;
4796
4797		/* the ftrace system is special, skip it */
4798		if (strcmp(system->name, "ftrace") == 0)
4799			continue;
4800
4801		pr_info("Testing event system %s: ", system->name);
4802
4803		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 1, NULL);
4804		if (WARN_ON_ONCE(ret)) {
4805			pr_warn("error enabling system %s\n",
4806				system->name);
4807			continue;
4808		}
4809
4810		event_test_stuff();
4811
4812		ret = __ftrace_set_clr_event(tr, NULL, system->name, NULL, 0, NULL);
4813		if (WARN_ON_ONCE(ret)) {
4814			pr_warn("error disabling system %s\n",
4815				system->name);
4816			continue;
4817		}
4818
4819		pr_cont("OK\n");
4820	}
4821
4822	/* Test with all events enabled */
4823
4824	pr_info("Running tests on all trace events:\n");
4825	pr_info("Testing all events: ");
4826
4827	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 1, NULL);
4828	if (WARN_ON_ONCE(ret)) {
4829		pr_warn("error enabling all events\n");
4830		return;
4831	}
4832
4833	event_test_stuff();
4834
4835	/* reset sysname */
4836	ret = __ftrace_set_clr_event(tr, NULL, NULL, NULL, 0, NULL);
4837	if (WARN_ON_ONCE(ret)) {
4838		pr_warn("error disabling all events\n");
4839		return;
4840	}
4841
4842	pr_cont("OK\n");
4843}
4844
4845#ifdef CONFIG_FUNCTION_TRACER
4846
4847static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
4848
4849static struct trace_event_file event_trace_file __initdata;
4850
4851static void __init
4852function_test_events_call(unsigned long ip, unsigned long parent_ip,
4853			  struct ftrace_ops *op, struct ftrace_regs *regs)
4854{
4855	struct trace_buffer *buffer;
4856	struct ring_buffer_event *event;
4857	struct ftrace_entry *entry;
4858	unsigned int trace_ctx;
4859	long disabled;
4860	int cpu;
4861
4862	trace_ctx = tracing_gen_ctx();
4863	preempt_disable_notrace();
4864	cpu = raw_smp_processor_id();
4865	disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
4866
4867	if (disabled != 1)
4868		goto out;
4869
4870	event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file,
4871						TRACE_FN, sizeof(*entry),
4872						trace_ctx);
4873	if (!event)
4874		goto out;
4875	entry	= ring_buffer_event_data(event);
4876	entry->ip			= ip;
4877	entry->parent_ip		= parent_ip;
4878
4879	event_trigger_unlock_commit(&event_trace_file, buffer, event,
4880				    entry, trace_ctx);
4881 out:
4882	atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
4883	preempt_enable_notrace();
4884}
4885
4886static struct ftrace_ops trace_ops __initdata  =
4887{
4888	.func = function_test_events_call,
4889};
4890
4891static __init void event_trace_self_test_with_function(void)
4892{
4893	int ret;
4894
4895	event_trace_file.tr = top_trace_array();
4896	if (WARN_ON(!event_trace_file.tr))
4897		return;
4898
4899	ret = register_ftrace_function(&trace_ops);
4900	if (WARN_ON(ret < 0)) {
4901		pr_info("Failed to enable function tracer for event tests\n");
4902		return;
4903	}
4904	pr_info("Running tests again, along with the function tracer\n");
4905	event_trace_self_tests();
4906	unregister_ftrace_function(&trace_ops);
4907}
4908#else
4909static __init void event_trace_self_test_with_function(void)
4910{
4911}
4912#endif
4913
4914static __init int event_trace_self_tests_init(void)
4915{
4916	if (!tracing_selftest_disabled) {
4917		event_trace_self_tests();
4918		event_trace_self_test_with_function();
4919	}
4920
4921	return 0;
4922}
4923
4924late_initcall(event_trace_self_tests_init);
4925
4926#endif