trace_kprobe.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Kprobes-based tracing events
   4 *
   5 * Created by Masami Hiramatsu <mhiramat@redhat.com>
   6 *
   7 */
   8#define pr_fmt(fmt)	"trace_kprobe: " fmt
   9
  10#include <linux/bpf-cgroup.h>
  11#include <linux/cleanup.h>
  12#include <linux/error-injection.h>
  13#include <linux/module.h>
  14#include <linux/rculist.h>
  15#include <linux/security.h>
  16#include <linux/uaccess.h>
  17
  18#include <asm/setup.h>  /* for COMMAND_LINE_SIZE */
  19
  20#include "trace_dynevent.h"
  21#include "trace_kprobe_selftest.h"
  22#include "trace_probe.h"
  23#include "trace_probe_kernel.h"
  24#include "trace_probe_tmpl.h"
  25
  26#define KPROBE_EVENT_SYSTEM "kprobes"
  27#define KRETPROBE_MAXACTIVE_MAX 4096
  28
  29/* Kprobe early definition from command line */
  30static char kprobe_boot_events_buf[COMMAND_LINE_SIZE] __initdata;
  31
  32static int __init set_kprobe_boot_events(char *str)
  33{
  34	strscpy(kprobe_boot_events_buf, str, COMMAND_LINE_SIZE);
  35	disable_tracing_selftest("running kprobe events");
  36
  37	return 1;
  38}
  39__setup("kprobe_event=", set_kprobe_boot_events);
  40
  41static int trace_kprobe_create(const char *raw_command);
  42static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev);
  43static int trace_kprobe_release(struct dyn_event *ev);
  44static bool trace_kprobe_is_busy(struct dyn_event *ev);
  45static bool trace_kprobe_match(const char *system, const char *event,
  46			int argc, const char **argv, struct dyn_event *ev);
  47
  48static struct dyn_event_operations trace_kprobe_ops = {
  49	.create = trace_kprobe_create,
  50	.show = trace_kprobe_show,
  51	.is_busy = trace_kprobe_is_busy,
  52	.free = trace_kprobe_release,
  53	.match = trace_kprobe_match,
  54};
  55
  56/*
  57 * Kprobe event core functions
  58 */
  59struct trace_kprobe {
  60	struct dyn_event	devent;
  61	struct kretprobe	rp;	/* Use rp.kp for kprobe use */
  62	unsigned long __percpu *nhit;
  63	const char		*symbol;	/* symbol name */
  64	struct trace_probe	tp;
  65};
  66
  67static bool is_trace_kprobe(struct dyn_event *ev)
  68{
  69	return ev->ops == &trace_kprobe_ops;
  70}
  71
  72static struct trace_kprobe *to_trace_kprobe(struct dyn_event *ev)
  73{
  74	return container_of(ev, struct trace_kprobe, devent);
  75}
  76
  77/**
  78 * for_each_trace_kprobe - iterate over the trace_kprobe list
  79 * @pos:	the struct trace_kprobe * for each entry
  80 * @dpos:	the struct dyn_event * to use as a loop cursor
  81 */
  82#define for_each_trace_kprobe(pos, dpos)	\
  83	for_each_dyn_event(dpos)		\
  84		if (is_trace_kprobe(dpos) && (pos = to_trace_kprobe(dpos)))
  85
  86static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
  87{
  88	return tk->rp.handler != NULL;
  89}
  90
  91static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
  92{
  93	return tk->symbol ? tk->symbol : "unknown";
  94}
  95
  96static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
  97{
  98	return tk->rp.kp.offset;
  99}
 100
 101static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
 102{
 103	return kprobe_gone(&tk->rp.kp);
 104}
 105
 106static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
 107						 struct module *mod)
 108{
 109	int len = strlen(module_name(mod));
 110	const char *name = trace_kprobe_symbol(tk);
 111
 112	return strncmp(module_name(mod), name, len) == 0 && name[len] == ':';
 113}
 114
 115#ifdef CONFIG_MODULES
 116static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
 117{
 118	char *p;
 119	bool ret;
 120
 121	if (!tk->symbol)
 122		return false;
 123	p = strchr(tk->symbol, ':');
 124	if (!p)
 125		return true;
 126	*p = '\0';
 127	scoped_guard(rcu)
 128		ret = !!find_module(tk->symbol);
 129	*p = ':';
 130
 131	return ret;
 132}
 133#else
 134static inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
 135{
 136	return false;
 137}
 138#endif
 139
 140static bool trace_kprobe_is_busy(struct dyn_event *ev)
 141{
 142	struct trace_kprobe *tk = to_trace_kprobe(ev);
 143
 144	return trace_probe_is_enabled(&tk->tp);
 145}
 146
 147static bool trace_kprobe_match_command_head(struct trace_kprobe *tk,
 148					    int argc, const char **argv)
 149{
 150	char buf[MAX_ARGSTR_LEN + 1];
 151
 152	if (!argc)
 153		return true;
 154
 155	if (!tk->symbol)
 156		snprintf(buf, sizeof(buf), "0x%p", tk->rp.kp.addr);
 157	else if (tk->rp.kp.offset)
 158		snprintf(buf, sizeof(buf), "%s+%u",
 159			 trace_kprobe_symbol(tk), tk->rp.kp.offset);
 160	else
 161		snprintf(buf, sizeof(buf), "%s", trace_kprobe_symbol(tk));
 162	if (strcmp(buf, argv[0]))
 163		return false;
 164	argc--; argv++;
 165
 166	return trace_probe_match_command_args(&tk->tp, argc, argv);
 167}
 168
 169static bool trace_kprobe_match(const char *system, const char *event,
 170			int argc, const char **argv, struct dyn_event *ev)
 171{
 172	struct trace_kprobe *tk = to_trace_kprobe(ev);
 173
 174	return (event[0] == '\0' ||
 175		strcmp(trace_probe_name(&tk->tp), event) == 0) &&
 176	    (!system || strcmp(trace_probe_group_name(&tk->tp), system) == 0) &&
 177	    trace_kprobe_match_command_head(tk, argc, argv);
 178}
 179
 180static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
 181{
 182	unsigned long nhit = 0;
 183	int cpu;
 184
 185	for_each_possible_cpu(cpu)
 186		nhit += *per_cpu_ptr(tk->nhit, cpu);
 187
 188	return nhit;
 189}
 190
 191static nokprobe_inline bool trace_kprobe_is_registered(struct trace_kprobe *tk)
 192{
 193	return !(list_empty(&tk->rp.kp.list) &&
 194		 hlist_unhashed(&tk->rp.kp.hlist));
 195}
 196
 197/* Return 0 if it fails to find the symbol address */
 198static nokprobe_inline
 199unsigned long trace_kprobe_address(struct trace_kprobe *tk)
 200{
 201	unsigned long addr;
 202
 203	if (tk->symbol) {
 204		addr = (unsigned long)
 205			kallsyms_lookup_name(trace_kprobe_symbol(tk));
 206		if (addr)
 207			addr += tk->rp.kp.offset;
 208	} else {
 209		addr = (unsigned long)tk->rp.kp.addr;
 210	}
 211	return addr;
 212}
 213
 214static nokprobe_inline struct trace_kprobe *
 215trace_kprobe_primary_from_call(struct trace_event_call *call)
 216{
 217	struct trace_probe *tp;
 218
 219	tp = trace_probe_primary_from_call(call);
 220	if (WARN_ON_ONCE(!tp))
 221		return NULL;
 222
 223	return container_of(tp, struct trace_kprobe, tp);
 224}
 225
 226bool trace_kprobe_on_func_entry(struct trace_event_call *call)
 227{
 228	struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
 229
 230	return tk ? (kprobe_on_func_entry(tk->rp.kp.addr,
 231			tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
 232			tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0) : false;
 233}
 234
 235bool trace_kprobe_error_injectable(struct trace_event_call *call)
 236{
 237	struct trace_kprobe *tk = trace_kprobe_primary_from_call(call);
 238
 239	return tk ? within_error_injection_list(trace_kprobe_address(tk)) :
 240	       false;
 241}
 242
 243static int register_kprobe_event(struct trace_kprobe *tk);
 244static int unregister_kprobe_event(struct trace_kprobe *tk);
 245
 246static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
 247static int kretprobe_dispatcher(struct kretprobe_instance *ri,
 248				struct pt_regs *regs);
 249
 250static void free_trace_kprobe(struct trace_kprobe *tk)
 251{
 252	if (tk) {
 253		trace_probe_cleanup(&tk->tp);
 254		kfree(tk->symbol);
 255		free_percpu(tk->nhit);
 256		kfree(tk);
 257	}
 258}
 259
 260DEFINE_FREE(free_trace_kprobe, struct trace_kprobe *,
 261	if (!IS_ERR_OR_NULL(_T)) free_trace_kprobe(_T))
 262
 263/*
 264 * Allocate new trace_probe and initialize it (including kprobes).
 265 */
 266static struct trace_kprobe *alloc_trace_kprobe(const char *group,
 267					     const char *event,
 268					     void *addr,
 269					     const char *symbol,
 270					     unsigned long offs,
 271					     int maxactive,
 272					     int nargs, bool is_return)
 273{
 274	struct trace_kprobe *tk __free(free_trace_kprobe) = NULL;
 275	int ret = -ENOMEM;
 276
 277	tk = kzalloc(struct_size(tk, tp.args, nargs), GFP_KERNEL);
 278	if (!tk)
 279		return ERR_PTR(ret);
 280
 281	tk->nhit = alloc_percpu(unsigned long);
 282	if (!tk->nhit)
 283		return ERR_PTR(ret);
 284
 285	if (symbol) {
 286		tk->symbol = kstrdup(symbol, GFP_KERNEL);
 287		if (!tk->symbol)
 288			return ERR_PTR(ret);
 289		tk->rp.kp.symbol_name = tk->symbol;
 290		tk->rp.kp.offset = offs;
 291	} else
 292		tk->rp.kp.addr = addr;
 293
 294	if (is_return)
 295		tk->rp.handler = kretprobe_dispatcher;
 296	else
 297		tk->rp.kp.pre_handler = kprobe_dispatcher;
 298
 299	tk->rp.maxactive = maxactive;
 300	INIT_HLIST_NODE(&tk->rp.kp.hlist);
 301	INIT_LIST_HEAD(&tk->rp.kp.list);
 302
 303	ret = trace_probe_init(&tk->tp, event, group, false, nargs);
 304	if (ret < 0)
 305		return ERR_PTR(ret);
 306
 307	dyn_event_init(&tk->devent, &trace_kprobe_ops);
 308	return_ptr(tk);
 309}
 310
 311static struct trace_kprobe *find_trace_kprobe(const char *event,
 312					      const char *group)
 313{
 314	struct dyn_event *pos;
 315	struct trace_kprobe *tk;
 316
 317	for_each_trace_kprobe(tk, pos)
 318		if (strcmp(trace_probe_name(&tk->tp), event) == 0 &&
 319		    strcmp(trace_probe_group_name(&tk->tp), group) == 0)
 320			return tk;
 321	return NULL;
 322}
 323
 324static inline int __enable_trace_kprobe(struct trace_kprobe *tk)
 325{
 326	int ret = 0;
 327
 328	if (trace_kprobe_is_registered(tk) && !trace_kprobe_has_gone(tk)) {
 329		if (trace_kprobe_is_return(tk))
 330			ret = enable_kretprobe(&tk->rp);
 331		else
 332			ret = enable_kprobe(&tk->rp.kp);
 333	}
 334
 335	return ret;
 336}
 337
 338static void __disable_trace_kprobe(struct trace_probe *tp)
 339{
 340	struct trace_kprobe *tk;
 341
 342	list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {
 343		if (!trace_kprobe_is_registered(tk))
 344			continue;
 345		if (trace_kprobe_is_return(tk))
 346			disable_kretprobe(&tk->rp);
 347		else
 348			disable_kprobe(&tk->rp.kp);
 349	}
 350}
 351
 352/*
 353 * Enable trace_probe
 354 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
 355 */
 356static int enable_trace_kprobe(struct trace_event_call *call,
 357				struct trace_event_file *file)
 358{
 359	struct trace_probe *tp;
 360	struct trace_kprobe *tk;
 361	bool enabled;
 362	int ret = 0;
 363
 364	tp = trace_probe_primary_from_call(call);
 365	if (WARN_ON_ONCE(!tp))
 366		return -ENODEV;
 367	enabled = trace_probe_is_enabled(tp);
 368
 369	/* This also changes "enabled" state */
 370	if (file) {
 371		ret = trace_probe_add_file(tp, file);
 372		if (ret)
 373			return ret;
 374	} else
 375		trace_probe_set_flag(tp, TP_FLAG_PROFILE);
 376
 377	if (enabled)
 378		return 0;
 379
 380	list_for_each_entry(tk, trace_probe_probe_list(tp), tp.list) {
 381		if (trace_kprobe_has_gone(tk))
 382			continue;
 383		ret = __enable_trace_kprobe(tk);
 384		if (ret)
 385			break;
 386		enabled = true;
 387	}
 388
 389	if (ret) {
 390		/* Failed to enable one of them. Roll back all */
 391		if (enabled)
 392			__disable_trace_kprobe(tp);
 393		if (file)
 394			trace_probe_remove_file(tp, file);
 395		else
 396			trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
 397	}
 398
 399	return ret;
 400}
 401
 402/*
 403 * Disable trace_probe
 404 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
 405 */
 406static int disable_trace_kprobe(struct trace_event_call *call,
 407				struct trace_event_file *file)
 408{
 409	struct trace_probe *tp;
 410
 411	tp = trace_probe_primary_from_call(call);
 412	if (WARN_ON_ONCE(!tp))
 413		return -ENODEV;
 414
 415	if (file) {
 416		if (!trace_probe_get_file_link(tp, file))
 417			return -ENOENT;
 418		if (!trace_probe_has_single_file(tp))
 419			goto out;
 420		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
 421	} else
 422		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
 423
 424	if (!trace_probe_is_enabled(tp))
 425		__disable_trace_kprobe(tp);
 426
 427 out:
 428	if (file)
 429		/*
 430		 * Synchronization is done in below function. For perf event,
 431		 * file == NULL and perf_trace_event_unreg() calls
 432		 * tracepoint_synchronize_unregister() to ensure synchronize
 433		 * event. We don't need to care about it.
 434		 */
 435		trace_probe_remove_file(tp, file);
 436
 437	return 0;
 438}
 439
 440#if defined(CONFIG_DYNAMIC_FTRACE) && \
 441	!defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)
 442static bool __within_notrace_func(unsigned long addr)
 443{
 444	unsigned long offset, size;
 445
 446	if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset))
 447		return false;
 448
 449	/* Get the entry address of the target function */
 450	addr -= offset;
 451
 452	/*
 453	 * Since ftrace_location_range() does inclusive range check, we need
 454	 * to subtract 1 byte from the end address.
 455	 */
 456	return !ftrace_location_range(addr, addr + size - 1);
 457}
 458
 459static bool within_notrace_func(struct trace_kprobe *tk)
 460{
 461	unsigned long addr = trace_kprobe_address(tk);
 462	char symname[KSYM_NAME_LEN], *p;
 463
 464	if (!__within_notrace_func(addr))
 465		return false;
 466
 467	/* Check if the address is on a suffixed-symbol */
 468	if (!lookup_symbol_name(addr, symname)) {
 469		p = strchr(symname, '.');
 470		if (!p)
 471			return true;
 472		*p = '\0';
 473		addr = (unsigned long)kprobe_lookup_name(symname, 0);
 474		if (addr)
 475			return __within_notrace_func(addr);
 476	}
 477
 478	return true;
 479}
 480#else
 481#define within_notrace_func(tk)	(false)
 482#endif
 483
 484/* Internal register function - just handle k*probes and flags */
 485static int __register_trace_kprobe(struct trace_kprobe *tk)
 486{
 487	int i, ret;
 488
 489	ret = security_locked_down(LOCKDOWN_KPROBES);
 490	if (ret)
 491		return ret;
 492
 493	if (trace_kprobe_is_registered(tk))
 494		return -EINVAL;
 495
 496	if (within_notrace_func(tk)) {
 497		pr_warn("Could not probe notrace function %ps\n",
 498			(void *)trace_kprobe_address(tk));
 499		return -EINVAL;
 500	}
 501
 502	for (i = 0; i < tk->tp.nr_args; i++) {
 503		ret = traceprobe_update_arg(&tk->tp.args[i]);
 504		if (ret)
 505			return ret;
 506	}
 507
 508	/* Set/clear disabled flag according to tp->flag */
 509	if (trace_probe_is_enabled(&tk->tp))
 510		tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
 511	else
 512		tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
 513
 514	if (trace_kprobe_is_return(tk))
 515		ret = register_kretprobe(&tk->rp);
 516	else
 517		ret = register_kprobe(&tk->rp.kp);
 518
 519	return ret;
 520}
 521
 522/* Internal unregister function - just handle k*probes and flags */
 523static void __unregister_trace_kprobe(struct trace_kprobe *tk)
 524{
 525	if (trace_kprobe_is_registered(tk)) {
 526		if (trace_kprobe_is_return(tk))
 527			unregister_kretprobe(&tk->rp);
 528		else
 529			unregister_kprobe(&tk->rp.kp);
 530		/* Cleanup kprobe for reuse and mark it unregistered */
 531		INIT_HLIST_NODE(&tk->rp.kp.hlist);
 532		INIT_LIST_HEAD(&tk->rp.kp.list);
 533		if (tk->rp.kp.symbol_name)
 534			tk->rp.kp.addr = NULL;
 535	}
 536}
 537
 538/* Unregister a trace_probe and probe_event */
 539static int unregister_trace_kprobe(struct trace_kprobe *tk)
 540{
 541	/* If other probes are on the event, just unregister kprobe */
 542	if (trace_probe_has_sibling(&tk->tp))
 543		goto unreg;
 544
 545	/* Enabled event can not be unregistered */
 546	if (trace_probe_is_enabled(&tk->tp))
 547		return -EBUSY;
 548
 549	/* If there's a reference to the dynamic event */
 550	if (trace_event_dyn_busy(trace_probe_event_call(&tk->tp)))
 551		return -EBUSY;
 552
 553	/* Will fail if probe is being used by ftrace or perf */
 554	if (unregister_kprobe_event(tk))
 555		return -EBUSY;
 556
 557unreg:
 558	__unregister_trace_kprobe(tk);
 559	dyn_event_remove(&tk->devent);
 560	trace_probe_unlink(&tk->tp);
 561
 562	return 0;
 563}
 564
 565static bool trace_kprobe_has_same_kprobe(struct trace_kprobe *orig,
 566					 struct trace_kprobe *comp)
 567{
 568	struct trace_probe_event *tpe = orig->tp.event;
 569	int i;
 570
 571	list_for_each_entry(orig, &tpe->probes, tp.list) {
 572		if (strcmp(trace_kprobe_symbol(orig),
 573			   trace_kprobe_symbol(comp)) ||
 574		    trace_kprobe_offset(orig) != trace_kprobe_offset(comp))
 575			continue;
 576
 577		/*
 578		 * trace_probe_compare_arg_type() ensured that nr_args and
 579		 * each argument name and type are same. Let's compare comm.
 580		 */
 581		for (i = 0; i < orig->tp.nr_args; i++) {
 582			if (strcmp(orig->tp.args[i].comm,
 583				   comp->tp.args[i].comm))
 584				break;
 585		}
 586
 587		if (i == orig->tp.nr_args)
 588			return true;
 589	}
 590
 591	return false;
 592}
 593
 594static int append_trace_kprobe(struct trace_kprobe *tk, struct trace_kprobe *to)
 595{
 596	int ret;
 597
 598	ret = trace_probe_compare_arg_type(&tk->tp, &to->tp);
 599	if (ret) {
 600		/* Note that argument starts index = 2 */
 601		trace_probe_log_set_index(ret + 1);
 602		trace_probe_log_err(0, DIFF_ARG_TYPE);
 603		return -EEXIST;
 604	}
 605	if (trace_kprobe_has_same_kprobe(to, tk)) {
 606		trace_probe_log_set_index(0);
 607		trace_probe_log_err(0, SAME_PROBE);
 608		return -EEXIST;
 609	}
 610
 611	/* Append to existing event */
 612	ret = trace_probe_append(&tk->tp, &to->tp);
 613	if (ret)
 614		return ret;
 615
 616	/* Register k*probe */
 617	ret = __register_trace_kprobe(tk);
 618	if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
 619		pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
 620		ret = 0;
 621	}
 622
 623	if (ret)
 624		trace_probe_unlink(&tk->tp);
 625	else
 626		dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));
 627
 628	return ret;
 629}
 630
 631/* Register a trace_probe and probe_event */
 632static int register_trace_kprobe(struct trace_kprobe *tk)
 633{
 634	struct trace_kprobe *old_tk;
 635	int ret;
 636
 637	guard(mutex)(&event_mutex);
 638
 639	old_tk = find_trace_kprobe(trace_probe_name(&tk->tp),
 640				   trace_probe_group_name(&tk->tp));
 641	if (old_tk) {
 642		if (trace_kprobe_is_return(tk) != trace_kprobe_is_return(old_tk)) {
 643			trace_probe_log_set_index(0);
 644			trace_probe_log_err(0, DIFF_PROBE_TYPE);
 645			return -EEXIST;
 646		}
 647		return append_trace_kprobe(tk, old_tk);
 648	}
 649
 650	/* Register new event */
 651	ret = register_kprobe_event(tk);
 652	if (ret) {
 653		if (ret == -EEXIST) {
 654			trace_probe_log_set_index(0);
 655			trace_probe_log_err(0, EVENT_EXIST);
 656		} else
 657			pr_warn("Failed to register probe event(%d)\n", ret);
 658		return ret;
 659	}
 660
 661	/* Register k*probe */
 662	ret = __register_trace_kprobe(tk);
 663	if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
 664		pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
 665		ret = 0;
 666	}
 667
 668	if (ret < 0)
 669		unregister_kprobe_event(tk);
 670	else
 671		dyn_event_add(&tk->devent, trace_probe_event_call(&tk->tp));
 672
 673	return ret;
 674}
 675
 676#ifdef CONFIG_MODULES
 677static int validate_module_probe_symbol(const char *modname, const char *symbol);
 678
 679static int register_module_trace_kprobe(struct module *mod, struct trace_kprobe *tk)
 680{
 681	const char *p;
 682	int ret = 0;
 683
 684	p = strchr(trace_kprobe_symbol(tk), ':');
 685	if (p)
 686		ret = validate_module_probe_symbol(module_name(mod), p + 1);
 687	if (!ret)
 688		ret = __register_trace_kprobe(tk);
 689	return ret;
 690}
 691
 692/* Module notifier call back, checking event on the module */
 693static int trace_kprobe_module_callback(struct notifier_block *nb,
 694				       unsigned long val, void *data)
 695{
 696	struct module *mod = data;
 697	struct dyn_event *pos;
 698	struct trace_kprobe *tk;
 699	int ret;
 700
 701	if (val != MODULE_STATE_COMING)
 702		return NOTIFY_DONE;
 703
 704	/* Update probes on coming module */
 705	guard(mutex)(&event_mutex);
 706	for_each_trace_kprobe(tk, pos) {
 707		if (trace_kprobe_within_module(tk, mod)) {
 708			/* Don't need to check busy - this should have gone. */
 709			__unregister_trace_kprobe(tk);
 710			ret = register_module_trace_kprobe(mod, tk);
 711			if (ret)
 712				pr_warn("Failed to re-register probe %s on %s: %d\n",
 713					trace_probe_name(&tk->tp),
 714					module_name(mod), ret);
 715		}
 716	}
 717
 718	return NOTIFY_DONE;
 719}
 720
 721static struct notifier_block trace_kprobe_module_nb = {
 722	.notifier_call = trace_kprobe_module_callback,
 723	.priority = 2	/* Invoked after kprobe and jump_label module callback */
 724};
 725static int trace_kprobe_register_module_notifier(void)
 726{
 727	return register_module_notifier(&trace_kprobe_module_nb);
 728}
 729#else
 730static int trace_kprobe_register_module_notifier(void)
 731{
 732	return 0;
 733}
 734#endif /* CONFIG_MODULES */
 735
 736static int count_symbols(void *data, unsigned long unused)
 737{
 738	unsigned int *count = data;
 739
 740	(*count)++;
 741
 742	return 0;
 743}
 744
 745struct sym_count_ctx {
 746	unsigned int count;
 747	const char *name;
 748};
 749
 750static int count_mod_symbols(void *data, const char *name, unsigned long unused)
 751{
 752	struct sym_count_ctx *ctx = data;
 753
 754	if (strcmp(name, ctx->name) == 0)
 755		ctx->count++;
 756
 757	return 0;
 758}
 759
 760static unsigned int number_of_same_symbols(const char *mod, const char *func_name)
 761{
 762	struct sym_count_ctx ctx = { .count = 0, .name = func_name };
 763
 764	if (!mod)
 765		kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count);
 766
 767	module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx);
 768
 769	return ctx.count;
 770}
 771
 772static int validate_module_probe_symbol(const char *modname, const char *symbol)
 773{
 774	unsigned int count = number_of_same_symbols(modname, symbol);
 775
 776	if (count > 1) {
 777		/*
 778		 * Users should use ADDR to remove the ambiguity of
 779		 * using KSYM only.
 780		 */
 781		return -EADDRNOTAVAIL;
 782	} else if (count == 0) {
 783		/*
 784		 * We can return ENOENT earlier than when register the
 785		 * kprobe.
 786		 */
 787		return -ENOENT;
 788	}
 789	return 0;
 790}
 791
 792#ifdef CONFIG_MODULES
 793/* Return NULL if the module is not loaded or under unloading. */
 794static struct module *try_module_get_by_name(const char *name)
 795{
 796	struct module *mod;
 797
 798	guard(rcu)();
 799	mod = find_module(name);
 800	if (mod && !try_module_get(mod))
 801		mod = NULL;
 802	return mod;
 803}
 804#else
 805#define try_module_get_by_name(name)	(NULL)
 806#endif
 807
 808static int validate_probe_symbol(char *symbol)
 809{
 810	struct module *mod = NULL;
 811	char *modname = NULL, *p;
 812	int ret = 0;
 813
 814	p = strchr(symbol, ':');
 815	if (p) {
 816		modname = symbol;
 817		symbol = p + 1;
 818		*p = '\0';
 819		mod = try_module_get_by_name(modname);
 820		if (!mod)
 821			goto out;
 822	}
 823
 824	ret = validate_module_probe_symbol(modname, symbol);
 825out:
 826	if (p)
 827		*p = ':';
 828	if (mod)
 829		module_put(mod);
 830	return ret;
 831}
 832
 833static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,
 834				      struct pt_regs *regs);
 835
 836static int trace_kprobe_create_internal(int argc, const char *argv[],
 837					struct traceprobe_parse_context *ctx)
 838{
 839	/*
 840	 * Argument syntax:
 841	 *  - Add kprobe:
 842	 *      p[:[GRP/][EVENT]] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
 843	 *  - Add kretprobe:
 844	 *      r[MAXACTIVE][:[GRP/][EVENT]] [MOD:]KSYM[+0] [FETCHARGS]
 845	 *    Or
 846	 *      p[:[GRP/][EVENT]] [MOD:]KSYM[+0]%return [FETCHARGS]
 847	 *
 848	 * Fetch args:
 849	 *  $retval	: fetch return value
 850	 *  $stack	: fetch stack address
 851	 *  $stackN	: fetch Nth of stack (N:0-)
 852	 *  $comm       : fetch current task comm
 853	 *  @ADDR	: fetch memory at ADDR (ADDR should be in kernel)
 854	 *  @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
 855	 *  %REG	: fetch register REG
 856	 * Dereferencing memory fetch:
 857	 *  +|-offs(ARG) : fetch memory at ARG +|- offs address.
 858	 * Alias name of args:
 859	 *  NAME=FETCHARG : set NAME as alias of FETCHARG.
 860	 * Type of args:
 861	 *  FETCHARG:TYPE : use TYPE instead of unsigned long.
 862	 */
 863	struct trace_kprobe *tk __free(free_trace_kprobe) = NULL;
 864	const char *event = NULL, *group = KPROBE_EVENT_SYSTEM;
 865	const char **new_argv __free(kfree) = NULL;
 866	int i, len, new_argc = 0, ret = 0;
 867	char *symbol __free(kfree) = NULL;
 868	char *ebuf __free(kfree) = NULL;
 869	char *gbuf __free(kfree) = NULL;
 870	char *abuf __free(kfree) = NULL;
 871	char *dbuf __free(kfree) = NULL;
 872	enum probe_print_type ptype;
 873	bool is_return = false;
 874	int maxactive = 0;
 875	void *addr = NULL;
 876	char *tmp = NULL;
 877	long offset = 0;
 878
 879	switch (argv[0][0]) {
 880	case 'r':
 881		is_return = true;
 882		break;
 883	case 'p':
 884		break;
 885	default:
 886		return -ECANCELED;
 887	}
 888	if (argc < 2)
 889		return -ECANCELED;
 890
 891	event = strchr(&argv[0][1], ':');
 892	if (event)
 893		event++;
 894
 895	if (isdigit(argv[0][1])) {
 896		char *buf __free(kfree) = NULL;
 897
 898		if (!is_return) {
 899			trace_probe_log_err(1, BAD_MAXACT_TYPE);
 900			return -EINVAL;
 901		}
 902		if (event)
 903			len = event - &argv[0][1] - 1;
 904		else
 905			len = strlen(&argv[0][1]);
 906		if (len > MAX_EVENT_NAME_LEN - 1) {
 907			trace_probe_log_err(1, BAD_MAXACT);
 908			return -EINVAL;
 909		}
 910		buf = kmemdup(&argv[0][1], len + 1, GFP_KERNEL);
 911		if (!buf)
 912			return -ENOMEM;
 913		buf[len] = '\0';
 914		ret = kstrtouint(buf, 0, &maxactive);
 915		if (ret || !maxactive) {
 916			trace_probe_log_err(1, BAD_MAXACT);
 917			return -EINVAL;
 918		}
 919		/* kretprobes instances are iterated over via a list. The
 920		 * maximum should stay reasonable.
 921		 */
 922		if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
 923			trace_probe_log_err(1, MAXACT_TOO_BIG);
 924			return -EINVAL;
 925		}
 926	}
 927
 928	/* try to parse an address. if that fails, try to read the
 929	 * input as a symbol. */
 930	if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
 931		trace_probe_log_set_index(1);
 932		/* Check whether uprobe event specified */
 933		if (strchr(argv[1], '/') && strchr(argv[1], ':'))
 934			return -ECANCELED;
 935
 936		/* a symbol specified */
 937		symbol = kstrdup(argv[1], GFP_KERNEL);
 938		if (!symbol)
 939			return -ENOMEM;
 940
 941		tmp = strchr(symbol, '%');
 942		if (tmp) {
 943			if (!strcmp(tmp, "%return")) {
 944				*tmp = '\0';
 945				is_return = true;
 946			} else {
 947				trace_probe_log_err(tmp - symbol, BAD_ADDR_SUFFIX);
 948				return -EINVAL;
 949			}
 950		}
 951
 952		/* TODO: support .init module functions */
 953		ret = traceprobe_split_symbol_offset(symbol, &offset);
 954		if (ret || offset < 0 || offset > UINT_MAX) {
 955			trace_probe_log_err(0, BAD_PROBE_ADDR);
 956			return -EINVAL;
 957		}
 958		ret = validate_probe_symbol(symbol);
 959		if (ret) {
 960			if (ret == -EADDRNOTAVAIL)
 961				trace_probe_log_err(0, NON_UNIQ_SYMBOL);
 962			else
 963				trace_probe_log_err(0, BAD_PROBE_ADDR);
 964			return -EINVAL;
 965		}
 966		if (is_return)
 967			ctx->flags |= TPARG_FL_RETURN;
 968		ret = kprobe_on_func_entry(NULL, symbol, offset);
 969		if (ret == 0 && !is_return)
 970			ctx->flags |= TPARG_FL_FENTRY;
 971		/* Defer the ENOENT case until register kprobe */
 972		if (ret == -EINVAL && is_return) {
 973			trace_probe_log_err(0, BAD_RETPROBE);
 974			return -EINVAL;
 975		}
 976	}
 977
 978	trace_probe_log_set_index(0);
 979	if (event) {
 980		gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
 981		if (!gbuf)
 982			return -ENOMEM;
 983		ret = traceprobe_parse_event_name(&event, &group, gbuf,
 984						  event - argv[0]);
 985		if (ret)
 986			return ret;
 987	}
 988
 989	if (!event) {
 990		/* Make a new event name */
 991		ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
 992		if (!ebuf)
 993			return -ENOMEM;
 994		if (symbol)
 995			snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
 996				 is_return ? 'r' : 'p', symbol, offset);
 997		else
 998			snprintf(ebuf, MAX_EVENT_NAME_LEN, "%c_0x%p",
 999				 is_return ? 'r' : 'p', addr);
1000		sanitize_event_name(ebuf);
1001		event = ebuf;
1002	}
1003
1004	abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL);
1005	if (!abuf)
1006		return -ENOMEM;
1007	argc -= 2; argv += 2;
1008	ctx->funcname = symbol;
1009	new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
1010					       abuf, MAX_BTF_ARGS_LEN, ctx);
1011	if (IS_ERR(new_argv)) {
1012		ret = PTR_ERR(new_argv);
1013		new_argv = NULL;
1014		return ret;
1015	}
1016	if (new_argv) {
1017		argc = new_argc;
1018		argv = new_argv;
1019	}
1020	if (argc > MAX_TRACE_ARGS) {
1021		trace_probe_log_set_index(2);
1022		trace_probe_log_err(0, TOO_MANY_ARGS);
1023		return -E2BIG;
1024	}
1025
1026	ret = traceprobe_expand_dentry_args(argc, argv, &dbuf);
1027	if (ret)
1028		return ret;
1029
1030	/* setup a probe */
1031	tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
1032				argc, is_return);
1033	if (IS_ERR(tk)) {
1034		ret = PTR_ERR(tk);
1035		/* This must return -ENOMEM, else there is a bug */
1036		WARN_ON_ONCE(ret != -ENOMEM);
1037		return ret;	/* We know tk is not allocated */
1038	}
1039
1040	/* parse arguments */
1041	for (i = 0; i < argc; i++) {
1042		trace_probe_log_set_index(i + 2);
1043		ctx->offset = 0;
1044		ret = traceprobe_parse_probe_arg(&tk->tp, i, argv[i], ctx);
1045		if (ret)
1046			return ret;	/* This can be -ENOMEM */
1047	}
1048	/* entry handler for kretprobe */
1049	if (is_return && tk->tp.entry_arg) {
1050		tk->rp.entry_handler = trace_kprobe_entry_handler;
1051		tk->rp.data_size = traceprobe_get_entry_data_size(&tk->tp);
1052	}
1053
1054	ptype = is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1055	ret = traceprobe_set_print_fmt(&tk->tp, ptype);
1056	if (ret < 0)
1057		return ret;
1058
1059	ret = register_trace_kprobe(tk);
1060	if (ret) {
1061		trace_probe_log_set_index(1);
1062		if (ret == -EILSEQ)
1063			trace_probe_log_err(0, BAD_INSN_BNDRY);
1064		else if (ret == -ENOENT)
1065			trace_probe_log_err(0, BAD_PROBE_ADDR);
1066		else if (ret != -ENOMEM && ret != -EEXIST)
1067			trace_probe_log_err(0, FAIL_REG_PROBE);
1068		return ret;
1069	}
1070	/*
1071	 * Here, 'tk' has been registered to the list successfully,
1072	 * so we don't need to free it.
1073	 */
1074	tk = NULL;
1075
1076	return 0;
1077}
1078
1079static int trace_kprobe_create_cb(int argc, const char *argv[])
1080{
1081	struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
1082	int ret;
1083
1084	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1085	if (!ctx)
1086		return -ENOMEM;
1087	ctx->flags = TPARG_FL_KERNEL;
1088
1089	trace_probe_log_init("trace_kprobe", argc, argv);
1090
1091	ret = trace_kprobe_create_internal(argc, argv, ctx);
1092
1093	trace_probe_log_clear();
1094	return ret;
1095}
1096
1097static int trace_kprobe_create(const char *raw_command)
1098{
1099	return trace_probe_create(raw_command, trace_kprobe_create_cb);
1100}
1101
1102static int create_or_delete_trace_kprobe(const char *raw_command)
1103{
1104	int ret;
1105
1106	if (raw_command[0] == '-')
1107		return dyn_event_release(raw_command, &trace_kprobe_ops);
1108
1109	ret = dyn_event_create(raw_command, &trace_kprobe_ops);
1110	return ret == -ECANCELED ? -EINVAL : ret;
1111}
1112
1113static int trace_kprobe_run_command(struct dynevent_cmd *cmd)
1114{
1115	return create_or_delete_trace_kprobe(cmd->seq.buffer);
1116}
1117
1118/**
1119 * kprobe_event_cmd_init - Initialize a kprobe event command object
1120 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1121 * @buf: A pointer to the buffer used to build the command
1122 * @maxlen: The length of the buffer passed in @buf
1123 *
1124 * Initialize a synthetic event command object.  Use this before
1125 * calling any of the other kprobe_event functions.
1126 */
1127void kprobe_event_cmd_init(struct dynevent_cmd *cmd, char *buf, int maxlen)
1128{
1129	dynevent_cmd_init(cmd, buf, maxlen, DYNEVENT_TYPE_KPROBE,
1130			  trace_kprobe_run_command);
1131}
1132EXPORT_SYMBOL_GPL(kprobe_event_cmd_init);
1133
1134/**
1135 * __kprobe_event_gen_cmd_start - Generate a kprobe event command from arg list
1136 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1137 * @kretprobe: Is this a return probe?
1138 * @name: The name of the kprobe event
1139 * @loc: The location of the kprobe event
1140 * @...: Variable number of arg (pairs), one pair for each field
1141 *
1142 * NOTE: Users normally won't want to call this function directly, but
1143 * rather use the kprobe_event_gen_cmd_start() wrapper, which automatically
1144 * adds a NULL to the end of the arg list.  If this function is used
1145 * directly, make sure the last arg in the variable arg list is NULL.
1146 *
1147 * Generate a kprobe event command to be executed by
1148 * kprobe_event_gen_cmd_end().  This function can be used to generate the
1149 * complete command or only the first part of it; in the latter case,
1150 * kprobe_event_add_fields() can be used to add more fields following this.
1151 *
1152 * Unlikely the synth_event_gen_cmd_start(), @loc must be specified. This
1153 * returns -EINVAL if @loc == NULL.
1154 *
1155 * Return: 0 if successful, error otherwise.
1156 */
1157int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd, bool kretprobe,
1158				 const char *name, const char *loc, ...)
1159{
1160	char buf[MAX_EVENT_NAME_LEN];
1161	struct dynevent_arg arg;
1162	va_list args;
1163	int ret;
1164
1165	if (cmd->type != DYNEVENT_TYPE_KPROBE)
1166		return -EINVAL;
1167
1168	if (!loc)
1169		return -EINVAL;
1170
1171	if (kretprobe)
1172		snprintf(buf, MAX_EVENT_NAME_LEN, "r:kprobes/%s", name);
1173	else
1174		snprintf(buf, MAX_EVENT_NAME_LEN, "p:kprobes/%s", name);
1175
1176	ret = dynevent_str_add(cmd, buf);
1177	if (ret)
1178		return ret;
1179
1180	dynevent_arg_init(&arg, 0);
1181	arg.str = loc;
1182	ret = dynevent_arg_add(cmd, &arg, NULL);
1183	if (ret)
1184		return ret;
1185
1186	va_start(args, loc);
1187	for (;;) {
1188		const char *field;
1189
1190		field = va_arg(args, const char *);
1191		if (!field)
1192			break;
1193
1194		if (++cmd->n_fields > MAX_TRACE_ARGS) {
1195			ret = -EINVAL;
1196			break;
1197		}
1198
1199		arg.str = field;
1200		ret = dynevent_arg_add(cmd, &arg, NULL);
1201		if (ret)
1202			break;
1203	}
1204	va_end(args);
1205
1206	return ret;
1207}
1208EXPORT_SYMBOL_GPL(__kprobe_event_gen_cmd_start);
1209
1210/**
1211 * __kprobe_event_add_fields - Add probe fields to a kprobe command from arg list
1212 * @cmd: A pointer to the dynevent_cmd struct representing the new event
1213 * @...: Variable number of arg (pairs), one pair for each field
1214 *
1215 * NOTE: Users normally won't want to call this function directly, but
1216 * rather use the kprobe_event_add_fields() wrapper, which
1217 * automatically adds a NULL to the end of the arg list.  If this
1218 * function is used directly, make sure the last arg in the variable
1219 * arg list is NULL.
1220 *
1221 * Add probe fields to an existing kprobe command using a variable
1222 * list of args.  Fields are added in the same order they're listed.
1223 *
1224 * Return: 0 if successful, error otherwise.
1225 */
1226int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...)
1227{
1228	struct dynevent_arg arg;
1229	va_list args;
1230	int ret = 0;
1231
1232	if (cmd->type != DYNEVENT_TYPE_KPROBE)
1233		return -EINVAL;
1234
1235	dynevent_arg_init(&arg, 0);
1236
1237	va_start(args, cmd);
1238	for (;;) {
1239		const char *field;
1240
1241		field = va_arg(args, const char *);
1242		if (!field)
1243			break;
1244
1245		if (++cmd->n_fields > MAX_TRACE_ARGS) {
1246			ret = -EINVAL;
1247			break;
1248		}
1249
1250		arg.str = field;
1251		ret = dynevent_arg_add(cmd, &arg, NULL);
1252		if (ret)
1253			break;
1254	}
1255	va_end(args);
1256
1257	return ret;
1258}
1259EXPORT_SYMBOL_GPL(__kprobe_event_add_fields);
1260
1261/**
1262 * kprobe_event_delete - Delete a kprobe event
1263 * @name: The name of the kprobe event to delete
1264 *
1265 * Delete a kprobe event with the give @name from kernel code rather
1266 * than directly from the command line.
1267 *
1268 * Return: 0 if successful, error otherwise.
1269 */
1270int kprobe_event_delete(const char *name)
1271{
1272	char buf[MAX_EVENT_NAME_LEN];
1273
1274	snprintf(buf, MAX_EVENT_NAME_LEN, "-:%s", name);
1275
1276	return create_or_delete_trace_kprobe(buf);
1277}
1278EXPORT_SYMBOL_GPL(kprobe_event_delete);
1279
1280static int trace_kprobe_release(struct dyn_event *ev)
1281{
1282	struct trace_kprobe *tk = to_trace_kprobe(ev);
1283	int ret = unregister_trace_kprobe(tk);
1284
1285	if (!ret)
1286		free_trace_kprobe(tk);
1287	return ret;
1288}
1289
1290static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
1291{
1292	struct trace_kprobe *tk = to_trace_kprobe(ev);
1293	int i;
1294
1295	seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
1296	if (trace_kprobe_is_return(tk) && tk->rp.maxactive)
1297		seq_printf(m, "%d", tk->rp.maxactive);
1298	seq_printf(m, ":%s/%s", trace_probe_group_name(&tk->tp),
1299				trace_probe_name(&tk->tp));
1300
1301	if (!tk->symbol)
1302		seq_printf(m, " 0x%p", tk->rp.kp.addr);
1303	else if (tk->rp.kp.offset)
1304		seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
1305			   tk->rp.kp.offset);
1306	else
1307		seq_printf(m, " %s", trace_kprobe_symbol(tk));
1308
1309	for (i = 0; i < tk->tp.nr_args; i++)
1310		seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
1311	seq_putc(m, '\n');
1312
1313	return 0;
1314}
1315
1316static int probes_seq_show(struct seq_file *m, void *v)
1317{
1318	struct dyn_event *ev = v;
1319
1320	if (!is_trace_kprobe(ev))
1321		return 0;
1322
1323	return trace_kprobe_show(m, ev);
1324}
1325
1326static const struct seq_operations probes_seq_op = {
1327	.start  = dyn_event_seq_start,
1328	.next   = dyn_event_seq_next,
1329	.stop   = dyn_event_seq_stop,
1330	.show   = probes_seq_show
1331};
1332
1333static int probes_open(struct inode *inode, struct file *file)
1334{
1335	int ret;
1336
1337	ret = security_locked_down(LOCKDOWN_TRACEFS);
1338	if (ret)
1339		return ret;
1340
1341	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1342		ret = dyn_events_release_all(&trace_kprobe_ops);
1343		if (ret < 0)
1344			return ret;
1345	}
1346
1347	return seq_open(file, &probes_seq_op);
1348}
1349
1350static ssize_t probes_write(struct file *file, const char __user *buffer,
1351			    size_t count, loff_t *ppos)
1352{
1353	return trace_parse_run_command(file, buffer, count, ppos,
1354				       create_or_delete_trace_kprobe);
1355}
1356
1357static const struct file_operations kprobe_events_ops = {
1358	.owner          = THIS_MODULE,
1359	.open           = probes_open,
1360	.read           = seq_read,
1361	.llseek         = seq_lseek,
1362	.release        = seq_release,
1363	.write		= probes_write,
1364};
1365
1366static unsigned long trace_kprobe_missed(struct trace_kprobe *tk)
1367{
1368	return trace_kprobe_is_return(tk) ?
1369		tk->rp.kp.nmissed + tk->rp.nmissed : tk->rp.kp.nmissed;
1370}
1371
1372/* Probes profiling interfaces */
1373static int probes_profile_seq_show(struct seq_file *m, void *v)
1374{
1375	struct dyn_event *ev = v;
1376	struct trace_kprobe *tk;
1377	unsigned long nmissed;
1378
1379	if (!is_trace_kprobe(ev))
1380		return 0;
1381
1382	tk = to_trace_kprobe(ev);
1383	nmissed = trace_kprobe_missed(tk);
1384	seq_printf(m, "  %-44s %15lu %15lu\n",
1385		   trace_probe_name(&tk->tp),
1386		   trace_kprobe_nhit(tk),
1387		   nmissed);
1388
1389	return 0;
1390}
1391
1392static const struct seq_operations profile_seq_op = {
1393	.start  = dyn_event_seq_start,
1394	.next   = dyn_event_seq_next,
1395	.stop   = dyn_event_seq_stop,
1396	.show   = probes_profile_seq_show
1397};
1398
1399static int profile_open(struct inode *inode, struct file *file)
1400{
1401	int ret;
1402
1403	ret = security_locked_down(LOCKDOWN_TRACEFS);
1404	if (ret)
1405		return ret;
1406
1407	return seq_open(file, &profile_seq_op);
1408}
1409
1410static const struct file_operations kprobe_profile_ops = {
1411	.owner          = THIS_MODULE,
1412	.open           = profile_open,
1413	.read           = seq_read,
1414	.llseek         = seq_lseek,
1415	.release        = seq_release,
1416};
1417
1418/* Note that we don't verify it, since the code does not come from user space */
1419static int
1420process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
1421		   void *dest, void *base)
1422{
1423	struct pt_regs *regs = rec;
1424	unsigned long val;
1425	int ret;
1426
1427retry:
1428	/* 1st stage: get value from context */
1429	switch (code->op) {
1430	case FETCH_OP_REG:
1431		val = regs_get_register(regs, code->param);
1432		break;
1433	case FETCH_OP_STACK:
1434		val = regs_get_kernel_stack_nth(regs, code->param);
1435		break;
1436	case FETCH_OP_STACKP:
1437		val = kernel_stack_pointer(regs);
1438		break;
1439	case FETCH_OP_RETVAL:
1440		val = regs_return_value(regs);
1441		break;
1442#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1443	case FETCH_OP_ARG:
1444		val = regs_get_kernel_argument(regs, code->param);
1445		break;
1446	case FETCH_OP_EDATA:
1447		val = *(unsigned long *)((unsigned long)edata + code->offset);
1448		break;
1449#endif
1450	case FETCH_NOP_SYMBOL:	/* Ignore a place holder */
1451		code++;
1452		goto retry;
1453	default:
1454		ret = process_common_fetch_insn(code, &val);
1455		if (ret < 0)
1456			return ret;
1457	}
1458	code++;
1459
1460	return process_fetch_insn_bottom(code, val, dest, base);
1461}
1462NOKPROBE_SYMBOL(process_fetch_insn)
1463
1464/* Kprobe handler */
1465static nokprobe_inline void
1466__kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
1467		    struct trace_event_file *trace_file)
1468{
1469	struct kprobe_trace_entry_head *entry;
1470	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1471	struct trace_event_buffer fbuffer;
1472	int dsize;
1473
1474	WARN_ON(call != trace_file->event_call);
1475
1476	if (trace_trigger_soft_disabled(trace_file))
1477		return;
1478
1479	dsize = __get_data_size(&tk->tp, regs, NULL);
1480
1481	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
1482					   sizeof(*entry) + tk->tp.size + dsize);
1483	if (!entry)
1484		return;
1485
1486	fbuffer.regs = regs;
1487	entry->ip = (unsigned long)tk->rp.kp.addr;
1488	store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);
1489
1490	trace_event_buffer_commit(&fbuffer);
1491}
1492
1493static void
1494kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
1495{
1496	struct event_file_link *link;
1497
1498	trace_probe_for_each_link_rcu(link, &tk->tp)
1499		__kprobe_trace_func(tk, regs, link->file);
1500}
1501NOKPROBE_SYMBOL(kprobe_trace_func);
1502
1503/* Kretprobe handler */
1504
1505static int trace_kprobe_entry_handler(struct kretprobe_instance *ri,
1506				      struct pt_regs *regs)
1507{
1508	struct kretprobe *rp = get_kretprobe(ri);
1509	struct trace_kprobe *tk;
1510
1511	/*
1512	 * There is a small chance that get_kretprobe(ri) returns NULL when
1513	 * the kretprobe is unregister on another CPU between kretprobe's
1514	 * trampoline_handler and this function.
1515	 */
1516	if (unlikely(!rp))
1517		return -ENOENT;
1518
1519	tk = container_of(rp, struct trace_kprobe, rp);
1520
1521	/* store argument values into ri->data as entry data */
1522	if (tk->tp.entry_arg)
1523		store_trace_entry_data(ri->data, &tk->tp, regs);
1524
1525	return 0;
1526}
1527
1528
1529static nokprobe_inline void
1530__kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1531		       struct pt_regs *regs,
1532		       struct trace_event_file *trace_file)
1533{
1534	struct kretprobe_trace_entry_head *entry;
1535	struct trace_event_buffer fbuffer;
1536	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1537	int dsize;
1538
1539	WARN_ON(call != trace_file->event_call);
1540
1541	if (trace_trigger_soft_disabled(trace_file))
1542		return;
1543
1544	dsize = __get_data_size(&tk->tp, regs, ri->data);
1545
1546	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
1547					   sizeof(*entry) + tk->tp.size + dsize);
1548	if (!entry)
1549		return;
1550
1551	fbuffer.regs = regs;
1552	entry->func = (unsigned long)tk->rp.kp.addr;
1553	entry->ret_ip = get_kretprobe_retaddr(ri);
1554	store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize);
1555
1556	trace_event_buffer_commit(&fbuffer);
1557}
1558
1559static void
1560kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1561		     struct pt_regs *regs)
1562{
1563	struct event_file_link *link;
1564
1565	trace_probe_for_each_link_rcu(link, &tk->tp)
1566		__kretprobe_trace_func(tk, ri, regs, link->file);
1567}
1568NOKPROBE_SYMBOL(kretprobe_trace_func);
1569
1570/* Event entry printers */
1571static enum print_line_t
1572print_kprobe_event(struct trace_iterator *iter, int flags,
1573		   struct trace_event *event)
1574{
1575	struct kprobe_trace_entry_head *field;
1576	struct trace_seq *s = &iter->seq;
1577	struct trace_probe *tp;
1578
1579	field = (struct kprobe_trace_entry_head *)iter->ent;
1580	tp = trace_probe_primary_from_call(
1581		container_of(event, struct trace_event_call, event));
1582	if (WARN_ON_ONCE(!tp))
1583		goto out;
1584
1585	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
1586
1587	if (!seq_print_ip_sym_offset(s, field->ip, flags))
1588		goto out;
1589
1590	trace_seq_putc(s, ')');
1591
1592	if (trace_probe_print_args(s, tp->args, tp->nr_args,
1593			     (u8 *)&field[1], field) < 0)
1594		goto out;
1595
1596	trace_seq_putc(s, '\n');
1597 out:
1598	return trace_handle_return(s);
1599}
1600
1601static enum print_line_t
1602print_kretprobe_event(struct trace_iterator *iter, int flags,
1603		      struct trace_event *event)
1604{
1605	struct kretprobe_trace_entry_head *field;
1606	struct trace_seq *s = &iter->seq;
1607	struct trace_probe *tp;
1608
1609	field = (struct kretprobe_trace_entry_head *)iter->ent;
1610	tp = trace_probe_primary_from_call(
1611		container_of(event, struct trace_event_call, event));
1612	if (WARN_ON_ONCE(!tp))
1613		goto out;
1614
1615	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
1616
1617	if (!seq_print_ip_sym_offset(s, field->ret_ip, flags))
1618		goto out;
1619
1620	trace_seq_puts(s, " <- ");
1621
1622	if (!seq_print_ip_sym_no_offset(s, field->func, flags))
1623		goto out;
1624
1625	trace_seq_putc(s, ')');
1626
1627	if (trace_probe_print_args(s, tp->args, tp->nr_args,
1628			     (u8 *)&field[1], field) < 0)
1629		goto out;
1630
1631	trace_seq_putc(s, '\n');
1632
1633 out:
1634	return trace_handle_return(s);
1635}
1636
1637
1638static int kprobe_event_define_fields(struct trace_event_call *event_call)
1639{
1640	int ret;
1641	struct kprobe_trace_entry_head field;
1642	struct trace_probe *tp;
1643
1644	tp = trace_probe_primary_from_call(event_call);
1645	if (WARN_ON_ONCE(!tp))
1646		return -ENOENT;
1647
1648	DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1649
1650	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1651}
1652
1653static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1654{
1655	int ret;
1656	struct kretprobe_trace_entry_head field;
1657	struct trace_probe *tp;
1658
1659	tp = trace_probe_primary_from_call(event_call);
1660	if (WARN_ON_ONCE(!tp))
1661		return -ENOENT;
1662
1663	DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1664	DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1665
1666	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
1667}
1668
1669#ifdef CONFIG_PERF_EVENTS
1670
1671/* Kprobe profile handler */
1672static int
1673kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1674{
1675	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1676	struct kprobe_trace_entry_head *entry;
1677	struct hlist_head *head;
1678	int size, __size, dsize;
1679	int rctx;
1680
1681	if (bpf_prog_array_valid(call)) {
1682		unsigned long orig_ip = instruction_pointer(regs);
1683		int ret;
1684
1685		ret = trace_call_bpf(call, regs);
1686
1687		/*
1688		 * We need to check and see if we modified the pc of the
1689		 * pt_regs, and if so return 1 so that we don't do the
1690		 * single stepping.
1691		 */
1692		if (orig_ip != instruction_pointer(regs))
1693			return 1;
1694		if (!ret)
1695			return 0;
1696	}
1697
1698	head = this_cpu_ptr(call->perf_events);
1699	if (hlist_empty(head))
1700		return 0;
1701
1702	dsize = __get_data_size(&tk->tp, regs, NULL);
1703	__size = sizeof(*entry) + tk->tp.size + dsize;
1704	size = ALIGN(__size + sizeof(u32), sizeof(u64));
1705	size -= sizeof(u32);
1706
1707	entry = perf_trace_buf_alloc(size, NULL, &rctx);
1708	if (!entry)
1709		return 0;
1710
1711	entry->ip = (unsigned long)tk->rp.kp.addr;
1712	memset(&entry[1], 0, dsize);
1713	store_trace_args(&entry[1], &tk->tp, regs, NULL, sizeof(*entry), dsize);
1714	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1715			      head, NULL);
1716	return 0;
1717}
1718NOKPROBE_SYMBOL(kprobe_perf_func);
1719
1720/* Kretprobe profile handler */
1721static void
1722kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1723		    struct pt_regs *regs)
1724{
1725	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1726	struct kretprobe_trace_entry_head *entry;
1727	struct hlist_head *head;
1728	int size, __size, dsize;
1729	int rctx;
1730
1731	if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1732		return;
1733
1734	head = this_cpu_ptr(call->perf_events);
1735	if (hlist_empty(head))
1736		return;
1737
1738	dsize = __get_data_size(&tk->tp, regs, ri->data);
1739	__size = sizeof(*entry) + tk->tp.size + dsize;
1740	size = ALIGN(__size + sizeof(u32), sizeof(u64));
1741	size -= sizeof(u32);
1742
1743	entry = perf_trace_buf_alloc(size, NULL, &rctx);
1744	if (!entry)
1745		return;
1746
1747	entry->func = (unsigned long)tk->rp.kp.addr;
1748	entry->ret_ip = get_kretprobe_retaddr(ri);
1749	store_trace_args(&entry[1], &tk->tp, regs, ri->data, sizeof(*entry), dsize);
1750	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1751			      head, NULL);
1752}
1753NOKPROBE_SYMBOL(kretprobe_perf_func);
1754
1755int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
1756			const char **symbol, u64 *probe_offset,
1757			u64 *probe_addr, unsigned long *missed,
1758			bool perf_type_tracepoint)
1759{
1760	const char *pevent = trace_event_name(event->tp_event);
1761	const char *group = event->tp_event->class->system;
1762	struct trace_kprobe *tk;
1763
1764	if (perf_type_tracepoint)
1765		tk = find_trace_kprobe(pevent, group);
1766	else
1767		tk = trace_kprobe_primary_from_call(event->tp_event);
1768	if (!tk)
1769		return -EINVAL;
1770
1771	*fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE
1772					      : BPF_FD_TYPE_KPROBE;
1773	*probe_offset = tk->rp.kp.offset;
1774	*probe_addr = kallsyms_show_value(current_cred()) ?
1775		      (unsigned long)tk->rp.kp.addr : 0;
1776	*symbol = tk->symbol;
1777	if (missed)
1778		*missed = trace_kprobe_missed(tk);
1779	return 0;
1780}
1781#endif	/* CONFIG_PERF_EVENTS */
1782
1783/*
1784 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1785 *
1786 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1787 * lockless, but we can't race with this __init function.
1788 */
1789static int kprobe_register(struct trace_event_call *event,
1790			   enum trace_reg type, void *data)
1791{
1792	struct trace_event_file *file = data;
1793
1794	switch (type) {
1795	case TRACE_REG_REGISTER:
1796		return enable_trace_kprobe(event, file);
1797	case TRACE_REG_UNREGISTER:
1798		return disable_trace_kprobe(event, file);
1799
1800#ifdef CONFIG_PERF_EVENTS
1801	case TRACE_REG_PERF_REGISTER:
1802		return enable_trace_kprobe(event, NULL);
1803	case TRACE_REG_PERF_UNREGISTER:
1804		return disable_trace_kprobe(event, NULL);
1805	case TRACE_REG_PERF_OPEN:
1806	case TRACE_REG_PERF_CLOSE:
1807	case TRACE_REG_PERF_ADD:
1808	case TRACE_REG_PERF_DEL:
1809		return 0;
1810#endif
1811	}
1812	return 0;
1813}
1814
1815static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1816{
1817	struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1818	unsigned int flags = trace_probe_load_flag(&tk->tp);
1819	int ret = 0;
1820
1821	raw_cpu_inc(*tk->nhit);
1822
1823	if (flags & TP_FLAG_TRACE)
1824		kprobe_trace_func(tk, regs);
1825#ifdef CONFIG_PERF_EVENTS
1826	if (flags & TP_FLAG_PROFILE)
1827		ret = kprobe_perf_func(tk, regs);
1828#endif
1829	return ret;
1830}
1831NOKPROBE_SYMBOL(kprobe_dispatcher);
1832
1833static int
1834kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1835{
1836	struct kretprobe *rp = get_kretprobe(ri);
1837	struct trace_kprobe *tk;
1838	unsigned int flags;
1839
1840	/*
1841	 * There is a small chance that get_kretprobe(ri) returns NULL when
1842	 * the kretprobe is unregister on another CPU between kretprobe's
1843	 * trampoline_handler and this function.
1844	 */
1845	if (unlikely(!rp))
1846		return 0;
1847
1848	tk = container_of(rp, struct trace_kprobe, rp);
1849	raw_cpu_inc(*tk->nhit);
1850
1851	flags = trace_probe_load_flag(&tk->tp);
1852	if (flags & TP_FLAG_TRACE)
1853		kretprobe_trace_func(tk, ri, regs);
1854#ifdef CONFIG_PERF_EVENTS
1855	if (flags & TP_FLAG_PROFILE)
1856		kretprobe_perf_func(tk, ri, regs);
1857#endif
1858	return 0;	/* We don't tweak kernel, so just return 0 */
1859}
1860NOKPROBE_SYMBOL(kretprobe_dispatcher);
1861
1862static struct trace_event_functions kretprobe_funcs = {
1863	.trace		= print_kretprobe_event
1864};
1865
1866static struct trace_event_functions kprobe_funcs = {
1867	.trace		= print_kprobe_event
1868};
1869
1870static struct trace_event_fields kretprobe_fields_array[] = {
1871	{ .type = TRACE_FUNCTION_TYPE,
1872	  .define_fields = kretprobe_event_define_fields },
1873	{}
1874};
1875
1876static struct trace_event_fields kprobe_fields_array[] = {
1877	{ .type = TRACE_FUNCTION_TYPE,
1878	  .define_fields = kprobe_event_define_fields },
1879	{}
1880};
1881
1882static inline void init_trace_event_call(struct trace_kprobe *tk)
1883{
1884	struct trace_event_call *call = trace_probe_event_call(&tk->tp);
1885
1886	if (trace_kprobe_is_return(tk)) {
1887		call->event.funcs = &kretprobe_funcs;
1888		call->class->fields_array = kretprobe_fields_array;
1889	} else {
1890		call->event.funcs = &kprobe_funcs;
1891		call->class->fields_array = kprobe_fields_array;
1892	}
1893
1894	call->flags = TRACE_EVENT_FL_KPROBE;
1895	call->class->reg = kprobe_register;
1896}
1897
1898static int register_kprobe_event(struct trace_kprobe *tk)
1899{
1900	init_trace_event_call(tk);
1901
1902	return trace_probe_register_event_call(&tk->tp);
1903}
1904
1905static int unregister_kprobe_event(struct trace_kprobe *tk)
1906{
1907	return trace_probe_unregister_event_call(&tk->tp);
1908}
1909
1910#ifdef CONFIG_PERF_EVENTS
1911
1912/* create a trace_kprobe, but don't add it to global lists */
1913struct trace_event_call *
1914create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1915			  bool is_return)
1916{
1917	enum probe_print_type ptype;
1918	struct trace_kprobe *tk __free(free_trace_kprobe) = NULL;
1919	int ret;
1920	char *event;
1921
1922	if (func) {
1923		ret = validate_probe_symbol(func);
1924		if (ret)
1925			return ERR_PTR(ret);
1926	}
1927
1928	/*
1929	 * local trace_kprobes are not added to dyn_event, so they are never
1930	 * searched in find_trace_kprobe(). Therefore, there is no concern of
1931	 * duplicated name here.
1932	 */
1933	event = func ? func : "DUMMY_EVENT";
1934
1935	tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1936				offs, 0 /* maxactive */, 0 /* nargs */,
1937				is_return);
1938
1939	if (IS_ERR(tk)) {
1940		pr_info("Failed to allocate trace_probe.(%d)\n",
1941			(int)PTR_ERR(tk));
1942		return ERR_CAST(tk);
1943	}
1944
1945	init_trace_event_call(tk);
1946
1947	ptype = trace_kprobe_is_return(tk) ?
1948		PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1949	if (traceprobe_set_print_fmt(&tk->tp, ptype) < 0)
1950		return ERR_PTR(-ENOMEM);
1951
1952	ret = __register_trace_kprobe(tk);
1953	if (ret < 0)
1954		return ERR_PTR(ret);
1955
1956	return trace_probe_event_call(&(no_free_ptr(tk)->tp));
1957}
1958
1959void destroy_local_trace_kprobe(struct trace_event_call *event_call)
1960{
1961	struct trace_kprobe *tk;
1962
1963	tk = trace_kprobe_primary_from_call(event_call);
1964	if (unlikely(!tk))
1965		return;
1966
1967	if (trace_probe_is_enabled(&tk->tp)) {
1968		WARN_ON(1);
1969		return;
1970	}
1971
1972	__unregister_trace_kprobe(tk);
1973
1974	free_trace_kprobe(tk);
1975}
1976#endif /* CONFIG_PERF_EVENTS */
1977
1978static __init void enable_boot_kprobe_events(void)
1979{
1980	struct trace_array *tr = top_trace_array();
1981	struct trace_event_file *file;
1982	struct trace_kprobe *tk;
1983	struct dyn_event *pos;
1984
1985	guard(mutex)(&event_mutex);
1986	for_each_trace_kprobe(tk, pos) {
1987		list_for_each_entry(file, &tr->events, list)
1988			if (file->event_call == trace_probe_event_call(&tk->tp))
1989				trace_event_enable_disable(file, 1, 0);
1990	}
1991}
1992
1993static __init void setup_boot_kprobe_events(void)
1994{
1995	char *p, *cmd = kprobe_boot_events_buf;
1996	int ret;
1997
1998	strreplace(kprobe_boot_events_buf, ',', ' ');
1999
2000	while (cmd && *cmd != '\0') {
2001		p = strchr(cmd, ';');
2002		if (p)
2003			*p++ = '\0';
2004
2005		ret = create_or_delete_trace_kprobe(cmd);
2006		if (ret)
2007			pr_warn("Failed to add event(%d): %s\n", ret, cmd);
2008
2009		cmd = p;
2010	}
2011
2012	enable_boot_kprobe_events();
2013}
2014
2015/*
2016 * Register dynevent at core_initcall. This allows kernel to setup kprobe
2017 * events in postcore_initcall without tracefs.
2018 */
2019static __init int init_kprobe_trace_early(void)
2020{
2021	int ret;
2022
2023	ret = dyn_event_register(&trace_kprobe_ops);
2024	if (ret)
2025		return ret;
2026
2027	if (trace_kprobe_register_module_notifier())
2028		return -EINVAL;
2029
2030	return 0;
2031}
2032core_initcall(init_kprobe_trace_early);
2033
2034/* Make a tracefs interface for controlling probe points */
2035static __init int init_kprobe_trace(void)
2036{
2037	int ret;
2038
2039	ret = tracing_init_dentry();
2040	if (ret)
2041		return 0;
2042
2043	/* Event list interface */
2044	trace_create_file("kprobe_events", TRACE_MODE_WRITE,
2045			  NULL, NULL, &kprobe_events_ops);
2046
2047	/* Profile interface */
2048	trace_create_file("kprobe_profile", TRACE_MODE_READ,
2049			  NULL, NULL, &kprobe_profile_ops);
2050
2051	setup_boot_kprobe_events();
2052
2053	return 0;
2054}
2055fs_initcall(init_kprobe_trace);
2056
2057
2058#ifdef CONFIG_FTRACE_STARTUP_TEST
2059static __init struct trace_event_file *
2060find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
2061{
2062	struct trace_event_file *file;
2063
2064	list_for_each_entry(file, &tr->events, list)
2065		if (file->event_call == trace_probe_event_call(&tk->tp))
2066			return file;
2067
2068	return NULL;
2069}
2070
2071/*
2072 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
2073 * stage, we can do this lockless.
2074 */
2075static __init int kprobe_trace_self_tests_init(void)
2076{
2077	int ret, warn = 0;
2078	int (*target)(int, int, int, int, int, int);
2079	struct trace_kprobe *tk;
2080	struct trace_event_file *file;
2081
2082	if (tracing_is_disabled())
2083		return -ENODEV;
2084
2085	if (tracing_selftest_disabled)
2086		return 0;
2087
2088	target = kprobe_trace_selftest_target;
2089
2090	pr_info("Testing kprobe tracing: ");
2091
2092	ret = create_or_delete_trace_kprobe("p:testprobe kprobe_trace_selftest_target $stack $stack0 +0($stack)");
2093	if (WARN_ONCE(ret, "error on probing function entry.")) {
2094		warn++;
2095	} else {
2096		/* Enable trace point */
2097		tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2098		if (WARN_ONCE(tk == NULL, "error on probing function entry.")) {
2099			warn++;
2100		} else {
2101			file = find_trace_probe_file(tk, top_trace_array());
2102			if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2103				warn++;
2104			} else
2105				enable_trace_kprobe(
2106					trace_probe_event_call(&tk->tp), file);
2107		}
2108	}
2109
2110	ret = create_or_delete_trace_kprobe("r:testprobe2 kprobe_trace_selftest_target $retval");
2111	if (WARN_ONCE(ret, "error on probing function return.")) {
2112		warn++;
2113	} else {
2114		/* Enable trace point */
2115		tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2116		if (WARN_ONCE(tk == NULL, "error on getting 2nd new probe.")) {
2117			warn++;
2118		} else {
2119			file = find_trace_probe_file(tk, top_trace_array());
2120			if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2121				warn++;
2122			} else
2123				enable_trace_kprobe(
2124					trace_probe_event_call(&tk->tp), file);
2125		}
2126	}
2127
2128	if (warn)
2129		goto end;
2130
2131	ret = target(1, 2, 3, 4, 5, 6);
2132
2133	/*
2134	 * Not expecting an error here, the check is only to prevent the
2135	 * optimizer from removing the call to target() as otherwise there
2136	 * are no side-effects and the call is never performed.
2137	 */
2138	if (ret != 21)
2139		warn++;
2140
2141	/* Disable trace points before removing it */
2142	tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
2143	if (WARN_ONCE(tk == NULL, "error on getting test probe.")) {
2144		warn++;
2145	} else {
2146		if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
2147				 "incorrect number of testprobe hits."))
2148			warn++;
2149
2150		file = find_trace_probe_file(tk, top_trace_array());
2151		if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2152			warn++;
2153		} else
2154			disable_trace_kprobe(
2155				trace_probe_event_call(&tk->tp), file);
2156	}
2157
2158	tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
2159	if (WARN_ONCE(tk == NULL, "error on getting 2nd test probe.")) {
2160		warn++;
2161	} else {
2162		if (WARN_ONCE(trace_kprobe_nhit(tk) != 1,
2163				 "incorrect number of testprobe2 hits."))
2164			warn++;
2165
2166		file = find_trace_probe_file(tk, top_trace_array());
2167		if (WARN_ONCE(file == NULL, "error on getting probe file.")) {
2168			warn++;
2169		} else
2170			disable_trace_kprobe(
2171				trace_probe_event_call(&tk->tp), file);
2172	}
2173
2174	ret = create_or_delete_trace_kprobe("-:testprobe");
2175	if (WARN_ONCE(ret, "error on deleting a probe."))
2176		warn++;
2177
2178	ret = create_or_delete_trace_kprobe("-:testprobe2");
2179	if (WARN_ONCE(ret, "error on deleting a probe."))
2180		warn++;
2181
2182
2183end:
2184	/*
2185	 * Wait for the optimizer work to finish. Otherwise it might fiddle
2186	 * with probes in already freed __init text.
2187	 */
2188	wait_for_kprobe_optimizer();
2189	if (warn)
2190		pr_cont("NG: Some tests are failed. Please check them.\n");
2191	else
2192		pr_cont("OK\n");
2193	return 0;
2194}
2195
2196late_initcall(kprobe_trace_self_tests_init);
2197
2198#endif