trace_fprobe.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Fprobe-based tracing events
   4 * Copyright (C) 2022 Google LLC.
   5 */
   6#define pr_fmt(fmt)	"trace_fprobe: " fmt
   7
   8#include <linux/fprobe.h>
   9#include <linux/list.h>
  10#include <linux/module.h>
  11#include <linux/mutex.h>
  12#include <linux/rculist.h>
  13#include <linux/security.h>
  14#include <linux/tracepoint.h>
  15#include <linux/uaccess.h>
  16
  17#include <asm/ptrace.h>
  18
  19#include "trace_dynevent.h"
  20#include "trace_probe.h"
  21#include "trace_probe_kernel.h"
  22#include "trace_probe_tmpl.h"
  23
  24#define FPROBE_EVENT_SYSTEM "fprobes"
  25#define TRACEPOINT_EVENT_SYSTEM "tracepoints"
  26#define RETHOOK_MAXACTIVE_MAX 4096
  27
  28static int trace_fprobe_create(const char *raw_command);
  29static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev);
  30static int trace_fprobe_release(struct dyn_event *ev);
  31static bool trace_fprobe_is_busy(struct dyn_event *ev);
  32static bool trace_fprobe_match(const char *system, const char *event,
  33			int argc, const char **argv, struct dyn_event *ev);
  34
  35static struct dyn_event_operations trace_fprobe_ops = {
  36	.create = trace_fprobe_create,
  37	.show = trace_fprobe_show,
  38	.is_busy = trace_fprobe_is_busy,
  39	.free = trace_fprobe_release,
  40	.match = trace_fprobe_match,
  41};
  42
  43/* List of tracepoint_user */
  44static LIST_HEAD(tracepoint_user_list);
  45static DEFINE_MUTEX(tracepoint_user_mutex);
  46
  47/* While living tracepoint_user, @tpoint can be NULL and @refcount != 0. */
  48struct tracepoint_user {
  49	struct list_head	list;
  50	const char		*name;
  51	struct tracepoint	*tpoint;
  52	unsigned int		refcount;
  53};
  54
  55/* NOTE: you must lock tracepoint_user_mutex. */
  56#define for_each_tracepoint_user(tuser)		\
  57	list_for_each_entry(tuser, &tracepoint_user_list, list)
  58
  59static int tracepoint_user_register(struct tracepoint_user *tuser)
  60{
  61	struct tracepoint *tpoint = tuser->tpoint;
  62
  63	if (!tpoint)
  64		return 0;
  65
  66	return tracepoint_probe_register_prio_may_exist(tpoint,
  67					tpoint->probestub, NULL, 0);
  68}
  69
  70static void tracepoint_user_unregister(struct tracepoint_user *tuser)
  71{
  72	if (!tuser->tpoint)
  73		return;
  74
  75	WARN_ON_ONCE(tracepoint_probe_unregister(tuser->tpoint, tuser->tpoint->probestub, NULL));
  76	tuser->tpoint = NULL;
  77}
  78
  79static unsigned long tracepoint_user_ip(struct tracepoint_user *tuser)
  80{
  81	if (!tuser->tpoint)
  82		return 0UL;
  83
  84	return (unsigned long)tuser->tpoint->probestub;
  85}
  86
  87static void __tracepoint_user_free(struct tracepoint_user *tuser)
  88{
  89	if (!tuser)
  90		return;
  91	kfree(tuser->name);
  92	kfree(tuser);
  93}
  94
  95DEFINE_FREE(tuser_free, struct tracepoint_user *, __tracepoint_user_free(_T))
  96
  97static struct tracepoint_user *__tracepoint_user_init(const char *name, struct tracepoint *tpoint)
  98{
  99	struct tracepoint_user *tuser __free(tuser_free) = NULL;
 100	int ret;
 101
 102	tuser = kzalloc(sizeof(*tuser), GFP_KERNEL);
 103	if (!tuser)
 104		return NULL;
 105	tuser->name = kstrdup(name, GFP_KERNEL);
 106	if (!tuser->name)
 107		return NULL;
 108
 109	/* Register tracepoint if it is loaded. */
 110	if (tpoint) {
 111		tuser->tpoint = tpoint;
 112		ret = tracepoint_user_register(tuser);
 113		if (ret)
 114			return ERR_PTR(ret);
 115	}
 116
 117	tuser->refcount = 1;
 118	INIT_LIST_HEAD(&tuser->list);
 119	list_add(&tuser->list, &tracepoint_user_list);
 120
 121	return_ptr(tuser);
 122}
 123
 124static struct tracepoint *find_tracepoint(const char *tp_name,
 125	struct module **tp_mod);
 126
 127/*
 128 * Get tracepoint_user if exist, or allocate new one and register it.
 129 * If tracepoint is on a module, get its refcounter too.
 130 * This returns errno or NULL (not loaded yet) or tracepoint_user.
 131 */
 132static struct tracepoint_user *tracepoint_user_find_get(const char *name, struct module **pmod)
 133{
 134	struct module *mod __free(module_put) = NULL;
 135	struct tracepoint_user *tuser;
 136	struct tracepoint *tpoint;
 137
 138	if (!name || !pmod)
 139		return ERR_PTR(-EINVAL);
 140
 141	/* Get and lock the module which has tracepoint. */
 142	tpoint = find_tracepoint(name, &mod);
 143
 144	guard(mutex)(&tracepoint_user_mutex);
 145	/* Search existing tracepoint_user */
 146	for_each_tracepoint_user(tuser) {
 147		if (!strcmp(tuser->name, name)) {
 148			tuser->refcount++;
 149			*pmod = no_free_ptr(mod);
 150			return tuser;
 151		}
 152	}
 153
 154	/* The corresponding tracepoint_user is not found. */
 155	tuser = __tracepoint_user_init(name, tpoint);
 156	if (!IS_ERR_OR_NULL(tuser))
 157		*pmod = no_free_ptr(mod);
 158
 159	return tuser;
 160}
 161
 162static void tracepoint_user_put(struct tracepoint_user *tuser)
 163{
 164	scoped_guard(mutex, &tracepoint_user_mutex) {
 165		if (--tuser->refcount > 0)
 166			return;
 167
 168		list_del(&tuser->list);
 169		tracepoint_user_unregister(tuser);
 170	}
 171
 172	__tracepoint_user_free(tuser);
 173}
 174
 175DEFINE_FREE(tuser_put, struct tracepoint_user *,
 176	if (!IS_ERR_OR_NULL(_T))
 177		tracepoint_user_put(_T))
 178
 179/*
 180 * Fprobe event core functions
 181 */
 182
 183/*
 184 * @tprobe is true for tracepoint probe.
 185 * @tuser can be NULL if the trace_fprobe is disabled or the tracepoint is not
 186 * loaded with a module. If @tuser != NULL, this trace_fprobe is enabled.
 187 */
 188struct trace_fprobe {
 189	struct dyn_event	devent;
 190	struct fprobe		fp;
 191	const char		*symbol;
 192	bool			tprobe;
 193	struct tracepoint_user	*tuser;
 194	struct trace_probe	tp;
 195};
 196
 197static bool is_trace_fprobe(struct dyn_event *ev)
 198{
 199	return ev->ops == &trace_fprobe_ops;
 200}
 201
 202static struct trace_fprobe *to_trace_fprobe(struct dyn_event *ev)
 203{
 204	return container_of(ev, struct trace_fprobe, devent);
 205}
 206
 207/**
 208 * for_each_trace_fprobe - iterate over the trace_fprobe list
 209 * @pos:	the struct trace_fprobe * for each entry
 210 * @dpos:	the struct dyn_event * to use as a loop cursor
 211 */
 212#define for_each_trace_fprobe(pos, dpos)	\
 213	for_each_dyn_event(dpos)		\
 214		if (is_trace_fprobe(dpos) && (pos = to_trace_fprobe(dpos)))
 215
 216static bool trace_fprobe_is_return(struct trace_fprobe *tf)
 217{
 218	return tf->fp.exit_handler != NULL;
 219}
 220
 221static bool trace_fprobe_is_tracepoint(struct trace_fprobe *tf)
 222{
 223	return tf->tprobe;
 224}
 225
 226static const char *trace_fprobe_symbol(struct trace_fprobe *tf)
 227{
 228	return tf->symbol ? tf->symbol : "unknown";
 229}
 230
 231static bool trace_fprobe_is_busy(struct dyn_event *ev)
 232{
 233	struct trace_fprobe *tf = to_trace_fprobe(ev);
 234
 235	return trace_probe_is_enabled(&tf->tp);
 236}
 237
 238static bool trace_fprobe_match_command_head(struct trace_fprobe *tf,
 239					    int argc, const char **argv)
 240{
 241	char buf[MAX_ARGSTR_LEN + 1];
 242
 243	if (!argc)
 244		return true;
 245
 246	snprintf(buf, sizeof(buf), "%s", trace_fprobe_symbol(tf));
 247	if (strcmp(buf, argv[0]))
 248		return false;
 249	argc--; argv++;
 250
 251	return trace_probe_match_command_args(&tf->tp, argc, argv);
 252}
 253
 254static bool trace_fprobe_match(const char *system, const char *event,
 255			int argc, const char **argv, struct dyn_event *ev)
 256{
 257	struct trace_fprobe *tf = to_trace_fprobe(ev);
 258
 259	if (event[0] != '\0' && strcmp(trace_probe_name(&tf->tp), event))
 260		return false;
 261
 262	if (system && strcmp(trace_probe_group_name(&tf->tp), system))
 263		return false;
 264
 265	return trace_fprobe_match_command_head(tf, argc, argv);
 266}
 267
 268static bool trace_fprobe_is_registered(struct trace_fprobe *tf)
 269{
 270	return fprobe_is_registered(&tf->fp);
 271}
 272
 273/*
 274 * Note that we don't verify the fetch_insn code, since it does not come
 275 * from user space.
 276 */
 277static int
 278process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
 279		   void *dest, void *base)
 280{
 281	struct ftrace_regs *fregs = rec;
 282	unsigned long val;
 283	int ret;
 284
 285retry:
 286	/* 1st stage: get value from context */
 287	switch (code->op) {
 288	case FETCH_OP_STACK:
 289		val = ftrace_regs_get_kernel_stack_nth(fregs, code->param);
 290		break;
 291	case FETCH_OP_STACKP:
 292		val = ftrace_regs_get_stack_pointer(fregs);
 293		break;
 294	case FETCH_OP_RETVAL:
 295		val = ftrace_regs_get_return_value(fregs);
 296		break;
 297#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
 298	case FETCH_OP_ARG:
 299		val = ftrace_regs_get_argument(fregs, code->param);
 300		break;
 301	case FETCH_OP_EDATA:
 302		val = *(unsigned long *)((unsigned long)edata + code->offset);
 303		break;
 304#endif
 305	case FETCH_NOP_SYMBOL:	/* Ignore a place holder */
 306		code++;
 307		goto retry;
 308	default:
 309		ret = process_common_fetch_insn(code, &val);
 310		if (ret < 0)
 311			return ret;
 312	}
 313	code++;
 314
 315	return process_fetch_insn_bottom(code, val, dest, base);
 316}
 317NOKPROBE_SYMBOL(process_fetch_insn)
 318
 319/* function entry handler */
 320static nokprobe_inline void
 321__fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
 322		    struct ftrace_regs *fregs,
 323		    struct trace_event_file *trace_file)
 324{
 325	struct fentry_trace_entry_head *entry;
 326	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
 327	struct trace_event_buffer fbuffer;
 328	int dsize;
 329
 330	if (WARN_ON_ONCE(call != trace_file->event_call))
 331		return;
 332
 333	if (trace_trigger_soft_disabled(trace_file))
 334		return;
 335
 336	dsize = __get_data_size(&tf->tp, fregs, NULL);
 337
 338	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 339					   sizeof(*entry) + tf->tp.size + dsize);
 340	if (!entry)
 341		return;
 342
 343	fbuffer.regs = ftrace_get_regs(fregs);
 344	entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
 345	entry->ip = entry_ip;
 346	store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize);
 347
 348	trace_event_buffer_commit(&fbuffer);
 349}
 350
 351static void
 352fentry_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
 353		  struct ftrace_regs *fregs)
 354{
 355	struct event_file_link *link;
 356
 357	trace_probe_for_each_link_rcu(link, &tf->tp)
 358		__fentry_trace_func(tf, entry_ip, fregs, link->file);
 359}
 360NOKPROBE_SYMBOL(fentry_trace_func);
 361
 362static nokprobe_inline
 363void store_fprobe_entry_data(void *edata, struct trace_probe *tp, struct ftrace_regs *fregs)
 364{
 365	struct probe_entry_arg *earg = tp->entry_arg;
 366	unsigned long val = 0;
 367	int i;
 368
 369	if (!earg)
 370		return;
 371
 372	for (i = 0; i < earg->size; i++) {
 373		struct fetch_insn *code = &earg->code[i];
 374
 375		switch (code->op) {
 376		case FETCH_OP_ARG:
 377			val = ftrace_regs_get_argument(fregs, code->param);
 378			break;
 379		case FETCH_OP_ST_EDATA:
 380			*(unsigned long *)((unsigned long)edata + code->offset) = val;
 381			break;
 382		case FETCH_OP_END:
 383			goto end;
 384		default:
 385			break;
 386		}
 387	}
 388end:
 389	return;
 390}
 391
 392/* function exit handler */
 393static int trace_fprobe_entry_handler(struct fprobe *fp, unsigned long entry_ip,
 394				unsigned long ret_ip, struct ftrace_regs *fregs,
 395				void *entry_data)
 396{
 397	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
 398
 399	if (tf->tp.entry_arg)
 400		store_fprobe_entry_data(entry_data, &tf->tp, fregs);
 401
 402	return 0;
 403}
 404NOKPROBE_SYMBOL(trace_fprobe_entry_handler)
 405
 406static nokprobe_inline void
 407__fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
 408		   unsigned long ret_ip, struct ftrace_regs *fregs,
 409		   void *entry_data, struct trace_event_file *trace_file)
 410{
 411	struct fexit_trace_entry_head *entry;
 412	struct trace_event_buffer fbuffer;
 413	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
 414	int dsize;
 415
 416	if (WARN_ON_ONCE(call != trace_file->event_call))
 417		return;
 418
 419	if (trace_trigger_soft_disabled(trace_file))
 420		return;
 421
 422	dsize = __get_data_size(&tf->tp, fregs, entry_data);
 423
 424	entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 425					   sizeof(*entry) + tf->tp.size + dsize);
 426	if (!entry)
 427		return;
 428
 429	fbuffer.regs = ftrace_get_regs(fregs);
 430	entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
 431	entry->func = entry_ip;
 432	entry->ret_ip = ret_ip;
 433	store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize);
 434
 435	trace_event_buffer_commit(&fbuffer);
 436}
 437
 438static void
 439fexit_trace_func(struct trace_fprobe *tf, unsigned long entry_ip,
 440		 unsigned long ret_ip, struct ftrace_regs *fregs, void *entry_data)
 441{
 442	struct event_file_link *link;
 443
 444	trace_probe_for_each_link_rcu(link, &tf->tp)
 445		__fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data, link->file);
 446}
 447NOKPROBE_SYMBOL(fexit_trace_func);
 448
 449#ifdef CONFIG_PERF_EVENTS
 450
 451static int fentry_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
 452			    struct ftrace_regs *fregs)
 453{
 454	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
 455	struct fentry_trace_entry_head *entry;
 456	struct hlist_head *head;
 457	int size, __size, dsize;
 458	struct pt_regs *regs;
 459	int rctx;
 460
 461	head = this_cpu_ptr(call->perf_events);
 462	if (hlist_empty(head))
 463		return 0;
 464
 465	dsize = __get_data_size(&tf->tp, fregs, NULL);
 466	__size = sizeof(*entry) + tf->tp.size + dsize;
 467	size = ALIGN(__size + sizeof(u32), sizeof(u64));
 468	size -= sizeof(u32);
 469
 470	entry = perf_trace_buf_alloc(size, &regs, &rctx);
 471	if (!entry)
 472		return 0;
 473
 474	regs = ftrace_fill_perf_regs(fregs, regs);
 475
 476	entry->ip = entry_ip;
 477	memset(&entry[1], 0, dsize);
 478	store_trace_args(&entry[1], &tf->tp, fregs, NULL, sizeof(*entry), dsize);
 479	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
 480			      head, NULL);
 481	return 0;
 482}
 483NOKPROBE_SYMBOL(fentry_perf_func);
 484
 485static void
 486fexit_perf_func(struct trace_fprobe *tf, unsigned long entry_ip,
 487		unsigned long ret_ip, struct ftrace_regs *fregs,
 488		void *entry_data)
 489{
 490	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
 491	struct fexit_trace_entry_head *entry;
 492	struct hlist_head *head;
 493	int size, __size, dsize;
 494	struct pt_regs *regs;
 495	int rctx;
 496
 497	head = this_cpu_ptr(call->perf_events);
 498	if (hlist_empty(head))
 499		return;
 500
 501	dsize = __get_data_size(&tf->tp, fregs, entry_data);
 502	__size = sizeof(*entry) + tf->tp.size + dsize;
 503	size = ALIGN(__size + sizeof(u32), sizeof(u64));
 504	size -= sizeof(u32);
 505
 506	entry = perf_trace_buf_alloc(size, &regs, &rctx);
 507	if (!entry)
 508		return;
 509
 510	regs = ftrace_fill_perf_regs(fregs, regs);
 511
 512	entry->func = entry_ip;
 513	entry->ret_ip = ret_ip;
 514	store_trace_args(&entry[1], &tf->tp, fregs, entry_data, sizeof(*entry), dsize);
 515	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
 516			      head, NULL);
 517}
 518NOKPROBE_SYMBOL(fexit_perf_func);
 519#endif	/* CONFIG_PERF_EVENTS */
 520
 521static int fentry_dispatcher(struct fprobe *fp, unsigned long entry_ip,
 522			     unsigned long ret_ip, struct ftrace_regs *fregs,
 523			     void *entry_data)
 524{
 525	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
 526	unsigned int flags = trace_probe_load_flag(&tf->tp);
 527	int ret = 0;
 528
 529	if (flags & TP_FLAG_TRACE)
 530		fentry_trace_func(tf, entry_ip, fregs);
 531
 532#ifdef CONFIG_PERF_EVENTS
 533	if (flags & TP_FLAG_PROFILE)
 534		ret = fentry_perf_func(tf, entry_ip, fregs);
 535#endif
 536	return ret;
 537}
 538NOKPROBE_SYMBOL(fentry_dispatcher);
 539
 540static void fexit_dispatcher(struct fprobe *fp, unsigned long entry_ip,
 541			     unsigned long ret_ip, struct ftrace_regs *fregs,
 542			     void *entry_data)
 543{
 544	struct trace_fprobe *tf = container_of(fp, struct trace_fprobe, fp);
 545	unsigned int flags = trace_probe_load_flag(&tf->tp);
 546
 547	if (flags & TP_FLAG_TRACE)
 548		fexit_trace_func(tf, entry_ip, ret_ip, fregs, entry_data);
 549#ifdef CONFIG_PERF_EVENTS
 550	if (flags & TP_FLAG_PROFILE)
 551		fexit_perf_func(tf, entry_ip, ret_ip, fregs, entry_data);
 552#endif
 553}
 554NOKPROBE_SYMBOL(fexit_dispatcher);
 555
 556static void free_trace_fprobe(struct trace_fprobe *tf)
 557{
 558	if (tf) {
 559		trace_probe_cleanup(&tf->tp);
 560		if (tf->tuser)
 561			tracepoint_user_put(tf->tuser);
 562		kfree(tf->symbol);
 563		kfree(tf);
 564	}
 565}
 566
 567/* Since alloc_trace_fprobe() can return error, check the pointer is ERR too. */
 568DEFINE_FREE(free_trace_fprobe, struct trace_fprobe *, if (!IS_ERR_OR_NULL(_T)) free_trace_fprobe(_T))
 569
 570/*
 571 * Allocate new trace_probe and initialize it (including fprobe).
 572 */
 573static struct trace_fprobe *alloc_trace_fprobe(const char *group,
 574					       const char *event,
 575					       const char *symbol,
 576					       int nargs, bool is_return,
 577					       bool is_tracepoint)
 578{
 579	struct trace_fprobe *tf __free(free_trace_fprobe) = NULL;
 580	int ret = -ENOMEM;
 581
 582	tf = kzalloc(struct_size(tf, tp.args, nargs), GFP_KERNEL);
 583	if (!tf)
 584		return ERR_PTR(ret);
 585
 586	tf->symbol = kstrdup(symbol, GFP_KERNEL);
 587	if (!tf->symbol)
 588		return ERR_PTR(-ENOMEM);
 589
 590	if (is_return)
 591		tf->fp.exit_handler = fexit_dispatcher;
 592	else
 593		tf->fp.entry_handler = fentry_dispatcher;
 594
 595	tf->tprobe = is_tracepoint;
 596
 597	ret = trace_probe_init(&tf->tp, event, group, false, nargs);
 598	if (ret < 0)
 599		return ERR_PTR(ret);
 600
 601	dyn_event_init(&tf->devent, &trace_fprobe_ops);
 602	return_ptr(tf);
 603}
 604
 605static struct trace_fprobe *find_trace_fprobe(const char *event,
 606					      const char *group)
 607{
 608	struct dyn_event *pos;
 609	struct trace_fprobe *tf;
 610
 611	for_each_trace_fprobe(tf, pos)
 612		if (strcmp(trace_probe_name(&tf->tp), event) == 0 &&
 613		    strcmp(trace_probe_group_name(&tf->tp), group) == 0)
 614			return tf;
 615	return NULL;
 616}
 617
 618/* Event entry printers */
 619static enum print_line_t
 620print_fentry_event(struct trace_iterator *iter, int flags,
 621		   struct trace_event *event)
 622{
 623	struct fentry_trace_entry_head *field;
 624	struct trace_seq *s = &iter->seq;
 625	struct trace_probe *tp;
 626
 627	field = (struct fentry_trace_entry_head *)iter->ent;
 628	tp = trace_probe_primary_from_call(
 629		container_of(event, struct trace_event_call, event));
 630	if (WARN_ON_ONCE(!tp))
 631		goto out;
 632
 633	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
 634
 635	if (!seq_print_ip_sym_offset(s, field->ip, flags))
 636		goto out;
 637
 638	trace_seq_putc(s, ')');
 639
 640	if (trace_probe_print_args(s, tp->args, tp->nr_args,
 641			     (u8 *)&field[1], field) < 0)
 642		goto out;
 643
 644	trace_seq_putc(s, '\n');
 645 out:
 646	return trace_handle_return(s);
 647}
 648
 649static enum print_line_t
 650print_fexit_event(struct trace_iterator *iter, int flags,
 651		  struct trace_event *event)
 652{
 653	struct fexit_trace_entry_head *field;
 654	struct trace_seq *s = &iter->seq;
 655	struct trace_probe *tp;
 656
 657	field = (struct fexit_trace_entry_head *)iter->ent;
 658	tp = trace_probe_primary_from_call(
 659		container_of(event, struct trace_event_call, event));
 660	if (WARN_ON_ONCE(!tp))
 661		goto out;
 662
 663	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
 664
 665	if (!seq_print_ip_sym_offset(s, field->ret_ip, flags))
 666		goto out;
 667
 668	trace_seq_puts(s, " <- ");
 669
 670	if (!seq_print_ip_sym_no_offset(s, field->func, flags))
 671		goto out;
 672
 673	trace_seq_putc(s, ')');
 674
 675	if (trace_probe_print_args(s, tp->args, tp->nr_args,
 676			     (u8 *)&field[1], field) < 0)
 677		goto out;
 678
 679	trace_seq_putc(s, '\n');
 680
 681 out:
 682	return trace_handle_return(s);
 683}
 684
 685static int fentry_event_define_fields(struct trace_event_call *event_call)
 686{
 687	int ret;
 688	struct fentry_trace_entry_head field;
 689	struct trace_probe *tp;
 690
 691	tp = trace_probe_primary_from_call(event_call);
 692	if (WARN_ON_ONCE(!tp))
 693		return -ENOENT;
 694
 695	DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
 696
 697	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
 698}
 699
 700static int fexit_event_define_fields(struct trace_event_call *event_call)
 701{
 702	int ret;
 703	struct fexit_trace_entry_head field;
 704	struct trace_probe *tp;
 705
 706	tp = trace_probe_primary_from_call(event_call);
 707	if (WARN_ON_ONCE(!tp))
 708		return -ENOENT;
 709
 710	DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
 711	DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
 712
 713	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
 714}
 715
 716static struct trace_event_functions fentry_funcs = {
 717	.trace		= print_fentry_event
 718};
 719
 720static struct trace_event_functions fexit_funcs = {
 721	.trace		= print_fexit_event
 722};
 723
 724static struct trace_event_fields fentry_fields_array[] = {
 725	{ .type = TRACE_FUNCTION_TYPE,
 726	  .define_fields = fentry_event_define_fields },
 727	{}
 728};
 729
 730static struct trace_event_fields fexit_fields_array[] = {
 731	{ .type = TRACE_FUNCTION_TYPE,
 732	  .define_fields = fexit_event_define_fields },
 733	{}
 734};
 735
 736static int fprobe_register(struct trace_event_call *event,
 737			   enum trace_reg type, void *data);
 738
 739static inline void init_trace_event_call(struct trace_fprobe *tf)
 740{
 741	struct trace_event_call *call = trace_probe_event_call(&tf->tp);
 742
 743	if (trace_fprobe_is_return(tf)) {
 744		call->event.funcs = &fexit_funcs;
 745		call->class->fields_array = fexit_fields_array;
 746	} else {
 747		call->event.funcs = &fentry_funcs;
 748		call->class->fields_array = fentry_fields_array;
 749	}
 750
 751	call->flags = TRACE_EVENT_FL_FPROBE;
 752	call->class->reg = fprobe_register;
 753}
 754
 755static int register_fprobe_event(struct trace_fprobe *tf)
 756{
 757	init_trace_event_call(tf);
 758
 759	return trace_probe_register_event_call(&tf->tp);
 760}
 761
 762static int unregister_fprobe_event(struct trace_fprobe *tf)
 763{
 764	return trace_probe_unregister_event_call(&tf->tp);
 765}
 766
 767static int __regsiter_tracepoint_fprobe(struct trace_fprobe *tf)
 768{
 769	struct tracepoint_user *tuser __free(tuser_put) = NULL;
 770	struct module *mod __free(module_put) = NULL;
 771	unsigned long ip;
 772	int ret;
 773
 774	if (WARN_ON_ONCE(tf->tuser))
 775		return -EINVAL;
 776
 777	/* If the tracepoint is in a module, it must be locked in this function. */
 778	tuser = tracepoint_user_find_get(tf->symbol, &mod);
 779	/* This tracepoint is not loaded yet */
 780	if (IS_ERR(tuser))
 781		return PTR_ERR(tuser);
 782	if (!tuser)
 783		return -ENOMEM;
 784
 785	/* Register fprobe only if the tracepoint is loaded. */
 786	if (tuser->tpoint) {
 787		ip = tracepoint_user_ip(tuser);
 788		if (WARN_ON_ONCE(!ip))
 789			return -ENOENT;
 790
 791		ret = register_fprobe_ips(&tf->fp, &ip, 1);
 792		if (ret < 0)
 793			return ret;
 794	}
 795
 796	tf->tuser = no_free_ptr(tuser);
 797	return 0;
 798}
 799
 800/* Returns an error if the target function is not available, or 0 */
 801static int trace_fprobe_verify_target(struct trace_fprobe *tf)
 802{
 803	int ret;
 804
 805	/* Tracepoint should have a stub function. */
 806	if (trace_fprobe_is_tracepoint(tf))
 807		return 0;
 808
 809	/*
 810	 * Note: since we don't lock the module, even if this succeeded,
 811	 * register_fprobe() later can fail.
 812	 */
 813	ret = fprobe_count_ips_from_filter(tf->symbol, NULL);
 814	return (ret < 0) ? ret : 0;
 815}
 816
 817/* Internal register function - just handle fprobe and flags */
 818static int __register_trace_fprobe(struct trace_fprobe *tf)
 819{
 820	int i, ret;
 821
 822	/* Should we need new LOCKDOWN flag for fprobe? */
 823	ret = security_locked_down(LOCKDOWN_KPROBES);
 824	if (ret)
 825		return ret;
 826
 827	if (trace_fprobe_is_registered(tf))
 828		return -EINVAL;
 829
 830	for (i = 0; i < tf->tp.nr_args; i++) {
 831		ret = traceprobe_update_arg(&tf->tp.args[i]);
 832		if (ret)
 833			return ret;
 834	}
 835
 836	tf->fp.flags &= ~FPROBE_FL_DISABLED;
 837
 838	if (trace_fprobe_is_tracepoint(tf))
 839		return __regsiter_tracepoint_fprobe(tf);
 840
 841	/* TODO: handle filter, nofilter or symbol list */
 842	return register_fprobe(&tf->fp, tf->symbol, NULL);
 843}
 844
 845/* Internal unregister function - just handle fprobe and flags */
 846static void __unregister_trace_fprobe(struct trace_fprobe *tf)
 847{
 848	if (trace_fprobe_is_registered(tf))
 849		unregister_fprobe(&tf->fp);
 850	if (tf->tuser) {
 851		tracepoint_user_put(tf->tuser);
 852		tf->tuser = NULL;
 853	}
 854}
 855
 856/* TODO: make this trace_*probe common function */
 857/* Unregister a trace_probe and probe_event */
 858static int unregister_trace_fprobe(struct trace_fprobe *tf)
 859{
 860	/* If other probes are on the event, just unregister fprobe */
 861	if (trace_probe_has_sibling(&tf->tp))
 862		goto unreg;
 863
 864	/* Enabled event can not be unregistered */
 865	if (trace_probe_is_enabled(&tf->tp))
 866		return -EBUSY;
 867
 868	/* If there's a reference to the dynamic event */
 869	if (trace_event_dyn_busy(trace_probe_event_call(&tf->tp)))
 870		return -EBUSY;
 871
 872	/* Will fail if probe is being used by ftrace or perf */
 873	if (unregister_fprobe_event(tf))
 874		return -EBUSY;
 875
 876unreg:
 877	__unregister_trace_fprobe(tf);
 878	dyn_event_remove(&tf->devent);
 879	trace_probe_unlink(&tf->tp);
 880
 881	return 0;
 882}
 883
 884static bool trace_fprobe_has_same_fprobe(struct trace_fprobe *orig,
 885					 struct trace_fprobe *comp)
 886{
 887	struct trace_probe_event *tpe = orig->tp.event;
 888	int i;
 889
 890	list_for_each_entry(orig, &tpe->probes, tp.list) {
 891		if (strcmp(trace_fprobe_symbol(orig),
 892			   trace_fprobe_symbol(comp)))
 893			continue;
 894
 895		/*
 896		 * trace_probe_compare_arg_type() ensured that nr_args and
 897		 * each argument name and type are same. Let's compare comm.
 898		 */
 899		for (i = 0; i < orig->tp.nr_args; i++) {
 900			if (strcmp(orig->tp.args[i].comm,
 901				   comp->tp.args[i].comm))
 902				break;
 903		}
 904
 905		if (i == orig->tp.nr_args)
 906			return true;
 907	}
 908
 909	return false;
 910}
 911
 912static int append_trace_fprobe_event(struct trace_fprobe *tf, struct trace_fprobe *to)
 913{
 914	int ret;
 915
 916	if (trace_fprobe_is_return(tf) != trace_fprobe_is_return(to) ||
 917	    trace_fprobe_is_tracepoint(tf) != trace_fprobe_is_tracepoint(to)) {
 918		trace_probe_log_set_index(0);
 919		trace_probe_log_err(0, DIFF_PROBE_TYPE);
 920		return -EEXIST;
 921	}
 922	ret = trace_probe_compare_arg_type(&tf->tp, &to->tp);
 923	if (ret) {
 924		/* Note that argument starts index = 2 */
 925		trace_probe_log_set_index(ret + 1);
 926		trace_probe_log_err(0, DIFF_ARG_TYPE);
 927		return -EEXIST;
 928	}
 929	if (trace_fprobe_has_same_fprobe(to, tf)) {
 930		trace_probe_log_set_index(0);
 931		trace_probe_log_err(0, SAME_PROBE);
 932		return -EEXIST;
 933	}
 934
 935	/* Append to existing event */
 936	ret = trace_probe_append(&tf->tp, &to->tp);
 937	if (ret)
 938		return ret;
 939
 940	ret = trace_fprobe_verify_target(tf);
 941	if (ret)
 942		trace_probe_unlink(&tf->tp);
 943	else
 944		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
 945
 946	return ret;
 947}
 948
 949/* Register a trace_probe and probe_event, and check the fprobe is available. */
 950static int register_trace_fprobe_event(struct trace_fprobe *tf)
 951{
 952	struct trace_fprobe *old_tf;
 953	int ret;
 954
 955	guard(mutex)(&event_mutex);
 956
 957	old_tf = find_trace_fprobe(trace_probe_name(&tf->tp),
 958				   trace_probe_group_name(&tf->tp));
 959	if (old_tf)
 960		return append_trace_fprobe_event(tf, old_tf);
 961
 962	/* Register new event */
 963	ret = register_fprobe_event(tf);
 964	if (ret) {
 965		if (ret == -EEXIST) {
 966			trace_probe_log_set_index(0);
 967			trace_probe_log_err(0, EVENT_EXIST);
 968		} else
 969			pr_warn("Failed to register probe event(%d)\n", ret);
 970		return ret;
 971	}
 972
 973	/* Verify fprobe is sane. */
 974	ret = trace_fprobe_verify_target(tf);
 975	if (ret < 0)
 976		unregister_fprobe_event(tf);
 977	else
 978		dyn_event_add(&tf->devent, trace_probe_event_call(&tf->tp));
 979
 980	return ret;
 981}
 982
 983struct __find_tracepoint_cb_data {
 984	const char *tp_name;
 985	struct tracepoint *tpoint;
 986	struct module *mod;
 987};
 988
 989static void __find_tracepoint_module_cb(struct tracepoint *tp, struct module *mod, void *priv)
 990{
 991	struct __find_tracepoint_cb_data *data = priv;
 992
 993	if (!data->tpoint && !strcmp(data->tp_name, tp->name)) {
 994		/* If module is not specified, try getting module refcount. */
 995		if (!data->mod && mod) {
 996			/* If failed to get refcount, ignore this tracepoint. */
 997			if (!try_module_get(mod))
 998				return;
 999
1000			data->mod = mod;
1001		}
1002		data->tpoint = tp;
1003	}
1004}
1005
1006static void __find_tracepoint_cb(struct tracepoint *tp, void *priv)
1007{
1008	struct __find_tracepoint_cb_data *data = priv;
1009
1010	if (!data->tpoint && !strcmp(data->tp_name, tp->name))
1011		data->tpoint = tp;
1012}
1013
1014/*
1015 * Find a tracepoint from kernel and module. If the tracepoint is on the module,
1016 * the module's refcount is incremented and returned as *@tp_mod. Thus, if it is
1017 * not NULL, caller must call module_put(*tp_mod) after used the tracepoint.
1018 */
1019static struct tracepoint *find_tracepoint(const char *tp_name,
1020					  struct module **tp_mod)
1021{
1022	struct __find_tracepoint_cb_data data = {
1023		.tp_name = tp_name,
1024		.mod = NULL,
1025	};
1026
1027	for_each_kernel_tracepoint(__find_tracepoint_cb, &data);
1028
1029	if (!data.tpoint && IS_ENABLED(CONFIG_MODULES)) {
1030		for_each_module_tracepoint(__find_tracepoint_module_cb, &data);
1031		*tp_mod = data.mod;
1032	}
1033
1034	return data.tpoint;
1035}
1036
1037#ifdef CONFIG_MODULES
1038/*
1039 * Find a tracepoint from specified module. In this case, this does not get the
1040 * module's refcount. The caller must ensure the module is not freed.
1041 */
1042static struct tracepoint *find_tracepoint_in_module(struct module *mod,
1043						    const char *tp_name)
1044{
1045	struct __find_tracepoint_cb_data data = {
1046		.tp_name = tp_name,
1047		.mod = mod,
1048	};
1049
1050	for_each_tracepoint_in_module(mod, __find_tracepoint_module_cb, &data);
1051	return data.tpoint;
1052}
1053
1054/* These are CONFIG_MODULES=y specific functions. */
1055static bool tracepoint_user_within_module(struct tracepoint_user *tuser,
1056					  struct module *mod)
1057{
1058	return within_module(tracepoint_user_ip(tuser), mod);
1059}
1060
1061static int tracepoint_user_register_again(struct tracepoint_user *tuser,
1062					  struct tracepoint *tpoint)
1063{
1064	tuser->tpoint = tpoint;
1065	return tracepoint_user_register(tuser);
1066}
1067
1068static void tracepoint_user_unregister_clear(struct tracepoint_user *tuser)
1069{
1070	tracepoint_user_unregister(tuser);
1071	tuser->tpoint = NULL;
1072}
1073
1074/* module callback for tracepoint_user */
1075static int __tracepoint_probe_module_cb(struct notifier_block *self,
1076					unsigned long val, void *data)
1077{
1078	struct tp_module *tp_mod = data;
1079	struct tracepoint_user *tuser;
1080	struct tracepoint *tpoint;
1081
1082	if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING)
1083		return NOTIFY_DONE;
1084
1085	mutex_lock(&tracepoint_user_mutex);
1086	for_each_tracepoint_user(tuser) {
1087		if (val == MODULE_STATE_COMING) {
1088			/* This is not a tracepoint in this module. Skip it. */
1089			tpoint = find_tracepoint_in_module(tp_mod->mod, tuser->name);
1090			if (!tpoint)
1091				continue;
1092			WARN_ON_ONCE(tracepoint_user_register_again(tuser, tpoint));
1093		} else if (val == MODULE_STATE_GOING &&
1094			  tracepoint_user_within_module(tuser, tp_mod->mod)) {
1095			/* Unregister all tracepoint_user in this module. */
1096			tracepoint_user_unregister_clear(tuser);
1097		}
1098	}
1099	mutex_unlock(&tracepoint_user_mutex);
1100
1101	return NOTIFY_DONE;
1102}
1103
1104static struct notifier_block tracepoint_module_nb = {
1105	.notifier_call = __tracepoint_probe_module_cb,
1106};
1107
1108/* module callback for tprobe events */
1109static int __tprobe_event_module_cb(struct notifier_block *self,
1110				     unsigned long val, void *data)
1111{
1112	struct trace_fprobe *tf;
1113	struct dyn_event *pos;
1114	struct module *mod = data;
1115
1116	if (val != MODULE_STATE_GOING && val != MODULE_STATE_COMING)
1117		return NOTIFY_DONE;
1118
1119	mutex_lock(&event_mutex);
1120	for_each_trace_fprobe(tf, pos) {
1121		/* Skip fprobe and disabled tprobe events. */
1122		if (!trace_fprobe_is_tracepoint(tf) || !tf->tuser)
1123			continue;
1124
1125		/* Before this notification, tracepoint notifier has already done. */
1126		if (val == MODULE_STATE_COMING &&
1127		    tracepoint_user_within_module(tf->tuser, mod)) {
1128			unsigned long ip = tracepoint_user_ip(tf->tuser);
1129
1130			WARN_ON_ONCE(register_fprobe_ips(&tf->fp, &ip, 1));
1131		} else if (val == MODULE_STATE_GOING &&
1132			   /*
1133			    * tracepoint_user_within_module() does not work here because
1134			    * tracepoint_user is already unregistered and cleared tpoint.
1135			    * Instead, checking whether the fprobe is registered but
1136			    * tpoint is cleared(unregistered). Such unbalance probes
1137			    * must be adjusted anyway.
1138			    */
1139			    trace_fprobe_is_registered(tf) &&
1140			    !tf->tuser->tpoint) {
1141			unregister_fprobe(&tf->fp);
1142		}
1143	}
1144	mutex_unlock(&event_mutex);
1145
1146	return NOTIFY_DONE;
1147}
1148
1149/* NOTE: this must be called after tracepoint callback */
1150static struct notifier_block tprobe_event_module_nb = {
1151	.notifier_call = __tprobe_event_module_cb,
1152	/* Make sure this is later than tracepoint module notifier. */
1153	.priority = -10,
1154};
1155#endif /* CONFIG_MODULES */
1156
1157static int parse_symbol_and_return(int argc, const char *argv[],
1158				   char **symbol, bool *is_return,
1159				   bool is_tracepoint)
1160{
1161	char *tmp = strchr(argv[1], '%');
1162	int i;
1163
1164	if (tmp) {
1165		int len = tmp - argv[1];
1166
1167		if (!is_tracepoint && !strcmp(tmp, "%return")) {
1168			*is_return = true;
1169		} else {
1170			trace_probe_log_err(len, BAD_ADDR_SUFFIX);
1171			return -EINVAL;
1172		}
1173		*symbol = kmemdup_nul(argv[1], len, GFP_KERNEL);
1174	} else
1175		*symbol = kstrdup(argv[1], GFP_KERNEL);
1176	if (!*symbol)
1177		return -ENOMEM;
1178
1179	if (*is_return)
1180		return 0;
1181
1182	if (is_tracepoint) {
1183		tmp = *symbol;
1184		while (*tmp && (isalnum(*tmp) || *tmp == '_'))
1185			tmp++;
1186		if (*tmp) {
1187			/* find a wrong character. */
1188			trace_probe_log_err(tmp - *symbol, BAD_TP_NAME);
1189			kfree(*symbol);
1190			*symbol = NULL;
1191			return -EINVAL;
1192		}
1193	}
1194
1195	/* If there is $retval, this should be a return fprobe. */
1196	for (i = 2; i < argc; i++) {
1197		tmp = strstr(argv[i], "$retval");
1198		if (tmp && !isalnum(tmp[7]) && tmp[7] != '_') {
1199			if (is_tracepoint) {
1200				trace_probe_log_set_index(i);
1201				trace_probe_log_err(tmp - argv[i], RETVAL_ON_PROBE);
1202				kfree(*symbol);
1203				*symbol = NULL;
1204				return -EINVAL;
1205			}
1206			*is_return = true;
1207			break;
1208		}
1209	}
1210	return 0;
1211}
1212
1213static int trace_fprobe_create_internal(int argc, const char *argv[],
1214					struct traceprobe_parse_context *ctx)
1215{
1216	/*
1217	 * Argument syntax:
1218	 *  - Add fentry probe:
1219	 *      f[:[GRP/][EVENT]] [MOD:]KSYM [FETCHARGS]
1220	 *  - Add fexit probe:
1221	 *      f[N][:[GRP/][EVENT]] [MOD:]KSYM%return [FETCHARGS]
1222	 *  - Add tracepoint probe:
1223	 *      t[:[GRP/][EVENT]] TRACEPOINT [FETCHARGS]
1224	 *
1225	 * Fetch args:
1226	 *  $retval	: fetch return value
1227	 *  $stack	: fetch stack address
1228	 *  $stackN	: fetch Nth entry of stack (N:0-)
1229	 *  $argN	: fetch Nth argument (N:1-)
1230	 *  $comm       : fetch current task comm
1231	 *  @ADDR	: fetch memory at ADDR (ADDR should be in kernel)
1232	 *  @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
1233	 * Dereferencing memory fetch:
1234	 *  +|-offs(ARG) : fetch memory at ARG +|- offs address.
1235	 * Alias name of args:
1236	 *  NAME=FETCHARG : set NAME as alias of FETCHARG.
1237	 * Type of args:
1238	 *  FETCHARG:TYPE : use TYPE instead of unsigned long.
1239	 */
1240	struct trace_fprobe *tf __free(free_trace_fprobe) = NULL;
1241	const char *event = NULL, *group = FPROBE_EVENT_SYSTEM;
1242	struct module *mod __free(module_put) = NULL;
1243	const char **new_argv __free(kfree) = NULL;
1244	char *symbol __free(kfree) = NULL;
1245	char *ebuf __free(kfree) = NULL;
1246	char *gbuf __free(kfree) = NULL;
1247	char *sbuf __free(kfree) = NULL;
1248	char *abuf __free(kfree) = NULL;
1249	char *dbuf __free(kfree) = NULL;
1250	int i, new_argc = 0, ret = 0;
1251	bool is_tracepoint = false;
1252	bool is_return = false;
1253
1254	if ((argv[0][0] != 'f' && argv[0][0] != 't') || argc < 2)
1255		return -ECANCELED;
1256
1257	if (argv[0][0] == 't') {
1258		is_tracepoint = true;
1259		group = TRACEPOINT_EVENT_SYSTEM;
1260	}
1261
1262	if (argv[0][1] != '\0') {
1263		if (argv[0][1] != ':') {
1264			trace_probe_log_set_index(0);
1265			trace_probe_log_err(1, BAD_MAXACT);
1266			return -EINVAL;
1267		}
1268		event = &argv[0][2];
1269	}
1270
1271	trace_probe_log_set_index(1);
1272
1273	/* a symbol(or tracepoint) must be specified */
1274	ret = parse_symbol_and_return(argc, argv, &symbol, &is_return, is_tracepoint);
1275	if (ret < 0)
1276		return -EINVAL;
1277
1278	trace_probe_log_set_index(0);
1279	if (event) {
1280		gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
1281		if (!gbuf)
1282			return -ENOMEM;
1283		ret = traceprobe_parse_event_name(&event, &group, gbuf,
1284						  event - argv[0]);
1285		if (ret)
1286			return -EINVAL;
1287	}
1288
1289	if (!event) {
1290		ebuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
1291		if (!ebuf)
1292			return -ENOMEM;
1293		/* Make a new event name */
1294		if (is_tracepoint)
1295			snprintf(ebuf, MAX_EVENT_NAME_LEN, "%s%s",
1296				 isdigit(*symbol) ? "_" : "", symbol);
1297		else
1298			snprintf(ebuf, MAX_EVENT_NAME_LEN, "%s__%s", symbol,
1299				 is_return ? "exit" : "entry");
1300		sanitize_event_name(ebuf);
1301		event = ebuf;
1302	}
1303
1304	if (is_return)
1305		ctx->flags |= TPARG_FL_RETURN;
1306	else
1307		ctx->flags |= TPARG_FL_FENTRY;
1308
1309	ctx->funcname = NULL;
1310	if (is_tracepoint) {
1311		/* Get tracepoint and lock its module until the end of the registration. */
1312		struct tracepoint *tpoint;
1313
1314		ctx->flags |= TPARG_FL_TPOINT;
1315		mod = NULL;
1316		tpoint = find_tracepoint(symbol, &mod);
1317		if (tpoint) {
1318			sbuf = kmalloc(KSYM_NAME_LEN, GFP_KERNEL);
1319			if (!sbuf)
1320				return -ENOMEM;
1321			ctx->funcname = kallsyms_lookup((unsigned long)tpoint->probestub,
1322							NULL, NULL, NULL, sbuf);
1323		}
1324	}
1325	if (!ctx->funcname)
1326		ctx->funcname = symbol;
1327
1328	abuf = kmalloc(MAX_BTF_ARGS_LEN, GFP_KERNEL);
1329	if (!abuf)
1330		return -ENOMEM;
1331	argc -= 2; argv += 2;
1332	new_argv = traceprobe_expand_meta_args(argc, argv, &new_argc,
1333					       abuf, MAX_BTF_ARGS_LEN, ctx);
1334	if (IS_ERR(new_argv))
1335		return PTR_ERR(new_argv);
1336	if (new_argv) {
1337		argc = new_argc;
1338		argv = new_argv;
1339	}
1340	if (argc > MAX_TRACE_ARGS) {
1341		trace_probe_log_set_index(2);
1342		trace_probe_log_err(0, TOO_MANY_ARGS);
1343		return -E2BIG;
1344	}
1345
1346	ret = traceprobe_expand_dentry_args(argc, argv, &dbuf);
1347	if (ret)
1348		return ret;
1349
1350	/* setup a probe */
1351	tf = alloc_trace_fprobe(group, event, symbol, argc, is_return, is_tracepoint);
1352	if (IS_ERR(tf)) {
1353		ret = PTR_ERR(tf);
1354		/* This must return -ENOMEM, else there is a bug */
1355		WARN_ON_ONCE(ret != -ENOMEM);
1356		return ret;
1357	}
1358
1359	/* parse arguments */
1360	for (i = 0; i < argc; i++) {
1361		trace_probe_log_set_index(i + 2);
1362		ctx->offset = 0;
1363		ret = traceprobe_parse_probe_arg(&tf->tp, i, argv[i], ctx);
1364		if (ret)
1365			return ret;	/* This can be -ENOMEM */
1366	}
1367
1368	if (is_return && tf->tp.entry_arg) {
1369		tf->fp.entry_handler = trace_fprobe_entry_handler;
1370		tf->fp.entry_data_size = traceprobe_get_entry_data_size(&tf->tp);
1371		if (ALIGN(tf->fp.entry_data_size, sizeof(long)) > MAX_FPROBE_DATA_SIZE) {
1372			trace_probe_log_set_index(2);
1373			trace_probe_log_err(0, TOO_MANY_EARGS);
1374			return -E2BIG;
1375		}
1376	}
1377
1378	ret = traceprobe_set_print_fmt(&tf->tp,
1379			is_return ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL);
1380	if (ret < 0)
1381		return ret;
1382
1383	ret = register_trace_fprobe_event(tf);
1384	if (ret) {
1385		trace_probe_log_set_index(1);
1386		if (ret == -EILSEQ)
1387			trace_probe_log_err(0, BAD_INSN_BNDRY);
1388		else if (ret == -ENOENT)
1389			trace_probe_log_err(0, BAD_PROBE_ADDR);
1390		else if (ret != -ENOMEM && ret != -EEXIST)
1391			trace_probe_log_err(0, FAIL_REG_PROBE);
1392		return -EINVAL;
1393	}
1394
1395	/* 'tf' is successfully registered. To avoid freeing, assign NULL. */
1396	tf = NULL;
1397
1398	return 0;
1399}
1400
1401static int trace_fprobe_create_cb(int argc, const char *argv[])
1402{
1403	struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
1404	int ret;
1405
1406	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
1407	if (!ctx)
1408		return -ENOMEM;
1409
1410	ctx->flags = TPARG_FL_KERNEL | TPARG_FL_FPROBE;
1411
1412	trace_probe_log_init("trace_fprobe", argc, argv);
1413	ret = trace_fprobe_create_internal(argc, argv, ctx);
1414	trace_probe_log_clear();
1415	return ret;
1416}
1417
1418static int trace_fprobe_create(const char *raw_command)
1419{
1420	return trace_probe_create(raw_command, trace_fprobe_create_cb);
1421}
1422
1423static int trace_fprobe_release(struct dyn_event *ev)
1424{
1425	struct trace_fprobe *tf = to_trace_fprobe(ev);
1426	int ret = unregister_trace_fprobe(tf);
1427
1428	if (!ret)
1429		free_trace_fprobe(tf);
1430	return ret;
1431}
1432
1433static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
1434{
1435	struct trace_fprobe *tf = to_trace_fprobe(ev);
1436	int i;
1437
1438	if (trace_fprobe_is_tracepoint(tf))
1439		seq_putc(m, 't');
1440	else
1441		seq_putc(m, 'f');
1442	seq_printf(m, ":%s/%s", trace_probe_group_name(&tf->tp),
1443				trace_probe_name(&tf->tp));
1444
1445	seq_printf(m, " %s%s", trace_fprobe_symbol(tf),
1446			       trace_fprobe_is_return(tf) ? "%return" : "");
1447
1448	for (i = 0; i < tf->tp.nr_args; i++)
1449		seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
1450	seq_putc(m, '\n');
1451
1452	return 0;
1453}
1454
1455/*
1456 * Enable trace_probe
1457 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
1458 */
1459static int enable_trace_fprobe(struct trace_event_call *call,
1460			       struct trace_event_file *file)
1461{
1462	struct trace_probe *tp;
1463	struct trace_fprobe *tf;
1464	bool enabled;
1465	int ret = 0;
1466
1467	tp = trace_probe_primary_from_call(call);
1468	if (WARN_ON_ONCE(!tp))
1469		return -ENODEV;
1470	enabled = trace_probe_is_enabled(tp);
1471
1472	/* This also changes "enabled" state */
1473	if (file) {
1474		ret = trace_probe_add_file(tp, file);
1475		if (ret)
1476			return ret;
1477	} else
1478		trace_probe_set_flag(tp, TP_FLAG_PROFILE);
1479
1480	if (!enabled) {
1481		list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
1482			ret = __register_trace_fprobe(tf);
1483			if (ret < 0)
1484				return ret;
1485		}
1486	}
1487
1488	return 0;
1489}
1490
1491/*
1492 * Disable trace_probe
1493 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
1494 */
1495static int disable_trace_fprobe(struct trace_event_call *call,
1496				struct trace_event_file *file)
1497{
1498	struct trace_fprobe *tf;
1499	struct trace_probe *tp;
1500
1501	tp = trace_probe_primary_from_call(call);
1502	if (WARN_ON_ONCE(!tp))
1503		return -ENODEV;
1504
1505	if (file) {
1506		if (!trace_probe_get_file_link(tp, file))
1507			return -ENOENT;
1508		if (!trace_probe_has_single_file(tp))
1509			goto out;
1510		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
1511	} else
1512		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1513
1514	if (!trace_probe_is_enabled(tp)) {
1515		list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) {
1516			unregister_fprobe(&tf->fp);
1517			if (tf->tuser) {
1518				tracepoint_user_put(tf->tuser);
1519				tf->tuser = NULL;
1520			}
1521		}
1522	}
1523
1524 out:
1525	if (file)
1526		/*
1527		 * Synchronization is done in below function. For perf event,
1528		 * file == NULL and perf_trace_event_unreg() calls
1529		 * tracepoint_synchronize_unregister() to ensure synchronize
1530		 * event. We don't need to care about it.
1531		 */
1532		trace_probe_remove_file(tp, file);
1533
1534	return 0;
1535}
1536
1537/*
1538 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1539 */
1540static int fprobe_register(struct trace_event_call *event,
1541			   enum trace_reg type, void *data)
1542{
1543	struct trace_event_file *file = data;
1544
1545	switch (type) {
1546	case TRACE_REG_REGISTER:
1547		return enable_trace_fprobe(event, file);
1548	case TRACE_REG_UNREGISTER:
1549		return disable_trace_fprobe(event, file);
1550
1551#ifdef CONFIG_PERF_EVENTS
1552	case TRACE_REG_PERF_REGISTER:
1553		return enable_trace_fprobe(event, NULL);
1554	case TRACE_REG_PERF_UNREGISTER:
1555		return disable_trace_fprobe(event, NULL);
1556	case TRACE_REG_PERF_OPEN:
1557	case TRACE_REG_PERF_CLOSE:
1558	case TRACE_REG_PERF_ADD:
1559	case TRACE_REG_PERF_DEL:
1560		return 0;
1561#endif
1562	}
1563	return 0;
1564}
1565
1566/*
1567 * Register dynevent at core_initcall. This allows kernel to setup fprobe
1568 * events in postcore_initcall without tracefs.
1569 */
1570static __init int init_fprobe_trace_early(void)
1571{
1572	int ret;
1573
1574	ret = dyn_event_register(&trace_fprobe_ops);
1575	if (ret)
1576		return ret;
1577
1578#ifdef CONFIG_MODULES
1579	ret = register_tracepoint_module_notifier(&tracepoint_module_nb);
1580	if (ret)
1581		return ret;
1582	ret = register_module_notifier(&tprobe_event_module_nb);
1583	if (ret)
1584		return ret;
1585#endif
1586
1587	return 0;
1588}
1589core_initcall(init_fprobe_trace_early);