trace_uprobe.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * uprobes-based tracing events
   4 *
   5 * Copyright (C) IBM Corporation, 2010-2012
   6 * Author:	Srikar Dronamraju <srikar@linux.vnet.ibm.com>
   7 */
   8#define pr_fmt(fmt)	"trace_uprobe: " fmt
   9
  10#include <linux/bpf-cgroup.h>
  11#include <linux/cleanup.h>
  12#include <linux/ctype.h>
  13#include <linux/filter.h>
  14#include <linux/module.h>
  15#include <linux/namei.h>
  16#include <linux/percpu.h>
  17#include <linux/rculist.h>
  18#include <linux/security.h>
  19#include <linux/string.h>
  20#include <linux/uaccess.h>
  21#include <linux/uprobes.h>
  22
  23#include "trace.h"
  24#include "trace_dynevent.h"
  25#include "trace_probe.h"
  26#include "trace_probe_tmpl.h"
  27
  28#define UPROBE_EVENT_SYSTEM	"uprobes"
  29
  30struct uprobe_trace_entry_head {
  31	struct trace_entry	ent;
  32	unsigned long		vaddr[];
  33};
  34
  35#define SIZEOF_TRACE_ENTRY(is_return)			\
  36	(sizeof(struct uprobe_trace_entry_head) +	\
  37	 sizeof(unsigned long) * (is_return ? 2 : 1))
  38
  39#define DATAOF_TRACE_ENTRY(entry, is_return)		\
  40	((void*)(entry) + SIZEOF_TRACE_ENTRY(is_return))
  41
  42static int trace_uprobe_create(const char *raw_command);
  43static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev);
  44static int trace_uprobe_release(struct dyn_event *ev);
  45static bool trace_uprobe_is_busy(struct dyn_event *ev);
  46static bool trace_uprobe_match(const char *system, const char *event,
  47			int argc, const char **argv, struct dyn_event *ev);
  48
  49static struct dyn_event_operations trace_uprobe_ops = {
  50	.create = trace_uprobe_create,
  51	.show = trace_uprobe_show,
  52	.is_busy = trace_uprobe_is_busy,
  53	.free = trace_uprobe_release,
  54	.match = trace_uprobe_match,
  55};
  56
  57/*
  58 * uprobe event core functions
  59 */
  60struct trace_uprobe {
  61	struct dyn_event		devent;
  62	struct uprobe_consumer		consumer;
  63	struct path			path;
  64	char				*filename;
  65	struct uprobe			*uprobe;
  66	unsigned long			offset;
  67	unsigned long			ref_ctr_offset;
  68	unsigned long __percpu		*nhits;
  69	struct trace_probe		tp;
  70};
  71
  72static bool is_trace_uprobe(struct dyn_event *ev)
  73{
  74	return ev->ops == &trace_uprobe_ops;
  75}
  76
  77static struct trace_uprobe *to_trace_uprobe(struct dyn_event *ev)
  78{
  79	return container_of(ev, struct trace_uprobe, devent);
  80}
  81
  82/**
  83 * for_each_trace_uprobe - iterate over the trace_uprobe list
  84 * @pos:	the struct trace_uprobe * for each entry
  85 * @dpos:	the struct dyn_event * to use as a loop cursor
  86 */
  87#define for_each_trace_uprobe(pos, dpos)	\
  88	for_each_dyn_event(dpos)		\
  89		if (is_trace_uprobe(dpos) && (pos = to_trace_uprobe(dpos)))
  90
  91static int register_uprobe_event(struct trace_uprobe *tu);
  92static int unregister_uprobe_event(struct trace_uprobe *tu);
  93
  94static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs,
  95			     __u64 *data);
  96static int uretprobe_dispatcher(struct uprobe_consumer *con,
  97				unsigned long func, struct pt_regs *regs,
  98				__u64 *data);
  99
 100#ifdef CONFIG_STACK_GROWSUP
 101static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
 102{
 103	return addr - (n * sizeof(long));
 104}
 105#else
 106static unsigned long adjust_stack_addr(unsigned long addr, unsigned int n)
 107{
 108	return addr + (n * sizeof(long));
 109}
 110#endif
 111
 112static unsigned long get_user_stack_nth(struct pt_regs *regs, unsigned int n)
 113{
 114	unsigned long ret;
 115	unsigned long addr = user_stack_pointer(regs);
 116
 117	addr = adjust_stack_addr(addr, n);
 118
 119	if (copy_from_user(&ret, (void __force __user *) addr, sizeof(ret)))
 120		return 0;
 121
 122	return ret;
 123}
 124
 125/*
 126 * Uprobes-specific fetch functions
 127 */
 128static nokprobe_inline int
 129probe_mem_read(void *dest, void *src, size_t size)
 130{
 131	void __user *vaddr = (void __force __user *)src;
 132
 133	return copy_from_user(dest, vaddr, size) ? -EFAULT : 0;
 134}
 135
 136static nokprobe_inline int
 137probe_mem_read_user(void *dest, void *src, size_t size)
 138{
 139	return probe_mem_read(dest, src, size);
 140}
 141
 142/*
 143 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
 144 * length and relative data location.
 145 */
 146static nokprobe_inline int
 147fetch_store_string(unsigned long addr, void *dest, void *base)
 148{
 149	long ret;
 150	u32 loc = *(u32 *)dest;
 151	int maxlen  = get_loc_len(loc);
 152	u8 *dst = get_loc_data(dest, base);
 153	void __user *src = (void __force __user *) addr;
 154
 155	if (unlikely(!maxlen))
 156		return -ENOMEM;
 157
 158	if (addr == FETCH_TOKEN_COMM)
 159		ret = strscpy(dst, current->comm, maxlen);
 160	else
 161		ret = strncpy_from_user(dst, src, maxlen);
 162	if (ret >= 0) {
 163		if (ret == maxlen)
 164			dst[ret - 1] = '\0';
 165		else
 166			/*
 167			 * Include the terminating null byte. In this case it
 168			 * was copied by strncpy_from_user but not accounted
 169			 * for in ret.
 170			 */
 171			ret++;
 172		*(u32 *)dest = make_data_loc(ret, (void *)dst - base);
 173	} else
 174		*(u32 *)dest = make_data_loc(0, (void *)dst - base);
 175
 176	return ret;
 177}
 178
 179static nokprobe_inline int
 180fetch_store_string_user(unsigned long addr, void *dest, void *base)
 181{
 182	return fetch_store_string(addr, dest, base);
 183}
 184
 185/* Return the length of string -- including null terminal byte */
 186static nokprobe_inline int
 187fetch_store_strlen(unsigned long addr)
 188{
 189	int len;
 190	void __user *vaddr = (void __force __user *) addr;
 191
 192	if (addr == FETCH_TOKEN_COMM)
 193		len = strlen(current->comm) + 1;
 194	else
 195		len = strnlen_user(vaddr, MAX_STRING_SIZE);
 196
 197	return (len > MAX_STRING_SIZE) ? 0 : len;
 198}
 199
 200static nokprobe_inline int
 201fetch_store_strlen_user(unsigned long addr)
 202{
 203	return fetch_store_strlen(addr);
 204}
 205
 206static unsigned long translate_user_vaddr(unsigned long file_offset)
 207{
 208	unsigned long base_addr;
 209	struct uprobe_dispatch_data *udd;
 210
 211	udd = (void *) current->utask->vaddr;
 212
 213	base_addr = udd->bp_addr - udd->tu->offset;
 214	return base_addr + file_offset;
 215}
 216
 217/* Note that we don't verify it, since the code does not come from user space */
 218static int
 219process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
 220		   void *dest, void *base)
 221{
 222	struct pt_regs *regs = rec;
 223	unsigned long val;
 224	int ret;
 225
 226	/* 1st stage: get value from context */
 227	switch (code->op) {
 228	case FETCH_OP_REG:
 229		val = regs_get_register(regs, code->param);
 230		break;
 231	case FETCH_OP_STACK:
 232		val = get_user_stack_nth(regs, code->param);
 233		break;
 234	case FETCH_OP_STACKP:
 235		val = user_stack_pointer(regs);
 236		break;
 237	case FETCH_OP_RETVAL:
 238		val = regs_return_value(regs);
 239		break;
 240	case FETCH_OP_COMM:
 241		val = FETCH_TOKEN_COMM;
 242		break;
 243	case FETCH_OP_FOFFS:
 244		val = translate_user_vaddr(code->immediate);
 245		break;
 246	default:
 247		ret = process_common_fetch_insn(code, &val);
 248		if (ret < 0)
 249			return ret;
 250	}
 251	code++;
 252
 253	return process_fetch_insn_bottom(code, val, dest, base);
 254}
 255NOKPROBE_SYMBOL(process_fetch_insn)
 256
 257static inline void init_trace_uprobe_filter(struct trace_uprobe_filter *filter)
 258{
 259	rwlock_init(&filter->rwlock);
 260	filter->nr_systemwide = 0;
 261	INIT_LIST_HEAD(&filter->perf_events);
 262}
 263
 264static inline bool uprobe_filter_is_empty(struct trace_uprobe_filter *filter)
 265{
 266	return !filter->nr_systemwide && list_empty(&filter->perf_events);
 267}
 268
 269static inline bool is_ret_probe(struct trace_uprobe *tu)
 270{
 271	return tu->consumer.ret_handler != NULL;
 272}
 273
 274static bool trace_uprobe_is_busy(struct dyn_event *ev)
 275{
 276	struct trace_uprobe *tu = to_trace_uprobe(ev);
 277
 278	return trace_probe_is_enabled(&tu->tp);
 279}
 280
 281static bool trace_uprobe_match_command_head(struct trace_uprobe *tu,
 282					    int argc, const char **argv)
 283{
 284	char buf[MAX_ARGSTR_LEN + 1];
 285	int len;
 286
 287	if (!argc)
 288		return true;
 289
 290	len = strlen(tu->filename);
 291	if (strncmp(tu->filename, argv[0], len) || argv[0][len] != ':')
 292		return false;
 293
 294	if (tu->ref_ctr_offset == 0)
 295		snprintf(buf, sizeof(buf), "0x%0*lx",
 296				(int)(sizeof(void *) * 2), tu->offset);
 297	else
 298		snprintf(buf, sizeof(buf), "0x%0*lx(0x%lx)",
 299				(int)(sizeof(void *) * 2), tu->offset,
 300				tu->ref_ctr_offset);
 301	if (strcmp(buf, &argv[0][len + 1]))
 302		return false;
 303
 304	argc--; argv++;
 305
 306	return trace_probe_match_command_args(&tu->tp, argc, argv);
 307}
 308
 309static bool trace_uprobe_match(const char *system, const char *event,
 310			int argc, const char **argv, struct dyn_event *ev)
 311{
 312	struct trace_uprobe *tu = to_trace_uprobe(ev);
 313
 314	return (event[0] == '\0' ||
 315		strcmp(trace_probe_name(&tu->tp), event) == 0) &&
 316	   (!system || strcmp(trace_probe_group_name(&tu->tp), system) == 0) &&
 317	   trace_uprobe_match_command_head(tu, argc, argv);
 318}
 319
 320static nokprobe_inline struct trace_uprobe *
 321trace_uprobe_primary_from_call(struct trace_event_call *call)
 322{
 323	struct trace_probe *tp;
 324
 325	tp = trace_probe_primary_from_call(call);
 326	if (WARN_ON_ONCE(!tp))
 327		return NULL;
 328
 329	return container_of(tp, struct trace_uprobe, tp);
 330}
 331
 332/*
 333 * Allocate new trace_uprobe and initialize it (including uprobes).
 334 */
 335static struct trace_uprobe *
 336alloc_trace_uprobe(const char *group, const char *event, int nargs, bool is_ret)
 337{
 338	struct trace_uprobe *tu;
 339	int ret;
 340
 341	tu = kzalloc(struct_size(tu, tp.args, nargs), GFP_KERNEL);
 342	if (!tu)
 343		return ERR_PTR(-ENOMEM);
 344
 345	tu->nhits = alloc_percpu(unsigned long);
 346	if (!tu->nhits) {
 347		ret = -ENOMEM;
 348		goto error;
 349	}
 350
 351	ret = trace_probe_init(&tu->tp, event, group, true, nargs);
 352	if (ret < 0)
 353		goto error;
 354
 355	dyn_event_init(&tu->devent, &trace_uprobe_ops);
 356	tu->consumer.handler = uprobe_dispatcher;
 357	if (is_ret)
 358		tu->consumer.ret_handler = uretprobe_dispatcher;
 359	init_trace_uprobe_filter(tu->tp.event->filter);
 360	return tu;
 361
 362error:
 363	free_percpu(tu->nhits);
 364	kfree(tu);
 365
 366	return ERR_PTR(ret);
 367}
 368
 369static void free_trace_uprobe(struct trace_uprobe *tu)
 370{
 371	if (!tu)
 372		return;
 373
 374	path_put(&tu->path);
 375	trace_probe_cleanup(&tu->tp);
 376	kfree(tu->filename);
 377	free_percpu(tu->nhits);
 378	kfree(tu);
 379}
 380
 381static struct trace_uprobe *find_probe_event(const char *event, const char *group)
 382{
 383	struct dyn_event *pos;
 384	struct trace_uprobe *tu;
 385
 386	for_each_trace_uprobe(tu, pos)
 387		if (strcmp(trace_probe_name(&tu->tp), event) == 0 &&
 388		    strcmp(trace_probe_group_name(&tu->tp), group) == 0)
 389			return tu;
 390
 391	return NULL;
 392}
 393
 394/* Unregister a trace_uprobe and probe_event */
 395static int unregister_trace_uprobe(struct trace_uprobe *tu)
 396{
 397	int ret;
 398
 399	if (trace_probe_has_sibling(&tu->tp))
 400		goto unreg;
 401
 402	/* If there's a reference to the dynamic event */
 403	if (trace_event_dyn_busy(trace_probe_event_call(&tu->tp)))
 404		return -EBUSY;
 405
 406	ret = unregister_uprobe_event(tu);
 407	if (ret)
 408		return ret;
 409
 410unreg:
 411	dyn_event_remove(&tu->devent);
 412	trace_probe_unlink(&tu->tp);
 413	free_trace_uprobe(tu);
 414	return 0;
 415}
 416
 417static bool trace_uprobe_has_same_uprobe(struct trace_uprobe *orig,
 418					 struct trace_uprobe *comp)
 419{
 420	struct trace_probe_event *tpe = orig->tp.event;
 421	struct inode *comp_inode = d_real_inode(comp->path.dentry);
 422	int i;
 423
 424	list_for_each_entry(orig, &tpe->probes, tp.list) {
 425		if (comp_inode != d_real_inode(orig->path.dentry) ||
 426		    comp->offset != orig->offset)
 427			continue;
 428
 429		/*
 430		 * trace_probe_compare_arg_type() ensured that nr_args and
 431		 * each argument name and type are same. Let's compare comm.
 432		 */
 433		for (i = 0; i < orig->tp.nr_args; i++) {
 434			if (strcmp(orig->tp.args[i].comm,
 435				   comp->tp.args[i].comm))
 436				break;
 437		}
 438
 439		if (i == orig->tp.nr_args)
 440			return true;
 441	}
 442
 443	return false;
 444}
 445
 446static int append_trace_uprobe(struct trace_uprobe *tu, struct trace_uprobe *to)
 447{
 448	int ret;
 449
 450	ret = trace_probe_compare_arg_type(&tu->tp, &to->tp);
 451	if (ret) {
 452		/* Note that argument starts index = 2 */
 453		trace_probe_log_set_index(ret + 1);
 454		trace_probe_log_err(0, DIFF_ARG_TYPE);
 455		return -EEXIST;
 456	}
 457	if (trace_uprobe_has_same_uprobe(to, tu)) {
 458		trace_probe_log_set_index(0);
 459		trace_probe_log_err(0, SAME_PROBE);
 460		return -EEXIST;
 461	}
 462
 463	/* Append to existing event */
 464	ret = trace_probe_append(&tu->tp, &to->tp);
 465	if (!ret)
 466		dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));
 467
 468	return ret;
 469}
 470
 471/*
 472 * Uprobe with multiple reference counter is not allowed. i.e.
 473 * If inode and offset matches, reference counter offset *must*
 474 * match as well. Though, there is one exception: If user is
 475 * replacing old trace_uprobe with new one(same group/event),
 476 * then we allow same uprobe with new reference counter as far
 477 * as the new one does not conflict with any other existing
 478 * ones.
 479 */
 480static int validate_ref_ctr_offset(struct trace_uprobe *new)
 481{
 482	struct dyn_event *pos;
 483	struct trace_uprobe *tmp;
 484	struct inode *new_inode = d_real_inode(new->path.dentry);
 485
 486	for_each_trace_uprobe(tmp, pos) {
 487		if (new_inode == d_real_inode(tmp->path.dentry) &&
 488		    new->offset == tmp->offset &&
 489		    new->ref_ctr_offset != tmp->ref_ctr_offset) {
 490			pr_warn("Reference counter offset mismatch.");
 491			return -EINVAL;
 492		}
 493	}
 494	return 0;
 495}
 496
 497/* Register a trace_uprobe and probe_event */
 498static int register_trace_uprobe(struct trace_uprobe *tu)
 499{
 500	struct trace_uprobe *old_tu;
 501	int ret;
 502
 503	guard(mutex)(&event_mutex);
 504
 505	ret = validate_ref_ctr_offset(tu);
 506	if (ret)
 507		return ret;
 508
 509	/* register as an event */
 510	old_tu = find_probe_event(trace_probe_name(&tu->tp),
 511				  trace_probe_group_name(&tu->tp));
 512	if (old_tu) {
 513		if (is_ret_probe(tu) != is_ret_probe(old_tu)) {
 514			trace_probe_log_set_index(0);
 515			trace_probe_log_err(0, DIFF_PROBE_TYPE);
 516			return -EEXIST;
 517		}
 518		return append_trace_uprobe(tu, old_tu);
 519	}
 520
 521	ret = register_uprobe_event(tu);
 522	if (ret) {
 523		if (ret == -EEXIST) {
 524			trace_probe_log_set_index(0);
 525			trace_probe_log_err(0, EVENT_EXIST);
 526		} else
 527			pr_warn("Failed to register probe event(%d)\n", ret);
 528		return ret;
 529	}
 530
 531	dyn_event_add(&tu->devent, trace_probe_event_call(&tu->tp));
 532
 533	return ret;
 534}
 535
 536DEFINE_FREE(free_trace_uprobe, struct trace_uprobe *, if (_T) free_trace_uprobe(_T))
 537
 538/*
 539 * Argument syntax:
 540 *  - Add uprobe: p|r[:[GRP/][EVENT]] PATH:OFFSET[%return][(REF)] [FETCHARGS]
 541 */
 542static int __trace_uprobe_create(int argc, const char **argv)
 543{
 544	struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
 545	struct trace_uprobe *tu __free(free_trace_uprobe) = NULL;
 546	const char *trlog __free(trace_probe_log_clear) = NULL;
 547	const char *event = NULL, *group = UPROBE_EVENT_SYSTEM;
 548	struct path path __free(path_put) = {};
 549	unsigned long offset, ref_ctr_offset;
 550	char *filename __free(kfree) = NULL;
 551	char *arg, *rctr, *rctr_end, *tmp;
 552	char *gbuf __free(kfree) = NULL;
 553	char *buf __free(kfree) = NULL;
 554	enum probe_print_type ptype;
 555	bool is_return = false;
 556	int i, ret;
 557
 558	ref_ctr_offset = 0;
 559
 560	switch (argv[0][0]) {
 561	case 'r':
 562		is_return = true;
 563		break;
 564	case 'p':
 565		break;
 566	default:
 567		return -ECANCELED;
 568	}
 569
 570	if (argc < 2)
 571		return -ECANCELED;
 572
 573	trlog = trace_probe_log_init("trace_uprobe", argc, argv);
 574
 575	if (argc - 2 > MAX_TRACE_ARGS) {
 576		trace_probe_log_set_index(2);
 577		trace_probe_log_err(0, TOO_MANY_ARGS);
 578		return -E2BIG;
 579	}
 580
 581	if (argv[0][1] == ':')
 582		event = &argv[0][2];
 583
 584	if (!strchr(argv[1], '/'))
 585		return -ECANCELED;
 586
 587	filename = kstrdup(argv[1], GFP_KERNEL);
 588	if (!filename)
 589		return -ENOMEM;
 590
 591	/* Find the last occurrence, in case the path contains ':' too. */
 592	arg = strrchr(filename, ':');
 593	if (!arg || !isdigit(arg[1]))
 594		return -ECANCELED;
 595
 596	trace_probe_log_set_index(1);	/* filename is the 2nd argument */
 597
 598	*arg++ = '\0';
 599	ret = kern_path(filename, LOOKUP_FOLLOW, &path);
 600	if (ret) {
 601		trace_probe_log_err(0, FILE_NOT_FOUND);
 602		return ret;
 603	}
 604	if (!d_is_reg(path.dentry)) {
 605		trace_probe_log_err(0, NO_REGULAR_FILE);
 606		return -EINVAL;
 607	}
 608
 609	/* Parse reference counter offset if specified. */
 610	rctr = strchr(arg, '(');
 611	if (rctr) {
 612		rctr_end = strchr(rctr, ')');
 613		if (!rctr_end) {
 614			rctr_end = rctr + strlen(rctr);
 615			trace_probe_log_err(rctr_end - filename,
 616					    REFCNT_OPEN_BRACE);
 617			return -EINVAL;
 618		} else if (rctr_end[1] != '\0') {
 619			trace_probe_log_err(rctr_end + 1 - filename,
 620					    BAD_REFCNT_SUFFIX);
 621			return -EINVAL;
 622		}
 623
 624		*rctr++ = '\0';
 625		*rctr_end = '\0';
 626		ret = kstrtoul(rctr, 0, &ref_ctr_offset);
 627		if (ret) {
 628			trace_probe_log_err(rctr - filename, BAD_REFCNT);
 629			return ret;
 630		}
 631	}
 632
 633	/* Check if there is %return suffix */
 634	tmp = strchr(arg, '%');
 635	if (tmp) {
 636		if (!strcmp(tmp, "%return")) {
 637			*tmp = '\0';
 638			is_return = true;
 639		} else {
 640			trace_probe_log_err(tmp - filename, BAD_ADDR_SUFFIX);
 641			return -EINVAL;
 642		}
 643	}
 644
 645	/* Parse uprobe offset. */
 646	ret = kstrtoul(arg, 0, &offset);
 647	if (ret) {
 648		trace_probe_log_err(arg - filename, BAD_UPROBE_OFFS);
 649		return ret;
 650	}
 651
 652	/* setup a probe */
 653	trace_probe_log_set_index(0);
 654	if (event) {
 655		gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
 656		if (!gbuf)
 657			return -ENOMEM;
 658
 659		ret = traceprobe_parse_event_name(&event, &group, gbuf,
 660						  event - argv[0]);
 661		if (ret)
 662			return ret;
 663	}
 664
 665	if (!event) {
 666		char *tail;
 667		char *ptr;
 668
 669		tail = kstrdup(kbasename(filename), GFP_KERNEL);
 670		if (!tail)
 671			return -ENOMEM;
 672
 673		ptr = strpbrk(tail, ".-_");
 674		if (ptr)
 675			*ptr = '\0';
 676
 677		buf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
 678		if (!buf)
 679			return -ENOMEM;
 680		snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_0x%lx", 'p', tail, offset);
 681		event = buf;
 682		kfree(tail);
 683	}
 684
 685	argc -= 2;
 686	argv += 2;
 687
 688	tu = alloc_trace_uprobe(group, event, argc, is_return);
 689	if (IS_ERR(tu)) {
 690		ret = PTR_ERR(tu);
 691		/* This must return -ENOMEM otherwise there is a bug */
 692		WARN_ON_ONCE(ret != -ENOMEM);
 693		return ret;
 694	}
 695	tu->offset = offset;
 696	tu->ref_ctr_offset = ref_ctr_offset;
 697	tu->path = path;
 698	/* Clear @path so that it will not freed by path_put() */
 699	memset(&path, 0, sizeof(path));
 700	tu->filename = no_free_ptr(filename);
 701
 702	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 703	if (!ctx)
 704		return -ENOMEM;
 705	ctx->flags = (is_return ? TPARG_FL_RETURN : 0) | TPARG_FL_USER;
 706
 707	/* parse arguments */
 708	for (i = 0; i < argc; i++) {
 709		trace_probe_log_set_index(i + 2);
 710		ret = traceprobe_parse_probe_arg(&tu->tp, i, argv[i], ctx);
 711		if (ret)
 712			return ret;
 713	}
 714
 715	ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
 716	ret = traceprobe_set_print_fmt(&tu->tp, ptype);
 717	if (ret < 0)
 718		return ret;
 719
 720	ret = register_trace_uprobe(tu);
 721	if (!ret)
 722		tu = NULL;
 723
 724	return ret;
 725}
 726
 727int trace_uprobe_create(const char *raw_command)
 728{
 729	return trace_probe_create(raw_command, __trace_uprobe_create);
 730}
 731
 732static int create_or_delete_trace_uprobe(const char *raw_command)
 733{
 734	int ret;
 735
 736	if (raw_command[0] == '-')
 737		return dyn_event_release(raw_command, &trace_uprobe_ops);
 738
 739	ret = dyn_event_create(raw_command, &trace_uprobe_ops);
 740	return ret == -ECANCELED ? -EINVAL : ret;
 741}
 742
 743static int trace_uprobe_release(struct dyn_event *ev)
 744{
 745	struct trace_uprobe *tu = to_trace_uprobe(ev);
 746
 747	return unregister_trace_uprobe(tu);
 748}
 749
 750/* Probes listing interfaces */
 751static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
 752{
 753	struct trace_uprobe *tu = to_trace_uprobe(ev);
 754	char c = is_ret_probe(tu) ? 'r' : 'p';
 755	int i;
 756
 757	seq_printf(m, "%c:%s/%s %s:0x%0*lx", c, trace_probe_group_name(&tu->tp),
 758			trace_probe_name(&tu->tp), tu->filename,
 759			(int)(sizeof(void *) * 2), tu->offset);
 760
 761	if (tu->ref_ctr_offset)
 762		seq_printf(m, "(0x%lx)", tu->ref_ctr_offset);
 763
 764	for (i = 0; i < tu->tp.nr_args; i++)
 765		seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
 766
 767	seq_putc(m, '\n');
 768	return 0;
 769}
 770
 771static int probes_seq_show(struct seq_file *m, void *v)
 772{
 773	struct dyn_event *ev = v;
 774
 775	if (!is_trace_uprobe(ev))
 776		return 0;
 777
 778	return trace_uprobe_show(m, ev);
 779}
 780
 781static const struct seq_operations probes_seq_op = {
 782	.start  = dyn_event_seq_start,
 783	.next   = dyn_event_seq_next,
 784	.stop   = dyn_event_seq_stop,
 785	.show   = probes_seq_show
 786};
 787
 788static int probes_open(struct inode *inode, struct file *file)
 789{
 790	int ret;
 791
 792	ret = security_locked_down(LOCKDOWN_TRACEFS);
 793	if (ret)
 794		return ret;
 795
 796	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
 797		ret = dyn_events_release_all(&trace_uprobe_ops);
 798		if (ret)
 799			return ret;
 800	}
 801
 802	return seq_open(file, &probes_seq_op);
 803}
 804
 805static ssize_t probes_write(struct file *file, const char __user *buffer,
 806			    size_t count, loff_t *ppos)
 807{
 808	return trace_parse_run_command(file, buffer, count, ppos,
 809					create_or_delete_trace_uprobe);
 810}
 811
 812static const struct file_operations uprobe_events_ops = {
 813	.owner		= THIS_MODULE,
 814	.open		= probes_open,
 815	.read		= seq_read,
 816	.llseek		= seq_lseek,
 817	.release	= seq_release,
 818	.write		= probes_write,
 819};
 820
 821/* Probes profiling interfaces */
 822static int probes_profile_seq_show(struct seq_file *m, void *v)
 823{
 824	struct dyn_event *ev = v;
 825	struct trace_uprobe *tu;
 826	unsigned long nhits;
 827	int cpu;
 828
 829	if (!is_trace_uprobe(ev))
 830		return 0;
 831
 832	tu = to_trace_uprobe(ev);
 833
 834	nhits = 0;
 835	for_each_possible_cpu(cpu) {
 836		nhits += per_cpu(*tu->nhits, cpu);
 837	}
 838
 839	seq_printf(m, "  %s %-44s %15lu\n", tu->filename,
 840		   trace_probe_name(&tu->tp), nhits);
 841	return 0;
 842}
 843
 844static const struct seq_operations profile_seq_op = {
 845	.start  = dyn_event_seq_start,
 846	.next   = dyn_event_seq_next,
 847	.stop   = dyn_event_seq_stop,
 848	.show	= probes_profile_seq_show
 849};
 850
 851static int profile_open(struct inode *inode, struct file *file)
 852{
 853	int ret;
 854
 855	ret = security_locked_down(LOCKDOWN_TRACEFS);
 856	if (ret)
 857		return ret;
 858
 859	return seq_open(file, &profile_seq_op);
 860}
 861
 862static const struct file_operations uprobe_profile_ops = {
 863	.owner		= THIS_MODULE,
 864	.open		= profile_open,
 865	.read		= seq_read,
 866	.llseek		= seq_lseek,
 867	.release	= seq_release,
 868};
 869
 870struct uprobe_cpu_buffer {
 871	struct mutex mutex;
 872	void *buf;
 873	int dsize;
 874};
 875static struct uprobe_cpu_buffer __percpu *uprobe_cpu_buffer;
 876static int uprobe_buffer_refcnt;
 877#define MAX_UCB_BUFFER_SIZE PAGE_SIZE
 878
 879static int uprobe_buffer_init(void)
 880{
 881	int cpu, err_cpu;
 882
 883	uprobe_cpu_buffer = alloc_percpu(struct uprobe_cpu_buffer);
 884	if (uprobe_cpu_buffer == NULL)
 885		return -ENOMEM;
 886
 887	for_each_possible_cpu(cpu) {
 888		struct page *p = alloc_pages_node(cpu_to_node(cpu),
 889						  GFP_KERNEL, 0);
 890		if (p == NULL) {
 891			err_cpu = cpu;
 892			goto err;
 893		}
 894		per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf = page_address(p);
 895		mutex_init(&per_cpu_ptr(uprobe_cpu_buffer, cpu)->mutex);
 896	}
 897
 898	return 0;
 899
 900err:
 901	for_each_possible_cpu(cpu) {
 902		if (cpu == err_cpu)
 903			break;
 904		free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer, cpu)->buf);
 905	}
 906
 907	free_percpu(uprobe_cpu_buffer);
 908	return -ENOMEM;
 909}
 910
 911static int uprobe_buffer_enable(void)
 912{
 913	int ret = 0;
 914
 915	BUG_ON(!mutex_is_locked(&event_mutex));
 916
 917	if (uprobe_buffer_refcnt++ == 0) {
 918		ret = uprobe_buffer_init();
 919		if (ret < 0)
 920			uprobe_buffer_refcnt--;
 921	}
 922
 923	return ret;
 924}
 925
 926static void uprobe_buffer_disable(void)
 927{
 928	int cpu;
 929
 930	BUG_ON(!mutex_is_locked(&event_mutex));
 931
 932	if (--uprobe_buffer_refcnt == 0) {
 933		for_each_possible_cpu(cpu)
 934			free_page((unsigned long)per_cpu_ptr(uprobe_cpu_buffer,
 935							     cpu)->buf);
 936
 937		free_percpu(uprobe_cpu_buffer);
 938		uprobe_cpu_buffer = NULL;
 939	}
 940}
 941
 942static struct uprobe_cpu_buffer *uprobe_buffer_get(void)
 943{
 944	struct uprobe_cpu_buffer *ucb;
 945	int cpu;
 946
 947	cpu = raw_smp_processor_id();
 948	ucb = per_cpu_ptr(uprobe_cpu_buffer, cpu);
 949
 950	/*
 951	 * Use per-cpu buffers for fastest access, but we might migrate
 952	 * so the mutex makes sure we have sole access to it.
 953	 */
 954	mutex_lock(&ucb->mutex);
 955
 956	return ucb;
 957}
 958
 959static void uprobe_buffer_put(struct uprobe_cpu_buffer *ucb)
 960{
 961	if (!ucb)
 962		return;
 963	mutex_unlock(&ucb->mutex);
 964}
 965
 966static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu,
 967						       struct pt_regs *regs,
 968						       struct uprobe_cpu_buffer **ucbp)
 969{
 970	struct uprobe_cpu_buffer *ucb;
 971	int dsize, esize;
 972
 973	if (*ucbp)
 974		return *ucbp;
 975
 976	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
 977	dsize = __get_data_size(&tu->tp, regs, NULL);
 978
 979	ucb = uprobe_buffer_get();
 980	ucb->dsize = tu->tp.size + dsize;
 981
 982	if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) {
 983		ucb->dsize = MAX_UCB_BUFFER_SIZE;
 984		dsize = MAX_UCB_BUFFER_SIZE - tu->tp.size;
 985	}
 986
 987	store_trace_args(ucb->buf, &tu->tp, regs, NULL, esize, dsize);
 988
 989	*ucbp = ucb;
 990	return ucb;
 991}
 992
 993static void __uprobe_trace_func(struct trace_uprobe *tu,
 994				unsigned long func, struct pt_regs *regs,
 995				struct uprobe_cpu_buffer *ucb,
 996				struct trace_event_file *trace_file)
 997{
 998	struct uprobe_trace_entry_head *entry;
 999	struct trace_event_buffer fbuffer;
1000	void *data;
1001	int size, esize;
1002	struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1003
1004	WARN_ON(call != trace_file->event_call);
1005
1006	if (trace_trigger_soft_disabled(trace_file))
1007		return;
1008
1009	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1010	size = esize + ucb->dsize;
1011	entry = trace_event_buffer_reserve(&fbuffer, trace_file, size);
1012	if (!entry)
1013		return;
1014
1015	if (is_ret_probe(tu)) {
1016		entry->vaddr[0] = func;
1017		entry->vaddr[1] = instruction_pointer(regs);
1018		data = DATAOF_TRACE_ENTRY(entry, true);
1019	} else {
1020		entry->vaddr[0] = instruction_pointer(regs);
1021		data = DATAOF_TRACE_ENTRY(entry, false);
1022	}
1023
1024	memcpy(data, ucb->buf, ucb->dsize);
1025
1026	trace_event_buffer_commit(&fbuffer);
1027}
1028
1029/* uprobe handler */
1030static int uprobe_trace_func(struct trace_uprobe *tu, struct pt_regs *regs,
1031			     struct uprobe_cpu_buffer **ucbp)
1032{
1033	struct event_file_link *link;
1034	struct uprobe_cpu_buffer *ucb;
1035
1036	if (is_ret_probe(tu))
1037		return 0;
1038
1039	ucb = prepare_uprobe_buffer(tu, regs, ucbp);
1040
1041	rcu_read_lock();
1042	trace_probe_for_each_link_rcu(link, &tu->tp)
1043		__uprobe_trace_func(tu, 0, regs, ucb, link->file);
1044	rcu_read_unlock();
1045
1046	return 0;
1047}
1048
1049static void uretprobe_trace_func(struct trace_uprobe *tu, unsigned long func,
1050				 struct pt_regs *regs,
1051				 struct uprobe_cpu_buffer **ucbp)
1052{
1053	struct event_file_link *link;
1054	struct uprobe_cpu_buffer *ucb;
1055
1056	ucb = prepare_uprobe_buffer(tu, regs, ucbp);
1057
1058	rcu_read_lock();
1059	trace_probe_for_each_link_rcu(link, &tu->tp)
1060		__uprobe_trace_func(tu, func, regs, ucb, link->file);
1061	rcu_read_unlock();
1062}
1063
1064/* Event entry printers */
1065static enum print_line_t
1066print_uprobe_event(struct trace_iterator *iter, int flags, struct trace_event *event)
1067{
1068	struct uprobe_trace_entry_head *entry;
1069	struct trace_seq *s = &iter->seq;
1070	struct trace_uprobe *tu;
1071	u8 *data;
1072
1073	entry = (struct uprobe_trace_entry_head *)iter->ent;
1074	tu = trace_uprobe_primary_from_call(
1075		container_of(event, struct trace_event_call, event));
1076	if (unlikely(!tu))
1077		goto out;
1078
1079	if (is_ret_probe(tu)) {
1080		trace_seq_printf(s, "%s: (0x%lx <- 0x%lx)",
1081				 trace_probe_name(&tu->tp),
1082				 entry->vaddr[1], entry->vaddr[0]);
1083		data = DATAOF_TRACE_ENTRY(entry, true);
1084	} else {
1085		trace_seq_printf(s, "%s: (0x%lx)",
1086				 trace_probe_name(&tu->tp),
1087				 entry->vaddr[0]);
1088		data = DATAOF_TRACE_ENTRY(entry, false);
1089	}
1090
1091	if (trace_probe_print_args(s, tu->tp.args, tu->tp.nr_args, data, entry) < 0)
1092		goto out;
1093
1094	trace_seq_putc(s, '\n');
1095
1096 out:
1097	return trace_handle_return(s);
1098}
1099
1100typedef bool (*filter_func_t)(struct uprobe_consumer *self, struct mm_struct *mm);
1101
1102static int trace_uprobe_enable(struct trace_uprobe *tu, filter_func_t filter)
1103{
1104	struct inode *inode = d_real_inode(tu->path.dentry);
1105	struct uprobe *uprobe;
1106
1107	tu->consumer.filter = filter;
1108	uprobe = uprobe_register(inode, tu->offset, tu->ref_ctr_offset, &tu->consumer);
1109	if (IS_ERR(uprobe))
1110		return PTR_ERR(uprobe);
1111
1112	tu->uprobe = uprobe;
1113	return 0;
1114}
1115
1116static void __probe_event_disable(struct trace_probe *tp)
1117{
1118	struct trace_uprobe *tu;
1119	bool sync = false;
1120
1121	tu = container_of(tp, struct trace_uprobe, tp);
1122	WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
1123
1124	list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1125		if (!tu->uprobe)
1126			continue;
1127
1128		uprobe_unregister_nosync(tu->uprobe, &tu->consumer);
1129		sync = true;
1130		tu->uprobe = NULL;
1131	}
1132	if (sync)
1133		uprobe_unregister_sync();
1134}
1135
1136static int probe_event_enable(struct trace_event_call *call,
1137			struct trace_event_file *file, filter_func_t filter)
1138{
1139	struct trace_probe *tp;
1140	struct trace_uprobe *tu;
1141	bool enabled;
1142	int ret;
1143
1144	tp = trace_probe_primary_from_call(call);
1145	if (WARN_ON_ONCE(!tp))
1146		return -ENODEV;
1147	enabled = trace_probe_is_enabled(tp);
1148
1149	/* This may also change "enabled" state */
1150	if (file) {
1151		if (trace_probe_test_flag(tp, TP_FLAG_PROFILE))
1152			return -EINTR;
1153
1154		ret = trace_probe_add_file(tp, file);
1155		if (ret < 0)
1156			return ret;
1157	} else {
1158		if (trace_probe_test_flag(tp, TP_FLAG_TRACE))
1159			return -EINTR;
1160
1161		trace_probe_set_flag(tp, TP_FLAG_PROFILE);
1162	}
1163
1164	tu = container_of(tp, struct trace_uprobe, tp);
1165	WARN_ON(!uprobe_filter_is_empty(tu->tp.event->filter));
1166
1167	if (enabled)
1168		return 0;
1169
1170	ret = uprobe_buffer_enable();
1171	if (ret)
1172		goto err_flags;
1173
1174	list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1175		ret = trace_uprobe_enable(tu, filter);
1176		if (ret) {
1177			__probe_event_disable(tp);
1178			goto err_buffer;
1179		}
1180	}
1181
1182	return 0;
1183
1184 err_buffer:
1185	uprobe_buffer_disable();
1186
1187 err_flags:
1188	if (file)
1189		trace_probe_remove_file(tp, file);
1190	else
1191		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1192
1193	return ret;
1194}
1195
1196static void probe_event_disable(struct trace_event_call *call,
1197				struct trace_event_file *file)
1198{
1199	struct trace_probe *tp;
1200
1201	tp = trace_probe_primary_from_call(call);
1202	if (WARN_ON_ONCE(!tp))
1203		return;
1204
1205	if (!trace_probe_is_enabled(tp))
1206		return;
1207
1208	if (file) {
1209		if (trace_probe_remove_file(tp, file) < 0)
1210			return;
1211
1212		if (trace_probe_is_enabled(tp))
1213			return;
1214	} else
1215		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
1216
1217	__probe_event_disable(tp);
1218	uprobe_buffer_disable();
1219}
1220
1221static int uprobe_event_define_fields(struct trace_event_call *event_call)
1222{
1223	int ret, size;
1224	struct uprobe_trace_entry_head field;
1225	struct trace_uprobe *tu;
1226
1227	tu = trace_uprobe_primary_from_call(event_call);
1228	if (unlikely(!tu))
1229		return -ENODEV;
1230
1231	if (is_ret_probe(tu)) {
1232		DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_FUNC, 0);
1233		DEFINE_FIELD(unsigned long, vaddr[1], FIELD_STRING_RETIP, 0);
1234		size = SIZEOF_TRACE_ENTRY(true);
1235	} else {
1236		DEFINE_FIELD(unsigned long, vaddr[0], FIELD_STRING_IP, 0);
1237		size = SIZEOF_TRACE_ENTRY(false);
1238	}
1239
1240	return traceprobe_define_arg_fields(event_call, size, &tu->tp);
1241}
1242
1243#ifdef CONFIG_PERF_EVENTS
1244static bool
1245__uprobe_perf_filter(struct trace_uprobe_filter *filter, struct mm_struct *mm)
1246{
1247	struct perf_event *event;
1248
1249	list_for_each_entry(event, &filter->perf_events, hw.tp_list) {
1250		if (event->hw.target->mm == mm)
1251			return true;
1252	}
1253
1254	return false;
1255}
1256
1257static inline bool
1258trace_uprobe_filter_event(struct trace_uprobe_filter *filter,
1259			  struct perf_event *event)
1260{
1261	return __uprobe_perf_filter(filter, event->hw.target->mm);
1262}
1263
1264static bool trace_uprobe_filter_remove(struct trace_uprobe_filter *filter,
1265				       struct perf_event *event)
1266{
1267	bool done;
1268
1269	write_lock(&filter->rwlock);
1270	if (event->hw.target) {
1271		list_del(&event->hw.tp_list);
1272		done = filter->nr_systemwide ||
1273			(event->hw.target->flags & PF_EXITING) ||
1274			trace_uprobe_filter_event(filter, event);
1275	} else {
1276		filter->nr_systemwide--;
1277		done = filter->nr_systemwide;
1278	}
1279	write_unlock(&filter->rwlock);
1280
1281	return done;
1282}
1283
1284/* This returns true if the filter always covers target mm */
1285static bool trace_uprobe_filter_add(struct trace_uprobe_filter *filter,
1286				    struct perf_event *event)
1287{
1288	bool done;
1289
1290	write_lock(&filter->rwlock);
1291	if (event->hw.target) {
1292		/*
1293		 * event->parent != NULL means copy_process(), we can avoid
1294		 * uprobe_apply(). current->mm must be probed and we can rely
1295		 * on dup_mmap() which preserves the already installed bp's.
1296		 *
1297		 * attr.enable_on_exec means that exec/mmap will install the
1298		 * breakpoints we need.
1299		 */
1300		done = filter->nr_systemwide ||
1301			event->parent || event->attr.enable_on_exec ||
1302			trace_uprobe_filter_event(filter, event);
1303		list_add(&event->hw.tp_list, &filter->perf_events);
1304	} else {
1305		done = filter->nr_systemwide;
1306		filter->nr_systemwide++;
1307	}
1308	write_unlock(&filter->rwlock);
1309
1310	return done;
1311}
1312
1313static int uprobe_perf_close(struct trace_event_call *call,
1314			     struct perf_event *event)
1315{
1316	struct trace_probe *tp;
1317	struct trace_uprobe *tu;
1318	int ret = 0;
1319
1320	tp = trace_probe_primary_from_call(call);
1321	if (WARN_ON_ONCE(!tp))
1322		return -ENODEV;
1323
1324	tu = container_of(tp, struct trace_uprobe, tp);
1325	if (trace_uprobe_filter_remove(tu->tp.event->filter, event))
1326		return 0;
1327
1328	list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1329		ret = uprobe_apply(tu->uprobe, &tu->consumer, false);
1330		if (ret)
1331			break;
1332	}
1333
1334	return ret;
1335}
1336
1337static int uprobe_perf_open(struct trace_event_call *call,
1338			    struct perf_event *event)
1339{
1340	struct trace_probe *tp;
1341	struct trace_uprobe *tu;
1342	int err = 0;
1343
1344	tp = trace_probe_primary_from_call(call);
1345	if (WARN_ON_ONCE(!tp))
1346		return -ENODEV;
1347
1348	tu = container_of(tp, struct trace_uprobe, tp);
1349	if (trace_uprobe_filter_add(tu->tp.event->filter, event))
1350		return 0;
1351
1352	list_for_each_entry(tu, trace_probe_probe_list(tp), tp.list) {
1353		err = uprobe_apply(tu->uprobe, &tu->consumer, true);
1354		if (err) {
1355			uprobe_perf_close(call, event);
1356			break;
1357		}
1358	}
1359
1360	return err;
1361}
1362
1363static bool uprobe_perf_filter(struct uprobe_consumer *uc, struct mm_struct *mm)
1364{
1365	struct trace_uprobe_filter *filter;
1366	struct trace_uprobe *tu;
1367	int ret;
1368
1369	tu = container_of(uc, struct trace_uprobe, consumer);
1370	filter = tu->tp.event->filter;
1371
1372	/*
1373	 * speculative short-circuiting check to avoid unnecessarily taking
1374	 * filter->rwlock below, if the uprobe has system-wide consumer
1375	 */
1376	if (READ_ONCE(filter->nr_systemwide))
1377		return true;
1378
1379	read_lock(&filter->rwlock);
1380	ret = __uprobe_perf_filter(filter, mm);
1381	read_unlock(&filter->rwlock);
1382
1383	return ret;
1384}
1385
1386static void __uprobe_perf_func(struct trace_uprobe *tu,
1387			       unsigned long func, struct pt_regs *regs,
1388			       struct uprobe_cpu_buffer **ucbp)
1389{
1390	struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1391	struct uprobe_trace_entry_head *entry;
1392	struct uprobe_cpu_buffer *ucb;
1393	struct hlist_head *head;
1394	void *data;
1395	int size, esize;
1396	int rctx;
1397
1398#ifdef CONFIG_BPF_EVENTS
1399	if (bpf_prog_array_valid(call)) {
1400		const struct bpf_prog_array *array;
1401		u32 ret;
1402
1403		rcu_read_lock_trace();
1404		array = rcu_dereference_check(call->prog_array, rcu_read_lock_trace_held());
1405		ret = bpf_prog_run_array_uprobe(array, regs, bpf_prog_run);
1406		rcu_read_unlock_trace();
1407		if (!ret)
1408			return;
1409	}
1410#endif /* CONFIG_BPF_EVENTS */
1411
1412	esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu));
1413
1414	ucb = prepare_uprobe_buffer(tu, regs, ucbp);
1415	size = esize + ucb->dsize;
1416	size = ALIGN(size + sizeof(u32), sizeof(u64)) - sizeof(u32);
1417	if (WARN_ONCE(size > PERF_MAX_TRACE_SIZE, "profile buffer not large enough"))
1418		return;
1419
1420	preempt_disable();
1421	head = this_cpu_ptr(call->perf_events);
1422	if (hlist_empty(head))
1423		goto out;
1424
1425	entry = perf_trace_buf_alloc(size, NULL, &rctx);
1426	if (!entry)
1427		goto out;
1428
1429	if (is_ret_probe(tu)) {
1430		entry->vaddr[0] = func;
1431		entry->vaddr[1] = instruction_pointer(regs);
1432		data = DATAOF_TRACE_ENTRY(entry, true);
1433	} else {
1434		entry->vaddr[0] = instruction_pointer(regs);
1435		data = DATAOF_TRACE_ENTRY(entry, false);
1436	}
1437
1438	memcpy(data, ucb->buf, ucb->dsize);
1439
1440	if (size - esize > ucb->dsize)
1441		memset(data + ucb->dsize, 0, size - esize - ucb->dsize);
1442
1443	perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1444			      head, NULL);
1445 out:
1446	preempt_enable();
1447}
1448
1449/* uprobe profile handler */
1450static int uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs,
1451			    struct uprobe_cpu_buffer **ucbp)
1452{
1453	if (!uprobe_perf_filter(&tu->consumer, current->mm))
1454		return UPROBE_HANDLER_REMOVE;
1455
1456	if (!is_ret_probe(tu))
1457		__uprobe_perf_func(tu, 0, regs, ucbp);
1458	return 0;
1459}
1460
1461static void uretprobe_perf_func(struct trace_uprobe *tu, unsigned long func,
1462				struct pt_regs *regs,
1463				struct uprobe_cpu_buffer **ucbp)
1464{
1465	__uprobe_perf_func(tu, func, regs, ucbp);
1466}
1467
1468int bpf_get_uprobe_info(const struct perf_event *event, u32 *fd_type,
1469			const char **filename, u64 *probe_offset,
1470			u64 *probe_addr, bool perf_type_tracepoint)
1471{
1472	const char *pevent = trace_event_name(event->tp_event);
1473	const char *group = event->tp_event->class->system;
1474	struct trace_uprobe *tu;
1475
1476	if (perf_type_tracepoint)
1477		tu = find_probe_event(pevent, group);
1478	else
1479		tu = trace_uprobe_primary_from_call(event->tp_event);
1480	if (!tu)
1481		return -EINVAL;
1482
1483	*fd_type = is_ret_probe(tu) ? BPF_FD_TYPE_URETPROBE
1484				    : BPF_FD_TYPE_UPROBE;
1485	*filename = tu->filename;
1486	*probe_offset = tu->offset;
1487	*probe_addr = tu->ref_ctr_offset;
1488	return 0;
1489}
1490#endif	/* CONFIG_PERF_EVENTS */
1491
1492static int
1493trace_uprobe_register(struct trace_event_call *event, enum trace_reg type,
1494		      void *data)
1495{
1496	struct trace_event_file *file = data;
1497
1498	switch (type) {
1499	case TRACE_REG_REGISTER:
1500		return probe_event_enable(event, file, NULL);
1501
1502	case TRACE_REG_UNREGISTER:
1503		probe_event_disable(event, file);
1504		return 0;
1505
1506#ifdef CONFIG_PERF_EVENTS
1507	case TRACE_REG_PERF_REGISTER:
1508		return probe_event_enable(event, NULL, uprobe_perf_filter);
1509
1510	case TRACE_REG_PERF_UNREGISTER:
1511		probe_event_disable(event, NULL);
1512		return 0;
1513
1514	case TRACE_REG_PERF_OPEN:
1515		return uprobe_perf_open(event, data);
1516
1517	case TRACE_REG_PERF_CLOSE:
1518		return uprobe_perf_close(event, data);
1519
1520#endif
1521	default:
1522		return 0;
1523	}
1524}
1525
1526static int uprobe_dispatcher(struct uprobe_consumer *con, struct pt_regs *regs,
1527			     __u64 *data)
1528{
1529	struct trace_uprobe *tu;
1530	struct uprobe_dispatch_data udd;
1531	struct uprobe_cpu_buffer *ucb = NULL;
1532	unsigned int flags;
1533	int ret = 0;
1534
1535	tu = container_of(con, struct trace_uprobe, consumer);
1536
1537	this_cpu_inc(*tu->nhits);
1538
1539	udd.tu = tu;
1540	udd.bp_addr = instruction_pointer(regs);
1541
1542	current->utask->vaddr = (unsigned long) &udd;
1543
1544	if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1545		return 0;
1546
1547	flags = trace_probe_load_flag(&tu->tp);
1548	if (flags & TP_FLAG_TRACE)
1549		ret |= uprobe_trace_func(tu, regs, &ucb);
1550
1551#ifdef CONFIG_PERF_EVENTS
1552	if (flags & TP_FLAG_PROFILE)
1553		ret |= uprobe_perf_func(tu, regs, &ucb);
1554#endif
1555	uprobe_buffer_put(ucb);
1556	return ret;
1557}
1558
1559static int uretprobe_dispatcher(struct uprobe_consumer *con,
1560				unsigned long func, struct pt_regs *regs,
1561				__u64 *data)
1562{
1563	struct trace_uprobe *tu;
1564	struct uprobe_dispatch_data udd;
1565	struct uprobe_cpu_buffer *ucb = NULL;
1566	unsigned int flags;
1567
1568	tu = container_of(con, struct trace_uprobe, consumer);
1569
1570	udd.tu = tu;
1571	udd.bp_addr = func;
1572
1573	current->utask->vaddr = (unsigned long) &udd;
1574
1575	if (WARN_ON_ONCE(!uprobe_cpu_buffer))
1576		return 0;
1577
1578	flags = trace_probe_load_flag(&tu->tp);
1579	if (flags & TP_FLAG_TRACE)
1580		uretprobe_trace_func(tu, func, regs, &ucb);
1581
1582#ifdef CONFIG_PERF_EVENTS
1583	if (flags & TP_FLAG_PROFILE)
1584		uretprobe_perf_func(tu, func, regs, &ucb);
1585#endif
1586	uprobe_buffer_put(ucb);
1587	return 0;
1588}
1589
1590static struct trace_event_functions uprobe_funcs = {
1591	.trace		= print_uprobe_event
1592};
1593
1594static struct trace_event_fields uprobe_fields_array[] = {
1595	{ .type = TRACE_FUNCTION_TYPE,
1596	  .define_fields = uprobe_event_define_fields },
1597	{}
1598};
1599
1600static inline void init_trace_event_call(struct trace_uprobe *tu)
1601{
1602	struct trace_event_call *call = trace_probe_event_call(&tu->tp);
1603	call->event.funcs = &uprobe_funcs;
1604	call->class->fields_array = uprobe_fields_array;
1605
1606	call->flags = TRACE_EVENT_FL_UPROBE | TRACE_EVENT_FL_CAP_ANY;
1607	call->class->reg = trace_uprobe_register;
1608}
1609
1610static int register_uprobe_event(struct trace_uprobe *tu)
1611{
1612	init_trace_event_call(tu);
1613
1614	return trace_probe_register_event_call(&tu->tp);
1615}
1616
1617static int unregister_uprobe_event(struct trace_uprobe *tu)
1618{
1619	return trace_probe_unregister_event_call(&tu->tp);
1620}
1621
1622#ifdef CONFIG_PERF_EVENTS
1623struct trace_event_call *
1624create_local_trace_uprobe(char *name, unsigned long offs,
1625			  unsigned long ref_ctr_offset, bool is_return)
1626{
1627	enum probe_print_type ptype;
1628	struct trace_uprobe *tu;
1629	struct path path;
1630	int ret;
1631
1632	ret = kern_path(name, LOOKUP_FOLLOW, &path);
1633	if (ret)
1634		return ERR_PTR(ret);
1635
1636	if (!d_is_reg(path.dentry)) {
1637		path_put(&path);
1638		return ERR_PTR(-EINVAL);
1639	}
1640
1641	/*
1642	 * local trace_kprobes are not added to dyn_event, so they are never
1643	 * searched in find_trace_kprobe(). Therefore, there is no concern of
1644	 * duplicated name "DUMMY_EVENT" here.
1645	 */
1646	tu = alloc_trace_uprobe(UPROBE_EVENT_SYSTEM, "DUMMY_EVENT", 0,
1647				is_return);
1648
1649	if (IS_ERR(tu)) {
1650		pr_info("Failed to allocate trace_uprobe.(%d)\n",
1651			(int)PTR_ERR(tu));
1652		path_put(&path);
1653		return ERR_CAST(tu);
1654	}
1655
1656	tu->offset = offs;
1657	tu->path = path;
1658	tu->ref_ctr_offset = ref_ctr_offset;
1659	tu->filename = kstrdup(name, GFP_KERNEL);
1660	if (!tu->filename) {
1661		ret = -ENOMEM;
1662		goto error;
1663	}
1664
1665	init_trace_event_call(tu);
1666
1667	ptype = is_ret_probe(tu) ? PROBE_PRINT_RETURN : PROBE_PRINT_NORMAL;
1668	if (traceprobe_set_print_fmt(&tu->tp, ptype) < 0) {
1669		ret = -ENOMEM;
1670		goto error;
1671	}
1672
1673	return trace_probe_event_call(&tu->tp);
1674error:
1675	free_trace_uprobe(tu);
1676	return ERR_PTR(ret);
1677}
1678
1679void destroy_local_trace_uprobe(struct trace_event_call *event_call)
1680{
1681	struct trace_uprobe *tu;
1682
1683	tu = trace_uprobe_primary_from_call(event_call);
1684
1685	free_trace_uprobe(tu);
1686}
1687#endif /* CONFIG_PERF_EVENTS */
1688
1689/* Make a trace interface for controlling probe points */
1690static __init int init_uprobe_trace(void)
1691{
1692	int ret;
1693
1694	ret = dyn_event_register(&trace_uprobe_ops);
1695	if (ret)
1696		return ret;
1697
1698	ret = tracing_init_dentry();
1699	if (ret)
1700		return 0;
1701
1702	trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL,
1703				    NULL, &uprobe_events_ops);
1704	/* Profile interface */
1705	trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL,
1706				    NULL, &uprobe_profile_ops);
1707	return 0;
1708}
1709
1710fs_initcall(init_uprobe_trace);