trace_eprobe.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * event probes
  4 *
  5 * Part of this code was copied from kernel/trace/trace_kprobe.c written by
  6 * Masami Hiramatsu <mhiramat@kernel.org>
  7 *
  8 * Copyright (C) 2021, VMware Inc, Steven Rostedt <rostedt@goodmis.org>
  9 * Copyright (C) 2021, VMware Inc, Tzvetomir Stoyanov tz.stoyanov@gmail.com>
 10 *
 11 */
 12#include <linux/cleanup.h>
 13#include <linux/ftrace.h>
 14#include <linux/module.h>
 15#include <linux/mutex.h>
 16
 17#include "trace_dynevent.h"
 18#include "trace_probe.h"
 19#include "trace_probe_kernel.h"
 20#include "trace_probe_tmpl.h"
 21
 22#define EPROBE_EVENT_SYSTEM "eprobes"
 23
 24struct trace_eprobe {
 25	/* tracepoint system */
 26	const char *event_system;
 27
 28	/* tracepoint event */
 29	const char *event_name;
 30
 31	/* filter string for the tracepoint */
 32	char *filter_str;
 33
 34	struct trace_event_call *event;
 35
 36	struct dyn_event	devent;
 37	struct trace_probe	tp;
 38};
 39
 40struct eprobe_data {
 41	struct trace_event_file	*file;
 42	struct trace_eprobe	*ep;
 43};
 44
 45
 46#define for_each_trace_eprobe_tp(ep, _tp) \
 47	list_for_each_entry(ep, trace_probe_probe_list(_tp), tp.list)
 48
 49static int __trace_eprobe_create(int argc, const char *argv[]);
 50
 51static void trace_event_probe_cleanup(struct trace_eprobe *ep)
 52{
 53	if (!ep)
 54		return;
 55	trace_probe_cleanup(&ep->tp);
 56	kfree(ep->event_name);
 57	kfree(ep->event_system);
 58	if (ep->event)
 59		trace_event_put_ref(ep->event);
 60	kfree(ep->filter_str);
 61	kfree(ep);
 62}
 63
 64DEFINE_FREE(trace_event_probe_cleanup, struct trace_eprobe *,
 65		if (!IS_ERR_OR_NULL(_T)) trace_event_probe_cleanup(_T))
 66
 67static struct trace_eprobe *to_trace_eprobe(struct dyn_event *ev)
 68{
 69	return container_of(ev, struct trace_eprobe, devent);
 70}
 71
 72static int eprobe_dyn_event_create(const char *raw_command)
 73{
 74	return trace_probe_create(raw_command, __trace_eprobe_create);
 75}
 76
 77static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev)
 78{
 79	struct trace_eprobe *ep = to_trace_eprobe(ev);
 80	int i;
 81
 82	seq_printf(m, "e:%s/%s", trace_probe_group_name(&ep->tp),
 83				trace_probe_name(&ep->tp));
 84	seq_printf(m, " %s.%s", ep->event_system, ep->event_name);
 85
 86	for (i = 0; i < ep->tp.nr_args; i++)
 87		seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm);
 88	seq_putc(m, '\n');
 89
 90	return 0;
 91}
 92
 93static int unregister_trace_eprobe(struct trace_eprobe *ep)
 94{
 95	/* If other probes are on the event, just unregister eprobe */
 96	if (trace_probe_has_sibling(&ep->tp))
 97		goto unreg;
 98
 99	/* Enabled event can not be unregistered */
100	if (trace_probe_is_enabled(&ep->tp))
101		return -EBUSY;
102
103	/* Will fail if probe is being used by ftrace or perf */
104	if (trace_probe_unregister_event_call(&ep->tp))
105		return -EBUSY;
106
107unreg:
108	dyn_event_remove(&ep->devent);
109	trace_probe_unlink(&ep->tp);
110
111	return 0;
112}
113
114static int eprobe_dyn_event_release(struct dyn_event *ev)
115{
116	struct trace_eprobe *ep = to_trace_eprobe(ev);
117	int ret = unregister_trace_eprobe(ep);
118
119	if (!ret)
120		trace_event_probe_cleanup(ep);
121	return ret;
122}
123
124static bool eprobe_dyn_event_is_busy(struct dyn_event *ev)
125{
126	struct trace_eprobe *ep = to_trace_eprobe(ev);
127
128	return trace_probe_is_enabled(&ep->tp);
129}
130
131static bool eprobe_dyn_event_match(const char *system, const char *event,
132			int argc, const char **argv, struct dyn_event *ev)
133{
134	struct trace_eprobe *ep = to_trace_eprobe(ev);
135	const char *slash;
136
137	/*
138	 * We match the following:
139	 *  event only			- match all eprobes with event name
140	 *  system and event only	- match all system/event probes
141	 *  system only			- match all system probes
142	 *
143	 * The below has the above satisfied with more arguments:
144	 *
145	 *  attached system/event	- If the arg has the system and event
146	 *				  the probe is attached to, match
147	 *				  probes with the attachment.
148	 *
149	 *  If any more args are given, then it requires a full match.
150	 */
151
152	/*
153	 * If system exists, but this probe is not part of that system
154	 * do not match.
155	 */
156	if (system && strcmp(trace_probe_group_name(&ep->tp), system) != 0)
157		return false;
158
159	/* Must match the event name */
160	if (event[0] != '\0' && strcmp(trace_probe_name(&ep->tp), event) != 0)
161		return false;
162
163	/* No arguments match all */
164	if (argc < 1)
165		return true;
166
167	/* First argument is the system/event the probe is attached to */
168
169	slash = strchr(argv[0], '/');
170	if (!slash)
171		slash = strchr(argv[0], '.');
172	if (!slash)
173		return false;
174
175	if (strncmp(ep->event_system, argv[0], slash - argv[0]))
176		return false;
177	if (strcmp(ep->event_name, slash + 1))
178		return false;
179
180	argc--;
181	argv++;
182
183	/* If there are no other args, then match */
184	if (argc < 1)
185		return true;
186
187	return trace_probe_match_command_args(&ep->tp, argc, argv);
188}
189
190static struct dyn_event_operations eprobe_dyn_event_ops = {
191	.create = eprobe_dyn_event_create,
192	.show = eprobe_dyn_event_show,
193	.is_busy = eprobe_dyn_event_is_busy,
194	.free = eprobe_dyn_event_release,
195	.match = eprobe_dyn_event_match,
196};
197
198static struct trace_eprobe *alloc_event_probe(const char *group,
199					      const char *this_event,
200					      struct trace_event_call *event,
201					      int nargs)
202{
203	struct trace_eprobe *ep __free(trace_event_probe_cleanup) = NULL;
204	const char *event_name;
205	const char *sys_name;
206	int ret;
207
208	if (!event)
209		return ERR_PTR(-ENODEV);
210
211	sys_name = event->class->system;
212	event_name = trace_event_name(event);
213
214	ep = kzalloc(struct_size(ep, tp.args, nargs), GFP_KERNEL);
215	if (!ep) {
216		trace_event_put_ref(event);
217		return ERR_PTR(-ENOMEM);
218	}
219	ep->event = event;
220	ep->event_name = kstrdup(event_name, GFP_KERNEL);
221	if (!ep->event_name)
222		return ERR_PTR(-ENOMEM);
223	ep->event_system = kstrdup(sys_name, GFP_KERNEL);
224	if (!ep->event_system)
225		return ERR_PTR(-ENOMEM);
226
227	ret = trace_probe_init(&ep->tp, this_event, group, false, nargs);
228	if (ret < 0)
229		return ERR_PTR(ret);
230
231	dyn_event_init(&ep->devent, &eprobe_dyn_event_ops);
232	return_ptr(ep);
233}
234
235static int eprobe_event_define_fields(struct trace_event_call *event_call)
236{
237	struct eprobe_trace_entry_head field;
238	struct trace_probe *tp;
239
240	tp = trace_probe_primary_from_call(event_call);
241	if (WARN_ON_ONCE(!tp))
242		return -ENOENT;
243
244	return traceprobe_define_arg_fields(event_call, sizeof(field), tp);
245}
246
247static struct trace_event_fields eprobe_fields_array[] = {
248	{ .type = TRACE_FUNCTION_TYPE,
249	  .define_fields = eprobe_event_define_fields },
250	{}
251};
252
253/* Event entry printers */
254static enum print_line_t
255print_eprobe_event(struct trace_iterator *iter, int flags,
256		   struct trace_event *event)
257{
258	struct eprobe_trace_entry_head *field;
259	struct trace_event_call *pevent;
260	struct trace_event *probed_event;
261	struct trace_seq *s = &iter->seq;
262	struct trace_eprobe *ep;
263	struct trace_probe *tp;
264	unsigned int type;
265
266	field = (struct eprobe_trace_entry_head *)iter->ent;
267	tp = trace_probe_primary_from_call(
268		container_of(event, struct trace_event_call, event));
269	if (WARN_ON_ONCE(!tp))
270		goto out;
271
272	ep = container_of(tp, struct trace_eprobe, tp);
273	type = ep->event->event.type;
274
275	trace_seq_printf(s, "%s: (", trace_probe_name(tp));
276
277	probed_event = ftrace_find_event(type);
278	if (probed_event) {
279		pevent = container_of(probed_event, struct trace_event_call, event);
280		trace_seq_printf(s, "%s.%s", pevent->class->system,
281				 trace_event_name(pevent));
282	} else {
283		trace_seq_printf(s, "%u", type);
284	}
285
286	trace_seq_putc(s, ')');
287
288	if (trace_probe_print_args(s, tp->args, tp->nr_args,
289			     (u8 *)&field[1], field) < 0)
290		goto out;
291
292	trace_seq_putc(s, '\n');
293 out:
294	return trace_handle_return(s);
295}
296
297static nokprobe_inline unsigned long
298get_event_field(struct fetch_insn *code, void *rec)
299{
300	struct ftrace_event_field *field = code->data;
301	unsigned long val;
302	void *addr;
303
304	addr = rec + field->offset;
305
306	if (is_string_field(field)) {
307		switch (field->filter_type) {
308		case FILTER_DYN_STRING:
309			val = (unsigned long)(rec + (*(unsigned int *)addr & 0xffff));
310			break;
311		case FILTER_RDYN_STRING:
312			val = (unsigned long)(addr + (*(unsigned int *)addr & 0xffff));
313			break;
314		case FILTER_STATIC_STRING:
315			val = (unsigned long)addr;
316			break;
317		case FILTER_PTR_STRING:
318			val = (unsigned long)(*(char *)addr);
319			break;
320		default:
321			WARN_ON_ONCE(1);
322			return 0;
323		}
324		return val;
325	}
326
327	switch (field->size) {
328	case 1:
329		if (field->is_signed)
330			val = *(char *)addr;
331		else
332			val = *(unsigned char *)addr;
333		break;
334	case 2:
335		if (field->is_signed)
336			val = *(short *)addr;
337		else
338			val = *(unsigned short *)addr;
339		break;
340	case 4:
341		if (field->is_signed)
342			val = *(int *)addr;
343		else
344			val = *(unsigned int *)addr;
345		break;
346	default:
347		if (field->size == sizeof(long)) {
348			if (field->is_signed)
349				val = *(long *)addr;
350			else
351				val = *(unsigned long *)addr;
352			break;
353		}
354		/* This is an array, point to the addr itself */
355		val = (unsigned long)addr;
356		break;
357	}
358	return val;
359}
360
361static int get_eprobe_size(struct trace_probe *tp, void *rec)
362{
363	struct fetch_insn *code;
364	struct probe_arg *arg;
365	int i, len, ret = 0;
366
367	for (i = 0; i < tp->nr_args; i++) {
368		arg = tp->args + i;
369		if (arg->dynamic) {
370			unsigned long val;
371
372			code = arg->code;
373 retry:
374			switch (code->op) {
375			case FETCH_OP_TP_ARG:
376				val = get_event_field(code, rec);
377				break;
378			case FETCH_NOP_SYMBOL:	/* Ignore a place holder */
379				code++;
380				goto retry;
381			default:
382				if (process_common_fetch_insn(code, &val) < 0)
383					continue;
384			}
385			code++;
386			len = process_fetch_insn_bottom(code, val, NULL, NULL);
387			if (len > 0)
388				ret += len;
389		}
390	}
391
392	return ret;
393}
394
395/* Kprobe specific fetch functions */
396
397/* Note that we don't verify it, since the code does not come from user space */
398static int
399process_fetch_insn(struct fetch_insn *code, void *rec, void *edata,
400		   void *dest, void *base)
401{
402	unsigned long val;
403	int ret;
404
405 retry:
406	switch (code->op) {
407	case FETCH_OP_TP_ARG:
408		val = get_event_field(code, rec);
409		break;
410	case FETCH_NOP_SYMBOL:	/* Ignore a place holder */
411		code++;
412		goto retry;
413	default:
414		ret = process_common_fetch_insn(code, &val);
415		if (ret < 0)
416			return ret;
417	}
418	code++;
419	return process_fetch_insn_bottom(code, val, dest, base);
420}
421NOKPROBE_SYMBOL(process_fetch_insn)
422
423/* eprobe handler */
424static inline void
425__eprobe_trace_func(struct eprobe_data *edata, void *rec)
426{
427	struct eprobe_trace_entry_head *entry;
428	struct trace_event_call *call = trace_probe_event_call(&edata->ep->tp);
429	struct trace_event_buffer fbuffer;
430	int dsize;
431
432	if (WARN_ON_ONCE(call != edata->file->event_call))
433		return;
434
435	if (trace_trigger_soft_disabled(edata->file))
436		return;
437
438	dsize = get_eprobe_size(&edata->ep->tp, rec);
439
440	entry = trace_event_buffer_reserve(&fbuffer, edata->file,
441					   sizeof(*entry) + edata->ep->tp.size + dsize);
442
443	if (!entry)
444		return;
445
446	entry = fbuffer.entry = ring_buffer_event_data(fbuffer.event);
447	store_trace_args(&entry[1], &edata->ep->tp, rec, NULL, sizeof(*entry), dsize);
448
449	trace_event_buffer_commit(&fbuffer);
450}
451
452/*
453 * The event probe implementation uses event triggers to get access to
454 * the event it is attached to, but is not an actual trigger. The below
455 * functions are just stubs to fulfill what is needed to use the trigger
456 * infrastructure.
457 */
458static int eprobe_trigger_init(struct event_trigger_data *data)
459{
460	return 0;
461}
462
463static void eprobe_trigger_free(struct event_trigger_data *data)
464{
465
466}
467
468static int eprobe_trigger_print(struct seq_file *m,
469				struct event_trigger_data *data)
470{
471	/* Do not print eprobe event triggers */
472	return 0;
473}
474
475static void eprobe_trigger_func(struct event_trigger_data *data,
476				struct trace_buffer *buffer, void *rec,
477				struct ring_buffer_event *rbe)
478{
479	struct eprobe_data *edata = data->private_data;
480
481	if (unlikely(!rec))
482		return;
483
484	__eprobe_trace_func(edata, rec);
485}
486
487static int eprobe_trigger_cmd_parse(struct event_command *cmd_ops,
488				    struct trace_event_file *file,
489				    char *glob, char *cmd,
490				    char *param_and_filter)
491{
492	return -1;
493}
494
495static int eprobe_trigger_reg_func(char *glob,
496				   struct event_trigger_data *data,
497				   struct trace_event_file *file)
498{
499	return -1;
500}
501
502static void eprobe_trigger_unreg_func(char *glob,
503				      struct event_trigger_data *data,
504				      struct trace_event_file *file)
505{
506
507}
508
509static struct event_command event_trigger_cmd = {
510	.name			= "eprobe",
511	.trigger_type		= ETT_EVENT_EPROBE,
512	.flags			= EVENT_CMD_FL_NEEDS_REC,
513	.parse			= eprobe_trigger_cmd_parse,
514	.reg			= eprobe_trigger_reg_func,
515	.unreg			= eprobe_trigger_unreg_func,
516	.unreg_all		= NULL,
517	.set_filter		= NULL,
518	.trigger		= eprobe_trigger_func,
519	.print			= eprobe_trigger_print,
520	.init			= eprobe_trigger_init,
521	.free			= eprobe_trigger_free,
522};
523
524static struct event_trigger_data *
525new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file)
526{
527	struct event_trigger_data *trigger;
528	struct event_filter *filter = NULL;
529	struct eprobe_data *edata;
530	int ret;
531
532	edata = kzalloc(sizeof(*edata), GFP_KERNEL);
533	trigger = kzalloc(sizeof(*trigger), GFP_KERNEL);
534	if (!trigger || !edata) {
535		ret = -ENOMEM;
536		goto error;
537	}
538
539	trigger->flags = EVENT_TRIGGER_FL_PROBE;
540	trigger->count = -1;
541
542	/*
543	 * EVENT PROBE triggers are not registered as commands with
544	 * register_event_command(), as they are not controlled by the user
545	 * from the trigger file
546	 */
547	trigger->cmd_ops = &event_trigger_cmd;
548
549	INIT_LIST_HEAD(&trigger->list);
550
551	if (ep->filter_str) {
552		ret = create_event_filter(file->tr, ep->event,
553					ep->filter_str, false, &filter);
554		if (ret)
555			goto error;
556	}
557	RCU_INIT_POINTER(trigger->filter, filter);
558
559	edata->file = file;
560	edata->ep = ep;
561	trigger->private_data = edata;
562
563	return trigger;
564error:
565	free_event_filter(filter);
566	kfree(edata);
567	kfree(trigger);
568	return ERR_PTR(ret);
569}
570
571static int enable_eprobe(struct trace_eprobe *ep,
572			 struct trace_event_file *eprobe_file)
573{
574	struct event_trigger_data *trigger;
575	struct trace_event_file *file;
576	struct trace_array *tr = eprobe_file->tr;
577
578	file = find_event_file(tr, ep->event_system, ep->event_name);
579	if (!file)
580		return -ENOENT;
581	trigger = new_eprobe_trigger(ep, eprobe_file);
582	if (IS_ERR(trigger))
583		return PTR_ERR(trigger);
584
585	list_add_tail_rcu(&trigger->list, &file->triggers);
586
587	trace_event_trigger_enable_disable(file, 1);
588	update_cond_flag(file);
589
590	return 0;
591}
592
593static struct trace_event_functions eprobe_funcs = {
594	.trace		= print_eprobe_event
595};
596
597static int disable_eprobe(struct trace_eprobe *ep,
598			  struct trace_array *tr)
599{
600	struct event_trigger_data *trigger = NULL, *iter;
601	struct trace_event_file *file;
602	struct event_filter *filter;
603	struct eprobe_data *edata;
604
605	file = find_event_file(tr, ep->event_system, ep->event_name);
606	if (!file)
607		return -ENOENT;
608
609	list_for_each_entry(iter, &file->triggers, list) {
610		if (!(iter->flags & EVENT_TRIGGER_FL_PROBE))
611			continue;
612		edata = iter->private_data;
613		if (edata->ep == ep) {
614			trigger = iter;
615			break;
616		}
617	}
618	if (!trigger)
619		return -ENODEV;
620
621	list_del_rcu(&trigger->list);
622
623	trace_event_trigger_enable_disable(file, 0);
624	update_cond_flag(file);
625
626	/* Make sure nothing is using the edata or trigger */
627	tracepoint_synchronize_unregister();
628
629	filter = rcu_access_pointer(trigger->filter);
630
631	if (filter)
632		free_event_filter(filter);
633	kfree(edata);
634	kfree(trigger);
635
636	return 0;
637}
638
639static int enable_trace_eprobe(struct trace_event_call *call,
640			       struct trace_event_file *file)
641{
642	struct trace_probe *tp;
643	struct trace_eprobe *ep;
644	bool enabled;
645	int ret = 0;
646	int cnt = 0;
647
648	tp = trace_probe_primary_from_call(call);
649	if (WARN_ON_ONCE(!tp))
650		return -ENODEV;
651	enabled = trace_probe_is_enabled(tp);
652
653	/* This also changes "enabled" state */
654	if (file) {
655		ret = trace_probe_add_file(tp, file);
656		if (ret)
657			return ret;
658	} else
659		trace_probe_set_flag(tp, TP_FLAG_PROFILE);
660
661	if (enabled)
662		return 0;
663
664	for_each_trace_eprobe_tp(ep, tp) {
665		ret = enable_eprobe(ep, file);
666		if (ret)
667			break;
668		enabled = true;
669		cnt++;
670	}
671
672	if (ret) {
673		/* Failed to enable one of them. Roll back all */
674		if (enabled) {
675			/*
676			 * It's a bug if one failed for something other than memory
677			 * not being available but another eprobe succeeded.
678			 */
679			WARN_ON_ONCE(ret != -ENOMEM);
680
681			for_each_trace_eprobe_tp(ep, tp) {
682				disable_eprobe(ep, file->tr);
683				if (!--cnt)
684					break;
685			}
686		}
687		if (file)
688			trace_probe_remove_file(tp, file);
689		else
690			trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
691	}
692
693	return ret;
694}
695
696static int disable_trace_eprobe(struct trace_event_call *call,
697				struct trace_event_file *file)
698{
699	struct trace_probe *tp;
700	struct trace_eprobe *ep;
701
702	tp = trace_probe_primary_from_call(call);
703	if (WARN_ON_ONCE(!tp))
704		return -ENODEV;
705
706	if (file) {
707		if (!trace_probe_get_file_link(tp, file))
708			return -ENOENT;
709		if (!trace_probe_has_single_file(tp))
710			goto out;
711		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
712	} else
713		trace_probe_clear_flag(tp, TP_FLAG_PROFILE);
714
715	if (!trace_probe_is_enabled(tp)) {
716		for_each_trace_eprobe_tp(ep, tp)
717			disable_eprobe(ep, file->tr);
718	}
719
720 out:
721	if (file)
722		/*
723		 * Synchronization is done in below function. For perf event,
724		 * file == NULL and perf_trace_event_unreg() calls
725		 * tracepoint_synchronize_unregister() to ensure synchronize
726		 * event. We don't need to care about it.
727		 */
728		trace_probe_remove_file(tp, file);
729
730	return 0;
731}
732
733static int eprobe_register(struct trace_event_call *event,
734			   enum trace_reg type, void *data)
735{
736	struct trace_event_file *file = data;
737
738	switch (type) {
739	case TRACE_REG_REGISTER:
740		return enable_trace_eprobe(event, file);
741	case TRACE_REG_UNREGISTER:
742		return disable_trace_eprobe(event, file);
743#ifdef CONFIG_PERF_EVENTS
744	case TRACE_REG_PERF_REGISTER:
745	case TRACE_REG_PERF_UNREGISTER:
746	case TRACE_REG_PERF_OPEN:
747	case TRACE_REG_PERF_CLOSE:
748	case TRACE_REG_PERF_ADD:
749	case TRACE_REG_PERF_DEL:
750		return 0;
751#endif
752	}
753	return 0;
754}
755
756static inline void init_trace_eprobe_call(struct trace_eprobe *ep)
757{
758	struct trace_event_call *call = trace_probe_event_call(&ep->tp);
759
760	call->flags = TRACE_EVENT_FL_EPROBE;
761	call->event.funcs = &eprobe_funcs;
762	call->class->fields_array = eprobe_fields_array;
763	call->class->reg = eprobe_register;
764}
765
766static struct trace_event_call *
767find_and_get_event(const char *system, const char *event_name)
768{
769	struct trace_event_call *tp_event;
770	const char *name;
771
772	list_for_each_entry(tp_event, &ftrace_events, list) {
773		/* Skip other probes and ftrace events */
774		if (tp_event->flags &
775		    (TRACE_EVENT_FL_IGNORE_ENABLE |
776		     TRACE_EVENT_FL_KPROBE |
777		     TRACE_EVENT_FL_UPROBE |
778		     TRACE_EVENT_FL_EPROBE))
779			continue;
780		if (!tp_event->class->system ||
781		    strcmp(system, tp_event->class->system))
782			continue;
783		name = trace_event_name(tp_event);
784		if (!name || strcmp(event_name, name))
785			continue;
786		if (!trace_event_try_get_ref(tp_event))
787			return NULL;
788		return tp_event;
789	}
790	return NULL;
791}
792
793static int trace_eprobe_parse_filter(struct trace_eprobe *ep, int argc, const char *argv[])
794{
795	struct event_filter *dummy = NULL;
796	int i, ret, len = 0;
797	char *p;
798
799	if (argc == 0) {
800		trace_probe_log_err(0, NO_EP_FILTER);
801		return -EINVAL;
802	}
803
804	/* Recover the filter string */
805	for (i = 0; i < argc; i++)
806		len += strlen(argv[i]) + 1;
807
808	ep->filter_str = kzalloc(len, GFP_KERNEL);
809	if (!ep->filter_str)
810		return -ENOMEM;
811
812	p = ep->filter_str;
813	for (i = 0; i < argc; i++) {
814		if (i)
815			ret = snprintf(p, len, " %s", argv[i]);
816		else
817			ret = snprintf(p, len, "%s", argv[i]);
818		p += ret;
819		len -= ret;
820	}
821
822	/*
823	 * Ensure the filter string can be parsed correctly. Note, this
824	 * filter string is for the original event, not for the eprobe.
825	 */
826	ret = create_event_filter(top_trace_array(), ep->event, ep->filter_str,
827				  true, &dummy);
828	free_event_filter(dummy);
829	if (ret) {
830		kfree(ep->filter_str);
831		ep->filter_str = NULL;
832	}
833	return ret;
834}
835
836static int __trace_eprobe_create(int argc, const char *argv[])
837{
838	/*
839	 * Argument syntax:
840	 *      e[:[GRP/][ENAME]] SYSTEM.EVENT [FETCHARGS] [if FILTER]
841	 * Fetch args (no space):
842	 *  <name>=$<field>[:TYPE]
843	 */
844	struct traceprobe_parse_context *ctx __free(traceprobe_parse_context) = NULL;
845	struct trace_eprobe *ep __free(trace_event_probe_cleanup) = NULL;
846	const char *trlog __free(trace_probe_log_clear) = NULL;
847	const char *event = NULL, *group = EPROBE_EVENT_SYSTEM;
848	const char *sys_event = NULL, *sys_name = NULL;
849	struct trace_event_call *event_call;
850	char *buf1 __free(kfree) = NULL;
851	char *buf2 __free(kfree) = NULL;
852	char *gbuf __free(kfree) = NULL;
853	int ret = 0, filter_idx = 0;
854	int i, filter_cnt;
855
856	if (argc < 2 || argv[0][0] != 'e')
857		return -ECANCELED;
858
859	trlog = trace_probe_log_init("event_probe", argc, argv);
860
861	event = strchr(&argv[0][1], ':');
862	if (event) {
863		gbuf = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
864		if (!gbuf)
865			return -ENOMEM;
866		event++;
867		ret = traceprobe_parse_event_name(&event, &group, gbuf,
868						  event - argv[0]);
869		if (ret)
870			return -EINVAL;
871	}
872
873	trace_probe_log_set_index(1);
874	sys_event = argv[1];
875
876	buf2 = kmalloc(MAX_EVENT_NAME_LEN, GFP_KERNEL);
877	if (!buf2)
878		return -ENOMEM;
879
880	ret = traceprobe_parse_event_name(&sys_event, &sys_name, buf2, 0);
881	if (ret || !sys_event || !sys_name) {
882		trace_probe_log_err(0, NO_EVENT_INFO);
883		return -EINVAL;
884	}
885
886	if (!event) {
887		buf1 = kstrdup(sys_event, GFP_KERNEL);
888		if (!buf1)
889			return -ENOMEM;
890		event = buf1;
891	}
892
893	for (i = 2; i < argc; i++) {
894		if (!strcmp(argv[i], "if")) {
895			filter_idx = i + 1;
896			filter_cnt = argc - filter_idx;
897			argc = i;
898			break;
899		}
900	}
901
902	if (argc - 2 > MAX_TRACE_ARGS) {
903		trace_probe_log_set_index(2);
904		trace_probe_log_err(0, TOO_MANY_ARGS);
905		return -E2BIG;
906	}
907
908	scoped_guard(mutex, &event_mutex) {
909		event_call = find_and_get_event(sys_name, sys_event);
910		ep = alloc_event_probe(group, event, event_call, argc - 2);
911	}
912
913	if (IS_ERR(ep)) {
914		ret = PTR_ERR(ep);
915		if (ret == -ENODEV)
916			trace_probe_log_err(0, BAD_ATTACH_EVENT);
917		/* This must return -ENOMEM or missing event, else there is a bug */
918		WARN_ON_ONCE(ret != -ENOMEM && ret != -ENODEV);
919		return ret;
920	}
921
922	if (filter_idx) {
923		trace_probe_log_set_index(filter_idx);
924		ret = trace_eprobe_parse_filter(ep, filter_cnt, argv + filter_idx);
925		if (ret)
926			return -EINVAL;
927	} else
928		ep->filter_str = NULL;
929
930	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
931	if (!ctx)
932		return -ENOMEM;
933	ctx->event = ep->event;
934	ctx->flags = TPARG_FL_KERNEL | TPARG_FL_TEVENT;
935
936	argc -= 2; argv += 2;
937	/* parse arguments */
938	for (i = 0; i < argc; i++) {
939		trace_probe_log_set_index(i + 2);
940
941		ret = traceprobe_parse_probe_arg(&ep->tp, i, argv[i], ctx);
942		/* Handle symbols "@" */
943		if (!ret)
944			ret = traceprobe_update_arg(&ep->tp.args[i]);
945		if (ret)
946			return ret;
947	}
948	ret = traceprobe_set_print_fmt(&ep->tp, PROBE_PRINT_EVENT);
949	if (ret < 0)
950		return ret;
951
952	init_trace_eprobe_call(ep);
953	scoped_guard(mutex, &event_mutex) {
954		ret = trace_probe_register_event_call(&ep->tp);
955		if (ret) {
956			if (ret == -EEXIST) {
957				trace_probe_log_set_index(0);
958				trace_probe_log_err(0, EVENT_EXIST);
959			}
960			return ret;
961		}
962		ret = dyn_event_add(&ep->devent, &ep->tp.event->call);
963		if (ret < 0) {
964			trace_probe_unregister_event_call(&ep->tp);
965			return ret;
966		}
967		/* To avoid freeing registered eprobe event, clear ep. */
968		ep = NULL;
969	}
970	return ret;
971}
972
973/*
974 * Register dynevent at core_initcall. This allows kernel to setup eprobe
975 * events in postcore_initcall without tracefs.
976 */
977static __init int trace_events_eprobe_init_early(void)
978{
979	int err = 0;
980
981	err = dyn_event_register(&eprobe_dyn_event_ops);
982	if (err)
983		pr_warn("Could not register eprobe_dyn_event_ops\n");
984
985	return err;
986}
987core_initcall(trace_events_eprobe_init_early);