trace_events.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2
  3#ifndef _LINUX_TRACE_EVENT_H
  4#define _LINUX_TRACE_EVENT_H
  5
  6#include <linux/ring_buffer.h>
  7#include <linux/trace_seq.h>
  8#include <linux/percpu.h>
  9#include <linux/hardirq.h>
 10#include <linux/perf_event.h>
 11#include <linux/tracepoint.h>
 12
 13struct trace_array;
 14struct array_buffer;
 15struct tracer;
 16struct dentry;
 17struct bpf_prog;
 18union bpf_attr;
 19
 20/* Used for event string fields when they are NULL */
 21#define EVENT_NULL_STR		"(null)"
 22
 23const char *trace_print_flags_seq(struct trace_seq *p, const char *delim,
 24				  unsigned long flags,
 25				  const struct trace_print_flags *flag_array);
 26
 27const char *trace_print_symbols_seq(struct trace_seq *p, unsigned long val,
 28				    const struct trace_print_flags *symbol_array);
 29
 30#if BITS_PER_LONG == 32
 31const char *trace_print_flags_seq_u64(struct trace_seq *p, const char *delim,
 32		      unsigned long long flags,
 33		      const struct trace_print_flags_u64 *flag_array);
 34
 35const char *trace_print_symbols_seq_u64(struct trace_seq *p,
 36					unsigned long long val,
 37					const struct trace_print_flags_u64
 38								 *symbol_array);
 39#endif
 40
 41const char *trace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr,
 42				    unsigned int bitmask_size);
 43
 44const char *trace_print_hex_seq(struct trace_seq *p,
 45				const unsigned char *buf, int len,
 46				bool concatenate);
 47
 48const char *trace_print_array_seq(struct trace_seq *p,
 49				   const void *buf, int count,
 50				   size_t el_size);
 51
 52const char *
 53trace_print_hex_dump_seq(struct trace_seq *p, const char *prefix_str,
 54			 int prefix_type, int rowsize, int groupsize,
 55			 const void *buf, size_t len, bool ascii);
 56
 57struct trace_iterator;
 58struct trace_event;
 59
 60int trace_raw_output_prep(struct trace_iterator *iter,
 61			  struct trace_event *event);
 62extern __printf(2, 3)
 63void trace_event_printf(struct trace_iterator *iter, const char *fmt, ...);
 64
 65/* Used to find the offset and length of dynamic fields in trace events */
 66struct trace_dynamic_info {
 67#ifdef CONFIG_CPU_BIG_ENDIAN
 68	u16	len;
 69	u16	offset;
 70#else
 71	u16	offset;
 72	u16	len;
 73#endif
 74} __packed;
 75
 76/*
 77 * The trace entry - the most basic unit of tracing. This is what
 78 * is printed in the end as a single line in the trace output, such as:
 79 *
 80 *     bash-15816 [01]   235.197585: idle_cpu <- irq_enter
 81 */
 82struct trace_entry {
 83	unsigned short		type;
 84	unsigned char		flags;
 85	unsigned char		preempt_count;
 86	int			pid;
 87};
 88
 89#define TRACE_EVENT_TYPE_MAX						\
 90	((1 << (sizeof(((struct trace_entry *)0)->type) * 8)) - 1)
 91
 92/*
 93 * Trace iterator - used by printout routines who present trace
 94 * results to users and which routines might sleep, etc:
 95 */
 96struct trace_iterator {
 97	struct trace_array	*tr;
 98	struct tracer		*trace;
 99	struct array_buffer	*array_buffer;
100	void			*private;
101	int			cpu_file;
102	struct mutex		mutex;
103	struct ring_buffer_iter	**buffer_iter;
104	unsigned long		iter_flags;
105	void			*temp;	/* temp holder */
106	unsigned int		temp_size;
107	char			*fmt;	/* modified format holder */
108	unsigned int		fmt_size;
109	atomic_t		wait_index;
110
111	/* trace_seq for __print_flags() and __print_symbolic() etc. */
112	struct trace_seq	tmp_seq;
113
114	cpumask_var_t		started;
115
116	/* Set when the file is closed to prevent new waiters */
117	bool			closed;
118
119	/* it's true when current open file is snapshot */
120	bool			snapshot;
121
122	/* The below is zeroed out in pipe_read */
123	struct trace_seq	seq;
124	struct trace_entry	*ent;
125	unsigned long		lost_events;
126	int			leftover;
127	int			ent_size;
128	int			cpu;
129	u64			ts;
130
131	loff_t			pos;
132	long			idx;
133
134	/* All new field here will be zeroed out in pipe_read */
135};
136
137enum trace_iter_flags {
138	TRACE_FILE_LAT_FMT	= 1,
139	TRACE_FILE_ANNOTATE	= 2,
140	TRACE_FILE_TIME_IN_NS	= 4,
141	TRACE_FILE_PAUSE	= 8,
142};
143
144
145typedef enum print_line_t (*trace_print_func)(struct trace_iterator *iter,
146				      int flags, struct trace_event *event);
147
148struct trace_event_functions {
149	trace_print_func	trace;
150	trace_print_func	raw;
151	trace_print_func	hex;
152	trace_print_func	binary;
153};
154
155struct trace_event {
156	struct hlist_node		node;
157	int				type;
158	struct trace_event_functions	*funcs;
159};
160
161extern int register_trace_event(struct trace_event *event);
162extern int unregister_trace_event(struct trace_event *event);
163
164/* Return values for print_line callback */
165enum print_line_t {
166	TRACE_TYPE_PARTIAL_LINE	= 0,	/* Retry after flushing the seq */
167	TRACE_TYPE_HANDLED	= 1,
168	TRACE_TYPE_UNHANDLED	= 2,	/* Relay to other output functions */
169	TRACE_TYPE_NO_CONSUME	= 3	/* Handled but ask to not consume */
170};
171
172enum print_line_t trace_handle_return(struct trace_seq *s);
173
174static inline void tracing_generic_entry_update(struct trace_entry *entry,
175						unsigned short type,
176						unsigned int trace_ctx)
177{
178	entry->preempt_count		= trace_ctx & 0xff;
179	entry->pid			= current->pid;
180	entry->type			= type;
181	entry->flags =			trace_ctx >> 16;
182}
183
184unsigned int tracing_gen_ctx_irq_test(unsigned int irqs_status);
185
186enum trace_flag_type {
187	TRACE_FLAG_IRQS_OFF		= 0x01,
188	TRACE_FLAG_NEED_RESCHED_LAZY	= 0x02,
189	TRACE_FLAG_NEED_RESCHED		= 0x04,
190	TRACE_FLAG_HARDIRQ		= 0x08,
191	TRACE_FLAG_SOFTIRQ		= 0x10,
192	TRACE_FLAG_PREEMPT_RESCHED	= 0x20,
193	TRACE_FLAG_NMI			= 0x40,
194	TRACE_FLAG_BH_OFF		= 0x80,
195};
196
197static inline unsigned int tracing_gen_ctx_flags(unsigned long irqflags)
198{
199	unsigned int irq_status = irqs_disabled_flags(irqflags) ?
200		TRACE_FLAG_IRQS_OFF : 0;
201	return tracing_gen_ctx_irq_test(irq_status);
202}
203static inline unsigned int tracing_gen_ctx(void)
204{
205	unsigned long irqflags;
206
207	local_save_flags(irqflags);
208	return tracing_gen_ctx_flags(irqflags);
209}
210
211static inline unsigned int tracing_gen_ctx_dec(void)
212{
213	unsigned int trace_ctx;
214
215	trace_ctx = tracing_gen_ctx();
216	/*
217	 * Subtract one from the preemption counter if preemption is enabled,
218	 * see trace_event_buffer_reserve()for details.
219	 */
220	if (IS_ENABLED(CONFIG_PREEMPTION))
221		trace_ctx--;
222	return trace_ctx;
223}
224
225struct trace_event_file;
226
227struct ring_buffer_event *
228trace_event_buffer_lock_reserve(struct trace_buffer **current_buffer,
229				struct trace_event_file *trace_file,
230				int type, unsigned long len,
231				unsigned int trace_ctx);
232
233#define TRACE_RECORD_CMDLINE	BIT(0)
234#define TRACE_RECORD_TGID	BIT(1)
235
236void tracing_record_taskinfo(struct task_struct *task, int flags);
237void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
238					  struct task_struct *next, int flags);
239
240void tracing_record_cmdline(struct task_struct *task);
241void tracing_record_tgid(struct task_struct *task);
242
243int trace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...)
244	 __printf(3, 4);
245
246struct event_filter;
247
248enum trace_reg {
249	TRACE_REG_REGISTER,
250	TRACE_REG_UNREGISTER,
251#ifdef CONFIG_PERF_EVENTS
252	TRACE_REG_PERF_REGISTER,
253	TRACE_REG_PERF_UNREGISTER,
254	TRACE_REG_PERF_OPEN,
255	TRACE_REG_PERF_CLOSE,
256	/*
257	 * These (ADD/DEL) use a 'boolean' return value, where 1 (true) means a
258	 * custom action was taken and the default action is not to be
259	 * performed.
260	 */
261	TRACE_REG_PERF_ADD,
262	TRACE_REG_PERF_DEL,
263#endif
264};
265
266struct trace_event_call;
267
268#define TRACE_FUNCTION_TYPE ((const char *)~0UL)
269
270struct trace_event_fields {
271	const char *type;
272	union {
273		struct {
274			const char *name;
275			const int  size;
276			const int  align;
277			const unsigned int is_signed:1;
278			unsigned int needs_test:1;
279			const int  filter_type;
280			const int  len;
281		};
282		int (*define_fields)(struct trace_event_call *);
283	};
284};
285
286struct trace_event_class {
287	const char		*system;
288	void			*probe;
289#ifdef CONFIG_PERF_EVENTS
290	void			*perf_probe;
291#endif
292	int			(*reg)(struct trace_event_call *event,
293				       enum trace_reg type, void *data);
294	struct trace_event_fields *fields_array;
295	struct list_head	*(*get_fields)(struct trace_event_call *);
296	struct list_head	fields;
297	int			(*raw_init)(struct trace_event_call *);
298};
299
300extern int trace_event_reg(struct trace_event_call *event,
301			    enum trace_reg type, void *data);
302
303struct trace_event_buffer {
304	struct trace_buffer		*buffer;
305	struct ring_buffer_event	*event;
306	struct trace_event_file		*trace_file;
307	void				*entry;
308	unsigned int			trace_ctx;
309	struct pt_regs			*regs;
310};
311
312void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer,
313				  struct trace_event_file *trace_file,
314				  unsigned long len);
315
316void trace_event_buffer_commit(struct trace_event_buffer *fbuffer);
317
318enum {
319	TRACE_EVENT_FL_CAP_ANY_BIT,
320	TRACE_EVENT_FL_NO_SET_FILTER_BIT,
321	TRACE_EVENT_FL_IGNORE_ENABLE_BIT,
322	TRACE_EVENT_FL_TRACEPOINT_BIT,
323	TRACE_EVENT_FL_DYNAMIC_BIT,
324	TRACE_EVENT_FL_KPROBE_BIT,
325	TRACE_EVENT_FL_UPROBE_BIT,
326	TRACE_EVENT_FL_EPROBE_BIT,
327	TRACE_EVENT_FL_FPROBE_BIT,
328	TRACE_EVENT_FL_CUSTOM_BIT,
329	TRACE_EVENT_FL_TEST_STR_BIT,
330};
331
332/*
333 * Event flags:
334 *  CAP_ANY	  - Any user can enable for perf
335 *  NO_SET_FILTER - Set when filter has error and is to be ignored
336 *  IGNORE_ENABLE - For trace internal events, do not enable with debugfs file
337 *  TRACEPOINT    - Event is a tracepoint
338 *  DYNAMIC       - Event is a dynamic event (created at run time)
339 *  KPROBE        - Event is a kprobe
340 *  UPROBE        - Event is a uprobe
341 *  EPROBE        - Event is an event probe
342 *  FPROBE        - Event is an function probe
343 *  CUSTOM        - Event is a custom event (to be attached to an exsiting tracepoint)
344 *                   This is set when the custom event has not been attached
345 *                   to a tracepoint yet, then it is cleared when it is.
346 *  TEST_STR      - The event has a "%s" that points to a string outside the event
347 */
348enum {
349	TRACE_EVENT_FL_CAP_ANY		= (1 << TRACE_EVENT_FL_CAP_ANY_BIT),
350	TRACE_EVENT_FL_NO_SET_FILTER	= (1 << TRACE_EVENT_FL_NO_SET_FILTER_BIT),
351	TRACE_EVENT_FL_IGNORE_ENABLE	= (1 << TRACE_EVENT_FL_IGNORE_ENABLE_BIT),
352	TRACE_EVENT_FL_TRACEPOINT	= (1 << TRACE_EVENT_FL_TRACEPOINT_BIT),
353	TRACE_EVENT_FL_DYNAMIC		= (1 << TRACE_EVENT_FL_DYNAMIC_BIT),
354	TRACE_EVENT_FL_KPROBE		= (1 << TRACE_EVENT_FL_KPROBE_BIT),
355	TRACE_EVENT_FL_UPROBE		= (1 << TRACE_EVENT_FL_UPROBE_BIT),
356	TRACE_EVENT_FL_EPROBE		= (1 << TRACE_EVENT_FL_EPROBE_BIT),
357	TRACE_EVENT_FL_FPROBE		= (1 << TRACE_EVENT_FL_FPROBE_BIT),
358	TRACE_EVENT_FL_CUSTOM		= (1 << TRACE_EVENT_FL_CUSTOM_BIT),
359	TRACE_EVENT_FL_TEST_STR		= (1 << TRACE_EVENT_FL_TEST_STR_BIT),
360};
361
362#define TRACE_EVENT_FL_UKPROBE (TRACE_EVENT_FL_KPROBE | TRACE_EVENT_FL_UPROBE)
363
364struct trace_event_call {
365	struct list_head	list;
366	struct trace_event_class *class;
367	union {
368		const char		*name;
369		/* Set TRACE_EVENT_FL_TRACEPOINT flag when using "tp" */
370		struct tracepoint	*tp;
371	};
372	struct trace_event	event;
373	char			*print_fmt;
374	/*
375	 * Static events can disappear with modules,
376	 * where as dynamic ones need their own ref count.
377	 */
378	union {
379		void				*module;
380		atomic_t			refcnt;
381	};
382	void			*data;
383
384	/* See the TRACE_EVENT_FL_* flags above */
385	int			flags; /* static flags of different events */
386
387#ifdef CONFIG_PERF_EVENTS
388	int				perf_refcount;
389	struct hlist_head __percpu	*perf_events;
390	struct bpf_prog_array __rcu	*prog_array;
391
392	int	(*perf_perm)(struct trace_event_call *,
393			     struct perf_event *);
394#endif
395};
396
397#ifdef CONFIG_DYNAMIC_EVENTS
398bool trace_event_dyn_try_get_ref(struct trace_event_call *call);
399void trace_event_dyn_put_ref(struct trace_event_call *call);
400bool trace_event_dyn_busy(struct trace_event_call *call);
401#else
402static inline bool trace_event_dyn_try_get_ref(struct trace_event_call *call)
403{
404	/* Without DYNAMIC_EVENTS configured, nothing should be calling this */
405	return false;
406}
407static inline void trace_event_dyn_put_ref(struct trace_event_call *call)
408{
409}
410static inline bool trace_event_dyn_busy(struct trace_event_call *call)
411{
412	/* Nothing should call this without DYNAIMIC_EVENTS configured. */
413	return true;
414}
415#endif
416
417static inline bool trace_event_try_get_ref(struct trace_event_call *call)
418{
419	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
420		return trace_event_dyn_try_get_ref(call);
421	else
422		return try_module_get(call->module);
423}
424
425static inline void trace_event_put_ref(struct trace_event_call *call)
426{
427	if (call->flags & TRACE_EVENT_FL_DYNAMIC)
428		trace_event_dyn_put_ref(call);
429	else
430		module_put(call->module);
431}
432
433#ifdef CONFIG_PERF_EVENTS
434static inline bool bpf_prog_array_valid(struct trace_event_call *call)
435{
436	/*
437	 * This inline function checks whether call->prog_array
438	 * is valid or not. The function is called in various places,
439	 * outside rcu_read_lock/unlock, as a heuristic to speed up execution.
440	 *
441	 * If this function returns true, and later call->prog_array
442	 * becomes false inside rcu_read_lock/unlock region,
443	 * we bail out then. If this function return false,
444	 * there is a risk that we might miss a few events if the checking
445	 * were delayed until inside rcu_read_lock/unlock region and
446	 * call->prog_array happened to become non-NULL then.
447	 *
448	 * Here, READ_ONCE() is used instead of rcu_access_pointer().
449	 * rcu_access_pointer() requires the actual definition of
450	 * "struct bpf_prog_array" while READ_ONCE() only needs
451	 * a declaration of the same type.
452	 */
453	return !!READ_ONCE(call->prog_array);
454}
455#endif
456
457static inline const char *
458trace_event_name(struct trace_event_call *call)
459{
460	if (call->flags & TRACE_EVENT_FL_CUSTOM)
461		return call->name;
462	else if (call->flags & TRACE_EVENT_FL_TRACEPOINT)
463		return call->tp ? call->tp->name : NULL;
464	else
465		return call->name;
466}
467
468static inline struct list_head *
469trace_get_fields(struct trace_event_call *event_call)
470{
471	if (!event_call->class->get_fields)
472		return &event_call->class->fields;
473	return event_call->class->get_fields(event_call);
474}
475
476struct trace_subsystem_dir;
477
478enum {
479	EVENT_FILE_FL_ENABLED_BIT,
480	EVENT_FILE_FL_RECORDED_CMD_BIT,
481	EVENT_FILE_FL_RECORDED_TGID_BIT,
482	EVENT_FILE_FL_FILTERED_BIT,
483	EVENT_FILE_FL_NO_SET_FILTER_BIT,
484	EVENT_FILE_FL_SOFT_DISABLED_BIT,
485	EVENT_FILE_FL_TRIGGER_MODE_BIT,
486	EVENT_FILE_FL_TRIGGER_COND_BIT,
487	EVENT_FILE_FL_PID_FILTER_BIT,
488	EVENT_FILE_FL_WAS_ENABLED_BIT,
489	EVENT_FILE_FL_FREED_BIT,
490};
491
492extern struct trace_event_file *trace_get_event_file(const char *instance,
493						     const char *system,
494						     const char *event);
495extern void trace_put_event_file(struct trace_event_file *file);
496
497#define MAX_DYNEVENT_CMD_LEN	(2048)
498
499enum dynevent_type {
500	DYNEVENT_TYPE_SYNTH = 1,
501	DYNEVENT_TYPE_KPROBE,
502	DYNEVENT_TYPE_NONE,
503};
504
505struct dynevent_cmd;
506
507typedef int (*dynevent_create_fn_t)(struct dynevent_cmd *cmd);
508
509struct dynevent_cmd {
510	struct seq_buf		seq;
511	const char		*event_name;
512	unsigned int		n_fields;
513	enum dynevent_type	type;
514	dynevent_create_fn_t	run_command;
515	void			*private_data;
516};
517
518extern int dynevent_create(struct dynevent_cmd *cmd);
519
520extern int synth_event_delete(const char *name);
521
522extern void synth_event_cmd_init(struct dynevent_cmd *cmd,
523				 char *buf, int maxlen);
524
525extern int __synth_event_gen_cmd_start(struct dynevent_cmd *cmd,
526				       const char *name,
527				       struct module *mod, ...);
528
529#define synth_event_gen_cmd_start(cmd, name, mod, ...)	\
530	__synth_event_gen_cmd_start(cmd, name, mod, ## __VA_ARGS__, NULL)
531
532struct synth_field_desc {
533	const char *type;
534	const char *name;
535};
536
537extern int synth_event_gen_cmd_array_start(struct dynevent_cmd *cmd,
538					   const char *name,
539					   struct module *mod,
540					   struct synth_field_desc *fields,
541					   unsigned int n_fields);
542extern int synth_event_create(const char *name,
543			      struct synth_field_desc *fields,
544			      unsigned int n_fields, struct module *mod);
545
546extern int synth_event_add_field(struct dynevent_cmd *cmd,
547				 const char *type,
548				 const char *name);
549extern int synth_event_add_field_str(struct dynevent_cmd *cmd,
550				     const char *type_name);
551extern int synth_event_add_fields(struct dynevent_cmd *cmd,
552				  struct synth_field_desc *fields,
553				  unsigned int n_fields);
554
555#define synth_event_gen_cmd_end(cmd)	\
556	dynevent_create(cmd)
557
558struct synth_event;
559
560struct synth_event_trace_state {
561	struct trace_event_buffer fbuffer;
562	struct synth_trace_event *entry;
563	struct trace_buffer *buffer;
564	struct synth_event *event;
565	unsigned int cur_field;
566	unsigned int n_u64;
567	bool disabled;
568	bool add_next;
569	bool add_name;
570};
571
572extern int synth_event_trace(struct trace_event_file *file,
573			     unsigned int n_vals, ...);
574extern int synth_event_trace_array(struct trace_event_file *file, u64 *vals,
575				   unsigned int n_vals);
576extern int synth_event_trace_start(struct trace_event_file *file,
577				   struct synth_event_trace_state *trace_state);
578extern int synth_event_add_next_val(u64 val,
579				    struct synth_event_trace_state *trace_state);
580extern int synth_event_add_val(const char *field_name, u64 val,
581			       struct synth_event_trace_state *trace_state);
582extern int synth_event_trace_end(struct synth_event_trace_state *trace_state);
583
584extern int kprobe_event_delete(const char *name);
585
586extern void kprobe_event_cmd_init(struct dynevent_cmd *cmd,
587				  char *buf, int maxlen);
588
589#define kprobe_event_gen_cmd_start(cmd, name, loc, ...)			\
590	__kprobe_event_gen_cmd_start(cmd, false, name, loc, ## __VA_ARGS__, NULL)
591
592#define kretprobe_event_gen_cmd_start(cmd, name, loc, ...)		\
593	__kprobe_event_gen_cmd_start(cmd, true, name, loc, ## __VA_ARGS__, NULL)
594
595extern int __kprobe_event_gen_cmd_start(struct dynevent_cmd *cmd,
596					bool kretprobe,
597					const char *name,
598					const char *loc, ...);
599
600#define kprobe_event_add_fields(cmd, ...)	\
601	__kprobe_event_add_fields(cmd, ## __VA_ARGS__, NULL)
602
603#define kprobe_event_add_field(cmd, field)	\
604	__kprobe_event_add_fields(cmd, field, NULL)
605
606extern int __kprobe_event_add_fields(struct dynevent_cmd *cmd, ...);
607
608#define kprobe_event_gen_cmd_end(cmd)		\
609	dynevent_create(cmd)
610
611#define kretprobe_event_gen_cmd_end(cmd)	\
612	dynevent_create(cmd)
613
614/*
615 * Event file flags:
616 *  ENABLED	  - The event is enabled
617 *  RECORDED_CMD  - The comms should be recorded at sched_switch
618 *  RECORDED_TGID - The tgids should be recorded at sched_switch
619 *  FILTERED	  - The event has a filter attached
620 *  NO_SET_FILTER - Set when filter has error and is to be ignored
621 *  SOFT_DISABLED - When set, do not trace the event (even though its
622 *                   tracepoint may be enabled)
623 *  TRIGGER_MODE  - When set, invoke the triggers associated with the event
624 *  TRIGGER_COND  - When set, one or more triggers has an associated filter
625 *  PID_FILTER    - When set, the event is filtered based on pid
626 *  WAS_ENABLED   - Set when enabled to know to clear trace on module removal
627 *  FREED         - File descriptor is freed, all fields should be considered invalid
628 */
629enum {
630	EVENT_FILE_FL_ENABLED		= (1 << EVENT_FILE_FL_ENABLED_BIT),
631	EVENT_FILE_FL_RECORDED_CMD	= (1 << EVENT_FILE_FL_RECORDED_CMD_BIT),
632	EVENT_FILE_FL_RECORDED_TGID	= (1 << EVENT_FILE_FL_RECORDED_TGID_BIT),
633	EVENT_FILE_FL_FILTERED		= (1 << EVENT_FILE_FL_FILTERED_BIT),
634	EVENT_FILE_FL_NO_SET_FILTER	= (1 << EVENT_FILE_FL_NO_SET_FILTER_BIT),
635	EVENT_FILE_FL_SOFT_DISABLED	= (1 << EVENT_FILE_FL_SOFT_DISABLED_BIT),
636	EVENT_FILE_FL_TRIGGER_MODE	= (1 << EVENT_FILE_FL_TRIGGER_MODE_BIT),
637	EVENT_FILE_FL_TRIGGER_COND	= (1 << EVENT_FILE_FL_TRIGGER_COND_BIT),
638	EVENT_FILE_FL_PID_FILTER	= (1 << EVENT_FILE_FL_PID_FILTER_BIT),
639	EVENT_FILE_FL_WAS_ENABLED	= (1 << EVENT_FILE_FL_WAS_ENABLED_BIT),
640	EVENT_FILE_FL_FREED		= (1 << EVENT_FILE_FL_FREED_BIT),
641};
642
643struct trace_event_file {
644	struct list_head		list;
645	struct trace_event_call		*event_call;
646	struct event_filter __rcu	*filter;
647	struct eventfs_inode		*ei;
648	struct trace_array		*tr;
649	struct trace_subsystem_dir	*system;
650	struct list_head		triggers;
651
652	/*
653	 * 32 bit flags:
654	 *   bit 0:		enabled
655	 *   bit 1:		enabled cmd record
656	 *   bit 2:		enable/disable with the soft disable bit
657	 *   bit 3:		soft disabled
658	 *   bit 4:		trigger enabled
659	 *
660	 * Note: The bits must be set atomically to prevent races
661	 * from other writers. Reads of flags do not need to be in
662	 * sync as they occur in critical sections. But the way flags
663	 * is currently used, these changes do not affect the code
664	 * except that when a change is made, it may have a slight
665	 * delay in propagating the changes to other CPUs due to
666	 * caching and such. Which is mostly OK ;-)
667	 */
668	unsigned long		flags;
669	refcount_t		ref;	/* ref count for opened files */
670	atomic_t		sm_ref;	/* soft-mode reference counter */
671	atomic_t		tm_ref;	/* trigger-mode reference counter */
672};
673
674#ifdef CONFIG_HIST_TRIGGERS
675extern struct irq_work hist_poll_work;
676extern wait_queue_head_t hist_poll_wq;
677
678static inline void hist_poll_wakeup(void)
679{
680	if (wq_has_sleeper(&hist_poll_wq))
681		irq_work_queue(&hist_poll_work);
682}
683
684#define hist_poll_wait(file, wait)	\
685	poll_wait(file, &hist_poll_wq, wait)
686
687#else
688static inline void hist_poll_wakeup(void)
689{
690}
691#endif
692
693#define __TRACE_EVENT_FLAGS(name, value)				\
694	static int __init trace_init_flags_##name(void)			\
695	{								\
696		event_##name.flags |= value;				\
697		return 0;						\
698	}								\
699	early_initcall(trace_init_flags_##name);
700
701#define __TRACE_EVENT_PERF_PERM(name, expr...)				\
702	static int perf_perm_##name(struct trace_event_call *tp_event, \
703				    struct perf_event *p_event)		\
704	{								\
705		return ({ expr; });					\
706	}								\
707	static int __init trace_init_perf_perm_##name(void)		\
708	{								\
709		event_##name.perf_perm = &perf_perm_##name;		\
710		return 0;						\
711	}								\
712	early_initcall(trace_init_perf_perm_##name);
713
714#define PERF_MAX_TRACE_SIZE	8192
715
716#define MAX_FILTER_STR_VAL	256U	/* Should handle KSYM_SYMBOL_LEN */
717
718enum event_trigger_type {
719	ETT_NONE		= (0),
720	ETT_TRACE_ONOFF		= (1 << 0),
721	ETT_SNAPSHOT		= (1 << 1),
722	ETT_STACKTRACE		= (1 << 2),
723	ETT_EVENT_ENABLE	= (1 << 3),
724	ETT_EVENT_HIST		= (1 << 4),
725	ETT_HIST_ENABLE		= (1 << 5),
726	ETT_EVENT_EPROBE	= (1 << 6),
727};
728
729extern int filter_match_preds(struct event_filter *filter, void *rec);
730
731extern enum event_trigger_type
732event_triggers_call(struct trace_event_file *file,
733		    struct trace_buffer *buffer, void *rec,
734		    struct ring_buffer_event *event);
735extern void
736event_triggers_post_call(struct trace_event_file *file,
737			 enum event_trigger_type tt);
738
739bool trace_event_ignore_this_pid(struct trace_event_file *trace_file);
740
741bool __trace_trigger_soft_disabled(struct trace_event_file *file);
742
743/**
744 * trace_trigger_soft_disabled - do triggers and test if soft disabled
745 * @file: The file pointer of the event to test
746 *
747 * If any triggers without filters are attached to this event, they
748 * will be called here. If the event is soft disabled and has no
749 * triggers that require testing the fields, it will return true,
750 * otherwise false.
751 */
752static __always_inline bool
753trace_trigger_soft_disabled(struct trace_event_file *file)
754{
755	unsigned long eflags = file->flags;
756
757	if (likely(!(eflags & (EVENT_FILE_FL_TRIGGER_MODE |
758			       EVENT_FILE_FL_SOFT_DISABLED |
759			       EVENT_FILE_FL_PID_FILTER))))
760		return false;
761
762	if (likely(eflags & EVENT_FILE_FL_TRIGGER_COND))
763		return false;
764
765	return __trace_trigger_soft_disabled(file);
766}
767
768#ifdef CONFIG_BPF_EVENTS
769unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);
770int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
771void perf_event_detach_bpf_prog(struct perf_event *event);
772int perf_event_query_prog_array(struct perf_event *event, void __user *info);
773
774struct bpf_raw_tp_link;
775int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link);
776int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link);
777
778struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name);
779void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);
780int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
781			    u32 *fd_type, const char **buf,
782			    u64 *probe_offset, u64 *probe_addr,
783			    unsigned long *missed);
784int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
785int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
786#else
787static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
788{
789	return 1;
790}
791
792static inline int
793perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie)
794{
795	return -EOPNOTSUPP;
796}
797
798static inline void perf_event_detach_bpf_prog(struct perf_event *event) { }
799
800static inline int
801perf_event_query_prog_array(struct perf_event *event, void __user *info)
802{
803	return -EOPNOTSUPP;
804}
805struct bpf_raw_tp_link;
806static inline int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
807{
808	return -EOPNOTSUPP;
809}
810static inline int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
811{
812	return -EOPNOTSUPP;
813}
814static inline struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name)
815{
816	return NULL;
817}
818static inline void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
819{
820}
821static inline int bpf_get_perf_event_info(const struct perf_event *event,
822					  u32 *prog_id, u32 *fd_type,
823					  const char **buf, u64 *probe_offset,
824					  u64 *probe_addr, unsigned long *missed)
825{
826	return -EOPNOTSUPP;
827}
828static inline int
829bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
830{
831	return -EOPNOTSUPP;
832}
833static inline int
834bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
835{
836	return -EOPNOTSUPP;
837}
838#endif
839
840enum {
841	FILTER_OTHER = 0,
842	FILTER_STATIC_STRING,
843	FILTER_DYN_STRING,
844	FILTER_RDYN_STRING,
845	FILTER_PTR_STRING,
846	FILTER_TRACE_FN,
847	FILTER_CPUMASK,
848	FILTER_COMM,
849	FILTER_CPU,
850	FILTER_STACKTRACE,
851};
852
853extern int trace_event_raw_init(struct trace_event_call *call);
854extern int trace_define_field(struct trace_event_call *call, const char *type,
855			      const char *name, int offset, int size,
856			      int is_signed, int filter_type);
857extern int trace_add_event_call(struct trace_event_call *call);
858extern int trace_remove_event_call(struct trace_event_call *call);
859extern int trace_event_get_offsets(struct trace_event_call *call);
860
861int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set);
862int trace_set_clr_event(const char *system, const char *event, int set);
863int trace_array_set_clr_event(struct trace_array *tr, const char *system,
864		const char *event, bool enable);
865
866#ifdef CONFIG_PERF_EVENTS
867struct perf_event;
868
869DECLARE_PER_CPU(struct pt_regs, perf_trace_regs);
870
871extern int  perf_trace_init(struct perf_event *event);
872extern void perf_trace_destroy(struct perf_event *event);
873extern int  perf_trace_add(struct perf_event *event, int flags);
874extern void perf_trace_del(struct perf_event *event, int flags);
875#ifdef CONFIG_KPROBE_EVENTS
876extern int  perf_kprobe_init(struct perf_event *event, bool is_retprobe);
877extern void perf_kprobe_destroy(struct perf_event *event);
878extern int bpf_get_kprobe_info(const struct perf_event *event,
879			       u32 *fd_type, const char **symbol,
880			       u64 *probe_offset, u64 *probe_addr,
881			       unsigned long *missed,
882			       bool perf_type_tracepoint);
883#endif
884#ifdef CONFIG_UPROBE_EVENTS
885extern int  perf_uprobe_init(struct perf_event *event,
886			     unsigned long ref_ctr_offset, bool is_retprobe);
887extern void perf_uprobe_destroy(struct perf_event *event);
888extern int bpf_get_uprobe_info(const struct perf_event *event,
889			       u32 *fd_type, const char **filename,
890			       u64 *probe_offset, u64 *probe_addr,
891			       bool perf_type_tracepoint);
892#endif
893extern int  ftrace_profile_set_filter(struct perf_event *event, int event_id,
894				     char *filter_str);
895extern void ftrace_profile_free_filter(struct perf_event *event);
896void perf_trace_buf_update(void *record, u16 type);
897void *perf_trace_buf_alloc(int size, struct pt_regs **regs, int *rctxp);
898
899int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
900void perf_event_free_bpf_prog(struct perf_event *event);
901
902void bpf_trace_run1(struct bpf_raw_tp_link *link, u64 arg1);
903void bpf_trace_run2(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2);
904void bpf_trace_run3(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
905		    u64 arg3);
906void bpf_trace_run4(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
907		    u64 arg3, u64 arg4);
908void bpf_trace_run5(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
909		    u64 arg3, u64 arg4, u64 arg5);
910void bpf_trace_run6(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
911		    u64 arg3, u64 arg4, u64 arg5, u64 arg6);
912void bpf_trace_run7(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
913		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7);
914void bpf_trace_run8(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
915		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
916		    u64 arg8);
917void bpf_trace_run9(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
918		    u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
919		    u64 arg8, u64 arg9);
920void bpf_trace_run10(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
921		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
922		     u64 arg8, u64 arg9, u64 arg10);
923void bpf_trace_run11(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
924		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
925		     u64 arg8, u64 arg9, u64 arg10, u64 arg11);
926void bpf_trace_run12(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
927		     u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
928		     u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12);
929void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,
930			       struct trace_event_call *call, u64 count,
931			       struct pt_regs *regs, struct hlist_head *head,
932			       struct task_struct *task);
933
934static inline void
935perf_trace_buf_submit(void *raw_data, int size, int rctx, u16 type,
936		       u64 count, struct pt_regs *regs, void *head,
937		       struct task_struct *task)
938{
939	perf_tp_event(type, count, raw_data, size, regs, head, rctx, task);
940}
941
942#endif
943
944#define TRACE_EVENT_STR_MAX	512
945
946/*
947 * gcc warns that you can not use a va_list in an inlined
948 * function. But lets me make it into a macro :-/
949 */
950#define __trace_event_vstr_len(fmt, va)			\
951({							\
952	va_list __ap;					\
953	int __ret;					\
954							\
955	va_copy(__ap, *(va));				\
956	__ret = vsnprintf(NULL, 0, fmt, __ap) + 1;	\
957	va_end(__ap);					\
958							\
959	min(__ret, TRACE_EVENT_STR_MAX);		\
960})
961
962#endif /* _LINUX_TRACE_EVENT_H */
963
964/*
965 * Note: we keep the TRACE_CUSTOM_EVENT outside the include file ifdef protection.
966 *  This is due to the way trace custom events work. If a file includes two
967 *  trace event headers under one "CREATE_CUSTOM_TRACE_EVENTS" the first include
968 *  will override the TRACE_CUSTOM_EVENT and break the second include.
969 */
970
971#ifndef TRACE_CUSTOM_EVENT
972
973#define DECLARE_CUSTOM_EVENT_CLASS(name, proto, args, tstruct, assign, print)
974#define DEFINE_CUSTOM_EVENT(template, name, proto, args)
975#define TRACE_CUSTOM_EVENT(name, proto, args, struct, assign, print)
976
977#endif /* ifdef TRACE_CUSTOM_EVENT (see note above) */