trace_recursion.h

  1/* SPDX-License-Identifier: GPL-2.0 */
  2#ifndef _LINUX_TRACE_RECURSION_H
  3#define _LINUX_TRACE_RECURSION_H
  4
  5#include <linux/interrupt.h>
  6#include <linux/sched.h>
  7
  8#ifdef CONFIG_TRACING
  9
 10/* Only current can touch trace_recursion */
 11
 12/*
 13 * For function tracing recursion:
 14 *  The order of these bits are important.
 15 *
 16 *  When function tracing occurs, the following steps are made:
 17 *   If arch does not support a ftrace feature:
 18 *    call internal function (uses INTERNAL bits) which calls...
 19 *   The function callback, which can use the FTRACE bits to
 20 *    check for recursion.
 21 */
 22enum {
 23	/* Function recursion bits */
 24	TRACE_FTRACE_BIT,
 25	TRACE_FTRACE_NMI_BIT,
 26	TRACE_FTRACE_IRQ_BIT,
 27	TRACE_FTRACE_SIRQ_BIT,
 28	TRACE_FTRACE_TRANSITION_BIT,
 29
 30	/* Internal use recursion bits */
 31	TRACE_INTERNAL_BIT,
 32	TRACE_INTERNAL_NMI_BIT,
 33	TRACE_INTERNAL_IRQ_BIT,
 34	TRACE_INTERNAL_SIRQ_BIT,
 35	TRACE_INTERNAL_TRANSITION_BIT,
 36
 37	/* Internal event use recursion bits */
 38	TRACE_INTERNAL_EVENT_BIT,
 39	TRACE_INTERNAL_EVENT_NMI_BIT,
 40	TRACE_INTERNAL_EVENT_IRQ_BIT,
 41	TRACE_INTERNAL_EVENT_SIRQ_BIT,
 42	TRACE_INTERNAL_EVENT_TRANSITION_BIT,
 43
 44	TRACE_BRANCH_BIT,
 45/*
 46 * Abuse of the trace_recursion.
 47 * As we need a way to maintain state if we are tracing the function
 48 * graph in irq because we want to trace a particular function that
 49 * was called in irq context but we have irq tracing off. Since this
 50 * can only be modified by current, we can reuse trace_recursion.
 51 */
 52	TRACE_IRQ_BIT,
 53
 54	/* Used to prevent recursion recording from recursing. */
 55	TRACE_RECORD_RECURSION_BIT,
 56};
 57
 58#define trace_recursion_set(bit)	do { (current)->trace_recursion |= (1<<(bit)); } while (0)
 59#define trace_recursion_clear(bit)	do { (current)->trace_recursion &= ~(1<<(bit)); } while (0)
 60#define trace_recursion_test(bit)	((current)->trace_recursion & (1<<(bit)))
 61
 62#define TRACE_CONTEXT_BITS	4
 63
 64#define TRACE_FTRACE_START	TRACE_FTRACE_BIT
 65
 66#define TRACE_LIST_START	TRACE_INTERNAL_BIT
 67
 68#define TRACE_EVENT_START	TRACE_INTERNAL_EVENT_BIT
 69
 70#define TRACE_CONTEXT_MASK	((1 << (TRACE_LIST_START + TRACE_CONTEXT_BITS)) - 1)
 71
 72/*
 73 * Used for setting context
 74 *  NMI     = 0
 75 *  IRQ     = 1
 76 *  SOFTIRQ = 2
 77 *  NORMAL  = 3
 78 */
 79enum {
 80	TRACE_CTX_NMI,
 81	TRACE_CTX_IRQ,
 82	TRACE_CTX_SOFTIRQ,
 83	TRACE_CTX_NORMAL,
 84	TRACE_CTX_TRANSITION,
 85};
 86
 87static __always_inline int trace_get_context_bit(void)
 88{
 89	unsigned char bit = interrupt_context_level();
 90
 91	return TRACE_CTX_NORMAL - bit;
 92}
 93
 94#ifdef CONFIG_FTRACE_RECORD_RECURSION
 95extern void ftrace_record_recursion(unsigned long ip, unsigned long parent_ip);
 96# define do_ftrace_record_recursion(ip, pip)				\
 97	do {								\
 98		if (!trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
 99			trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
100			ftrace_record_recursion(ip, pip);		\
101			trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
102		}							\
103	} while (0)
104#else
105# define do_ftrace_record_recursion(ip, pip)	do { } while (0)
106#endif
107
108#ifdef CONFIG_FTRACE_VALIDATE_RCU_IS_WATCHING
109# define trace_warn_on_no_rcu(ip)					\
110	({								\
111		bool __ret = !rcu_is_watching();			\
112		if (__ret && !trace_recursion_test(TRACE_RECORD_RECURSION_BIT)) { \
113			trace_recursion_set(TRACE_RECORD_RECURSION_BIT); \
114			WARN_ONCE(true, "RCU not on for: %pS\n", (void *)ip); \
115			trace_recursion_clear(TRACE_RECORD_RECURSION_BIT); \
116		}							\
117		__ret;							\
118	})
119#else
120# define trace_warn_on_no_rcu(ip)	false
121#endif
122
123/*
124 * Preemption is promised to be disabled when return bit >= 0.
125 */
126static __always_inline int trace_test_and_set_recursion(unsigned long ip, unsigned long pip,
127							int start)
128{
129	unsigned int val = READ_ONCE(current->trace_recursion);
130	int bit;
131
132	if (trace_warn_on_no_rcu(ip))
133		return -1;
134
135	bit = trace_get_context_bit() + start;
136	if (unlikely(val & (1 << bit))) {
137		/*
138		 * If an interrupt occurs during a trace, and another trace
139		 * happens in that interrupt but before the preempt_count is
140		 * updated to reflect the new interrupt context, then this
141		 * will think a recursion occurred, and the event will be dropped.
142		 * Let a single instance happen via the TRANSITION_BIT to
143		 * not drop those events.
144		 */
145		bit = TRACE_CTX_TRANSITION + start;
146		if (val & (1 << bit)) {
147			do_ftrace_record_recursion(ip, pip);
148			return -1;
149		}
150	}
151
152	val |= 1 << bit;
153	current->trace_recursion = val;
154	barrier();
155
156	preempt_disable_notrace();
157
158	return bit;
159}
160
161/*
162 * Preemption will be enabled (if it was previously enabled).
163 */
164static __always_inline void trace_clear_recursion(int bit)
165{
166	preempt_enable_notrace();
167	barrier();
168	trace_recursion_clear(bit);
169}
170
171/**
172 * ftrace_test_recursion_trylock - tests for recursion in same context
173 *
174 * Use this for ftrace callbacks. This will detect if the function
175 * tracing recursed in the same context (normal vs interrupt),
176 *
177 * Returns: -1 if a recursion happened.
178 *           >= 0 if no recursion.
179 */
180static __always_inline int ftrace_test_recursion_trylock(unsigned long ip,
181							 unsigned long parent_ip)
182{
183	return trace_test_and_set_recursion(ip, parent_ip, TRACE_FTRACE_START);
184}
185
186/**
187 * ftrace_test_recursion_unlock - called when function callback is complete
188 * @bit: The return of a successful ftrace_test_recursion_trylock()
189 *
190 * This is used at the end of a ftrace callback.
191 */
192static __always_inline void ftrace_test_recursion_unlock(int bit)
193{
194	trace_clear_recursion(bit);
195}
196
197#endif /* CONFIG_TRACING */
198#endif /* _LINUX_TRACE_RECURSION_H */