trace_branch.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * unlikely profiler
  4 *
  5 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
  6 */
  7#include <linux/kallsyms.h>
  8#include <linux/seq_file.h>
  9#include <linux/spinlock.h>
 10#include <linux/irqflags.h>
 11#include <linux/uaccess.h>
 12#include <linux/module.h>
 13#include <linux/ftrace.h>
 14#include <linux/hash.h>
 15#include <linux/fs.h>
 16#include <asm/local.h>
 17
 18#include "trace.h"
 19#include "trace_stat.h"
 20#include "trace_output.h"
 21
 22#ifdef CONFIG_BRANCH_TRACER
 23
 24static struct tracer branch_trace;
 25static int branch_tracing_enabled __read_mostly;
 26static DEFINE_MUTEX(branch_tracing_mutex);
 27
 28static struct trace_array *branch_tracer;
 29
 30static void
 31probe_likely_condition(struct ftrace_likely_data *f, int val, int expect)
 32{
 33	struct trace_array *tr = branch_tracer;
 34	struct trace_buffer *buffer;
 35	struct ring_buffer_event *event;
 36	struct trace_branch *entry;
 37	unsigned long flags;
 38	unsigned int trace_ctx;
 39	const char *p;
 40
 41	if (current->trace_recursion & TRACE_BRANCH_BIT)
 42		return;
 43
 44	/*
 45	 * I would love to save just the ftrace_likely_data pointer, but
 46	 * this code can also be used by modules. Ugly things can happen
 47	 * if the module is unloaded, and then we go and read the
 48	 * pointer.  This is slower, but much safer.
 49	 */
 50
 51	if (unlikely(!tr))
 52		return;
 53
 54	raw_local_irq_save(flags);
 55	current->trace_recursion |= TRACE_BRANCH_BIT;
 56	if (!tracer_tracing_is_on_cpu(tr, raw_smp_processor_id()))
 57		goto out;
 58
 59	trace_ctx = tracing_gen_ctx_flags(flags);
 60	buffer = tr->array_buffer.buffer;
 61	event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH,
 62					  sizeof(*entry), trace_ctx);
 63	if (!event)
 64		goto out;
 65
 66	entry	= ring_buffer_event_data(event);
 67
 68	/* Strip off the path, only save the file */
 69	p = f->data.file + strlen(f->data.file);
 70	while (p >= f->data.file && *p != '/')
 71		p--;
 72	p++;
 73
 74	strscpy(entry->func, f->data.func);
 75	strscpy(entry->file, p);
 76	entry->constant = f->constant;
 77	entry->line = f->data.line;
 78	entry->correct = val == expect;
 79
 80	trace_buffer_unlock_commit_nostack(buffer, event);
 81
 82 out:
 83	current->trace_recursion &= ~TRACE_BRANCH_BIT;
 84	raw_local_irq_restore(flags);
 85}
 86
 87static inline
 88void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
 89{
 90	if (!branch_tracing_enabled)
 91		return;
 92
 93	probe_likely_condition(f, val, expect);
 94}
 95
 96int enable_branch_tracing(struct trace_array *tr)
 97{
 98	mutex_lock(&branch_tracing_mutex);
 99	branch_tracer = tr;
100	/*
101	 * Must be seen before enabling. The reader is a condition
102	 * where we do not need a matching rmb()
103	 */
104	smp_wmb();
105	branch_tracing_enabled++;
106	mutex_unlock(&branch_tracing_mutex);
107
108	return 0;
109}
110
111void disable_branch_tracing(void)
112{
113	mutex_lock(&branch_tracing_mutex);
114
115	if (!branch_tracing_enabled)
116		goto out_unlock;
117
118	branch_tracing_enabled--;
119
120 out_unlock:
121	mutex_unlock(&branch_tracing_mutex);
122}
123
124static int branch_trace_init(struct trace_array *tr)
125{
126	return enable_branch_tracing(tr);
127}
128
129static void branch_trace_reset(struct trace_array *tr)
130{
131	disable_branch_tracing();
132}
133
134static enum print_line_t trace_branch_print(struct trace_iterator *iter,
135					    int flags, struct trace_event *event)
136{
137	struct trace_branch *field;
138
139	trace_assign_type(field, iter->ent);
140
141	trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
142			 field->correct ? "  ok  " : " MISS ",
143			 field->func,
144			 field->file,
145			 field->line);
146
147	return trace_handle_return(&iter->seq);
148}
149
150static void branch_print_header(struct seq_file *s)
151{
152	seq_puts(s, "#           TASK-PID    CPU#    TIMESTAMP  CORRECT"
153		    "  FUNC:FILE:LINE\n"
154		    "#              | |       |          |         |   "
155		    "    |\n");
156}
157
158static struct trace_event_functions trace_branch_funcs = {
159	.trace		= trace_branch_print,
160};
161
162static struct trace_event trace_branch_event = {
163	.type		= TRACE_BRANCH,
164	.funcs		= &trace_branch_funcs,
165};
166
167static struct tracer branch_trace __read_mostly =
168{
169	.name		= "branch",
170	.init		= branch_trace_init,
171	.reset		= branch_trace_reset,
172#ifdef CONFIG_FTRACE_SELFTEST
173	.selftest	= trace_selftest_startup_branch,
174#endif /* CONFIG_FTRACE_SELFTEST */
175	.print_header	= branch_print_header,
176};
177
178__init static int init_branch_tracer(void)
179{
180	int ret;
181
182	ret = register_trace_event(&trace_branch_event);
183	if (!ret) {
184		printk(KERN_WARNING "Warning: could not register "
185				    "branch events\n");
186		return 1;
187	}
188	return register_tracer(&branch_trace);
189}
190core_initcall(init_branch_tracer);
191
192#else
193static inline
194void trace_likely_condition(struct ftrace_likely_data *f, int val, int expect)
195{
196}
197#endif /* CONFIG_BRANCH_TRACER */
198
199void ftrace_likely_update(struct ftrace_likely_data *f, int val,
200			  int expect, int is_constant)
201{
202	unsigned long flags = user_access_save();
203
204	/* A constant is always correct */
205	if (is_constant) {
206		f->constant++;
207		val = expect;
208	}
209	/*
210	 * I would love to have a trace point here instead, but the
211	 * trace point code is so inundated with unlikely and likely
212	 * conditions that the recursive nightmare that exists is too
213	 * much to try to get working. At least for now.
214	 */
215	trace_likely_condition(f, val, expect);
216
217	/* FIXME: Make this atomic! */
218	if (val == expect)
219		f->data.correct++;
220	else
221		f->data.incorrect++;
222
223	user_access_restore(flags);
224}
225EXPORT_SYMBOL(ftrace_likely_update);
226
227extern unsigned long __start_annotated_branch_profile[];
228extern unsigned long __stop_annotated_branch_profile[];
229
230static int annotated_branch_stat_headers(struct seq_file *m)
231{
232	seq_puts(m, " correct incorrect  % "
233		    "       Function                "
234		    "  File              Line\n"
235		    " ------- ---------  - "
236		    "       --------                "
237		    "  ----              ----\n");
238	return 0;
239}
240
241static inline long get_incorrect_percent(const struct ftrace_branch_data *p)
242{
243	long percent;
244
245	if (p->correct) {
246		percent = p->incorrect * 100;
247		percent /= p->correct + p->incorrect;
248	} else
249		percent = p->incorrect ? 100 : -1;
250
251	return percent;
252}
253
254static const char *branch_stat_process_file(struct ftrace_branch_data *p)
255{
256	const char *f;
257
258	/* Only print the file, not the path */
259	f = p->file + strlen(p->file);
260	while (f >= p->file && *f != '/')
261		f--;
262	return ++f;
263}
264
265static void branch_stat_show(struct seq_file *m,
266			     struct ftrace_branch_data *p, const char *f)
267{
268	long percent;
269
270	/*
271	 * The miss is overlayed on correct, and hit on incorrect.
272	 */
273	percent = get_incorrect_percent(p);
274
275	if (percent < 0)
276		seq_puts(m, "  X ");
277	else
278		seq_printf(m, "%3ld ", percent);
279
280	seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
281}
282
283static int branch_stat_show_normal(struct seq_file *m,
284				   struct ftrace_branch_data *p, const char *f)
285{
286	seq_printf(m, "%8lu %8lu ",  p->correct, p->incorrect);
287	branch_stat_show(m, p, f);
288	return 0;
289}
290
291static int annotate_branch_stat_show(struct seq_file *m, void *v)
292{
293	struct ftrace_likely_data *p = v;
294	const char *f;
295	int l;
296
297	f = branch_stat_process_file(&p->data);
298
299	if (!p->constant)
300		return branch_stat_show_normal(m, &p->data, f);
301
302	l = snprintf(NULL, 0, "/%lu", p->constant);
303	l = l > 8 ? 0 : 8 - l;
304
305	seq_printf(m, "%8lu/%lu %*lu ",
306		   p->data.correct, p->constant, l, p->data.incorrect);
307	branch_stat_show(m, &p->data, f);
308	return 0;
309}
310
311static void *annotated_branch_stat_start(struct tracer_stat *trace)
312{
313	return __start_annotated_branch_profile;
314}
315
316static void *
317annotated_branch_stat_next(void *v, int idx)
318{
319	struct ftrace_likely_data *p = v;
320
321	++p;
322
323	if ((void *)p >= (void *)__stop_annotated_branch_profile)
324		return NULL;
325
326	return p;
327}
328
329static int annotated_branch_stat_cmp(const void *p1, const void *p2)
330{
331	const struct ftrace_branch_data *a = p1;
332	const struct ftrace_branch_data *b = p2;
333
334	long percent_a, percent_b;
335
336	percent_a = get_incorrect_percent(a);
337	percent_b = get_incorrect_percent(b);
338
339	if (percent_a < percent_b)
340		return -1;
341	if (percent_a > percent_b)
342		return 1;
343
344	if (a->incorrect < b->incorrect)
345		return -1;
346	if (a->incorrect > b->incorrect)
347		return 1;
348
349	/*
350	 * Since the above shows worse (incorrect) cases
351	 * first, we continue that by showing best (correct)
352	 * cases last.
353	 */
354	if (a->correct > b->correct)
355		return -1;
356	if (a->correct < b->correct)
357		return 1;
358
359	return 0;
360}
361
362static struct tracer_stat annotated_branch_stats = {
363	.name = "branch_annotated",
364	.stat_start = annotated_branch_stat_start,
365	.stat_next = annotated_branch_stat_next,
366	.stat_cmp = annotated_branch_stat_cmp,
367	.stat_headers = annotated_branch_stat_headers,
368	.stat_show = annotate_branch_stat_show
369};
370
371__init static int init_annotated_branch_stats(void)
372{
373	int ret;
374
375	ret = register_stat_tracer(&annotated_branch_stats);
376	if (!ret) {
377		printk(KERN_WARNING "Warning: could not register "
378				    "annotated branches stats\n");
379		return 1;
380	}
381	return 0;
382}
383fs_initcall(init_annotated_branch_stats);
384
385#ifdef CONFIG_PROFILE_ALL_BRANCHES
386
387extern unsigned long __start_branch_profile[];
388extern unsigned long __stop_branch_profile[];
389
390static int all_branch_stat_headers(struct seq_file *m)
391{
392	seq_puts(m, "   miss      hit    % "
393		    "       Function                "
394		    "  File              Line\n"
395		    " ------- ---------  - "
396		    "       --------                "
397		    "  ----              ----\n");
398	return 0;
399}
400
401static void *all_branch_stat_start(struct tracer_stat *trace)
402{
403	return __start_branch_profile;
404}
405
406static void *
407all_branch_stat_next(void *v, int idx)
408{
409	struct ftrace_branch_data *p = v;
410
411	++p;
412
413	if ((void *)p >= (void *)__stop_branch_profile)
414		return NULL;
415
416	return p;
417}
418
419static int all_branch_stat_show(struct seq_file *m, void *v)
420{
421	struct ftrace_branch_data *p = v;
422	const char *f;
423
424	f = branch_stat_process_file(p);
425	return branch_stat_show_normal(m, p, f);
426}
427
428static struct tracer_stat all_branch_stats = {
429	.name = "branch_all",
430	.stat_start = all_branch_stat_start,
431	.stat_next = all_branch_stat_next,
432	.stat_headers = all_branch_stat_headers,
433	.stat_show = all_branch_stat_show
434};
435
436__init static int all_annotated_branch_stats(void)
437{
438	int ret;
439
440	ret = register_stat_tracer(&all_branch_stats);
441	if (!ret) {
442		printk(KERN_WARNING "Warning: could not register "
443				    "all branches stats\n");
444		return 1;
445	}
446	return 0;
447}
448fs_initcall(all_annotated_branch_stats);
449#endif /* CONFIG_PROFILE_ALL_BRANCHES */