trace_functions.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * ring buffer based function tracer
   4 *
   5 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   7 *
   8 * Based on code from the latency_tracer, that is:
   9 *
  10 *  Copyright (C) 2004-2006 Ingo Molnar
  11 *  Copyright (C) 2004 Nadia Yvette Chambers
  12 */
  13#include <linux/ring_buffer.h>
  14#include <linux/debugfs.h>
  15#include <linux/uaccess.h>
  16#include <linux/ftrace.h>
  17#include <linux/slab.h>
  18#include <linux/fs.h>
  19
  20#include "trace.h"
  21
  22static void tracing_start_function_trace(struct trace_array *tr);
  23static void tracing_stop_function_trace(struct trace_array *tr);
  24static void
  25function_trace_call(unsigned long ip, unsigned long parent_ip,
  26		    struct ftrace_ops *op, struct ftrace_regs *fregs);
  27static void
  28function_args_trace_call(unsigned long ip, unsigned long parent_ip,
  29			 struct ftrace_ops *op, struct ftrace_regs *fregs);
  30static void
  31function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
  32			  struct ftrace_ops *op, struct ftrace_regs *fregs);
  33static void
  34function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
  35			       struct ftrace_ops *op, struct ftrace_regs *fregs);
  36static void
  37function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
  38				     struct ftrace_ops *op,
  39				     struct ftrace_regs *fregs);
  40static struct tracer_flags func_flags;
  41
  42/* Our option */
  43enum {
  44
  45	TRACE_FUNC_NO_OPTS		= 0x0, /* No flags set. */
  46	TRACE_FUNC_OPT_STACK		= 0x1,
  47	TRACE_FUNC_OPT_NO_REPEATS	= 0x2,
  48	TRACE_FUNC_OPT_ARGS		= 0x4,
  49
  50	/* Update this to next highest bit. */
  51	TRACE_FUNC_OPT_HIGHEST_BIT	= 0x8
  52};
  53
  54#define TRACE_FUNC_OPT_MASK	(TRACE_FUNC_OPT_HIGHEST_BIT - 1)
  55
  56int ftrace_allocate_ftrace_ops(struct trace_array *tr)
  57{
  58	struct ftrace_ops *ops;
  59
  60	/* The top level array uses the "global_ops" */
  61	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  62		return 0;
  63
  64	ops = kzalloc(sizeof(*ops), GFP_KERNEL);
  65	if (!ops)
  66		return -ENOMEM;
  67
  68	/* Currently only the non stack version is supported */
  69	ops->func = function_trace_call;
  70	ops->flags = FTRACE_OPS_FL_PID;
  71
  72	tr->ops = ops;
  73	ops->private = tr;
  74
  75	return 0;
  76}
  77
  78void ftrace_free_ftrace_ops(struct trace_array *tr)
  79{
  80	kfree(tr->ops);
  81	tr->ops = NULL;
  82}
  83
  84int ftrace_create_function_files(struct trace_array *tr,
  85				 struct dentry *parent)
  86{
  87	int ret;
  88	/*
  89	 * The top level array uses the "global_ops", and the files are
  90	 * created on boot up.
  91	 */
  92	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  93		return 0;
  94
  95	if (!tr->ops)
  96		return -EINVAL;
  97
  98	ret = allocate_fgraph_ops(tr, tr->ops);
  99	if (ret) {
 100		kfree(tr->ops);
 101		return ret;
 102	}
 103
 104	ftrace_create_filter_files(tr->ops, parent);
 105
 106	return 0;
 107}
 108
 109void ftrace_destroy_function_files(struct trace_array *tr)
 110{
 111	ftrace_destroy_filter_files(tr->ops);
 112	ftrace_free_ftrace_ops(tr);
 113	free_fgraph_ops(tr);
 114}
 115
 116static ftrace_func_t select_trace_function(u32 flags_val)
 117{
 118	switch (flags_val & TRACE_FUNC_OPT_MASK) {
 119	case TRACE_FUNC_NO_OPTS:
 120		return function_trace_call;
 121	case TRACE_FUNC_OPT_ARGS:
 122		return function_args_trace_call;
 123	case TRACE_FUNC_OPT_STACK:
 124		return function_stack_trace_call;
 125	case TRACE_FUNC_OPT_NO_REPEATS:
 126		return function_no_repeats_trace_call;
 127	case TRACE_FUNC_OPT_STACK | TRACE_FUNC_OPT_NO_REPEATS:
 128		return function_stack_no_repeats_trace_call;
 129	default:
 130		return NULL;
 131	}
 132}
 133
 134static bool handle_func_repeats(struct trace_array *tr, u32 flags_val)
 135{
 136	if (!tr->last_func_repeats &&
 137	    (flags_val & TRACE_FUNC_OPT_NO_REPEATS)) {
 138		tr->last_func_repeats = alloc_percpu(struct trace_func_repeats);
 139		if (!tr->last_func_repeats)
 140			return false;
 141	}
 142
 143	return true;
 144}
 145
 146static int function_trace_init(struct trace_array *tr)
 147{
 148	ftrace_func_t func;
 149	/*
 150	 * Instance trace_arrays get their ops allocated
 151	 * at instance creation. Unless it failed
 152	 * the allocation.
 153	 */
 154	if (!tr->ops)
 155		return -ENOMEM;
 156
 157	func = select_trace_function(tr->current_trace_flags->val);
 158	if (!func)
 159		return -EINVAL;
 160
 161	if (!handle_func_repeats(tr, tr->current_trace_flags->val))
 162		return -ENOMEM;
 163
 164	ftrace_init_array_ops(tr, func);
 165
 166	tr->array_buffer.cpu = raw_smp_processor_id();
 167
 168	tracing_start_cmdline_record();
 169	tracing_start_function_trace(tr);
 170	return 0;
 171}
 172
 173static void function_trace_reset(struct trace_array *tr)
 174{
 175	tracing_stop_function_trace(tr);
 176	tracing_stop_cmdline_record();
 177	ftrace_reset_array_ops(tr);
 178}
 179
 180static void function_trace_start(struct trace_array *tr)
 181{
 182	tracing_reset_online_cpus(&tr->array_buffer);
 183}
 184
 185/* fregs are guaranteed not to be NULL if HAVE_DYNAMIC_FTRACE_WITH_ARGS is set */
 186#if defined(CONFIG_FUNCTION_GRAPH_TRACER) && defined(CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS)
 187static __always_inline unsigned long
 188function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs)
 189{
 190	unsigned long true_parent_ip;
 191	int idx = 0;
 192
 193	true_parent_ip = parent_ip;
 194	if (unlikely(parent_ip == (unsigned long)&return_to_handler) && fregs)
 195		true_parent_ip = ftrace_graph_ret_addr(current, &idx, parent_ip,
 196				(unsigned long *)ftrace_regs_get_stack_pointer(fregs));
 197	return true_parent_ip;
 198}
 199#else
 200static __always_inline unsigned long
 201function_get_true_parent_ip(unsigned long parent_ip, struct ftrace_regs *fregs)
 202{
 203	return parent_ip;
 204}
 205#endif
 206
 207static void
 208function_trace_call(unsigned long ip, unsigned long parent_ip,
 209		    struct ftrace_ops *op, struct ftrace_regs *fregs)
 210{
 211	struct trace_array *tr = op->private;
 212	unsigned int trace_ctx;
 213	int bit;
 214
 215	if (unlikely(!tr->function_enabled))
 216		return;
 217
 218	bit = ftrace_test_recursion_trylock(ip, parent_ip);
 219	if (bit < 0)
 220		return;
 221
 222	parent_ip = function_get_true_parent_ip(parent_ip, fregs);
 223
 224	trace_ctx = tracing_gen_ctx_dec();
 225
 226	trace_function(tr, ip, parent_ip, trace_ctx, NULL);
 227
 228	ftrace_test_recursion_unlock(bit);
 229}
 230
 231static void
 232function_args_trace_call(unsigned long ip, unsigned long parent_ip,
 233			 struct ftrace_ops *op, struct ftrace_regs *fregs)
 234{
 235	struct trace_array *tr = op->private;
 236	unsigned int trace_ctx;
 237	int bit;
 238
 239	if (unlikely(!tr->function_enabled))
 240		return;
 241
 242	bit = ftrace_test_recursion_trylock(ip, parent_ip);
 243	if (bit < 0)
 244		return;
 245
 246	trace_ctx = tracing_gen_ctx();
 247
 248	trace_function(tr, ip, parent_ip, trace_ctx, fregs);
 249
 250	ftrace_test_recursion_unlock(bit);
 251}
 252
 253#ifdef CONFIG_UNWINDER_ORC
 254/*
 255 * Skip 2:
 256 *
 257 *   function_stack_trace_call()
 258 *   ftrace_call()
 259 */
 260#define STACK_SKIP 2
 261#else
 262/*
 263 * Skip 3:
 264 *   __trace_stack()
 265 *   function_stack_trace_call()
 266 *   ftrace_call()
 267 */
 268#define STACK_SKIP 3
 269#endif
 270
 271static void
 272function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
 273			  struct ftrace_ops *op, struct ftrace_regs *fregs)
 274{
 275	struct trace_array *tr = op->private;
 276	struct trace_array_cpu *data;
 277	unsigned long flags;
 278	long disabled;
 279	int cpu;
 280	unsigned int trace_ctx;
 281	int skip = STACK_SKIP;
 282
 283	if (unlikely(!tr->function_enabled))
 284		return;
 285
 286	/*
 287	 * Need to use raw, since this must be called before the
 288	 * recursive protection is performed.
 289	 */
 290	local_irq_save(flags);
 291	parent_ip = function_get_true_parent_ip(parent_ip, fregs);
 292	cpu = raw_smp_processor_id();
 293	data = per_cpu_ptr(tr->array_buffer.data, cpu);
 294	disabled = local_inc_return(&data->disabled);
 295
 296	if (likely(disabled == 1)) {
 297		trace_ctx = tracing_gen_ctx_flags(flags);
 298		trace_function(tr, ip, parent_ip, trace_ctx, NULL);
 299#ifdef CONFIG_UNWINDER_FRAME_POINTER
 300		if (ftrace_pids_enabled(op))
 301			skip++;
 302#endif
 303		__trace_stack(tr, trace_ctx, skip);
 304	}
 305
 306	local_dec(&data->disabled);
 307	local_irq_restore(flags);
 308}
 309
 310static inline bool is_repeat_check(struct trace_array *tr,
 311				   struct trace_func_repeats *last_info,
 312				   unsigned long ip, unsigned long parent_ip)
 313{
 314	if (last_info->ip == ip &&
 315	    last_info->parent_ip == parent_ip &&
 316	    last_info->count < U16_MAX) {
 317		last_info->ts_last_call =
 318			ring_buffer_time_stamp(tr->array_buffer.buffer);
 319		last_info->count++;
 320		return true;
 321	}
 322
 323	return false;
 324}
 325
 326static inline void process_repeats(struct trace_array *tr,
 327				   unsigned long ip, unsigned long parent_ip,
 328				   struct trace_func_repeats *last_info,
 329				   unsigned int trace_ctx)
 330{
 331	if (last_info->count) {
 332		trace_last_func_repeats(tr, last_info, trace_ctx);
 333		last_info->count = 0;
 334	}
 335
 336	last_info->ip = ip;
 337	last_info->parent_ip = parent_ip;
 338}
 339
 340static void
 341function_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
 342			       struct ftrace_ops *op,
 343			       struct ftrace_regs *fregs)
 344{
 345	struct trace_func_repeats *last_info;
 346	struct trace_array *tr = op->private;
 347	unsigned int trace_ctx;
 348	int bit;
 349
 350	if (unlikely(!tr->function_enabled))
 351		return;
 352
 353	bit = ftrace_test_recursion_trylock(ip, parent_ip);
 354	if (bit < 0)
 355		return;
 356
 357	parent_ip = function_get_true_parent_ip(parent_ip, fregs);
 358	if (!tracer_tracing_is_on(tr))
 359		goto out;
 360
 361	/*
 362	 * An interrupt may happen at any place here. But as far as I can see,
 363	 * the only damage that this can cause is to mess up the repetition
 364	 * counter without valuable data being lost.
 365	 * TODO: think about a solution that is better than just hoping to be
 366	 * lucky.
 367	 */
 368	last_info = this_cpu_ptr(tr->last_func_repeats);
 369	if (is_repeat_check(tr, last_info, ip, parent_ip))
 370		goto out;
 371
 372	trace_ctx = tracing_gen_ctx_dec();
 373	process_repeats(tr, ip, parent_ip, last_info, trace_ctx);
 374
 375	trace_function(tr, ip, parent_ip, trace_ctx, NULL);
 376
 377out:
 378	ftrace_test_recursion_unlock(bit);
 379}
 380
 381static void
 382function_stack_no_repeats_trace_call(unsigned long ip, unsigned long parent_ip,
 383				     struct ftrace_ops *op,
 384				     struct ftrace_regs *fregs)
 385{
 386	struct trace_func_repeats *last_info;
 387	struct trace_array *tr = op->private;
 388	struct trace_array_cpu *data;
 389	unsigned long flags;
 390	long disabled;
 391	int cpu;
 392	unsigned int trace_ctx;
 393
 394	if (unlikely(!tr->function_enabled))
 395		return;
 396
 397	/*
 398	 * Need to use raw, since this must be called before the
 399	 * recursive protection is performed.
 400	 */
 401	local_irq_save(flags);
 402	parent_ip = function_get_true_parent_ip(parent_ip, fregs);
 403	cpu = raw_smp_processor_id();
 404	data = per_cpu_ptr(tr->array_buffer.data, cpu);
 405	disabled = local_inc_return(&data->disabled);
 406
 407	if (likely(disabled == 1)) {
 408		last_info = per_cpu_ptr(tr->last_func_repeats, cpu);
 409		if (is_repeat_check(tr, last_info, ip, parent_ip))
 410			goto out;
 411
 412		trace_ctx = tracing_gen_ctx_flags(flags);
 413		process_repeats(tr, ip, parent_ip, last_info, trace_ctx);
 414
 415		trace_function(tr, ip, parent_ip, trace_ctx, NULL);
 416		__trace_stack(tr, trace_ctx, STACK_SKIP);
 417	}
 418
 419 out:
 420	local_dec(&data->disabled);
 421	local_irq_restore(flags);
 422}
 423
 424static struct tracer_opt func_opts[] = {
 425#ifdef CONFIG_STACKTRACE
 426	{ TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
 427#endif
 428	{ TRACER_OPT(func-no-repeats, TRACE_FUNC_OPT_NO_REPEATS) },
 429#ifdef CONFIG_FUNCTION_TRACE_ARGS
 430	{ TRACER_OPT(func-args, TRACE_FUNC_OPT_ARGS) },
 431#endif
 432	{ } /* Always set a last empty entry */
 433};
 434
 435static struct tracer_flags func_flags = {
 436	.val = TRACE_FUNC_NO_OPTS, /* By default: all flags disabled */
 437	.opts = func_opts
 438};
 439
 440static void tracing_start_function_trace(struct trace_array *tr)
 441{
 442	tr->function_enabled = 0;
 443	register_ftrace_function(tr->ops);
 444	tr->function_enabled = 1;
 445}
 446
 447static void tracing_stop_function_trace(struct trace_array *tr)
 448{
 449	tr->function_enabled = 0;
 450	unregister_ftrace_function(tr->ops);
 451}
 452
 453static struct tracer function_trace;
 454
 455static int
 456func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
 457{
 458	ftrace_func_t func;
 459	u32 new_flags;
 460
 461	/* Do nothing if already set. */
 462	if (!!set == !!(tr->current_trace_flags->val & bit))
 463		return 0;
 464
 465	/* We can change this flag only when not running. */
 466	if (tr->current_trace != &function_trace)
 467		return 0;
 468
 469	new_flags = (tr->current_trace_flags->val & ~bit) | (set ? bit : 0);
 470	func = select_trace_function(new_flags);
 471	if (!func)
 472		return -EINVAL;
 473
 474	/* Check if there's anything to change. */
 475	if (tr->ops->func == func)
 476		return 0;
 477
 478	if (!handle_func_repeats(tr, new_flags))
 479		return -ENOMEM;
 480
 481	unregister_ftrace_function(tr->ops);
 482	tr->ops->func = func;
 483	register_ftrace_function(tr->ops);
 484
 485	return 0;
 486}
 487
 488static struct tracer function_trace __tracer_data =
 489{
 490	.name		= "function",
 491	.init		= function_trace_init,
 492	.reset		= function_trace_reset,
 493	.start		= function_trace_start,
 494	.default_flags	= &func_flags,
 495	.set_flag	= func_set_flag,
 496	.allow_instances = true,
 497#ifdef CONFIG_FTRACE_SELFTEST
 498	.selftest	= trace_selftest_startup_function,
 499#endif
 500};
 501
 502#ifdef CONFIG_DYNAMIC_FTRACE
 503static void update_traceon_count(struct ftrace_probe_ops *ops,
 504				 unsigned long ip,
 505				 struct trace_array *tr, bool on,
 506				 void *data)
 507{
 508	struct ftrace_func_mapper *mapper = data;
 509	long *count;
 510	long old_count;
 511
 512	/*
 513	 * Tracing gets disabled (or enabled) once per count.
 514	 * This function can be called at the same time on multiple CPUs.
 515	 * It is fine if both disable (or enable) tracing, as disabling
 516	 * (or enabling) the second time doesn't do anything as the
 517	 * state of the tracer is already disabled (or enabled).
 518	 * What needs to be synchronized in this case is that the count
 519	 * only gets decremented once, even if the tracer is disabled
 520	 * (or enabled) twice, as the second one is really a nop.
 521	 *
 522	 * The memory barriers guarantee that we only decrement the
 523	 * counter once. First the count is read to a local variable
 524	 * and a read barrier is used to make sure that it is loaded
 525	 * before checking if the tracer is in the state we want.
 526	 * If the tracer is not in the state we want, then the count
 527	 * is guaranteed to be the old count.
 528	 *
 529	 * Next the tracer is set to the state we want (disabled or enabled)
 530	 * then a write memory barrier is used to make sure that
 531	 * the new state is visible before changing the counter by
 532	 * one minus the old counter. This guarantees that another CPU
 533	 * executing this code will see the new state before seeing
 534	 * the new counter value, and would not do anything if the new
 535	 * counter is seen.
 536	 *
 537	 * Note, there is no synchronization between this and a user
 538	 * setting the tracing_on file. But we currently don't care
 539	 * about that.
 540	 */
 541	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
 542	old_count = *count;
 543
 544	if (old_count <= 0)
 545		return;
 546
 547	/* Make sure we see count before checking tracing state */
 548	smp_rmb();
 549
 550	if (on == !!tracer_tracing_is_on(tr))
 551		return;
 552
 553	if (on)
 554		tracer_tracing_on(tr);
 555	else
 556		tracer_tracing_off(tr);
 557
 558	/* Make sure tracing state is visible before updating count */
 559	smp_wmb();
 560
 561	*count = old_count - 1;
 562}
 563
 564static void
 565ftrace_traceon_count(unsigned long ip, unsigned long parent_ip,
 566		     struct trace_array *tr, struct ftrace_probe_ops *ops,
 567		     void *data)
 568{
 569	update_traceon_count(ops, ip, tr, 1, data);
 570}
 571
 572static void
 573ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip,
 574		      struct trace_array *tr, struct ftrace_probe_ops *ops,
 575		      void *data)
 576{
 577	update_traceon_count(ops, ip, tr, 0, data);
 578}
 579
 580static void
 581ftrace_traceon(unsigned long ip, unsigned long parent_ip,
 582	       struct trace_array *tr, struct ftrace_probe_ops *ops,
 583	       void *data)
 584{
 585	if (tracer_tracing_is_on(tr))
 586		return;
 587
 588	tracer_tracing_on(tr);
 589}
 590
 591static void
 592ftrace_traceoff(unsigned long ip, unsigned long parent_ip,
 593		struct trace_array *tr, struct ftrace_probe_ops *ops,
 594		void *data)
 595{
 596	if (!tracer_tracing_is_on(tr))
 597		return;
 598
 599	tracer_tracing_off(tr);
 600}
 601
 602#ifdef CONFIG_UNWINDER_ORC
 603/*
 604 * Skip 3:
 605 *
 606 *   function_trace_probe_call()
 607 *   ftrace_ops_assist_func()
 608 *   ftrace_call()
 609 */
 610#define FTRACE_STACK_SKIP 3
 611#else
 612/*
 613 * Skip 5:
 614 *
 615 *   __trace_stack()
 616 *   ftrace_stacktrace()
 617 *   function_trace_probe_call()
 618 *   ftrace_ops_assist_func()
 619 *   ftrace_call()
 620 */
 621#define FTRACE_STACK_SKIP 5
 622#endif
 623
 624static __always_inline void trace_stack(struct trace_array *tr)
 625{
 626	__trace_stack(tr, tracing_gen_ctx_dec(), FTRACE_STACK_SKIP);
 627}
 628
 629static void
 630ftrace_stacktrace(unsigned long ip, unsigned long parent_ip,
 631		  struct trace_array *tr, struct ftrace_probe_ops *ops,
 632		  void *data)
 633{
 634	trace_stack(tr);
 635}
 636
 637static void
 638ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip,
 639			struct trace_array *tr, struct ftrace_probe_ops *ops,
 640			void *data)
 641{
 642	struct ftrace_func_mapper *mapper = data;
 643	long *count;
 644	long old_count;
 645	long new_count;
 646
 647	if (!tracing_is_on())
 648		return;
 649
 650	/* unlimited? */
 651	if (!mapper) {
 652		trace_stack(tr);
 653		return;
 654	}
 655
 656	count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
 657
 658	/*
 659	 * Stack traces should only execute the number of times the
 660	 * user specified in the counter.
 661	 */
 662	do {
 663		old_count = *count;
 664
 665		if (!old_count)
 666			return;
 667
 668		new_count = old_count - 1;
 669		new_count = cmpxchg(count, old_count, new_count);
 670		if (new_count == old_count)
 671			trace_stack(tr);
 672
 673		if (!tracing_is_on())
 674			return;
 675
 676	} while (new_count != old_count);
 677}
 678
 679static int update_count(struct ftrace_probe_ops *ops, unsigned long ip,
 680			void *data)
 681{
 682	struct ftrace_func_mapper *mapper = data;
 683	long *count = NULL;
 684
 685	if (mapper)
 686		count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
 687
 688	if (count) {
 689		if (*count <= 0)
 690			return 0;
 691		(*count)--;
 692	}
 693
 694	return 1;
 695}
 696
 697static void
 698ftrace_dump_probe(unsigned long ip, unsigned long parent_ip,
 699		  struct trace_array *tr, struct ftrace_probe_ops *ops,
 700		  void *data)
 701{
 702	if (update_count(ops, ip, data))
 703		ftrace_dump(DUMP_ALL);
 704}
 705
 706/* Only dump the current CPU buffer. */
 707static void
 708ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip,
 709		     struct trace_array *tr, struct ftrace_probe_ops *ops,
 710		     void *data)
 711{
 712	if (update_count(ops, ip, data))
 713		ftrace_dump(DUMP_ORIG);
 714}
 715
 716static int
 717ftrace_probe_print(const char *name, struct seq_file *m,
 718		   unsigned long ip, struct ftrace_probe_ops *ops,
 719		   void *data)
 720{
 721	struct ftrace_func_mapper *mapper = data;
 722	long *count = NULL;
 723
 724	seq_printf(m, "%ps:%s", (void *)ip, name);
 725
 726	if (mapper)
 727		count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
 728
 729	if (count)
 730		seq_printf(m, ":count=%ld\n", *count);
 731	else
 732		seq_puts(m, ":unlimited\n");
 733
 734	return 0;
 735}
 736
 737static int
 738ftrace_traceon_print(struct seq_file *m, unsigned long ip,
 739		     struct ftrace_probe_ops *ops,
 740		     void *data)
 741{
 742	return ftrace_probe_print("traceon", m, ip, ops, data);
 743}
 744
 745static int
 746ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
 747			 struct ftrace_probe_ops *ops, void *data)
 748{
 749	return ftrace_probe_print("traceoff", m, ip, ops, data);
 750}
 751
 752static int
 753ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
 754			struct ftrace_probe_ops *ops, void *data)
 755{
 756	return ftrace_probe_print("stacktrace", m, ip, ops, data);
 757}
 758
 759static int
 760ftrace_dump_print(struct seq_file *m, unsigned long ip,
 761			struct ftrace_probe_ops *ops, void *data)
 762{
 763	return ftrace_probe_print("dump", m, ip, ops, data);
 764}
 765
 766static int
 767ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
 768			struct ftrace_probe_ops *ops, void *data)
 769{
 770	return ftrace_probe_print("cpudump", m, ip, ops, data);
 771}
 772
 773
 774static int
 775ftrace_count_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
 776		  unsigned long ip, void *init_data, void **data)
 777{
 778	struct ftrace_func_mapper *mapper = *data;
 779
 780	if (!mapper) {
 781		mapper = allocate_ftrace_func_mapper();
 782		if (!mapper)
 783			return -ENOMEM;
 784		*data = mapper;
 785	}
 786
 787	return ftrace_func_mapper_add_ip(mapper, ip, init_data);
 788}
 789
 790static void
 791ftrace_count_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
 792		  unsigned long ip, void *data)
 793{
 794	struct ftrace_func_mapper *mapper = data;
 795
 796	if (!ip) {
 797		free_ftrace_func_mapper(mapper, NULL);
 798		return;
 799	}
 800
 801	ftrace_func_mapper_remove_ip(mapper, ip);
 802}
 803
 804static struct ftrace_probe_ops traceon_count_probe_ops = {
 805	.func			= ftrace_traceon_count,
 806	.print			= ftrace_traceon_print,
 807	.init			= ftrace_count_init,
 808	.free			= ftrace_count_free,
 809};
 810
 811static struct ftrace_probe_ops traceoff_count_probe_ops = {
 812	.func			= ftrace_traceoff_count,
 813	.print			= ftrace_traceoff_print,
 814	.init			= ftrace_count_init,
 815	.free			= ftrace_count_free,
 816};
 817
 818static struct ftrace_probe_ops stacktrace_count_probe_ops = {
 819	.func			= ftrace_stacktrace_count,
 820	.print			= ftrace_stacktrace_print,
 821	.init			= ftrace_count_init,
 822	.free			= ftrace_count_free,
 823};
 824
 825static struct ftrace_probe_ops dump_probe_ops = {
 826	.func			= ftrace_dump_probe,
 827	.print			= ftrace_dump_print,
 828	.init			= ftrace_count_init,
 829	.free			= ftrace_count_free,
 830};
 831
 832static struct ftrace_probe_ops cpudump_probe_ops = {
 833	.func			= ftrace_cpudump_probe,
 834	.print			= ftrace_cpudump_print,
 835};
 836
 837static struct ftrace_probe_ops traceon_probe_ops = {
 838	.func			= ftrace_traceon,
 839	.print			= ftrace_traceon_print,
 840};
 841
 842static struct ftrace_probe_ops traceoff_probe_ops = {
 843	.func			= ftrace_traceoff,
 844	.print			= ftrace_traceoff_print,
 845};
 846
 847static struct ftrace_probe_ops stacktrace_probe_ops = {
 848	.func			= ftrace_stacktrace,
 849	.print			= ftrace_stacktrace_print,
 850};
 851
 852static int
 853ftrace_trace_probe_callback(struct trace_array *tr,
 854			    struct ftrace_probe_ops *ops,
 855			    struct ftrace_hash *hash, char *glob,
 856			    char *cmd, char *param, int enable)
 857{
 858	void *count = (void *)-1;
 859	char *number;
 860	int ret;
 861
 862	/* hash funcs only work with set_ftrace_filter */
 863	if (!enable)
 864		return -EINVAL;
 865
 866	if (glob[0] == '!')
 867		return unregister_ftrace_function_probe_func(glob+1, tr, ops);
 868
 869	if (!param)
 870		goto out_reg;
 871
 872	number = strsep(&param, ":");
 873
 874	if (!strlen(number))
 875		goto out_reg;
 876
 877	/*
 878	 * We use the callback data field (which is a pointer)
 879	 * as our counter.
 880	 */
 881	ret = kstrtoul(number, 0, (unsigned long *)&count);
 882	if (ret)
 883		return ret;
 884
 885 out_reg:
 886	ret = register_ftrace_function_probe(glob, tr, ops, count);
 887
 888	return ret < 0 ? ret : 0;
 889}
 890
 891static int
 892ftrace_trace_onoff_callback(struct trace_array *tr, struct ftrace_hash *hash,
 893			    char *glob, char *cmd, char *param, int enable)
 894{
 895	struct ftrace_probe_ops *ops;
 896
 897	if (!tr)
 898		return -ENODEV;
 899
 900	/* we register both traceon and traceoff to this callback */
 901	if (strcmp(cmd, "traceon") == 0)
 902		ops = param ? &traceon_count_probe_ops : &traceon_probe_ops;
 903	else
 904		ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops;
 905
 906	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
 907					   param, enable);
 908}
 909
 910static int
 911ftrace_stacktrace_callback(struct trace_array *tr, struct ftrace_hash *hash,
 912			   char *glob, char *cmd, char *param, int enable)
 913{
 914	struct ftrace_probe_ops *ops;
 915
 916	if (!tr)
 917		return -ENODEV;
 918
 919	ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops;
 920
 921	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
 922					   param, enable);
 923}
 924
 925static int
 926ftrace_dump_callback(struct trace_array *tr, struct ftrace_hash *hash,
 927			   char *glob, char *cmd, char *param, int enable)
 928{
 929	struct ftrace_probe_ops *ops;
 930
 931	if (!tr)
 932		return -ENODEV;
 933
 934	ops = &dump_probe_ops;
 935
 936	/* Only dump once. */
 937	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
 938					   "1", enable);
 939}
 940
 941static int
 942ftrace_cpudump_callback(struct trace_array *tr, struct ftrace_hash *hash,
 943			   char *glob, char *cmd, char *param, int enable)
 944{
 945	struct ftrace_probe_ops *ops;
 946
 947	if (!tr)
 948		return -ENODEV;
 949
 950	ops = &cpudump_probe_ops;
 951
 952	/* Only dump once. */
 953	return ftrace_trace_probe_callback(tr, ops, hash, glob, cmd,
 954					   "1", enable);
 955}
 956
 957static struct ftrace_func_command ftrace_traceon_cmd = {
 958	.name			= "traceon",
 959	.func			= ftrace_trace_onoff_callback,
 960};
 961
 962static struct ftrace_func_command ftrace_traceoff_cmd = {
 963	.name			= "traceoff",
 964	.func			= ftrace_trace_onoff_callback,
 965};
 966
 967static struct ftrace_func_command ftrace_stacktrace_cmd = {
 968	.name			= "stacktrace",
 969	.func			= ftrace_stacktrace_callback,
 970};
 971
 972static struct ftrace_func_command ftrace_dump_cmd = {
 973	.name			= "dump",
 974	.func			= ftrace_dump_callback,
 975};
 976
 977static struct ftrace_func_command ftrace_cpudump_cmd = {
 978	.name			= "cpudump",
 979	.func			= ftrace_cpudump_callback,
 980};
 981
 982static int __init init_func_cmd_traceon(void)
 983{
 984	int ret;
 985
 986	ret = register_ftrace_command(&ftrace_traceoff_cmd);
 987	if (ret)
 988		return ret;
 989
 990	ret = register_ftrace_command(&ftrace_traceon_cmd);
 991	if (ret)
 992		goto out_free_traceoff;
 993
 994	ret = register_ftrace_command(&ftrace_stacktrace_cmd);
 995	if (ret)
 996		goto out_free_traceon;
 997
 998	ret = register_ftrace_command(&ftrace_dump_cmd);
 999	if (ret)
1000		goto out_free_stacktrace;
1001
1002	ret = register_ftrace_command(&ftrace_cpudump_cmd);
1003	if (ret)
1004		goto out_free_dump;
1005
1006	return 0;
1007
1008 out_free_dump:
1009	unregister_ftrace_command(&ftrace_dump_cmd);
1010 out_free_stacktrace:
1011	unregister_ftrace_command(&ftrace_stacktrace_cmd);
1012 out_free_traceon:
1013	unregister_ftrace_command(&ftrace_traceon_cmd);
1014 out_free_traceoff:
1015	unregister_ftrace_command(&ftrace_traceoff_cmd);
1016
1017	return ret;
1018}
1019#else
1020static inline int init_func_cmd_traceon(void)
1021{
1022	return 0;
1023}
1024#endif /* CONFIG_DYNAMIC_FTRACE */
1025
1026__init int init_function_trace(void)
1027{
1028	init_func_cmd_traceon();
1029	return register_tracer(&function_trace);
1030}