trace_output.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * trace_output.c
   4 *
   5 * Copyright (C) 2008 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
   6 *
   7 */
   8#include "trace.h"
   9#include <linux/module.h>
  10#include <linux/mutex.h>
  11#include <linux/ftrace.h>
  12#include <linux/kprobes.h>
  13#include <linux/sched/clock.h>
  14#include <linux/sched/mm.h>
  15#include <linux/idr.h>
  16#include <linux/btf.h>
  17#include <linux/bpf.h>
  18#include <linux/hashtable.h>
  19
  20#include "trace_output.h"
  21#include "trace_btf.h"
  22
  23/* 2^7 = 128 */
  24#define EVENT_HASH_BITS 7
  25
  26DECLARE_RWSEM(trace_event_sem);
  27
  28static DEFINE_HASHTABLE(event_hash, EVENT_HASH_BITS);
  29
  30enum print_line_t trace_print_bputs_msg_only(struct trace_iterator *iter)
  31{
  32	struct trace_seq *s = &iter->seq;
  33	struct trace_entry *entry = iter->ent;
  34	struct bputs_entry *field;
  35
  36	trace_assign_type(field, entry);
  37
  38	trace_seq_puts(s, field->str);
  39
  40	return trace_handle_return(s);
  41}
  42
  43enum print_line_t trace_print_bprintk_msg_only(struct trace_iterator *iter)
  44{
  45	struct trace_seq *s = &iter->seq;
  46	struct trace_entry *entry = iter->ent;
  47	struct bprint_entry *field;
  48
  49	trace_assign_type(field, entry);
  50
  51	trace_seq_bprintf(s, field->fmt, field->buf);
  52
  53	return trace_handle_return(s);
  54}
  55
  56enum print_line_t trace_print_printk_msg_only(struct trace_iterator *iter)
  57{
  58	struct trace_seq *s = &iter->seq;
  59	struct trace_entry *entry = iter->ent;
  60	struct print_entry *field;
  61
  62	trace_assign_type(field, entry);
  63
  64	trace_seq_puts(s, field->buf);
  65
  66	return trace_handle_return(s);
  67}
  68
  69const char *
  70trace_print_flags_seq(struct trace_seq *p, const char *delim,
  71		      unsigned long flags,
  72		      const struct trace_print_flags *flag_array)
  73{
  74	unsigned long mask;
  75	const char *str;
  76	const char *ret = trace_seq_buffer_ptr(p);
  77	int i, first = 1;
  78
  79	for (i = 0;  flag_array[i].name && flags; i++) {
  80
  81		mask = flag_array[i].mask;
  82		if ((flags & mask) != mask)
  83			continue;
  84
  85		str = flag_array[i].name;
  86		flags &= ~mask;
  87		if (!first && delim)
  88			trace_seq_puts(p, delim);
  89		else
  90			first = 0;
  91		trace_seq_puts(p, str);
  92	}
  93
  94	/* check for left over flags */
  95	if (flags) {
  96		if (!first && delim)
  97			trace_seq_puts(p, delim);
  98		trace_seq_printf(p, "0x%lx", flags);
  99	}
 100
 101	trace_seq_putc(p, 0);
 102
 103	return ret;
 104}
 105EXPORT_SYMBOL(trace_print_flags_seq);
 106
 107const char *
 108trace_print_symbols_seq(struct trace_seq *p, unsigned long val,
 109			const struct trace_print_flags *symbol_array)
 110{
 111	int i;
 112	const char *ret = trace_seq_buffer_ptr(p);
 113
 114	for (i = 0;  symbol_array[i].name; i++) {
 115
 116		if (val != symbol_array[i].mask)
 117			continue;
 118
 119		trace_seq_puts(p, symbol_array[i].name);
 120		break;
 121	}
 122
 123	if (ret == (const char *)(trace_seq_buffer_ptr(p)))
 124		trace_seq_printf(p, "0x%lx", val);
 125
 126	trace_seq_putc(p, 0);
 127
 128	return ret;
 129}
 130EXPORT_SYMBOL(trace_print_symbols_seq);
 131
 132#if BITS_PER_LONG == 32
 133const char *
 134trace_print_flags_seq_u64(struct trace_seq *p, const char *delim,
 135		      unsigned long long flags,
 136		      const struct trace_print_flags_u64 *flag_array)
 137{
 138	unsigned long long mask;
 139	const char *str;
 140	const char *ret = trace_seq_buffer_ptr(p);
 141	int i, first = 1;
 142
 143	for (i = 0;  flag_array[i].name && flags; i++) {
 144
 145		mask = flag_array[i].mask;
 146		if ((flags & mask) != mask)
 147			continue;
 148
 149		str = flag_array[i].name;
 150		flags &= ~mask;
 151		if (!first && delim)
 152			trace_seq_puts(p, delim);
 153		else
 154			first = 0;
 155		trace_seq_puts(p, str);
 156	}
 157
 158	/* check for left over flags */
 159	if (flags) {
 160		if (!first && delim)
 161			trace_seq_puts(p, delim);
 162		trace_seq_printf(p, "0x%llx", flags);
 163	}
 164
 165	trace_seq_putc(p, 0);
 166
 167	return ret;
 168}
 169EXPORT_SYMBOL(trace_print_flags_seq_u64);
 170
 171const char *
 172trace_print_symbols_seq_u64(struct trace_seq *p, unsigned long long val,
 173			 const struct trace_print_flags_u64 *symbol_array)
 174{
 175	int i;
 176	const char *ret = trace_seq_buffer_ptr(p);
 177
 178	for (i = 0;  symbol_array[i].name; i++) {
 179
 180		if (val != symbol_array[i].mask)
 181			continue;
 182
 183		trace_seq_puts(p, symbol_array[i].name);
 184		break;
 185	}
 186
 187	if (ret == (const char *)(trace_seq_buffer_ptr(p)))
 188		trace_seq_printf(p, "0x%llx", val);
 189
 190	trace_seq_putc(p, 0);
 191
 192	return ret;
 193}
 194EXPORT_SYMBOL(trace_print_symbols_seq_u64);
 195#endif
 196
 197const char *
 198trace_print_bitmask_seq(struct trace_seq *p, void *bitmask_ptr,
 199			unsigned int bitmask_size)
 200{
 201	const char *ret = trace_seq_buffer_ptr(p);
 202
 203	trace_seq_bitmask(p, bitmask_ptr, bitmask_size * 8);
 204	trace_seq_putc(p, 0);
 205
 206	return ret;
 207}
 208EXPORT_SYMBOL_GPL(trace_print_bitmask_seq);
 209
 210/**
 211 * trace_print_hex_seq - print buffer as hex sequence
 212 * @p: trace seq struct to write to
 213 * @buf: The buffer to print
 214 * @buf_len: Length of @buf in bytes
 215 * @concatenate: Print @buf as single hex string or with spacing
 216 *
 217 * Prints the passed buffer as a hex sequence either as a whole,
 218 * single hex string if @concatenate is true or with spacing after
 219 * each byte in case @concatenate is false.
 220 */
 221const char *
 222trace_print_hex_seq(struct trace_seq *p, const unsigned char *buf, int buf_len,
 223		    bool concatenate)
 224{
 225	int i;
 226	const char *ret = trace_seq_buffer_ptr(p);
 227	const char *fmt = concatenate ? "%*phN" : "%*ph";
 228
 229	for (i = 0; i < buf_len; i += 16) {
 230		if (!concatenate && i != 0)
 231			trace_seq_putc(p, ' ');
 232		trace_seq_printf(p, fmt, min(buf_len - i, 16), &buf[i]);
 233	}
 234	trace_seq_putc(p, 0);
 235
 236	return ret;
 237}
 238EXPORT_SYMBOL(trace_print_hex_seq);
 239
 240const char *
 241trace_print_array_seq(struct trace_seq *p, const void *buf, int count,
 242		      size_t el_size)
 243{
 244	const char *ret = trace_seq_buffer_ptr(p);
 245	const char *prefix = "";
 246	void *ptr = (void *)buf;
 247	size_t buf_len = count * el_size;
 248
 249	trace_seq_putc(p, '{');
 250
 251	while (ptr < buf + buf_len) {
 252		switch (el_size) {
 253		case 1:
 254			trace_seq_printf(p, "%s0x%x", prefix,
 255					 *(u8 *)ptr);
 256			break;
 257		case 2:
 258			trace_seq_printf(p, "%s0x%x", prefix,
 259					 *(u16 *)ptr);
 260			break;
 261		case 4:
 262			trace_seq_printf(p, "%s0x%x", prefix,
 263					 *(u32 *)ptr);
 264			break;
 265		case 8:
 266			trace_seq_printf(p, "%s0x%llx", prefix,
 267					 *(u64 *)ptr);
 268			break;
 269		default:
 270			trace_seq_printf(p, "BAD SIZE:%zu 0x%x", el_size,
 271					 *(u8 *)ptr);
 272			el_size = 1;
 273		}
 274		prefix = ",";
 275		ptr += el_size;
 276	}
 277
 278	trace_seq_putc(p, '}');
 279	trace_seq_putc(p, 0);
 280
 281	return ret;
 282}
 283EXPORT_SYMBOL(trace_print_array_seq);
 284
 285const char *
 286trace_print_hex_dump_seq(struct trace_seq *p, const char *prefix_str,
 287			 int prefix_type, int rowsize, int groupsize,
 288			 const void *buf, size_t len, bool ascii)
 289{
 290	const char *ret = trace_seq_buffer_ptr(p);
 291
 292	trace_seq_putc(p, '\n');
 293	trace_seq_hex_dump(p, prefix_str, prefix_type,
 294			   rowsize, groupsize, buf, len, ascii);
 295	trace_seq_putc(p, 0);
 296	return ret;
 297}
 298EXPORT_SYMBOL(trace_print_hex_dump_seq);
 299
 300int trace_raw_output_prep(struct trace_iterator *iter,
 301			  struct trace_event *trace_event)
 302{
 303	struct trace_event_call *event;
 304	struct trace_seq *s = &iter->seq;
 305	struct trace_seq *p = &iter->tmp_seq;
 306	struct trace_entry *entry;
 307
 308	event = container_of(trace_event, struct trace_event_call, event);
 309	entry = iter->ent;
 310
 311	if (entry->type != event->event.type) {
 312		WARN_ON_ONCE(1);
 313		return TRACE_TYPE_UNHANDLED;
 314	}
 315
 316	trace_seq_init(p);
 317	trace_seq_printf(s, "%s: ", trace_event_name(event));
 318
 319	return trace_handle_return(s);
 320}
 321EXPORT_SYMBOL(trace_raw_output_prep);
 322
 323void trace_event_printf(struct trace_iterator *iter, const char *fmt, ...)
 324{
 325	struct trace_seq *s = &iter->seq;
 326	va_list ap;
 327
 328	if (ignore_event(iter))
 329		return;
 330
 331	va_start(ap, fmt);
 332	trace_seq_vprintf(s, trace_event_format(iter, fmt), ap);
 333	va_end(ap);
 334}
 335EXPORT_SYMBOL(trace_event_printf);
 336
 337static __printf(3, 0)
 338int trace_output_raw(struct trace_iterator *iter, char *name,
 339		     char *fmt, va_list ap)
 340{
 341	struct trace_seq *s = &iter->seq;
 342
 343	trace_seq_printf(s, "%s: ", name);
 344	trace_seq_vprintf(s, trace_event_format(iter, fmt), ap);
 345
 346	return trace_handle_return(s);
 347}
 348
 349int trace_output_call(struct trace_iterator *iter, char *name, char *fmt, ...)
 350{
 351	va_list ap;
 352	int ret;
 353
 354	va_start(ap, fmt);
 355	ret = trace_output_raw(iter, name, fmt, ap);
 356	va_end(ap);
 357
 358	return ret;
 359}
 360EXPORT_SYMBOL_GPL(trace_output_call);
 361
 362static inline const char *kretprobed(const char *name, unsigned long addr)
 363{
 364	if (is_kretprobe_trampoline(addr))
 365		return "[unknown/kretprobe'd]";
 366	return name;
 367}
 368
 369void
 370trace_seq_print_sym(struct trace_seq *s, unsigned long address, bool offset)
 371{
 372#ifdef CONFIG_KALLSYMS
 373	char str[KSYM_SYMBOL_LEN];
 374	const char *name;
 375
 376	if (offset)
 377		sprint_symbol(str, address);
 378	else
 379		kallsyms_lookup(address, NULL, NULL, NULL, str);
 380	name = kretprobed(str, address);
 381
 382	if (name && strlen(name)) {
 383		trace_seq_puts(s, name);
 384		return;
 385	}
 386#endif
 387	trace_seq_printf(s, "0x%08lx", address);
 388}
 389
 390#ifndef CONFIG_64BIT
 391# define IP_FMT "%08lx"
 392#else
 393# define IP_FMT "%016lx"
 394#endif
 395
 396static int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
 397			     unsigned long ip, unsigned long sym_flags)
 398{
 399	struct file *file = NULL;
 400	unsigned long vmstart = 0;
 401	int ret = 1;
 402
 403	if (s->full)
 404		return 0;
 405
 406	if (mm) {
 407		const struct vm_area_struct *vma;
 408
 409		mmap_read_lock(mm);
 410		vma = find_vma(mm, ip);
 411		if (vma) {
 412			file = vma->vm_file;
 413			vmstart = vma->vm_start;
 414		}
 415		if (file) {
 416			ret = trace_seq_path(s, file_user_path(file));
 417			if (ret)
 418				trace_seq_printf(s, "[+0x%lx]",
 419						 ip - vmstart);
 420		}
 421		mmap_read_unlock(mm);
 422	}
 423	if (ret && ((sym_flags & TRACE_ITER(SYM_ADDR)) || !file))
 424		trace_seq_printf(s, " <" IP_FMT ">", ip);
 425	return !trace_seq_has_overflowed(s);
 426}
 427
 428int
 429seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
 430{
 431	if (!ip) {
 432		trace_seq_putc(s, '0');
 433		goto out;
 434	}
 435
 436	trace_seq_print_sym(s, ip, sym_flags & TRACE_ITER(SYM_OFFSET));
 437
 438	if (sym_flags & TRACE_ITER(SYM_ADDR))
 439		trace_seq_printf(s, " <" IP_FMT ">", ip);
 440
 441 out:
 442	return !trace_seq_has_overflowed(s);
 443}
 444
 445/**
 446 * trace_print_lat_fmt - print the irq, preempt and lockdep fields
 447 * @s: trace seq struct to write to
 448 * @entry: The trace entry field from the ring buffer
 449 *
 450 * Prints the generic fields of irqs off, in hard or softirq, preempt
 451 * count.
 452 */
 453int trace_print_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
 454{
 455	char hardsoft_irq;
 456	char need_resched;
 457	char irqs_off;
 458	int hardirq;
 459	int softirq;
 460	int bh_off;
 461	int nmi;
 462
 463	nmi = entry->flags & TRACE_FLAG_NMI;
 464	hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
 465	softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
 466	bh_off = entry->flags & TRACE_FLAG_BH_OFF;
 467
 468	irqs_off =
 469		(entry->flags & TRACE_FLAG_IRQS_OFF && bh_off) ? 'D' :
 470		(entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
 471		bh_off ? 'b' :
 472		'.';
 473
 474	switch (entry->flags & (TRACE_FLAG_NEED_RESCHED | TRACE_FLAG_NEED_RESCHED_LAZY |
 475				TRACE_FLAG_PREEMPT_RESCHED)) {
 476	case TRACE_FLAG_NEED_RESCHED | TRACE_FLAG_NEED_RESCHED_LAZY | TRACE_FLAG_PREEMPT_RESCHED:
 477		need_resched = 'B';
 478		break;
 479	case TRACE_FLAG_NEED_RESCHED | TRACE_FLAG_PREEMPT_RESCHED:
 480		need_resched = 'N';
 481		break;
 482	case TRACE_FLAG_NEED_RESCHED_LAZY | TRACE_FLAG_PREEMPT_RESCHED:
 483		need_resched = 'L';
 484		break;
 485	case TRACE_FLAG_NEED_RESCHED | TRACE_FLAG_NEED_RESCHED_LAZY:
 486		need_resched = 'b';
 487		break;
 488	case TRACE_FLAG_NEED_RESCHED:
 489		need_resched = 'n';
 490		break;
 491	case TRACE_FLAG_PREEMPT_RESCHED:
 492		need_resched = 'p';
 493		break;
 494	case TRACE_FLAG_NEED_RESCHED_LAZY:
 495		need_resched = 'l';
 496		break;
 497	default:
 498		need_resched = '.';
 499		break;
 500	}
 501
 502	hardsoft_irq =
 503		(nmi && hardirq)     ? 'Z' :
 504		nmi                  ? 'z' :
 505		(hardirq && softirq) ? 'H' :
 506		hardirq              ? 'h' :
 507		softirq              ? 's' :
 508		                       '.' ;
 509
 510	trace_seq_printf(s, "%c%c%c",
 511			 irqs_off, need_resched, hardsoft_irq);
 512
 513	if (entry->preempt_count & 0xf)
 514		trace_seq_printf(s, "%x", entry->preempt_count & 0xf);
 515	else
 516		trace_seq_putc(s, '.');
 517
 518	if (entry->preempt_count & 0xf0)
 519		trace_seq_printf(s, "%x", entry->preempt_count >> 4);
 520	else
 521		trace_seq_putc(s, '.');
 522
 523	return !trace_seq_has_overflowed(s);
 524}
 525
 526static int
 527lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
 528{
 529	char comm[TASK_COMM_LEN];
 530
 531	trace_find_cmdline(entry->pid, comm);
 532
 533	trace_seq_printf(s, "%8.8s-%-7d %3d",
 534			 comm, entry->pid, cpu);
 535
 536	return trace_print_lat_fmt(s, entry);
 537}
 538
 539#undef MARK
 540#define MARK(v, s) {.val = v, .sym = s}
 541/* trace overhead mark */
 542static const struct trace_mark {
 543	unsigned long long	val; /* unit: nsec */
 544	char			sym;
 545} mark[] = {
 546	MARK(1000000000ULL	, '$'), /* 1 sec */
 547	MARK(100000000ULL	, '@'), /* 100 msec */
 548	MARK(10000000ULL	, '*'), /* 10 msec */
 549	MARK(1000000ULL		, '#'), /* 1000 usecs */
 550	MARK(100000ULL		, '!'), /* 100 usecs */
 551	MARK(10000ULL		, '+'), /* 10 usecs */
 552};
 553#undef MARK
 554
 555char trace_find_mark(unsigned long long d)
 556{
 557	int i;
 558	int size = ARRAY_SIZE(mark);
 559
 560	for (i = 0; i < size; i++) {
 561		if (d > mark[i].val)
 562			break;
 563	}
 564
 565	return (i == size) ? ' ' : mark[i].sym;
 566}
 567
 568static int
 569lat_print_timestamp(struct trace_iterator *iter, u64 next_ts)
 570{
 571	struct trace_array *tr = iter->tr;
 572	unsigned long verbose = tr->trace_flags & TRACE_ITER(VERBOSE);
 573	unsigned long in_ns = iter->iter_flags & TRACE_FILE_TIME_IN_NS;
 574	unsigned long long abs_ts = iter->ts - iter->array_buffer->time_start;
 575	unsigned long long rel_ts = next_ts - iter->ts;
 576	struct trace_seq *s = &iter->seq;
 577
 578	if (in_ns) {
 579		abs_ts = ns2usecs(abs_ts);
 580		rel_ts = ns2usecs(rel_ts);
 581	}
 582
 583	if (verbose && in_ns) {
 584		unsigned long abs_usec = do_div(abs_ts, USEC_PER_MSEC);
 585		unsigned long abs_msec = (unsigned long)abs_ts;
 586		unsigned long rel_usec = do_div(rel_ts, USEC_PER_MSEC);
 587		unsigned long rel_msec = (unsigned long)rel_ts;
 588
 589		trace_seq_printf(
 590			s, "[%08llx] %ld.%03ldms (+%ld.%03ldms): ",
 591			ns2usecs(iter->ts),
 592			abs_msec, abs_usec,
 593			rel_msec, rel_usec);
 594
 595	} else if (verbose && !in_ns) {
 596		trace_seq_printf(
 597			s, "[%016llx] %lld (+%lld): ",
 598			iter->ts, abs_ts, rel_ts);
 599
 600	} else if (!verbose && in_ns) {
 601		trace_seq_printf(
 602			s, " %4lldus%c: ",
 603			abs_ts,
 604			trace_find_mark(rel_ts * NSEC_PER_USEC));
 605
 606	} else { /* !verbose && !in_ns */
 607		trace_seq_printf(s, " %4lld: ", abs_ts);
 608	}
 609
 610	return !trace_seq_has_overflowed(s);
 611}
 612
 613static void trace_print_time(struct trace_seq *s, struct trace_iterator *iter,
 614			     unsigned long long ts)
 615{
 616	unsigned long secs, usec_rem;
 617	unsigned long long t;
 618
 619	if (iter->iter_flags & TRACE_FILE_TIME_IN_NS) {
 620		t = ns2usecs(ts);
 621		usec_rem = do_div(t, USEC_PER_SEC);
 622		secs = (unsigned long)t;
 623		trace_seq_printf(s, " %5lu.%06lu", secs, usec_rem);
 624	} else
 625		trace_seq_printf(s, " %12llu", ts);
 626}
 627
 628int trace_print_context(struct trace_iterator *iter)
 629{
 630	struct trace_array *tr = iter->tr;
 631	struct trace_seq *s = &iter->seq;
 632	struct trace_entry *entry = iter->ent;
 633	char comm[TASK_COMM_LEN];
 634
 635	trace_find_cmdline(entry->pid, comm);
 636
 637	trace_seq_printf(s, "%16s-%-7d ", comm, entry->pid);
 638
 639	if (tr->trace_flags & TRACE_ITER(RECORD_TGID)) {
 640		unsigned int tgid = trace_find_tgid(entry->pid);
 641
 642		if (!tgid)
 643			trace_seq_printf(s, "(-------) ");
 644		else
 645			trace_seq_printf(s, "(%7d) ", tgid);
 646	}
 647
 648	trace_seq_printf(s, "[%03d] ", iter->cpu);
 649
 650	if (tr->trace_flags & TRACE_ITER(IRQ_INFO))
 651		trace_print_lat_fmt(s, entry);
 652
 653	trace_print_time(s, iter, iter->ts);
 654	trace_seq_puts(s, ": ");
 655
 656	return !trace_seq_has_overflowed(s);
 657}
 658
 659int trace_print_lat_context(struct trace_iterator *iter)
 660{
 661	struct trace_entry *entry, *next_entry;
 662	struct trace_array *tr = iter->tr;
 663	struct trace_seq *s = &iter->seq;
 664	unsigned long verbose = (tr->trace_flags & TRACE_ITER(VERBOSE));
 665	u64 next_ts;
 666
 667	next_entry = trace_find_next_entry(iter, NULL, &next_ts);
 668	if (!next_entry)
 669		next_ts = iter->ts;
 670
 671	/* trace_find_next_entry() may change iter->ent */
 672	entry = iter->ent;
 673
 674	if (verbose) {
 675		char comm[TASK_COMM_LEN];
 676
 677		trace_find_cmdline(entry->pid, comm);
 678
 679		trace_seq_printf(
 680			s, "%16s %7d %3d %d %08x %08lx ",
 681			comm, entry->pid, iter->cpu, entry->flags,
 682			entry->preempt_count & 0xf, iter->idx);
 683	} else {
 684		lat_print_generic(s, entry, iter->cpu);
 685	}
 686
 687	lat_print_timestamp(iter, next_ts);
 688
 689	return !trace_seq_has_overflowed(s);
 690}
 691
 692#ifdef CONFIG_FUNCTION_TRACE_ARGS
 693void print_function_args(struct trace_seq *s, unsigned long *args,
 694			 unsigned long func)
 695{
 696	const struct btf_param *param;
 697	const struct btf_type *t;
 698	const char *param_name;
 699	char name[KSYM_NAME_LEN];
 700	unsigned long arg;
 701	struct btf *btf;
 702	s32 tid, nr = 0;
 703	int a, p, x;
 704	u16 encode;
 705
 706	trace_seq_printf(s, "(");
 707
 708	if (!args)
 709		goto out;
 710	if (lookup_symbol_name(func, name))
 711		goto out;
 712
 713	/* TODO: Pass module name here too */
 714	t = btf_find_func_proto(name, &btf);
 715	if (IS_ERR_OR_NULL(t))
 716		goto out;
 717
 718	param = btf_get_func_param(t, &nr);
 719	if (!param)
 720		goto out_put;
 721
 722	for (a = 0, p = 0; p < nr; a++, p++) {
 723		if (p)
 724			trace_seq_puts(s, ", ");
 725
 726		/* This only prints what the arch allows (6 args by default) */
 727		if (a == FTRACE_REGS_MAX_ARGS) {
 728			trace_seq_puts(s, "...");
 729			break;
 730		}
 731
 732		arg = args[a];
 733
 734		param_name = btf_name_by_offset(btf, param[p].name_off);
 735		if (param_name)
 736			trace_seq_printf(s, "%s=", param_name);
 737		t = btf_type_skip_modifiers(btf, param[p].type, &tid);
 738
 739		switch (t ? BTF_INFO_KIND(t->info) : BTF_KIND_UNKN) {
 740		case BTF_KIND_UNKN:
 741			trace_seq_putc(s, '?');
 742			/* Still print unknown type values */
 743			fallthrough;
 744		case BTF_KIND_PTR:
 745			trace_seq_printf(s, "0x%lx", arg);
 746			break;
 747		case BTF_KIND_INT:
 748			encode = btf_int_encoding(t);
 749			/* Print unsigned ints as hex */
 750			if (encode & BTF_INT_SIGNED)
 751				trace_seq_printf(s, "%ld", arg);
 752			else
 753				trace_seq_printf(s, "0x%lx", arg);
 754			break;
 755		case BTF_KIND_ENUM:
 756			trace_seq_printf(s, "%ld", arg);
 757			break;
 758		default:
 759			/* This does not handle complex arguments */
 760			trace_seq_printf(s, "(%s)[0x%lx", btf_type_str(t), arg);
 761			for (x = sizeof(long); x < t->size; x += sizeof(long)) {
 762				trace_seq_putc(s, ':');
 763				if (++a == FTRACE_REGS_MAX_ARGS) {
 764					trace_seq_puts(s, "...]");
 765					goto out_put;
 766				}
 767				trace_seq_printf(s, "0x%lx", args[a]);
 768			}
 769			trace_seq_putc(s, ']');
 770			break;
 771		}
 772	}
 773out_put:
 774	btf_put(btf);
 775out:
 776	trace_seq_printf(s, ")");
 777}
 778#endif
 779
 780/**
 781 * ftrace_find_event - find a registered event
 782 * @type: the type of event to look for
 783 *
 784 * Returns an event of type @type otherwise NULL
 785 * Called with trace_event_read_lock() held.
 786 */
 787struct trace_event *ftrace_find_event(int type)
 788{
 789	struct trace_event *event;
 790
 791	hash_for_each_possible(event_hash, event, node, type) {
 792		if (event->type == type)
 793			return event;
 794	}
 795
 796	return NULL;
 797}
 798
 799static DEFINE_IDA(trace_event_ida);
 800
 801static void free_trace_event_type(int type)
 802{
 803	if (type >= __TRACE_LAST_TYPE)
 804		ida_free(&trace_event_ida, type);
 805}
 806
 807static int alloc_trace_event_type(void)
 808{
 809	int next;
 810
 811	/* Skip static defined type numbers */
 812	next = ida_alloc_range(&trace_event_ida, __TRACE_LAST_TYPE,
 813			       TRACE_EVENT_TYPE_MAX, GFP_KERNEL);
 814	if (next < 0)
 815		return 0;
 816	return next;
 817}
 818
 819void trace_event_read_lock(void)
 820{
 821	down_read(&trace_event_sem);
 822}
 823
 824void trace_event_read_unlock(void)
 825{
 826	up_read(&trace_event_sem);
 827}
 828
 829/**
 830 * register_trace_event - register output for an event type
 831 * @event: the event type to register
 832 *
 833 * Event types are stored in a hash and this hash is used to
 834 * find a way to print an event. If the @event->type is set
 835 * then it will use that type, otherwise it will assign a
 836 * type to use.
 837 *
 838 * If you assign your own type, please make sure it is added
 839 * to the trace_type enum in trace.h, to avoid collisions
 840 * with the dynamic types.
 841 *
 842 * Returns the event type number or zero on error.
 843 */
 844int register_trace_event(struct trace_event *event)
 845{
 846	int ret = 0;
 847
 848	down_write(&trace_event_sem);
 849
 850	if (WARN_ON(!event))
 851		goto out;
 852
 853	if (WARN_ON(!event->funcs))
 854		goto out;
 855
 856	if (!event->type) {
 857		event->type = alloc_trace_event_type();
 858		if (!event->type)
 859			goto out;
 860	} else if (WARN(event->type > __TRACE_LAST_TYPE,
 861			"Need to add type to trace.h")) {
 862		goto out;
 863	} else {
 864		/* Is this event already used */
 865		if (ftrace_find_event(event->type))
 866			goto out;
 867	}
 868
 869	if (event->funcs->trace == NULL)
 870		event->funcs->trace = trace_nop_print;
 871	if (event->funcs->raw == NULL)
 872		event->funcs->raw = trace_nop_print;
 873	if (event->funcs->hex == NULL)
 874		event->funcs->hex = trace_nop_print;
 875	if (event->funcs->binary == NULL)
 876		event->funcs->binary = trace_nop_print;
 877
 878	hash_add(event_hash, &event->node, event->type);
 879
 880	ret = event->type;
 881 out:
 882	up_write(&trace_event_sem);
 883
 884	return ret;
 885}
 886EXPORT_SYMBOL_GPL(register_trace_event);
 887
 888/*
 889 * Used by module code with the trace_event_sem held for write.
 890 */
 891int __unregister_trace_event(struct trace_event *event)
 892{
 893	hash_del(&event->node);
 894	free_trace_event_type(event->type);
 895	return 0;
 896}
 897
 898/**
 899 * unregister_trace_event - remove a no longer used event
 900 * @event: the event to remove
 901 */
 902int unregister_trace_event(struct trace_event *event)
 903{
 904	down_write(&trace_event_sem);
 905	__unregister_trace_event(event);
 906	up_write(&trace_event_sem);
 907
 908	return 0;
 909}
 910EXPORT_SYMBOL_GPL(unregister_trace_event);
 911
 912/*
 913 * Standard events
 914 */
 915
 916static void print_array(struct trace_iterator *iter, void *pos,
 917			struct ftrace_event_field *field)
 918{
 919	int offset;
 920	int len;
 921	int i;
 922
 923	offset = *(int *)pos & 0xffff;
 924	len = *(int *)pos >> 16;
 925
 926	if (field)
 927		offset += field->offset + sizeof(int);
 928
 929	if (offset + len > iter->ent_size) {
 930		trace_seq_puts(&iter->seq, "<OVERFLOW>");
 931		return;
 932	}
 933
 934	pos = (void *)iter->ent + offset;
 935
 936	for (i = 0; i < len; i++, pos++) {
 937		if (i)
 938			trace_seq_putc(&iter->seq, ',');
 939		trace_seq_printf(&iter->seq, "%02x", *(unsigned char *)pos);
 940	}
 941}
 942
 943static void print_fields(struct trace_iterator *iter, struct trace_event_call *call,
 944			 struct list_head *head)
 945{
 946	struct ftrace_event_field *field;
 947	struct trace_array *tr = iter->tr;
 948	unsigned long long laddr;
 949	unsigned long addr;
 950	int offset;
 951	int len;
 952	int ret;
 953	int i;
 954	void *pos;
 955	char *str;
 956
 957	list_for_each_entry_reverse(field, head, link) {
 958		trace_seq_printf(&iter->seq, " %s=", field->name);
 959		if (field->offset + field->size > iter->ent_size) {
 960			trace_seq_puts(&iter->seq, "<OVERFLOW>");
 961			continue;
 962		}
 963		pos = (void *)iter->ent + field->offset;
 964
 965		switch (field->filter_type) {
 966		case FILTER_COMM:
 967		case FILTER_STATIC_STRING:
 968			trace_seq_printf(&iter->seq, "%.*s", field->size, (char *)pos);
 969			break;
 970		case FILTER_RDYN_STRING:
 971		case FILTER_DYN_STRING:
 972			offset = *(int *)pos & 0xffff;
 973			len = *(int *)pos >> 16;
 974
 975			if (field->filter_type == FILTER_RDYN_STRING)
 976				offset += field->offset + sizeof(int);
 977
 978			if (offset + len > iter->ent_size) {
 979				trace_seq_puts(&iter->seq, "<OVERFLOW>");
 980				break;
 981			}
 982			str = (char *)iter->ent + offset;
 983			/* Check if there's any non printable strings */
 984			for (i = 0; i < len; i++) {
 985				if (str[i] && !(isascii(str[i]) && isprint(str[i])))
 986					break;
 987			}
 988			if (i < len) {
 989				for (i = 0; i < len; i++) {
 990					if (isascii(str[i]) && isprint(str[i]))
 991						trace_seq_putc(&iter->seq, str[i]);
 992					else
 993						trace_seq_putc(&iter->seq, '.');
 994				}
 995				trace_seq_puts(&iter->seq, " (");
 996				for (i = 0; i < len; i++) {
 997					if (i)
 998						trace_seq_putc(&iter->seq, ':');
 999					trace_seq_printf(&iter->seq, "%02x", str[i]);
1000				}
1001				trace_seq_putc(&iter->seq, ')');
1002			} else {
1003				trace_seq_printf(&iter->seq, "%.*s", len, str);
1004			}
1005			break;
1006		case FILTER_PTR_STRING:
1007			if (!iter->fmt_size)
1008				trace_iter_expand_format(iter);
1009			addr = trace_adjust_address(tr, *(unsigned long *)pos);
1010			ret = strncpy_from_kernel_nofault(iter->fmt, (void *)addr,
1011							  iter->fmt_size);
1012			if (ret < 0)
1013				trace_seq_printf(&iter->seq, "(0x%px)", pos);
1014			else
1015				trace_seq_printf(&iter->seq, "(0x%px:%s)",
1016						 pos, iter->fmt);
1017			break;
1018		case FILTER_TRACE_FN:
1019			addr = trace_adjust_address(tr, *(unsigned long *)pos);
1020			trace_seq_printf(&iter->seq, "%pS", (void *)addr);
1021			break;
1022		case FILTER_CPU:
1023		case FILTER_OTHER:
1024			switch (field->size) {
1025			case 1:
1026				if (isprint(*(char *)pos)) {
1027					trace_seq_printf(&iter->seq, "'%c'",
1028						 *(unsigned char *)pos);
1029				}
1030				trace_seq_printf(&iter->seq, "(%d)",
1031						 *(unsigned char *)pos);
1032				break;
1033			case 2:
1034				trace_seq_printf(&iter->seq, "0x%x (%d)",
1035						 *(unsigned short *)pos,
1036						 *(unsigned short *)pos);
1037				break;
1038			case 4:
1039				/* dynamic array info is 4 bytes */
1040				if (strstr(field->type, "__data_loc")) {
1041					print_array(iter, pos, NULL);
1042					break;
1043				}
1044
1045				if (strstr(field->type, "__rel_loc")) {
1046					print_array(iter, pos, field);
1047					break;
1048				}
1049
1050				addr = *(unsigned int *)pos;
1051
1052				/* Some fields reference offset from _stext. */
1053				if (!strcmp(field->name, "caller_offs") ||
1054				    !strcmp(field->name, "parent_offs")) {
1055					unsigned long ip;
1056
1057					ip = addr + (unsigned long)_stext;
1058					ip = trace_adjust_address(tr, ip);
1059					trace_seq_printf(&iter->seq, "%pS ", (void *)ip);
1060				}
1061
1062				if (sizeof(long) == 4) {
1063					addr = trace_adjust_address(tr, addr);
1064					trace_seq_printf(&iter->seq, "%pS (%d)",
1065							 (void *)addr, (int)addr);
1066				} else {
1067					trace_seq_printf(&iter->seq, "0x%x (%d)",
1068							 (unsigned int)addr, (int)addr);
1069				}
1070				break;
1071			case 8:
1072				laddr = *(unsigned long long *)pos;
1073				if (sizeof(long) == 8) {
1074					laddr = trace_adjust_address(tr, (unsigned long)laddr);
1075					trace_seq_printf(&iter->seq, "%pS (%lld)",
1076							 (void *)(long)laddr, laddr);
1077				} else {
1078					trace_seq_printf(&iter->seq, "0x%llx (%lld)", laddr, laddr);
1079				}
1080				break;
1081			default:
1082				trace_seq_puts(&iter->seq, "<INVALID-SIZE>");
1083				break;
1084			}
1085			break;
1086		default:
1087			trace_seq_puts(&iter->seq, "<INVALID-TYPE>");
1088		}
1089	}
1090	trace_seq_putc(&iter->seq, '\n');
1091}
1092
1093enum print_line_t print_event_fields(struct trace_iterator *iter,
1094				     struct trace_event *event)
1095{
1096	struct trace_event_call *call;
1097	struct list_head *head;
1098
1099	lockdep_assert_held_read(&trace_event_sem);
1100
1101	/* ftrace defined events have separate call structures */
1102	if (event->type <= __TRACE_LAST_TYPE) {
1103		bool found = false;
1104
1105		list_for_each_entry(call, &ftrace_events, list) {
1106			if (call->event.type == event->type) {
1107				found = true;
1108				break;
1109			}
1110			/* No need to search all events */
1111			if (call->event.type > __TRACE_LAST_TYPE)
1112				break;
1113		}
1114		if (!found) {
1115			trace_seq_printf(&iter->seq, "UNKNOWN TYPE %d\n", event->type);
1116			goto out;
1117		}
1118	} else {
1119		call = container_of(event, struct trace_event_call, event);
1120	}
1121	head = trace_get_fields(call);
1122
1123	trace_seq_printf(&iter->seq, "%s:", trace_event_name(call));
1124
1125	if (head && !list_empty(head))
1126		print_fields(iter, call, head);
1127	else
1128		trace_seq_puts(&iter->seq, "No fields found\n");
1129
1130 out:
1131	return trace_handle_return(&iter->seq);
1132}
1133
1134enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags,
1135				  struct trace_event *event)
1136{
1137	trace_seq_printf(&iter->seq, "type: %d\n", iter->ent->type);
1138
1139	return trace_handle_return(&iter->seq);
1140}
1141
1142static void print_fn_trace(struct trace_seq *s, unsigned long ip,
1143			   unsigned long parent_ip, unsigned long *args,
1144			   struct trace_array *tr, int flags)
1145{
1146	ip = trace_adjust_address(tr, ip);
1147	parent_ip = trace_adjust_address(tr, parent_ip);
1148
1149	seq_print_ip_sym(s, ip, flags);
1150	if (args)
1151		print_function_args(s, args, ip);
1152
1153	if ((flags & TRACE_ITER(PRINT_PARENT)) && parent_ip) {
1154		trace_seq_puts(s, " <-");
1155		seq_print_ip_sym(s, parent_ip, flags);
1156	}
1157}
1158
1159/* TRACE_FN */
1160static enum print_line_t trace_fn_trace(struct trace_iterator *iter, int flags,
1161					struct trace_event *event)
1162{
1163	struct ftrace_entry *field;
1164	struct trace_seq *s = &iter->seq;
1165	unsigned long *args;
1166	int args_size;
1167
1168	trace_assign_type(field, iter->ent);
1169
1170	args_size = iter->ent_size - offsetof(struct ftrace_entry, args);
1171	if (args_size >= FTRACE_REGS_MAX_ARGS * sizeof(long))
1172		args = field->args;
1173	else
1174		args = NULL;
1175
1176	print_fn_trace(s, field->ip, field->parent_ip, args, iter->tr, flags);
1177	trace_seq_putc(s, '\n');
1178
1179	return trace_handle_return(s);
1180}
1181
1182static enum print_line_t trace_fn_raw(struct trace_iterator *iter, int flags,
1183				      struct trace_event *event)
1184{
1185	struct ftrace_entry *field;
1186
1187	trace_assign_type(field, iter->ent);
1188
1189	trace_seq_printf(&iter->seq, "%lx %lx\n",
1190			 field->ip,
1191			 field->parent_ip);
1192
1193	return trace_handle_return(&iter->seq);
1194}
1195
1196static enum print_line_t trace_fn_hex(struct trace_iterator *iter, int flags,
1197				      struct trace_event *event)
1198{
1199	struct ftrace_entry *field;
1200	struct trace_seq *s = &iter->seq;
1201
1202	trace_assign_type(field, iter->ent);
1203
1204	SEQ_PUT_HEX_FIELD(s, field->ip);
1205	SEQ_PUT_HEX_FIELD(s, field->parent_ip);
1206
1207	return trace_handle_return(s);
1208}
1209
1210static enum print_line_t trace_fn_bin(struct trace_iterator *iter, int flags,
1211				      struct trace_event *event)
1212{
1213	struct ftrace_entry *field;
1214	struct trace_seq *s = &iter->seq;
1215
1216	trace_assign_type(field, iter->ent);
1217
1218	SEQ_PUT_FIELD(s, field->ip);
1219	SEQ_PUT_FIELD(s, field->parent_ip);
1220
1221	return trace_handle_return(s);
1222}
1223
1224static struct trace_event_functions trace_fn_funcs = {
1225	.trace		= trace_fn_trace,
1226	.raw		= trace_fn_raw,
1227	.hex		= trace_fn_hex,
1228	.binary		= trace_fn_bin,
1229};
1230
1231static struct trace_event trace_fn_event = {
1232	.type		= TRACE_FN,
1233	.funcs		= &trace_fn_funcs,
1234};
1235
1236/* TRACE_CTX an TRACE_WAKE */
1237static enum print_line_t trace_ctxwake_print(struct trace_iterator *iter,
1238					     char *delim)
1239{
1240	struct ctx_switch_entry *field;
1241	char comm[TASK_COMM_LEN];
1242	int S, T;
1243
1244
1245	trace_assign_type(field, iter->ent);
1246
1247	T = task_index_to_char(field->next_state);
1248	S = task_index_to_char(field->prev_state);
1249	trace_find_cmdline(field->next_pid, comm);
1250	trace_seq_printf(&iter->seq,
1251			 " %7d:%3d:%c %s [%03d] %7d:%3d:%c %s\n",
1252			 field->prev_pid,
1253			 field->prev_prio,
1254			 S, delim,
1255			 field->next_cpu,
1256			 field->next_pid,
1257			 field->next_prio,
1258			 T, comm);
1259
1260	return trace_handle_return(&iter->seq);
1261}
1262
1263static enum print_line_t trace_ctx_print(struct trace_iterator *iter, int flags,
1264					 struct trace_event *event)
1265{
1266	return trace_ctxwake_print(iter, "==>");
1267}
1268
1269static enum print_line_t trace_wake_print(struct trace_iterator *iter,
1270					  int flags, struct trace_event *event)
1271{
1272	return trace_ctxwake_print(iter, "  +");
1273}
1274
1275static int trace_ctxwake_raw(struct trace_iterator *iter, char S)
1276{
1277	struct ctx_switch_entry *field;
1278	int T;
1279
1280	trace_assign_type(field, iter->ent);
1281
1282	if (!S)
1283		S = task_index_to_char(field->prev_state);
1284	T = task_index_to_char(field->next_state);
1285	trace_seq_printf(&iter->seq, "%d %d %c %d %d %d %c\n",
1286			 field->prev_pid,
1287			 field->prev_prio,
1288			 S,
1289			 field->next_cpu,
1290			 field->next_pid,
1291			 field->next_prio,
1292			 T);
1293
1294	return trace_handle_return(&iter->seq);
1295}
1296
1297static enum print_line_t trace_ctx_raw(struct trace_iterator *iter, int flags,
1298				       struct trace_event *event)
1299{
1300	return trace_ctxwake_raw(iter, 0);
1301}
1302
1303static enum print_line_t trace_wake_raw(struct trace_iterator *iter, int flags,
1304					struct trace_event *event)
1305{
1306	return trace_ctxwake_raw(iter, '+');
1307}
1308
1309
1310static int trace_ctxwake_hex(struct trace_iterator *iter, char S)
1311{
1312	struct ctx_switch_entry *field;
1313	struct trace_seq *s = &iter->seq;
1314	int T;
1315
1316	trace_assign_type(field, iter->ent);
1317
1318	if (!S)
1319		S = task_index_to_char(field->prev_state);
1320	T = task_index_to_char(field->next_state);
1321
1322	SEQ_PUT_HEX_FIELD(s, field->prev_pid);
1323	SEQ_PUT_HEX_FIELD(s, field->prev_prio);
1324	SEQ_PUT_HEX_FIELD(s, S);
1325	SEQ_PUT_HEX_FIELD(s, field->next_cpu);
1326	SEQ_PUT_HEX_FIELD(s, field->next_pid);
1327	SEQ_PUT_HEX_FIELD(s, field->next_prio);
1328	SEQ_PUT_HEX_FIELD(s, T);
1329
1330	return trace_handle_return(s);
1331}
1332
1333static enum print_line_t trace_ctx_hex(struct trace_iterator *iter, int flags,
1334				       struct trace_event *event)
1335{
1336	return trace_ctxwake_hex(iter, 0);
1337}
1338
1339static enum print_line_t trace_wake_hex(struct trace_iterator *iter, int flags,
1340					struct trace_event *event)
1341{
1342	return trace_ctxwake_hex(iter, '+');
1343}
1344
1345static enum print_line_t trace_ctxwake_bin(struct trace_iterator *iter,
1346					   int flags, struct trace_event *event)
1347{
1348	struct ctx_switch_entry *field;
1349	struct trace_seq *s = &iter->seq;
1350
1351	trace_assign_type(field, iter->ent);
1352
1353	SEQ_PUT_FIELD(s, field->prev_pid);
1354	SEQ_PUT_FIELD(s, field->prev_prio);
1355	SEQ_PUT_FIELD(s, field->prev_state);
1356	SEQ_PUT_FIELD(s, field->next_cpu);
1357	SEQ_PUT_FIELD(s, field->next_pid);
1358	SEQ_PUT_FIELD(s, field->next_prio);
1359	SEQ_PUT_FIELD(s, field->next_state);
1360
1361	return trace_handle_return(s);
1362}
1363
1364static struct trace_event_functions trace_ctx_funcs = {
1365	.trace		= trace_ctx_print,
1366	.raw		= trace_ctx_raw,
1367	.hex		= trace_ctx_hex,
1368	.binary		= trace_ctxwake_bin,
1369};
1370
1371static struct trace_event trace_ctx_event = {
1372	.type		= TRACE_CTX,
1373	.funcs		= &trace_ctx_funcs,
1374};
1375
1376static struct trace_event_functions trace_wake_funcs = {
1377	.trace		= trace_wake_print,
1378	.raw		= trace_wake_raw,
1379	.hex		= trace_wake_hex,
1380	.binary		= trace_ctxwake_bin,
1381};
1382
1383static struct trace_event trace_wake_event = {
1384	.type		= TRACE_WAKE,
1385	.funcs		= &trace_wake_funcs,
1386};
1387
1388/* TRACE_STACK */
1389
1390static enum print_line_t trace_stack_print(struct trace_iterator *iter,
1391					   int flags, struct trace_event *event)
1392{
1393	struct stack_entry *field;
1394	struct trace_seq *s = &iter->seq;
1395	unsigned long *p;
1396	unsigned long *end;
1397
1398	trace_assign_type(field, iter->ent);
1399	end = (unsigned long *)((long)iter->ent + iter->ent_size);
1400
1401	trace_seq_puts(s, "<stack trace>\n");
1402
1403	for (p = field->caller; p && p < end && *p != ULONG_MAX; p++) {
1404
1405		if (trace_seq_has_overflowed(s))
1406			break;
1407
1408		trace_seq_puts(s, " => ");
1409		if ((*p) == FTRACE_TRAMPOLINE_MARKER) {
1410			trace_seq_puts(s, "[FTRACE TRAMPOLINE]\n");
1411			continue;
1412		}
1413		seq_print_ip_sym(s, trace_adjust_address(iter->tr, *p), flags);
1414		trace_seq_putc(s, '\n');
1415	}
1416
1417	return trace_handle_return(s);
1418}
1419
1420static struct trace_event_functions trace_stack_funcs = {
1421	.trace		= trace_stack_print,
1422};
1423
1424static struct trace_event trace_stack_event = {
1425	.type		= TRACE_STACK,
1426	.funcs		= &trace_stack_funcs,
1427};
1428
1429/* TRACE_USER_STACK */
1430static enum print_line_t trace_user_stack_print(struct trace_iterator *iter,
1431						int flags, struct trace_event *event)
1432{
1433	struct trace_array *tr = iter->tr;
1434	struct userstack_entry *field;
1435	struct trace_seq *s = &iter->seq;
1436	struct mm_struct *mm = NULL;
1437	unsigned int i;
1438
1439	trace_assign_type(field, iter->ent);
1440
1441	trace_seq_puts(s, "<user stack trace>\n");
1442
1443	if (tr->trace_flags & TRACE_ITER(SYM_USEROBJ)) {
1444		struct task_struct *task;
1445		/*
1446		 * we do the lookup on the thread group leader,
1447		 * since individual threads might have already quit!
1448		 */
1449		rcu_read_lock();
1450		task = find_task_by_vpid(field->tgid);
1451		if (task)
1452			mm = get_task_mm(task);
1453		rcu_read_unlock();
1454	}
1455
1456	for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1457		unsigned long ip = field->caller[i];
1458
1459		if (!ip || trace_seq_has_overflowed(s))
1460			break;
1461
1462		trace_seq_puts(s, " => ");
1463		seq_print_user_ip(s, mm, ip, flags);
1464		trace_seq_putc(s, '\n');
1465	}
1466
1467	if (mm)
1468		mmput(mm);
1469
1470	return trace_handle_return(s);
1471}
1472
1473static struct trace_event_functions trace_user_stack_funcs = {
1474	.trace		= trace_user_stack_print,
1475};
1476
1477static struct trace_event trace_user_stack_event = {
1478	.type		= TRACE_USER_STACK,
1479	.funcs		= &trace_user_stack_funcs,
1480};
1481
1482/* TRACE_HWLAT */
1483static enum print_line_t
1484trace_hwlat_print(struct trace_iterator *iter, int flags,
1485		  struct trace_event *event)
1486{
1487	struct trace_entry *entry = iter->ent;
1488	struct trace_seq *s = &iter->seq;
1489	struct hwlat_entry *field;
1490
1491	trace_assign_type(field, entry);
1492
1493	trace_seq_printf(s, "#%-5u inner/outer(us): %4llu/%-5llu ts:%ptSp count:%d",
1494			 field->seqnum,
1495			 field->duration,
1496			 field->outer_duration,
1497			 &field->timestamp,
1498			 field->count);
1499
1500	if (field->nmi_count) {
1501		/*
1502		 * The generic sched_clock() is not NMI safe, thus
1503		 * we only record the count and not the time.
1504		 */
1505		if (!IS_ENABLED(CONFIG_GENERIC_SCHED_CLOCK))
1506			trace_seq_printf(s, " nmi-total:%llu",
1507					 field->nmi_total_ts);
1508		trace_seq_printf(s, " nmi-count:%u",
1509				 field->nmi_count);
1510	}
1511
1512	trace_seq_putc(s, '\n');
1513
1514	return trace_handle_return(s);
1515}
1516
1517static enum print_line_t
1518trace_hwlat_raw(struct trace_iterator *iter, int flags,
1519		struct trace_event *event)
1520{
1521	struct hwlat_entry *field;
1522	struct trace_seq *s = &iter->seq;
1523
1524	trace_assign_type(field, iter->ent);
1525
1526	trace_seq_printf(s, "%llu %lld %lld %09ld %u\n",
1527			 field->duration,
1528			 field->outer_duration,
1529			 (long long)field->timestamp.tv_sec,
1530			 field->timestamp.tv_nsec,
1531			 field->seqnum);
1532
1533	return trace_handle_return(s);
1534}
1535
1536static struct trace_event_functions trace_hwlat_funcs = {
1537	.trace		= trace_hwlat_print,
1538	.raw		= trace_hwlat_raw,
1539};
1540
1541static struct trace_event trace_hwlat_event = {
1542	.type		= TRACE_HWLAT,
1543	.funcs		= &trace_hwlat_funcs,
1544};
1545
1546/* TRACE_OSNOISE */
1547static enum print_line_t
1548trace_osnoise_print(struct trace_iterator *iter, int flags,
1549		    struct trace_event *event)
1550{
1551	struct trace_entry *entry = iter->ent;
1552	struct trace_seq *s = &iter->seq;
1553	struct osnoise_entry *field;
1554	u64 ratio, ratio_dec;
1555	u64 net_runtime;
1556
1557	trace_assign_type(field, entry);
1558
1559	/*
1560	 * compute the available % of cpu time.
1561	 */
1562	net_runtime = field->runtime - field->noise;
1563	ratio = net_runtime * 10000000;
1564	do_div(ratio, field->runtime);
1565	ratio_dec = do_div(ratio, 100000);
1566
1567	trace_seq_printf(s, "%llu %10llu %3llu.%05llu %7llu",
1568			 field->runtime,
1569			 field->noise,
1570			 ratio, ratio_dec,
1571			 field->max_sample);
1572
1573	trace_seq_printf(s, " %6u", field->hw_count);
1574	trace_seq_printf(s, " %6u", field->nmi_count);
1575	trace_seq_printf(s, " %6u", field->irq_count);
1576	trace_seq_printf(s, " %6u", field->softirq_count);
1577	trace_seq_printf(s, " %6u", field->thread_count);
1578
1579	trace_seq_putc(s, '\n');
1580
1581	return trace_handle_return(s);
1582}
1583
1584static enum print_line_t
1585trace_osnoise_raw(struct trace_iterator *iter, int flags,
1586		  struct trace_event *event)
1587{
1588	struct osnoise_entry *field;
1589	struct trace_seq *s = &iter->seq;
1590
1591	trace_assign_type(field, iter->ent);
1592
1593	trace_seq_printf(s, "%lld %llu %llu %u %u %u %u %u\n",
1594			 field->runtime,
1595			 field->noise,
1596			 field->max_sample,
1597			 field->hw_count,
1598			 field->nmi_count,
1599			 field->irq_count,
1600			 field->softirq_count,
1601			 field->thread_count);
1602
1603	return trace_handle_return(s);
1604}
1605
1606static struct trace_event_functions trace_osnoise_funcs = {
1607	.trace		= trace_osnoise_print,
1608	.raw		= trace_osnoise_raw,
1609};
1610
1611static struct trace_event trace_osnoise_event = {
1612	.type		= TRACE_OSNOISE,
1613	.funcs		= &trace_osnoise_funcs,
1614};
1615
1616/* TRACE_TIMERLAT */
1617
1618static char *timerlat_lat_context[] = {"irq", "thread", "user-ret"};
1619static enum print_line_t
1620trace_timerlat_print(struct trace_iterator *iter, int flags,
1621		     struct trace_event *event)
1622{
1623	struct trace_entry *entry = iter->ent;
1624	struct trace_seq *s = &iter->seq;
1625	struct timerlat_entry *field;
1626
1627	trace_assign_type(field, entry);
1628
1629	trace_seq_printf(s, "#%-5u context %6s timer_latency %9llu ns\n",
1630			 field->seqnum,
1631			 timerlat_lat_context[field->context],
1632			 field->timer_latency);
1633
1634	return trace_handle_return(s);
1635}
1636
1637static enum print_line_t
1638trace_timerlat_raw(struct trace_iterator *iter, int flags,
1639		   struct trace_event *event)
1640{
1641	struct timerlat_entry *field;
1642	struct trace_seq *s = &iter->seq;
1643
1644	trace_assign_type(field, iter->ent);
1645
1646	trace_seq_printf(s, "%u %d %llu\n",
1647			 field->seqnum,
1648			 field->context,
1649			 field->timer_latency);
1650
1651	return trace_handle_return(s);
1652}
1653
1654static struct trace_event_functions trace_timerlat_funcs = {
1655	.trace		= trace_timerlat_print,
1656	.raw		= trace_timerlat_raw,
1657};
1658
1659static struct trace_event trace_timerlat_event = {
1660	.type		= TRACE_TIMERLAT,
1661	.funcs		= &trace_timerlat_funcs,
1662};
1663
1664/* TRACE_BPUTS */
1665static enum print_line_t
1666trace_bputs_print(struct trace_iterator *iter, int flags,
1667		   struct trace_event *event)
1668{
1669	struct trace_entry *entry = iter->ent;
1670	struct trace_seq *s = &iter->seq;
1671	struct bputs_entry *field;
1672
1673	trace_assign_type(field, entry);
1674
1675	seq_print_ip_sym(s, field->ip, flags);
1676	trace_seq_puts(s, ": ");
1677	trace_seq_puts(s, field->str);
1678
1679	return trace_handle_return(s);
1680}
1681
1682
1683static enum print_line_t
1684trace_bputs_raw(struct trace_iterator *iter, int flags,
1685		struct trace_event *event)
1686{
1687	struct bputs_entry *field;
1688	struct trace_seq *s = &iter->seq;
1689
1690	trace_assign_type(field, iter->ent);
1691
1692	trace_seq_printf(s, ": %lx : ", field->ip);
1693	trace_seq_puts(s, field->str);
1694
1695	return trace_handle_return(s);
1696}
1697
1698static struct trace_event_functions trace_bputs_funcs = {
1699	.trace		= trace_bputs_print,
1700	.raw		= trace_bputs_raw,
1701};
1702
1703static struct trace_event trace_bputs_event = {
1704	.type		= TRACE_BPUTS,
1705	.funcs		= &trace_bputs_funcs,
1706};
1707
1708/* TRACE_BPRINT */
1709static enum print_line_t
1710trace_bprint_print(struct trace_iterator *iter, int flags,
1711		   struct trace_event *event)
1712{
1713	struct trace_entry *entry = iter->ent;
1714	struct trace_seq *s = &iter->seq;
1715	struct bprint_entry *field;
1716
1717	trace_assign_type(field, entry);
1718
1719	seq_print_ip_sym(s, field->ip, flags);
1720	trace_seq_puts(s, ": ");
1721	trace_seq_bprintf(s, field->fmt, field->buf);
1722
1723	return trace_handle_return(s);
1724}
1725
1726
1727static enum print_line_t
1728trace_bprint_raw(struct trace_iterator *iter, int flags,
1729		 struct trace_event *event)
1730{
1731	struct bprint_entry *field;
1732	struct trace_seq *s = &iter->seq;
1733
1734	trace_assign_type(field, iter->ent);
1735
1736	trace_seq_printf(s, ": %lx : ", field->ip);
1737	trace_seq_bprintf(s, field->fmt, field->buf);
1738
1739	return trace_handle_return(s);
1740}
1741
1742static struct trace_event_functions trace_bprint_funcs = {
1743	.trace		= trace_bprint_print,
1744	.raw		= trace_bprint_raw,
1745};
1746
1747static struct trace_event trace_bprint_event = {
1748	.type		= TRACE_BPRINT,
1749	.funcs		= &trace_bprint_funcs,
1750};
1751
1752/* TRACE_PRINT */
1753static enum print_line_t trace_print_print(struct trace_iterator *iter,
1754					   int flags, struct trace_event *event)
1755{
1756	struct print_entry *field;
1757	struct trace_seq *s = &iter->seq;
1758	unsigned long ip;
1759
1760	trace_assign_type(field, iter->ent);
1761
1762	ip = trace_adjust_address(iter->tr, field->ip);
1763
1764	seq_print_ip_sym(s, ip, flags);
1765	trace_seq_printf(s, ": %s", field->buf);
1766
1767	return trace_handle_return(s);
1768}
1769
1770static enum print_line_t trace_print_raw(struct trace_iterator *iter, int flags,
1771					 struct trace_event *event)
1772{
1773	struct print_entry *field;
1774
1775	trace_assign_type(field, iter->ent);
1776
1777	trace_seq_printf(&iter->seq, "# %lx %s", field->ip, field->buf);
1778
1779	return trace_handle_return(&iter->seq);
1780}
1781
1782static struct trace_event_functions trace_print_funcs = {
1783	.trace		= trace_print_print,
1784	.raw		= trace_print_raw,
1785};
1786
1787static struct trace_event trace_print_event = {
1788	.type	 	= TRACE_PRINT,
1789	.funcs		= &trace_print_funcs,
1790};
1791
1792static enum print_line_t trace_raw_data(struct trace_iterator *iter, int flags,
1793					 struct trace_event *event)
1794{
1795	struct raw_data_entry *field;
1796	int i;
1797
1798	trace_assign_type(field, iter->ent);
1799
1800	trace_seq_printf(&iter->seq, "# %x buf:", field->id);
1801
1802	for (i = 0; i < iter->ent_size - offsetof(struct raw_data_entry, buf); i++)
1803		trace_seq_printf(&iter->seq, " %02x",
1804				 (unsigned char)field->buf[i]);
1805
1806	trace_seq_putc(&iter->seq, '\n');
1807
1808	return trace_handle_return(&iter->seq);
1809}
1810
1811static struct trace_event_functions trace_raw_data_funcs = {
1812	.trace		= trace_raw_data,
1813	.raw		= trace_raw_data,
1814};
1815
1816static struct trace_event trace_raw_data_event = {
1817	.type	 	= TRACE_RAW_DATA,
1818	.funcs		= &trace_raw_data_funcs,
1819};
1820
1821static enum print_line_t
1822trace_func_repeats_raw(struct trace_iterator *iter, int flags,
1823			 struct trace_event *event)
1824{
1825	struct func_repeats_entry *field;
1826	struct trace_seq *s = &iter->seq;
1827
1828	trace_assign_type(field, iter->ent);
1829
1830	trace_seq_printf(s, "%lu %lu %u %llu\n",
1831			 field->ip,
1832			 field->parent_ip,
1833			 field->count,
1834			 FUNC_REPEATS_GET_DELTA_TS(field));
1835
1836	return trace_handle_return(s);
1837}
1838
1839static enum print_line_t
1840trace_func_repeats_print(struct trace_iterator *iter, int flags,
1841			 struct trace_event *event)
1842{
1843	struct func_repeats_entry *field;
1844	struct trace_seq *s = &iter->seq;
1845
1846	trace_assign_type(field, iter->ent);
1847
1848	print_fn_trace(s, field->ip, field->parent_ip, NULL, iter->tr, flags);
1849	trace_seq_printf(s, " (repeats: %u, last_ts:", field->count);
1850	trace_print_time(s, iter,
1851			 iter->ts - FUNC_REPEATS_GET_DELTA_TS(field));
1852	trace_seq_puts(s, ")\n");
1853
1854	return trace_handle_return(s);
1855}
1856
1857static struct trace_event_functions trace_func_repeats_funcs = {
1858	.trace		= trace_func_repeats_print,
1859	.raw		= trace_func_repeats_raw,
1860};
1861
1862static struct trace_event trace_func_repeats_event = {
1863	.type	 	= TRACE_FUNC_REPEATS,
1864	.funcs		= &trace_func_repeats_funcs,
1865};
1866
1867static struct trace_event *events[] __initdata = {
1868	&trace_fn_event,
1869	&trace_ctx_event,
1870	&trace_wake_event,
1871	&trace_stack_event,
1872	&trace_user_stack_event,
1873	&trace_bputs_event,
1874	&trace_bprint_event,
1875	&trace_print_event,
1876	&trace_hwlat_event,
1877	&trace_osnoise_event,
1878	&trace_timerlat_event,
1879	&trace_raw_data_event,
1880	&trace_func_repeats_event,
1881	NULL
1882};
1883
1884__init int init_events(void)
1885{
1886	struct trace_event *event;
1887	int i, ret;
1888
1889	for (i = 0; events[i]; i++) {
1890		event = events[i];
1891		ret = register_trace_event(event);
1892		WARN_ONCE(!ret, "event %d failed to register", event->type);
1893	}
1894
1895	return 0;
1896}