builtin-kwork.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * builtin-kwork.c
   4 *
   5 * Copyright (c) 2022  Huawei Inc,  Yang Jihong <yangjihong1@huawei.com>
   6 */
   7
   8#include "builtin.h"
   9#include "perf.h"
  10
  11#include "util/data.h"
  12#include "util/evlist.h"
  13#include "util/evsel.h"
  14#include "util/header.h"
  15#include "util/kwork.h"
  16#include "util/debug.h"
  17#include "util/session.h"
  18#include "util/symbol.h"
  19#include "util/thread.h"
  20#include "util/string2.h"
  21#include "util/callchain.h"
  22#include "util/evsel_fprintf.h"
  23#include "util/util.h"
  24
  25#include <subcmd/pager.h>
  26#include <subcmd/parse-options.h>
  27#include <event-parse.h>
  28
  29#include <errno.h>
  30#include <inttypes.h>
  31#include <signal.h>
  32#include <linux/err.h>
  33#include <linux/time64.h>
  34#include <linux/zalloc.h>
  35
  36/*
  37 * report header elements width
  38 */
  39#define PRINT_CPU_WIDTH 4
  40#define PRINT_COUNT_WIDTH 9
  41#define PRINT_RUNTIME_WIDTH 10
  42#define PRINT_LATENCY_WIDTH 10
  43#define PRINT_TIMESTAMP_WIDTH 17
  44#define PRINT_KWORK_NAME_WIDTH 30
  45#define RPINT_DECIMAL_WIDTH 3
  46#define PRINT_BRACKETPAIR_WIDTH 2
  47#define PRINT_TIME_UNIT_SEC_WIDTH 2
  48#define PRINT_TIME_UNIT_MESC_WIDTH 3
  49#define PRINT_PID_WIDTH 7
  50#define PRINT_TASK_NAME_WIDTH 16
  51#define PRINT_CPU_USAGE_WIDTH 6
  52#define PRINT_CPU_USAGE_DECIMAL_WIDTH 2
  53#define PRINT_CPU_USAGE_HIST_WIDTH 30
  54#define PRINT_RUNTIME_HEADER_WIDTH (PRINT_RUNTIME_WIDTH + PRINT_TIME_UNIT_MESC_WIDTH)
  55#define PRINT_LATENCY_HEADER_WIDTH (PRINT_LATENCY_WIDTH + PRINT_TIME_UNIT_MESC_WIDTH)
  56#define PRINT_TIMEHIST_CPU_WIDTH (PRINT_CPU_WIDTH + PRINT_BRACKETPAIR_WIDTH)
  57#define PRINT_TIMESTAMP_HEADER_WIDTH (PRINT_TIMESTAMP_WIDTH + PRINT_TIME_UNIT_SEC_WIDTH)
  58
  59struct sort_dimension {
  60	const char      *name;
  61	int             (*cmp)(struct kwork_work *l, struct kwork_work *r);
  62	struct          list_head list;
  63};
  64
  65static int id_cmp(struct kwork_work *l, struct kwork_work *r)
  66{
  67	if (l->cpu > r->cpu)
  68		return 1;
  69	if (l->cpu < r->cpu)
  70		return -1;
  71
  72	if (l->id > r->id)
  73		return 1;
  74	if (l->id < r->id)
  75		return -1;
  76
  77	return 0;
  78}
  79
  80static int count_cmp(struct kwork_work *l, struct kwork_work *r)
  81{
  82	if (l->nr_atoms > r->nr_atoms)
  83		return 1;
  84	if (l->nr_atoms < r->nr_atoms)
  85		return -1;
  86
  87	return 0;
  88}
  89
  90static int runtime_cmp(struct kwork_work *l, struct kwork_work *r)
  91{
  92	if (l->total_runtime > r->total_runtime)
  93		return 1;
  94	if (l->total_runtime < r->total_runtime)
  95		return -1;
  96
  97	return 0;
  98}
  99
 100static int max_runtime_cmp(struct kwork_work *l, struct kwork_work *r)
 101{
 102	if (l->max_runtime > r->max_runtime)
 103		return 1;
 104	if (l->max_runtime < r->max_runtime)
 105		return -1;
 106
 107	return 0;
 108}
 109
 110static int avg_latency_cmp(struct kwork_work *l, struct kwork_work *r)
 111{
 112	u64 avgl, avgr;
 113
 114	if (!r->nr_atoms)
 115		return 1;
 116	if (!l->nr_atoms)
 117		return -1;
 118
 119	avgl = l->total_latency / l->nr_atoms;
 120	avgr = r->total_latency / r->nr_atoms;
 121
 122	if (avgl > avgr)
 123		return 1;
 124	if (avgl < avgr)
 125		return -1;
 126
 127	return 0;
 128}
 129
 130static int max_latency_cmp(struct kwork_work *l, struct kwork_work *r)
 131{
 132	if (l->max_latency > r->max_latency)
 133		return 1;
 134	if (l->max_latency < r->max_latency)
 135		return -1;
 136
 137	return 0;
 138}
 139
 140static int cpu_usage_cmp(struct kwork_work *l, struct kwork_work *r)
 141{
 142	if (l->cpu_usage > r->cpu_usage)
 143		return 1;
 144	if (l->cpu_usage < r->cpu_usage)
 145		return -1;
 146
 147	return 0;
 148}
 149
 150static int id_or_cpu_r_cmp(struct kwork_work *l, struct kwork_work *r)
 151{
 152	if (l->id < r->id)
 153		return 1;
 154	if (l->id > r->id)
 155		return -1;
 156
 157	if (l->id != 0)
 158		return 0;
 159
 160	if (l->cpu < r->cpu)
 161		return 1;
 162	if (l->cpu > r->cpu)
 163		return -1;
 164
 165	return 0;
 166}
 167
 168static int sort_dimension__add(struct perf_kwork *kwork __maybe_unused,
 169			       const char *tok, struct list_head *list)
 170{
 171	size_t i;
 172	static struct sort_dimension max_sort_dimension = {
 173		.name = "max",
 174		.cmp  = max_runtime_cmp,
 175	};
 176	static struct sort_dimension id_sort_dimension = {
 177		.name = "id",
 178		.cmp  = id_cmp,
 179	};
 180	static struct sort_dimension runtime_sort_dimension = {
 181		.name = "runtime",
 182		.cmp  = runtime_cmp,
 183	};
 184	static struct sort_dimension count_sort_dimension = {
 185		.name = "count",
 186		.cmp  = count_cmp,
 187	};
 188	static struct sort_dimension avg_sort_dimension = {
 189		.name = "avg",
 190		.cmp  = avg_latency_cmp,
 191	};
 192	static struct sort_dimension rate_sort_dimension = {
 193		.name = "rate",
 194		.cmp  = cpu_usage_cmp,
 195	};
 196	static struct sort_dimension tid_sort_dimension = {
 197		.name = "tid",
 198		.cmp  = id_or_cpu_r_cmp,
 199	};
 200	struct sort_dimension *available_sorts[] = {
 201		&id_sort_dimension,
 202		&max_sort_dimension,
 203		&count_sort_dimension,
 204		&runtime_sort_dimension,
 205		&avg_sort_dimension,
 206		&rate_sort_dimension,
 207		&tid_sort_dimension,
 208	};
 209
 210	if (kwork->report == KWORK_REPORT_LATENCY)
 211		max_sort_dimension.cmp = max_latency_cmp;
 212
 213	for (i = 0; i < ARRAY_SIZE(available_sorts); i++) {
 214		if (!strcmp(available_sorts[i]->name, tok)) {
 215			list_add_tail(&available_sorts[i]->list, list);
 216			return 0;
 217		}
 218	}
 219
 220	return -1;
 221}
 222
 223static void setup_sorting(struct perf_kwork *kwork,
 224			  const struct option *options,
 225			  const char * const usage_msg[])
 226{
 227	char *tmp, *tok, *str = strdup(kwork->sort_order);
 228
 229	for (tok = strtok_r(str, ", ", &tmp);
 230	     tok; tok = strtok_r(NULL, ", ", &tmp)) {
 231		if (sort_dimension__add(kwork, tok, &kwork->sort_list) < 0)
 232			usage_with_options_msg(usage_msg, options,
 233					       "Unknown --sort key: `%s'", tok);
 234	}
 235
 236	pr_debug("Sort order: %s\n", kwork->sort_order);
 237	free(str);
 238}
 239
 240static struct kwork_atom *atom_new(struct perf_kwork *kwork,
 241				   struct perf_sample *sample)
 242{
 243	unsigned long i;
 244	struct kwork_atom_page *page;
 245	struct kwork_atom *atom = NULL;
 246
 247	list_for_each_entry(page, &kwork->atom_page_list, list) {
 248		if (!bitmap_full(page->bitmap, NR_ATOM_PER_PAGE)) {
 249			i = find_first_zero_bit(page->bitmap, NR_ATOM_PER_PAGE);
 250			BUG_ON(i >= NR_ATOM_PER_PAGE);
 251			atom = &page->atoms[i];
 252			goto found_atom;
 253		}
 254	}
 255
 256	/*
 257	 * new page
 258	 */
 259	page = zalloc(sizeof(*page));
 260	if (page == NULL) {
 261		pr_err("Failed to zalloc kwork atom page\n");
 262		return NULL;
 263	}
 264
 265	i = 0;
 266	atom = &page->atoms[0];
 267	list_add_tail(&page->list, &kwork->atom_page_list);
 268
 269found_atom:
 270	__set_bit(i, page->bitmap);
 271	atom->time = sample->time;
 272	atom->prev = NULL;
 273	atom->page_addr = page;
 274	atom->bit_inpage = i;
 275	return atom;
 276}
 277
 278static void atom_free(struct kwork_atom *atom)
 279{
 280	if (atom->prev != NULL)
 281		atom_free(atom->prev);
 282
 283	__clear_bit(atom->bit_inpage,
 284		    ((struct kwork_atom_page *)atom->page_addr)->bitmap);
 285}
 286
 287static void atom_del(struct kwork_atom *atom)
 288{
 289	list_del(&atom->list);
 290	atom_free(atom);
 291}
 292
 293static int work_cmp(struct list_head *list,
 294		    struct kwork_work *l, struct kwork_work *r)
 295{
 296	int ret = 0;
 297	struct sort_dimension *sort;
 298
 299	BUG_ON(list_empty(list));
 300
 301	list_for_each_entry(sort, list, list) {
 302		ret = sort->cmp(l, r);
 303		if (ret)
 304			return ret;
 305	}
 306
 307	return ret;
 308}
 309
 310static struct kwork_work *work_search(struct rb_root_cached *root,
 311				      struct kwork_work *key,
 312				      struct list_head *sort_list)
 313{
 314	int cmp;
 315	struct kwork_work *work;
 316	struct rb_node *node = root->rb_root.rb_node;
 317
 318	while (node) {
 319		work = container_of(node, struct kwork_work, node);
 320		cmp = work_cmp(sort_list, key, work);
 321		if (cmp > 0)
 322			node = node->rb_left;
 323		else if (cmp < 0)
 324			node = node->rb_right;
 325		else {
 326			if (work->name == NULL)
 327				work->name = key->name;
 328			return work;
 329		}
 330	}
 331	return NULL;
 332}
 333
 334static void work_insert(struct rb_root_cached *root,
 335			struct kwork_work *key, struct list_head *sort_list)
 336{
 337	int cmp;
 338	bool leftmost = true;
 339	struct kwork_work *cur;
 340	struct rb_node **new = &(root->rb_root.rb_node), *parent = NULL;
 341
 342	while (*new) {
 343		cur = container_of(*new, struct kwork_work, node);
 344		parent = *new;
 345		cmp = work_cmp(sort_list, key, cur);
 346
 347		if (cmp > 0)
 348			new = &((*new)->rb_left);
 349		else {
 350			new = &((*new)->rb_right);
 351			leftmost = false;
 352		}
 353	}
 354
 355	rb_link_node(&key->node, parent, new);
 356	rb_insert_color_cached(&key->node, root, leftmost);
 357}
 358
 359static struct kwork_work *work_new(struct kwork_work *key)
 360{
 361	int i;
 362	struct kwork_work *work = zalloc(sizeof(*work));
 363
 364	if (work == NULL) {
 365		pr_err("Failed to zalloc kwork work\n");
 366		return NULL;
 367	}
 368
 369	for (i = 0; i < KWORK_TRACE_MAX; i++)
 370		INIT_LIST_HEAD(&work->atom_list[i]);
 371
 372	work->id = key->id;
 373	work->cpu = key->cpu;
 374	work->name = key->name;
 375	work->class = key->class;
 376	return work;
 377}
 378
 379static struct kwork_work *work_findnew(struct rb_root_cached *root,
 380				       struct kwork_work *key,
 381				       struct list_head *sort_list)
 382{
 383	struct kwork_work *work = work_search(root, key, sort_list);
 384
 385	if (work != NULL)
 386		return work;
 387
 388	work = work_new(key);
 389	if (work)
 390		work_insert(root, work, sort_list);
 391
 392	return work;
 393}
 394
 395static void profile_update_timespan(struct perf_kwork *kwork,
 396				    struct perf_sample *sample)
 397{
 398	if (!kwork->summary)
 399		return;
 400
 401	if ((kwork->timestart == 0) || (kwork->timestart > sample->time))
 402		kwork->timestart = sample->time;
 403
 404	if (kwork->timeend < sample->time)
 405		kwork->timeend = sample->time;
 406}
 407
 408static bool profile_name_match(struct perf_kwork *kwork,
 409			       struct kwork_work *work)
 410{
 411	if (kwork->profile_name && work->name &&
 412	    (strcmp(work->name, kwork->profile_name) != 0)) {
 413		return false;
 414	}
 415
 416	return true;
 417}
 418
 419static bool profile_event_match(struct perf_kwork *kwork,
 420				struct kwork_work *work,
 421				struct perf_sample *sample)
 422{
 423	int cpu = work->cpu;
 424	u64 time = sample->time;
 425	struct perf_time_interval *ptime = &kwork->ptime;
 426
 427	if ((kwork->cpu_list != NULL) && !test_bit(cpu, kwork->cpu_bitmap))
 428		return false;
 429
 430	if (((ptime->start != 0) && (ptime->start > time)) ||
 431	    ((ptime->end != 0) && (ptime->end < time)))
 432		return false;
 433
 434	/*
 435	 * report top needs to collect the runtime of all tasks to
 436	 * calculate the load of each core.
 437	 */
 438	if ((kwork->report != KWORK_REPORT_TOP) &&
 439	    !profile_name_match(kwork, work)) {
 440		return false;
 441	}
 442
 443	profile_update_timespan(kwork, sample);
 444	return true;
 445}
 446
 447static int work_push_atom(struct perf_kwork *kwork,
 448			  struct kwork_class *class,
 449			  enum kwork_trace_type src_type,
 450			  enum kwork_trace_type dst_type,
 451			  struct evsel *evsel,
 452			  struct perf_sample *sample,
 453			  struct machine *machine,
 454			  struct kwork_work **ret_work,
 455			  bool overwrite)
 456{
 457	struct kwork_atom *atom, *dst_atom, *last_atom;
 458	struct kwork_work *work, key;
 459
 460	BUG_ON(class->work_init == NULL);
 461	class->work_init(kwork, class, &key, src_type, evsel, sample, machine);
 462
 463	atom = atom_new(kwork, sample);
 464	if (atom == NULL)
 465		return -1;
 466
 467	work = work_findnew(&class->work_root, &key, &kwork->cmp_id);
 468	if (work == NULL) {
 469		atom_free(atom);
 470		return -1;
 471	}
 472
 473	if (!profile_event_match(kwork, work, sample)) {
 474		atom_free(atom);
 475		return 0;
 476	}
 477
 478	if (dst_type < KWORK_TRACE_MAX) {
 479		dst_atom = list_last_entry_or_null(&work->atom_list[dst_type],
 480						   struct kwork_atom, list);
 481		if (dst_atom != NULL) {
 482			atom->prev = dst_atom;
 483			list_del(&dst_atom->list);
 484		}
 485	}
 486
 487	if (ret_work != NULL)
 488		*ret_work = work;
 489
 490	if (overwrite) {
 491		last_atom = list_last_entry_or_null(&work->atom_list[src_type],
 492						    struct kwork_atom, list);
 493		if (last_atom) {
 494			atom_del(last_atom);
 495
 496			kwork->nr_skipped_events[src_type]++;
 497			kwork->nr_skipped_events[KWORK_TRACE_MAX]++;
 498		}
 499	}
 500
 501	list_add_tail(&atom->list, &work->atom_list[src_type]);
 502
 503	return 0;
 504}
 505
 506static struct kwork_atom *work_pop_atom(struct perf_kwork *kwork,
 507					struct kwork_class *class,
 508					enum kwork_trace_type src_type,
 509					enum kwork_trace_type dst_type,
 510					struct evsel *evsel,
 511					struct perf_sample *sample,
 512					struct machine *machine,
 513					struct kwork_work **ret_work)
 514{
 515	struct kwork_atom *atom, *src_atom;
 516	struct kwork_work *work, key;
 517
 518	BUG_ON(class->work_init == NULL);
 519	class->work_init(kwork, class, &key, src_type, evsel, sample, machine);
 520
 521	work = work_findnew(&class->work_root, &key, &kwork->cmp_id);
 522	if (ret_work != NULL)
 523		*ret_work = work;
 524
 525	if (work == NULL)
 526		return NULL;
 527
 528	if (!profile_event_match(kwork, work, sample))
 529		return NULL;
 530
 531	atom = list_last_entry_or_null(&work->atom_list[dst_type],
 532				       struct kwork_atom, list);
 533	if (atom != NULL)
 534		return atom;
 535
 536	src_atom = atom_new(kwork, sample);
 537	if (src_atom != NULL)
 538		list_add_tail(&src_atom->list, &work->atom_list[src_type]);
 539	else {
 540		if (ret_work != NULL)
 541			*ret_work = NULL;
 542	}
 543
 544	return NULL;
 545}
 546
 547static struct kwork_work *find_work_by_id(struct rb_root_cached *root,
 548					  u64 id, int cpu)
 549{
 550	struct rb_node *next;
 551	struct kwork_work *work;
 552
 553	next = rb_first_cached(root);
 554	while (next) {
 555		work = rb_entry(next, struct kwork_work, node);
 556		if ((cpu != -1 && work->id == id && work->cpu == cpu) ||
 557		    (cpu == -1 && work->id == id))
 558			return work;
 559
 560		next = rb_next(next);
 561	}
 562
 563	return NULL;
 564}
 565
 566static struct kwork_class *get_kwork_class(struct perf_kwork *kwork,
 567					   enum kwork_class_type type)
 568{
 569	struct kwork_class *class;
 570
 571	list_for_each_entry(class, &kwork->class_list, list) {
 572		if (class->type == type)
 573			return class;
 574	}
 575
 576	return NULL;
 577}
 578
 579static void report_update_exit_event(struct kwork_work *work,
 580				     struct kwork_atom *atom,
 581				     struct perf_sample *sample)
 582{
 583	u64 delta;
 584	u64 exit_time = sample->time;
 585	u64 entry_time = atom->time;
 586
 587	if ((entry_time != 0) && (exit_time >= entry_time)) {
 588		delta = exit_time - entry_time;
 589		if ((delta > work->max_runtime) ||
 590		    (work->max_runtime == 0)) {
 591			work->max_runtime = delta;
 592			work->max_runtime_start = entry_time;
 593			work->max_runtime_end = exit_time;
 594		}
 595		work->total_runtime += delta;
 596		work->nr_atoms++;
 597	}
 598}
 599
 600static int report_entry_event(struct perf_kwork *kwork,
 601			      struct kwork_class *class,
 602			      struct evsel *evsel,
 603			      struct perf_sample *sample,
 604			      struct machine *machine)
 605{
 606	return work_push_atom(kwork, class, KWORK_TRACE_ENTRY,
 607			      KWORK_TRACE_MAX, evsel, sample,
 608			      machine, NULL, true);
 609}
 610
 611static int report_exit_event(struct perf_kwork *kwork,
 612			     struct kwork_class *class,
 613			     struct evsel *evsel,
 614			     struct perf_sample *sample,
 615			     struct machine *machine)
 616{
 617	struct kwork_atom *atom = NULL;
 618	struct kwork_work *work = NULL;
 619
 620	atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT,
 621			     KWORK_TRACE_ENTRY, evsel, sample,
 622			     machine, &work);
 623	if (work == NULL)
 624		return -1;
 625
 626	if (atom != NULL) {
 627		report_update_exit_event(work, atom, sample);
 628		atom_del(atom);
 629	}
 630
 631	return 0;
 632}
 633
 634static void latency_update_entry_event(struct kwork_work *work,
 635				       struct kwork_atom *atom,
 636				       struct perf_sample *sample)
 637{
 638	u64 delta;
 639	u64 entry_time = sample->time;
 640	u64 raise_time = atom->time;
 641
 642	if ((raise_time != 0) && (entry_time >= raise_time)) {
 643		delta = entry_time - raise_time;
 644		if ((delta > work->max_latency) ||
 645		    (work->max_latency == 0)) {
 646			work->max_latency = delta;
 647			work->max_latency_start = raise_time;
 648			work->max_latency_end = entry_time;
 649		}
 650		work->total_latency += delta;
 651		work->nr_atoms++;
 652	}
 653}
 654
 655static int latency_raise_event(struct perf_kwork *kwork,
 656			       struct kwork_class *class,
 657			       struct evsel *evsel,
 658			       struct perf_sample *sample,
 659			       struct machine *machine)
 660{
 661	return work_push_atom(kwork, class, KWORK_TRACE_RAISE,
 662			      KWORK_TRACE_MAX, evsel, sample,
 663			      machine, NULL, true);
 664}
 665
 666static int latency_entry_event(struct perf_kwork *kwork,
 667			       struct kwork_class *class,
 668			       struct evsel *evsel,
 669			       struct perf_sample *sample,
 670			       struct machine *machine)
 671{
 672	struct kwork_atom *atom = NULL;
 673	struct kwork_work *work = NULL;
 674
 675	atom = work_pop_atom(kwork, class, KWORK_TRACE_ENTRY,
 676			     KWORK_TRACE_RAISE, evsel, sample,
 677			     machine, &work);
 678	if (work == NULL)
 679		return -1;
 680
 681	if (atom != NULL) {
 682		latency_update_entry_event(work, atom, sample);
 683		atom_del(atom);
 684	}
 685
 686	return 0;
 687}
 688
 689static void timehist_save_callchain(struct perf_kwork *kwork,
 690				    struct perf_sample *sample,
 691				    struct evsel *evsel,
 692				    struct machine *machine)
 693{
 694	struct symbol *sym;
 695	struct thread *thread;
 696	struct callchain_cursor_node *node;
 697	struct callchain_cursor *cursor;
 698
 699	if (!kwork->show_callchain || sample->callchain == NULL)
 700		return;
 701
 702	/* want main thread for process - has maps */
 703	thread = machine__findnew_thread(machine, sample->pid, sample->pid);
 704	if (thread == NULL) {
 705		pr_debug("Failed to get thread for pid %d\n", sample->pid);
 706		return;
 707	}
 708
 709	cursor = get_tls_callchain_cursor();
 710
 711	if (thread__resolve_callchain(thread, cursor, evsel, sample,
 712				      NULL, NULL, kwork->max_stack + 2) != 0) {
 713		pr_debug("Failed to resolve callchain, skipping\n");
 714		goto out_put;
 715	}
 716
 717	callchain_cursor_commit(cursor);
 718
 719	while (true) {
 720		node = callchain_cursor_current(cursor);
 721		if (node == NULL)
 722			break;
 723
 724		sym = node->ms.sym;
 725		if (sym) {
 726			if (!strcmp(sym->name, "__softirqentry_text_start") ||
 727			    !strcmp(sym->name, "__do_softirq"))
 728				sym->ignore = 1;
 729		}
 730
 731		callchain_cursor_advance(cursor);
 732	}
 733
 734out_put:
 735	thread__put(thread);
 736}
 737
 738static void timehist_print_event(struct perf_kwork *kwork,
 739				 struct kwork_work *work,
 740				 struct kwork_atom *atom,
 741				 struct perf_sample *sample,
 742				 struct addr_location *al)
 743{
 744	char entrytime[32], exittime[32];
 745	char kwork_name[PRINT_KWORK_NAME_WIDTH];
 746
 747	/*
 748	 * runtime start
 749	 */
 750	timestamp__scnprintf_usec(atom->time,
 751				  entrytime, sizeof(entrytime));
 752	printf(" %*s ", PRINT_TIMESTAMP_WIDTH, entrytime);
 753
 754	/*
 755	 * runtime end
 756	 */
 757	timestamp__scnprintf_usec(sample->time,
 758				  exittime, sizeof(exittime));
 759	printf(" %*s ", PRINT_TIMESTAMP_WIDTH, exittime);
 760
 761	/*
 762	 * cpu
 763	 */
 764	printf(" [%0*d] ", PRINT_CPU_WIDTH, work->cpu);
 765
 766	/*
 767	 * kwork name
 768	 */
 769	if (work->class && work->class->work_name) {
 770		work->class->work_name(work, kwork_name,
 771				       PRINT_KWORK_NAME_WIDTH);
 772		printf(" %-*s ", PRINT_KWORK_NAME_WIDTH, kwork_name);
 773	} else
 774		printf(" %-*s ", PRINT_KWORK_NAME_WIDTH, "");
 775
 776	/*
 777	 *runtime
 778	 */
 779	printf(" %*.*f ",
 780	       PRINT_RUNTIME_WIDTH, RPINT_DECIMAL_WIDTH,
 781	       (double)(sample->time - atom->time) / NSEC_PER_MSEC);
 782
 783	/*
 784	 * delaytime
 785	 */
 786	if (atom->prev != NULL)
 787		printf(" %*.*f ", PRINT_LATENCY_WIDTH, RPINT_DECIMAL_WIDTH,
 788		       (double)(atom->time - atom->prev->time) / NSEC_PER_MSEC);
 789	else
 790		printf(" %*s ", PRINT_LATENCY_WIDTH, " ");
 791
 792	/*
 793	 * callchain
 794	 */
 795	if (kwork->show_callchain) {
 796		struct callchain_cursor *cursor = get_tls_callchain_cursor();
 797
 798		if (cursor == NULL)
 799			return;
 800
 801		printf(" ");
 802
 803		sample__fprintf_sym(sample, al, 0,
 804				    EVSEL__PRINT_SYM | EVSEL__PRINT_ONELINE |
 805				    EVSEL__PRINT_CALLCHAIN_ARROW |
 806				    EVSEL__PRINT_SKIP_IGNORED,
 807				    cursor, symbol_conf.bt_stop_list,
 808				    stdout);
 809	}
 810
 811	printf("\n");
 812}
 813
 814static int timehist_raise_event(struct perf_kwork *kwork,
 815				struct kwork_class *class,
 816				struct evsel *evsel,
 817				struct perf_sample *sample,
 818				struct machine *machine)
 819{
 820	return work_push_atom(kwork, class, KWORK_TRACE_RAISE,
 821			      KWORK_TRACE_MAX, evsel, sample,
 822			      machine, NULL, true);
 823}
 824
 825static int timehist_entry_event(struct perf_kwork *kwork,
 826				struct kwork_class *class,
 827				struct evsel *evsel,
 828				struct perf_sample *sample,
 829				struct machine *machine)
 830{
 831	int ret;
 832	struct kwork_work *work = NULL;
 833
 834	ret = work_push_atom(kwork, class, KWORK_TRACE_ENTRY,
 835			     KWORK_TRACE_RAISE, evsel, sample,
 836			     machine, &work, true);
 837	if (ret)
 838		return ret;
 839
 840	if (work != NULL)
 841		timehist_save_callchain(kwork, sample, evsel, machine);
 842
 843	return 0;
 844}
 845
 846static int timehist_exit_event(struct perf_kwork *kwork,
 847			       struct kwork_class *class,
 848			       struct evsel *evsel,
 849			       struct perf_sample *sample,
 850			       struct machine *machine)
 851{
 852	struct kwork_atom *atom = NULL;
 853	struct kwork_work *work = NULL;
 854	struct addr_location al;
 855	int ret = 0;
 856
 857	addr_location__init(&al);
 858	if (machine__resolve(machine, &al, sample) < 0) {
 859		pr_debug("Problem processing event, skipping it\n");
 860		ret = -1;
 861		goto out;
 862	}
 863
 864	atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT,
 865			     KWORK_TRACE_ENTRY, evsel, sample,
 866			     machine, &work);
 867	if (work == NULL) {
 868		ret = -1;
 869		goto out;
 870	}
 871
 872	if (atom != NULL) {
 873		work->nr_atoms++;
 874		timehist_print_event(kwork, work, atom, sample, &al);
 875		atom_del(atom);
 876	}
 877
 878out:
 879	addr_location__exit(&al);
 880	return ret;
 881}
 882
 883static void top_update_runtime(struct kwork_work *work,
 884			       struct kwork_atom *atom,
 885			       struct perf_sample *sample)
 886{
 887	u64 delta;
 888	u64 exit_time = sample->time;
 889	u64 entry_time = atom->time;
 890
 891	if ((entry_time != 0) && (exit_time >= entry_time)) {
 892		delta = exit_time - entry_time;
 893		work->total_runtime += delta;
 894	}
 895}
 896
 897static int top_entry_event(struct perf_kwork *kwork,
 898			   struct kwork_class *class,
 899			   struct evsel *evsel,
 900			   struct perf_sample *sample,
 901			   struct machine *machine)
 902{
 903	return work_push_atom(kwork, class, KWORK_TRACE_ENTRY,
 904			      KWORK_TRACE_MAX, evsel, sample,
 905			      machine, NULL, true);
 906}
 907
 908static int top_exit_event(struct perf_kwork *kwork,
 909			  struct kwork_class *class,
 910			  struct evsel *evsel,
 911			  struct perf_sample *sample,
 912			  struct machine *machine)
 913{
 914	struct kwork_work *work, *sched_work;
 915	struct kwork_class *sched_class;
 916	struct kwork_atom *atom;
 917
 918	atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT,
 919			     KWORK_TRACE_ENTRY, evsel, sample,
 920			     machine, &work);
 921	if (!work)
 922		return -1;
 923
 924	if (atom) {
 925		sched_class = get_kwork_class(kwork, KWORK_CLASS_SCHED);
 926		if (sched_class) {
 927			sched_work = find_work_by_id(&sched_class->work_root,
 928						     work->id, work->cpu);
 929			if (sched_work)
 930				top_update_runtime(work, atom, sample);
 931		}
 932		atom_del(atom);
 933	}
 934
 935	return 0;
 936}
 937
 938static int top_sched_switch_event(struct perf_kwork *kwork,
 939				  struct kwork_class *class,
 940				  struct evsel *evsel,
 941				  struct perf_sample *sample,
 942				  struct machine *machine)
 943{
 944	struct kwork_atom *atom;
 945	struct kwork_work *work;
 946
 947	atom = work_pop_atom(kwork, class, KWORK_TRACE_EXIT,
 948			     KWORK_TRACE_ENTRY, evsel, sample,
 949			     machine, &work);
 950	if (!work)
 951		return -1;
 952
 953	if (atom) {
 954		top_update_runtime(work, atom, sample);
 955		atom_del(atom);
 956	}
 957
 958	return top_entry_event(kwork, class, evsel, sample, machine);
 959}
 960
 961static struct kwork_class kwork_irq;
 962static int process_irq_handler_entry_event(const struct perf_tool *tool,
 963					   struct evsel *evsel,
 964					   struct perf_sample *sample,
 965					   struct machine *machine)
 966{
 967	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
 968
 969	if (kwork->tp_handler->entry_event)
 970		return kwork->tp_handler->entry_event(kwork, &kwork_irq,
 971						      evsel, sample, machine);
 972	return 0;
 973}
 974
 975static int process_irq_handler_exit_event(const struct perf_tool *tool,
 976					  struct evsel *evsel,
 977					  struct perf_sample *sample,
 978					  struct machine *machine)
 979{
 980	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
 981
 982	if (kwork->tp_handler->exit_event)
 983		return kwork->tp_handler->exit_event(kwork, &kwork_irq,
 984						     evsel, sample, machine);
 985	return 0;
 986}
 987
 988const struct evsel_str_handler irq_tp_handlers[] = {
 989	{ "irq:irq_handler_entry", process_irq_handler_entry_event, },
 990	{ "irq:irq_handler_exit",  process_irq_handler_exit_event,  },
 991};
 992
 993static int irq_class_init(struct kwork_class *class,
 994			  struct perf_session *session)
 995{
 996	if (perf_session__set_tracepoints_handlers(session, irq_tp_handlers)) {
 997		pr_err("Failed to set irq tracepoints handlers\n");
 998		return -1;
 999	}
1000
1001	class->work_root = RB_ROOT_CACHED;
1002	return 0;
1003}
1004
1005static void irq_work_init(struct perf_kwork *kwork,
1006			  struct kwork_class *class,
1007			  struct kwork_work *work,
1008			  enum kwork_trace_type src_type __maybe_unused,
1009			  struct evsel *evsel,
1010			  struct perf_sample *sample,
1011			  struct machine *machine __maybe_unused)
1012{
1013	work->class = class;
1014	work->cpu = sample->cpu;
1015
1016	if (kwork->report == KWORK_REPORT_TOP) {
1017		work->id = evsel__intval_common(evsel, sample, "common_pid");
1018		work->name = NULL;
1019	} else {
1020		work->id = evsel__intval(evsel, sample, "irq");
1021		work->name = evsel__strval(evsel, sample, "name");
1022	}
1023}
1024
1025static void irq_work_name(struct kwork_work *work, char *buf, int len)
1026{
1027	snprintf(buf, len, "%s:%" PRIu64 "", work->name, work->id);
1028}
1029
1030static struct kwork_class kwork_irq = {
1031	.name           = "irq",
1032	.type           = KWORK_CLASS_IRQ,
1033	.nr_tracepoints = 2,
1034	.tp_handlers    = irq_tp_handlers,
1035	.class_init     = irq_class_init,
1036	.work_init      = irq_work_init,
1037	.work_name      = irq_work_name,
1038};
1039
1040static struct kwork_class kwork_softirq;
1041static int process_softirq_raise_event(const struct perf_tool *tool,
1042				       struct evsel *evsel,
1043				       struct perf_sample *sample,
1044				       struct machine *machine)
1045{
1046	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
1047
1048	if (kwork->tp_handler->raise_event)
1049		return kwork->tp_handler->raise_event(kwork, &kwork_softirq,
1050						      evsel, sample, machine);
1051
1052	return 0;
1053}
1054
1055static int process_softirq_entry_event(const struct perf_tool *tool,
1056				       struct evsel *evsel,
1057				       struct perf_sample *sample,
1058				       struct machine *machine)
1059{
1060	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
1061
1062	if (kwork->tp_handler->entry_event)
1063		return kwork->tp_handler->entry_event(kwork, &kwork_softirq,
1064						      evsel, sample, machine);
1065
1066	return 0;
1067}
1068
1069static int process_softirq_exit_event(const struct perf_tool *tool,
1070				      struct evsel *evsel,
1071				      struct perf_sample *sample,
1072				      struct machine *machine)
1073{
1074	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
1075
1076	if (kwork->tp_handler->exit_event)
1077		return kwork->tp_handler->exit_event(kwork, &kwork_softirq,
1078						     evsel, sample, machine);
1079
1080	return 0;
1081}
1082
1083const struct evsel_str_handler softirq_tp_handlers[] = {
1084	{ "irq:softirq_raise", process_softirq_raise_event, },
1085	{ "irq:softirq_entry", process_softirq_entry_event, },
1086	{ "irq:softirq_exit",  process_softirq_exit_event,  },
1087};
1088
1089static int softirq_class_init(struct kwork_class *class,
1090			      struct perf_session *session)
1091{
1092	if (perf_session__set_tracepoints_handlers(session,
1093						   softirq_tp_handlers)) {
1094		pr_err("Failed to set softirq tracepoints handlers\n");
1095		return -1;
1096	}
1097
1098	class->work_root = RB_ROOT_CACHED;
1099	return 0;
1100}
1101
1102static char *evsel__softirq_name(struct evsel *evsel, u64 num)
1103{
1104	char *name = NULL;
1105	bool found = false;
1106	struct tep_print_flag_sym *sym = NULL;
1107	const struct tep_event *tp_format = evsel__tp_format(evsel);
1108	struct tep_print_arg *args = tp_format ? tp_format->print_fmt.args : NULL;
1109
1110	if ((args == NULL) || (args->next == NULL))
1111		return NULL;
1112
1113	/* skip softirq field: "REC->vec" */
1114	for (sym = args->next->symbol.symbols; sym != NULL; sym = sym->next) {
1115		if ((eval_flag(sym->value) == (unsigned long long)num) &&
1116		    (strlen(sym->str) != 0)) {
1117			found = true;
1118			break;
1119		}
1120	}
1121
1122	if (!found)
1123		return NULL;
1124
1125	name = strdup(sym->str);
1126	if (name == NULL) {
1127		pr_err("Failed to copy symbol name\n");
1128		return NULL;
1129	}
1130	return name;
1131}
1132
1133static void softirq_work_init(struct perf_kwork *kwork,
1134			      struct kwork_class *class,
1135			      struct kwork_work *work,
1136			      enum kwork_trace_type src_type __maybe_unused,
1137			      struct evsel *evsel,
1138			      struct perf_sample *sample,
1139			      struct machine *machine __maybe_unused)
1140{
1141	u64 num;
1142
1143	work->class = class;
1144	work->cpu = sample->cpu;
1145
1146	if (kwork->report == KWORK_REPORT_TOP) {
1147		work->id = evsel__intval_common(evsel, sample, "common_pid");
1148		work->name = NULL;
1149	} else {
1150		num = evsel__intval(evsel, sample, "vec");
1151		work->id = num;
1152		work->name = evsel__softirq_name(evsel, num);
1153	}
1154}
1155
1156static void softirq_work_name(struct kwork_work *work, char *buf, int len)
1157{
1158	snprintf(buf, len, "(s)%s:%" PRIu64 "", work->name, work->id);
1159}
1160
1161static struct kwork_class kwork_softirq = {
1162	.name           = "softirq",
1163	.type           = KWORK_CLASS_SOFTIRQ,
1164	.nr_tracepoints = 3,
1165	.tp_handlers    = softirq_tp_handlers,
1166	.class_init     = softirq_class_init,
1167	.work_init      = softirq_work_init,
1168	.work_name      = softirq_work_name,
1169};
1170
1171static struct kwork_class kwork_workqueue;
1172static int process_workqueue_activate_work_event(const struct perf_tool *tool,
1173						 struct evsel *evsel,
1174						 struct perf_sample *sample,
1175						 struct machine *machine)
1176{
1177	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
1178
1179	if (kwork->tp_handler->raise_event)
1180		return kwork->tp_handler->raise_event(kwork, &kwork_workqueue,
1181						    evsel, sample, machine);
1182
1183	return 0;
1184}
1185
1186static int process_workqueue_execute_start_event(const struct perf_tool *tool,
1187						 struct evsel *evsel,
1188						 struct perf_sample *sample,
1189						 struct machine *machine)
1190{
1191	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
1192
1193	if (kwork->tp_handler->entry_event)
1194		return kwork->tp_handler->entry_event(kwork, &kwork_workqueue,
1195						    evsel, sample, machine);
1196
1197	return 0;
1198}
1199
1200static int process_workqueue_execute_end_event(const struct perf_tool *tool,
1201					       struct evsel *evsel,
1202					       struct perf_sample *sample,
1203					       struct machine *machine)
1204{
1205	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
1206
1207	if (kwork->tp_handler->exit_event)
1208		return kwork->tp_handler->exit_event(kwork, &kwork_workqueue,
1209						   evsel, sample, machine);
1210
1211	return 0;
1212}
1213
1214const struct evsel_str_handler workqueue_tp_handlers[] = {
1215	{ "workqueue:workqueue_activate_work", process_workqueue_activate_work_event, },
1216	{ "workqueue:workqueue_execute_start", process_workqueue_execute_start_event, },
1217	{ "workqueue:workqueue_execute_end",   process_workqueue_execute_end_event,   },
1218};
1219
1220static int workqueue_class_init(struct kwork_class *class,
1221				struct perf_session *session)
1222{
1223	if (perf_session__set_tracepoints_handlers(session,
1224						   workqueue_tp_handlers)) {
1225		pr_err("Failed to set workqueue tracepoints handlers\n");
1226		return -1;
1227	}
1228
1229	class->work_root = RB_ROOT_CACHED;
1230	return 0;
1231}
1232
1233static void workqueue_work_init(struct perf_kwork *kwork __maybe_unused,
1234				struct kwork_class *class,
1235				struct kwork_work *work,
1236				enum kwork_trace_type src_type __maybe_unused,
1237				struct evsel *evsel,
1238				struct perf_sample *sample,
1239				struct machine *machine)
1240{
1241	char *modp = NULL;
1242	unsigned long long function_addr = evsel__intval(evsel,
1243							 sample, "function");
1244
1245	work->class = class;
1246	work->cpu = sample->cpu;
1247	work->id = evsel__intval(evsel, sample, "work");
1248	work->name = function_addr == 0 ? NULL :
1249		machine__resolve_kernel_addr(machine, &function_addr, &modp);
1250}
1251
1252static void workqueue_work_name(struct kwork_work *work, char *buf, int len)
1253{
1254	if (work->name != NULL)
1255		snprintf(buf, len, "(w)%s", work->name);
1256	else
1257		snprintf(buf, len, "(w)0x%" PRIx64, work->id);
1258}
1259
1260static struct kwork_class kwork_workqueue = {
1261	.name           = "workqueue",
1262	.type           = KWORK_CLASS_WORKQUEUE,
1263	.nr_tracepoints = 3,
1264	.tp_handlers    = workqueue_tp_handlers,
1265	.class_init     = workqueue_class_init,
1266	.work_init      = workqueue_work_init,
1267	.work_name      = workqueue_work_name,
1268};
1269
1270static struct kwork_class kwork_sched;
1271static int process_sched_switch_event(const struct perf_tool *tool,
1272				      struct evsel *evsel,
1273				      struct perf_sample *sample,
1274				      struct machine *machine)
1275{
1276	struct perf_kwork *kwork = container_of(tool, struct perf_kwork, tool);
1277
1278	if (kwork->tp_handler->sched_switch_event)
1279		return kwork->tp_handler->sched_switch_event(kwork, &kwork_sched,
1280							     evsel, sample, machine);
1281	return 0;
1282}
1283
1284const struct evsel_str_handler sched_tp_handlers[] = {
1285	{ "sched:sched_switch",  process_sched_switch_event, },
1286};
1287
1288static int sched_class_init(struct kwork_class *class,
1289			    struct perf_session *session)
1290{
1291	if (perf_session__set_tracepoints_handlers(session,
1292						   sched_tp_handlers)) {
1293		pr_err("Failed to set sched tracepoints handlers\n");
1294		return -1;
1295	}
1296
1297	class->work_root = RB_ROOT_CACHED;
1298	return 0;
1299}
1300
1301static void sched_work_init(struct perf_kwork *kwork __maybe_unused,
1302			    struct kwork_class *class,
1303			    struct kwork_work *work,
1304			    enum kwork_trace_type src_type,
1305			    struct evsel *evsel,
1306			    struct perf_sample *sample,
1307			    struct machine *machine __maybe_unused)
1308{
1309	work->class = class;
1310	work->cpu = sample->cpu;
1311
1312	if (src_type == KWORK_TRACE_EXIT) {
1313		work->id = evsel__intval(evsel, sample, "prev_pid");
1314		work->name = strdup(evsel__strval(evsel, sample, "prev_comm"));
1315	} else if (src_type == KWORK_TRACE_ENTRY) {
1316		work->id = evsel__intval(evsel, sample, "next_pid");
1317		work->name = strdup(evsel__strval(evsel, sample, "next_comm"));
1318	}
1319}
1320
1321static void sched_work_name(struct kwork_work *work, char *buf, int len)
1322{
1323	snprintf(buf, len, "%s", work->name);
1324}
1325
1326static struct kwork_class kwork_sched = {
1327	.name		= "sched",
1328	.type		= KWORK_CLASS_SCHED,
1329	.nr_tracepoints	= ARRAY_SIZE(sched_tp_handlers),
1330	.tp_handlers	= sched_tp_handlers,
1331	.class_init	= sched_class_init,
1332	.work_init	= sched_work_init,
1333	.work_name	= sched_work_name,
1334};
1335
1336static struct kwork_class *kwork_class_supported_list[KWORK_CLASS_MAX] = {
1337	[KWORK_CLASS_IRQ]       = &kwork_irq,
1338	[KWORK_CLASS_SOFTIRQ]   = &kwork_softirq,
1339	[KWORK_CLASS_WORKQUEUE] = &kwork_workqueue,
1340	[KWORK_CLASS_SCHED]     = &kwork_sched,
1341};
1342
1343static void print_separator(int len)
1344{
1345	printf(" %.*s\n", len, graph_dotted_line);
1346}
1347
1348static int report_print_work(struct perf_kwork *kwork, struct kwork_work *work)
1349{
1350	int ret = 0;
1351	char kwork_name[PRINT_KWORK_NAME_WIDTH];
1352	char max_runtime_start[32], max_runtime_end[32];
1353	char max_latency_start[32], max_latency_end[32];
1354
1355	printf(" ");
1356
1357	/*
1358	 * kwork name
1359	 */
1360	if (work->class && work->class->work_name) {
1361		work->class->work_name(work, kwork_name,
1362				       PRINT_KWORK_NAME_WIDTH);
1363		ret += printf(" %-*s |", PRINT_KWORK_NAME_WIDTH, kwork_name);
1364	} else {
1365		ret += printf(" %-*s |", PRINT_KWORK_NAME_WIDTH, "");
1366	}
1367
1368	/*
1369	 * cpu
1370	 */
1371	ret += printf(" %0*d |", PRINT_CPU_WIDTH, work->cpu);
1372
1373	/*
1374	 * total runtime
1375	 */
1376	if (kwork->report == KWORK_REPORT_RUNTIME) {
1377		ret += printf(" %*.*f ms |",
1378			      PRINT_RUNTIME_WIDTH, RPINT_DECIMAL_WIDTH,
1379			      (double)work->total_runtime / NSEC_PER_MSEC);
1380	} else if (kwork->report == KWORK_REPORT_LATENCY) { // avg delay
1381		ret += printf(" %*.*f ms |",
1382			      PRINT_LATENCY_WIDTH, RPINT_DECIMAL_WIDTH,
1383			      (double)work->total_latency /
1384			      work->nr_atoms / NSEC_PER_MSEC);
1385	}
1386
1387	/*
1388	 * count
1389	 */
1390	ret += printf(" %*" PRIu64 " |", PRINT_COUNT_WIDTH, work->nr_atoms);
1391
1392	/*
1393	 * max runtime, max runtime start, max runtime end
1394	 */
1395	if (kwork->report == KWORK_REPORT_RUNTIME) {
1396		timestamp__scnprintf_usec(work->max_runtime_start,
1397					  max_runtime_start,
1398					  sizeof(max_runtime_start));
1399		timestamp__scnprintf_usec(work->max_runtime_end,
1400					  max_runtime_end,
1401					  sizeof(max_runtime_end));
1402		ret += printf(" %*.*f ms | %*s s | %*s s |",
1403			      PRINT_RUNTIME_WIDTH, RPINT_DECIMAL_WIDTH,
1404			      (double)work->max_runtime / NSEC_PER_MSEC,
1405			      PRINT_TIMESTAMP_WIDTH, max_runtime_start,
1406			      PRINT_TIMESTAMP_WIDTH, max_runtime_end);
1407	}
1408	/*
1409	 * max delay, max delay start, max delay end
1410	 */
1411	else if (kwork->report == KWORK_REPORT_LATENCY) {
1412		timestamp__scnprintf_usec(work->max_latency_start,
1413					  max_latency_start,
1414					  sizeof(max_latency_start));
1415		timestamp__scnprintf_usec(work->max_latency_end,
1416					  max_latency_end,
1417					  sizeof(max_latency_end));
1418		ret += printf(" %*.*f ms | %*s s | %*s s |",
1419			      PRINT_LATENCY_WIDTH, RPINT_DECIMAL_WIDTH,
1420			      (double)work->max_latency / NSEC_PER_MSEC,
1421			      PRINT_TIMESTAMP_WIDTH, max_latency_start,
1422			      PRINT_TIMESTAMP_WIDTH, max_latency_end);
1423	}
1424
1425	printf("\n");
1426	return ret;
1427}
1428
1429static int report_print_header(struct perf_kwork *kwork)
1430{
1431	int ret;
1432
1433	printf("\n ");
1434	ret = printf(" %-*s | %-*s |",
1435		     PRINT_KWORK_NAME_WIDTH, "Kwork Name",
1436		     PRINT_CPU_WIDTH, "Cpu");
1437
1438	if (kwork->report == KWORK_REPORT_RUNTIME) {
1439		ret += printf(" %-*s |",
1440			      PRINT_RUNTIME_HEADER_WIDTH, "Total Runtime");
1441	} else if (kwork->report == KWORK_REPORT_LATENCY) {
1442		ret += printf(" %-*s |",
1443			      PRINT_LATENCY_HEADER_WIDTH, "Avg delay");
1444	}
1445
1446	ret += printf(" %-*s |", PRINT_COUNT_WIDTH, "Count");
1447
1448	if (kwork->report == KWORK_REPORT_RUNTIME) {
1449		ret += printf(" %-*s | %-*s | %-*s |",
1450			      PRINT_RUNTIME_HEADER_WIDTH, "Max runtime",
1451			      PRINT_TIMESTAMP_HEADER_WIDTH, "Max runtime start",
1452			      PRINT_TIMESTAMP_HEADER_WIDTH, "Max runtime end");
1453	} else if (kwork->report == KWORK_REPORT_LATENCY) {
1454		ret += printf(" %-*s | %-*s | %-*s |",
1455			      PRINT_LATENCY_HEADER_WIDTH, "Max delay",
1456			      PRINT_TIMESTAMP_HEADER_WIDTH, "Max delay start",
1457			      PRINT_TIMESTAMP_HEADER_WIDTH, "Max delay end");
1458	}
1459
1460	printf("\n");
1461	print_separator(ret);
1462	return ret;
1463}
1464
1465static void timehist_print_header(void)
1466{
1467	/*
1468	 * header row
1469	 */
1470	printf(" %-*s  %-*s  %-*s  %-*s  %-*s  %-*s\n",
1471	       PRINT_TIMESTAMP_WIDTH, "Runtime start",
1472	       PRINT_TIMESTAMP_WIDTH, "Runtime end",
1473	       PRINT_TIMEHIST_CPU_WIDTH, "Cpu",
1474	       PRINT_KWORK_NAME_WIDTH, "Kwork name",
1475	       PRINT_RUNTIME_WIDTH, "Runtime",
1476	       PRINT_RUNTIME_WIDTH, "Delaytime");
1477
1478	/*
1479	 * units row
1480	 */
1481	printf(" %-*s  %-*s  %-*s  %-*s  %-*s  %-*s\n",
1482	       PRINT_TIMESTAMP_WIDTH, "",
1483	       PRINT_TIMESTAMP_WIDTH, "",
1484	       PRINT_TIMEHIST_CPU_WIDTH, "",
1485	       PRINT_KWORK_NAME_WIDTH, "(TYPE)NAME:NUM",
1486	       PRINT_RUNTIME_WIDTH, "(msec)",
1487	       PRINT_RUNTIME_WIDTH, "(msec)");
1488
1489	/*
1490	 * separator
1491	 */
1492	printf(" %.*s  %.*s  %.*s  %.*s  %.*s  %.*s\n",
1493	       PRINT_TIMESTAMP_WIDTH, graph_dotted_line,
1494	       PRINT_TIMESTAMP_WIDTH, graph_dotted_line,
1495	       PRINT_TIMEHIST_CPU_WIDTH, graph_dotted_line,
1496	       PRINT_KWORK_NAME_WIDTH, graph_dotted_line,
1497	       PRINT_RUNTIME_WIDTH, graph_dotted_line,
1498	       PRINT_RUNTIME_WIDTH, graph_dotted_line);
1499}
1500
1501static void print_summary(struct perf_kwork *kwork)
1502{
1503	u64 time = kwork->timeend - kwork->timestart;
1504
1505	printf("  Total count            : %9" PRIu64 "\n", kwork->all_count);
1506	printf("  Total runtime   (msec) : %9.3f (%.3f%% load average)\n",
1507	       (double)kwork->all_runtime / NSEC_PER_MSEC,
1508	       time == 0 ? 0 : (double)kwork->all_runtime / time);
1509	printf("  Total time span (msec) : %9.3f\n",
1510	       (double)time / NSEC_PER_MSEC);
1511}
1512
1513static unsigned long long nr_list_entry(struct list_head *head)
1514{
1515	struct list_head *pos;
1516	unsigned long long n = 0;
1517
1518	list_for_each(pos, head)
1519		n++;
1520
1521	return n;
1522}
1523
1524static void print_skipped_events(struct perf_kwork *kwork)
1525{
1526	int i;
1527	const char *const kwork_event_str[] = {
1528		[KWORK_TRACE_RAISE] = "raise",
1529		[KWORK_TRACE_ENTRY] = "entry",
1530		[KWORK_TRACE_EXIT]  = "exit",
1531	};
1532
1533	if ((kwork->nr_skipped_events[KWORK_TRACE_MAX] != 0) &&
1534	    (kwork->nr_events != 0)) {
1535		printf("  INFO: %.3f%% skipped events (%" PRIu64 " including ",
1536		       (double)kwork->nr_skipped_events[KWORK_TRACE_MAX] /
1537		       (double)kwork->nr_events * 100.0,
1538		       kwork->nr_skipped_events[KWORK_TRACE_MAX]);
1539
1540		for (i = 0; i < KWORK_TRACE_MAX; i++) {
1541			printf("%" PRIu64 " %s%s",
1542			       kwork->nr_skipped_events[i],
1543			       kwork_event_str[i],
1544			       (i == KWORK_TRACE_MAX - 1) ? ")\n" : ", ");
1545		}
1546	}
1547
1548	if (verbose > 0)
1549		printf("  INFO: use %lld atom pages\n",
1550		       nr_list_entry(&kwork->atom_page_list));
1551}
1552
1553static void print_bad_events(struct perf_kwork *kwork)
1554{
1555	if ((kwork->nr_lost_events != 0) && (kwork->nr_events != 0)) {
1556		printf("  INFO: %.3f%% lost events (%ld out of %ld, in %ld chunks)\n",
1557		       (double)kwork->nr_lost_events /
1558		       (double)kwork->nr_events * 100.0,
1559		       kwork->nr_lost_events, kwork->nr_events,
1560		       kwork->nr_lost_chunks);
1561	}
1562}
1563
1564const char *graph_load = "||||||||||||||||||||||||||||||||||||||||||||||||";
1565const char *graph_idle = "                                                ";
1566static void top_print_per_cpu_load(struct perf_kwork *kwork)
1567{
1568	int i, load_width;
1569	u64 total, load, load_ratio;
1570	struct kwork_top_stat *stat = &kwork->top_stat;
1571
1572	for (i = 0; i < MAX_NR_CPUS; i++) {
1573		total = stat->cpus_runtime[i].total;
1574		load = stat->cpus_runtime[i].load;
1575		if (test_bit(i, stat->all_cpus_bitmap) && total) {
1576			load_ratio = load * 10000 / total;
1577			load_width = PRINT_CPU_USAGE_HIST_WIDTH *
1578				load_ratio / 10000;
1579
1580			printf("%%Cpu%-*d[%.*s%.*s %*.*f%%]\n",
1581			       PRINT_CPU_WIDTH, i,
1582			       load_width, graph_load,
1583			       PRINT_CPU_USAGE_HIST_WIDTH - load_width,
1584			       graph_idle,
1585			       PRINT_CPU_USAGE_WIDTH,
1586			       PRINT_CPU_USAGE_DECIMAL_WIDTH,
1587			       (double)load_ratio / 100);
1588		}
1589	}
1590}
1591
1592static void top_print_cpu_usage(struct perf_kwork *kwork)
1593{
1594	struct kwork_top_stat *stat = &kwork->top_stat;
1595	u64 idle_time = stat->cpus_runtime[MAX_NR_CPUS].idle;
1596	u64 hardirq_time = stat->cpus_runtime[MAX_NR_CPUS].irq;
1597	u64 softirq_time = stat->cpus_runtime[MAX_NR_CPUS].softirq;
1598	int cpus_nr = bitmap_weight(stat->all_cpus_bitmap, MAX_NR_CPUS);
1599	u64 cpus_total_time = stat->cpus_runtime[MAX_NR_CPUS].total;
1600
1601	printf("Total  : %*.*f ms, %d cpus\n",
1602	       PRINT_RUNTIME_WIDTH, RPINT_DECIMAL_WIDTH,
1603	       (double)cpus_total_time / NSEC_PER_MSEC,
1604	       cpus_nr);
1605
1606	printf("%%Cpu(s): %*.*f%% id, %*.*f%% hi, %*.*f%% si\n",
1607	       PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH,
1608	       cpus_total_time ? (double)idle_time * 100 / cpus_total_time : 0,
1609
1610	       PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH,
1611	       cpus_total_time ? (double)hardirq_time * 100 / cpus_total_time : 0,
1612
1613	       PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH,
1614	       cpus_total_time ? (double)softirq_time * 100 / cpus_total_time : 0);
1615
1616	top_print_per_cpu_load(kwork);
1617}
1618
1619static void top_print_header(struct perf_kwork *kwork __maybe_unused)
1620{
1621	int ret;
1622
1623	printf("\n ");
1624	ret = printf(" %*s %s%*s%s %*s  %*s  %-*s",
1625		     PRINT_PID_WIDTH, "PID",
1626
1627		     kwork->use_bpf ? " " : "",
1628		     kwork->use_bpf ? PRINT_PID_WIDTH : 0,
1629		     kwork->use_bpf ? "SPID" : "",
1630		     kwork->use_bpf ? " " : "",
1631
1632		     PRINT_CPU_USAGE_WIDTH, "%CPU",
1633		     PRINT_RUNTIME_HEADER_WIDTH + RPINT_DECIMAL_WIDTH, "RUNTIME",
1634		     PRINT_TASK_NAME_WIDTH, "COMMAND");
1635	printf("\n ");
1636	print_separator(ret);
1637}
1638
1639static int top_print_work(struct perf_kwork *kwork __maybe_unused, struct kwork_work *work)
1640{
1641	int ret = 0;
1642
1643	printf(" ");
1644
1645	/*
1646	 * pid
1647	 */
1648	ret += printf(" %*" PRIu64 " ", PRINT_PID_WIDTH, work->id);
1649
1650	/*
1651	 * tgid
1652	 */
1653	if (kwork->use_bpf)
1654		ret += printf(" %*d ", PRINT_PID_WIDTH, work->tgid);
1655
1656	/*
1657	 * cpu usage
1658	 */
1659	ret += printf(" %*.*f ",
1660		      PRINT_CPU_USAGE_WIDTH, PRINT_CPU_USAGE_DECIMAL_WIDTH,
1661		      (double)work->cpu_usage / 100);
1662
1663	/*
1664	 * total runtime
1665	 */
1666	ret += printf(" %*.*f ms ",
1667		      PRINT_RUNTIME_WIDTH + RPINT_DECIMAL_WIDTH, RPINT_DECIMAL_WIDTH,
1668		      (double)work->total_runtime / NSEC_PER_MSEC);
1669
1670	/*
1671	 * command
1672	 */
1673	if (kwork->use_bpf)
1674		ret += printf(" %s%s%s",
1675			      work->is_kthread ? "[" : "",
1676			      work->name,
1677			      work->is_kthread ? "]" : "");
1678	else
1679		ret += printf(" %-*s", PRINT_TASK_NAME_WIDTH, work->name);
1680
1681	printf("\n");
1682	return ret;
1683}
1684
1685static void work_sort(struct perf_kwork *kwork,
1686		      struct kwork_class *class, struct rb_root_cached *root)
1687{
1688	struct rb_node *node;
1689	struct kwork_work *data;
1690
1691	pr_debug("Sorting %s ...\n", class->name);
1692	for (;;) {
1693		node = rb_first_cached(root);
1694		if (!node)
1695			break;
1696
1697		rb_erase_cached(node, root);
1698		data = rb_entry(node, struct kwork_work, node);
1699		work_insert(&kwork->sorted_work_root,
1700			       data, &kwork->sort_list);
1701	}
1702}
1703
1704static void perf_kwork__sort(struct perf_kwork *kwork)
1705{
1706	struct kwork_class *class;
1707
1708	list_for_each_entry(class, &kwork->class_list, list)
1709		work_sort(kwork, class, &class->work_root);
1710}
1711
1712static int perf_kwork__check_config(struct perf_kwork *kwork,
1713				    struct perf_session *session)
1714{
1715	int ret;
1716	struct evsel *evsel;
1717	struct kwork_class *class;
1718
1719	static struct trace_kwork_handler report_ops = {
1720		.entry_event = report_entry_event,
1721		.exit_event  = report_exit_event,
1722	};
1723	static struct trace_kwork_handler latency_ops = {
1724		.raise_event = latency_raise_event,
1725		.entry_event = latency_entry_event,
1726	};
1727	static struct trace_kwork_handler timehist_ops = {
1728		.raise_event = timehist_raise_event,
1729		.entry_event = timehist_entry_event,
1730		.exit_event  = timehist_exit_event,
1731	};
1732	static struct trace_kwork_handler top_ops = {
1733		.entry_event        = timehist_entry_event,
1734		.exit_event         = top_exit_event,
1735		.sched_switch_event = top_sched_switch_event,
1736	};
1737
1738	switch (kwork->report) {
1739	case KWORK_REPORT_RUNTIME:
1740		kwork->tp_handler = &report_ops;
1741		break;
1742	case KWORK_REPORT_LATENCY:
1743		kwork->tp_handler = &latency_ops;
1744		break;
1745	case KWORK_REPORT_TIMEHIST:
1746		kwork->tp_handler = &timehist_ops;
1747		break;
1748	case KWORK_REPORT_TOP:
1749		kwork->tp_handler = &top_ops;
1750		break;
1751	default:
1752		pr_debug("Invalid report type %d\n", kwork->report);
1753		return -1;
1754	}
1755
1756	list_for_each_entry(class, &kwork->class_list, list)
1757		if ((class->class_init != NULL) &&
1758		    (class->class_init(class, session) != 0))
1759			return -1;
1760
1761	if (kwork->cpu_list != NULL) {
1762		ret = perf_session__cpu_bitmap(session,
1763					       kwork->cpu_list,
1764					       kwork->cpu_bitmap);
1765		if (ret < 0) {
1766			pr_err("Invalid cpu bitmap\n");
1767			return -1;
1768		}
1769	}
1770
1771	if (kwork->time_str != NULL) {
1772		ret = perf_time__parse_str(&kwork->ptime, kwork->time_str);
1773		if (ret != 0) {
1774			pr_err("Invalid time span\n");
1775			return -1;
1776		}
1777	}
1778
1779	list_for_each_entry(evsel, &session->evlist->core.entries, core.node) {
1780		if (kwork->show_callchain && !evsel__has_callchain(evsel)) {
1781			pr_debug("Samples do not have callchains\n");
1782			kwork->show_callchain = 0;
1783			symbol_conf.use_callchain = 0;
1784		}
1785	}
1786
1787	return 0;
1788}
1789
1790static int perf_kwork__read_events(struct perf_kwork *kwork)
1791{
1792	int ret = -1;
1793	struct perf_session *session = NULL;
1794
1795	struct perf_data data = {
1796		.path  = input_name,
1797		.mode  = PERF_DATA_MODE_READ,
1798		.force = kwork->force,
1799	};
1800
1801	session = perf_session__new(&data, &kwork->tool);
1802	if (IS_ERR(session)) {
1803		pr_debug("Error creating perf session\n");
1804		return PTR_ERR(session);
1805	}
1806
1807	symbol__init(perf_session__env(session));
1808
1809	if (perf_kwork__check_config(kwork, session) != 0)
1810		goto out_delete;
1811
1812	if (session->tevent.pevent &&
1813	    tep_set_function_resolver(session->tevent.pevent,
1814				      machine__resolve_kernel_addr,
1815				      &session->machines.host) < 0) {
1816		pr_err("Failed to set libtraceevent function resolver\n");
1817		goto out_delete;
1818	}
1819
1820	if (kwork->report == KWORK_REPORT_TIMEHIST)
1821		timehist_print_header();
1822
1823	ret = perf_session__process_events(session);
1824	if (ret) {
1825		pr_debug("Failed to process events, error %d\n", ret);
1826		goto out_delete;
1827	}
1828
1829	kwork->nr_events      = session->evlist->stats.nr_events[0];
1830	kwork->nr_lost_events = session->evlist->stats.total_lost;
1831	kwork->nr_lost_chunks = session->evlist->stats.nr_events[PERF_RECORD_LOST];
1832
1833out_delete:
1834	perf_session__delete(session);
1835	return ret;
1836}
1837
1838static void process_skipped_events(struct perf_kwork *kwork,
1839				   struct kwork_work *work)
1840{
1841	int i;
1842	unsigned long long count;
1843
1844	for (i = 0; i < KWORK_TRACE_MAX; i++) {
1845		count = nr_list_entry(&work->atom_list[i]);
1846		kwork->nr_skipped_events[i] += count;
1847		kwork->nr_skipped_events[KWORK_TRACE_MAX] += count;
1848	}
1849}
1850
1851static struct kwork_work *perf_kwork_add_work(struct perf_kwork *kwork,
1852				       struct kwork_class *class,
1853				       struct kwork_work *key)
1854{
1855	struct kwork_work *work = NULL;
1856
1857	work = work_new(key);
1858	if (work == NULL)
1859		return NULL;
1860
1861	work_insert(&class->work_root, work, &kwork->cmp_id);
1862	return work;
1863}
1864
1865static void sig_handler(int sig)
1866{
1867	/*
1868	 * Simply capture termination signal so that
1869	 * the program can continue after pause returns
1870	 */
1871	pr_debug("Capture signal %d\n", sig);
1872}
1873
1874static int perf_kwork__report_bpf(struct perf_kwork *kwork)
1875{
1876	int ret;
1877
1878	signal(SIGINT, sig_handler);
1879	signal(SIGTERM, sig_handler);
1880
1881	ret = perf_kwork__trace_prepare_bpf(kwork);
1882	if (ret)
1883		return -1;
1884
1885	printf("Starting trace, Hit <Ctrl+C> to stop and report\n");
1886
1887	perf_kwork__trace_start();
1888
1889	/*
1890	 * a simple pause, wait here for stop signal
1891	 */
1892	pause();
1893
1894	perf_kwork__trace_finish();
1895
1896	perf_kwork__report_read_bpf(kwork);
1897
1898	perf_kwork__report_cleanup_bpf();
1899
1900	return 0;
1901}
1902
1903static int perf_kwork__report(struct perf_kwork *kwork)
1904{
1905	int ret;
1906	struct rb_node *next;
1907	struct kwork_work *work;
1908
1909	if (kwork->use_bpf)
1910		ret = perf_kwork__report_bpf(kwork);
1911	else
1912		ret = perf_kwork__read_events(kwork);
1913
1914	if (ret != 0)
1915		return -1;
1916
1917	perf_kwork__sort(kwork);
1918
1919	setup_pager();
1920
1921	ret = report_print_header(kwork);
1922	next = rb_first_cached(&kwork->sorted_work_root);
1923	while (next) {
1924		work = rb_entry(next, struct kwork_work, node);
1925		process_skipped_events(kwork, work);
1926
1927		if (work->nr_atoms != 0) {
1928			report_print_work(kwork, work);
1929			if (kwork->summary) {
1930				kwork->all_runtime += work->total_runtime;
1931				kwork->all_count += work->nr_atoms;
1932			}
1933		}
1934		next = rb_next(next);
1935	}
1936	print_separator(ret);
1937
1938	if (kwork->summary) {
1939		print_summary(kwork);
1940		print_separator(ret);
1941	}
1942
1943	print_bad_events(kwork);
1944	print_skipped_events(kwork);
1945	printf("\n");
1946
1947	return 0;
1948}
1949
1950typedef int (*tracepoint_handler)(const struct perf_tool *tool,
1951				  struct evsel *evsel,
1952				  struct perf_sample *sample,
1953				  struct machine *machine);
1954
1955static int perf_kwork__process_tracepoint_sample(const struct perf_tool *tool,
1956						 union perf_event *event __maybe_unused,
1957						 struct perf_sample *sample,
1958						 struct evsel *evsel,
1959						 struct machine *machine)
1960{
1961	int err = 0;
1962
1963	if (evsel->handler != NULL) {
1964		tracepoint_handler f = evsel->handler;
1965
1966		err = f(tool, evsel, sample, machine);
1967	}
1968
1969	return err;
1970}
1971
1972static int perf_kwork__timehist(struct perf_kwork *kwork)
1973{
1974	/*
1975	 * event handlers for timehist option
1976	 */
1977	kwork->tool.comm	 = perf_event__process_comm;
1978	kwork->tool.exit	 = perf_event__process_exit;
1979	kwork->tool.fork	 = perf_event__process_fork;
1980	kwork->tool.attr	 = perf_event__process_attr;
1981	kwork->tool.tracing_data = perf_event__process_tracing_data;
1982	kwork->tool.build_id	 = perf_event__process_build_id;
1983	kwork->tool.ordered_events = true;
1984	kwork->tool.ordering_requires_timestamps = true;
1985	symbol_conf.use_callchain = kwork->show_callchain;
1986
1987	if (symbol__validate_sym_arguments()) {
1988		pr_err("Failed to validate sym arguments\n");
1989		return -1;
1990	}
1991
1992	setup_pager();
1993
1994	return perf_kwork__read_events(kwork);
1995}
1996
1997static void top_calc_total_runtime(struct perf_kwork *kwork)
1998{
1999	struct kwork_class *class;
2000	struct kwork_work *work;
2001	struct rb_node *next;
2002	struct kwork_top_stat *stat = &kwork->top_stat;
2003
2004	class = get_kwork_class(kwork, KWORK_CLASS_SCHED);
2005	if (!class)
2006		return;
2007
2008	next = rb_first_cached(&class->work_root);
2009	while (next) {
2010		work = rb_entry(next, struct kwork_work, node);
2011		BUG_ON(work->cpu >= MAX_NR_CPUS);
2012		stat->cpus_runtime[work->cpu].total += work->total_runtime;
2013		stat->cpus_runtime[MAX_NR_CPUS].total += work->total_runtime;
2014		next = rb_next(next);
2015	}
2016}
2017
2018static void top_calc_idle_time(struct perf_kwork *kwork,
2019				struct kwork_work *work)
2020{
2021	struct kwork_top_stat *stat = &kwork->top_stat;
2022
2023	if (work->id == 0) {
2024		stat->cpus_runtime[work->cpu].idle += work->total_runtime;
2025		stat->cpus_runtime[MAX_NR_CPUS].idle += work->total_runtime;
2026	}
2027}
2028
2029static void top_calc_irq_runtime(struct perf_kwork *kwork,
2030				 enum kwork_class_type type,
2031				 struct kwork_work *work)
2032{
2033	struct kwork_top_stat *stat = &kwork->top_stat;
2034
2035	if (type == KWORK_CLASS_IRQ) {
2036		stat->cpus_runtime[work->cpu].irq += work->total_runtime;
2037		stat->cpus_runtime[MAX_NR_CPUS].irq += work->total_runtime;
2038	} else if (type == KWORK_CLASS_SOFTIRQ) {
2039		stat->cpus_runtime[work->cpu].softirq += work->total_runtime;
2040		stat->cpus_runtime[MAX_NR_CPUS].softirq += work->total_runtime;
2041	}
2042}
2043
2044static void top_subtract_irq_runtime(struct perf_kwork *kwork,
2045				     struct kwork_work *work)
2046{
2047	struct kwork_class *class;
2048	struct kwork_work *data;
2049	unsigned int i;
2050	int irq_class_list[] = {KWORK_CLASS_IRQ, KWORK_CLASS_SOFTIRQ};
2051
2052	for (i = 0; i < ARRAY_SIZE(irq_class_list); i++) {
2053		class = get_kwork_class(kwork, irq_class_list[i]);
2054		if (!class)
2055			continue;
2056
2057		data = find_work_by_id(&class->work_root,
2058				       work->id, work->cpu);
2059		if (!data)
2060			continue;
2061
2062		if (work->total_runtime > data->total_runtime) {
2063			work->total_runtime -= data->total_runtime;
2064			top_calc_irq_runtime(kwork, irq_class_list[i], data);
2065		}
2066	}
2067}
2068
2069static void top_calc_cpu_usage(struct perf_kwork *kwork)
2070{
2071	struct kwork_class *class;
2072	struct kwork_work *work;
2073	struct rb_node *next;
2074	struct kwork_top_stat *stat = &kwork->top_stat;
2075
2076	class = get_kwork_class(kwork, KWORK_CLASS_SCHED);
2077	if (!class)
2078		return;
2079
2080	next = rb_first_cached(&class->work_root);
2081	while (next) {
2082		work = rb_entry(next, struct kwork_work, node);
2083
2084		if (work->total_runtime == 0)
2085			goto next;
2086
2087		__set_bit(work->cpu, stat->all_cpus_bitmap);
2088
2089		top_subtract_irq_runtime(kwork, work);
2090
2091		work->cpu_usage = work->total_runtime * 10000 /
2092			stat->cpus_runtime[work->cpu].total;
2093
2094		top_calc_idle_time(kwork, work);
2095next:
2096		next = rb_next(next);
2097	}
2098}
2099
2100static void top_calc_load_runtime(struct perf_kwork *kwork,
2101				  struct kwork_work *work)
2102{
2103	struct kwork_top_stat *stat = &kwork->top_stat;
2104
2105	if (work->id != 0) {
2106		stat->cpus_runtime[work->cpu].load += work->total_runtime;
2107		stat->cpus_runtime[MAX_NR_CPUS].load += work->total_runtime;
2108	}
2109}
2110
2111static void top_merge_tasks(struct perf_kwork *kwork)
2112{
2113	struct kwork_work *merged_work, *data;
2114	struct kwork_class *class;
2115	struct rb_node *node;
2116	int cpu;
2117	struct rb_root_cached merged_root = RB_ROOT_CACHED;
2118
2119	class = get_kwork_class(kwork, KWORK_CLASS_SCHED);
2120	if (!class)
2121		return;
2122
2123	for (;;) {
2124		node = rb_first_cached(&class->work_root);
2125		if (!node)
2126			break;
2127
2128		rb_erase_cached(node, &class->work_root);
2129		data = rb_entry(node, struct kwork_work, node);
2130
2131		if (!profile_name_match(kwork, data))
2132			continue;
2133
2134		cpu = data->cpu;
2135		merged_work = find_work_by_id(&merged_root, data->id,
2136					      data->id == 0 ? cpu : -1);
2137		if (!merged_work) {
2138			work_insert(&merged_root, data, &kwork->cmp_id);
2139		} else {
2140			merged_work->total_runtime += data->total_runtime;
2141			merged_work->cpu_usage += data->cpu_usage;
2142		}
2143
2144		top_calc_load_runtime(kwork, data);
2145	}
2146
2147	work_sort(kwork, class, &merged_root);
2148}
2149
2150static void perf_kwork__top_report(struct perf_kwork *kwork)
2151{
2152	struct kwork_work *work;
2153	struct rb_node *next;
2154
2155	printf("\n");
2156
2157	top_print_cpu_usage(kwork);
2158	top_print_header(kwork);
2159	next = rb_first_cached(&kwork->sorted_work_root);
2160	while (next) {
2161		work = rb_entry(next, struct kwork_work, node);
2162		process_skipped_events(kwork, work);
2163
2164		if (work->total_runtime == 0)
2165			goto next;
2166
2167		top_print_work(kwork, work);
2168
2169next:
2170		next = rb_next(next);
2171	}
2172
2173	printf("\n");
2174}
2175
2176static int perf_kwork__top_bpf(struct perf_kwork *kwork)
2177{
2178	int ret;
2179
2180	signal(SIGINT, sig_handler);
2181	signal(SIGTERM, sig_handler);
2182
2183	ret = perf_kwork__top_prepare_bpf(kwork);
2184	if (ret)
2185		return -1;
2186
2187	printf("Starting trace, Hit <Ctrl+C> to stop and report\n");
2188
2189	perf_kwork__top_start();
2190
2191	/*
2192	 * a simple pause, wait here for stop signal
2193	 */
2194	pause();
2195
2196	perf_kwork__top_finish();
2197
2198	perf_kwork__top_read_bpf(kwork);
2199
2200	perf_kwork__top_cleanup_bpf();
2201
2202	return 0;
2203
2204}
2205
2206static int perf_kwork__top(struct perf_kwork *kwork)
2207{
2208	struct __top_cpus_runtime *cpus_runtime;
2209	int ret = 0;
2210
2211	cpus_runtime = zalloc(sizeof(struct __top_cpus_runtime) * (MAX_NR_CPUS + 1));
2212	if (!cpus_runtime)
2213		return -1;
2214
2215	kwork->top_stat.cpus_runtime = cpus_runtime;
2216	bitmap_zero(kwork->top_stat.all_cpus_bitmap, MAX_NR_CPUS);
2217
2218	if (kwork->use_bpf)
2219		ret = perf_kwork__top_bpf(kwork);
2220	else
2221		ret = perf_kwork__read_events(kwork);
2222
2223	if (ret)
2224		goto out;
2225
2226	top_calc_total_runtime(kwork);
2227	top_calc_cpu_usage(kwork);
2228	top_merge_tasks(kwork);
2229
2230	setup_pager();
2231
2232	perf_kwork__top_report(kwork);
2233
2234out:
2235	zfree(&kwork->top_stat.cpus_runtime);
2236	return ret;
2237}
2238
2239static void setup_event_list(struct perf_kwork *kwork,
2240			     const struct option *options,
2241			     const char * const usage_msg[])
2242{
2243	int i;
2244	struct kwork_class *class;
2245	char *tmp, *tok, *str;
2246
2247	/*
2248	 * set default events list if not specified
2249	 */
2250	if (kwork->event_list_str == NULL)
2251		kwork->event_list_str = "irq, softirq, workqueue";
2252
2253	str = strdup(kwork->event_list_str);
2254	for (tok = strtok_r(str, ", ", &tmp);
2255	     tok; tok = strtok_r(NULL, ", ", &tmp)) {
2256		for (i = 0; i < KWORK_CLASS_MAX; i++) {
2257			class = kwork_class_supported_list[i];
2258			if (strcmp(tok, class->name) == 0) {
2259				list_add_tail(&class->list, &kwork->class_list);
2260				break;
2261			}
2262		}
2263		if (i == KWORK_CLASS_MAX) {
2264			usage_with_options_msg(usage_msg, options,
2265					       "Unknown --event key: `%s'", tok);
2266		}
2267	}
2268	free(str);
2269
2270	pr_debug("Config event list:");
2271	list_for_each_entry(class, &kwork->class_list, list)
2272		pr_debug(" %s", class->name);
2273	pr_debug("\n");
2274}
2275
2276#define STRDUP_FAIL_EXIT(s)		\
2277	({	char *_p;		\
2278		_p = strdup(s);		\
2279		if (!_p) {		\
2280			ret = -ENOMEM;	\
2281			goto EXIT;	\
2282		}			\
2283		_p;			\
2284	})
2285
2286static int perf_kwork__record(struct perf_kwork *kwork,
2287			      int argc, const char **argv)
2288{
2289	const char **rec_argv;
2290	unsigned int rec_argc, i, j;
2291	struct kwork_class *class;
2292	int ret;
2293
2294	const char *const record_args[] = {
2295		"record",
2296		"-a",
2297		"-R",
2298		"-m", "1024",
2299		"-c", "1",
2300	};
2301
2302	rec_argc = ARRAY_SIZE(record_args) + argc - 1;
2303
2304	list_for_each_entry(class, &kwork->class_list, list)
2305		rec_argc += 2 * class->nr_tracepoints;
2306
2307	rec_argv = calloc(rec_argc + 1, sizeof(char *));
2308	if (rec_argv == NULL)
2309		return -ENOMEM;
2310
2311	for (i = 0; i < ARRAY_SIZE(record_args); i++)
2312		rec_argv[i] = STRDUP_FAIL_EXIT(record_args[i]);
2313
2314	list_for_each_entry(class, &kwork->class_list, list) {
2315		for (j = 0; j < class->nr_tracepoints; j++) {
2316			rec_argv[i++] = STRDUP_FAIL_EXIT("-e");
2317			rec_argv[i++] = STRDUP_FAIL_EXIT(class->tp_handlers[j].name);
2318		}
2319	}
2320
2321	for (j = 1; j < (unsigned int)argc; j++, i++)
2322		rec_argv[i] = STRDUP_FAIL_EXIT(argv[j]);
2323
2324	BUG_ON(i != rec_argc);
2325
2326	pr_debug("record comm: ");
2327	for (j = 0; j < rec_argc; j++)
2328		pr_debug("%s ", rec_argv[j]);
2329	pr_debug("\n");
2330
2331	ret = cmd_record(i, rec_argv);
2332
2333EXIT:
2334	for (i = 0; i < rec_argc; i++)
2335		free((void *)rec_argv[i]);
2336	free(rec_argv);
2337	return ret;
2338}
2339
2340int cmd_kwork(int argc, const char **argv)
2341{
2342	static struct perf_kwork kwork = {
2343		.class_list          = LIST_HEAD_INIT(kwork.class_list),
2344		.atom_page_list      = LIST_HEAD_INIT(kwork.atom_page_list),
2345		.sort_list           = LIST_HEAD_INIT(kwork.sort_list),
2346		.cmp_id              = LIST_HEAD_INIT(kwork.cmp_id),
2347		.sorted_work_root    = RB_ROOT_CACHED,
2348		.tp_handler          = NULL,
2349		.profile_name        = NULL,
2350		.cpu_list            = NULL,
2351		.time_str            = NULL,
2352		.force               = false,
2353		.event_list_str      = NULL,
2354		.summary             = false,
2355		.sort_order          = NULL,
2356		.show_callchain      = false,
2357		.max_stack           = 5,
2358		.timestart           = 0,
2359		.timeend             = 0,
2360		.nr_events           = 0,
2361		.nr_lost_chunks      = 0,
2362		.nr_lost_events      = 0,
2363		.all_runtime         = 0,
2364		.all_count           = 0,
2365		.nr_skipped_events   = { 0 },
2366		.add_work            = perf_kwork_add_work,
2367	};
2368	static const char default_report_sort_order[] = "runtime, max, count";
2369	static const char default_latency_sort_order[] = "avg, max, count";
2370	static const char default_top_sort_order[] = "rate, runtime";
2371	const struct option kwork_options[] = {
2372	OPT_INCR('v', "verbose", &verbose,
2373		 "be more verbose (show symbol address, etc)"),
2374	OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
2375		    "dump raw trace in ASCII"),
2376	OPT_STRING('k', "kwork", &kwork.event_list_str, "kwork",
2377		   "list of kwork to profile (irq, softirq, workqueue, sched, etc)"),
2378	OPT_BOOLEAN('f', "force", &kwork.force, "don't complain, do it"),
2379	OPT_END()
2380	};
2381	const struct option report_options[] = {
2382	OPT_STRING('s', "sort", &kwork.sort_order, "key[,key2...]",
2383		   "sort by key(s): runtime, max, count"),
2384	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",
2385		   "list of cpus to profile"),
2386	OPT_STRING('n', "name", &kwork.profile_name, "name",
2387		   "event name to profile"),
2388	OPT_STRING(0, "time", &kwork.time_str, "str",
2389		   "Time span for analysis (start,stop)"),
2390	OPT_STRING('i', "input", &input_name, "file",
2391		   "input file name"),
2392	OPT_BOOLEAN('S', "with-summary", &kwork.summary,
2393		    "Show summary with statistics"),
2394#ifdef HAVE_BPF_SKEL
2395	OPT_BOOLEAN('b', "use-bpf", &kwork.use_bpf,
2396		    "Use BPF to measure kwork runtime"),
2397#endif
2398	OPT_PARENT(kwork_options)
2399	};
2400	const struct option latency_options[] = {
2401	OPT_STRING('s', "sort", &kwork.sort_order, "key[,key2...]",
2402		   "sort by key(s): avg, max, count"),
2403	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",
2404		   "list of cpus to profile"),
2405	OPT_STRING('n', "name", &kwork.profile_name, "name",
2406		   "event name to profile"),
2407	OPT_STRING(0, "time", &kwork.time_str, "str",
2408		   "Time span for analysis (start,stop)"),
2409	OPT_STRING('i', "input", &input_name, "file",
2410		   "input file name"),
2411#ifdef HAVE_BPF_SKEL
2412	OPT_BOOLEAN('b', "use-bpf", &kwork.use_bpf,
2413		    "Use BPF to measure kwork latency"),
2414#endif
2415	OPT_PARENT(kwork_options)
2416	};
2417	const struct option timehist_options[] = {
2418	OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
2419		   "file", "vmlinux pathname"),
2420	OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name,
2421		   "file", "kallsyms pathname"),
2422	OPT_BOOLEAN('g', "call-graph", &kwork.show_callchain,
2423		    "Display call chains if present"),
2424	OPT_UINTEGER(0, "max-stack", &kwork.max_stack,
2425		   "Maximum number of functions to display backtrace."),
2426	OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
2427		    "Look for files with symbols relative to this directory"),
2428	OPT_STRING(0, "time", &kwork.time_str, "str",
2429		   "Time span for analysis (start,stop)"),
2430	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",
2431		   "list of cpus to profile"),
2432	OPT_STRING('n', "name", &kwork.profile_name, "name",
2433		   "event name to profile"),
2434	OPT_STRING('i', "input", &input_name, "file",
2435		   "input file name"),
2436	OPT_PARENT(kwork_options)
2437	};
2438	const struct option top_options[] = {
2439	OPT_STRING('s', "sort", &kwork.sort_order, "key[,key2...]",
2440		   "sort by key(s): rate, runtime, tid"),
2441	OPT_STRING('C', "cpu", &kwork.cpu_list, "cpu",
2442		   "list of cpus to profile"),
2443	OPT_STRING('n', "name", &kwork.profile_name, "name",
2444		   "event name to profile"),
2445	OPT_STRING(0, "time", &kwork.time_str, "str",
2446		   "Time span for analysis (start,stop)"),
2447	OPT_STRING('i', "input", &input_name, "file",
2448		   "input file name"),
2449#ifdef HAVE_BPF_SKEL
2450	OPT_BOOLEAN('b', "use-bpf", &kwork.use_bpf,
2451		    "Use BPF to measure task cpu usage"),
2452#endif
2453	OPT_PARENT(kwork_options)
2454	};
2455	const char *kwork_usage[] = {
2456		NULL,
2457		NULL
2458	};
2459	const char * const report_usage[] = {
2460		"perf kwork report [<options>]",
2461		NULL
2462	};
2463	const char * const latency_usage[] = {
2464		"perf kwork latency [<options>]",
2465		NULL
2466	};
2467	const char * const timehist_usage[] = {
2468		"perf kwork timehist [<options>]",
2469		NULL
2470	};
2471	const char * const top_usage[] = {
2472		"perf kwork top [<options>]",
2473		NULL
2474	};
2475	const char *const kwork_subcommands[] = {
2476		"record", "report", "latency", "timehist", "top", NULL
2477	};
2478
2479	perf_tool__init(&kwork.tool, /*ordered_events=*/true);
2480	kwork.tool.mmap	  = perf_event__process_mmap;
2481	kwork.tool.mmap2  = perf_event__process_mmap2;
2482	kwork.tool.sample = perf_kwork__process_tracepoint_sample;
2483
2484	argc = parse_options_subcommand(argc, argv, kwork_options,
2485					kwork_subcommands, kwork_usage,
2486					PARSE_OPT_STOP_AT_NON_OPTION);
2487	if (!argc)
2488		usage_with_options(kwork_usage, kwork_options);
2489
2490	sort_dimension__add(&kwork, "id", &kwork.cmp_id);
2491
2492	if (strlen(argv[0]) > 2 && strstarts("record", argv[0])) {
2493		setup_event_list(&kwork, kwork_options, kwork_usage);
2494		return perf_kwork__record(&kwork, argc, argv);
2495	} else if (strlen(argv[0]) > 2 && strstarts("report", argv[0])) {
2496		kwork.sort_order = default_report_sort_order;
2497		if (argc > 1) {
2498			argc = parse_options(argc, argv, report_options, report_usage, 0);
2499			if (argc)
2500				usage_with_options(report_usage, report_options);
2501		}
2502		kwork.report = KWORK_REPORT_RUNTIME;
2503		setup_sorting(&kwork, report_options, report_usage);
2504		setup_event_list(&kwork, kwork_options, kwork_usage);
2505		return perf_kwork__report(&kwork);
2506	} else if (strlen(argv[0]) > 2 && strstarts("latency", argv[0])) {
2507		kwork.sort_order = default_latency_sort_order;
2508		if (argc > 1) {
2509			argc = parse_options(argc, argv, latency_options, latency_usage, 0);
2510			if (argc)
2511				usage_with_options(latency_usage, latency_options);
2512		}
2513		kwork.report = KWORK_REPORT_LATENCY;
2514		setup_sorting(&kwork, latency_options, latency_usage);
2515		setup_event_list(&kwork, kwork_options, kwork_usage);
2516		return perf_kwork__report(&kwork);
2517	} else if (strlen(argv[0]) > 2 && strstarts("timehist", argv[0])) {
2518		if (argc > 1) {
2519			argc = parse_options(argc, argv, timehist_options, timehist_usage, 0);
2520			if (argc)
2521				usage_with_options(timehist_usage, timehist_options);
2522		}
2523		kwork.report = KWORK_REPORT_TIMEHIST;
2524		setup_event_list(&kwork, kwork_options, kwork_usage);
2525		return perf_kwork__timehist(&kwork);
2526	} else if (strlen(argv[0]) > 2 && strstarts("top", argv[0])) {
2527		kwork.sort_order = default_top_sort_order;
2528		if (argc > 1) {
2529			argc = parse_options(argc, argv, top_options, top_usage, 0);
2530			if (argc)
2531				usage_with_options(top_usage, top_options);
2532		}
2533		kwork.report = KWORK_REPORT_TOP;
2534		if (!kwork.event_list_str)
2535			kwork.event_list_str = "sched, irq, softirq";
2536		setup_event_list(&kwork, kwork_options, kwork_usage);
2537		setup_sorting(&kwork, top_options, top_usage);
2538		return perf_kwork__top(&kwork);
2539	} else
2540		usage_with_options(kwork_usage, kwork_options);
2541
2542	/* free usage string allocated by parse_options_subcommand */
2543	free((void *)kwork_usage[0]);
2544
2545	return 0;
2546}