trace_events_hist.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * trace_events_hist - trace event hist triggers
   4 *
   5 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/kallsyms.h>
  10#include <linux/security.h>
  11#include <linux/mutex.h>
  12#include <linux/slab.h>
  13#include <linux/stacktrace.h>
  14#include <linux/rculist.h>
  15#include <linux/tracefs.h>
  16
  17/* for gfp flag names */
  18#include <linux/trace_events.h>
  19#include <trace/events/mmflags.h>
  20
  21#include "tracing_map.h"
  22#include "trace_synth.h"
  23
  24#define ERRORS								\
  25	C(NONE,			"No error"),				\
  26	C(DUPLICATE_VAR,	"Variable already defined"),		\
  27	C(VAR_NOT_UNIQUE,	"Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28	C(TOO_MANY_VARS,	"Too many variables defined"),		\
  29	C(MALFORMED_ASSIGNMENT,	"Malformed assignment"),		\
  30	C(NAMED_MISMATCH,	"Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31	C(TRIGGER_EEXIST,	"Hist trigger already exists"),		\
  32	C(TRIGGER_ENOENT_CLEAR,	"Can't clear or continue a nonexistent hist trigger"), \
  33	C(SET_CLOCK_FAIL,	"Couldn't set trace_clock"),		\
  34	C(BAD_FIELD_MODIFIER,	"Invalid field modifier"),		\
  35	C(TOO_MANY_SUBEXPR,	"Too many subexpressions (3 max)"),	\
  36	C(TIMESTAMP_MISMATCH,	"Timestamp units in expression don't match"), \
  37	C(TOO_MANY_FIELD_VARS,	"Too many field variables defined"),	\
  38	C(EVENT_FILE_NOT_FOUND,	"Event file not found"),		\
  39	C(HIST_NOT_FOUND,	"Matching event histogram not found"),	\
  40	C(HIST_CREATE_FAIL,	"Couldn't create histogram for field"),	\
  41	C(SYNTH_VAR_NOT_FOUND,	"Couldn't find synthetic variable"),	\
  42	C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),	\
  43	C(SYNTH_TYPE_MISMATCH,	"Param type doesn't match synthetic event field type"), \
  44	C(SYNTH_COUNT_MISMATCH,	"Param count doesn't match synthetic event field count"), \
  45	C(FIELD_VAR_PARSE_FAIL,	"Couldn't parse field variable"),	\
  46	C(VAR_CREATE_FIND_FAIL,	"Couldn't create or find variable"),	\
  47	C(ONX_NOT_VAR,		"For onmax(x) or onchange(x), x must be a variable"), \
  48	C(ONX_VAR_NOT_FOUND,	"Couldn't find onmax or onchange variable"), \
  49	C(ONX_VAR_CREATE_FAIL,	"Couldn't create onmax or onchange variable"), \
  50	C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),	\
  51	C(TOO_MANY_PARAMS,	"Too many action params"),		\
  52	C(PARAM_NOT_FOUND,	"Couldn't find param"),			\
  53	C(INVALID_PARAM,	"Invalid action param"),		\
  54	C(ACTION_NOT_FOUND,	"No action found"),			\
  55	C(NO_SAVE_PARAMS,	"No params found for save()"),		\
  56	C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57	C(ACTION_MISMATCH,	"Handler doesn't support action"),	\
  58	C(NO_CLOSING_PAREN,	"No closing paren found"),		\
  59	C(SUBSYS_NOT_FOUND,	"Missing subsystem"),			\
  60	C(INVALID_SUBSYS_EVENT,	"Invalid subsystem or event name"),	\
  61	C(INVALID_REF_KEY,	"Using variable references in keys not supported"), \
  62	C(VAR_NOT_FOUND,	"Couldn't find variable"),		\
  63	C(FIELD_NOT_FOUND,	"Couldn't find field"),			\
  64	C(EMPTY_ASSIGNMENT,	"Empty assignment"),			\
  65	C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),		\
  66	C(EMPTY_SORT_FIELD,	"Empty sort field"),			\
  67	C(TOO_MANY_SORT_FIELDS,	"Too many sort fields (Max = 2)"),	\
  68	C(INVALID_SORT_FIELD,	"Sort field must be a key or a val"),	\
  69	C(INVALID_STR_OPERAND,	"String type can not be an operand in expression"), \
  70	C(EXPECT_NUMBER,	"Expecting numeric literal"),		\
  71	C(UNARY_MINUS_SUBEXPR,	"Unary minus not supported in sub-expressions"), \
  72	C(DIVISION_BY_ZERO,	"Division by zero"),			\
  73	C(NEED_NOHC_VAL,	"Non-hitcount value is required for 'nohitcount'"),
  74
  75#undef C
  76#define C(a, b)		HIST_ERR_##a
  77
  78enum { ERRORS };
  79
  80#undef C
  81#define C(a, b)		b
  82
  83static const char *err_text[] = { ERRORS };
  84
  85struct hist_field;
  86
  87typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  88				struct tracing_map_elt *elt,
  89				struct trace_buffer *buffer,
  90				struct ring_buffer_event *rbe,
  91				void *event);
  92
  93#define HIST_FIELD_OPERANDS_MAX	2
  94#define HIST_FIELDS_MAX		(TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95#define HIST_ACTIONS_MAX	8
  96#define HIST_CONST_DIGITS_MAX	21
  97#define HIST_DIV_SHIFT		20  /* For optimizing division by constants */
  98
  99enum field_op_id {
 100	FIELD_OP_NONE,
 101	FIELD_OP_PLUS,
 102	FIELD_OP_MINUS,
 103	FIELD_OP_UNARY_MINUS,
 104	FIELD_OP_DIV,
 105	FIELD_OP_MULT,
 106};
 107
 108enum hist_field_fn {
 109	HIST_FIELD_FN_NOP,
 110	HIST_FIELD_FN_VAR_REF,
 111	HIST_FIELD_FN_COUNTER,
 112	HIST_FIELD_FN_CONST,
 113	HIST_FIELD_FN_LOG2,
 114	HIST_FIELD_FN_BUCKET,
 115	HIST_FIELD_FN_TIMESTAMP,
 116	HIST_FIELD_FN_CPU,
 117	HIST_FIELD_FN_COMM,
 118	HIST_FIELD_FN_STRING,
 119	HIST_FIELD_FN_DYNSTRING,
 120	HIST_FIELD_FN_RELDYNSTRING,
 121	HIST_FIELD_FN_PSTRING,
 122	HIST_FIELD_FN_S64,
 123	HIST_FIELD_FN_U64,
 124	HIST_FIELD_FN_S32,
 125	HIST_FIELD_FN_U32,
 126	HIST_FIELD_FN_S16,
 127	HIST_FIELD_FN_U16,
 128	HIST_FIELD_FN_S8,
 129	HIST_FIELD_FN_U8,
 130	HIST_FIELD_FN_UMINUS,
 131	HIST_FIELD_FN_MINUS,
 132	HIST_FIELD_FN_PLUS,
 133	HIST_FIELD_FN_DIV,
 134	HIST_FIELD_FN_MULT,
 135	HIST_FIELD_FN_DIV_POWER2,
 136	HIST_FIELD_FN_DIV_NOT_POWER2,
 137	HIST_FIELD_FN_DIV_MULT_SHIFT,
 138	HIST_FIELD_FN_EXECNAME,
 139	HIST_FIELD_FN_STACK,
 140};
 141
 142/*
 143 * A hist_var (histogram variable) contains variable information for
 144 * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 145 * flag set.  A hist_var has a variable name e.g. ts0, and is
 146 * associated with a given histogram trigger, as specified by
 147 * hist_data.  The hist_var idx is the unique index assigned to the
 148 * variable by the hist trigger's tracing_map.  The idx is what is
 149 * used to set a variable's value and, by a variable reference, to
 150 * retrieve it.
 151 */
 152struct hist_var {
 153	char				*name;
 154	struct hist_trigger_data	*hist_data;
 155	unsigned int			idx;
 156};
 157
 158struct hist_field {
 159	struct ftrace_event_field	*field;
 160	unsigned long			flags;
 161	unsigned long			buckets;
 162	const char			*type;
 163	struct hist_field		*operands[HIST_FIELD_OPERANDS_MAX];
 164	struct hist_trigger_data	*hist_data;
 165	enum hist_field_fn		fn_num;
 166	unsigned int			ref;
 167	unsigned int			size;
 168	unsigned int			offset;
 169	unsigned int                    is_signed;
 170
 171	/*
 172	 * Variable fields contain variable-specific info in var.
 173	 */
 174	struct hist_var			var;
 175	enum field_op_id		operator;
 176	char				*system;
 177	char				*event_name;
 178
 179	/*
 180	 * The name field is used for EXPR and VAR_REF fields.  VAR
 181	 * fields contain the variable name in var.name.
 182	 */
 183	char				*name;
 184
 185	/*
 186	 * When a histogram trigger is hit, if it has any references
 187	 * to variables, the values of those variables are collected
 188	 * into a var_ref_vals array by resolve_var_refs().  The
 189	 * current value of each variable is read from the tracing_map
 190	 * using the hist field's hist_var.idx and entered into the
 191	 * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 192	 */
 193	unsigned int			var_ref_idx;
 194	bool                            read_once;
 195
 196	unsigned int			var_str_idx;
 197
 198	/* Numeric literals are represented as u64 */
 199	u64				constant;
 200	/* Used to optimize division by constants */
 201	u64				div_multiplier;
 202};
 203
 204static u64 hist_fn_call(struct hist_field *hist_field,
 205			struct tracing_map_elt *elt,
 206			struct trace_buffer *buffer,
 207			struct ring_buffer_event *rbe,
 208			void *event);
 209
 210static u64 hist_field_const(struct hist_field *field,
 211			   struct tracing_map_elt *elt,
 212			   struct trace_buffer *buffer,
 213			   struct ring_buffer_event *rbe,
 214			   void *event)
 215{
 216	return field->constant;
 217}
 218
 219static u64 hist_field_counter(struct hist_field *field,
 220			      struct tracing_map_elt *elt,
 221			      struct trace_buffer *buffer,
 222			      struct ring_buffer_event *rbe,
 223			      void *event)
 224{
 225	return 1;
 226}
 227
 228static u64 hist_field_string(struct hist_field *hist_field,
 229			     struct tracing_map_elt *elt,
 230			     struct trace_buffer *buffer,
 231			     struct ring_buffer_event *rbe,
 232			     void *event)
 233{
 234	char *addr = (char *)(event + hist_field->field->offset);
 235
 236	return (u64)(unsigned long)addr;
 237}
 238
 239static u64 hist_field_dynstring(struct hist_field *hist_field,
 240				struct tracing_map_elt *elt,
 241				struct trace_buffer *buffer,
 242				struct ring_buffer_event *rbe,
 243				void *event)
 244{
 245	u32 str_item = *(u32 *)(event + hist_field->field->offset);
 246	int str_loc = str_item & 0xffff;
 247	char *addr = (char *)(event + str_loc);
 248
 249	return (u64)(unsigned long)addr;
 250}
 251
 252static u64 hist_field_reldynstring(struct hist_field *hist_field,
 253				   struct tracing_map_elt *elt,
 254				   struct trace_buffer *buffer,
 255				   struct ring_buffer_event *rbe,
 256				   void *event)
 257{
 258	u32 *item = event + hist_field->field->offset;
 259	u32 str_item = *item;
 260	int str_loc = str_item & 0xffff;
 261	char *addr = (char *)&item[1] + str_loc;
 262
 263	return (u64)(unsigned long)addr;
 264}
 265
 266static u64 hist_field_pstring(struct hist_field *hist_field,
 267			      struct tracing_map_elt *elt,
 268			      struct trace_buffer *buffer,
 269			      struct ring_buffer_event *rbe,
 270			      void *event)
 271{
 272	char **addr = (char **)(event + hist_field->field->offset);
 273
 274	return (u64)(unsigned long)*addr;
 275}
 276
 277static u64 hist_field_log2(struct hist_field *hist_field,
 278			   struct tracing_map_elt *elt,
 279			   struct trace_buffer *buffer,
 280			   struct ring_buffer_event *rbe,
 281			   void *event)
 282{
 283	struct hist_field *operand = hist_field->operands[0];
 284
 285	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 286
 287	return (u64) ilog2(roundup_pow_of_two(val));
 288}
 289
 290static u64 hist_field_bucket(struct hist_field *hist_field,
 291			     struct tracing_map_elt *elt,
 292			     struct trace_buffer *buffer,
 293			     struct ring_buffer_event *rbe,
 294			     void *event)
 295{
 296	struct hist_field *operand = hist_field->operands[0];
 297	unsigned long buckets = hist_field->buckets;
 298
 299	u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 300
 301	if (WARN_ON_ONCE(!buckets))
 302		return val;
 303
 304	if (val >= LONG_MAX)
 305		val = div64_ul(val, buckets);
 306	else
 307		val = (u64)((unsigned long)val / buckets);
 308	return val * buckets;
 309}
 310
 311static u64 hist_field_plus(struct hist_field *hist_field,
 312			   struct tracing_map_elt *elt,
 313			   struct trace_buffer *buffer,
 314			   struct ring_buffer_event *rbe,
 315			   void *event)
 316{
 317	struct hist_field *operand1 = hist_field->operands[0];
 318	struct hist_field *operand2 = hist_field->operands[1];
 319
 320	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 321	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 322
 323	return val1 + val2;
 324}
 325
 326static u64 hist_field_minus(struct hist_field *hist_field,
 327			    struct tracing_map_elt *elt,
 328			    struct trace_buffer *buffer,
 329			    struct ring_buffer_event *rbe,
 330			    void *event)
 331{
 332	struct hist_field *operand1 = hist_field->operands[0];
 333	struct hist_field *operand2 = hist_field->operands[1];
 334
 335	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 336	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 337
 338	return val1 - val2;
 339}
 340
 341static u64 hist_field_div(struct hist_field *hist_field,
 342			   struct tracing_map_elt *elt,
 343			   struct trace_buffer *buffer,
 344			   struct ring_buffer_event *rbe,
 345			   void *event)
 346{
 347	struct hist_field *operand1 = hist_field->operands[0];
 348	struct hist_field *operand2 = hist_field->operands[1];
 349
 350	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 351	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 352
 353	/* Return -1 for the undefined case */
 354	if (!val2)
 355		return -1;
 356
 357	/* Use shift if the divisor is a power of 2 */
 358	if (!(val2 & (val2 - 1)))
 359		return val1 >> __ffs64(val2);
 360
 361	return div64_u64(val1, val2);
 362}
 363
 364static u64 div_by_power_of_two(struct hist_field *hist_field,
 365				struct tracing_map_elt *elt,
 366				struct trace_buffer *buffer,
 367				struct ring_buffer_event *rbe,
 368				void *event)
 369{
 370	struct hist_field *operand1 = hist_field->operands[0];
 371	struct hist_field *operand2 = hist_field->operands[1];
 372
 373	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 374
 375	return val1 >> __ffs64(operand2->constant);
 376}
 377
 378static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 379				struct tracing_map_elt *elt,
 380				struct trace_buffer *buffer,
 381				struct ring_buffer_event *rbe,
 382				void *event)
 383{
 384	struct hist_field *operand1 = hist_field->operands[0];
 385	struct hist_field *operand2 = hist_field->operands[1];
 386
 387	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 388
 389	return div64_u64(val1, operand2->constant);
 390}
 391
 392static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 393				struct tracing_map_elt *elt,
 394				struct trace_buffer *buffer,
 395				struct ring_buffer_event *rbe,
 396				void *event)
 397{
 398	struct hist_field *operand1 = hist_field->operands[0];
 399	struct hist_field *operand2 = hist_field->operands[1];
 400
 401	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 402
 403	/*
 404	 * If the divisor is a constant, do a multiplication and shift instead.
 405	 *
 406	 * Choose Z = some power of 2. If Y <= Z, then:
 407	 *     X / Y = (X * (Z / Y)) / Z
 408	 *
 409	 * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 410	 *     X / Y = (X * mult) / Z
 411	 *
 412	 * The division by Z can be replaced by a shift since Z is a power of 2:
 413	 *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 414	 *
 415	 * As long, as X < Z the results will not be off by more than 1.
 416	 */
 417	if (val1 < (1 << HIST_DIV_SHIFT)) {
 418		u64 mult = operand2->div_multiplier;
 419
 420		return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 421	}
 422
 423	return div64_u64(val1, operand2->constant);
 424}
 425
 426static u64 hist_field_mult(struct hist_field *hist_field,
 427			   struct tracing_map_elt *elt,
 428			   struct trace_buffer *buffer,
 429			   struct ring_buffer_event *rbe,
 430			   void *event)
 431{
 432	struct hist_field *operand1 = hist_field->operands[0];
 433	struct hist_field *operand2 = hist_field->operands[1];
 434
 435	u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 436	u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 437
 438	return val1 * val2;
 439}
 440
 441static u64 hist_field_unary_minus(struct hist_field *hist_field,
 442				  struct tracing_map_elt *elt,
 443				  struct trace_buffer *buffer,
 444				  struct ring_buffer_event *rbe,
 445				  void *event)
 446{
 447	struct hist_field *operand = hist_field->operands[0];
 448
 449	s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 450	u64 val = (u64)-sval;
 451
 452	return val;
 453}
 454
 455#define DEFINE_HIST_FIELD_FN(type)					\
 456	static u64 hist_field_##type(struct hist_field *hist_field,	\
 457				     struct tracing_map_elt *elt,	\
 458				     struct trace_buffer *buffer,	\
 459				     struct ring_buffer_event *rbe,	\
 460				     void *event)			\
 461{									\
 462	type *addr = (type *)(event + hist_field->field->offset);	\
 463									\
 464	return (u64)(unsigned long)*addr;				\
 465}
 466
 467DEFINE_HIST_FIELD_FN(s64);
 468DEFINE_HIST_FIELD_FN(u64);
 469DEFINE_HIST_FIELD_FN(s32);
 470DEFINE_HIST_FIELD_FN(u32);
 471DEFINE_HIST_FIELD_FN(s16);
 472DEFINE_HIST_FIELD_FN(u16);
 473DEFINE_HIST_FIELD_FN(s8);
 474DEFINE_HIST_FIELD_FN(u8);
 475
 476#define for_each_hist_field(i, hist_data)	\
 477	for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 478
 479#define for_each_hist_val_field(i, hist_data)	\
 480	for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 481
 482#define for_each_hist_key_field(i, hist_data)	\
 483	for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 484
 485#define HITCOUNT_IDX		0
 486#define HIST_KEY_SIZE_MAX	(MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 487
 488enum hist_field_flags {
 489	HIST_FIELD_FL_HITCOUNT		= 1 << 0,
 490	HIST_FIELD_FL_KEY		= 1 << 1,
 491	HIST_FIELD_FL_STRING		= 1 << 2,
 492	HIST_FIELD_FL_HEX		= 1 << 3,
 493	HIST_FIELD_FL_SYM		= 1 << 4,
 494	HIST_FIELD_FL_SYM_OFFSET	= 1 << 5,
 495	HIST_FIELD_FL_EXECNAME		= 1 << 6,
 496	HIST_FIELD_FL_SYSCALL		= 1 << 7,
 497	HIST_FIELD_FL_STACKTRACE	= 1 << 8,
 498	HIST_FIELD_FL_LOG2		= 1 << 9,
 499	HIST_FIELD_FL_TIMESTAMP		= 1 << 10,
 500	HIST_FIELD_FL_TIMESTAMP_USECS	= 1 << 11,
 501	HIST_FIELD_FL_VAR		= 1 << 12,
 502	HIST_FIELD_FL_EXPR		= 1 << 13,
 503	HIST_FIELD_FL_VAR_REF		= 1 << 14,
 504	HIST_FIELD_FL_CPU		= 1 << 15,
 505	HIST_FIELD_FL_ALIAS		= 1 << 16,
 506	HIST_FIELD_FL_BUCKET		= 1 << 17,
 507	HIST_FIELD_FL_CONST		= 1 << 18,
 508	HIST_FIELD_FL_PERCENT		= 1 << 19,
 509	HIST_FIELD_FL_GRAPH		= 1 << 20,
 510	HIST_FIELD_FL_COMM		= 1 << 21,
 511};
 512
 513struct var_defs {
 514	unsigned int	n_vars;
 515	char		*name[TRACING_MAP_VARS_MAX];
 516	char		*expr[TRACING_MAP_VARS_MAX];
 517};
 518
 519struct hist_trigger_attrs {
 520	char		*keys_str;
 521	char		*vals_str;
 522	char		*sort_key_str;
 523	char		*name;
 524	char		*clock;
 525	bool		pause;
 526	bool		cont;
 527	bool		clear;
 528	bool		ts_in_usecs;
 529	bool		no_hitcount;
 530	unsigned int	map_bits;
 531
 532	char		*assignment_str[TRACING_MAP_VARS_MAX];
 533	unsigned int	n_assignments;
 534
 535	char		*action_str[HIST_ACTIONS_MAX];
 536	unsigned int	n_actions;
 537
 538	struct var_defs	var_defs;
 539};
 540
 541struct field_var {
 542	struct hist_field	*var;
 543	struct hist_field	*val;
 544};
 545
 546struct field_var_hist {
 547	struct hist_trigger_data	*hist_data;
 548	char				*cmd;
 549};
 550
 551struct hist_trigger_data {
 552	struct hist_field               *fields[HIST_FIELDS_MAX];
 553	unsigned int			n_vals;
 554	unsigned int			n_keys;
 555	unsigned int			n_fields;
 556	unsigned int			n_vars;
 557	unsigned int			n_var_str;
 558	unsigned int			key_size;
 559	struct tracing_map_sort_key	sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 560	unsigned int			n_sort_keys;
 561	struct trace_event_file		*event_file;
 562	struct hist_trigger_attrs	*attrs;
 563	struct tracing_map		*map;
 564	bool				enable_timestamps;
 565	bool				remove;
 566	struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 567	unsigned int			n_var_refs;
 568
 569	struct action_data		*actions[HIST_ACTIONS_MAX];
 570	unsigned int			n_actions;
 571
 572	struct field_var		*field_vars[SYNTH_FIELDS_MAX];
 573	unsigned int			n_field_vars;
 574	unsigned int			n_field_var_str;
 575	struct field_var_hist		*field_var_hists[SYNTH_FIELDS_MAX];
 576	unsigned int			n_field_var_hists;
 577
 578	struct field_var		*save_vars[SYNTH_FIELDS_MAX];
 579	unsigned int			n_save_vars;
 580	unsigned int			n_save_var_str;
 581};
 582
 583struct action_data;
 584
 585typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 586			     struct tracing_map_elt *elt,
 587			     struct trace_buffer *buffer, void *rec,
 588			     struct ring_buffer_event *rbe, void *key,
 589			     struct action_data *data, u64 *var_ref_vals);
 590
 591typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 592
 593enum handler_id {
 594	HANDLER_ONMATCH = 1,
 595	HANDLER_ONMAX,
 596	HANDLER_ONCHANGE,
 597};
 598
 599enum action_id {
 600	ACTION_SAVE = 1,
 601	ACTION_TRACE,
 602	ACTION_SNAPSHOT,
 603};
 604
 605struct action_data {
 606	enum handler_id		handler;
 607	enum action_id		action;
 608	char			*action_name;
 609	action_fn_t		fn;
 610
 611	unsigned int		n_params;
 612	char			*params[SYNTH_FIELDS_MAX];
 613
 614	/*
 615	 * When a histogram trigger is hit, the values of any
 616	 * references to variables, including variables being passed
 617	 * as parameters to synthetic events, are collected into a
 618	 * var_ref_vals array.  This var_ref_idx array is an array of
 619	 * indices into the var_ref_vals array, one for each synthetic
 620	 * event param, and is passed to the synthetic event
 621	 * invocation.
 622	 */
 623	unsigned int		var_ref_idx[SYNTH_FIELDS_MAX];
 624	struct synth_event	*synth_event;
 625	bool			use_trace_keyword;
 626	char			*synth_event_name;
 627
 628	union {
 629		struct {
 630			char			*event;
 631			char			*event_system;
 632		} match_data;
 633
 634		struct {
 635			/*
 636			 * var_str contains the $-unstripped variable
 637			 * name referenced by var_ref, and used when
 638			 * printing the action.  Because var_ref
 639			 * creation is deferred to create_actions(),
 640			 * we need a per-action way to save it until
 641			 * then, thus var_str.
 642			 */
 643			char			*var_str;
 644
 645			/*
 646			 * var_ref refers to the variable being
 647			 * tracked e.g onmax($var).
 648			 */
 649			struct hist_field	*var_ref;
 650
 651			/*
 652			 * track_var contains the 'invisible' tracking
 653			 * variable created to keep the current
 654			 * e.g. max value.
 655			 */
 656			struct hist_field	*track_var;
 657
 658			check_track_val_fn_t	check_val;
 659			action_fn_t		save_data;
 660		} track_data;
 661	};
 662};
 663
 664struct track_data {
 665	u64				track_val;
 666	bool				updated;
 667
 668	unsigned int			key_len;
 669	void				*key;
 670	struct tracing_map_elt		elt;
 671
 672	struct action_data		*action_data;
 673	struct hist_trigger_data	*hist_data;
 674};
 675
 676struct hist_elt_data {
 677	char *comm;
 678	u64 *var_ref_vals;
 679	char **field_var_str;
 680	int n_field_var_str;
 681};
 682
 683struct snapshot_context {
 684	struct tracing_map_elt	*elt;
 685	void			*key;
 686};
 687
 688/*
 689 * Returns the specific division function to use if the divisor
 690 * is constant. This avoids extra branches when the trigger is hit.
 691 */
 692static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 693{
 694	u64 div = divisor->constant;
 695
 696	if (!(div & (div - 1)))
 697		return HIST_FIELD_FN_DIV_POWER2;
 698
 699	/* If the divisor is too large, do a regular division */
 700	if (div > (1 << HIST_DIV_SHIFT))
 701		return HIST_FIELD_FN_DIV_NOT_POWER2;
 702
 703	divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 704	return HIST_FIELD_FN_DIV_MULT_SHIFT;
 705}
 706
 707static void track_data_free(struct track_data *track_data)
 708{
 709	struct hist_elt_data *elt_data;
 710
 711	if (!track_data)
 712		return;
 713
 714	kfree(track_data->key);
 715
 716	elt_data = track_data->elt.private_data;
 717	if (elt_data) {
 718		kfree(elt_data->comm);
 719		kfree(elt_data);
 720	}
 721
 722	kfree(track_data);
 723}
 724
 725static struct track_data *track_data_alloc(unsigned int key_len,
 726					   struct action_data *action_data,
 727					   struct hist_trigger_data *hist_data)
 728{
 729	struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 730	struct hist_elt_data *elt_data;
 731
 732	if (!data)
 733		return ERR_PTR(-ENOMEM);
 734
 735	data->key = kzalloc(key_len, GFP_KERNEL);
 736	if (!data->key) {
 737		track_data_free(data);
 738		return ERR_PTR(-ENOMEM);
 739	}
 740
 741	data->key_len = key_len;
 742	data->action_data = action_data;
 743	data->hist_data = hist_data;
 744
 745	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 746	if (!elt_data) {
 747		track_data_free(data);
 748		return ERR_PTR(-ENOMEM);
 749	}
 750
 751	data->elt.private_data = elt_data;
 752
 753	elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 754	if (!elt_data->comm) {
 755		track_data_free(data);
 756		return ERR_PTR(-ENOMEM);
 757	}
 758
 759	return data;
 760}
 761
 762#define HIST_PREFIX "hist:"
 763
 764static char *last_cmd;
 765static char last_cmd_loc[MAX_FILTER_STR_VAL];
 766
 767static int errpos(char *str)
 768{
 769	if (!str || !last_cmd)
 770		return 0;
 771
 772	return err_pos(last_cmd, str);
 773}
 774
 775static void last_cmd_set(struct trace_event_file *file, char *str)
 776{
 777	const char *system = NULL, *name = NULL;
 778	struct trace_event_call *call;
 779
 780	if (!str)
 781		return;
 782
 783	kfree(last_cmd);
 784
 785	last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str);
 786	if (!last_cmd)
 787		return;
 788
 789	if (file) {
 790		call = file->event_call;
 791		system = call->class->system;
 792		if (system) {
 793			name = trace_event_name(call);
 794			if (!name)
 795				system = NULL;
 796		}
 797	}
 798
 799	if (system)
 800		snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 801}
 802
 803static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 804{
 805	if (!last_cmd)
 806		return;
 807
 808	tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 809			err_type, err_pos);
 810}
 811
 812static void hist_err_clear(void)
 813{
 814	if (last_cmd)
 815		last_cmd[0] = '\0';
 816	last_cmd_loc[0] = '\0';
 817}
 818
 819typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 820				    unsigned int *var_ref_idx);
 821
 822static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 823			       unsigned int *var_ref_idx)
 824{
 825	struct tracepoint *tp = event->tp;
 826
 827	if (unlikely(static_key_enabled(&tp->key))) {
 828		struct tracepoint_func *probe_func_ptr;
 829		synth_probe_func_t probe_func;
 830		void *__data;
 831
 832		if (!(cpu_online(raw_smp_processor_id())))
 833			return;
 834
 835		probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 836		if (probe_func_ptr) {
 837			do {
 838				probe_func = probe_func_ptr->func;
 839				__data = probe_func_ptr->data;
 840				probe_func(__data, var_ref_vals, var_ref_idx);
 841			} while ((++probe_func_ptr)->func);
 842		}
 843	}
 844}
 845
 846static void action_trace(struct hist_trigger_data *hist_data,
 847			 struct tracing_map_elt *elt,
 848			 struct trace_buffer *buffer, void *rec,
 849			 struct ring_buffer_event *rbe, void *key,
 850			 struct action_data *data, u64 *var_ref_vals)
 851{
 852	struct synth_event *event = data->synth_event;
 853
 854	trace_synth(event, var_ref_vals, data->var_ref_idx);
 855}
 856
 857struct hist_var_data {
 858	struct list_head list;
 859	struct hist_trigger_data *hist_data;
 860};
 861
 862static u64 hist_field_timestamp(struct hist_field *hist_field,
 863				struct tracing_map_elt *elt,
 864				struct trace_buffer *buffer,
 865				struct ring_buffer_event *rbe,
 866				void *event)
 867{
 868	struct hist_trigger_data *hist_data = hist_field->hist_data;
 869	struct trace_array *tr = hist_data->event_file->tr;
 870
 871	u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 872
 873	if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 874		ts = ns2usecs(ts);
 875
 876	return ts;
 877}
 878
 879static u64 hist_field_cpu(struct hist_field *hist_field,
 880			  struct tracing_map_elt *elt,
 881			  struct trace_buffer *buffer,
 882			  struct ring_buffer_event *rbe,
 883			  void *event)
 884{
 885	int cpu = smp_processor_id();
 886
 887	return cpu;
 888}
 889
 890static u64 hist_field_comm(struct hist_field *hist_field,
 891			  struct tracing_map_elt *elt,
 892			  struct trace_buffer *buffer,
 893			  struct ring_buffer_event *rbe,
 894			  void *event)
 895{
 896	return (u64)(unsigned long)current->comm;
 897}
 898
 899/**
 900 * check_field_for_var_ref - Check if a VAR_REF field references a variable
 901 * @hist_field: The VAR_REF field to check
 902 * @var_data: The hist trigger that owns the variable
 903 * @var_idx: The trigger variable identifier
 904 *
 905 * Check the given VAR_REF field to see whether or not it references
 906 * the given variable associated with the given trigger.
 907 *
 908 * Return: The VAR_REF field if it does reference the variable, NULL if not
 909 */
 910static struct hist_field *
 911check_field_for_var_ref(struct hist_field *hist_field,
 912			struct hist_trigger_data *var_data,
 913			unsigned int var_idx)
 914{
 915	WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 916
 917	if (hist_field && hist_field->var.idx == var_idx &&
 918	    hist_field->var.hist_data == var_data)
 919		return hist_field;
 920
 921	return NULL;
 922}
 923
 924/**
 925 * find_var_ref - Check if a trigger has a reference to a trigger variable
 926 * @hist_data: The hist trigger that might have a reference to the variable
 927 * @var_data: The hist trigger that owns the variable
 928 * @var_idx: The trigger variable identifier
 929 *
 930 * Check the list of var_refs[] on the first hist trigger to see
 931 * whether any of them are references to the variable on the second
 932 * trigger.
 933 *
 934 * Return: The VAR_REF field referencing the variable if so, NULL if not
 935 */
 936static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 937				       struct hist_trigger_data *var_data,
 938				       unsigned int var_idx)
 939{
 940	struct hist_field *hist_field;
 941	unsigned int i;
 942
 943	for (i = 0; i < hist_data->n_var_refs; i++) {
 944		hist_field = hist_data->var_refs[i];
 945		if (check_field_for_var_ref(hist_field, var_data, var_idx))
 946			return hist_field;
 947	}
 948
 949	return NULL;
 950}
 951
 952/**
 953 * find_any_var_ref - Check if there is a reference to a given trigger variable
 954 * @hist_data: The hist trigger
 955 * @var_idx: The trigger variable identifier
 956 *
 957 * Check to see whether the given variable is currently referenced by
 958 * any other trigger.
 959 *
 960 * The trigger the variable is defined on is explicitly excluded - the
 961 * assumption being that a self-reference doesn't prevent a trigger
 962 * from being removed.
 963 *
 964 * Return: The VAR_REF field referencing the variable if so, NULL if not
 965 */
 966static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 967					   unsigned int var_idx)
 968{
 969	struct trace_array *tr = hist_data->event_file->tr;
 970	struct hist_field *found = NULL;
 971	struct hist_var_data *var_data;
 972
 973	list_for_each_entry(var_data, &tr->hist_vars, list) {
 974		if (var_data->hist_data == hist_data)
 975			continue;
 976		found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 977		if (found)
 978			break;
 979	}
 980
 981	return found;
 982}
 983
 984/**
 985 * check_var_refs - Check if there is a reference to any of trigger's variables
 986 * @hist_data: The hist trigger
 987 *
 988 * A trigger can define one or more variables.  If any one of them is
 989 * currently referenced by any other trigger, this function will
 990 * determine that.
 991 *
 992 * Typically used to determine whether or not a trigger can be removed
 993 * - if there are any references to a trigger's variables, it cannot.
 994 *
 995 * Return: True if there is a reference to any of trigger's variables
 996 */
 997static bool check_var_refs(struct hist_trigger_data *hist_data)
 998{
 999	struct hist_field *field;
1000	bool found = false;
1001	int i;
1002
1003	for_each_hist_field(i, hist_data) {
1004		field = hist_data->fields[i];
1005		if (field && field->flags & HIST_FIELD_FL_VAR) {
1006			if (find_any_var_ref(hist_data, field->var.idx)) {
1007				found = true;
1008				break;
1009			}
1010		}
1011	}
1012
1013	return found;
1014}
1015
1016static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1017{
1018	struct trace_array *tr = hist_data->event_file->tr;
1019	struct hist_var_data *var_data, *found = NULL;
1020
1021	list_for_each_entry(var_data, &tr->hist_vars, list) {
1022		if (var_data->hist_data == hist_data) {
1023			found = var_data;
1024			break;
1025		}
1026	}
1027
1028	return found;
1029}
1030
1031static bool field_has_hist_vars(struct hist_field *hist_field,
1032				unsigned int level)
1033{
1034	int i;
1035
1036	if (level > 3)
1037		return false;
1038
1039	if (!hist_field)
1040		return false;
1041
1042	if (hist_field->flags & HIST_FIELD_FL_VAR ||
1043	    hist_field->flags & HIST_FIELD_FL_VAR_REF)
1044		return true;
1045
1046	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1047		struct hist_field *operand;
1048
1049		operand = hist_field->operands[i];
1050		if (field_has_hist_vars(operand, level + 1))
1051			return true;
1052	}
1053
1054	return false;
1055}
1056
1057static bool has_hist_vars(struct hist_trigger_data *hist_data)
1058{
1059	struct hist_field *hist_field;
1060	int i;
1061
1062	for_each_hist_field(i, hist_data) {
1063		hist_field = hist_data->fields[i];
1064		if (field_has_hist_vars(hist_field, 0))
1065			return true;
1066	}
1067
1068	return false;
1069}
1070
1071static int save_hist_vars(struct hist_trigger_data *hist_data)
1072{
1073	struct trace_array *tr = hist_data->event_file->tr;
1074	struct hist_var_data *var_data;
1075
1076	var_data = find_hist_vars(hist_data);
1077	if (var_data)
1078		return 0;
1079
1080	if (tracing_check_open_get_tr(tr))
1081		return -ENODEV;
1082
1083	var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1084	if (!var_data) {
1085		trace_array_put(tr);
1086		return -ENOMEM;
1087	}
1088
1089	var_data->hist_data = hist_data;
1090	list_add(&var_data->list, &tr->hist_vars);
1091
1092	return 0;
1093}
1094
1095static void remove_hist_vars(struct hist_trigger_data *hist_data)
1096{
1097	struct trace_array *tr = hist_data->event_file->tr;
1098	struct hist_var_data *var_data;
1099
1100	var_data = find_hist_vars(hist_data);
1101	if (!var_data)
1102		return;
1103
1104	if (WARN_ON(check_var_refs(hist_data)))
1105		return;
1106
1107	list_del(&var_data->list);
1108
1109	kfree(var_data);
1110
1111	trace_array_put(tr);
1112}
1113
1114static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1115					 const char *var_name)
1116{
1117	struct hist_field *hist_field, *found = NULL;
1118	int i;
1119
1120	for_each_hist_field(i, hist_data) {
1121		hist_field = hist_data->fields[i];
1122		if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1123		    strcmp(hist_field->var.name, var_name) == 0) {
1124			found = hist_field;
1125			break;
1126		}
1127	}
1128
1129	return found;
1130}
1131
1132static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1133				   struct trace_event_file *file,
1134				   const char *var_name)
1135{
1136	struct hist_trigger_data *test_data;
1137	struct event_trigger_data *test;
1138	struct hist_field *hist_field;
1139
1140	lockdep_assert_held(&event_mutex);
1141
1142	hist_field = find_var_field(hist_data, var_name);
1143	if (hist_field)
1144		return hist_field;
1145
1146	list_for_each_entry(test, &file->triggers, list) {
1147		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1148			test_data = test->private_data;
1149			hist_field = find_var_field(test_data, var_name);
1150			if (hist_field)
1151				return hist_field;
1152		}
1153	}
1154
1155	return NULL;
1156}
1157
1158static struct trace_event_file *find_var_file(struct trace_array *tr,
1159					      char *system,
1160					      char *event_name,
1161					      char *var_name)
1162{
1163	struct hist_trigger_data *var_hist_data;
1164	struct hist_var_data *var_data;
1165	struct trace_event_file *file, *found = NULL;
1166
1167	if (system)
1168		return find_event_file(tr, system, event_name);
1169
1170	list_for_each_entry(var_data, &tr->hist_vars, list) {
1171		var_hist_data = var_data->hist_data;
1172		file = var_hist_data->event_file;
1173		if (file == found)
1174			continue;
1175
1176		if (find_var_field(var_hist_data, var_name)) {
1177			if (found) {
1178				hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1179				return NULL;
1180			}
1181
1182			found = file;
1183		}
1184	}
1185
1186	return found;
1187}
1188
1189static struct hist_field *find_file_var(struct trace_event_file *file,
1190					const char *var_name)
1191{
1192	struct hist_trigger_data *test_data;
1193	struct event_trigger_data *test;
1194	struct hist_field *hist_field;
1195
1196	lockdep_assert_held(&event_mutex);
1197
1198	list_for_each_entry(test, &file->triggers, list) {
1199		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1200			test_data = test->private_data;
1201			hist_field = find_var_field(test_data, var_name);
1202			if (hist_field)
1203				return hist_field;
1204		}
1205	}
1206
1207	return NULL;
1208}
1209
1210static struct hist_field *
1211find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1212{
1213	struct trace_array *tr = hist_data->event_file->tr;
1214	struct hist_field *hist_field, *found = NULL;
1215	struct trace_event_file *file;
1216	unsigned int i;
1217
1218	for (i = 0; i < hist_data->n_actions; i++) {
1219		struct action_data *data = hist_data->actions[i];
1220
1221		if (data->handler == HANDLER_ONMATCH) {
1222			char *system = data->match_data.event_system;
1223			char *event_name = data->match_data.event;
1224
1225			file = find_var_file(tr, system, event_name, var_name);
1226			if (!file)
1227				continue;
1228			hist_field = find_file_var(file, var_name);
1229			if (hist_field) {
1230				if (found) {
1231					hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1232						 errpos(var_name));
1233					return ERR_PTR(-EINVAL);
1234				}
1235
1236				found = hist_field;
1237			}
1238		}
1239	}
1240	return found;
1241}
1242
1243static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1244					 char *system,
1245					 char *event_name,
1246					 char *var_name)
1247{
1248	struct trace_array *tr = hist_data->event_file->tr;
1249	struct hist_field *hist_field = NULL;
1250	struct trace_event_file *file;
1251
1252	if (!system || !event_name) {
1253		hist_field = find_match_var(hist_data, var_name);
1254		if (IS_ERR(hist_field))
1255			return NULL;
1256		if (hist_field)
1257			return hist_field;
1258	}
1259
1260	file = find_var_file(tr, system, event_name, var_name);
1261	if (!file)
1262		return NULL;
1263
1264	hist_field = find_file_var(file, var_name);
1265
1266	return hist_field;
1267}
1268
1269static u64 hist_field_var_ref(struct hist_field *hist_field,
1270			      struct tracing_map_elt *elt,
1271			      struct trace_buffer *buffer,
1272			      struct ring_buffer_event *rbe,
1273			      void *event)
1274{
1275	struct hist_elt_data *elt_data;
1276	u64 var_val = 0;
1277
1278	if (WARN_ON_ONCE(!elt))
1279		return var_val;
1280
1281	elt_data = elt->private_data;
1282	var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1283
1284	return var_val;
1285}
1286
1287static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1288			     u64 *var_ref_vals, bool self)
1289{
1290	struct hist_trigger_data *var_data;
1291	struct tracing_map_elt *var_elt;
1292	struct hist_field *hist_field;
1293	unsigned int i, var_idx;
1294	bool resolved = true;
1295	u64 var_val = 0;
1296
1297	for (i = 0; i < hist_data->n_var_refs; i++) {
1298		hist_field = hist_data->var_refs[i];
1299		var_idx = hist_field->var.idx;
1300		var_data = hist_field->var.hist_data;
1301
1302		if (var_data == NULL) {
1303			resolved = false;
1304			break;
1305		}
1306
1307		if ((self && var_data != hist_data) ||
1308		    (!self && var_data == hist_data))
1309			continue;
1310
1311		var_elt = tracing_map_lookup(var_data->map, key);
1312		if (!var_elt) {
1313			resolved = false;
1314			break;
1315		}
1316
1317		if (!tracing_map_var_set(var_elt, var_idx)) {
1318			resolved = false;
1319			break;
1320		}
1321
1322		if (self || !hist_field->read_once)
1323			var_val = tracing_map_read_var(var_elt, var_idx);
1324		else
1325			var_val = tracing_map_read_var_once(var_elt, var_idx);
1326
1327		var_ref_vals[i] = var_val;
1328	}
1329
1330	return resolved;
1331}
1332
1333static const char *hist_field_name(struct hist_field *field,
1334				   unsigned int level)
1335{
1336	const char *field_name = "";
1337
1338	if (WARN_ON_ONCE(!field))
1339		return field_name;
1340
1341	if (level > 1)
1342		return field_name;
1343
1344	if (field->field)
1345		field_name = field->field->name;
1346	else if (field->flags & HIST_FIELD_FL_LOG2 ||
1347		 field->flags & HIST_FIELD_FL_ALIAS ||
1348		 field->flags & HIST_FIELD_FL_BUCKET)
1349		field_name = hist_field_name(field->operands[0], ++level);
1350	else if (field->flags & HIST_FIELD_FL_CPU)
1351		field_name = "common_cpu";
1352	else if (field->flags & HIST_FIELD_FL_COMM)
1353		field_name = "common_comm";
1354	else if (field->flags & HIST_FIELD_FL_EXPR ||
1355		 field->flags & HIST_FIELD_FL_VAR_REF) {
1356		if (field->system) {
1357			static char full_name[MAX_FILTER_STR_VAL];
1358
1359			strcat(full_name, field->system);
1360			strcat(full_name, ".");
1361			strcat(full_name, field->event_name);
1362			strcat(full_name, ".");
1363			strcat(full_name, field->name);
1364			field_name = full_name;
1365		} else
1366			field_name = field->name;
1367	} else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1368		field_name = "common_timestamp";
1369	else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1370		field_name = "common_stacktrace";
1371	} else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1372		field_name = "hitcount";
1373
1374	if (field_name == NULL)
1375		field_name = "";
1376
1377	return field_name;
1378}
1379
1380static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1381{
1382	switch (field_size) {
1383	case 8:
1384		if (field_is_signed)
1385			return HIST_FIELD_FN_S64;
1386		else
1387			return HIST_FIELD_FN_U64;
1388	case 4:
1389		if (field_is_signed)
1390			return HIST_FIELD_FN_S32;
1391		else
1392			return HIST_FIELD_FN_U32;
1393	case 2:
1394		if (field_is_signed)
1395			return HIST_FIELD_FN_S16;
1396		else
1397			return HIST_FIELD_FN_U16;
1398	case 1:
1399		if (field_is_signed)
1400			return HIST_FIELD_FN_S8;
1401		else
1402			return HIST_FIELD_FN_U8;
1403	}
1404
1405	return HIST_FIELD_FN_NOP;
1406}
1407
1408static int parse_map_size(char *str)
1409{
1410	unsigned long size, map_bits;
1411	int ret;
1412
1413	ret = kstrtoul(str, 0, &size);
1414	if (ret)
1415		goto out;
1416
1417	map_bits = ilog2(roundup_pow_of_two(size));
1418	if (map_bits < TRACING_MAP_BITS_MIN ||
1419	    map_bits > TRACING_MAP_BITS_MAX)
1420		ret = -EINVAL;
1421	else
1422		ret = map_bits;
1423 out:
1424	return ret;
1425}
1426
1427static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1428{
1429	unsigned int i;
1430
1431	if (!attrs)
1432		return;
1433
1434	for (i = 0; i < attrs->n_assignments; i++)
1435		kfree(attrs->assignment_str[i]);
1436
1437	for (i = 0; i < attrs->n_actions; i++)
1438		kfree(attrs->action_str[i]);
1439
1440	kfree(attrs->name);
1441	kfree(attrs->sort_key_str);
1442	kfree(attrs->keys_str);
1443	kfree(attrs->vals_str);
1444	kfree(attrs->clock);
1445	kfree(attrs);
1446}
1447
1448static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1449{
1450	int ret = -EINVAL;
1451
1452	if (attrs->n_actions >= HIST_ACTIONS_MAX)
1453		return ret;
1454
1455	if ((str_has_prefix(str, "onmatch(")) ||
1456	    (str_has_prefix(str, "onmax(")) ||
1457	    (str_has_prefix(str, "onchange("))) {
1458		attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1459		if (!attrs->action_str[attrs->n_actions]) {
1460			ret = -ENOMEM;
1461			return ret;
1462		}
1463		attrs->n_actions++;
1464		ret = 0;
1465	}
1466	return ret;
1467}
1468
1469static int parse_assignment(struct trace_array *tr,
1470			    char *str, struct hist_trigger_attrs *attrs)
1471{
1472	int len, ret = 0;
1473
1474	if ((len = str_has_prefix(str, "key=")) ||
1475	    (len = str_has_prefix(str, "keys="))) {
1476		attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1477		if (!attrs->keys_str) {
1478			ret = -ENOMEM;
1479			goto out;
1480		}
1481	} else if ((len = str_has_prefix(str, "val=")) ||
1482		   (len = str_has_prefix(str, "vals=")) ||
1483		   (len = str_has_prefix(str, "values="))) {
1484		attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1485		if (!attrs->vals_str) {
1486			ret = -ENOMEM;
1487			goto out;
1488		}
1489	} else if ((len = str_has_prefix(str, "sort="))) {
1490		attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1491		if (!attrs->sort_key_str) {
1492			ret = -ENOMEM;
1493			goto out;
1494		}
1495	} else if (str_has_prefix(str, "name=")) {
1496		attrs->name = kstrdup(str, GFP_KERNEL);
1497		if (!attrs->name) {
1498			ret = -ENOMEM;
1499			goto out;
1500		}
1501	} else if ((len = str_has_prefix(str, "clock="))) {
1502		str += len;
1503
1504		str = strstrip(str);
1505		attrs->clock = kstrdup(str, GFP_KERNEL);
1506		if (!attrs->clock) {
1507			ret = -ENOMEM;
1508			goto out;
1509		}
1510	} else if ((len = str_has_prefix(str, "size="))) {
1511		int map_bits = parse_map_size(str + len);
1512
1513		if (map_bits < 0) {
1514			ret = map_bits;
1515			goto out;
1516		}
1517		attrs->map_bits = map_bits;
1518	} else {
1519		char *assignment;
1520
1521		if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1522			hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1523			ret = -EINVAL;
1524			goto out;
1525		}
1526
1527		assignment = kstrdup(str, GFP_KERNEL);
1528		if (!assignment) {
1529			ret = -ENOMEM;
1530			goto out;
1531		}
1532
1533		attrs->assignment_str[attrs->n_assignments++] = assignment;
1534	}
1535 out:
1536	return ret;
1537}
1538
1539static struct hist_trigger_attrs *
1540parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1541{
1542	struct hist_trigger_attrs *attrs;
1543	int ret = 0;
1544
1545	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1546	if (!attrs)
1547		return ERR_PTR(-ENOMEM);
1548
1549	while (trigger_str) {
1550		char *str = strsep(&trigger_str, ":");
1551		char *rhs;
1552
1553		rhs = strchr(str, '=');
1554		if (rhs) {
1555			if (!strlen(++rhs)) {
1556				ret = -EINVAL;
1557				hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1558				goto free;
1559			}
1560			ret = parse_assignment(tr, str, attrs);
1561			if (ret)
1562				goto free;
1563		} else if (strcmp(str, "nohitcount") == 0 ||
1564			   strcmp(str, "NOHC") == 0)
1565			attrs->no_hitcount = true;
1566		else if (strcmp(str, "pause") == 0)
1567			attrs->pause = true;
1568		else if ((strcmp(str, "cont") == 0) ||
1569			 (strcmp(str, "continue") == 0))
1570			attrs->cont = true;
1571		else if (strcmp(str, "clear") == 0)
1572			attrs->clear = true;
1573		else {
1574			ret = parse_action(str, attrs);
1575			if (ret)
1576				goto free;
1577		}
1578	}
1579
1580	if (!attrs->keys_str) {
1581		ret = -EINVAL;
1582		goto free;
1583	}
1584
1585	if (!attrs->clock) {
1586		attrs->clock = kstrdup("global", GFP_KERNEL);
1587		if (!attrs->clock) {
1588			ret = -ENOMEM;
1589			goto free;
1590		}
1591	}
1592
1593	return attrs;
1594 free:
1595	destroy_hist_trigger_attrs(attrs);
1596
1597	return ERR_PTR(ret);
1598}
1599
1600static inline void save_comm(char *comm, struct task_struct *task)
1601{
1602	if (!task->pid) {
1603		strcpy(comm, "<idle>");
1604		return;
1605	}
1606
1607	if (WARN_ON_ONCE(task->pid < 0)) {
1608		strcpy(comm, "<XXX>");
1609		return;
1610	}
1611
1612	strscpy(comm, task->comm, TASK_COMM_LEN);
1613}
1614
1615static void hist_elt_data_free(struct hist_elt_data *elt_data)
1616{
1617	unsigned int i;
1618
1619	for (i = 0; i < elt_data->n_field_var_str; i++)
1620		kfree(elt_data->field_var_str[i]);
1621
1622	kfree(elt_data->field_var_str);
1623
1624	kfree(elt_data->comm);
1625	kfree(elt_data);
1626}
1627
1628static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1629{
1630	struct hist_elt_data *elt_data = elt->private_data;
1631
1632	hist_elt_data_free(elt_data);
1633}
1634
1635static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1636{
1637	struct hist_trigger_data *hist_data = elt->map->private_data;
1638	unsigned int size = TASK_COMM_LEN;
1639	struct hist_elt_data *elt_data;
1640	struct hist_field *hist_field;
1641	unsigned int i, n_str;
1642
1643	elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1644	if (!elt_data)
1645		return -ENOMEM;
1646
1647	for_each_hist_field(i, hist_data) {
1648		hist_field = hist_data->fields[i];
1649
1650		if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1651			elt_data->comm = kzalloc(size, GFP_KERNEL);
1652			if (!elt_data->comm) {
1653				kfree(elt_data);
1654				return -ENOMEM;
1655			}
1656			break;
1657		}
1658	}
1659
1660	n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1661		hist_data->n_var_str;
1662	if (n_str > SYNTH_FIELDS_MAX) {
1663		hist_elt_data_free(elt_data);
1664		return -EINVAL;
1665	}
1666
1667	BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1668
1669	size = STR_VAR_LEN_MAX;
1670
1671	elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1672	if (!elt_data->field_var_str) {
1673		hist_elt_data_free(elt_data);
1674		return -EINVAL;
1675	}
1676	elt_data->n_field_var_str = n_str;
1677
1678	for (i = 0; i < n_str; i++) {
1679		elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1680		if (!elt_data->field_var_str[i]) {
1681			hist_elt_data_free(elt_data);
1682			return -ENOMEM;
1683		}
1684	}
1685
1686	elt->private_data = elt_data;
1687
1688	return 0;
1689}
1690
1691static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1692{
1693	struct hist_elt_data *elt_data = elt->private_data;
1694
1695	if (elt_data->comm)
1696		save_comm(elt_data->comm, current);
1697}
1698
1699static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1700	.elt_alloc	= hist_trigger_elt_data_alloc,
1701	.elt_free	= hist_trigger_elt_data_free,
1702	.elt_init	= hist_trigger_elt_data_init,
1703};
1704
1705static const char *get_hist_field_flags(struct hist_field *hist_field)
1706{
1707	const char *flags_str = NULL;
1708
1709	if (hist_field->flags & HIST_FIELD_FL_HEX)
1710		flags_str = "hex";
1711	else if (hist_field->flags & HIST_FIELD_FL_SYM)
1712		flags_str = "sym";
1713	else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1714		flags_str = "sym-offset";
1715	else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1716		flags_str = "execname";
1717	else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1718		flags_str = "syscall";
1719	else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1720		flags_str = "log2";
1721	else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1722		flags_str = "buckets";
1723	else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1724		flags_str = "usecs";
1725	else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1726		flags_str = "percent";
1727	else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1728		flags_str = "graph";
1729	else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1730		flags_str = "stacktrace";
1731
1732	return flags_str;
1733}
1734
1735static void expr_field_str(struct hist_field *field, char *expr)
1736{
1737	if (field->flags & HIST_FIELD_FL_VAR_REF)
1738		strcat(expr, "$");
1739	else if (field->flags & HIST_FIELD_FL_CONST) {
1740		char str[HIST_CONST_DIGITS_MAX];
1741
1742		snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1743		strcat(expr, str);
1744	}
1745
1746	strcat(expr, hist_field_name(field, 0));
1747
1748	if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1749		const char *flags_str = get_hist_field_flags(field);
1750
1751		if (flags_str) {
1752			strcat(expr, ".");
1753			strcat(expr, flags_str);
1754		}
1755	}
1756}
1757
1758static char *expr_str(struct hist_field *field, unsigned int level)
1759{
1760	char *expr;
1761
1762	if (level > 1)
1763		return NULL;
1764
1765	expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1766	if (!expr)
1767		return NULL;
1768
1769	if (!field->operands[0]) {
1770		expr_field_str(field, expr);
1771		return expr;
1772	}
1773
1774	if (field->operator == FIELD_OP_UNARY_MINUS) {
1775		char *subexpr;
1776
1777		strcat(expr, "-(");
1778		subexpr = expr_str(field->operands[0], ++level);
1779		if (!subexpr) {
1780			kfree(expr);
1781			return NULL;
1782		}
1783		strcat(expr, subexpr);
1784		strcat(expr, ")");
1785
1786		kfree(subexpr);
1787
1788		return expr;
1789	}
1790
1791	expr_field_str(field->operands[0], expr);
1792
1793	switch (field->operator) {
1794	case FIELD_OP_MINUS:
1795		strcat(expr, "-");
1796		break;
1797	case FIELD_OP_PLUS:
1798		strcat(expr, "+");
1799		break;
1800	case FIELD_OP_DIV:
1801		strcat(expr, "/");
1802		break;
1803	case FIELD_OP_MULT:
1804		strcat(expr, "*");
1805		break;
1806	default:
1807		kfree(expr);
1808		return NULL;
1809	}
1810
1811	expr_field_str(field->operands[1], expr);
1812
1813	return expr;
1814}
1815
1816/*
1817 * If field_op != FIELD_OP_NONE, *sep points to the root operator
1818 * of the expression tree to be evaluated.
1819 */
1820static int contains_operator(char *str, char **sep)
1821{
1822	enum field_op_id field_op = FIELD_OP_NONE;
1823	char *minus_op, *plus_op, *div_op, *mult_op;
1824
1825
1826	/*
1827	 * Report the last occurrence of the operators first, so that the
1828	 * expression is evaluated left to right. This is important since
1829	 * subtraction and division are not associative.
1830	 *
1831	 *	e.g
1832	 *		64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1833	 *		14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1834	 */
1835
1836	/*
1837	 * First, find lower precedence addition and subtraction
1838	 * since the expression will be evaluated recursively.
1839	 */
1840	minus_op = strrchr(str, '-');
1841	if (minus_op) {
1842		/*
1843		 * Unary minus is not supported in sub-expressions. If
1844		 * present, it is always the next root operator.
1845		 */
1846		if (minus_op == str) {
1847			field_op = FIELD_OP_UNARY_MINUS;
1848			goto out;
1849		}
1850
1851		field_op = FIELD_OP_MINUS;
1852	}
1853
1854	plus_op = strrchr(str, '+');
1855	if (plus_op || minus_op) {
1856		/*
1857		 * For operators of the same precedence use to rightmost as the
1858		 * root, so that the expression is evaluated left to right.
1859		 */
1860		if (plus_op > minus_op)
1861			field_op = FIELD_OP_PLUS;
1862		goto out;
1863	}
1864
1865	/*
1866	 * Multiplication and division have higher precedence than addition and
1867	 * subtraction.
1868	 */
1869	div_op = strrchr(str, '/');
1870	if (div_op)
1871		field_op = FIELD_OP_DIV;
1872
1873	mult_op = strrchr(str, '*');
1874	/*
1875	 * For operators of the same precedence use to rightmost as the
1876	 * root, so that the expression is evaluated left to right.
1877	 */
1878	if (mult_op > div_op)
1879		field_op = FIELD_OP_MULT;
1880
1881out:
1882	if (sep) {
1883		switch (field_op) {
1884		case FIELD_OP_UNARY_MINUS:
1885		case FIELD_OP_MINUS:
1886			*sep = minus_op;
1887			break;
1888		case FIELD_OP_PLUS:
1889			*sep = plus_op;
1890			break;
1891		case FIELD_OP_DIV:
1892			*sep = div_op;
1893			break;
1894		case FIELD_OP_MULT:
1895			*sep = mult_op;
1896			break;
1897		case FIELD_OP_NONE:
1898		default:
1899			*sep = NULL;
1900			break;
1901		}
1902	}
1903
1904	return field_op;
1905}
1906
1907static void get_hist_field(struct hist_field *hist_field)
1908{
1909	hist_field->ref++;
1910}
1911
1912static void __destroy_hist_field(struct hist_field *hist_field)
1913{
1914	if (--hist_field->ref > 1)
1915		return;
1916
1917	kfree(hist_field->var.name);
1918	kfree(hist_field->name);
1919
1920	/* Can likely be a const */
1921	kfree_const(hist_field->type);
1922
1923	kfree(hist_field->system);
1924	kfree(hist_field->event_name);
1925
1926	kfree(hist_field);
1927}
1928
1929static void destroy_hist_field(struct hist_field *hist_field,
1930			       unsigned int level)
1931{
1932	unsigned int i;
1933
1934	if (level > 3)
1935		return;
1936
1937	if (!hist_field)
1938		return;
1939
1940	if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1941		return; /* var refs will be destroyed separately */
1942
1943	for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1944		destroy_hist_field(hist_field->operands[i], level + 1);
1945
1946	__destroy_hist_field(hist_field);
1947}
1948
1949static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1950					    struct ftrace_event_field *field,
1951					    unsigned long flags,
1952					    char *var_name)
1953{
1954	struct hist_field *hist_field;
1955
1956	if (field && is_function_field(field))
1957		return NULL;
1958
1959	hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1960	if (!hist_field)
1961		return NULL;
1962
1963	hist_field->ref = 1;
1964
1965	hist_field->hist_data = hist_data;
1966
1967	if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1968		goto out; /* caller will populate */
1969
1970	if (flags & HIST_FIELD_FL_VAR_REF) {
1971		hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1972		goto out;
1973	}
1974
1975	if (flags & HIST_FIELD_FL_HITCOUNT) {
1976		hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1977		hist_field->size = sizeof(u64);
1978		hist_field->type = "u64";
1979		goto out;
1980	}
1981
1982	if (flags & HIST_FIELD_FL_CONST) {
1983		hist_field->fn_num = HIST_FIELD_FN_CONST;
1984		hist_field->size = sizeof(u64);
1985		hist_field->type = kstrdup("u64", GFP_KERNEL);
1986		if (!hist_field->type)
1987			goto free;
1988		goto out;
1989	}
1990
1991	if (flags & HIST_FIELD_FL_STACKTRACE) {
1992		if (field)
1993			hist_field->fn_num = HIST_FIELD_FN_STACK;
1994		else
1995			hist_field->fn_num = HIST_FIELD_FN_NOP;
1996		hist_field->size = HIST_STACKTRACE_SIZE;
1997		hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1998		if (!hist_field->type)
1999			goto free;
2000		goto out;
2001	}
2002
2003	if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
2004		unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
2005		hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
2006			HIST_FIELD_FN_BUCKET;
2007		hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2008		if (!hist_field->operands[0])
2009			goto free;
2010		hist_field->size = hist_field->operands[0]->size;
2011		hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
2012		if (!hist_field->type)
2013			goto free;
2014		goto out;
2015	}
2016
2017	if (flags & HIST_FIELD_FL_TIMESTAMP) {
2018		hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2019		hist_field->size = sizeof(u64);
2020		hist_field->type = "u64";
2021		goto out;
2022	}
2023
2024	if (flags & HIST_FIELD_FL_CPU) {
2025		hist_field->fn_num = HIST_FIELD_FN_CPU;
2026		hist_field->size = sizeof(int);
2027		hist_field->type = "unsigned int";
2028		goto out;
2029	}
2030
2031	if (flags & HIST_FIELD_FL_COMM) {
2032		hist_field->fn_num = HIST_FIELD_FN_COMM;
2033		hist_field->size = MAX_FILTER_STR_VAL;
2034		hist_field->type = "char[]";
2035		goto out;
2036	}
2037
2038	if (WARN_ON_ONCE(!field))
2039		goto out;
2040
2041	/* Pointers to strings are just pointers and dangerous to dereference */
2042	if (is_string_field(field) &&
2043	    (field->filter_type != FILTER_PTR_STRING)) {
2044		flags |= HIST_FIELD_FL_STRING;
2045
2046		hist_field->size = MAX_FILTER_STR_VAL;
2047		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2048		if (!hist_field->type)
2049			goto free;
2050
2051		if (field->filter_type == FILTER_STATIC_STRING) {
2052			hist_field->fn_num = HIST_FIELD_FN_STRING;
2053			hist_field->size = field->size;
2054		} else if (field->filter_type == FILTER_DYN_STRING) {
2055			hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2056		} else if (field->filter_type == FILTER_RDYN_STRING)
2057			hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2058		else
2059			hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2060	} else if (field->filter_type == FILTER_STACKTRACE) {
2061		flags |= HIST_FIELD_FL_STACKTRACE;
2062
2063		hist_field->size = MAX_FILTER_STR_VAL;
2064		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2065		if (!hist_field->type)
2066			goto free;
2067
2068		hist_field->fn_num = HIST_FIELD_FN_STACK;
2069	} else {
2070		hist_field->size = field->size;
2071		hist_field->is_signed = field->is_signed;
2072		hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2073		if (!hist_field->type)
2074			goto free;
2075
2076		hist_field->fn_num = select_value_fn(field->size,
2077						     field->is_signed);
2078		if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2079			destroy_hist_field(hist_field, 0);
2080			return NULL;
2081		}
2082	}
2083 out:
2084	hist_field->field = field;
2085	hist_field->flags = flags;
2086
2087	if (var_name) {
2088		hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2089		if (!hist_field->var.name)
2090			goto free;
2091	}
2092
2093	return hist_field;
2094 free:
2095	destroy_hist_field(hist_field, 0);
2096	return NULL;
2097}
2098
2099static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2100{
2101	unsigned int i;
2102
2103	for (i = 0; i < HIST_FIELDS_MAX; i++) {
2104		if (hist_data->fields[i]) {
2105			destroy_hist_field(hist_data->fields[i], 0);
2106			hist_data->fields[i] = NULL;
2107		}
2108	}
2109
2110	for (i = 0; i < hist_data->n_var_refs; i++) {
2111		WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2112		__destroy_hist_field(hist_data->var_refs[i]);
2113		hist_data->var_refs[i] = NULL;
2114	}
2115}
2116
2117static int init_var_ref(struct hist_field *ref_field,
2118			struct hist_field *var_field,
2119			char *system, char *event_name)
2120{
2121	int err = 0;
2122
2123	ref_field->var.idx = var_field->var.idx;
2124	ref_field->var.hist_data = var_field->hist_data;
2125	ref_field->size = var_field->size;
2126	ref_field->is_signed = var_field->is_signed;
2127	ref_field->flags |= var_field->flags &
2128		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2129
2130	if (system) {
2131		ref_field->system = kstrdup(system, GFP_KERNEL);
2132		if (!ref_field->system)
2133			return -ENOMEM;
2134	}
2135
2136	if (event_name) {
2137		ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2138		if (!ref_field->event_name) {
2139			err = -ENOMEM;
2140			goto free;
2141		}
2142	}
2143
2144	if (var_field->var.name) {
2145		ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2146		if (!ref_field->name) {
2147			err = -ENOMEM;
2148			goto free;
2149		}
2150	} else if (var_field->name) {
2151		ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2152		if (!ref_field->name) {
2153			err = -ENOMEM;
2154			goto free;
2155		}
2156	}
2157
2158	ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2159	if (!ref_field->type) {
2160		err = -ENOMEM;
2161		goto free;
2162	}
2163 out:
2164	return err;
2165 free:
2166	kfree(ref_field->system);
2167	ref_field->system = NULL;
2168	kfree(ref_field->event_name);
2169	ref_field->event_name = NULL;
2170	kfree(ref_field->name);
2171	ref_field->name = NULL;
2172
2173	goto out;
2174}
2175
2176static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2177			    struct hist_field *var_field)
2178{
2179	struct hist_field *ref_field;
2180	int i;
2181
2182	for (i = 0; i < hist_data->n_var_refs; i++) {
2183		ref_field = hist_data->var_refs[i];
2184		if (ref_field->var.idx == var_field->var.idx &&
2185		    ref_field->var.hist_data == var_field->hist_data)
2186			return i;
2187	}
2188
2189	return -ENOENT;
2190}
2191
2192/**
2193 * create_var_ref - Create a variable reference and attach it to trigger
2194 * @hist_data: The trigger that will be referencing the variable
2195 * @var_field: The VAR field to create a reference to
2196 * @system: The optional system string
2197 * @event_name: The optional event_name string
2198 *
2199 * Given a variable hist_field, create a VAR_REF hist_field that
2200 * represents a reference to it.
2201 *
2202 * This function also adds the reference to the trigger that
2203 * now references the variable.
2204 *
2205 * Return: The VAR_REF field if successful, NULL if not
2206 */
2207static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2208					 struct hist_field *var_field,
2209					 char *system, char *event_name)
2210{
2211	unsigned long flags = HIST_FIELD_FL_VAR_REF;
2212	struct hist_field *ref_field;
2213	int i;
2214
2215	/* Check if the variable already exists */
2216	for (i = 0; i < hist_data->n_var_refs; i++) {
2217		ref_field = hist_data->var_refs[i];
2218		if (ref_field->var.idx == var_field->var.idx &&
2219		    ref_field->var.hist_data == var_field->hist_data) {
2220			get_hist_field(ref_field);
2221			return ref_field;
2222		}
2223	}
2224	/* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2225	if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2226		return NULL;
2227	ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2228	if (ref_field) {
2229		if (init_var_ref(ref_field, var_field, system, event_name)) {
2230			destroy_hist_field(ref_field, 0);
2231			return NULL;
2232		}
2233
2234		hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2235		ref_field->var_ref_idx = hist_data->n_var_refs++;
2236	}
2237
2238	return ref_field;
2239}
2240
2241static bool is_var_ref(char *var_name)
2242{
2243	if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2244		return false;
2245
2246	return true;
2247}
2248
2249static char *field_name_from_var(struct hist_trigger_data *hist_data,
2250				 char *var_name)
2251{
2252	char *name, *field;
2253	unsigned int i;
2254
2255	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2256		name = hist_data->attrs->var_defs.name[i];
2257
2258		if (strcmp(var_name, name) == 0) {
2259			field = hist_data->attrs->var_defs.expr[i];
2260			if (contains_operator(field, NULL) || is_var_ref(field))
2261				continue;
2262			return field;
2263		}
2264	}
2265
2266	return NULL;
2267}
2268
2269static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2270				 char *system, char *event_name,
2271				 char *var_name)
2272{
2273	struct trace_event_call *call;
2274
2275	if (system && event_name) {
2276		call = hist_data->event_file->event_call;
2277
2278		if (strcmp(system, call->class->system) != 0)
2279			return NULL;
2280
2281		if (strcmp(event_name, trace_event_name(call)) != 0)
2282			return NULL;
2283	}
2284
2285	if (!!system != !!event_name)
2286		return NULL;
2287
2288	if (!is_var_ref(var_name))
2289		return NULL;
2290
2291	var_name++;
2292
2293	return field_name_from_var(hist_data, var_name);
2294}
2295
2296static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2297					char *system, char *event_name,
2298					char *var_name)
2299{
2300	struct hist_field *var_field = NULL, *ref_field = NULL;
2301	struct trace_array *tr = hist_data->event_file->tr;
2302
2303	if (!is_var_ref(var_name))
2304		return NULL;
2305
2306	var_name++;
2307
2308	var_field = find_event_var(hist_data, system, event_name, var_name);
2309	if (var_field)
2310		ref_field = create_var_ref(hist_data, var_field,
2311					   system, event_name);
2312
2313	if (!ref_field)
2314		hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2315
2316	return ref_field;
2317}
2318
2319static struct ftrace_event_field *
2320parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2321	    char *field_str, unsigned long *flags, unsigned long *buckets)
2322{
2323	struct ftrace_event_field *field = NULL;
2324	char *field_name, *modifier, *str;
2325	struct trace_array *tr = file->tr;
2326
2327	modifier = str = kstrdup(field_str, GFP_KERNEL);
2328	if (!modifier)
2329		return ERR_PTR(-ENOMEM);
2330
2331	field_name = strsep(&modifier, ".");
2332	if (modifier) {
2333		if (strcmp(modifier, "hex") == 0)
2334			*flags |= HIST_FIELD_FL_HEX;
2335		else if (strcmp(modifier, "sym") == 0)
2336			*flags |= HIST_FIELD_FL_SYM;
2337		/*
2338		 * 'sym-offset' occurrences in the trigger string are modified
2339		 * to 'symXoffset' to simplify arithmetic expression parsing.
2340		 */
2341		else if (strcmp(modifier, "symXoffset") == 0)
2342			*flags |= HIST_FIELD_FL_SYM_OFFSET;
2343		else if ((strcmp(modifier, "execname") == 0) &&
2344			 (strcmp(field_name, "common_pid") == 0))
2345			*flags |= HIST_FIELD_FL_EXECNAME;
2346		else if (strcmp(modifier, "syscall") == 0)
2347			*flags |= HIST_FIELD_FL_SYSCALL;
2348		else if (strcmp(modifier, "stacktrace") == 0)
2349			*flags |= HIST_FIELD_FL_STACKTRACE;
2350		else if (strcmp(modifier, "log2") == 0)
2351			*flags |= HIST_FIELD_FL_LOG2;
2352		else if (strcmp(modifier, "usecs") == 0)
2353			*flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2354		else if (strncmp(modifier, "bucket", 6) == 0) {
2355			int ret;
2356
2357			modifier += 6;
2358
2359			if (*modifier == 's')
2360				modifier++;
2361			if (*modifier != '=')
2362				goto error;
2363			modifier++;
2364			ret = kstrtoul(modifier, 0, buckets);
2365			if (ret || !(*buckets))
2366				goto error;
2367			*flags |= HIST_FIELD_FL_BUCKET;
2368		} else if (strncmp(modifier, "percent", 7) == 0) {
2369			if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2370				goto error;
2371			*flags |= HIST_FIELD_FL_PERCENT;
2372		} else if (strncmp(modifier, "graph", 5) == 0) {
2373			if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2374				goto error;
2375			*flags |= HIST_FIELD_FL_GRAPH;
2376		} else {
2377 error:
2378			hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2379			field = ERR_PTR(-EINVAL);
2380			goto out;
2381		}
2382	}
2383
2384	if (strcmp(field_name, "common_timestamp") == 0) {
2385		*flags |= HIST_FIELD_FL_TIMESTAMP;
2386		hist_data->enable_timestamps = true;
2387		if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2388			hist_data->attrs->ts_in_usecs = true;
2389	} else if (strcmp(field_name, "common_stacktrace") == 0) {
2390		*flags |= HIST_FIELD_FL_STACKTRACE;
2391	} else if (strcmp(field_name, "common_cpu") == 0) {
2392		*flags |= HIST_FIELD_FL_CPU;
2393	} else if (strcmp(field_name, "common_comm") == 0) {
2394		*flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING;
2395	} else if (strcmp(field_name, "hitcount") == 0)
2396		*flags |= HIST_FIELD_FL_HITCOUNT;
2397	else {
2398		field = trace_find_event_field(file->event_call, field_name);
2399		if (!field || !field->size) {
2400			/*
2401			 * For backward compatibility, if field_name
2402			 * was "cpu" or "stacktrace", then we treat this
2403			 * the same as common_cpu and common_stacktrace
2404			 * respectively. This also works for "CPU", and
2405			 * "STACKTRACE".
2406			 */
2407			if (field && field->filter_type == FILTER_CPU) {
2408				*flags |= HIST_FIELD_FL_CPU;
2409			} else if (field && field->filter_type == FILTER_STACKTRACE) {
2410				*flags |= HIST_FIELD_FL_STACKTRACE;
2411			} else if (field && field->filter_type == FILTER_COMM) {
2412				*flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING;
2413			} else {
2414				hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2415					 errpos(field_name));
2416				field = ERR_PTR(-EINVAL);
2417				goto out;
2418			}
2419		}
2420	}
2421 out:
2422	kfree(str);
2423
2424	return field;
2425}
2426
2427static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2428				       struct hist_field *var_ref,
2429				       char *var_name)
2430{
2431	struct hist_field *alias = NULL;
2432	unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2433
2434	alias = create_hist_field(hist_data, NULL, flags, var_name);
2435	if (!alias)
2436		return NULL;
2437
2438	alias->fn_num = var_ref->fn_num;
2439	alias->operands[0] = var_ref;
2440
2441	if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2442		destroy_hist_field(alias, 0);
2443		return NULL;
2444	}
2445
2446	alias->var_ref_idx = var_ref->var_ref_idx;
2447
2448	return alias;
2449}
2450
2451static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2452				      char *str, char *var_name,
2453				      unsigned long *flags)
2454{
2455	struct trace_array *tr = hist_data->event_file->tr;
2456	struct hist_field *field = NULL;
2457	u64 constant;
2458
2459	if (kstrtoull(str, 0, &constant)) {
2460		hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2461		return NULL;
2462	}
2463
2464	*flags |= HIST_FIELD_FL_CONST;
2465	field = create_hist_field(hist_data, NULL, *flags, var_name);
2466	if (!field)
2467		return NULL;
2468
2469	field->constant = constant;
2470
2471	return field;
2472}
2473
2474static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2475				     struct trace_event_file *file, char *str,
2476				     unsigned long *flags, char *var_name)
2477{
2478	char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2479	struct ftrace_event_field *field = NULL;
2480	struct hist_field *hist_field = NULL;
2481	unsigned long buckets = 0;
2482	int ret = 0;
2483
2484	if (isdigit(str[0])) {
2485		hist_field = parse_const(hist_data, str, var_name, flags);
2486		if (!hist_field) {
2487			ret = -EINVAL;
2488			goto out;
2489		}
2490		return hist_field;
2491	}
2492
2493	s = strchr(str, '.');
2494	if (s) {
2495		s = strchr(++s, '.');
2496		if (s) {
2497			ref_system = strsep(&str, ".");
2498			if (!str) {
2499				ret = -EINVAL;
2500				goto out;
2501			}
2502			ref_event = strsep(&str, ".");
2503			if (!str) {
2504				ret = -EINVAL;
2505				goto out;
2506			}
2507			ref_var = str;
2508		}
2509	}
2510
2511	s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2512	if (!s) {
2513		hist_field = parse_var_ref(hist_data, ref_system,
2514					   ref_event, ref_var);
2515		if (hist_field) {
2516			if (var_name) {
2517				hist_field = create_alias(hist_data, hist_field, var_name);
2518				if (!hist_field) {
2519					ret = -ENOMEM;
2520					goto out;
2521				}
2522			}
2523			return hist_field;
2524		}
2525	} else
2526		str = s;
2527
2528	field = parse_field(hist_data, file, str, flags, &buckets);
2529	if (IS_ERR(field)) {
2530		ret = PTR_ERR(field);
2531		goto out;
2532	}
2533
2534	hist_field = create_hist_field(hist_data, field, *flags, var_name);
2535	if (!hist_field) {
2536		ret = -ENOMEM;
2537		goto out;
2538	}
2539	hist_field->buckets = buckets;
2540
2541	return hist_field;
2542 out:
2543	return ERR_PTR(ret);
2544}
2545
2546static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2547				     struct trace_event_file *file,
2548				     char *str, unsigned long flags,
2549				     char *var_name, unsigned int *n_subexprs);
2550
2551static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2552				      struct trace_event_file *file,
2553				      char *str, unsigned long flags,
2554				      char *var_name, unsigned int *n_subexprs)
2555{
2556	struct hist_field *operand1, *expr = NULL;
2557	unsigned long operand_flags;
2558	int ret = 0;
2559	char *s;
2560
2561	/* Unary minus operator, increment n_subexprs */
2562	++*n_subexprs;
2563
2564	/* we support only -(xxx) i.e. explicit parens required */
2565
2566	if (*n_subexprs > 3) {
2567		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2568		ret = -EINVAL;
2569		goto free;
2570	}
2571
2572	str++; /* skip leading '-' */
2573
2574	s = strchr(str, '(');
2575	if (s)
2576		str++;
2577	else {
2578		ret = -EINVAL;
2579		goto free;
2580	}
2581
2582	s = strrchr(str, ')');
2583	if (s) {
2584		 /* unary minus not supported in sub-expressions */
2585		if (*(s+1) != '\0') {
2586			hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2587				 errpos(str));
2588			ret = -EINVAL;
2589			goto free;
2590		}
2591		*s = '\0';
2592	}
2593	else {
2594		ret = -EINVAL; /* no closing ')' */
2595		goto free;
2596	}
2597
2598	flags |= HIST_FIELD_FL_EXPR;
2599	expr = create_hist_field(hist_data, NULL, flags, var_name);
2600	if (!expr) {
2601		ret = -ENOMEM;
2602		goto free;
2603	}
2604
2605	operand_flags = 0;
2606	operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2607	if (IS_ERR(operand1)) {
2608		ret = PTR_ERR(operand1);
2609		goto free;
2610	}
2611	if (operand1->flags & HIST_FIELD_FL_STRING) {
2612		/* String type can not be the operand of unary operator. */
2613		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2614		destroy_hist_field(operand1, 0);
2615		ret = -EINVAL;
2616		goto free;
2617	}
2618
2619	expr->flags |= operand1->flags &
2620		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2621	expr->fn_num = HIST_FIELD_FN_UMINUS;
2622	expr->operands[0] = operand1;
2623	expr->size = operand1->size;
2624	expr->is_signed = operand1->is_signed;
2625	expr->operator = FIELD_OP_UNARY_MINUS;
2626	expr->name = expr_str(expr, 0);
2627	expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2628	if (!expr->type) {
2629		ret = -ENOMEM;
2630		goto free;
2631	}
2632
2633	return expr;
2634 free:
2635	destroy_hist_field(expr, 0);
2636	return ERR_PTR(ret);
2637}
2638
2639/*
2640 * If the operands are var refs, return pointers the
2641 * variable(s) referenced in var1 and var2, else NULL.
2642 */
2643static int check_expr_operands(struct trace_array *tr,
2644			       struct hist_field *operand1,
2645			       struct hist_field *operand2,
2646			       struct hist_field **var1,
2647			       struct hist_field **var2)
2648{
2649	unsigned long operand1_flags = operand1->flags;
2650	unsigned long operand2_flags = operand2->flags;
2651
2652	if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2653	    (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2654		struct hist_field *var;
2655
2656		var = find_var_field(operand1->var.hist_data, operand1->name);
2657		if (!var)
2658			return -EINVAL;
2659		operand1_flags = var->flags;
2660		*var1 = var;
2661	}
2662
2663	if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2664	    (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2665		struct hist_field *var;
2666
2667		var = find_var_field(operand2->var.hist_data, operand2->name);
2668		if (!var)
2669			return -EINVAL;
2670		operand2_flags = var->flags;
2671		*var2 = var;
2672	}
2673
2674	if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2675	    (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2676		hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2677		return -EINVAL;
2678	}
2679
2680	return 0;
2681}
2682
2683static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2684				     struct trace_event_file *file,
2685				     char *str, unsigned long flags,
2686				     char *var_name, unsigned int *n_subexprs)
2687{
2688	struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2689	struct hist_field *var1 = NULL, *var2 = NULL;
2690	unsigned long operand_flags, operand2_flags;
2691	int field_op, ret = -EINVAL;
2692	char *sep, *operand1_str;
2693	enum hist_field_fn op_fn;
2694	bool combine_consts;
2695
2696	if (*n_subexprs > 3) {
2697		hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2698		return ERR_PTR(-EINVAL);
2699	}
2700
2701	field_op = contains_operator(str, &sep);
2702
2703	if (field_op == FIELD_OP_NONE)
2704		return parse_atom(hist_data, file, str, &flags, var_name);
2705
2706	if (field_op == FIELD_OP_UNARY_MINUS)
2707		return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2708
2709	/* Binary operator found, increment n_subexprs */
2710	++*n_subexprs;
2711
2712	/* Split the expression string at the root operator */
2713	if (!sep)
2714		return ERR_PTR(-EINVAL);
2715
2716	*sep = '\0';
2717	operand1_str = str;
2718	str = sep+1;
2719
2720	/* Binary operator requires both operands */
2721	if (*operand1_str == '\0' || *str == '\0')
2722		return ERR_PTR(-EINVAL);
2723
2724	operand_flags = 0;
2725
2726	/* LHS of string is an expression e.g. a+b in a+b+c */
2727	operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2728	if (IS_ERR(operand1))
2729		return ERR_CAST(operand1);
2730
2731	if (operand1->flags & HIST_FIELD_FL_STRING) {
2732		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2733		ret = -EINVAL;
2734		goto free_op1;
2735	}
2736
2737	/* RHS of string is another expression e.g. c in a+b+c */
2738	operand_flags = 0;
2739	operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2740	if (IS_ERR(operand2)) {
2741		ret = PTR_ERR(operand2);
2742		goto free_op1;
2743	}
2744	if (operand2->flags & HIST_FIELD_FL_STRING) {
2745		hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2746		ret = -EINVAL;
2747		goto free_operands;
2748	}
2749
2750	switch (field_op) {
2751	case FIELD_OP_MINUS:
2752		op_fn = HIST_FIELD_FN_MINUS;
2753		break;
2754	case FIELD_OP_PLUS:
2755		op_fn = HIST_FIELD_FN_PLUS;
2756		break;
2757	case FIELD_OP_DIV:
2758		op_fn = HIST_FIELD_FN_DIV;
2759		break;
2760	case FIELD_OP_MULT:
2761		op_fn = HIST_FIELD_FN_MULT;
2762		break;
2763	default:
2764		ret = -EINVAL;
2765		goto free_operands;
2766	}
2767
2768	ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2769	if (ret)
2770		goto free_operands;
2771
2772	operand_flags = var1 ? var1->flags : operand1->flags;
2773	operand2_flags = var2 ? var2->flags : operand2->flags;
2774
2775	/*
2776	 * If both operands are constant, the expression can be
2777	 * collapsed to a single constant.
2778	 */
2779	combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2780
2781	flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2782
2783	flags |= operand1->flags &
2784		(HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2785
2786	expr = create_hist_field(hist_data, NULL, flags, var_name);
2787	if (!expr) {
2788		ret = -ENOMEM;
2789		goto free_operands;
2790	}
2791
2792	operand1->read_once = true;
2793	operand2->read_once = true;
2794
2795	/* The operands are now owned and free'd by 'expr' */
2796	expr->operands[0] = operand1;
2797	expr->operands[1] = operand2;
2798
2799	if (field_op == FIELD_OP_DIV &&
2800			operand2_flags & HIST_FIELD_FL_CONST) {
2801		u64 divisor = var2 ? var2->constant : operand2->constant;
2802
2803		if (!divisor) {
2804			hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2805			ret = -EDOM;
2806			goto free_expr;
2807		}
2808
2809		/*
2810		 * Copy the divisor here so we don't have to look it up
2811		 * later if this is a var ref
2812		 */
2813		operand2->constant = divisor;
2814		op_fn = hist_field_get_div_fn(operand2);
2815	}
2816
2817	expr->fn_num = op_fn;
2818
2819	if (combine_consts) {
2820		if (var1)
2821			expr->operands[0] = var1;
2822		if (var2)
2823			expr->operands[1] = var2;
2824
2825		expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2826		expr->fn_num = HIST_FIELD_FN_CONST;
2827
2828		expr->operands[0] = NULL;
2829		expr->operands[1] = NULL;
2830
2831		/*
2832		 * var refs won't be destroyed immediately
2833		 * See: destroy_hist_field()
2834		 */
2835		destroy_hist_field(operand2, 0);
2836		destroy_hist_field(operand1, 0);
2837
2838		expr->name = expr_str(expr, 0);
2839	} else {
2840		/* The operand sizes should be the same, so just pick one */
2841		expr->size = operand1->size;
2842		expr->is_signed = operand1->is_signed;
2843
2844		expr->operator = field_op;
2845		expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2846		if (!expr->type) {
2847			ret = -ENOMEM;
2848			goto free_expr;
2849		}
2850
2851		expr->name = expr_str(expr, 0);
2852	}
2853
2854	return expr;
2855
2856free_operands:
2857	destroy_hist_field(operand2, 0);
2858free_op1:
2859	destroy_hist_field(operand1, 0);
2860	return ERR_PTR(ret);
2861
2862free_expr:
2863	destroy_hist_field(expr, 0);
2864	return ERR_PTR(ret);
2865}
2866
2867static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2868				 struct trace_event_file *file)
2869{
2870	struct event_trigger_data *test;
2871
2872	lockdep_assert_held(&event_mutex);
2873
2874	list_for_each_entry(test, &file->triggers, list) {
2875		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2876			if (test->private_data == hist_data)
2877				return test->filter_str;
2878		}
2879	}
2880
2881	return NULL;
2882}
2883
2884static struct event_command trigger_hist_cmd;
2885static int event_hist_trigger_parse(struct event_command *cmd_ops,
2886				    struct trace_event_file *file,
2887				    char *glob, char *cmd,
2888				    char *param_and_filter);
2889
2890static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2891			    struct hist_trigger_data *hist_data,
2892			    unsigned int n_keys)
2893{
2894	struct hist_field *target_hist_field, *hist_field;
2895	unsigned int n, i, j;
2896
2897	if (hist_data->n_fields - hist_data->n_vals != n_keys)
2898		return false;
2899
2900	i = hist_data->n_vals;
2901	j = target_hist_data->n_vals;
2902
2903	for (n = 0; n < n_keys; n++) {
2904		hist_field = hist_data->fields[i + n];
2905		target_hist_field = target_hist_data->fields[j + n];
2906
2907		if (strcmp(hist_field->type, target_hist_field->type) != 0)
2908			return false;
2909		if (hist_field->size != target_hist_field->size)
2910			return false;
2911		if (hist_field->is_signed != target_hist_field->is_signed)
2912			return false;
2913	}
2914
2915	return true;
2916}
2917
2918static struct hist_trigger_data *
2919find_compatible_hist(struct hist_trigger_data *target_hist_data,
2920		     struct trace_event_file *file)
2921{
2922	struct hist_trigger_data *hist_data;
2923	struct event_trigger_data *test;
2924	unsigned int n_keys;
2925
2926	lockdep_assert_held(&event_mutex);
2927
2928	n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2929
2930	list_for_each_entry(test, &file->triggers, list) {
2931		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2932			hist_data = test->private_data;
2933
2934			if (compatible_keys(target_hist_data, hist_data, n_keys))
2935				return hist_data;
2936		}
2937	}
2938
2939	return NULL;
2940}
2941
2942static struct trace_event_file *event_file(struct trace_array *tr,
2943					   char *system, char *event_name)
2944{
2945	struct trace_event_file *file;
2946
2947	file = __find_event_file(tr, system, event_name);
2948	if (!file)
2949		return ERR_PTR(-EINVAL);
2950
2951	return file;
2952}
2953
2954static struct hist_field *
2955find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2956			 char *system, char *event_name, char *field_name)
2957{
2958	struct hist_field *event_var;
2959	char *synthetic_name;
2960
2961	synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2962	if (!synthetic_name)
2963		return ERR_PTR(-ENOMEM);
2964
2965	strcpy(synthetic_name, "synthetic_");
2966	strcat(synthetic_name, field_name);
2967
2968	event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2969
2970	kfree(synthetic_name);
2971
2972	return event_var;
2973}
2974
2975/**
2976 * create_field_var_hist - Automatically create a histogram and var for a field
2977 * @target_hist_data: The target hist trigger
2978 * @subsys_name: Optional subsystem name
2979 * @event_name: Optional event name
2980 * @field_name: The name of the field (and the resulting variable)
2981 *
2982 * Hist trigger actions fetch data from variables, not directly from
2983 * events.  However, for convenience, users are allowed to directly
2984 * specify an event field in an action, which will be automatically
2985 * converted into a variable on their behalf.
2986 *
2987 * If a user specifies a field on an event that isn't the event the
2988 * histogram currently being defined (the target event histogram), the
2989 * only way that can be accomplished is if a new hist trigger is
2990 * created and the field variable defined on that.
2991 *
2992 * This function creates a new histogram compatible with the target
2993 * event (meaning a histogram with the same key as the target
2994 * histogram), and creates a variable for the specified field, but
2995 * with 'synthetic_' prepended to the variable name in order to avoid
2996 * collision with normal field variables.
2997 *
2998 * Return: The variable created for the field.
2999 */
3000static struct hist_field *
3001create_field_var_hist(struct hist_trigger_data *target_hist_data,
3002		      char *subsys_name, char *event_name, char *field_name)
3003{
3004	struct trace_array *tr = target_hist_data->event_file->tr;
3005	struct hist_trigger_data *hist_data;
3006	unsigned int i, n, first = true;
3007	struct field_var_hist *var_hist;
3008	struct trace_event_file *file;
3009	struct hist_field *key_field;
3010	struct hist_field *event_var;
3011	char *saved_filter;
3012	char *cmd;
3013	int ret;
3014
3015	if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3016		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3017		return ERR_PTR(-EINVAL);
3018	}
3019
3020	file = event_file(tr, subsys_name, event_name);
3021
3022	if (IS_ERR(file)) {
3023		hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
3024		ret = PTR_ERR(file);
3025		return ERR_PTR(ret);
3026	}
3027
3028	/*
3029	 * Look for a histogram compatible with target.  We'll use the
3030	 * found histogram specification to create a new matching
3031	 * histogram with our variable on it.  target_hist_data is not
3032	 * yet a registered histogram so we can't use that.
3033	 */
3034	hist_data = find_compatible_hist(target_hist_data, file);
3035	if (!hist_data) {
3036		hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3037		return ERR_PTR(-EINVAL);
3038	}
3039
3040	/* See if a synthetic field variable has already been created */
3041	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3042					     event_name, field_name);
3043	if (!IS_ERR_OR_NULL(event_var))
3044		return event_var;
3045
3046	var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3047	if (!var_hist)
3048		return ERR_PTR(-ENOMEM);
3049
3050	cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3051	if (!cmd) {
3052		kfree(var_hist);
3053		return ERR_PTR(-ENOMEM);
3054	}
3055
3056	/* Use the same keys as the compatible histogram */
3057	strcat(cmd, "keys=");
3058
3059	for_each_hist_key_field(i, hist_data) {
3060		key_field = hist_data->fields[i];
3061		if (!first)
3062			strcat(cmd, ",");
3063		strcat(cmd, key_field->field->name);
3064		first = false;
3065	}
3066
3067	/* Create the synthetic field variable specification */
3068	strcat(cmd, ":synthetic_");
3069	strcat(cmd, field_name);
3070	strcat(cmd, "=");
3071	strcat(cmd, field_name);
3072
3073	/* Use the same filter as the compatible histogram */
3074	saved_filter = find_trigger_filter(hist_data, file);
3075	if (saved_filter) {
3076		strcat(cmd, " if ");
3077		strcat(cmd, saved_filter);
3078	}
3079
3080	var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3081	if (!var_hist->cmd) {
3082		kfree(cmd);
3083		kfree(var_hist);
3084		return ERR_PTR(-ENOMEM);
3085	}
3086
3087	/* Save the compatible histogram information */
3088	var_hist->hist_data = hist_data;
3089
3090	/* Create the new histogram with our variable */
3091	ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3092				       "", "hist", cmd);
3093	if (ret) {
3094		kfree(cmd);
3095		kfree(var_hist->cmd);
3096		kfree(var_hist);
3097		hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3098		return ERR_PTR(ret);
3099	}
3100
3101	kfree(cmd);
3102
3103	/* If we can't find the variable, something went wrong */
3104	event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3105					     event_name, field_name);
3106	if (IS_ERR_OR_NULL(event_var)) {
3107		kfree(var_hist->cmd);
3108		kfree(var_hist);
3109		hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3110		return ERR_PTR(-EINVAL);
3111	}
3112
3113	n = target_hist_data->n_field_var_hists;
3114	target_hist_data->field_var_hists[n] = var_hist;
3115	target_hist_data->n_field_var_hists++;
3116
3117	return event_var;
3118}
3119
3120static struct hist_field *
3121find_target_event_var(struct hist_trigger_data *hist_data,
3122		      char *subsys_name, char *event_name, char *var_name)
3123{
3124	struct trace_event_file *file = hist_data->event_file;
3125	struct hist_field *hist_field = NULL;
3126
3127	if (subsys_name) {
3128		struct trace_event_call *call;
3129
3130		if (!event_name)
3131			return NULL;
3132
3133		call = file->event_call;
3134
3135		if (strcmp(subsys_name, call->class->system) != 0)
3136			return NULL;
3137
3138		if (strcmp(event_name, trace_event_name(call)) != 0)
3139			return NULL;
3140	}
3141
3142	hist_field = find_var_field(hist_data, var_name);
3143
3144	return hist_field;
3145}
3146
3147static inline void __update_field_vars(struct tracing_map_elt *elt,
3148				       struct trace_buffer *buffer,
3149				       struct ring_buffer_event *rbe,
3150				       void *rec,
3151				       struct field_var **field_vars,
3152				       unsigned int n_field_vars,
3153				       unsigned int field_var_str_start)
3154{
3155	struct hist_elt_data *elt_data = elt->private_data;
3156	unsigned int i, j, var_idx;
3157	u64 var_val;
3158
3159	/* Make sure stacktrace can fit in the string variable length */
3160	BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX);
3161
3162	for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3163		struct field_var *field_var = field_vars[i];
3164		struct hist_field *var = field_var->var;
3165		struct hist_field *val = field_var->val;
3166
3167		var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3168		var_idx = var->var.idx;
3169
3170		if (val->flags & (HIST_FIELD_FL_STRING |
3171				  HIST_FIELD_FL_STACKTRACE)) {
3172			char *str = elt_data->field_var_str[j++];
3173			char *val_str = (char *)(uintptr_t)var_val;
3174			unsigned int size;
3175
3176			if (val->flags & HIST_FIELD_FL_STRING) {
3177				size = min(val->size, STR_VAR_LEN_MAX);
3178				strscpy(str, val_str, size);
3179			} else {
3180				char *stack_start = str + sizeof(unsigned long);
3181				int e;
3182
3183				e = stack_trace_save((void *)stack_start,
3184						     HIST_STACKTRACE_DEPTH,
3185						     HIST_STACKTRACE_SKIP);
3186				if (e < HIST_STACKTRACE_DEPTH - 1)
3187					((unsigned long *)stack_start)[e] = 0;
3188				*((unsigned long *)str) = e;
3189			}
3190			var_val = (u64)(uintptr_t)str;
3191		}
3192		tracing_map_set_var(elt, var_idx, var_val);
3193	}
3194}
3195
3196static void update_field_vars(struct hist_trigger_data *hist_data,
3197			      struct tracing_map_elt *elt,
3198			      struct trace_buffer *buffer,
3199			      struct ring_buffer_event *rbe,
3200			      void *rec)
3201{
3202	__update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3203			    hist_data->n_field_vars, 0);
3204}
3205
3206static void save_track_data_vars(struct hist_trigger_data *hist_data,
3207				 struct tracing_map_elt *elt,
3208				 struct trace_buffer *buffer,  void *rec,
3209				 struct ring_buffer_event *rbe, void *key,
3210				 struct action_data *data, u64 *var_ref_vals)
3211{
3212	__update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3213			    hist_data->n_save_vars, hist_data->n_field_var_str);
3214}
3215
3216static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3217				     struct trace_event_file *file,
3218				     char *name, int size, const char *type)
3219{
3220	struct hist_field *var;
3221	int idx;
3222
3223	if (find_var(hist_data, file, name) && !hist_data->remove) {
3224		var = ERR_PTR(-EINVAL);
3225		goto out;
3226	}
3227
3228	var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3229	if (!var) {
3230		var = ERR_PTR(-ENOMEM);
3231		goto out;
3232	}
3233
3234	idx = tracing_map_add_var(hist_data->map);
3235	if (idx < 0) {
3236		kfree(var);
3237		var = ERR_PTR(-EINVAL);
3238		goto out;
3239	}
3240
3241	var->ref = 1;
3242	var->flags = HIST_FIELD_FL_VAR;
3243	var->var.idx = idx;
3244	var->var.hist_data = var->hist_data = hist_data;
3245	var->size = size;
3246	var->var.name = kstrdup(name, GFP_KERNEL);
3247	var->type = kstrdup_const(type, GFP_KERNEL);
3248	if (!var->var.name || !var->type) {
3249		kfree_const(var->type);
3250		kfree(var->var.name);
3251		kfree(var);
3252		var = ERR_PTR(-ENOMEM);
3253	}
3254 out:
3255	return var;
3256}
3257
3258static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3259					  struct trace_event_file *file,
3260					  char *field_name)
3261{
3262	struct hist_field *val = NULL, *var = NULL;
3263	unsigned long flags = HIST_FIELD_FL_VAR;
3264	struct trace_array *tr = file->tr;
3265	struct field_var *field_var;
3266	int ret = 0;
3267
3268	if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3269		hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3270		ret = -EINVAL;
3271		goto err;
3272	}
3273
3274	val = parse_atom(hist_data, file, field_name, &flags, NULL);
3275	if (IS_ERR(val)) {
3276		hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3277		ret = PTR_ERR(val);
3278		goto err;
3279	}
3280
3281	var = create_var(hist_data, file, field_name, val->size, val->type);
3282	if (IS_ERR(var)) {
3283		hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3284		destroy_hist_field(val, 0);
3285		ret = PTR_ERR(var);
3286		goto err;
3287	}
3288
3289	field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3290	if (!field_var) {
3291		destroy_hist_field(val, 0);
3292		kfree_const(var->type);
3293		kfree(var->var.name);
3294		kfree(var);
3295		ret =  -ENOMEM;
3296		goto err;
3297	}
3298
3299	field_var->var = var;
3300	field_var->val = val;
3301 out:
3302	return field_var;
3303 err:
3304	field_var = ERR_PTR(ret);
3305	goto out;
3306}
3307
3308/**
3309 * create_target_field_var - Automatically create a variable for a field
3310 * @target_hist_data: The target hist trigger
3311 * @subsys_name: Optional subsystem name
3312 * @event_name: Optional event name
3313 * @var_name: The name of the field (and the resulting variable)
3314 *
3315 * Hist trigger actions fetch data from variables, not directly from
3316 * events.  However, for convenience, users are allowed to directly
3317 * specify an event field in an action, which will be automatically
3318 * converted into a variable on their behalf.
3319 *
3320 * This function creates a field variable with the name var_name on
3321 * the hist trigger currently being defined on the target event.  If
3322 * subsys_name and event_name are specified, this function simply
3323 * verifies that they do in fact match the target event subsystem and
3324 * event name.
3325 *
3326 * Return: The variable created for the field.
3327 */
3328static struct field_var *
3329create_target_field_var(struct hist_trigger_data *target_hist_data,
3330			char *subsys_name, char *event_name, char *var_name)
3331{
3332	struct trace_event_file *file = target_hist_data->event_file;
3333
3334	if (subsys_name) {
3335		struct trace_event_call *call;
3336
3337		if (!event_name)
3338			return NULL;
3339
3340		call = file->event_call;
3341
3342		if (strcmp(subsys_name, call->class->system) != 0)
3343			return NULL;
3344
3345		if (strcmp(event_name, trace_event_name(call)) != 0)
3346			return NULL;
3347	}
3348
3349	return create_field_var(target_hist_data, file, var_name);
3350}
3351
3352static bool check_track_val_max(u64 track_val, u64 var_val)
3353{
3354	if (var_val <= track_val)
3355		return false;
3356
3357	return true;
3358}
3359
3360static bool check_track_val_changed(u64 track_val, u64 var_val)
3361{
3362	if (var_val == track_val)
3363		return false;
3364
3365	return true;
3366}
3367
3368static u64 get_track_val(struct hist_trigger_data *hist_data,
3369			 struct tracing_map_elt *elt,
3370			 struct action_data *data)
3371{
3372	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3373	u64 track_val;
3374
3375	track_val = tracing_map_read_var(elt, track_var_idx);
3376
3377	return track_val;
3378}
3379
3380static void save_track_val(struct hist_trigger_data *hist_data,
3381			   struct tracing_map_elt *elt,
3382			   struct action_data *data, u64 var_val)
3383{
3384	unsigned int track_var_idx = data->track_data.track_var->var.idx;
3385
3386	tracing_map_set_var(elt, track_var_idx, var_val);
3387}
3388
3389static void save_track_data(struct hist_trigger_data *hist_data,
3390			    struct tracing_map_elt *elt,
3391			    struct trace_buffer *buffer, void *rec,
3392			    struct ring_buffer_event *rbe, void *key,
3393			    struct action_data *data, u64 *var_ref_vals)
3394{
3395	if (data->track_data.save_data)
3396		data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3397					   key, data, var_ref_vals);
3398}
3399
3400static bool check_track_val(struct tracing_map_elt *elt,
3401			    struct action_data *data,
3402			    u64 var_val)
3403{
3404	struct hist_trigger_data *hist_data;
3405	u64 track_val;
3406
3407	hist_data = data->track_data.track_var->hist_data;
3408	track_val = get_track_val(hist_data, elt, data);
3409
3410	return data->track_data.check_val(track_val, var_val);
3411}
3412
3413#ifdef CONFIG_TRACER_SNAPSHOT
3414static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3415{
3416	/* called with tr->max_lock held */
3417	struct track_data *track_data = tr->cond_snapshot->cond_data;
3418	struct hist_elt_data *elt_data, *track_elt_data;
3419	struct snapshot_context *context = cond_data;
3420	struct action_data *action;
3421	u64 track_val;
3422
3423	if (!track_data)
3424		return false;
3425
3426	action = track_data->action_data;
3427
3428	track_val = get_track_val(track_data->hist_data, context->elt,
3429				  track_data->action_data);
3430
3431	if (!action->track_data.check_val(track_data->track_val, track_val))
3432		return false;
3433
3434	track_data->track_val = track_val;
3435	memcpy(track_data->key, context->key, track_data->key_len);
3436
3437	elt_data = context->elt->private_data;
3438	track_elt_data = track_data->elt.private_data;
3439	if (elt_data->comm)
3440		strscpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3441
3442	track_data->updated = true;
3443
3444	return true;
3445}
3446
3447static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3448				     struct tracing_map_elt *elt,
3449				     struct trace_buffer *buffer, void *rec,
3450				     struct ring_buffer_event *rbe, void *key,
3451				     struct action_data *data,
3452				     u64 *var_ref_vals)
3453{
3454	struct trace_event_file *file = hist_data->event_file;
3455	struct snapshot_context context;
3456
3457	context.elt = elt;
3458	context.key = key;
3459
3460	tracing_snapshot_cond(file->tr, &context);
3461}
3462
3463static void hist_trigger_print_key(struct seq_file *m,
3464				   struct hist_trigger_data *hist_data,
3465				   void *key,
3466				   struct tracing_map_elt *elt);
3467
3468static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3469{
3470	unsigned int i;
3471
3472	if (!hist_data->n_actions)
3473		return NULL;
3474
3475	for (i = 0; i < hist_data->n_actions; i++) {
3476		struct action_data *data = hist_data->actions[i];
3477
3478		if (data->action == ACTION_SNAPSHOT)
3479			return data;
3480	}
3481
3482	return NULL;
3483}
3484
3485static void track_data_snapshot_print(struct seq_file *m,
3486				      struct hist_trigger_data *hist_data)
3487{
3488	struct trace_event_file *file = hist_data->event_file;
3489	struct track_data *track_data;
3490	struct action_data *action;
3491
3492	track_data = tracing_cond_snapshot_data(file->tr);
3493	if (!track_data)
3494		return;
3495
3496	if (!track_data->updated)
3497		return;
3498
3499	action = snapshot_action(hist_data);
3500	if (!action)
3501		return;
3502
3503	seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3504	seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3505		   action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3506		   action->track_data.var_str, track_data->track_val);
3507
3508	seq_puts(m, "\ttriggered by event with key: ");
3509	hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3510	seq_putc(m, '\n');
3511}
3512#else
3513static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3514{
3515	return false;
3516}
3517static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3518				     struct tracing_map_elt *elt,
3519				     struct trace_buffer *buffer, void *rec,
3520				     struct ring_buffer_event *rbe, void *key,
3521				     struct action_data *data,
3522				     u64 *var_ref_vals) {}
3523static void track_data_snapshot_print(struct seq_file *m,
3524				      struct hist_trigger_data *hist_data) {}
3525#endif /* CONFIG_TRACER_SNAPSHOT */
3526
3527static void track_data_print(struct seq_file *m,
3528			     struct hist_trigger_data *hist_data,
3529			     struct tracing_map_elt *elt,
3530			     struct action_data *data)
3531{
3532	u64 track_val = get_track_val(hist_data, elt, data);
3533	unsigned int i, save_var_idx;
3534
3535	if (data->handler == HANDLER_ONMAX)
3536		seq_printf(m, "\n\tmax: %10llu", track_val);
3537	else if (data->handler == HANDLER_ONCHANGE)
3538		seq_printf(m, "\n\tchanged: %10llu", track_val);
3539
3540	if (data->action == ACTION_SNAPSHOT)
3541		return;
3542
3543	for (i = 0; i < hist_data->n_save_vars; i++) {
3544		struct hist_field *save_val = hist_data->save_vars[i]->val;
3545		struct hist_field *save_var = hist_data->save_vars[i]->var;
3546		u64 val;
3547
3548		save_var_idx = save_var->var.idx;
3549
3550		val = tracing_map_read_var(elt, save_var_idx);
3551
3552		if (save_val->flags & HIST_FIELD_FL_STRING) {
3553			seq_printf(m, "  %s: %-32s", save_var->var.name,
3554				   (char *)(uintptr_t)(val));
3555		} else
3556			seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3557	}
3558}
3559
3560static void ontrack_action(struct hist_trigger_data *hist_data,
3561			   struct tracing_map_elt *elt,
3562			   struct trace_buffer *buffer, void *rec,
3563			   struct ring_buffer_event *rbe, void *key,
3564			   struct action_data *data, u64 *var_ref_vals)
3565{
3566	u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3567
3568	if (check_track_val(elt, data, var_val)) {
3569		save_track_val(hist_data, elt, data, var_val);
3570		save_track_data(hist_data, elt, buffer, rec, rbe,
3571				key, data, var_ref_vals);
3572	}
3573}
3574
3575static void action_data_destroy(struct action_data *data)
3576{
3577	unsigned int i;
3578
3579	lockdep_assert_held(&event_mutex);
3580
3581	kfree(data->action_name);
3582
3583	for (i = 0; i < data->n_params; i++)
3584		kfree(data->params[i]);
3585
3586	if (data->synth_event)
3587		data->synth_event->ref--;
3588
3589	kfree(data->synth_event_name);
3590
3591	kfree(data);
3592}
3593
3594static void track_data_destroy(struct hist_trigger_data *hist_data,
3595			       struct action_data *data)
3596{
3597	struct trace_event_file *file = hist_data->event_file;
3598
3599	destroy_hist_field(data->track_data.track_var, 0);
3600
3601	if (data->action == ACTION_SNAPSHOT) {
3602		struct track_data *track_data;
3603
3604		track_data = tracing_cond_snapshot_data(file->tr);
3605		if (track_data && track_data->hist_data == hist_data) {
3606			tracing_snapshot_cond_disable(file->tr);
3607			track_data_free(track_data);
3608		}
3609	}
3610
3611	kfree(data->track_data.var_str);
3612
3613	action_data_destroy(data);
3614}
3615
3616static int action_create(struct hist_trigger_data *hist_data,
3617			 struct action_data *data);
3618
3619static int track_data_create(struct hist_trigger_data *hist_data,
3620			     struct action_data *data)
3621{
3622	struct hist_field *var_field, *ref_field, *track_var = NULL;
3623	struct trace_event_file *file = hist_data->event_file;
3624	struct trace_array *tr = file->tr;
3625	char *track_data_var_str;
3626	int ret = 0;
3627
3628	track_data_var_str = data->track_data.var_str;
3629	if (track_data_var_str[0] != '$') {
3630		hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3631		return -EINVAL;
3632	}
3633	track_data_var_str++;
3634
3635	var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3636	if (!var_field) {
3637		hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3638		return -EINVAL;
3639	}
3640
3641	ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3642	if (!ref_field)
3643		return -ENOMEM;
3644
3645	data->track_data.var_ref = ref_field;
3646
3647	if (data->handler == HANDLER_ONMAX)
3648		track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3649	if (IS_ERR(track_var)) {
3650		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3651		ret = PTR_ERR(track_var);
3652		goto out;
3653	}
3654
3655	if (data->handler == HANDLER_ONCHANGE)
3656		track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3657	if (IS_ERR(track_var)) {
3658		hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3659		ret = PTR_ERR(track_var);
3660		goto out;
3661	}
3662	data->track_data.track_var = track_var;
3663
3664	ret = action_create(hist_data, data);
3665 out:
3666	return ret;
3667}
3668
3669static int parse_action_params(struct trace_array *tr, char *params,
3670			       struct action_data *data)
3671{
3672	char *param, *saved_param;
3673	bool first_param = true;
3674	int ret = 0;
3675
3676	while (params) {
3677		if (data->n_params >= SYNTH_FIELDS_MAX) {
3678			hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3679			ret = -EINVAL;
3680			goto out;
3681		}
3682
3683		param = strsep(&params, ",");
3684		if (!param) {
3685			hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3686			ret = -EINVAL;
3687			goto out;
3688		}
3689
3690		param = strstrip(param);
3691		if (strlen(param) < 2) {
3692			hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3693			ret = -EINVAL;
3694			goto out;
3695		}
3696
3697		saved_param = kstrdup(param, GFP_KERNEL);
3698		if (!saved_param) {
3699			ret = -ENOMEM;
3700			goto out;
3701		}
3702
3703		if (first_param && data->use_trace_keyword) {
3704			data->synth_event_name = saved_param;
3705			first_param = false;
3706			continue;
3707		}
3708		first_param = false;
3709
3710		data->params[data->n_params++] = saved_param;
3711	}
3712 out:
3713	return ret;
3714}
3715
3716static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3717			enum handler_id handler)
3718{
3719	char *action_name;
3720	int ret = 0;
3721
3722	strsep(&str, ".");
3723	if (!str) {
3724		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3725		ret = -EINVAL;
3726		goto out;
3727	}
3728
3729	action_name = strsep(&str, "(");
3730	if (!action_name || !str) {
3731		hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3732		ret = -EINVAL;
3733		goto out;
3734	}
3735
3736	if (str_has_prefix(action_name, "save")) {
3737		char *params = strsep(&str, ")");
3738
3739		if (!params) {
3740			hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3741			ret = -EINVAL;
3742			goto out;
3743		}
3744
3745		ret = parse_action_params(tr, params, data);
3746		if (ret)
3747			goto out;
3748
3749		if (handler == HANDLER_ONMAX)
3750			data->track_data.check_val = check_track_val_max;
3751		else if (handler == HANDLER_ONCHANGE)
3752			data->track_data.check_val = check_track_val_changed;
3753		else {
3754			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3755			ret = -EINVAL;
3756			goto out;
3757		}
3758
3759		data->track_data.save_data = save_track_data_vars;
3760		data->fn = ontrack_action;
3761		data->action = ACTION_SAVE;
3762	} else if (str_has_prefix(action_name, "snapshot")) {
3763		char *params = strsep(&str, ")");
3764
3765		if (!str) {
3766			hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3767			ret = -EINVAL;
3768			goto out;
3769		}
3770
3771		if (handler == HANDLER_ONMAX)
3772			data->track_data.check_val = check_track_val_max;
3773		else if (handler == HANDLER_ONCHANGE)
3774			data->track_data.check_val = check_track_val_changed;
3775		else {
3776			hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3777			ret = -EINVAL;
3778			goto out;
3779		}
3780
3781		data->track_data.save_data = save_track_data_snapshot;
3782		data->fn = ontrack_action;
3783		data->action = ACTION_SNAPSHOT;
3784	} else {
3785		char *params = strsep(&str, ")");
3786
3787		if (str_has_prefix(action_name, "trace"))
3788			data->use_trace_keyword = true;
3789
3790		if (params) {
3791			ret = parse_action_params(tr, params, data);
3792			if (ret)
3793				goto out;
3794		}
3795
3796		if (handler == HANDLER_ONMAX)
3797			data->track_data.check_val = check_track_val_max;
3798		else if (handler == HANDLER_ONCHANGE)
3799			data->track_data.check_val = check_track_val_changed;
3800
3801		if (handler != HANDLER_ONMATCH) {
3802			data->track_data.save_data = action_trace;
3803			data->fn = ontrack_action;
3804		} else
3805			data->fn = action_trace;
3806
3807		data->action = ACTION_TRACE;
3808	}
3809
3810	data->action_name = kstrdup(action_name, GFP_KERNEL);
3811	if (!data->action_name) {
3812		ret = -ENOMEM;
3813		goto out;
3814	}
3815
3816	data->handler = handler;
3817 out:
3818	return ret;
3819}
3820
3821static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3822					    char *str, enum handler_id handler)
3823{
3824	struct action_data *data;
3825	int ret = -EINVAL;
3826	char *var_str;
3827
3828	data = kzalloc(sizeof(*data), GFP_KERNEL);
3829	if (!data)
3830		return ERR_PTR(-ENOMEM);
3831
3832	var_str = strsep(&str, ")");
3833	if (!var_str || !str) {
3834		ret = -EINVAL;
3835		goto free;
3836	}
3837
3838	data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3839	if (!data->track_data.var_str) {
3840		ret = -ENOMEM;
3841		goto free;
3842	}
3843
3844	ret = action_parse(hist_data->event_file->tr, str, data, handler);
3845	if (ret)
3846		goto free;
3847 out:
3848	return data;
3849 free:
3850	track_data_destroy(hist_data, data);
3851	data = ERR_PTR(ret);
3852	goto out;
3853}
3854
3855static void onmatch_destroy(struct action_data *data)
3856{
3857	kfree(data->match_data.event);
3858	kfree(data->match_data.event_system);
3859
3860	action_data_destroy(data);
3861}
3862
3863static void destroy_field_var(struct field_var *field_var)
3864{
3865	if (!field_var)
3866		return;
3867
3868	destroy_hist_field(field_var->var, 0);
3869	destroy_hist_field(field_var->val, 0);
3870
3871	kfree(field_var);
3872}
3873
3874static void destroy_field_vars(struct hist_trigger_data *hist_data)
3875{
3876	unsigned int i;
3877
3878	for (i = 0; i < hist_data->n_field_vars; i++)
3879		destroy_field_var(hist_data->field_vars[i]);
3880
3881	for (i = 0; i < hist_data->n_save_vars; i++)
3882		destroy_field_var(hist_data->save_vars[i]);
3883}
3884
3885static void save_field_var(struct hist_trigger_data *hist_data,
3886			   struct field_var *field_var)
3887{
3888	hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3889
3890	/* Stack traces are saved in the string storage too */
3891	if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3892		hist_data->n_field_var_str++;
3893}
3894
3895
3896static int check_synth_field(struct synth_event *event,
3897			     struct hist_field *hist_field,
3898			     unsigned int field_pos)
3899{
3900	struct synth_field *field;
3901
3902	if (field_pos >= event->n_fields)
3903		return -EINVAL;
3904
3905	field = event->fields[field_pos];
3906
3907	/*
3908	 * A dynamic string synth field can accept static or
3909	 * dynamic. A static string synth field can only accept a
3910	 * same-sized static string, which is checked for later.
3911	 */
3912	if (strstr(hist_field->type, "char[") && field->is_string
3913	    && field->is_dynamic)
3914		return 0;
3915
3916	if (strstr(hist_field->type, "long[") && field->is_stack)
3917		return 0;
3918
3919	if (strcmp(field->type, hist_field->type) != 0) {
3920		if (field->size != hist_field->size ||
3921		    (!field->is_string && field->is_signed != hist_field->is_signed))
3922			return -EINVAL;
3923	}
3924
3925	return 0;
3926}
3927
3928static struct hist_field *
3929trace_action_find_var(struct hist_trigger_data *hist_data,
3930		      struct action_data *data,
3931		      char *system, char *event, char *var)
3932{
3933	struct trace_array *tr = hist_data->event_file->tr;
3934	struct hist_field *hist_field;
3935
3936	var++; /* skip '$' */
3937
3938	hist_field = find_target_event_var(hist_data, system, event, var);
3939	if (!hist_field) {
3940		if (!system && data->handler == HANDLER_ONMATCH) {
3941			system = data->match_data.event_system;
3942			event = data->match_data.event;
3943		}
3944
3945		hist_field = find_event_var(hist_data, system, event, var);
3946	}
3947
3948	if (!hist_field)
3949		hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3950
3951	return hist_field;
3952}
3953
3954static struct hist_field *
3955trace_action_create_field_var(struct hist_trigger_data *hist_data,
3956			      struct action_data *data, char *system,
3957			      char *event, char *var)
3958{
3959	struct hist_field *hist_field = NULL;
3960	struct field_var *field_var;
3961
3962	/*
3963	 * First try to create a field var on the target event (the
3964	 * currently being defined).  This will create a variable for
3965	 * unqualified fields on the target event, or if qualified,
3966	 * target fields that have qualified names matching the target.
3967	 */
3968	field_var = create_target_field_var(hist_data, system, event, var);
3969
3970	if (field_var && !IS_ERR(field_var)) {
3971		save_field_var(hist_data, field_var);
3972		hist_field = field_var->var;
3973	} else {
3974		field_var = NULL;
3975		/*
3976		 * If no explicit system.event is specified, default to
3977		 * looking for fields on the onmatch(system.event.xxx)
3978		 * event.
3979		 */
3980		if (!system && data->handler == HANDLER_ONMATCH) {
3981			system = data->match_data.event_system;
3982			event = data->match_data.event;
3983		}
3984
3985		if (!event)
3986			goto free;
3987		/*
3988		 * At this point, we're looking at a field on another
3989		 * event.  Because we can't modify a hist trigger on
3990		 * another event to add a variable for a field, we need
3991		 * to create a new trigger on that event and create the
3992		 * variable at the same time.
3993		 */
3994		hist_field = create_field_var_hist(hist_data, system, event, var);
3995		if (IS_ERR(hist_field))
3996			goto free;
3997	}
3998 out:
3999	return hist_field;
4000 free:
4001	destroy_field_var(field_var);
4002	hist_field = NULL;
4003	goto out;
4004}
4005
4006static int trace_action_create(struct hist_trigger_data *hist_data,
4007			       struct action_data *data)
4008{
4009	struct trace_array *tr = hist_data->event_file->tr;
4010	char *event_name, *param, *system = NULL;
4011	struct hist_field *hist_field, *var_ref;
4012	unsigned int i;
4013	unsigned int field_pos = 0;
4014	struct synth_event *event;
4015	char *synth_event_name;
4016	int var_ref_idx, ret = 0;
4017
4018	lockdep_assert_held(&event_mutex);
4019
4020	/* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
4021	if (data->n_params > SYNTH_FIELDS_MAX)
4022		return -EINVAL;
4023
4024	if (data->use_trace_keyword)
4025		synth_event_name = data->synth_event_name;
4026	else
4027		synth_event_name = data->action_name;
4028
4029	event = find_synth_event(synth_event_name);
4030	if (!event) {
4031		hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4032		return -EINVAL;
4033	}
4034
4035	event->ref++;
4036
4037	for (i = 0; i < data->n_params; i++) {
4038		char *p;
4039
4040		p = param = kstrdup(data->params[i], GFP_KERNEL);
4041		if (!param) {
4042			ret = -ENOMEM;
4043			goto err;
4044		}
4045
4046		system = strsep(&param, ".");
4047		if (!param) {
4048			param = (char *)system;
4049			system = event_name = NULL;
4050		} else {
4051			event_name = strsep(&param, ".");
4052			if (!param) {
4053				kfree(p);
4054				ret = -EINVAL;
4055				goto err;
4056			}
4057		}
4058
4059		if (param[0] == '$')
4060			hist_field = trace_action_find_var(hist_data, data,
4061							   system, event_name,
4062							   param);
4063		else
4064			hist_field = trace_action_create_field_var(hist_data,
4065								   data,
4066								   system,
4067								   event_name,
4068								   param);
4069
4070		if (!hist_field) {
4071			kfree(p);
4072			ret = -EINVAL;
4073			goto err;
4074		}
4075
4076		if (check_synth_field(event, hist_field, field_pos) == 0) {
4077			var_ref = create_var_ref(hist_data, hist_field,
4078						 system, event_name);
4079			if (!var_ref) {
4080				kfree(p);
4081				ret = -ENOMEM;
4082				goto err;
4083			}
4084
4085			var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4086			if (WARN_ON(var_ref_idx < 0)) {
4087				kfree(p);
4088				ret = var_ref_idx;
4089				goto err;
4090			}
4091
4092			data->var_ref_idx[i] = var_ref_idx;
4093
4094			field_pos++;
4095			kfree(p);
4096			continue;
4097		}
4098
4099		hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4100		kfree(p);
4101		ret = -EINVAL;
4102		goto err;
4103	}
4104
4105	if (field_pos != event->n_fields) {
4106		hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4107		ret = -EINVAL;
4108		goto err;
4109	}
4110
4111	data->synth_event = event;
4112 out:
4113	return ret;
4114 err:
4115	event->ref--;
4116
4117	goto out;
4118}
4119
4120static int action_create(struct hist_trigger_data *hist_data,
4121			 struct action_data *data)
4122{
4123	struct trace_event_file *file = hist_data->event_file;
4124	struct trace_array *tr = file->tr;
4125	struct track_data *track_data;
4126	struct field_var *field_var;
4127	unsigned int i;
4128	char *param;
4129	int ret = 0;
4130
4131	if (data->action == ACTION_TRACE)
4132		return trace_action_create(hist_data, data);
4133
4134	if (data->action == ACTION_SNAPSHOT) {
4135		track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4136		if (IS_ERR(track_data)) {
4137			ret = PTR_ERR(track_data);
4138			goto out;
4139		}
4140
4141		ret = tracing_snapshot_cond_enable(file->tr, track_data,
4142						   cond_snapshot_update);
4143		if (ret)
4144			track_data_free(track_data);
4145
4146		goto out;
4147	}
4148
4149	if (data->action == ACTION_SAVE) {
4150		if (hist_data->n_save_vars) {
4151			ret = -EEXIST;
4152			hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4153			goto out;
4154		}
4155
4156		for (i = 0; i < data->n_params; i++) {
4157			param = kstrdup(data->params[i], GFP_KERNEL);
4158			if (!param) {
4159				ret = -ENOMEM;
4160				goto out;
4161			}
4162
4163			field_var = create_target_field_var(hist_data, NULL, NULL, param);
4164			if (IS_ERR(field_var)) {
4165				hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4166					 errpos(param));
4167				ret = PTR_ERR(field_var);
4168				kfree(param);
4169				goto out;
4170			}
4171
4172			hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4173			if (field_var->val->flags &
4174			    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4175				hist_data->n_save_var_str++;
4176			kfree(param);
4177		}
4178	}
4179 out:
4180	return ret;
4181}
4182
4183static int onmatch_create(struct hist_trigger_data *hist_data,
4184			  struct action_data *data)
4185{
4186	return action_create(hist_data, data);
4187}
4188
4189static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4190{
4191	char *match_event, *match_event_system;
4192	struct action_data *data;
4193	int ret = -EINVAL;
4194
4195	data = kzalloc(sizeof(*data), GFP_KERNEL);
4196	if (!data)
4197		return ERR_PTR(-ENOMEM);
4198
4199	match_event = strsep(&str, ")");
4200	if (!match_event || !str) {
4201		hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4202		goto free;
4203	}
4204
4205	match_event_system = strsep(&match_event, ".");
4206	if (!match_event) {
4207		hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4208		goto free;
4209	}
4210
4211	if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4212		hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4213		goto free;
4214	}
4215
4216	data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4217	if (!data->match_data.event) {
4218		ret = -ENOMEM;
4219		goto free;
4220	}
4221
4222	data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4223	if (!data->match_data.event_system) {
4224		ret = -ENOMEM;
4225		goto free;
4226	}
4227
4228	ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4229	if (ret)
4230		goto free;
4231 out:
4232	return data;
4233 free:
4234	onmatch_destroy(data);
4235	data = ERR_PTR(ret);
4236	goto out;
4237}
4238
4239static int create_hitcount_val(struct hist_trigger_data *hist_data)
4240{
4241	hist_data->fields[HITCOUNT_IDX] =
4242		create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4243	if (!hist_data->fields[HITCOUNT_IDX])
4244		return -ENOMEM;
4245
4246	hist_data->n_vals++;
4247	hist_data->n_fields++;
4248
4249	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4250		return -EINVAL;
4251
4252	return 0;
4253}
4254
4255static int __create_val_field(struct hist_trigger_data *hist_data,
4256			      unsigned int val_idx,
4257			      struct trace_event_file *file,
4258			      char *var_name, char *field_str,
4259			      unsigned long flags)
4260{
4261	struct hist_field *hist_field;
4262	int ret = 0, n_subexprs = 0;
4263
4264	hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4265	if (IS_ERR(hist_field)) {
4266		ret = PTR_ERR(hist_field);
4267		goto out;
4268	}
4269
4270	/* values and variables should not have some modifiers */
4271	if (hist_field->flags & HIST_FIELD_FL_VAR) {
4272		/* Variable */
4273		if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4274					 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2))
4275			goto err;
4276	} else {
4277		/* Value */
4278		if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4279					 HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4280					 HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4281					 HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE))
4282			goto err;
4283	}
4284
4285	hist_data->fields[val_idx] = hist_field;
4286
4287	++hist_data->n_vals;
4288	++hist_data->n_fields;
4289
4290	if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4291		ret = -EINVAL;
4292 out:
4293	return ret;
4294 err:
4295	hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4296	return -EINVAL;
4297}
4298
4299static int create_val_field(struct hist_trigger_data *hist_data,
4300			    unsigned int val_idx,
4301			    struct trace_event_file *file,
4302			    char *field_str)
4303{
4304	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4305		return -EINVAL;
4306
4307	return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4308}
4309
4310static const char no_comm[] = "(no comm)";
4311
4312static u64 hist_field_execname(struct hist_field *hist_field,
4313			       struct tracing_map_elt *elt,
4314			       struct trace_buffer *buffer,
4315			       struct ring_buffer_event *rbe,
4316			       void *event)
4317{
4318	struct hist_elt_data *elt_data;
4319
4320	if (WARN_ON_ONCE(!elt))
4321		return (u64)(unsigned long)no_comm;
4322
4323	elt_data = elt->private_data;
4324
4325	if (WARN_ON_ONCE(!elt_data->comm))
4326		return (u64)(unsigned long)no_comm;
4327
4328	return (u64)(unsigned long)(elt_data->comm);
4329}
4330
4331static u64 hist_field_stack(struct hist_field *hist_field,
4332			    struct tracing_map_elt *elt,
4333			    struct trace_buffer *buffer,
4334			    struct ring_buffer_event *rbe,
4335			    void *event)
4336{
4337	u32 str_item = *(u32 *)(event + hist_field->field->offset);
4338	int str_loc = str_item & 0xffff;
4339	char *addr = (char *)(event + str_loc);
4340
4341	return (u64)(unsigned long)addr;
4342}
4343
4344static u64 hist_fn_call(struct hist_field *hist_field,
4345			struct tracing_map_elt *elt,
4346			struct trace_buffer *buffer,
4347			struct ring_buffer_event *rbe,
4348			void *event)
4349{
4350	switch (hist_field->fn_num) {
4351	case HIST_FIELD_FN_VAR_REF:
4352		return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4353	case HIST_FIELD_FN_COUNTER:
4354		return hist_field_counter(hist_field, elt, buffer, rbe, event);
4355	case HIST_FIELD_FN_CONST:
4356		return hist_field_const(hist_field, elt, buffer, rbe, event);
4357	case HIST_FIELD_FN_LOG2:
4358		return hist_field_log2(hist_field, elt, buffer, rbe, event);
4359	case HIST_FIELD_FN_BUCKET:
4360		return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4361	case HIST_FIELD_FN_TIMESTAMP:
4362		return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4363	case HIST_FIELD_FN_CPU:
4364		return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4365	case HIST_FIELD_FN_COMM:
4366		return hist_field_comm(hist_field, elt, buffer, rbe, event);
4367	case HIST_FIELD_FN_STRING:
4368		return hist_field_string(hist_field, elt, buffer, rbe, event);
4369	case HIST_FIELD_FN_DYNSTRING:
4370		return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4371	case HIST_FIELD_FN_RELDYNSTRING:
4372		return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4373	case HIST_FIELD_FN_PSTRING:
4374		return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4375	case HIST_FIELD_FN_S64:
4376		return hist_field_s64(hist_field, elt, buffer, rbe, event);
4377	case HIST_FIELD_FN_U64:
4378		return hist_field_u64(hist_field, elt, buffer, rbe, event);
4379	case HIST_FIELD_FN_S32:
4380		return hist_field_s32(hist_field, elt, buffer, rbe, event);
4381	case HIST_FIELD_FN_U32:
4382		return hist_field_u32(hist_field, elt, buffer, rbe, event);
4383	case HIST_FIELD_FN_S16:
4384		return hist_field_s16(hist_field, elt, buffer, rbe, event);
4385	case HIST_FIELD_FN_U16:
4386		return hist_field_u16(hist_field, elt, buffer, rbe, event);
4387	case HIST_FIELD_FN_S8:
4388		return hist_field_s8(hist_field, elt, buffer, rbe, event);
4389	case HIST_FIELD_FN_U8:
4390		return hist_field_u8(hist_field, elt, buffer, rbe, event);
4391	case HIST_FIELD_FN_UMINUS:
4392		return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4393	case HIST_FIELD_FN_MINUS:
4394		return hist_field_minus(hist_field, elt, buffer, rbe, event);
4395	case HIST_FIELD_FN_PLUS:
4396		return hist_field_plus(hist_field, elt, buffer, rbe, event);
4397	case HIST_FIELD_FN_DIV:
4398		return hist_field_div(hist_field, elt, buffer, rbe, event);
4399	case HIST_FIELD_FN_MULT:
4400		return hist_field_mult(hist_field, elt, buffer, rbe, event);
4401	case HIST_FIELD_FN_DIV_POWER2:
4402		return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4403	case HIST_FIELD_FN_DIV_NOT_POWER2:
4404		return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4405	case HIST_FIELD_FN_DIV_MULT_SHIFT:
4406		return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4407	case HIST_FIELD_FN_EXECNAME:
4408		return hist_field_execname(hist_field, elt, buffer, rbe, event);
4409	case HIST_FIELD_FN_STACK:
4410		return hist_field_stack(hist_field, elt, buffer, rbe, event);
4411	default:
4412		return 0;
4413	}
4414}
4415
4416/* Convert a var that points to common_pid.execname to a string */
4417static void update_var_execname(struct hist_field *hist_field)
4418{
4419	hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4420		HIST_FIELD_FL_EXECNAME;
4421	hist_field->size = MAX_FILTER_STR_VAL;
4422	hist_field->is_signed = 0;
4423
4424	kfree_const(hist_field->type);
4425	hist_field->type = "char[]";
4426
4427	hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4428}
4429
4430static int create_var_field(struct hist_trigger_data *hist_data,
4431			    unsigned int val_idx,
4432			    struct trace_event_file *file,
4433			    char *var_name, char *expr_str)
4434{
4435	struct trace_array *tr = hist_data->event_file->tr;
4436	unsigned long flags = 0;
4437	int ret;
4438
4439	if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4440		return -EINVAL;
4441
4442	if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4443		hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4444		return -EINVAL;
4445	}
4446
4447	flags |= HIST_FIELD_FL_VAR;
4448	hist_data->n_vars++;
4449	if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4450		return -EINVAL;
4451
4452	ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4453
4454	if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4455		update_var_execname(hist_data->fields[val_idx]);
4456
4457	if (!ret && hist_data->fields[val_idx]->flags &
4458	    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4459		hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4460
4461	return ret;
4462}
4463
4464static int create_val_fields(struct hist_trigger_data *hist_data,
4465			     struct trace_event_file *file)
4466{
4467	unsigned int i, j = 1, n_hitcount = 0;
4468	char *fields_str, *field_str;
4469	int ret;
4470
4471	ret = create_hitcount_val(hist_data);
4472	if (ret)
4473		goto out;
4474
4475	fields_str = hist_data->attrs->vals_str;
4476	if (!fields_str)
4477		goto out;
4478
4479	for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4480		     j < TRACING_MAP_VALS_MAX; i++) {
4481		field_str = strsep(&fields_str, ",");
4482		if (!field_str)
4483			break;
4484
4485		if (strcmp(field_str, "hitcount") == 0) {
4486			if (!n_hitcount++)
4487				continue;
4488		}
4489
4490		ret = create_val_field(hist_data, j++, file, field_str);
4491		if (ret)
4492			goto out;
4493	}
4494
4495	if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4496		ret = -EINVAL;
4497 out:
4498	/* There is only raw hitcount but nohitcount suppresses it. */
4499	if (j == 1 && hist_data->attrs->no_hitcount) {
4500		hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4501		ret = -ENOENT;
4502	}
4503
4504	return ret;
4505}
4506
4507static int create_key_field(struct hist_trigger_data *hist_data,
4508			    unsigned int key_idx,
4509			    unsigned int key_offset,
4510			    struct trace_event_file *file,
4511			    char *field_str)
4512{
4513	struct trace_array *tr = hist_data->event_file->tr;
4514	struct hist_field *hist_field = NULL;
4515	unsigned long flags = 0;
4516	unsigned int key_size;
4517	int ret = 0, n_subexprs = 0;
4518
4519	if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4520		return -EINVAL;
4521
4522	flags |= HIST_FIELD_FL_KEY;
4523
4524	if (strcmp(field_str, "stacktrace") == 0) {
4525		flags |= HIST_FIELD_FL_STACKTRACE;
4526		key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4527		hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4528	} else {
4529		hist_field = parse_expr(hist_data, file, field_str, flags,
4530					NULL, &n_subexprs);
4531		if (IS_ERR(hist_field)) {
4532			ret = PTR_ERR(hist_field);
4533			goto out;
4534		}
4535
4536		if (field_has_hist_vars(hist_field, 0))	{
4537			hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4538			destroy_hist_field(hist_field, 0);
4539			ret = -EINVAL;
4540			goto out;
4541		}
4542
4543		key_size = hist_field->size;
4544	}
4545
4546	hist_data->fields[key_idx] = hist_field;
4547
4548	key_size = ALIGN(key_size, sizeof(u64));
4549	hist_data->fields[key_idx]->size = key_size;
4550	hist_data->fields[key_idx]->offset = key_offset;
4551
4552	hist_data->key_size += key_size;
4553
4554	if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4555		ret = -EINVAL;
4556		goto out;
4557	}
4558
4559	hist_data->n_keys++;
4560	hist_data->n_fields++;
4561
4562	if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4563		return -EINVAL;
4564
4565	ret = key_size;
4566 out:
4567	return ret;
4568}
4569
4570static int create_key_fields(struct hist_trigger_data *hist_data,
4571			     struct trace_event_file *file)
4572{
4573	unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4574	char *fields_str, *field_str;
4575	int ret = -EINVAL;
4576
4577	fields_str = hist_data->attrs->keys_str;
4578	if (!fields_str)
4579		goto out;
4580
4581	for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4582		field_str = strsep(&fields_str, ",");
4583		if (!field_str)
4584			break;
4585		ret = create_key_field(hist_data, i, key_offset,
4586				       file, field_str);
4587		if (ret < 0)
4588			goto out;
4589		key_offset += ret;
4590	}
4591	if (fields_str) {
4592		ret = -EINVAL;
4593		goto out;
4594	}
4595	ret = 0;
4596 out:
4597	return ret;
4598}
4599
4600static int create_var_fields(struct hist_trigger_data *hist_data,
4601			     struct trace_event_file *file)
4602{
4603	unsigned int i, j = hist_data->n_vals;
4604	int ret = 0;
4605
4606	unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4607
4608	for (i = 0; i < n_vars; i++) {
4609		char *var_name = hist_data->attrs->var_defs.name[i];
4610		char *expr = hist_data->attrs->var_defs.expr[i];
4611
4612		ret = create_var_field(hist_data, j++, file, var_name, expr);
4613		if (ret)
4614			goto out;
4615	}
4616 out:
4617	return ret;
4618}
4619
4620static void free_var_defs(struct hist_trigger_data *hist_data)
4621{
4622	unsigned int i;
4623
4624	for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4625		kfree(hist_data->attrs->var_defs.name[i]);
4626		kfree(hist_data->attrs->var_defs.expr[i]);
4627	}
4628
4629	hist_data->attrs->var_defs.n_vars = 0;
4630}
4631
4632static int parse_var_defs(struct hist_trigger_data *hist_data)
4633{
4634	struct trace_array *tr = hist_data->event_file->tr;
4635	char *s, *str, *var_name, *field_str;
4636	unsigned int i, j, n_vars = 0;
4637	int ret = 0;
4638
4639	for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4640		str = hist_data->attrs->assignment_str[i];
4641		for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4642			field_str = strsep(&str, ",");
4643			if (!field_str)
4644				break;
4645
4646			var_name = strsep(&field_str, "=");
4647			if (!var_name || !field_str) {
4648				hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4649					 errpos(var_name));
4650				ret = -EINVAL;
4651				goto free;
4652			}
4653
4654			if (n_vars == TRACING_MAP_VARS_MAX) {
4655				hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4656				ret = -EINVAL;
4657				goto free;
4658			}
4659
4660			s = kstrdup(var_name, GFP_KERNEL);
4661			if (!s) {
4662				ret = -ENOMEM;
4663				goto free;
4664			}
4665			hist_data->attrs->var_defs.name[n_vars] = s;
4666
4667			s = kstrdup(field_str, GFP_KERNEL);
4668			if (!s) {
4669				kfree(hist_data->attrs->var_defs.name[n_vars]);
4670				hist_data->attrs->var_defs.name[n_vars] = NULL;
4671				ret = -ENOMEM;
4672				goto free;
4673			}
4674			hist_data->attrs->var_defs.expr[n_vars++] = s;
4675
4676			hist_data->attrs->var_defs.n_vars = n_vars;
4677		}
4678	}
4679
4680	return ret;
4681 free:
4682	free_var_defs(hist_data);
4683
4684	return ret;
4685}
4686
4687static int create_hist_fields(struct hist_trigger_data *hist_data,
4688			      struct trace_event_file *file)
4689{
4690	int ret;
4691
4692	ret = parse_var_defs(hist_data);
4693	if (ret)
4694		return ret;
4695
4696	ret = create_val_fields(hist_data, file);
4697	if (ret)
4698		goto out;
4699
4700	ret = create_var_fields(hist_data, file);
4701	if (ret)
4702		goto out;
4703
4704	ret = create_key_fields(hist_data, file);
4705
4706 out:
4707	free_var_defs(hist_data);
4708
4709	return ret;
4710}
4711
4712static int is_descending(struct trace_array *tr, const char *str)
4713{
4714	if (!str)
4715		return 0;
4716
4717	if (strcmp(str, "descending") == 0)
4718		return 1;
4719
4720	if (strcmp(str, "ascending") == 0)
4721		return 0;
4722
4723	hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4724
4725	return -EINVAL;
4726}
4727
4728static int create_sort_keys(struct hist_trigger_data *hist_data)
4729{
4730	struct trace_array *tr = hist_data->event_file->tr;
4731	char *fields_str = hist_data->attrs->sort_key_str;
4732	struct tracing_map_sort_key *sort_key;
4733	int descending, ret = 0;
4734	unsigned int i, j, k;
4735
4736	hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4737
4738	if (!fields_str)
4739		goto out;
4740
4741	for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4742		struct hist_field *hist_field;
4743		char *field_str, *field_name;
4744		const char *test_name;
4745
4746		sort_key = &hist_data->sort_keys[i];
4747
4748		field_str = strsep(&fields_str, ",");
4749		if (!field_str)
4750			break;
4751
4752		if (!*field_str) {
4753			ret = -EINVAL;
4754			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4755			break;
4756		}
4757
4758		if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4759			hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4760			ret = -EINVAL;
4761			break;
4762		}
4763
4764		field_name = strsep(&field_str, ".");
4765		if (!field_name || !*field_name) {
4766			ret = -EINVAL;
4767			hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4768			break;
4769		}
4770
4771		if (strcmp(field_name, "hitcount") == 0) {
4772			descending = is_descending(tr, field_str);
4773			if (descending < 0) {
4774				ret = descending;
4775				break;
4776			}
4777			sort_key->descending = descending;
4778			continue;
4779		}
4780
4781		for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4782			unsigned int idx;
4783
4784			hist_field = hist_data->fields[j];
4785			if (hist_field->flags & HIST_FIELD_FL_VAR)
4786				continue;
4787
4788			idx = k++;
4789
4790			test_name = hist_field_name(hist_field, 0);
4791
4792			if (strcmp(field_name, test_name) == 0) {
4793				sort_key->field_idx = idx;
4794				descending = is_descending(tr, field_str);
4795				if (descending < 0) {
4796					ret = descending;
4797					goto out;
4798				}
4799				sort_key->descending = descending;
4800				break;
4801			}
4802		}
4803		if (j == hist_data->n_fields) {
4804			ret = -EINVAL;
4805			hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4806			break;
4807		}
4808	}
4809
4810	hist_data->n_sort_keys = i;
4811 out:
4812	return ret;
4813}
4814
4815static void destroy_actions(struct hist_trigger_data *hist_data)
4816{
4817	unsigned int i;
4818
4819	for (i = 0; i < hist_data->n_actions; i++) {
4820		struct action_data *data = hist_data->actions[i];
4821
4822		if (data->handler == HANDLER_ONMATCH)
4823			onmatch_destroy(data);
4824		else if (data->handler == HANDLER_ONMAX ||
4825			 data->handler == HANDLER_ONCHANGE)
4826			track_data_destroy(hist_data, data);
4827		else
4828			kfree(data);
4829	}
4830}
4831
4832static int parse_actions(struct hist_trigger_data *hist_data)
4833{
4834	struct trace_array *tr = hist_data->event_file->tr;
4835	struct action_data *data;
4836	unsigned int i;
4837	int ret = 0;
4838	char *str;
4839	int len;
4840
4841	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4842		enum handler_id hid = 0;
4843		char *action_str;
4844
4845		str = hist_data->attrs->action_str[i];
4846
4847		if ((len = str_has_prefix(str, "onmatch(")))
4848			hid = HANDLER_ONMATCH;
4849		else if ((len = str_has_prefix(str, "onmax(")))
4850			hid = HANDLER_ONMAX;
4851		else if ((len = str_has_prefix(str, "onchange(")))
4852			hid = HANDLER_ONCHANGE;
4853
4854		action_str = str + len;
4855
4856		switch (hid) {
4857		case HANDLER_ONMATCH:
4858			data = onmatch_parse(tr, action_str);
4859			break;
4860		case HANDLER_ONMAX:
4861		case HANDLER_ONCHANGE:
4862			data = track_data_parse(hist_data, action_str, hid);
4863			break;
4864		default:
4865			data = ERR_PTR(-EINVAL);
4866			break;
4867		}
4868
4869		if (IS_ERR(data)) {
4870			ret = PTR_ERR(data);
4871			break;
4872		}
4873
4874		hist_data->actions[hist_data->n_actions++] = data;
4875	}
4876
4877	return ret;
4878}
4879
4880static int create_actions(struct hist_trigger_data *hist_data)
4881{
4882	struct action_data *data;
4883	unsigned int i;
4884	int ret = 0;
4885
4886	for (i = 0; i < hist_data->attrs->n_actions; i++) {
4887		data = hist_data->actions[i];
4888
4889		if (data->handler == HANDLER_ONMATCH) {
4890			ret = onmatch_create(hist_data, data);
4891			if (ret)
4892				break;
4893		} else if (data->handler == HANDLER_ONMAX ||
4894			   data->handler == HANDLER_ONCHANGE) {
4895			ret = track_data_create(hist_data, data);
4896			if (ret)
4897				break;
4898		} else {
4899			ret = -EINVAL;
4900			break;
4901		}
4902	}
4903
4904	return ret;
4905}
4906
4907static void print_actions(struct seq_file *m,
4908			  struct hist_trigger_data *hist_data,
4909			  struct tracing_map_elt *elt)
4910{
4911	unsigned int i;
4912
4913	for (i = 0; i < hist_data->n_actions; i++) {
4914		struct action_data *data = hist_data->actions[i];
4915
4916		if (data->action == ACTION_SNAPSHOT)
4917			continue;
4918
4919		if (data->handler == HANDLER_ONMAX ||
4920		    data->handler == HANDLER_ONCHANGE)
4921			track_data_print(m, hist_data, elt, data);
4922	}
4923}
4924
4925static void print_action_spec(struct seq_file *m,
4926			      struct hist_trigger_data *hist_data,
4927			      struct action_data *data)
4928{
4929	unsigned int i;
4930
4931	if (data->action == ACTION_SAVE) {
4932		for (i = 0; i < hist_data->n_save_vars; i++) {
4933			seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4934			if (i < hist_data->n_save_vars - 1)
4935				seq_puts(m, ",");
4936		}
4937	} else if (data->action == ACTION_TRACE) {
4938		if (data->use_trace_keyword)
4939			seq_printf(m, "%s", data->synth_event_name);
4940		for (i = 0; i < data->n_params; i++) {
4941			if (i || data->use_trace_keyword)
4942				seq_puts(m, ",");
4943			seq_printf(m, "%s", data->params[i]);
4944		}
4945	}
4946}
4947
4948static void print_track_data_spec(struct seq_file *m,
4949				  struct hist_trigger_data *hist_data,
4950				  struct action_data *data)
4951{
4952	if (data->handler == HANDLER_ONMAX)
4953		seq_puts(m, ":onmax(");
4954	else if (data->handler == HANDLER_ONCHANGE)
4955		seq_puts(m, ":onchange(");
4956	seq_printf(m, "%s", data->track_data.var_str);
4957	seq_printf(m, ").%s(", data->action_name);
4958
4959	print_action_spec(m, hist_data, data);
4960
4961	seq_puts(m, ")");
4962}
4963
4964static void print_onmatch_spec(struct seq_file *m,
4965			       struct hist_trigger_data *hist_data,
4966			       struct action_data *data)
4967{
4968	seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4969		   data->match_data.event);
4970
4971	seq_printf(m, "%s(", data->action_name);
4972
4973	print_action_spec(m, hist_data, data);
4974
4975	seq_puts(m, ")");
4976}
4977
4978static bool actions_match(struct hist_trigger_data *hist_data,
4979			  struct hist_trigger_data *hist_data_test)
4980{
4981	unsigned int i, j;
4982
4983	if (hist_data->n_actions != hist_data_test->n_actions)
4984		return false;
4985
4986	for (i = 0; i < hist_data->n_actions; i++) {
4987		struct action_data *data = hist_data->actions[i];
4988		struct action_data *data_test = hist_data_test->actions[i];
4989		char *action_name, *action_name_test;
4990
4991		if (data->handler != data_test->handler)
4992			return false;
4993		if (data->action != data_test->action)
4994			return false;
4995
4996		if (data->n_params != data_test->n_params)
4997			return false;
4998
4999		for (j = 0; j < data->n_params; j++) {
5000			if (strcmp(data->params[j], data_test->params[j]) != 0)
5001				return false;
5002		}
5003
5004		if (data->use_trace_keyword)
5005			action_name = data->synth_event_name;
5006		else
5007			action_name = data->action_name;
5008
5009		if (data_test->use_trace_keyword)
5010			action_name_test = data_test->synth_event_name;
5011		else
5012			action_name_test = data_test->action_name;
5013
5014		if (strcmp(action_name, action_name_test) != 0)
5015			return false;
5016
5017		if (data->handler == HANDLER_ONMATCH) {
5018			if (strcmp(data->match_data.event_system,
5019				   data_test->match_data.event_system) != 0)
5020				return false;
5021			if (strcmp(data->match_data.event,
5022				   data_test->match_data.event) != 0)
5023				return false;
5024		} else if (data->handler == HANDLER_ONMAX ||
5025			   data->handler == HANDLER_ONCHANGE) {
5026			if (strcmp(data->track_data.var_str,
5027				   data_test->track_data.var_str) != 0)
5028				return false;
5029		}
5030	}
5031
5032	return true;
5033}
5034
5035
5036static void print_actions_spec(struct seq_file *m,
5037			       struct hist_trigger_data *hist_data)
5038{
5039	unsigned int i;
5040
5041	for (i = 0; i < hist_data->n_actions; i++) {
5042		struct action_data *data = hist_data->actions[i];
5043
5044		if (data->handler == HANDLER_ONMATCH)
5045			print_onmatch_spec(m, hist_data, data);
5046		else if (data->handler == HANDLER_ONMAX ||
5047			 data->handler == HANDLER_ONCHANGE)
5048			print_track_data_spec(m, hist_data, data);
5049	}
5050}
5051
5052static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5053{
5054	unsigned int i;
5055
5056	for (i = 0; i < hist_data->n_field_var_hists; i++) {
5057		kfree(hist_data->field_var_hists[i]->cmd);
5058		kfree(hist_data->field_var_hists[i]);
5059	}
5060}
5061
5062static void destroy_hist_data(struct hist_trigger_data *hist_data)
5063{
5064	if (!hist_data)
5065		return;
5066
5067	destroy_hist_trigger_attrs(hist_data->attrs);
5068	destroy_hist_fields(hist_data);
5069	tracing_map_destroy(hist_data->map);
5070
5071	destroy_actions(hist_data);
5072	destroy_field_vars(hist_data);
5073	destroy_field_var_hists(hist_data);
5074
5075	kfree(hist_data);
5076}
5077
5078static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5079{
5080	struct tracing_map *map = hist_data->map;
5081	struct ftrace_event_field *field;
5082	struct hist_field *hist_field;
5083	int i, idx = 0;
5084
5085	for_each_hist_field(i, hist_data) {
5086		hist_field = hist_data->fields[i];
5087		if (hist_field->flags & HIST_FIELD_FL_KEY) {
5088			tracing_map_cmp_fn_t cmp_fn;
5089
5090			field = hist_field->field;
5091
5092			if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5093				cmp_fn = tracing_map_cmp_none;
5094			else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5095				cmp_fn = tracing_map_cmp_num(hist_field->size,
5096							     hist_field->is_signed);
5097			else if (is_string_field(field))
5098				cmp_fn = tracing_map_cmp_string;
5099			else
5100				cmp_fn = tracing_map_cmp_num(field->size,
5101							     field->is_signed);
5102			idx = tracing_map_add_key_field(map,
5103							hist_field->offset,
5104							cmp_fn);
5105		} else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5106			idx = tracing_map_add_sum_field(map);
5107
5108		if (idx < 0)
5109			return idx;
5110
5111		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5112			idx = tracing_map_add_var(map);
5113			if (idx < 0)
5114				return idx;
5115			hist_field->var.idx = idx;
5116			hist_field->var.hist_data = hist_data;
5117		}
5118	}
5119
5120	return 0;
5121}
5122
5123static struct hist_trigger_data *
5124create_hist_data(unsigned int map_bits,
5125		 struct hist_trigger_attrs *attrs,
5126		 struct trace_event_file *file,
5127		 bool remove)
5128{
5129	const struct tracing_map_ops *map_ops = NULL;
5130	struct hist_trigger_data *hist_data;
5131	int ret = 0;
5132
5133	hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5134	if (!hist_data)
5135		return ERR_PTR(-ENOMEM);
5136
5137	hist_data->attrs = attrs;
5138	hist_data->remove = remove;
5139	hist_data->event_file = file;
5140
5141	ret = parse_actions(hist_data);
5142	if (ret)
5143		goto free;
5144
5145	ret = create_hist_fields(hist_data, file);
5146	if (ret)
5147		goto free;
5148
5149	ret = create_sort_keys(hist_data);
5150	if (ret)
5151		goto free;
5152
5153	map_ops = &hist_trigger_elt_data_ops;
5154
5155	hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5156					    map_ops, hist_data);
5157	if (IS_ERR(hist_data->map)) {
5158		ret = PTR_ERR(hist_data->map);
5159		hist_data->map = NULL;
5160		goto free;
5161	}
5162
5163	ret = create_tracing_map_fields(hist_data);
5164	if (ret)
5165		goto free;
5166 out:
5167	return hist_data;
5168 free:
5169	hist_data->attrs = NULL;
5170
5171	destroy_hist_data(hist_data);
5172
5173	hist_data = ERR_PTR(ret);
5174
5175	goto out;
5176}
5177
5178static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5179				    struct tracing_map_elt *elt,
5180				    struct trace_buffer *buffer, void *rec,
5181				    struct ring_buffer_event *rbe,
5182				    u64 *var_ref_vals)
5183{
5184	struct hist_elt_data *elt_data;
5185	struct hist_field *hist_field;
5186	unsigned int i, var_idx;
5187	u64 hist_val;
5188
5189	elt_data = elt->private_data;
5190	elt_data->var_ref_vals = var_ref_vals;
5191
5192	for_each_hist_val_field(i, hist_data) {
5193		hist_field = hist_data->fields[i];
5194		hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5195		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5196			var_idx = hist_field->var.idx;
5197
5198			if (hist_field->flags &
5199			    (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5200				unsigned int str_start, var_str_idx, idx;
5201				char *str, *val_str;
5202				unsigned int size;
5203
5204				str_start = hist_data->n_field_var_str +
5205					hist_data->n_save_var_str;
5206				var_str_idx = hist_field->var_str_idx;
5207				idx = str_start + var_str_idx;
5208
5209				str = elt_data->field_var_str[idx];
5210				val_str = (char *)(uintptr_t)hist_val;
5211
5212				if (hist_field->flags & HIST_FIELD_FL_STRING) {
5213					size = min(hist_field->size, STR_VAR_LEN_MAX);
5214					strscpy(str, val_str, size);
5215				} else {
5216					char *stack_start = str + sizeof(unsigned long);
5217					int e;
5218
5219					e = stack_trace_save((void *)stack_start,
5220							     HIST_STACKTRACE_DEPTH,
5221							     HIST_STACKTRACE_SKIP);
5222					if (e < HIST_STACKTRACE_DEPTH - 1)
5223						((unsigned long *)stack_start)[e] = 0;
5224					*((unsigned long *)str) = e;
5225				}
5226				hist_val = (u64)(uintptr_t)str;
5227			}
5228			tracing_map_set_var(elt, var_idx, hist_val);
5229			continue;
5230		}
5231		tracing_map_update_sum(elt, i, hist_val);
5232	}
5233
5234	for_each_hist_key_field(i, hist_data) {
5235		hist_field = hist_data->fields[i];
5236		if (hist_field->flags & HIST_FIELD_FL_VAR) {
5237			hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5238			var_idx = hist_field->var.idx;
5239			tracing_map_set_var(elt, var_idx, hist_val);
5240		}
5241	}
5242
5243	update_field_vars(hist_data, elt, buffer, rbe, rec);
5244}
5245
5246static inline void add_to_key(char *compound_key, void *key,
5247			      struct hist_field *key_field, void *rec)
5248{
5249	size_t size = key_field->size;
5250
5251	if (key_field->flags & HIST_FIELD_FL_STRING) {
5252
5253		if (key_field->flags & HIST_FIELD_FL_COMM) {
5254			size = strlen((char *)key);
5255		} else {
5256			struct ftrace_event_field *field;
5257
5258			field = key_field->field;
5259			if (field->filter_type == FILTER_DYN_STRING ||
5260			    field->filter_type == FILTER_RDYN_STRING)
5261				size = *(u32 *)(rec + field->offset) >> 16;
5262			else if (field->filter_type == FILTER_STATIC_STRING)
5263				size = field->size;
5264		}
5265
5266		/* ensure NULL-termination */
5267		if (size > key_field->size - 1)
5268			size = key_field->size - 1;
5269	}
5270	memcpy(compound_key + key_field->offset, key, size);
5271}
5272
5273static void
5274hist_trigger_actions(struct hist_trigger_data *hist_data,
5275		     struct tracing_map_elt *elt,
5276		     struct trace_buffer *buffer, void *rec,
5277		     struct ring_buffer_event *rbe, void *key,
5278		     u64 *var_ref_vals)
5279{
5280	struct action_data *data;
5281	unsigned int i;
5282
5283	for (i = 0; i < hist_data->n_actions; i++) {
5284		data = hist_data->actions[i];
5285		data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5286	}
5287}
5288
5289/*
5290 * The hist_pad structure is used to save information to create
5291 * a histogram from the histogram trigger. It's too big to store
5292 * on the stack, so when the histogram trigger is initialized
5293 * a percpu array of 4 hist_pad structures is allocated.
5294 * This will cover every context from normal, softirq, irq and NMI
5295 * in the very unlikely event that a trigger happens at each of
5296 * these contexts and interrupts a currently active trigger.
5297 */
5298struct hist_pad {
5299	unsigned long		entries[HIST_STACKTRACE_DEPTH];
5300	u64			var_ref_vals[TRACING_MAP_VARS_MAX];
5301	char			compound_key[HIST_KEY_SIZE_MAX];
5302};
5303
5304static struct hist_pad __percpu *hist_pads;
5305static DEFINE_PER_CPU(int, hist_pad_cnt);
5306static refcount_t hist_pad_ref;
5307
5308/* One hist_pad for every context (normal, softirq, irq, NMI) */
5309#define MAX_HIST_CNT 4
5310
5311static int alloc_hist_pad(void)
5312{
5313	lockdep_assert_held(&event_mutex);
5314
5315	if (refcount_read(&hist_pad_ref)) {
5316		refcount_inc(&hist_pad_ref);
5317		return 0;
5318	}
5319
5320	hist_pads = __alloc_percpu(sizeof(struct hist_pad) * MAX_HIST_CNT,
5321				   __alignof__(struct hist_pad));
5322	if (!hist_pads)
5323		return -ENOMEM;
5324
5325	refcount_set(&hist_pad_ref, 1);
5326	return 0;
5327}
5328
5329static void free_hist_pad(void)
5330{
5331	lockdep_assert_held(&event_mutex);
5332
5333	if (!refcount_dec_and_test(&hist_pad_ref))
5334		return;
5335
5336	free_percpu(hist_pads);
5337	hist_pads = NULL;
5338}
5339
5340static struct hist_pad *get_hist_pad(void)
5341{
5342	struct hist_pad *hist_pad;
5343	int cnt;
5344
5345	if (WARN_ON_ONCE(!hist_pads))
5346		return NULL;
5347
5348	preempt_disable();
5349
5350	hist_pad = per_cpu_ptr(hist_pads, smp_processor_id());
5351
5352	if (this_cpu_read(hist_pad_cnt) == MAX_HIST_CNT) {
5353		preempt_enable();
5354		return NULL;
5355	}
5356
5357	cnt = this_cpu_inc_return(hist_pad_cnt) - 1;
5358
5359	return &hist_pad[cnt];
5360}
5361
5362static void put_hist_pad(void)
5363{
5364	this_cpu_dec(hist_pad_cnt);
5365	preempt_enable();
5366}
5367
5368static void event_hist_trigger(struct event_trigger_data *data,
5369			       struct trace_buffer *buffer, void *rec,
5370			       struct ring_buffer_event *rbe)
5371{
5372	struct hist_trigger_data *hist_data = data->private_data;
5373	bool use_compound_key = (hist_data->n_keys > 1);
5374	struct tracing_map_elt *elt = NULL;
5375	struct hist_field *key_field;
5376	struct hist_pad *hist_pad;
5377	u64 field_contents;
5378	void *key = NULL;
5379	unsigned int i;
5380
5381	if (unlikely(!rbe))
5382		return;
5383
5384	hist_pad = get_hist_pad();
5385	if (!hist_pad)
5386		return;
5387
5388	memset(hist_pad->compound_key, 0, hist_data->key_size);
5389
5390	for_each_hist_key_field(i, hist_data) {
5391		key_field = hist_data->fields[i];
5392
5393		if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5394			unsigned long *entries = hist_pad->entries;
5395
5396			memset(entries, 0, HIST_STACKTRACE_SIZE);
5397			if (key_field->field) {
5398				unsigned long *stack, n_entries;
5399
5400				field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5401				stack = (unsigned long *)(long)field_contents;
5402				n_entries = *stack;
5403				memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5404			} else {
5405				stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5406						 HIST_STACKTRACE_SKIP);
5407			}
5408			key = entries;
5409		} else {
5410			field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5411			if (key_field->flags & HIST_FIELD_FL_STRING) {
5412				key = (void *)(unsigned long)field_contents;
5413				use_compound_key = true;
5414			} else
5415				key = (void *)&field_contents;
5416		}
5417
5418		if (use_compound_key)
5419			add_to_key(hist_pad->compound_key, key, key_field, rec);
5420	}
5421
5422	if (use_compound_key)
5423		key = hist_pad->compound_key;
5424
5425	if (hist_data->n_var_refs &&
5426	    !resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, false))
5427		goto out;
5428
5429	elt = tracing_map_insert(hist_data->map, key);
5430	if (!elt)
5431		goto out;
5432
5433	hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, hist_pad->var_ref_vals);
5434
5435	if (resolve_var_refs(hist_data, key, hist_pad->var_ref_vals, true)) {
5436		hist_trigger_actions(hist_data, elt, buffer, rec, rbe,
5437				     key, hist_pad->var_ref_vals);
5438	}
5439
5440	hist_poll_wakeup();
5441
5442 out:
5443	put_hist_pad();
5444}
5445
5446static void hist_trigger_stacktrace_print(struct seq_file *m,
5447					  unsigned long *stacktrace_entries,
5448					  unsigned int max_entries)
5449{
5450	unsigned int spaces = 8;
5451	unsigned int i;
5452
5453	for (i = 0; i < max_entries; i++) {
5454		if (!stacktrace_entries[i])
5455			return;
5456
5457		seq_printf(m, "%*c", 1 + spaces, ' ');
5458		seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5459	}
5460}
5461
5462static void hist_trigger_print_key(struct seq_file *m,
5463				   struct hist_trigger_data *hist_data,
5464				   void *key,
5465				   struct tracing_map_elt *elt)
5466{
5467	struct hist_field *key_field;
5468	bool multiline = false;
5469	const char *field_name;
5470	unsigned int i;
5471	u64 uval;
5472
5473	seq_puts(m, "{ ");
5474
5475	for_each_hist_key_field(i, hist_data) {
5476		key_field = hist_data->fields[i];
5477
5478		if (i > hist_data->n_vals)
5479			seq_puts(m, ", ");
5480
5481		field_name = hist_field_name(key_field, 0);
5482
5483		if (key_field->flags & HIST_FIELD_FL_HEX) {
5484			uval = *(u64 *)(key + key_field->offset);
5485			seq_printf(m, "%s: %llx", field_name, uval);
5486		} else if (key_field->flags & HIST_FIELD_FL_SYM) {
5487			uval = *(u64 *)(key + key_field->offset);
5488			seq_printf(m, "%s: [%llx] %-45ps", field_name,
5489				   uval, (void *)(uintptr_t)uval);
5490		} else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5491			uval = *(u64 *)(key + key_field->offset);
5492			seq_printf(m, "%s: [%llx] %-55pS", field_name,
5493				   uval, (void *)(uintptr_t)uval);
5494		} else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5495			struct hist_elt_data *elt_data = elt->private_data;
5496			char *comm;
5497
5498			if (WARN_ON_ONCE(!elt_data))
5499				return;
5500
5501			comm = elt_data->comm;
5502
5503			uval = *(u64 *)(key + key_field->offset);
5504			seq_printf(m, "%s: %-16s[%10llu]", field_name,
5505				   comm, uval);
5506		} else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5507			const char *syscall_name;
5508
5509			uval = *(u64 *)(key + key_field->offset);
5510			syscall_name = get_syscall_name(uval);
5511			if (!syscall_name)
5512				syscall_name = "unknown_syscall";
5513
5514			seq_printf(m, "%s: %-30s[%3llu]", field_name,
5515				   syscall_name, uval);
5516		} else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5517			if (key_field->field)
5518				seq_printf(m, "%s.stacktrace", key_field->field->name);
5519			else
5520				seq_puts(m, "common_stacktrace:\n");
5521			hist_trigger_stacktrace_print(m,
5522						      key + key_field->offset,
5523						      HIST_STACKTRACE_DEPTH);
5524			multiline = true;
5525		} else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5526			seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5527				   *(u64 *)(key + key_field->offset));
5528		} else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5529			unsigned long buckets = key_field->buckets;
5530			uval = *(u64 *)(key + key_field->offset);
5531			seq_printf(m, "%s: ~ %llu-%llu", field_name,
5532				   uval, uval + buckets -1);
5533		} else if (key_field->flags & HIST_FIELD_FL_STRING) {
5534			seq_printf(m, "%s: %-50s", field_name,
5535				   (char *)(key + key_field->offset));
5536		} else {
5537			uval = *(u64 *)(key + key_field->offset);
5538			seq_printf(m, "%s: %10llu", field_name, uval);
5539		}
5540	}
5541
5542	if (!multiline)
5543		seq_puts(m, " ");
5544
5545	seq_puts(m, "}");
5546}
5547
5548/* Get the 100 times of the percentage of @val in @total */
5549static inline unsigned int __get_percentage(u64 val, u64 total)
5550{
5551	if (!total)
5552		goto div0;
5553
5554	if (val < (U64_MAX / 10000))
5555		return (unsigned int)div64_ul(val * 10000, total);
5556
5557	total = div64_u64(total, 10000);
5558	if (!total)
5559		goto div0;
5560
5561	return (unsigned int)div64_ul(val, total);
5562div0:
5563	return val ? UINT_MAX : 0;
5564}
5565
5566#define BAR_CHAR '#'
5567
5568static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5569{
5570	unsigned int len = __get_percentage(val, max);
5571	int i;
5572
5573	if (len == UINT_MAX) {
5574		snprintf(buf, size, "[ERROR]");
5575		return buf;
5576	}
5577
5578	len = len * size / 10000;
5579	for (i = 0; i < len && i < size; i++)
5580		buf[i] = BAR_CHAR;
5581	while (i < size)
5582		buf[i++] = ' ';
5583	buf[size] = '\0';
5584
5585	return buf;
5586}
5587
5588struct hist_val_stat {
5589	u64 max;
5590	u64 total;
5591};
5592
5593static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5594				   const char *field_name, unsigned long flags,
5595				   struct hist_val_stat *stats,
5596				   struct tracing_map_elt *elt)
5597{
5598	u64 val = tracing_map_read_sum(elt, idx);
5599	unsigned int pc;
5600	char bar[21];
5601
5602	if (flags & HIST_FIELD_FL_PERCENT) {
5603		pc = __get_percentage(val, stats[idx].total);
5604		if (pc == UINT_MAX)
5605			seq_printf(m, " %s (%%):[ERROR]", field_name);
5606		else
5607			seq_printf(m, " %s (%%): %3u.%02u", field_name,
5608					pc / 100, pc % 100);
5609	} else if (flags & HIST_FIELD_FL_GRAPH) {
5610		seq_printf(m, " %s: %20s", field_name,
5611			   __fill_bar_str(bar, 20, val, stats[idx].max));
5612	} else if (flags & HIST_FIELD_FL_HEX) {
5613		seq_printf(m, " %s: %10llx", field_name, val);
5614	} else {
5615		seq_printf(m, " %s: %10llu", field_name, val);
5616	}
5617}
5618
5619static void hist_trigger_entry_print(struct seq_file *m,
5620				     struct hist_trigger_data *hist_data,
5621				     struct hist_val_stat *stats,
5622				     void *key,
5623				     struct tracing_map_elt *elt)
5624{
5625	const char *field_name;
5626	unsigned int i = HITCOUNT_IDX;
5627	unsigned long flags;
5628
5629	hist_trigger_print_key(m, hist_data, key, elt);
5630
5631	/* At first, show the raw hitcount if !nohitcount */
5632	if (!hist_data->attrs->no_hitcount)
5633		hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5634
5635	for (i = 1; i < hist_data->n_vals; i++) {
5636		field_name = hist_field_name(hist_data->fields[i], 0);
5637		flags = hist_data->fields[i]->flags;
5638		if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5639			continue;
5640
5641		seq_puts(m, " ");
5642		hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5643	}
5644
5645	print_actions(m, hist_data, elt);
5646
5647	seq_puts(m, "\n");
5648}
5649
5650static int print_entries(struct seq_file *m,
5651			 struct hist_trigger_data *hist_data)
5652{
5653	struct tracing_map_sort_entry **sort_entries = NULL;
5654	struct tracing_map *map = hist_data->map;
5655	int i, j, n_entries;
5656	struct hist_val_stat *stats = NULL;
5657	u64 val;
5658
5659	n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5660					     hist_data->n_sort_keys,
5661					     &sort_entries);
5662	if (n_entries < 0)
5663		return n_entries;
5664
5665	/* Calculate the max and the total for each field if needed. */
5666	for (j = 0; j < hist_data->n_vals; j++) {
5667		if (!(hist_data->fields[j]->flags &
5668			(HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5669			continue;
5670		if (!stats) {
5671			stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5672				       GFP_KERNEL);
5673			if (!stats) {
5674				n_entries = -ENOMEM;
5675				goto out;
5676			}
5677		}
5678		for (i = 0; i < n_entries; i++) {
5679			val = tracing_map_read_sum(sort_entries[i]->elt, j);
5680			stats[j].total += val;
5681			if (stats[j].max < val)
5682				stats[j].max = val;
5683		}
5684	}
5685
5686	for (i = 0; i < n_entries; i++)
5687		hist_trigger_entry_print(m, hist_data, stats,
5688					 sort_entries[i]->key,
5689					 sort_entries[i]->elt);
5690
5691	kfree(stats);
5692out:
5693	tracing_map_destroy_sort_entries(sort_entries, n_entries);
5694
5695	return n_entries;
5696}
5697
5698static void hist_trigger_show(struct seq_file *m,
5699			      struct event_trigger_data *data, int n)
5700{
5701	struct hist_trigger_data *hist_data;
5702	int n_entries;
5703
5704	if (n > 0)
5705		seq_puts(m, "\n\n");
5706
5707	seq_puts(m, "# event histogram\n#\n# trigger info: ");
5708	data->cmd_ops->print(m, data);
5709	seq_puts(m, "#\n\n");
5710
5711	hist_data = data->private_data;
5712	n_entries = print_entries(m, hist_data);
5713	if (n_entries < 0)
5714		n_entries = 0;
5715
5716	track_data_snapshot_print(m, hist_data);
5717
5718	seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5719		   (u64)atomic64_read(&hist_data->map->hits),
5720		   n_entries, (u64)atomic64_read(&hist_data->map->drops));
5721}
5722
5723struct hist_file_data {
5724	struct file *file;
5725	u64 last_read;
5726	u64 last_act;
5727};
5728
5729static u64 get_hist_hit_count(struct trace_event_file *event_file)
5730{
5731	struct hist_trigger_data *hist_data;
5732	struct event_trigger_data *data;
5733	u64 ret = 0;
5734
5735	list_for_each_entry(data, &event_file->triggers, list) {
5736		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5737			hist_data = data->private_data;
5738			ret += atomic64_read(&hist_data->map->hits);
5739		}
5740	}
5741	return ret;
5742}
5743
5744static int hist_show(struct seq_file *m, void *v)
5745{
5746	struct hist_file_data *hist_file = m->private;
5747	struct event_trigger_data *data;
5748	struct trace_event_file *event_file;
5749	int n = 0;
5750
5751	guard(mutex)(&event_mutex);
5752
5753	event_file = event_file_file(hist_file->file);
5754	if (unlikely(!event_file))
5755		return -ENODEV;
5756
5757	list_for_each_entry(data, &event_file->triggers, list) {
5758		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5759			hist_trigger_show(m, data, n++);
5760	}
5761	hist_file->last_read = get_hist_hit_count(event_file);
5762	/*
5763	 * Update last_act too so that poll()/POLLPRI can wait for the next
5764	 * event after any syscall on hist file.
5765	 */
5766	hist_file->last_act = hist_file->last_read;
5767
5768	return 0;
5769}
5770
5771static __poll_t event_hist_poll(struct file *file, struct poll_table_struct *wait)
5772{
5773	struct trace_event_file *event_file;
5774	struct seq_file *m = file->private_data;
5775	struct hist_file_data *hist_file = m->private;
5776	__poll_t ret = 0;
5777	u64 cnt;
5778
5779	guard(mutex)(&event_mutex);
5780
5781	event_file = event_file_file(file);
5782	if (!event_file)
5783		return EPOLLERR;
5784
5785	hist_poll_wait(file, wait);
5786
5787	cnt = get_hist_hit_count(event_file);
5788	if (hist_file->last_read != cnt)
5789		ret |= EPOLLIN | EPOLLRDNORM;
5790	if (hist_file->last_act != cnt) {
5791		hist_file->last_act = cnt;
5792		ret |= EPOLLPRI;
5793	}
5794
5795	return ret;
5796}
5797
5798static int event_hist_release(struct inode *inode, struct file *file)
5799{
5800	struct seq_file *m = file->private_data;
5801	struct hist_file_data *hist_file = m->private;
5802
5803	kfree(hist_file);
5804	return tracing_single_release_file_tr(inode, file);
5805}
5806
5807static int event_hist_open(struct inode *inode, struct file *file)
5808{
5809	struct trace_event_file *event_file;
5810	struct hist_file_data *hist_file;
5811	int ret;
5812
5813	ret = tracing_open_file_tr(inode, file);
5814	if (ret)
5815		return ret;
5816
5817	guard(mutex)(&event_mutex);
5818
5819	event_file = event_file_file(file);
5820	if (!event_file) {
5821		ret = -ENODEV;
5822		goto err;
5823	}
5824
5825	hist_file = kzalloc(sizeof(*hist_file), GFP_KERNEL);
5826	if (!hist_file) {
5827		ret = -ENOMEM;
5828		goto err;
5829	}
5830
5831	hist_file->file = file;
5832	hist_file->last_act = get_hist_hit_count(event_file);
5833
5834	/* Clear private_data to avoid warning in single_open() */
5835	file->private_data = NULL;
5836	ret = single_open(file, hist_show, hist_file);
5837	if (ret) {
5838		kfree(hist_file);
5839		goto err;
5840	}
5841
5842	return 0;
5843err:
5844	tracing_release_file_tr(inode, file);
5845	return ret;
5846}
5847
5848const struct file_operations event_hist_fops = {
5849	.open = event_hist_open,
5850	.read = seq_read,
5851	.llseek = seq_lseek,
5852	.release = event_hist_release,
5853	.poll = event_hist_poll,
5854};
5855
5856#ifdef CONFIG_HIST_TRIGGERS_DEBUG
5857static void hist_field_debug_show_flags(struct seq_file *m,
5858					unsigned long flags)
5859{
5860	seq_puts(m, "      flags:\n");
5861
5862	if (flags & HIST_FIELD_FL_KEY)
5863		seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5864	else if (flags & HIST_FIELD_FL_HITCOUNT)
5865		seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5866	else if (flags & HIST_FIELD_FL_VAR)
5867		seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5868	else if (flags & HIST_FIELD_FL_VAR_REF)
5869		seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5870	else
5871		seq_puts(m, "        VAL: normal u64 value\n");
5872
5873	if (flags & HIST_FIELD_FL_ALIAS)
5874		seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5875	else if (flags & HIST_FIELD_FL_CONST)
5876		seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5877}
5878
5879static int hist_field_debug_show(struct seq_file *m,
5880				 struct hist_field *field, unsigned long flags)
5881{
5882	if ((field->flags & flags) != flags) {
5883		seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5884		return -EINVAL;
5885	}
5886
5887	hist_field_debug_show_flags(m, field->flags);
5888	if (field->field)
5889		seq_printf(m, "      ftrace_event_field name: %s\n",
5890			   field->field->name);
5891
5892	if (field->flags & HIST_FIELD_FL_VAR) {
5893		seq_printf(m, "      var.name: %s\n", field->var.name);
5894		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5895			   field->var.idx);
5896	}
5897
5898	if (field->flags & HIST_FIELD_FL_CONST)
5899		seq_printf(m, "      constant: %llu\n", field->constant);
5900
5901	if (field->flags & HIST_FIELD_FL_ALIAS)
5902		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5903			   field->var_ref_idx);
5904
5905	if (field->flags & HIST_FIELD_FL_VAR_REF) {
5906		seq_printf(m, "      name: %s\n", field->name);
5907		seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5908			   field->var.idx);
5909		seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5910		seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5911			   field->var_ref_idx);
5912		if (field->system)
5913			seq_printf(m, "      system: %s\n", field->system);
5914		if (field->event_name)
5915			seq_printf(m, "      event_name: %s\n", field->event_name);
5916	}
5917
5918	seq_printf(m, "      type: %s\n", field->type);
5919	seq_printf(m, "      size: %u\n", field->size);
5920	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5921
5922	return 0;
5923}
5924
5925static int field_var_debug_show(struct seq_file *m,
5926				struct field_var *field_var, unsigned int i,
5927				bool save_vars)
5928{
5929	const char *vars_name = save_vars ? "save_vars" : "field_vars";
5930	struct hist_field *field;
5931	int ret = 0;
5932
5933	seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5934
5935	field = field_var->var;
5936
5937	seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5938
5939	hist_field_debug_show_flags(m, field->flags);
5940	seq_printf(m, "      var.name: %s\n", field->var.name);
5941	seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5942		   field->var.idx);
5943
5944	field = field_var->val;
5945
5946	seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5947	if (field->field)
5948		seq_printf(m, "      ftrace_event_field name: %s\n",
5949			   field->field->name);
5950	else {
5951		ret = -EINVAL;
5952		goto out;
5953	}
5954
5955	seq_printf(m, "      type: %s\n", field->type);
5956	seq_printf(m, "      size: %u\n", field->size);
5957	seq_printf(m, "      is_signed: %u\n", field->is_signed);
5958out:
5959	return ret;
5960}
5961
5962static int hist_action_debug_show(struct seq_file *m,
5963				  struct action_data *data, int i)
5964{
5965	int ret = 0;
5966
5967	if (data->handler == HANDLER_ONMAX ||
5968	    data->handler == HANDLER_ONCHANGE) {
5969		seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5970		ret = hist_field_debug_show(m, data->track_data.var_ref,
5971					    HIST_FIELD_FL_VAR_REF);
5972		if (ret)
5973			goto out;
5974
5975		seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5976		ret = hist_field_debug_show(m, data->track_data.track_var,
5977					    HIST_FIELD_FL_VAR);
5978		if (ret)
5979			goto out;
5980	}
5981
5982	if (data->handler == HANDLER_ONMATCH) {
5983		seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5984			   i, data->match_data.event_system);
5985		seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5986			   i, data->match_data.event);
5987	}
5988out:
5989	return ret;
5990}
5991
5992static int hist_actions_debug_show(struct seq_file *m,
5993				   struct hist_trigger_data *hist_data)
5994{
5995	int i, ret = 0;
5996
5997	if (hist_data->n_actions)
5998		seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5999
6000	for (i = 0; i < hist_data->n_actions; i++) {
6001		struct action_data *action = hist_data->actions[i];
6002
6003		ret = hist_action_debug_show(m, action, i);
6004		if (ret)
6005			goto out;
6006	}
6007
6008	if (hist_data->n_save_vars)
6009		seq_puts(m, "\n  save action variables (save() params):\n");
6010
6011	for (i = 0; i < hist_data->n_save_vars; i++) {
6012		ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
6013		if (ret)
6014			goto out;
6015	}
6016out:
6017	return ret;
6018}
6019
6020static void hist_trigger_debug_show(struct seq_file *m,
6021				    struct event_trigger_data *data, int n)
6022{
6023	struct hist_trigger_data *hist_data;
6024	int i, ret;
6025
6026	if (n > 0)
6027		seq_puts(m, "\n\n");
6028
6029	seq_puts(m, "# event histogram\n#\n# trigger info: ");
6030	data->cmd_ops->print(m, data);
6031	seq_puts(m, "#\n\n");
6032
6033	hist_data = data->private_data;
6034
6035	seq_printf(m, "hist_data: %p\n\n", hist_data);
6036	seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
6037	seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
6038	seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
6039
6040	seq_puts(m, "\n  val fields:\n\n");
6041
6042	seq_puts(m, "    hist_data->fields[0]:\n");
6043	ret = hist_field_debug_show(m, hist_data->fields[0],
6044				    HIST_FIELD_FL_HITCOUNT);
6045	if (ret)
6046		return;
6047
6048	for (i = 1; i < hist_data->n_vals; i++) {
6049		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
6050		ret = hist_field_debug_show(m, hist_data->fields[i], 0);
6051		if (ret)
6052			return;
6053	}
6054
6055	seq_puts(m, "\n  key fields:\n");
6056
6057	for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
6058		seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
6059		ret = hist_field_debug_show(m, hist_data->fields[i],
6060					    HIST_FIELD_FL_KEY);
6061		if (ret)
6062			return;
6063	}
6064
6065	if (hist_data->n_var_refs)
6066		seq_puts(m, "\n  variable reference fields:\n");
6067
6068	for (i = 0; i < hist_data->n_var_refs; i++) {
6069		seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
6070		ret = hist_field_debug_show(m, hist_data->var_refs[i],
6071					    HIST_FIELD_FL_VAR_REF);
6072		if (ret)
6073			return;
6074	}
6075
6076	if (hist_data->n_field_vars)
6077		seq_puts(m, "\n  field variables:\n");
6078
6079	for (i = 0; i < hist_data->n_field_vars; i++) {
6080		ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
6081		if (ret)
6082			return;
6083	}
6084
6085	ret = hist_actions_debug_show(m, hist_data);
6086	if (ret)
6087		return;
6088}
6089
6090static int hist_debug_show(struct seq_file *m, void *v)
6091{
6092	struct event_trigger_data *data;
6093	struct trace_event_file *event_file;
6094	int n = 0;
6095
6096	guard(mutex)(&event_mutex);
6097
6098	event_file = event_file_file(m->private);
6099	if (unlikely(!event_file))
6100		return -ENODEV;
6101
6102	list_for_each_entry(data, &event_file->triggers, list) {
6103		if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
6104			hist_trigger_debug_show(m, data, n++);
6105	}
6106	return 0;
6107}
6108
6109static int event_hist_debug_open(struct inode *inode, struct file *file)
6110{
6111	int ret;
6112
6113	ret = tracing_open_file_tr(inode, file);
6114	if (ret)
6115		return ret;
6116
6117	/* Clear private_data to avoid warning in single_open() */
6118	file->private_data = NULL;
6119	ret = single_open(file, hist_debug_show, file);
6120	if (ret)
6121		tracing_release_file_tr(inode, file);
6122	return ret;
6123}
6124
6125const struct file_operations event_hist_debug_fops = {
6126	.open = event_hist_debug_open,
6127	.read = seq_read,
6128	.llseek = seq_lseek,
6129	.release = tracing_single_release_file_tr,
6130};
6131#endif
6132
6133static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
6134{
6135	const char *field_name = hist_field_name(hist_field, 0);
6136
6137	if (hist_field->var.name)
6138		seq_printf(m, "%s=", hist_field->var.name);
6139
6140	if (hist_field->flags & HIST_FIELD_FL_CPU)
6141		seq_puts(m, "common_cpu");
6142	if (hist_field->flags & HIST_FIELD_FL_COMM)
6143		seq_puts(m, "common_comm");
6144	else if (hist_field->flags & HIST_FIELD_FL_CONST)
6145		seq_printf(m, "%llu", hist_field->constant);
6146	else if (field_name) {
6147		if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
6148		    hist_field->flags & HIST_FIELD_FL_ALIAS)
6149			seq_putc(m, '$');
6150		seq_printf(m, "%s", field_name);
6151	} else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
6152		seq_puts(m, "common_timestamp");
6153
6154	if (hist_field->flags) {
6155		if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
6156		    !(hist_field->flags & HIST_FIELD_FL_EXPR) &&
6157		    !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
6158			const char *flags = get_hist_field_flags(hist_field);
6159
6160			if (flags)
6161				seq_printf(m, ".%s", flags);
6162		}
6163	}
6164	if (hist_field->buckets)
6165		seq_printf(m, "=%ld", hist_field->buckets);
6166}
6167
6168static int event_hist_trigger_print(struct seq_file *m,
6169				    struct event_trigger_data *data)
6170{
6171	struct hist_trigger_data *hist_data = data->private_data;
6172	struct hist_field *field;
6173	bool have_var = false;
6174	bool show_val = false;
6175	unsigned int i;
6176
6177	seq_puts(m, HIST_PREFIX);
6178
6179	if (data->name)
6180		seq_printf(m, "%s:", data->name);
6181
6182	seq_puts(m, "keys=");
6183
6184	for_each_hist_key_field(i, hist_data) {
6185		field = hist_data->fields[i];
6186
6187		if (i > hist_data->n_vals)
6188			seq_puts(m, ",");
6189
6190		if (field->flags & HIST_FIELD_FL_STACKTRACE) {
6191			if (field->field)
6192				seq_printf(m, "%s.stacktrace", field->field->name);
6193			else
6194				seq_puts(m, "common_stacktrace");
6195		} else
6196			hist_field_print(m, field);
6197	}
6198
6199	seq_puts(m, ":vals=");
6200
6201	for_each_hist_val_field(i, hist_data) {
6202		field = hist_data->fields[i];
6203		if (field->flags & HIST_FIELD_FL_VAR) {
6204			have_var = true;
6205			continue;
6206		}
6207
6208		if (i == HITCOUNT_IDX) {
6209			if (hist_data->attrs->no_hitcount)
6210				continue;
6211			seq_puts(m, "hitcount");
6212		} else {
6213			if (show_val)
6214				seq_puts(m, ",");
6215			hist_field_print(m, field);
6216		}
6217		show_val = true;
6218	}
6219
6220	if (have_var) {
6221		unsigned int n = 0;
6222
6223		seq_puts(m, ":");
6224
6225		for_each_hist_val_field(i, hist_data) {
6226			field = hist_data->fields[i];
6227
6228			if (field->flags & HIST_FIELD_FL_VAR) {
6229				if (n++)
6230					seq_puts(m, ",");
6231				hist_field_print(m, field);
6232			}
6233		}
6234	}
6235
6236	seq_puts(m, ":sort=");
6237
6238	for (i = 0; i < hist_data->n_sort_keys; i++) {
6239		struct tracing_map_sort_key *sort_key;
6240		unsigned int idx, first_key_idx;
6241
6242		/* skip VAR vals */
6243		first_key_idx = hist_data->n_vals - hist_data->n_vars;
6244
6245		sort_key = &hist_data->sort_keys[i];
6246		idx = sort_key->field_idx;
6247
6248		if (WARN_ON(idx >= HIST_FIELDS_MAX))
6249			return -EINVAL;
6250
6251		if (i > 0)
6252			seq_puts(m, ",");
6253
6254		if (idx == HITCOUNT_IDX)
6255			seq_puts(m, "hitcount");
6256		else {
6257			if (idx >= first_key_idx)
6258				idx += hist_data->n_vars;
6259			hist_field_print(m, hist_data->fields[idx]);
6260		}
6261
6262		if (sort_key->descending)
6263			seq_puts(m, ".descending");
6264	}
6265	seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6266	if (hist_data->enable_timestamps)
6267		seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6268	if (hist_data->attrs->no_hitcount)
6269		seq_puts(m, ":nohitcount");
6270
6271	print_actions_spec(m, hist_data);
6272
6273	if (data->filter_str)
6274		seq_printf(m, " if %s", data->filter_str);
6275
6276	if (data->paused)
6277		seq_puts(m, " [paused]");
6278	else
6279		seq_puts(m, " [active]");
6280
6281	seq_putc(m, '\n');
6282
6283	return 0;
6284}
6285
6286static int event_hist_trigger_init(struct event_trigger_data *data)
6287{
6288	struct hist_trigger_data *hist_data = data->private_data;
6289
6290	if (alloc_hist_pad() < 0)
6291		return -ENOMEM;
6292
6293	if (!data->ref && hist_data->attrs->name)
6294		save_named_trigger(hist_data->attrs->name, data);
6295
6296	data->ref++;
6297
6298	return 0;
6299}
6300
6301static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6302{
6303	struct trace_event_file *file;
6304	unsigned int i;
6305	char *cmd;
6306	int ret;
6307
6308	for (i = 0; i < hist_data->n_field_var_hists; i++) {
6309		file = hist_data->field_var_hists[i]->hist_data->event_file;
6310		cmd = hist_data->field_var_hists[i]->cmd;
6311		ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6312					       "!hist", "hist", cmd);
6313		WARN_ON_ONCE(ret < 0);
6314	}
6315}
6316
6317static void event_hist_trigger_free(struct event_trigger_data *data)
6318{
6319	struct hist_trigger_data *hist_data = data->private_data;
6320
6321	if (WARN_ON_ONCE(data->ref <= 0))
6322		return;
6323
6324	data->ref--;
6325	if (!data->ref) {
6326		if (data->name)
6327			del_named_trigger(data);
6328
6329		trigger_data_free(data);
6330
6331		remove_hist_vars(hist_data);
6332
6333		unregister_field_var_hists(hist_data);
6334
6335		destroy_hist_data(hist_data);
6336	}
6337	free_hist_pad();
6338}
6339
6340static int event_hist_trigger_named_init(struct event_trigger_data *data)
6341{
6342	int ret;
6343
6344	data->ref++;
6345
6346	save_named_trigger(data->named_data->name, data);
6347
6348	ret = event_hist_trigger_init(data->named_data);
6349	if (ret < 0) {
6350		kfree(data->cmd_ops);
6351		data->cmd_ops = &trigger_hist_cmd;
6352	}
6353
6354	return ret;
6355}
6356
6357static void event_hist_trigger_named_free(struct event_trigger_data *data)
6358{
6359	if (WARN_ON_ONCE(data->ref <= 0))
6360		return;
6361
6362	event_hist_trigger_free(data->named_data);
6363
6364	data->ref--;
6365	if (!data->ref) {
6366		struct event_command *cmd_ops = data->cmd_ops;
6367
6368		del_named_trigger(data);
6369		trigger_data_free(data);
6370		kfree(cmd_ops);
6371	}
6372}
6373
6374static void hist_clear(struct event_trigger_data *data)
6375{
6376	struct hist_trigger_data *hist_data = data->private_data;
6377
6378	if (data->name)
6379		pause_named_trigger(data);
6380
6381	tracepoint_synchronize_unregister();
6382
6383	tracing_map_clear(hist_data->map);
6384
6385	if (data->name)
6386		unpause_named_trigger(data);
6387}
6388
6389static bool compatible_field(struct ftrace_event_field *field,
6390			     struct ftrace_event_field *test_field)
6391{
6392	if (field == test_field)
6393		return true;
6394	if (field == NULL || test_field == NULL)
6395		return false;
6396	if (strcmp(field->name, test_field->name) != 0)
6397		return false;
6398	if (strcmp(field->type, test_field->type) != 0)
6399		return false;
6400	if (field->size != test_field->size)
6401		return false;
6402	if (field->is_signed != test_field->is_signed)
6403		return false;
6404
6405	return true;
6406}
6407
6408static bool hist_trigger_match(struct event_trigger_data *data,
6409			       struct event_trigger_data *data_test,
6410			       struct event_trigger_data *named_data,
6411			       bool ignore_filter)
6412{
6413	struct tracing_map_sort_key *sort_key, *sort_key_test;
6414	struct hist_trigger_data *hist_data, *hist_data_test;
6415	struct hist_field *key_field, *key_field_test;
6416	unsigned int i;
6417
6418	if (named_data && (named_data != data_test) &&
6419	    (named_data != data_test->named_data))
6420		return false;
6421
6422	if (!named_data && is_named_trigger(data_test))
6423		return false;
6424
6425	hist_data = data->private_data;
6426	hist_data_test = data_test->private_data;
6427
6428	if (hist_data->n_vals != hist_data_test->n_vals ||
6429	    hist_data->n_fields != hist_data_test->n_fields ||
6430	    hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6431		return false;
6432
6433	if (!ignore_filter) {
6434		if ((data->filter_str && !data_test->filter_str) ||
6435		   (!data->filter_str && data_test->filter_str))
6436			return false;
6437	}
6438
6439	for_each_hist_field(i, hist_data) {
6440		key_field = hist_data->fields[i];
6441		key_field_test = hist_data_test->fields[i];
6442
6443		if (key_field->flags != key_field_test->flags)
6444			return false;
6445		if (!compatible_field(key_field->field, key_field_test->field))
6446			return false;
6447		if (key_field->offset != key_field_test->offset)
6448			return false;
6449		if (key_field->size != key_field_test->size)
6450			return false;
6451		if (key_field->is_signed != key_field_test->is_signed)
6452			return false;
6453		if (!!key_field->var.name != !!key_field_test->var.name)
6454			return false;
6455		if (key_field->var.name &&
6456		    strcmp(key_field->var.name, key_field_test->var.name) != 0)
6457			return false;
6458	}
6459
6460	for (i = 0; i < hist_data->n_sort_keys; i++) {
6461		sort_key = &hist_data->sort_keys[i];
6462		sort_key_test = &hist_data_test->sort_keys[i];
6463
6464		if (sort_key->field_idx != sort_key_test->field_idx ||
6465		    sort_key->descending != sort_key_test->descending)
6466			return false;
6467	}
6468
6469	if (!ignore_filter && data->filter_str &&
6470	    (strcmp(data->filter_str, data_test->filter_str) != 0))
6471		return false;
6472
6473	if (!actions_match(hist_data, hist_data_test))
6474		return false;
6475
6476	return true;
6477}
6478
6479static bool existing_hist_update_only(char *glob,
6480				      struct event_trigger_data *data,
6481				      struct trace_event_file *file)
6482{
6483	struct hist_trigger_data *hist_data = data->private_data;
6484	struct event_trigger_data *test, *named_data = NULL;
6485	bool updated = false;
6486
6487	if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6488	    !hist_data->attrs->clear)
6489		goto out;
6490
6491	if (hist_data->attrs->name) {
6492		named_data = find_named_trigger(hist_data->attrs->name);
6493		if (named_data) {
6494			if (!hist_trigger_match(data, named_data, named_data,
6495						true))
6496				goto out;
6497		}
6498	}
6499
6500	if (hist_data->attrs->name && !named_data)
6501		goto out;
6502
6503	list_for_each_entry(test, &file->triggers, list) {
6504		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6505			if (!hist_trigger_match(data, test, named_data, false))
6506				continue;
6507			if (hist_data->attrs->pause)
6508				test->paused = true;
6509			else if (hist_data->attrs->cont)
6510				test->paused = false;
6511			else if (hist_data->attrs->clear)
6512				hist_clear(test);
6513			updated = true;
6514			goto out;
6515		}
6516	}
6517 out:
6518	return updated;
6519}
6520
6521static int hist_register_trigger(char *glob,
6522				 struct event_trigger_data *data,
6523				 struct trace_event_file *file)
6524{
6525	struct hist_trigger_data *hist_data = data->private_data;
6526	struct event_trigger_data *test, *named_data = NULL;
6527	struct trace_array *tr = file->tr;
6528	int ret = 0;
6529
6530	if (hist_data->attrs->name) {
6531		named_data = find_named_trigger(hist_data->attrs->name);
6532		if (named_data) {
6533			if (!hist_trigger_match(data, named_data, named_data,
6534						true)) {
6535				hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6536				ret = -EINVAL;
6537				goto out;
6538			}
6539		}
6540	}
6541
6542	if (hist_data->attrs->name && !named_data)
6543		goto new;
6544
6545	lockdep_assert_held(&event_mutex);
6546
6547	list_for_each_entry(test, &file->triggers, list) {
6548		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6549			if (hist_trigger_match(data, test, named_data, false)) {
6550				hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6551				ret = -EEXIST;
6552				goto out;
6553			}
6554		}
6555	}
6556 new:
6557	if (hist_data->attrs->cont || hist_data->attrs->clear) {
6558		hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6559		ret = -ENOENT;
6560		goto out;
6561	}
6562
6563	if (hist_data->attrs->pause)
6564		data->paused = true;
6565
6566	if (named_data) {
6567		struct event_command *cmd_ops;
6568
6569		data->private_data = named_data->private_data;
6570		set_named_trigger_data(data, named_data);
6571		/* Copy the command ops and update some of the functions */
6572		cmd_ops = kmalloc(sizeof(*cmd_ops), GFP_KERNEL);
6573		if (!cmd_ops) {
6574			ret = -ENOMEM;
6575			goto out;
6576		}
6577		*cmd_ops = *data->cmd_ops;
6578		cmd_ops->init = event_hist_trigger_named_init;
6579		cmd_ops->free = event_hist_trigger_named_free;
6580		data->cmd_ops = cmd_ops;
6581	}
6582
6583	if (data->cmd_ops->init) {
6584		ret = data->cmd_ops->init(data);
6585		if (ret < 0)
6586			goto out;
6587	}
6588
6589	if (hist_data->enable_timestamps) {
6590		char *clock = hist_data->attrs->clock;
6591
6592		ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6593		if (ret) {
6594			hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6595			goto out;
6596		}
6597
6598		tracing_set_filter_buffering(file->tr, true);
6599	}
6600
6601	if (named_data)
6602		destroy_hist_data(hist_data);
6603 out:
6604	return ret;
6605}
6606
6607static int hist_trigger_enable(struct event_trigger_data *data,
6608			       struct trace_event_file *file)
6609{
6610	int ret = 0;
6611
6612	list_add_tail_rcu(&data->list, &file->triggers);
6613
6614	update_cond_flag(file);
6615
6616	if (trace_event_trigger_enable_disable(file, 1) < 0) {
6617		list_del_rcu(&data->list);
6618		update_cond_flag(file);
6619		ret--;
6620	}
6621
6622	return ret;
6623}
6624
6625static bool have_hist_trigger_match(struct event_trigger_data *data,
6626				    struct trace_event_file *file)
6627{
6628	struct hist_trigger_data *hist_data = data->private_data;
6629	struct event_trigger_data *test, *named_data = NULL;
6630	bool match = false;
6631
6632	lockdep_assert_held(&event_mutex);
6633
6634	if (hist_data->attrs->name)
6635		named_data = find_named_trigger(hist_data->attrs->name);
6636
6637	list_for_each_entry(test, &file->triggers, list) {
6638		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6639			if (hist_trigger_match(data, test, named_data, false)) {
6640				match = true;
6641				break;
6642			}
6643		}
6644	}
6645
6646	return match;
6647}
6648
6649static bool hist_trigger_check_refs(struct event_trigger_data *data,
6650				    struct trace_event_file *file)
6651{
6652	struct hist_trigger_data *hist_data = data->private_data;
6653	struct event_trigger_data *test, *named_data = NULL;
6654
6655	lockdep_assert_held(&event_mutex);
6656
6657	if (hist_data->attrs->name)
6658		named_data = find_named_trigger(hist_data->attrs->name);
6659
6660	list_for_each_entry(test, &file->triggers, list) {
6661		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6662			if (!hist_trigger_match(data, test, named_data, false))
6663				continue;
6664			hist_data = test->private_data;
6665			if (check_var_refs(hist_data))
6666				return true;
6667			break;
6668		}
6669	}
6670
6671	return false;
6672}
6673
6674static void hist_unregister_trigger(char *glob,
6675				    struct event_trigger_data *data,
6676				    struct trace_event_file *file)
6677{
6678	struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6679	struct hist_trigger_data *hist_data = data->private_data;
6680
6681	lockdep_assert_held(&event_mutex);
6682
6683	if (hist_data->attrs->name)
6684		named_data = find_named_trigger(hist_data->attrs->name);
6685
6686	list_for_each_entry(iter, &file->triggers, list) {
6687		if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6688			if (!hist_trigger_match(data, iter, named_data, false))
6689				continue;
6690			test = iter;
6691			list_del_rcu(&test->list);
6692			trace_event_trigger_enable_disable(file, 0);
6693			update_cond_flag(file);
6694			break;
6695		}
6696	}
6697
6698	if (test && test->cmd_ops->free)
6699		test->cmd_ops->free(test);
6700
6701	if (hist_data->enable_timestamps) {
6702		if (!hist_data->remove || test)
6703			tracing_set_filter_buffering(file->tr, false);
6704	}
6705}
6706
6707static bool hist_file_check_refs(struct trace_event_file *file)
6708{
6709	struct hist_trigger_data *hist_data;
6710	struct event_trigger_data *test;
6711
6712	lockdep_assert_held(&event_mutex);
6713
6714	list_for_each_entry(test, &file->triggers, list) {
6715		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6716			hist_data = test->private_data;
6717			if (check_var_refs(hist_data))
6718				return true;
6719		}
6720	}
6721
6722	return false;
6723}
6724
6725static void hist_unreg_all(struct trace_event_file *file)
6726{
6727	struct event_trigger_data *test, *n;
6728	struct hist_trigger_data *hist_data;
6729	struct synth_event *se;
6730	const char *se_name;
6731
6732	lockdep_assert_held(&event_mutex);
6733
6734	if (hist_file_check_refs(file))
6735		return;
6736
6737	list_for_each_entry_safe(test, n, &file->triggers, list) {
6738		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6739			hist_data = test->private_data;
6740			list_del_rcu(&test->list);
6741			trace_event_trigger_enable_disable(file, 0);
6742
6743			se_name = trace_event_name(file->event_call);
6744			se = find_synth_event(se_name);
6745			if (se)
6746				se->ref--;
6747
6748			update_cond_flag(file);
6749			if (hist_data->enable_timestamps)
6750				tracing_set_filter_buffering(file->tr, false);
6751			if (test->cmd_ops->free)
6752				test->cmd_ops->free(test);
6753		}
6754	}
6755}
6756
6757static int event_hist_trigger_parse(struct event_command *cmd_ops,
6758				    struct trace_event_file *file,
6759				    char *glob, char *cmd,
6760				    char *param_and_filter)
6761{
6762	unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6763	struct event_trigger_data *trigger_data;
6764	struct hist_trigger_attrs *attrs;
6765	struct hist_trigger_data *hist_data;
6766	char *param, *filter, *p, *start;
6767	struct synth_event *se;
6768	const char *se_name;
6769	bool remove;
6770	int ret = 0;
6771
6772	lockdep_assert_held(&event_mutex);
6773
6774	if (WARN_ON(!glob))
6775		return -EINVAL;
6776
6777	if (glob[0]) {
6778		hist_err_clear();
6779		last_cmd_set(file, param_and_filter);
6780	}
6781
6782	remove = event_trigger_check_remove(glob);
6783
6784	if (event_trigger_empty_param(param_and_filter))
6785		return -EINVAL;
6786
6787	/*
6788	 * separate the trigger from the filter (k:v [if filter])
6789	 * allowing for whitespace in the trigger
6790	 */
6791	p = param = param_and_filter;
6792	do {
6793		p = strstr(p, "if");
6794		if (!p)
6795			break;
6796		if (p == param_and_filter)
6797			return -EINVAL;
6798		if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6799			p++;
6800			continue;
6801		}
6802		if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6803			return -EINVAL;
6804		if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6805			p++;
6806			continue;
6807		}
6808		break;
6809	} while (1);
6810
6811	if (!p)
6812		filter = NULL;
6813	else {
6814		*(p - 1) = '\0';
6815		filter = strstrip(p);
6816		param = strstrip(param);
6817	}
6818
6819	/*
6820	 * To simplify arithmetic expression parsing, replace occurrences of
6821	 * '.sym-offset' modifier with '.symXoffset'
6822	 */
6823	start = strstr(param, ".sym-offset");
6824	while (start) {
6825		*(start + 4) = 'X';
6826		start = strstr(start + 11, ".sym-offset");
6827	}
6828
6829	attrs = parse_hist_trigger_attrs(file->tr, param);
6830	if (IS_ERR(attrs))
6831		return PTR_ERR(attrs);
6832
6833	if (attrs->map_bits)
6834		hist_trigger_bits = attrs->map_bits;
6835
6836	hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6837	if (IS_ERR(hist_data)) {
6838		destroy_hist_trigger_attrs(attrs);
6839		return PTR_ERR(hist_data);
6840	}
6841
6842	trigger_data = trigger_data_alloc(cmd_ops, cmd, param, hist_data);
6843	if (!trigger_data) {
6844		ret = -ENOMEM;
6845		goto out_free;
6846	}
6847
6848	ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6849	if (ret < 0)
6850		goto out_free;
6851
6852	if (remove) {
6853		if (!have_hist_trigger_match(trigger_data, file))
6854			goto out_free;
6855
6856		if (hist_trigger_check_refs(trigger_data, file)) {
6857			ret = -EBUSY;
6858			goto out_free;
6859		}
6860
6861		event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6862		se_name = trace_event_name(file->event_call);
6863		se = find_synth_event(se_name);
6864		if (se)
6865			se->ref--;
6866		ret = 0;
6867		goto out_free;
6868	}
6869
6870	if (existing_hist_update_only(glob, trigger_data, file))
6871		goto out_free;
6872
6873	if (!get_named_trigger_data(trigger_data)) {
6874
6875		ret = create_actions(hist_data);
6876		if (ret)
6877			goto out_free;
6878
6879		if (has_hist_vars(hist_data) || hist_data->n_var_refs) {
6880			ret = save_hist_vars(hist_data);
6881			if (ret)
6882				goto out_free;
6883		}
6884
6885		ret = tracing_map_init(hist_data->map);
6886		if (ret)
6887			goto out_free;
6888	}
6889
6890	ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6891	if (ret < 0)
6892		goto out_free;
6893
6894	ret = hist_trigger_enable(trigger_data, file);
6895	if (ret)
6896		goto out_unreg;
6897
6898	se_name = trace_event_name(file->event_call);
6899	se = find_synth_event(se_name);
6900	if (se)
6901		se->ref++;
6902 out:
6903	if (ret == 0 && glob[0])
6904		hist_err_clear();
6905
6906	return ret;
6907 out_unreg:
6908	event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6909 out_free:
6910	event_trigger_reset_filter(cmd_ops, trigger_data);
6911
6912	remove_hist_vars(hist_data);
6913
6914	trigger_data_free(trigger_data);
6915
6916	destroy_hist_data(hist_data);
6917	goto out;
6918}
6919
6920static struct event_command trigger_hist_cmd = {
6921	.name			= "hist",
6922	.trigger_type		= ETT_EVENT_HIST,
6923	.flags			= EVENT_CMD_FL_NEEDS_REC,
6924	.parse			= event_hist_trigger_parse,
6925	.reg			= hist_register_trigger,
6926	.unreg			= hist_unregister_trigger,
6927	.unreg_all		= hist_unreg_all,
6928	.set_filter		= set_trigger_filter,
6929	.trigger		= event_hist_trigger,
6930	.print			= event_hist_trigger_print,
6931	.init			= event_hist_trigger_init,
6932	.free			= event_hist_trigger_free,
6933};
6934
6935__init int register_trigger_hist_cmd(void)
6936{
6937	int ret;
6938
6939	ret = register_event_command(&trigger_hist_cmd);
6940	WARN_ON(ret < 0);
6941
6942	return ret;
6943}
6944
6945static void
6946hist_enable_trigger(struct event_trigger_data *data,
6947		    struct trace_buffer *buffer,  void *rec,
6948		    struct ring_buffer_event *event)
6949{
6950	struct enable_trigger_data *enable_data = data->private_data;
6951	struct event_trigger_data *test;
6952
6953	list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6954				lockdep_is_held(&event_mutex)) {
6955		if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6956			if (enable_data->enable)
6957				test->paused = false;
6958			else
6959				test->paused = true;
6960		}
6961	}
6962}
6963
6964static void hist_enable_unreg_all(struct trace_event_file *file)
6965{
6966	struct event_trigger_data *test, *n;
6967
6968	list_for_each_entry_safe(test, n, &file->triggers, list) {
6969		if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6970			list_del_rcu(&test->list);
6971			update_cond_flag(file);
6972			trace_event_trigger_enable_disable(file, 0);
6973			if (test->cmd_ops->free)
6974				test->cmd_ops->free(test);
6975		}
6976	}
6977}
6978
6979static struct event_command trigger_hist_enable_cmd = {
6980	.name			= ENABLE_HIST_STR,
6981	.trigger_type		= ETT_HIST_ENABLE,
6982	.parse			= event_enable_trigger_parse,
6983	.reg			= event_enable_register_trigger,
6984	.unreg			= event_enable_unregister_trigger,
6985	.unreg_all		= hist_enable_unreg_all,
6986	.set_filter		= set_trigger_filter,
6987	.trigger		= hist_enable_trigger,
6988	.count_func		= event_trigger_count,
6989	.print			= event_enable_trigger_print,
6990	.init			= event_trigger_init,
6991	.free			= event_enable_trigger_free,
6992};
6993
6994static struct event_command trigger_hist_disable_cmd = {
6995	.name			= DISABLE_HIST_STR,
6996	.trigger_type		= ETT_HIST_ENABLE,
6997	.parse			= event_enable_trigger_parse,
6998	.reg			= event_enable_register_trigger,
6999	.unreg			= event_enable_unregister_trigger,
7000	.unreg_all		= hist_enable_unreg_all,
7001	.set_filter		= set_trigger_filter,
7002	.trigger		= hist_enable_trigger,
7003	.count_func		= event_trigger_count,
7004	.print			= event_enable_trigger_print,
7005	.init			= event_trigger_init,
7006	.free			= event_enable_trigger_free,
7007};
7008
7009static __init void unregister_trigger_hist_enable_disable_cmds(void)
7010{
7011	unregister_event_command(&trigger_hist_enable_cmd);
7012	unregister_event_command(&trigger_hist_disable_cmd);
7013}
7014
7015__init int register_trigger_hist_enable_disable_cmds(void)
7016{
7017	int ret;
7018
7019	ret = register_event_command(&trigger_hist_enable_cmd);
7020	if (WARN_ON(ret < 0))
7021		return ret;
7022	ret = register_event_command(&trigger_hist_disable_cmd);
7023	if (WARN_ON(ret < 0))
7024		unregister_trigger_hist_enable_disable_cmds();
7025
7026	return ret;
7027}