perf_event.h

   1/*
   2 * Performance events x86 architecture header
   3 *
   4 *  Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <tglx@kernel.org>
   5 *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6 *  Copyright (C) 2009 Jaswinder Singh Rajput
   7 *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8 *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
   9 *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10 *  Copyright (C) 2009 Google, Inc., Stephane Eranian
  11 *
  12 *  For licencing details see kernel-base/COPYING
  13 */
  14
  15#include <linux/perf_event.h>
  16
  17#include <asm/fpu/xstate.h>
  18#include <asm/intel_ds.h>
  19#include <asm/cpu.h>
  20#include <asm/msr.h>
  21
  22/* To enable MSR tracing please use the generic trace points. */
  23
  24/*
  25 *          |   NHM/WSM    |      SNB     |
  26 * register -------------------------------
  27 *          |  HT  | no HT |  HT  | no HT |
  28 *-----------------------------------------
  29 * offcore  | core | core  | cpu  | core  |
  30 * lbr_sel  | core | core  | cpu  | core  |
  31 * ld_lat   | cpu  | core  | cpu  | core  |
  32 *-----------------------------------------
  33 *
  34 * Given that there is a small number of shared regs,
  35 * we can pre-allocate their slot in the per-cpu
  36 * per-core reg tables.
  37 */
  38enum extra_reg_type {
  39	EXTRA_REG_NONE		= -1, /* not used */
  40
  41	EXTRA_REG_RSP_0		= 0,  /* offcore_response_0 */
  42	EXTRA_REG_RSP_1		= 1,  /* offcore_response_1 */
  43	EXTRA_REG_LBR		= 2,  /* lbr_select */
  44	EXTRA_REG_LDLAT		= 3,  /* ld_lat_threshold */
  45	EXTRA_REG_FE		= 4,  /* fe_* */
  46	EXTRA_REG_SNOOP_0	= 5,  /* snoop response 0 */
  47	EXTRA_REG_SNOOP_1	= 6,  /* snoop response 1 */
  48
  49	EXTRA_REG_MAX		      /* number of entries needed */
  50};
  51
  52struct event_constraint {
  53	union {
  54		unsigned long	idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  55		u64		idxmsk64;
  56	};
  57	u64		code;
  58	u64		cmask;
  59	int		weight;
  60	int		overlap;
  61	int		flags;
  62	unsigned int	size;
  63};
  64
  65static inline bool constraint_match(struct event_constraint *c, u64 ecode)
  66{
  67	return ((ecode & c->cmask) - c->code) <= (u64)c->size;
  68}
  69
  70#define PERF_ARCH(name, val)	\
  71	PERF_X86_EVENT_##name = val,
  72
  73/*
  74 * struct hw_perf_event.flags flags
  75 */
  76enum {
  77#include "perf_event_flags.h"
  78};
  79
  80#undef PERF_ARCH
  81
  82#define PERF_ARCH(name, val)						\
  83	static_assert((PERF_X86_EVENT_##name & PERF_EVENT_FLAG_ARCH) ==	\
  84		      PERF_X86_EVENT_##name);
  85
  86#include "perf_event_flags.h"
  87
  88#undef PERF_ARCH
  89
  90static inline bool is_topdown_count(struct perf_event *event)
  91{
  92	return event->hw.flags & PERF_X86_EVENT_TOPDOWN;
  93}
  94
  95static inline bool is_metric_event(struct perf_event *event)
  96{
  97	u64 config = event->attr.config;
  98
  99	return ((config & ARCH_PERFMON_EVENTSEL_EVENT) == 0) &&
 100		((config & INTEL_ARCH_EVENT_MASK) >= INTEL_TD_METRIC_RETIRING)  &&
 101		((config & INTEL_ARCH_EVENT_MASK) <= INTEL_TD_METRIC_MAX);
 102}
 103
 104static inline bool is_slots_event(struct perf_event *event)
 105{
 106	return (event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_TD_SLOTS;
 107}
 108
 109static inline bool is_topdown_event(struct perf_event *event)
 110{
 111	return is_metric_event(event) || is_slots_event(event);
 112}
 113
 114int is_x86_event(struct perf_event *event);
 115
 116static inline bool check_leader_group(struct perf_event *leader, int flags)
 117{
 118	return is_x86_event(leader) ? !!(leader->hw.flags & flags) : false;
 119}
 120
 121static inline bool is_branch_counters_group(struct perf_event *event)
 122{
 123	return check_leader_group(event->group_leader, PERF_X86_EVENT_BRANCH_COUNTERS);
 124}
 125
 126static inline bool is_pebs_counter_event_group(struct perf_event *event)
 127{
 128	return check_leader_group(event->group_leader, PERF_X86_EVENT_PEBS_CNTR);
 129}
 130
 131static inline bool is_acr_event_group(struct perf_event *event)
 132{
 133	return check_leader_group(event->group_leader, PERF_X86_EVENT_ACR);
 134}
 135
 136struct amd_nb {
 137	int nb_id;  /* NorthBridge id */
 138	int refcnt; /* reference count */
 139	struct perf_event *owners[X86_PMC_IDX_MAX];
 140	struct event_constraint event_constraints[X86_PMC_IDX_MAX];
 141};
 142
 143#define PEBS_COUNTER_MASK	((1ULL << MAX_PEBS_EVENTS) - 1)
 144#define PEBS_PMI_AFTER_EACH_RECORD BIT_ULL(60)
 145#define PEBS_OUTPUT_OFFSET	61
 146#define PEBS_OUTPUT_MASK	(3ull << PEBS_OUTPUT_OFFSET)
 147#define PEBS_OUTPUT_PT		(1ull << PEBS_OUTPUT_OFFSET)
 148#define PEBS_VIA_PT_MASK	(PEBS_OUTPUT_PT | PEBS_PMI_AFTER_EACH_RECORD)
 149
 150/*
 151 * Flags PEBS can handle without an PMI.
 152 *
 153 * TID can only be handled by flushing at context switch.
 154 * REGS_USER can be handled for events limited to ring 3.
 155 *
 156 */
 157#define LARGE_PEBS_FLAGS \
 158	(PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
 159	PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
 160	PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
 161	PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
 162	PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
 163	PERF_SAMPLE_PERIOD | PERF_SAMPLE_CODE_PAGE_SIZE | \
 164	PERF_SAMPLE_WEIGHT_TYPE)
 165
 166#define PEBS_GP_REGS			\
 167	((1ULL << PERF_REG_X86_AX)    | \
 168	 (1ULL << PERF_REG_X86_BX)    | \
 169	 (1ULL << PERF_REG_X86_CX)    | \
 170	 (1ULL << PERF_REG_X86_DX)    | \
 171	 (1ULL << PERF_REG_X86_DI)    | \
 172	 (1ULL << PERF_REG_X86_SI)    | \
 173	 (1ULL << PERF_REG_X86_SP)    | \
 174	 (1ULL << PERF_REG_X86_BP)    | \
 175	 (1ULL << PERF_REG_X86_IP)    | \
 176	 (1ULL << PERF_REG_X86_FLAGS) | \
 177	 (1ULL << PERF_REG_X86_R8)    | \
 178	 (1ULL << PERF_REG_X86_R9)    | \
 179	 (1ULL << PERF_REG_X86_R10)   | \
 180	 (1ULL << PERF_REG_X86_R11)   | \
 181	 (1ULL << PERF_REG_X86_R12)   | \
 182	 (1ULL << PERF_REG_X86_R13)   | \
 183	 (1ULL << PERF_REG_X86_R14)   | \
 184	 (1ULL << PERF_REG_X86_R15))
 185
 186/*
 187 * Per register state.
 188 */
 189struct er_account {
 190	raw_spinlock_t      lock;	/* per-core: protect structure */
 191	u64                 config;	/* extra MSR config */
 192	u64                 reg;	/* extra MSR number */
 193	atomic_t            ref;	/* reference count */
 194};
 195
 196/*
 197 * Per core/cpu state
 198 *
 199 * Used to coordinate shared registers between HT threads or
 200 * among events on a single PMU.
 201 */
 202struct intel_shared_regs {
 203	struct er_account       regs[EXTRA_REG_MAX];
 204	int                     refcnt;		/* per-core: #HT threads */
 205	unsigned                core_id;	/* per-core: core id */
 206};
 207
 208enum intel_excl_state_type {
 209	INTEL_EXCL_UNUSED    = 0, /* counter is unused */
 210	INTEL_EXCL_SHARED    = 1, /* counter can be used by both threads */
 211	INTEL_EXCL_EXCLUSIVE = 2, /* counter can be used by one thread only */
 212};
 213
 214struct intel_excl_states {
 215	enum intel_excl_state_type state[X86_PMC_IDX_MAX];
 216	bool sched_started; /* true if scheduling has started */
 217};
 218
 219struct intel_excl_cntrs {
 220	raw_spinlock_t	lock;
 221
 222	struct intel_excl_states states[2];
 223
 224	union {
 225		u16	has_exclusive[2];
 226		u32	exclusive_present;
 227	};
 228
 229	int		refcnt;		/* per-core: #HT threads */
 230	unsigned	core_id;	/* per-core: core id */
 231};
 232
 233struct x86_perf_task_context;
 234#define MAX_LBR_ENTRIES		32
 235
 236enum {
 237	LBR_FORMAT_32		= 0x00,
 238	LBR_FORMAT_LIP		= 0x01,
 239	LBR_FORMAT_EIP		= 0x02,
 240	LBR_FORMAT_EIP_FLAGS	= 0x03,
 241	LBR_FORMAT_EIP_FLAGS2	= 0x04,
 242	LBR_FORMAT_INFO		= 0x05,
 243	LBR_FORMAT_TIME		= 0x06,
 244	LBR_FORMAT_INFO2	= 0x07,
 245	LBR_FORMAT_MAX_KNOWN    = LBR_FORMAT_INFO2,
 246};
 247
 248enum {
 249	X86_PERF_KFREE_SHARED = 0,
 250	X86_PERF_KFREE_EXCL   = 1,
 251	X86_PERF_KFREE_MAX
 252};
 253
 254struct cpu_hw_events {
 255	/*
 256	 * Generic x86 PMC bits
 257	 */
 258	struct perf_event	*events[X86_PMC_IDX_MAX]; /* in counter order */
 259	unsigned long		active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 260	unsigned long		dirty[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 261	int			enabled;
 262
 263	int			n_events; /* the # of events in the below arrays */
 264	int			n_added;  /* the # last events in the below arrays;
 265					     they've never been enabled yet */
 266	int			n_txn;    /* the # last events in the below arrays;
 267					     added in the current transaction */
 268	int			n_txn_pair;
 269	int			n_txn_metric;
 270	int			assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
 271	u64			tags[X86_PMC_IDX_MAX];
 272
 273	struct perf_event	*event_list[X86_PMC_IDX_MAX]; /* in enabled order */
 274	struct event_constraint	*event_constraint[X86_PMC_IDX_MAX];
 275
 276	int			n_excl; /* the number of exclusive events */
 277	int			n_late_setup; /* the num of events needs late setup */
 278
 279	unsigned int		txn_flags;
 280	int			is_fake;
 281
 282	/*
 283	 * Intel DebugStore bits
 284	 */
 285	struct debug_store	*ds;
 286	void			*ds_bts_vaddr;
 287	/* DS based PEBS or arch-PEBS buffer address */
 288	void			*pebs_vaddr;
 289	u64			pebs_enabled;
 290	int			n_pebs;
 291	int			n_large_pebs;
 292	int			n_pebs_via_pt;
 293	int			pebs_output;
 294
 295	/* Current super set of events hardware configuration */
 296	u64			pebs_data_cfg;
 297	u64			active_pebs_data_cfg;
 298	int			pebs_record_size;
 299
 300	/* Intel Fixed counter configuration */
 301	u64			fixed_ctrl_val;
 302	u64			active_fixed_ctrl_val;
 303
 304	/* Intel ACR configuration */
 305	u64			acr_cfg_b[X86_PMC_IDX_MAX];
 306	u64			acr_cfg_c[X86_PMC_IDX_MAX];
 307	/* Cached CFG_C values */
 308	u64			cfg_c_val[X86_PMC_IDX_MAX];
 309
 310	/*
 311	 * Intel LBR bits
 312	 */
 313	int				lbr_users;
 314	int				lbr_pebs_users;
 315	struct perf_branch_stack	lbr_stack;
 316	struct perf_branch_entry	lbr_entries[MAX_LBR_ENTRIES];
 317	u64				lbr_counters[MAX_LBR_ENTRIES]; /* branch stack extra */
 318	union {
 319		struct er_account		*lbr_sel;
 320		struct er_account		*lbr_ctl;
 321	};
 322	u64				br_sel;
 323	void				*last_task_ctx;
 324	int				last_log_id;
 325	int				lbr_select;
 326	void				*lbr_xsave;
 327
 328	/*
 329	 * Intel host/guest exclude bits
 330	 */
 331	u64				intel_ctrl_guest_mask;
 332	u64				intel_ctrl_host_mask;
 333	struct perf_guest_switch_msr	guest_switch_msrs[X86_PMC_IDX_MAX];
 334
 335	/*
 336	 * Intel checkpoint mask
 337	 */
 338	u64				intel_cp_status;
 339
 340	/*
 341	 * manage shared (per-core, per-cpu) registers
 342	 * used on Intel NHM/WSM/SNB
 343	 */
 344	struct intel_shared_regs	*shared_regs;
 345	/*
 346	 * manage exclusive counter access between hyperthread
 347	 */
 348	struct event_constraint *constraint_list; /* in enable order */
 349	struct intel_excl_cntrs		*excl_cntrs;
 350	int excl_thread_id; /* 0 or 1 */
 351
 352	/*
 353	 * SKL TSX_FORCE_ABORT shadow
 354	 */
 355	u64				tfa_shadow;
 356
 357	/*
 358	 * Perf Metrics
 359	 */
 360	/* number of accepted metrics events */
 361	int				n_metric;
 362
 363	/*
 364	 * AMD specific bits
 365	 */
 366	struct amd_nb			*amd_nb;
 367	int				brs_active; /* BRS is enabled */
 368
 369	/* Inverted mask of bits to clear in the perf_ctr ctrl registers */
 370	u64				perf_ctr_virt_mask;
 371	int				n_pair; /* Large increment events */
 372
 373	void				*kfree_on_online[X86_PERF_KFREE_MAX];
 374
 375	struct pmu			*pmu;
 376};
 377
 378#define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) {	\
 379	{ .idxmsk64 = (n) },		\
 380	.code = (c),			\
 381	.size = (e) - (c),		\
 382	.cmask = (m),			\
 383	.weight = (w),			\
 384	.overlap = (o),			\
 385	.flags = f,			\
 386}
 387
 388#define __EVENT_CONSTRAINT(c, n, m, w, o, f) \
 389	__EVENT_CONSTRAINT_RANGE(c, c, n, m, w, o, f)
 390
 391#define EVENT_CONSTRAINT(c, n, m)	\
 392	__EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
 393
 394/*
 395 * The constraint_match() function only works for 'simple' event codes
 396 * and not for extended (AMD64_EVENTSEL_EVENT) events codes.
 397 */
 398#define EVENT_CONSTRAINT_RANGE(c, e, n, m) \
 399	__EVENT_CONSTRAINT_RANGE(c, e, n, m, HWEIGHT(n), 0, 0)
 400
 401#define INTEL_EXCLEVT_CONSTRAINT(c, n)	\
 402	__EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\
 403			   0, PERF_X86_EVENT_EXCL)
 404
 405/*
 406 * The overlap flag marks event constraints with overlapping counter
 407 * masks. This is the case if the counter mask of such an event is not
 408 * a subset of any other counter mask of a constraint with an equal or
 409 * higher weight, e.g.:
 410 *
 411 *  c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
 412 *  c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
 413 *  c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
 414 *
 415 * The event scheduler may not select the correct counter in the first
 416 * cycle because it needs to know which subsequent events will be
 417 * scheduled. It may fail to schedule the events then. So we set the
 418 * overlap flag for such constraints to give the scheduler a hint which
 419 * events to select for counter rescheduling.
 420 *
 421 * Care must be taken as the rescheduling algorithm is O(n!) which
 422 * will increase scheduling cycles for an over-committed system
 423 * dramatically.  The number of such EVENT_CONSTRAINT_OVERLAP() macros
 424 * and its counter masks must be kept at a minimum.
 425 */
 426#define EVENT_CONSTRAINT_OVERLAP(c, n, m)	\
 427	__EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
 428
 429/*
 430 * Constraint on the Event code.
 431 */
 432#define INTEL_EVENT_CONSTRAINT(c, n)	\
 433	EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
 434
 435/*
 436 * Constraint on a range of Event codes
 437 */
 438#define INTEL_EVENT_CONSTRAINT_RANGE(c, e, n)			\
 439	EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT)
 440
 441/*
 442 * Constraint on the Event code + UMask + fixed-mask
 443 *
 444 * filter mask to validate fixed counter events.
 445 * the following filters disqualify for fixed counters:
 446 *  - inv
 447 *  - edge
 448 *  - cnt-mask
 449 *  - in_tx
 450 *  - in_tx_checkpointed
 451 *  The other filters are supported by fixed counters.
 452 *  The any-thread option is supported starting with v3.
 453 */
 454#define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
 455#define FIXED_EVENT_CONSTRAINT(c, n)	\
 456	EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
 457
 458/*
 459 * The special metric counters do not actually exist. They are calculated from
 460 * the combination of the FxCtr3 + MSR_PERF_METRICS.
 461 *
 462 * The special metric counters are mapped to a dummy offset for the scheduler.
 463 * The sharing between multiple users of the same metric without multiplexing
 464 * is not allowed, even though the hardware supports that in principle.
 465 */
 466
 467#define METRIC_EVENT_CONSTRAINT(c, n)					\
 468	EVENT_CONSTRAINT(c, (1ULL << (INTEL_PMC_IDX_METRIC_BASE + n)),	\
 469			 INTEL_ARCH_EVENT_MASK)
 470
 471/*
 472 * Constraint on the Event code + UMask
 473 */
 474#define INTEL_UEVENT_CONSTRAINT(c, n)	\
 475	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
 476
 477/* Constraint on specific umask bit only + event */
 478#define INTEL_UBIT_EVENT_CONSTRAINT(c, n)	\
 479	EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c))
 480
 481/* Like UEVENT_CONSTRAINT, but match flags too */
 482#define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n)	\
 483	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
 484
 485#define INTEL_EXCLUEVT_CONSTRAINT(c, n)	\
 486	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
 487			   HWEIGHT(n), 0, PERF_X86_EVENT_EXCL)
 488
 489#define INTEL_PLD_CONSTRAINT(c, n)	\
 490	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 491			   HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
 492
 493#define INTEL_PSD_CONSTRAINT(c, n)	\
 494	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 495			   HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_STLAT)
 496
 497#define INTEL_PST_CONSTRAINT(c, n)	\
 498	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 499			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
 500
 501#define INTEL_HYBRID_LAT_CONSTRAINT(c, n)	\
 502	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 503			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID)
 504
 505#define INTEL_HYBRID_LDLAT_CONSTRAINT(c, n)	\
 506	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 507			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_LD_HSW)
 508
 509#define INTEL_HYBRID_STLAT_CONSTRAINT(c, n)	\
 510	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 511			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LAT_HYBRID|PERF_X86_EVENT_PEBS_ST_HSW)
 512
 513/* Event constraint, but match on all event flags too. */
 514#define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
 515	EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
 516
 517#define INTEL_FLAGS_EVENT_CONSTRAINT_RANGE(c, e, n)			\
 518	EVENT_CONSTRAINT_RANGE(c, e, n, ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS)
 519
 520/* Check only flags, but allow all event/umask */
 521#define INTEL_ALL_EVENT_CONSTRAINT(code, n)	\
 522	EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
 523
 524/* Check flags and event code, and set the HSW store flag */
 525#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
 526	__EVENT_CONSTRAINT(code, n, 			\
 527			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
 528			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
 529
 530/* Check flags and event code, and set the HSW load flag */
 531#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
 532	__EVENT_CONSTRAINT(code, n,			\
 533			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
 534			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
 535
 536#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(code, end, n) \
 537	__EVENT_CONSTRAINT_RANGE(code, end, n,				\
 538			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
 539			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
 540
 541#define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \
 542	__EVENT_CONSTRAINT(code, n,			\
 543			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
 544			  HWEIGHT(n), 0, \
 545			  PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
 546
 547/* Check flags and event code/umask, and set the HSW store flag */
 548#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
 549	__EVENT_CONSTRAINT(code, n, 			\
 550			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 551			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
 552
 553#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \
 554	__EVENT_CONSTRAINT(code, n,			\
 555			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 556			  HWEIGHT(n), 0, \
 557			  PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL)
 558
 559/* Check flags and event code/umask, and set the HSW load flag */
 560#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
 561	__EVENT_CONSTRAINT(code, n, 			\
 562			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 563			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
 564
 565#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \
 566	__EVENT_CONSTRAINT(code, n,			\
 567			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 568			  HWEIGHT(n), 0, \
 569			  PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
 570
 571/* Check flags and event code/umask, and set the HSW N/A flag */
 572#define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
 573	__EVENT_CONSTRAINT(code, n, 			\
 574			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
 575			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
 576
 577
 578/*
 579 * We define the end marker as having a weight of -1
 580 * to enable blacklisting of events using a counter bitmask
 581 * of zero and thus a weight of zero.
 582 * The end marker has a weight that cannot possibly be
 583 * obtained from counting the bits in the bitmask.
 584 */
 585#define EVENT_CONSTRAINT_END { .weight = -1 }
 586
 587/*
 588 * Check for end marker with weight == -1
 589 */
 590#define for_each_event_constraint(e, c)	\
 591	for ((e) = (c); (e)->weight != -1; (e)++)
 592
 593/*
 594 * Extra registers for specific events.
 595 *
 596 * Some events need large masks and require external MSRs.
 597 * Those extra MSRs end up being shared for all events on
 598 * a PMU and sometimes between PMU of sibling HT threads.
 599 * In either case, the kernel needs to handle conflicting
 600 * accesses to those extra, shared, regs. The data structure
 601 * to manage those registers is stored in cpu_hw_event.
 602 */
 603struct extra_reg {
 604	unsigned int		event;
 605	unsigned int		msr;
 606	u64			config_mask;
 607	u64			valid_mask;
 608	int			idx;  /* per_xxx->regs[] reg index */
 609	bool			extra_msr_access;
 610};
 611
 612#define EVENT_EXTRA_REG(e, ms, m, vm, i) {	\
 613	.event = (e),			\
 614	.msr = (ms),			\
 615	.config_mask = (m),		\
 616	.valid_mask = (vm),		\
 617	.idx = EXTRA_REG_##i,		\
 618	.extra_msr_access = true,	\
 619	}
 620
 621#define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx)	\
 622	EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
 623
 624#define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
 625	EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
 626			ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
 627
 628#define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
 629	INTEL_UEVENT_EXTRA_REG(c, \
 630			       MSR_PEBS_LD_LAT_THRESHOLD, \
 631			       0xffff, \
 632			       LDLAT)
 633
 634#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
 635
 636union perf_capabilities {
 637	struct {
 638		u64	lbr_format:6;
 639		u64	pebs_trap:1;
 640		u64	pebs_arch_reg:1;
 641		u64	pebs_format:4;
 642		u64	smm_freeze:1;
 643		/*
 644		 * PMU supports separate counter range for writing
 645		 * values > 32bit.
 646		 */
 647		u64	full_width_write:1;
 648		u64     pebs_baseline:1;
 649		u64	perf_metrics:1;
 650		u64	pebs_output_pt_available:1;
 651		u64	pebs_timing_info:1;
 652		u64	anythread_deprecated:1;
 653		u64	rdpmc_metrics_clear:1;
 654	};
 655	u64	capabilities;
 656};
 657
 658struct x86_pmu_quirk {
 659	struct x86_pmu_quirk *next;
 660	void (*func)(void);
 661};
 662
 663union x86_pmu_config {
 664	struct {
 665		u64 event:8,
 666		    umask:8,
 667		    usr:1,
 668		    os:1,
 669		    edge:1,
 670		    pc:1,
 671		    interrupt:1,
 672		    __reserved1:1,
 673		    en:1,
 674		    inv:1,
 675		    cmask:8,
 676		    event2:4,
 677		    __reserved2:4,
 678		    go:1,
 679		    ho:1;
 680	} bits;
 681	u64 value;
 682};
 683
 684#define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
 685
 686enum {
 687	x86_lbr_exclusive_lbr,
 688	x86_lbr_exclusive_bts,
 689	x86_lbr_exclusive_pt,
 690	x86_lbr_exclusive_max,
 691};
 692
 693#define PERF_PEBS_DATA_SOURCE_MAX	0x100
 694#define PERF_PEBS_DATA_SOURCE_MASK	(PERF_PEBS_DATA_SOURCE_MAX - 1)
 695#define PERF_PEBS_DATA_SOURCE_GRT_MAX	0x10
 696#define PERF_PEBS_DATA_SOURCE_GRT_MASK	(PERF_PEBS_DATA_SOURCE_GRT_MAX - 1)
 697
 698#define X86_HYBRID_PMU_ATOM_IDX		0
 699#define X86_HYBRID_PMU_CORE_IDX		1
 700#define X86_HYBRID_PMU_TINY_IDX		2
 701
 702enum hybrid_pmu_type {
 703	not_hybrid,
 704	hybrid_small		= BIT(X86_HYBRID_PMU_ATOM_IDX),
 705	hybrid_big		= BIT(X86_HYBRID_PMU_CORE_IDX),
 706	hybrid_tiny		= BIT(X86_HYBRID_PMU_TINY_IDX),
 707
 708	/* The belows are only used for matching */
 709	hybrid_big_small	= hybrid_big   | hybrid_small,
 710	hybrid_small_tiny	= hybrid_small | hybrid_tiny,
 711	hybrid_big_small_tiny	= hybrid_big   | hybrid_small_tiny,
 712};
 713
 714struct arch_pebs_cap {
 715	u64 caps;
 716	u64 counters;
 717	u64 pdists;
 718};
 719
 720struct x86_hybrid_pmu {
 721	struct pmu			pmu;
 722	const char			*name;
 723	enum hybrid_pmu_type		pmu_type;
 724	cpumask_t			supported_cpus;
 725	union perf_capabilities		intel_cap;
 726	u64				intel_ctrl;
 727	u64				pebs_events_mask;
 728	u64				config_mask;
 729	union {
 730			u64		cntr_mask64;
 731			unsigned long	cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 732	};
 733	union {
 734			u64		fixed_cntr_mask64;
 735			unsigned long	fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 736	};
 737
 738	union {
 739			u64		acr_cntr_mask64;
 740			unsigned long	acr_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 741	};
 742	union {
 743			u64		acr_cause_mask64;
 744			unsigned long	acr_cause_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 745	};
 746	struct event_constraint		unconstrained;
 747
 748	u64				hw_cache_event_ids
 749					[PERF_COUNT_HW_CACHE_MAX]
 750					[PERF_COUNT_HW_CACHE_OP_MAX]
 751					[PERF_COUNT_HW_CACHE_RESULT_MAX];
 752	u64				hw_cache_extra_regs
 753					[PERF_COUNT_HW_CACHE_MAX]
 754					[PERF_COUNT_HW_CACHE_OP_MAX]
 755					[PERF_COUNT_HW_CACHE_RESULT_MAX];
 756	struct event_constraint		*event_constraints;
 757	struct event_constraint		*pebs_constraints;
 758	struct extra_reg		*extra_regs;
 759
 760	unsigned int			late_ack	:1,
 761					mid_ack		:1,
 762					enabled_ack	:1;
 763
 764	struct arch_pebs_cap		arch_pebs_cap;
 765
 766	u64				pebs_data_source[PERF_PEBS_DATA_SOURCE_MAX];
 767};
 768
 769static __always_inline struct x86_hybrid_pmu *hybrid_pmu(struct pmu *pmu)
 770{
 771	return container_of(pmu, struct x86_hybrid_pmu, pmu);
 772}
 773
 774extern struct static_key_false perf_is_hybrid;
 775#define is_hybrid()		static_branch_unlikely(&perf_is_hybrid)
 776
 777#define hybrid(_pmu, _field)				\
 778(*({							\
 779	typeof(&x86_pmu._field) __Fp = &x86_pmu._field;	\
 780							\
 781	if (is_hybrid() && (_pmu))			\
 782		__Fp = &hybrid_pmu(_pmu)->_field;	\
 783							\
 784	__Fp;						\
 785}))
 786
 787#define hybrid_var(_pmu, _var)				\
 788(*({							\
 789	typeof(&_var) __Fp = &_var;			\
 790							\
 791	if (is_hybrid() && (_pmu))			\
 792		__Fp = &hybrid_pmu(_pmu)->_var;		\
 793							\
 794	__Fp;						\
 795}))
 796
 797#define hybrid_bit(_pmu, _field)			\
 798({							\
 799	bool __Fp = x86_pmu._field;			\
 800							\
 801	if (is_hybrid() && (_pmu))			\
 802		__Fp = hybrid_pmu(_pmu)->_field;	\
 803							\
 804	__Fp;						\
 805})
 806
 807/*
 808 * struct x86_pmu - generic x86 pmu
 809 */
 810struct x86_pmu {
 811	/*
 812	 * Generic x86 PMC bits
 813	 */
 814	const char	*name;
 815	int		version;
 816	int		(*handle_irq)(struct pt_regs *);
 817	void		(*disable_all)(void);
 818	void		(*enable_all)(int added);
 819	void		(*enable)(struct perf_event *);
 820	void		(*disable)(struct perf_event *);
 821	void		(*assign)(struct perf_event *event, int idx);
 822	void		(*add)(struct perf_event *);
 823	void		(*del)(struct perf_event *);
 824	void		(*read)(struct perf_event *event);
 825	int		(*set_period)(struct perf_event *event);
 826	u64		(*update)(struct perf_event *event);
 827	int		(*hw_config)(struct perf_event *event);
 828	int		(*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
 829	void		(*late_setup)(void);
 830	void		(*pebs_enable)(struct perf_event *event);
 831	void		(*pebs_disable)(struct perf_event *event);
 832	void		(*pebs_enable_all)(void);
 833	void		(*pebs_disable_all)(void);
 834	unsigned	eventsel;
 835	unsigned	perfctr;
 836	unsigned	fixedctr;
 837	int		(*addr_offset)(int index, bool eventsel);
 838	int		(*rdpmc_index)(int index);
 839	u64		(*event_map)(int);
 840	int		max_events;
 841	u64		config_mask;
 842	union {
 843			u64		cntr_mask64;
 844			unsigned long	cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 845	};
 846	union {
 847			u64		fixed_cntr_mask64;
 848			unsigned long	fixed_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 849	};
 850	union {
 851			u64		acr_cntr_mask64;
 852			unsigned long	acr_cntr_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 853	};
 854	union {
 855			u64		acr_cause_mask64;
 856			unsigned long	acr_cause_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
 857	};
 858	int		cntval_bits;
 859	u64		cntval_mask;
 860	union {
 861			unsigned long events_maskl;
 862			unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
 863	};
 864	int		events_mask_len;
 865	int		apic;
 866	u64		max_period;
 867	struct event_constraint *
 868			(*get_event_constraints)(struct cpu_hw_events *cpuc,
 869						 int idx,
 870						 struct perf_event *event);
 871
 872	void		(*put_event_constraints)(struct cpu_hw_events *cpuc,
 873						 struct perf_event *event);
 874
 875	void		(*start_scheduling)(struct cpu_hw_events *cpuc);
 876
 877	void		(*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr);
 878
 879	void		(*stop_scheduling)(struct cpu_hw_events *cpuc);
 880
 881	struct event_constraint *event_constraints;
 882	struct x86_pmu_quirk *quirks;
 883	void		(*limit_period)(struct perf_event *event, s64 *l);
 884
 885	/* PMI handler bits */
 886	unsigned int	late_ack		:1,
 887			mid_ack			:1,
 888			enabled_ack		:1;
 889	/*
 890	 * sysfs attrs
 891	 */
 892	int		attr_rdpmc_broken;
 893	int		attr_rdpmc;
 894	struct attribute **format_attrs;
 895
 896	ssize_t		(*events_sysfs_show)(char *page, u64 config);
 897	const struct attribute_group **attr_update;
 898
 899	unsigned long	attr_freeze_on_smi;
 900
 901	/*
 902	 * CPU Hotplug hooks
 903	 */
 904	int		(*cpu_prepare)(int cpu);
 905	void		(*cpu_starting)(int cpu);
 906	void		(*cpu_dying)(int cpu);
 907	void		(*cpu_dead)(int cpu);
 908
 909	void		(*check_microcode)(void);
 910	void		(*sched_task)(struct perf_event_pmu_context *pmu_ctx,
 911				      struct task_struct *task, bool sched_in);
 912
 913	/*
 914	 * Intel Arch Perfmon v2+
 915	 */
 916	u64			intel_ctrl;
 917	union perf_capabilities intel_cap;
 918
 919	/*
 920	 * Intel DebugStore and PEBS bits
 921	 */
 922	unsigned int	bts			:1,
 923			bts_active		:1,
 924			ds_pebs			:1,
 925			pebs_active		:1,
 926			pebs_broken		:1,
 927			pebs_prec_dist		:1,
 928			pebs_no_tlb		:1,
 929			pebs_no_isolation	:1,
 930			pebs_block		:1,
 931			pebs_ept		:1,
 932			arch_pebs		:1;
 933	int		pebs_record_size;
 934	int		pebs_buffer_size;
 935	u64		pebs_events_mask;
 936	void		(*drain_pebs)(struct pt_regs *regs, struct perf_sample_data *data);
 937	struct event_constraint *pebs_constraints;
 938	void		(*pebs_aliases)(struct perf_event *event);
 939	u64		(*pebs_latency_data)(struct perf_event *event, u64 status);
 940	unsigned long	large_pebs_flags;
 941	u64		rtm_abort_event;
 942	u64		pebs_capable;
 943
 944	/*
 945	 * Intel Architectural PEBS
 946	 */
 947	struct arch_pebs_cap arch_pebs_cap;
 948
 949	/*
 950	 * Intel LBR
 951	 */
 952	unsigned int	lbr_tos, lbr_from, lbr_to,
 953			lbr_info, lbr_nr;	   /* LBR base regs and size */
 954	union {
 955		u64	lbr_sel_mask;		   /* LBR_SELECT valid bits */
 956		u64	lbr_ctl_mask;		   /* LBR_CTL valid bits */
 957	};
 958	union {
 959		const int	*lbr_sel_map;	   /* lbr_select mappings */
 960		int		*lbr_ctl_map;	   /* LBR_CTL mappings */
 961	};
 962	u64		lbr_callstack_users;	   /* lbr callstack system wide users */
 963	bool		lbr_double_abort;	   /* duplicated lbr aborts */
 964	bool		lbr_pt_coexist;		   /* (LBR|BTS) may coexist with PT */
 965
 966	unsigned int	lbr_has_info:1;
 967	unsigned int	lbr_has_tsx:1;
 968	unsigned int	lbr_from_flags:1;
 969	unsigned int	lbr_to_cycles:1;
 970
 971	/*
 972	 * Intel Architectural LBR CPUID Enumeration
 973	 */
 974	unsigned int	lbr_depth_mask:8;
 975	unsigned int	lbr_deep_c_reset:1;
 976	unsigned int	lbr_lip:1;
 977	unsigned int	lbr_cpl:1;
 978	unsigned int	lbr_filter:1;
 979	unsigned int	lbr_call_stack:1;
 980	unsigned int	lbr_mispred:1;
 981	unsigned int	lbr_timed_lbr:1;
 982	unsigned int	lbr_br_type:1;
 983	unsigned int	lbr_counters:4;
 984
 985	void		(*lbr_reset)(void);
 986	void		(*lbr_read)(struct cpu_hw_events *cpuc);
 987	void		(*lbr_save)(void *ctx);
 988	void		(*lbr_restore)(void *ctx);
 989
 990	/*
 991	 * Intel PT/LBR/BTS are exclusive
 992	 */
 993	atomic_t	lbr_exclusive[x86_lbr_exclusive_max];
 994
 995	/*
 996	 * Intel perf metrics
 997	 */
 998	int		num_topdown_events;
 999
1000	/*
1001	 * AMD bits
1002	 */
1003	unsigned int	amd_nb_constraints : 1;
1004	u64		perf_ctr_pair_en;
1005
1006	/*
1007	 * Extra registers for events
1008	 */
1009	struct extra_reg *extra_regs;
1010	unsigned int flags;
1011
1012	/*
1013	 * Intel host/guest support (KVM)
1014	 */
1015	struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr, void *data);
1016
1017	/*
1018	 * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
1019	 */
1020	int (*check_period) (struct perf_event *event, u64 period);
1021
1022	int (*aux_output_match) (struct perf_event *event);
1023
1024	void (*filter)(struct pmu *pmu, int cpu, bool *ret);
1025	/*
1026	 * Hybrid support
1027	 *
1028	 * Most PMU capabilities are the same among different hybrid PMUs.
1029	 * The global x86_pmu saves the architecture capabilities, which
1030	 * are available for all PMUs. The hybrid_pmu only includes the
1031	 * unique capabilities.
1032	 */
1033	int				num_hybrid_pmus;
1034	struct x86_hybrid_pmu		*hybrid_pmu;
1035	enum intel_cpu_type (*get_hybrid_cpu_type)	(void);
1036};
1037
1038struct x86_perf_task_context_opt {
1039	int lbr_callstack_users;
1040	int lbr_stack_state;
1041	int log_id;
1042};
1043
1044struct x86_perf_task_context {
1045	u64 lbr_sel;
1046	int tos;
1047	int valid_lbrs;
1048	struct x86_perf_task_context_opt opt;
1049	struct lbr_entry lbr[MAX_LBR_ENTRIES];
1050};
1051
1052struct x86_perf_task_context_arch_lbr {
1053	struct x86_perf_task_context_opt opt;
1054	struct lbr_entry entries[];
1055};
1056
1057/*
1058 * Add padding to guarantee the 64-byte alignment of the state buffer.
1059 *
1060 * The structure is dynamically allocated. The size of the LBR state may vary
1061 * based on the number of LBR registers.
1062 *
1063 * Do not put anything after the LBR state.
1064 */
1065struct x86_perf_task_context_arch_lbr_xsave {
1066	struct x86_perf_task_context_opt		opt;
1067
1068	union {
1069		struct xregs_state			xsave;
1070		struct {
1071			struct fxregs_state		i387;
1072			struct xstate_header		header;
1073			struct arch_lbr_state		lbr;
1074		} __attribute__ ((packed, aligned (XSAVE_ALIGNMENT)));
1075	};
1076};
1077
1078#define x86_add_quirk(func_)						\
1079do {									\
1080	static struct x86_pmu_quirk __quirk __initdata = {		\
1081		.func = func_,						\
1082	};								\
1083	__quirk.next = x86_pmu.quirks;					\
1084	x86_pmu.quirks = &__quirk;					\
1085} while (0)
1086
1087/*
1088 * x86_pmu flags
1089 */
1090#define PMU_FL_NO_HT_SHARING	0x1 /* no hyper-threading resource sharing */
1091#define PMU_FL_HAS_RSP_1	0x2 /* has 2 equivalent offcore_rsp regs   */
1092#define PMU_FL_EXCL_CNTRS	0x4 /* has exclusive counter requirements  */
1093#define PMU_FL_EXCL_ENABLED	0x8 /* exclusive counter active */
1094#define PMU_FL_PEBS_ALL		0x10 /* all events are valid PEBS events */
1095#define PMU_FL_TFA		0x20 /* deal with TSX force abort */
1096#define PMU_FL_PAIR		0x40 /* merge counters for large incr. events */
1097#define PMU_FL_INSTR_LATENCY	0x80 /* Support Instruction Latency in PEBS Memory Info Record */
1098#define PMU_FL_MEM_LOADS_AUX	0x100 /* Require an auxiliary event for the complete memory info */
1099#define PMU_FL_RETIRE_LATENCY	0x200 /* Support Retire Latency in PEBS */
1100#define PMU_FL_BR_CNTR		0x400 /* Support branch counter logging */
1101#define PMU_FL_DYN_CONSTRAINT	0x800 /* Needs dynamic constraint */
1102
1103#define EVENT_VAR(_id)  event_attr_##_id
1104#define EVENT_PTR(_id) &event_attr_##_id.attr.attr
1105
1106#define EVENT_ATTR(_name, _id)						\
1107static struct perf_pmu_events_attr EVENT_VAR(_id) = {			\
1108	.attr		= __ATTR(_name, 0444, events_sysfs_show, NULL),	\
1109	.id		= PERF_COUNT_HW_##_id,				\
1110	.event_str	= NULL,						\
1111};
1112
1113#define EVENT_ATTR_STR(_name, v, str)					\
1114static struct perf_pmu_events_attr event_attr_##v = {			\
1115	.attr		= __ATTR(_name, 0444, events_sysfs_show, NULL),	\
1116	.id		= 0,						\
1117	.event_str	= str,						\
1118};
1119
1120#define EVENT_ATTR_STR_HT(_name, v, noht, ht)				\
1121static struct perf_pmu_events_ht_attr event_attr_##v = {		\
1122	.attr		= __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\
1123	.id		= 0,						\
1124	.event_str_noht	= noht,						\
1125	.event_str_ht	= ht,						\
1126}
1127
1128#define EVENT_ATTR_STR_HYBRID(_name, v, str, _pmu)			\
1129static struct perf_pmu_events_hybrid_attr event_attr_##v = {		\
1130	.attr		= __ATTR(_name, 0444, events_hybrid_sysfs_show, NULL),\
1131	.id		= 0,						\
1132	.event_str	= str,						\
1133	.pmu_type	= _pmu,						\
1134}
1135
1136#define FORMAT_HYBRID_PTR(_id) (&format_attr_hybrid_##_id.attr.attr)
1137
1138#define FORMAT_ATTR_HYBRID(_name, _pmu)					\
1139static struct perf_pmu_format_hybrid_attr format_attr_hybrid_##_name = {\
1140	.attr		= __ATTR_RO(_name),				\
1141	.pmu_type	= _pmu,						\
1142}
1143
1144struct pmu *x86_get_pmu(unsigned int cpu);
1145extern struct x86_pmu x86_pmu __read_mostly;
1146
1147DECLARE_STATIC_CALL(x86_pmu_set_period, *x86_pmu.set_period);
1148DECLARE_STATIC_CALL(x86_pmu_update,     *x86_pmu.update);
1149DECLARE_STATIC_CALL(x86_pmu_drain_pebs,	*x86_pmu.drain_pebs);
1150DECLARE_STATIC_CALL(x86_pmu_late_setup,	*x86_pmu.late_setup);
1151DECLARE_STATIC_CALL(x86_pmu_pebs_enable, *x86_pmu.pebs_enable);
1152DECLARE_STATIC_CALL(x86_pmu_pebs_disable, *x86_pmu.pebs_disable);
1153DECLARE_STATIC_CALL(x86_pmu_pebs_enable_all, *x86_pmu.pebs_enable_all);
1154DECLARE_STATIC_CALL(x86_pmu_pebs_disable_all, *x86_pmu.pebs_disable_all);
1155
1156static __always_inline struct x86_perf_task_context_opt *task_context_opt(void *ctx)
1157{
1158	if (static_cpu_has(X86_FEATURE_ARCH_LBR))
1159		return &((struct x86_perf_task_context_arch_lbr *)ctx)->opt;
1160
1161	return &((struct x86_perf_task_context *)ctx)->opt;
1162}
1163
1164static inline bool x86_pmu_has_lbr_callstack(void)
1165{
1166	return  x86_pmu.lbr_sel_map &&
1167		x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
1168}
1169
1170DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
1171DECLARE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1172
1173int x86_perf_event_set_period(struct perf_event *event);
1174
1175/*
1176 * Generalized hw caching related hw_event table, filled
1177 * in on a per model basis. A value of 0 means
1178 * 'not supported', -1 means 'hw_event makes no sense on
1179 * this CPU', any other value means the raw hw_event
1180 * ID.
1181 */
1182
1183#define C(x) PERF_COUNT_HW_CACHE_##x
1184
1185extern u64 __read_mostly hw_cache_event_ids
1186				[PERF_COUNT_HW_CACHE_MAX]
1187				[PERF_COUNT_HW_CACHE_OP_MAX]
1188				[PERF_COUNT_HW_CACHE_RESULT_MAX];
1189extern u64 __read_mostly hw_cache_extra_regs
1190				[PERF_COUNT_HW_CACHE_MAX]
1191				[PERF_COUNT_HW_CACHE_OP_MAX]
1192				[PERF_COUNT_HW_CACHE_RESULT_MAX];
1193
1194u64 x86_perf_event_update(struct perf_event *event);
1195
1196static inline u64 intel_pmu_topdown_event_update(struct perf_event *event, u64 *val)
1197{
1198	return x86_perf_event_update(event);
1199}
1200DECLARE_STATIC_CALL(intel_pmu_update_topdown_event, intel_pmu_topdown_event_update);
1201
1202static inline unsigned int x86_pmu_config_addr(int index)
1203{
1204	return x86_pmu.eventsel + (x86_pmu.addr_offset ?
1205				   x86_pmu.addr_offset(index, true) : index);
1206}
1207
1208static inline unsigned int x86_pmu_event_addr(int index)
1209{
1210	return x86_pmu.perfctr + (x86_pmu.addr_offset ?
1211				  x86_pmu.addr_offset(index, false) : index);
1212}
1213
1214static inline unsigned int x86_pmu_fixed_ctr_addr(int index)
1215{
1216	return x86_pmu.fixedctr + (x86_pmu.addr_offset ?
1217				   x86_pmu.addr_offset(index, false) : index);
1218}
1219
1220static inline int x86_pmu_rdpmc_index(int index)
1221{
1222	return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
1223}
1224
1225bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask,
1226		     unsigned long *fixed_cntr_mask);
1227
1228int x86_add_exclusive(unsigned int what);
1229
1230void x86_del_exclusive(unsigned int what);
1231
1232int x86_reserve_hardware(void);
1233
1234void x86_release_hardware(void);
1235
1236int x86_pmu_max_precise(struct pmu *pmu);
1237
1238void hw_perf_lbr_event_destroy(struct perf_event *event);
1239
1240int x86_setup_perfctr(struct perf_event *event);
1241
1242int x86_pmu_hw_config(struct perf_event *event);
1243
1244void x86_pmu_disable_all(void);
1245
1246static inline bool has_amd_brs(struct hw_perf_event *hwc)
1247{
1248	return hwc->flags & PERF_X86_EVENT_AMD_BRS;
1249}
1250
1251static inline bool is_counter_pair(struct hw_perf_event *hwc)
1252{
1253	return hwc->flags & PERF_X86_EVENT_PAIR;
1254}
1255
1256static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
1257					  u64 enable_mask)
1258{
1259	u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
1260
1261	if (hwc->extra_reg.reg)
1262		wrmsrq(hwc->extra_reg.reg, hwc->extra_reg.config);
1263
1264	/*
1265	 * Add enabled Merge event on next counter
1266	 * if large increment event being enabled on this counter
1267	 */
1268	if (is_counter_pair(hwc))
1269		wrmsrq(x86_pmu_config_addr(hwc->idx + 1), x86_pmu.perf_ctr_pair_en);
1270
1271	wrmsrq(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
1272}
1273
1274void x86_pmu_enable_all(int added);
1275
1276int perf_assign_events(struct event_constraint **constraints, int n,
1277			int wmin, int wmax, int gpmax, int *assign);
1278int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
1279
1280void x86_pmu_stop(struct perf_event *event, int flags);
1281
1282static inline void x86_pmu_disable_event(struct perf_event *event)
1283{
1284	u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
1285	struct hw_perf_event *hwc = &event->hw;
1286
1287	wrmsrq(hwc->config_base, hwc->config & ~disable_mask);
1288
1289	if (is_counter_pair(hwc))
1290		wrmsrq(x86_pmu_config_addr(hwc->idx + 1), 0);
1291}
1292
1293void x86_pmu_enable_event(struct perf_event *event);
1294
1295int x86_pmu_handle_irq(struct pt_regs *regs);
1296
1297void x86_pmu_show_pmu_cap(struct pmu *pmu);
1298
1299static inline int x86_pmu_num_counters(struct pmu *pmu)
1300{
1301	return hweight64(hybrid(pmu, cntr_mask64));
1302}
1303
1304static inline int x86_pmu_max_num_counters(struct pmu *pmu)
1305{
1306	return fls64(hybrid(pmu, cntr_mask64));
1307}
1308
1309static inline int x86_pmu_num_counters_fixed(struct pmu *pmu)
1310{
1311	return hweight64(hybrid(pmu, fixed_cntr_mask64));
1312}
1313
1314static inline int x86_pmu_max_num_counters_fixed(struct pmu *pmu)
1315{
1316	return fls64(hybrid(pmu, fixed_cntr_mask64));
1317}
1318
1319static inline u64 x86_pmu_get_event_config(struct perf_event *event)
1320{
1321	return event->attr.config & hybrid(event->pmu, config_mask);
1322}
1323
1324extern struct event_constraint emptyconstraint;
1325
1326extern struct event_constraint unconstrained;
1327
1328static inline bool kernel_ip(unsigned long ip)
1329{
1330#ifdef CONFIG_X86_32
1331	return ip > PAGE_OFFSET;
1332#else
1333	return (long)ip < 0;
1334#endif
1335}
1336
1337/*
1338 * Not all PMUs provide the right context information to place the reported IP
1339 * into full context. Specifically segment registers are typically not
1340 * supplied.
1341 *
1342 * Assuming the address is a linear address (it is for IBS), we fake the CS and
1343 * vm86 mode using the known zero-based code segment and 'fix up' the registers
1344 * to reflect this.
1345 *
1346 * Intel PEBS/LBR appear to typically provide the effective address, nothing
1347 * much we can do about that but pray and treat it like a linear address.
1348 */
1349static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
1350{
1351	regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
1352	if (regs->flags & X86_VM_MASK)
1353		regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
1354	regs->ip = ip;
1355}
1356
1357/*
1358 * x86control flow change classification
1359 * x86control flow changes include branches, interrupts, traps, faults
1360 */
1361enum {
1362	X86_BR_NONE		= 0,      /* unknown */
1363
1364	X86_BR_USER		= 1 << 0, /* branch target is user */
1365	X86_BR_KERNEL		= 1 << 1, /* branch target is kernel */
1366
1367	X86_BR_CALL		= 1 << 2, /* call */
1368	X86_BR_RET		= 1 << 3, /* return */
1369	X86_BR_SYSCALL		= 1 << 4, /* syscall */
1370	X86_BR_SYSRET		= 1 << 5, /* syscall return */
1371	X86_BR_INT		= 1 << 6, /* sw interrupt */
1372	X86_BR_IRET		= 1 << 7, /* return from interrupt */
1373	X86_BR_JCC		= 1 << 8, /* conditional */
1374	X86_BR_JMP		= 1 << 9, /* jump */
1375	X86_BR_IRQ		= 1 << 10,/* hw interrupt or trap or fault */
1376	X86_BR_IND_CALL		= 1 << 11,/* indirect calls */
1377	X86_BR_ABORT		= 1 << 12,/* transaction abort */
1378	X86_BR_IN_TX		= 1 << 13,/* in transaction */
1379	X86_BR_NO_TX		= 1 << 14,/* not in transaction */
1380	X86_BR_ZERO_CALL	= 1 << 15,/* zero length call */
1381	X86_BR_CALL_STACK	= 1 << 16,/* call stack */
1382	X86_BR_IND_JMP		= 1 << 17,/* indirect jump */
1383
1384	X86_BR_TYPE_SAVE	= 1 << 18,/* indicate to save branch type */
1385
1386};
1387
1388#define X86_BR_PLM (X86_BR_USER | X86_BR_KERNEL)
1389#define X86_BR_ANYTX (X86_BR_NO_TX | X86_BR_IN_TX)
1390
1391#define X86_BR_ANY       \
1392	(X86_BR_CALL    |\
1393	 X86_BR_RET     |\
1394	 X86_BR_SYSCALL |\
1395	 X86_BR_SYSRET  |\
1396	 X86_BR_INT     |\
1397	 X86_BR_IRET    |\
1398	 X86_BR_JCC     |\
1399	 X86_BR_JMP	 |\
1400	 X86_BR_IRQ	 |\
1401	 X86_BR_ABORT	 |\
1402	 X86_BR_IND_CALL |\
1403	 X86_BR_IND_JMP  |\
1404	 X86_BR_ZERO_CALL)
1405
1406#define X86_BR_ALL (X86_BR_PLM | X86_BR_ANY)
1407
1408#define X86_BR_ANY_CALL		 \
1409	(X86_BR_CALL		|\
1410	 X86_BR_IND_CALL	|\
1411	 X86_BR_ZERO_CALL	|\
1412	 X86_BR_SYSCALL		|\
1413	 X86_BR_IRQ		|\
1414	 X86_BR_INT)
1415
1416int common_branch_type(int type);
1417int branch_type(unsigned long from, unsigned long to, int abort);
1418int branch_type_fused(unsigned long from, unsigned long to, int abort,
1419		      int *offset);
1420
1421ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
1422ssize_t intel_event_sysfs_show(char *page, u64 config);
1423
1424ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
1425			  char *page);
1426ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1427			  char *page);
1428ssize_t events_hybrid_sysfs_show(struct device *dev,
1429				 struct device_attribute *attr,
1430				 char *page);
1431
1432static inline bool fixed_counter_disabled(int i, struct pmu *pmu)
1433{
1434	u64 intel_ctrl = hybrid(pmu, intel_ctrl);
1435
1436	return !(intel_ctrl >> (i + INTEL_PMC_IDX_FIXED));
1437}
1438
1439#ifdef CONFIG_CPU_SUP_AMD
1440
1441int amd_pmu_init(void);
1442
1443int amd_pmu_lbr_init(void);
1444void amd_pmu_lbr_reset(void);
1445void amd_pmu_lbr_read(void);
1446void amd_pmu_lbr_add(struct perf_event *event);
1447void amd_pmu_lbr_del(struct perf_event *event);
1448void amd_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
1449			    struct task_struct *task, bool sched_in);
1450void amd_pmu_lbr_enable_all(void);
1451void amd_pmu_lbr_disable_all(void);
1452int amd_pmu_lbr_hw_config(struct perf_event *event);
1453
1454static __always_inline void __amd_pmu_lbr_disable(void)
1455{
1456	u64 dbg_ctl, dbg_extn_cfg;
1457
1458	rdmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg);
1459	wrmsrq(MSR_AMD_DBG_EXTN_CFG, dbg_extn_cfg & ~DBG_EXTN_CFG_LBRV2EN);
1460
1461	if (cpu_feature_enabled(X86_FEATURE_AMD_LBR_PMC_FREEZE)) {
1462		rdmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl);
1463		wrmsrq(MSR_IA32_DEBUGCTLMSR, dbg_ctl & ~DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
1464	}
1465}
1466
1467#ifdef CONFIG_PERF_EVENTS_AMD_BRS
1468
1469#define AMD_FAM19H_BRS_EVENT 0xc4 /* RETIRED_TAKEN_BRANCH_INSTRUCTIONS */
1470
1471int amd_brs_init(void);
1472void amd_brs_disable(void);
1473void amd_brs_enable(void);
1474void amd_brs_enable_all(void);
1475void amd_brs_disable_all(void);
1476void amd_brs_drain(void);
1477void amd_brs_lopwr_init(void);
1478int amd_brs_hw_config(struct perf_event *event);
1479void amd_brs_reset(void);
1480
1481static inline void amd_pmu_brs_add(struct perf_event *event)
1482{
1483	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1484
1485	perf_sched_cb_inc(event->pmu);
1486	cpuc->lbr_users++;
1487	/*
1488	 * No need to reset BRS because it is reset
1489	 * on brs_enable() and it is saturating
1490	 */
1491}
1492
1493static inline void amd_pmu_brs_del(struct perf_event *event)
1494{
1495	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1496
1497	cpuc->lbr_users--;
1498	WARN_ON_ONCE(cpuc->lbr_users < 0);
1499
1500	perf_sched_cb_dec(event->pmu);
1501}
1502
1503void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
1504			    struct task_struct *task, bool sched_in);
1505#else
1506static inline int amd_brs_init(void)
1507{
1508	return 0;
1509}
1510static inline void amd_brs_disable(void) {}
1511static inline void amd_brs_enable(void) {}
1512static inline void amd_brs_drain(void) {}
1513static inline void amd_brs_lopwr_init(void) {}
1514static inline void amd_brs_disable_all(void) {}
1515static inline int amd_brs_hw_config(struct perf_event *event)
1516{
1517	return 0;
1518}
1519static inline void amd_brs_reset(void) {}
1520
1521static inline void amd_pmu_brs_add(struct perf_event *event)
1522{
1523}
1524
1525static inline void amd_pmu_brs_del(struct perf_event *event)
1526{
1527}
1528
1529static inline void amd_pmu_brs_sched_task(struct perf_event_pmu_context *pmu_ctx,
1530					  struct task_struct *task, bool sched_in)
1531{
1532}
1533
1534static inline void amd_brs_enable_all(void)
1535{
1536}
1537
1538#endif
1539
1540#else /* CONFIG_CPU_SUP_AMD */
1541
1542static inline int amd_pmu_init(void)
1543{
1544	return 0;
1545}
1546
1547static inline int amd_brs_init(void)
1548{
1549	return -EOPNOTSUPP;
1550}
1551
1552static inline void amd_brs_drain(void)
1553{
1554}
1555
1556static inline void amd_brs_enable_all(void)
1557{
1558}
1559
1560static inline void amd_brs_disable_all(void)
1561{
1562}
1563#endif /* CONFIG_CPU_SUP_AMD */
1564
1565static inline int is_pebs_pt(struct perf_event *event)
1566{
1567	return !!(event->hw.flags & PERF_X86_EVENT_PEBS_VIA_PT);
1568}
1569
1570#ifdef CONFIG_CPU_SUP_INTEL
1571
1572static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period)
1573{
1574	struct hw_perf_event *hwc = &event->hw;
1575	unsigned int hw_event, bts_event;
1576
1577	/*
1578	 * Only use BTS for fixed rate period==1 events.
1579	 */
1580	if (event->attr.freq || period != 1)
1581		return false;
1582
1583	/*
1584	 * BTS doesn't virtualize.
1585	 */
1586	if (event->attr.exclude_host)
1587		return false;
1588
1589	hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
1590	bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
1591
1592	return hw_event == bts_event;
1593}
1594
1595static inline bool intel_pmu_has_bts(struct perf_event *event)
1596{
1597	struct hw_perf_event *hwc = &event->hw;
1598
1599	return intel_pmu_has_bts_period(event, hwc->sample_period);
1600}
1601
1602static __always_inline void __intel_pmu_pebs_disable_all(void)
1603{
1604	wrmsrq(MSR_IA32_PEBS_ENABLE, 0);
1605}
1606
1607static __always_inline void __intel_pmu_arch_lbr_disable(void)
1608{
1609	wrmsrq(MSR_ARCH_LBR_CTL, 0);
1610}
1611
1612static __always_inline void __intel_pmu_lbr_disable(void)
1613{
1614	u64 debugctl;
1615
1616	rdmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
1617	debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
1618	wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
1619}
1620
1621int intel_pmu_save_and_restart(struct perf_event *event);
1622
1623struct event_constraint *
1624x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
1625			  struct perf_event *event);
1626
1627extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu);
1628extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
1629
1630int intel_pmu_init(void);
1631
1632int alloc_arch_pebs_buf_on_cpu(int cpu);
1633
1634void release_arch_pebs_buf_on_cpu(int cpu);
1635
1636void init_arch_pebs_on_cpu(int cpu);
1637
1638void fini_arch_pebs_on_cpu(int cpu);
1639
1640void init_debug_store_on_cpu(int cpu);
1641
1642void fini_debug_store_on_cpu(int cpu);
1643
1644void release_ds_buffers(void);
1645
1646void reserve_ds_buffers(void);
1647
1648void release_lbr_buffers(void);
1649
1650void reserve_lbr_buffers(void);
1651
1652extern struct event_constraint bts_constraint;
1653extern struct event_constraint vlbr_constraint;
1654
1655void intel_pmu_enable_bts(u64 config);
1656
1657void intel_pmu_disable_bts(void);
1658
1659int intel_pmu_drain_bts_buffer(void);
1660
1661void intel_pmu_late_setup(void);
1662
1663u64 grt_latency_data(struct perf_event *event, u64 status);
1664
1665u64 cmt_latency_data(struct perf_event *event, u64 status);
1666
1667u64 lnl_latency_data(struct perf_event *event, u64 status);
1668
1669u64 arl_h_latency_data(struct perf_event *event, u64 status);
1670
1671extern struct event_constraint intel_core2_pebs_event_constraints[];
1672
1673extern struct event_constraint intel_atom_pebs_event_constraints[];
1674
1675extern struct event_constraint intel_slm_pebs_event_constraints[];
1676
1677extern struct event_constraint intel_glm_pebs_event_constraints[];
1678
1679extern struct event_constraint intel_glp_pebs_event_constraints[];
1680
1681extern struct event_constraint intel_grt_pebs_event_constraints[];
1682
1683extern struct event_constraint intel_nehalem_pebs_event_constraints[];
1684
1685extern struct event_constraint intel_westmere_pebs_event_constraints[];
1686
1687extern struct event_constraint intel_snb_pebs_event_constraints[];
1688
1689extern struct event_constraint intel_ivb_pebs_event_constraints[];
1690
1691extern struct event_constraint intel_hsw_pebs_event_constraints[];
1692
1693extern struct event_constraint intel_bdw_pebs_event_constraints[];
1694
1695extern struct event_constraint intel_skl_pebs_event_constraints[];
1696
1697extern struct event_constraint intel_icl_pebs_event_constraints[];
1698
1699extern struct event_constraint intel_glc_pebs_event_constraints[];
1700
1701extern struct event_constraint intel_lnc_pebs_event_constraints[];
1702
1703struct event_constraint *intel_pebs_constraints(struct perf_event *event);
1704
1705void intel_pmu_pebs_add(struct perf_event *event);
1706
1707void intel_pmu_pebs_del(struct perf_event *event);
1708
1709void intel_pmu_pebs_enable(struct perf_event *event);
1710
1711void intel_pmu_pebs_disable(struct perf_event *event);
1712
1713void intel_pmu_pebs_enable_all(void);
1714
1715void intel_pmu_pebs_disable_all(void);
1716
1717void intel_pmu_pebs_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in);
1718
1719void intel_pmu_pebs_late_setup(struct cpu_hw_events *cpuc);
1720
1721void intel_pmu_drain_pebs_buffer(void);
1722
1723void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr);
1724
1725void intel_pebs_init(void);
1726
1727void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
1728				struct cpu_hw_events *cpuc,
1729				struct perf_event *event);
1730
1731void intel_pmu_lbr_sched_task(struct perf_event_pmu_context *pmu_ctx,
1732			      struct task_struct *task, bool sched_in);
1733
1734u64 lbr_from_signext_quirk_wr(u64 val);
1735
1736void intel_pmu_lbr_reset(void);
1737
1738void intel_pmu_lbr_reset_32(void);
1739
1740void intel_pmu_lbr_reset_64(void);
1741
1742void intel_pmu_lbr_add(struct perf_event *event);
1743
1744void intel_pmu_lbr_del(struct perf_event *event);
1745
1746void intel_pmu_lbr_enable_all(bool pmi);
1747
1748void intel_pmu_lbr_disable_all(void);
1749
1750void intel_pmu_lbr_read(void);
1751
1752void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc);
1753
1754void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc);
1755
1756void intel_pmu_lbr_save(void *ctx);
1757
1758void intel_pmu_lbr_restore(void *ctx);
1759
1760void intel_pmu_lbr_init_core(void);
1761
1762void intel_pmu_lbr_init_nhm(void);
1763
1764void intel_pmu_lbr_init_atom(void);
1765
1766void intel_pmu_lbr_init_slm(void);
1767
1768void intel_pmu_lbr_init_snb(void);
1769
1770void intel_pmu_lbr_init_hsw(void);
1771
1772void intel_pmu_lbr_init_skl(void);
1773
1774void intel_pmu_lbr_init_knl(void);
1775
1776void intel_pmu_lbr_init(void);
1777
1778void intel_pmu_arch_lbr_init(void);
1779
1780void intel_pmu_pebs_data_source_nhm(void);
1781
1782void intel_pmu_pebs_data_source_skl(bool pmem);
1783
1784void intel_pmu_pebs_data_source_adl(void);
1785
1786void intel_pmu_pebs_data_source_grt(void);
1787
1788void intel_pmu_pebs_data_source_mtl(void);
1789
1790void intel_pmu_pebs_data_source_arl_h(void);
1791
1792void intel_pmu_pebs_data_source_cmt(void);
1793
1794void intel_pmu_pebs_data_source_lnl(void);
1795
1796u64 intel_get_arch_pebs_data_config(struct perf_event *event);
1797
1798int intel_pmu_setup_lbr_filter(struct perf_event *event);
1799
1800void intel_pt_interrupt(void);
1801
1802int intel_bts_interrupt(void);
1803
1804void intel_bts_enable_local(void);
1805
1806void intel_bts_disable_local(void);
1807
1808int p4_pmu_init(void);
1809
1810int p6_pmu_init(void);
1811
1812int knc_pmu_init(void);
1813
1814static inline int is_ht_workaround_enabled(void)
1815{
1816	return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED);
1817}
1818
1819static inline u64 intel_pmu_pebs_mask(u64 cntr_mask)
1820{
1821	return MAX_PEBS_EVENTS_MASK & cntr_mask;
1822}
1823
1824static inline int intel_pmu_max_num_pebs(struct pmu *pmu)
1825{
1826	static_assert(MAX_PEBS_EVENTS == 32);
1827	return fls((u32)hybrid(pmu, pebs_events_mask));
1828}
1829
1830static inline bool intel_pmu_has_pebs(void)
1831{
1832	return x86_pmu.ds_pebs || x86_pmu.arch_pebs;
1833}
1834
1835#else /* CONFIG_CPU_SUP_INTEL */
1836
1837static inline void reserve_ds_buffers(void)
1838{
1839}
1840
1841static inline void release_ds_buffers(void)
1842{
1843}
1844
1845static inline void release_lbr_buffers(void)
1846{
1847}
1848
1849static inline void reserve_lbr_buffers(void)
1850{
1851}
1852
1853static inline int intel_pmu_init(void)
1854{
1855	return 0;
1856}
1857
1858static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
1859{
1860	return 0;
1861}
1862
1863static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc)
1864{
1865}
1866
1867static inline int is_ht_workaround_enabled(void)
1868{
1869	return 0;
1870}
1871#endif /* CONFIG_CPU_SUP_INTEL */
1872
1873#if ((defined CONFIG_CPU_SUP_CENTAUR) || (defined CONFIG_CPU_SUP_ZHAOXIN))
1874int zhaoxin_pmu_init(void);
1875#else
1876static inline int zhaoxin_pmu_init(void)
1877{
1878	return 0;
1879}
1880#endif /*CONFIG_CPU_SUP_CENTAUR or CONFIG_CPU_SUP_ZHAOXIN*/