arm_spe_pmu.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Perf support for the Statistical Profiling Extension, introduced as
   4 * part of ARMv8.2.
   5 *
   6 * Copyright (C) 2016 ARM Limited
   7 *
   8 * Author: Will Deacon <will.deacon@arm.com>
   9 */
  10
  11#define PMUNAME					"arm_spe"
  12#define DRVNAME					PMUNAME "_pmu"
  13#define pr_fmt(fmt)				DRVNAME ": " fmt
  14
  15#include <linux/bitfield.h>
  16#include <linux/bitops.h>
  17#include <linux/bug.h>
  18#include <linux/capability.h>
  19#include <linux/cpuhotplug.h>
  20#include <linux/cpumask.h>
  21#include <linux/device.h>
  22#include <linux/errno.h>
  23#include <linux/interrupt.h>
  24#include <linux/irq.h>
  25#include <linux/kernel.h>
  26#include <linux/list.h>
  27#include <linux/module.h>
  28#include <linux/of.h>
  29#include <linux/perf_event.h>
  30#include <linux/perf/arm_pmu.h>
  31#include <linux/platform_device.h>
  32#include <linux/printk.h>
  33#include <linux/slab.h>
  34#include <linux/smp.h>
  35#include <linux/vmalloc.h>
  36
  37#include <asm/barrier.h>
  38#include <asm/cpufeature.h>
  39#include <asm/mmu.h>
  40#include <asm/sysreg.h>
  41
  42/*
  43 * Cache if the event is allowed to trace Context information.
  44 * This allows us to perform the check, i.e, perf_allow_kernel(),
  45 * in the context of the event owner, once, during the event_init().
  46 */
  47#define SPE_PMU_HW_FLAGS_CX			0x00001
  48
  49static_assert((PERF_EVENT_FLAG_ARCH & SPE_PMU_HW_FLAGS_CX) == SPE_PMU_HW_FLAGS_CX);
  50
  51static void set_spe_event_has_cx(struct perf_event *event)
  52{
  53	if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && !perf_allow_kernel())
  54		event->hw.flags |= SPE_PMU_HW_FLAGS_CX;
  55}
  56
  57static bool get_spe_event_has_cx(struct perf_event *event)
  58{
  59	return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX);
  60}
  61
  62#define ARM_SPE_BUF_PAD_BYTE			0
  63
  64struct arm_spe_pmu_buf {
  65	int					nr_pages;
  66	bool					snapshot;
  67	void					*base;
  68};
  69
  70struct arm_spe_pmu {
  71	struct pmu				pmu;
  72	struct platform_device			*pdev;
  73	cpumask_t				supported_cpus;
  74	struct hlist_node			hotplug_node;
  75
  76	int					irq; /* PPI */
  77	u16					pmsver;
  78	u16					min_period;
  79	u16					counter_sz;
  80
  81#define SPE_PMU_FEAT_FILT_EVT			(1UL << 0)
  82#define SPE_PMU_FEAT_FILT_TYP			(1UL << 1)
  83#define SPE_PMU_FEAT_FILT_LAT			(1UL << 2)
  84#define SPE_PMU_FEAT_ARCH_INST			(1UL << 3)
  85#define SPE_PMU_FEAT_LDS			(1UL << 4)
  86#define SPE_PMU_FEAT_ERND			(1UL << 5)
  87#define SPE_PMU_FEAT_INV_FILT_EVT		(1UL << 6)
  88#define SPE_PMU_FEAT_DISCARD			(1UL << 7)
  89#define SPE_PMU_FEAT_EFT			(1UL << 8)
  90#define SPE_PMU_FEAT_FDS			(1UL << 9)
  91#define SPE_PMU_FEAT_DEV_PROBED			(1UL << 63)
  92	u64					features;
  93
  94	u64					pmsevfr_res0;
  95	u16					max_record_sz;
  96	u16					align;
  97	struct perf_output_handle __percpu	*handle;
  98};
  99
 100#define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu))
 101
 102/* Convert a free-running index from perf into an SPE buffer offset */
 103#define PERF_IDX2OFF(idx, buf) \
 104	((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
 105
 106/* Keep track of our dynamic hotplug state */
 107static enum cpuhp_state arm_spe_pmu_online;
 108
 109static void arm_spe_pmu_stop(struct perf_event *event, int flags);
 110
 111enum arm_spe_pmu_buf_fault_action {
 112	SPE_PMU_BUF_FAULT_ACT_SPURIOUS,
 113	SPE_PMU_BUF_FAULT_ACT_FATAL,
 114	SPE_PMU_BUF_FAULT_ACT_OK,
 115};
 116
 117/* This sysfs gunk was really good fun to write. */
 118enum arm_spe_pmu_capabilities {
 119	SPE_PMU_CAP_ARCH_INST = 0,
 120	SPE_PMU_CAP_ERND,
 121	SPE_PMU_CAP_FEAT_MAX,
 122	SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX,
 123	SPE_PMU_CAP_MIN_IVAL,
 124	SPE_PMU_CAP_EVENT_FILTER,
 125};
 126
 127static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = {
 128	[SPE_PMU_CAP_ARCH_INST]	= SPE_PMU_FEAT_ARCH_INST,
 129	[SPE_PMU_CAP_ERND]	= SPE_PMU_FEAT_ERND,
 130};
 131
 132static u64 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap)
 133{
 134	if (cap < SPE_PMU_CAP_FEAT_MAX)
 135		return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]);
 136
 137	switch (cap) {
 138	case SPE_PMU_CAP_CNT_SZ:
 139		return spe_pmu->counter_sz;
 140	case SPE_PMU_CAP_MIN_IVAL:
 141		return spe_pmu->min_period;
 142	case SPE_PMU_CAP_EVENT_FILTER:
 143		return ~spe_pmu->pmsevfr_res0;
 144	default:
 145		WARN(1, "unknown cap %d\n", cap);
 146	}
 147
 148	return 0;
 149}
 150
 151static ssize_t arm_spe_pmu_cap_show(struct device *dev,
 152				    struct device_attribute *attr,
 153				    char *buf)
 154{
 155	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
 156	struct dev_ext_attribute *ea =
 157		container_of(attr, struct dev_ext_attribute, attr);
 158	int cap = (long)ea->var;
 159
 160	return sysfs_emit(buf, "%llu\n", arm_spe_pmu_cap_get(spe_pmu, cap));
 161}
 162
 163static ssize_t arm_spe_pmu_cap_show_hex(struct device *dev,
 164					struct device_attribute *attr,
 165					char *buf)
 166{
 167	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
 168	struct dev_ext_attribute *ea =
 169		container_of(attr, struct dev_ext_attribute, attr);
 170	int cap = (long)ea->var;
 171
 172	return sysfs_emit(buf, "0x%llx\n", arm_spe_pmu_cap_get(spe_pmu, cap));
 173}
 174
 175#define SPE_EXT_ATTR_ENTRY(_name, _func, _var)				\
 176	&((struct dev_ext_attribute[]) {				\
 177		{ __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var }	\
 178	})[0].attr.attr
 179
 180#define SPE_CAP_EXT_ATTR_ENTRY(_name, _var)				\
 181	SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var)
 182#define SPE_CAP_EXT_ATTR_ENTRY_HEX(_name, _var)				\
 183	SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show_hex, _var)
 184
 185static struct attribute *arm_spe_pmu_cap_attr[] = {
 186	SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST),
 187	SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND),
 188	SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ),
 189	SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL),
 190	SPE_CAP_EXT_ATTR_ENTRY_HEX(event_filter, SPE_PMU_CAP_EVENT_FILTER),
 191	NULL,
 192};
 193
 194static const struct attribute_group arm_spe_pmu_cap_group = {
 195	.name	= "caps",
 196	.attrs	= arm_spe_pmu_cap_attr,
 197};
 198
 199/* User ABI */
 200#define ATTR_CFG_FLD_ts_enable_CFG		config	/* PMSCR_EL1.TS */
 201#define ATTR_CFG_FLD_ts_enable_LO		0
 202#define ATTR_CFG_FLD_ts_enable_HI		0
 203#define ATTR_CFG_FLD_pa_enable_CFG		config	/* PMSCR_EL1.PA */
 204#define ATTR_CFG_FLD_pa_enable_LO		1
 205#define ATTR_CFG_FLD_pa_enable_HI		1
 206#define ATTR_CFG_FLD_pct_enable_CFG		config	/* PMSCR_EL1.PCT */
 207#define ATTR_CFG_FLD_pct_enable_LO		2
 208#define ATTR_CFG_FLD_pct_enable_HI		2
 209#define ATTR_CFG_FLD_jitter_CFG			config	/* PMSIRR_EL1.RND */
 210#define ATTR_CFG_FLD_jitter_LO			16
 211#define ATTR_CFG_FLD_jitter_HI			16
 212#define ATTR_CFG_FLD_branch_filter_CFG		config	/* PMSFCR_EL1.B */
 213#define ATTR_CFG_FLD_branch_filter_LO		32
 214#define ATTR_CFG_FLD_branch_filter_HI		32
 215#define ATTR_CFG_FLD_load_filter_CFG		config	/* PMSFCR_EL1.LD */
 216#define ATTR_CFG_FLD_load_filter_LO		33
 217#define ATTR_CFG_FLD_load_filter_HI		33
 218#define ATTR_CFG_FLD_store_filter_CFG		config	/* PMSFCR_EL1.ST */
 219#define ATTR_CFG_FLD_store_filter_LO		34
 220#define ATTR_CFG_FLD_store_filter_HI		34
 221#define ATTR_CFG_FLD_discard_CFG		config	/* PMBLIMITR_EL1.FM = DISCARD */
 222#define ATTR_CFG_FLD_discard_LO			35
 223#define ATTR_CFG_FLD_discard_HI			35
 224#define ATTR_CFG_FLD_branch_filter_mask_CFG	config	/* PMSFCR_EL1.Bm */
 225#define ATTR_CFG_FLD_branch_filter_mask_LO	36
 226#define ATTR_CFG_FLD_branch_filter_mask_HI	36
 227#define ATTR_CFG_FLD_load_filter_mask_CFG	config	/* PMSFCR_EL1.LDm */
 228#define ATTR_CFG_FLD_load_filter_mask_LO	37
 229#define ATTR_CFG_FLD_load_filter_mask_HI	37
 230#define ATTR_CFG_FLD_store_filter_mask_CFG	config	/* PMSFCR_EL1.STm */
 231#define ATTR_CFG_FLD_store_filter_mask_LO	38
 232#define ATTR_CFG_FLD_store_filter_mask_HI	38
 233#define ATTR_CFG_FLD_simd_filter_CFG		config	/* PMSFCR_EL1.SIMD */
 234#define ATTR_CFG_FLD_simd_filter_LO		39
 235#define ATTR_CFG_FLD_simd_filter_HI		39
 236#define ATTR_CFG_FLD_simd_filter_mask_CFG	config	/* PMSFCR_EL1.SIMDm */
 237#define ATTR_CFG_FLD_simd_filter_mask_LO	40
 238#define ATTR_CFG_FLD_simd_filter_mask_HI	40
 239#define ATTR_CFG_FLD_float_filter_CFG		config	/* PMSFCR_EL1.FP */
 240#define ATTR_CFG_FLD_float_filter_LO		41
 241#define ATTR_CFG_FLD_float_filter_HI		41
 242#define ATTR_CFG_FLD_float_filter_mask_CFG	config	/* PMSFCR_EL1.FPm */
 243#define ATTR_CFG_FLD_float_filter_mask_LO	42
 244#define ATTR_CFG_FLD_float_filter_mask_HI	42
 245
 246#define ATTR_CFG_FLD_event_filter_CFG		config1	/* PMSEVFR_EL1 */
 247#define ATTR_CFG_FLD_event_filter_LO		0
 248#define ATTR_CFG_FLD_event_filter_HI		63
 249
 250#define ATTR_CFG_FLD_min_latency_CFG		config2	/* PMSLATFR_EL1.MINLAT */
 251#define ATTR_CFG_FLD_min_latency_LO		0
 252#define ATTR_CFG_FLD_min_latency_HI		11
 253
 254#define ATTR_CFG_FLD_inv_event_filter_CFG	config3	/* PMSNEVFR_EL1 */
 255#define ATTR_CFG_FLD_inv_event_filter_LO	0
 256#define ATTR_CFG_FLD_inv_event_filter_HI	63
 257
 258#define ATTR_CFG_FLD_inv_data_src_filter_CFG	config4	/* inverse of PMSDSFR_EL1 */
 259#define ATTR_CFG_FLD_inv_data_src_filter_LO	0
 260#define ATTR_CFG_FLD_inv_data_src_filter_HI	63
 261
 262GEN_PMU_FORMAT_ATTR(ts_enable);
 263GEN_PMU_FORMAT_ATTR(pa_enable);
 264GEN_PMU_FORMAT_ATTR(pct_enable);
 265GEN_PMU_FORMAT_ATTR(jitter);
 266GEN_PMU_FORMAT_ATTR(branch_filter);
 267GEN_PMU_FORMAT_ATTR(branch_filter_mask);
 268GEN_PMU_FORMAT_ATTR(load_filter);
 269GEN_PMU_FORMAT_ATTR(load_filter_mask);
 270GEN_PMU_FORMAT_ATTR(store_filter);
 271GEN_PMU_FORMAT_ATTR(store_filter_mask);
 272GEN_PMU_FORMAT_ATTR(simd_filter);
 273GEN_PMU_FORMAT_ATTR(simd_filter_mask);
 274GEN_PMU_FORMAT_ATTR(float_filter);
 275GEN_PMU_FORMAT_ATTR(float_filter_mask);
 276GEN_PMU_FORMAT_ATTR(event_filter);
 277GEN_PMU_FORMAT_ATTR(inv_event_filter);
 278GEN_PMU_FORMAT_ATTR(inv_data_src_filter);
 279GEN_PMU_FORMAT_ATTR(min_latency);
 280GEN_PMU_FORMAT_ATTR(discard);
 281
 282static struct attribute *arm_spe_pmu_formats_attr[] = {
 283	&format_attr_ts_enable.attr,
 284	&format_attr_pa_enable.attr,
 285	&format_attr_pct_enable.attr,
 286	&format_attr_jitter.attr,
 287	&format_attr_branch_filter.attr,
 288	&format_attr_branch_filter_mask.attr,
 289	&format_attr_load_filter.attr,
 290	&format_attr_load_filter_mask.attr,
 291	&format_attr_store_filter.attr,
 292	&format_attr_store_filter_mask.attr,
 293	&format_attr_simd_filter.attr,
 294	&format_attr_simd_filter_mask.attr,
 295	&format_attr_float_filter.attr,
 296	&format_attr_float_filter_mask.attr,
 297	&format_attr_event_filter.attr,
 298	&format_attr_inv_event_filter.attr,
 299	&format_attr_inv_data_src_filter.attr,
 300	&format_attr_min_latency.attr,
 301	&format_attr_discard.attr,
 302	NULL,
 303};
 304
 305static umode_t arm_spe_pmu_format_attr_is_visible(struct kobject *kobj,
 306						  struct attribute *attr,
 307						  int unused)
 308	{
 309	struct device *dev = kobj_to_dev(kobj);
 310	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
 311
 312	if (attr == &format_attr_discard.attr && !(spe_pmu->features & SPE_PMU_FEAT_DISCARD))
 313		return 0;
 314
 315	if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT))
 316		return 0;
 317
 318	if (attr == &format_attr_inv_data_src_filter.attr &&
 319	    !(spe_pmu->features & SPE_PMU_FEAT_FDS))
 320		return 0;
 321
 322	if ((attr == &format_attr_branch_filter_mask.attr ||
 323	     attr == &format_attr_load_filter_mask.attr ||
 324	     attr == &format_attr_store_filter_mask.attr ||
 325	     attr == &format_attr_simd_filter.attr ||
 326	     attr == &format_attr_simd_filter_mask.attr ||
 327	     attr == &format_attr_float_filter.attr ||
 328	     attr == &format_attr_float_filter_mask.attr) &&
 329	     !(spe_pmu->features & SPE_PMU_FEAT_EFT))
 330		return 0;
 331
 332	return attr->mode;
 333}
 334
 335static const struct attribute_group arm_spe_pmu_format_group = {
 336	.name	= "format",
 337	.is_visible = arm_spe_pmu_format_attr_is_visible,
 338	.attrs	= arm_spe_pmu_formats_attr,
 339};
 340
 341static ssize_t cpumask_show(struct device *dev,
 342			    struct device_attribute *attr, char *buf)
 343{
 344	struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
 345
 346	return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);
 347}
 348static DEVICE_ATTR_RO(cpumask);
 349
 350static struct attribute *arm_spe_pmu_attrs[] = {
 351	&dev_attr_cpumask.attr,
 352	NULL,
 353};
 354
 355static const struct attribute_group arm_spe_pmu_group = {
 356	.attrs	= arm_spe_pmu_attrs,
 357};
 358
 359static const struct attribute_group *arm_spe_pmu_attr_groups[] = {
 360	&arm_spe_pmu_group,
 361	&arm_spe_pmu_cap_group,
 362	&arm_spe_pmu_format_group,
 363	NULL,
 364};
 365
 366/* Convert between user ABI and register values */
 367static u64 arm_spe_event_to_pmscr(struct perf_event *event)
 368{
 369	struct perf_event_attr *attr = &event->attr;
 370	u64 reg = 0;
 371
 372	reg |= FIELD_PREP(PMSCR_EL1_TS, ATTR_CFG_GET_FLD(attr, ts_enable));
 373	reg |= FIELD_PREP(PMSCR_EL1_PA, ATTR_CFG_GET_FLD(attr, pa_enable));
 374	reg |= FIELD_PREP(PMSCR_EL1_PCT, ATTR_CFG_GET_FLD(attr, pct_enable));
 375
 376	if (!attr->exclude_user)
 377		reg |= PMSCR_EL1_E0SPE;
 378
 379	if (!attr->exclude_kernel)
 380		reg |= PMSCR_EL1_E1SPE;
 381
 382	if (get_spe_event_has_cx(event))
 383		reg |= PMSCR_EL1_CX;
 384
 385	return reg;
 386}
 387
 388static void arm_spe_event_sanitise_period(struct perf_event *event)
 389{
 390	u64 period = event->hw.sample_period;
 391	u64 max_period = PMSIRR_EL1_INTERVAL_MASK;
 392
 393	/*
 394	 * The PMSIDR_EL1.Interval field (stored in spe_pmu->min_period) is a
 395	 * recommendation for the minimum interval, not a hardware limitation.
 396	 *
 397	 * According to the Arm ARM (DDI 0487 L.a), section D24.7.12 PMSIRR_EL1,
 398	 * Sampling Interval Reload Register, the INTERVAL field (bits [31:8])
 399	 * states: "Software must set this to a nonzero value". Use 1 as the
 400	 * minimum value.
 401	 */
 402	u64 min_period = FIELD_PREP(PMSIRR_EL1_INTERVAL_MASK, 1);
 403
 404	period = clamp_t(u64, period, min_period, max_period) & max_period;
 405	event->hw.sample_period = period;
 406}
 407
 408static u64 arm_spe_event_to_pmsirr(struct perf_event *event)
 409{
 410	struct perf_event_attr *attr = &event->attr;
 411	u64 reg = 0;
 412
 413	arm_spe_event_sanitise_period(event);
 414
 415	reg |= FIELD_PREP(PMSIRR_EL1_RND, ATTR_CFG_GET_FLD(attr, jitter));
 416	reg |= event->hw.sample_period;
 417
 418	return reg;
 419}
 420
 421static u64 arm_spe_event_to_pmsfcr(struct perf_event *event)
 422{
 423	struct perf_event_attr *attr = &event->attr;
 424	u64 reg = 0;
 425
 426	reg |= FIELD_PREP(PMSFCR_EL1_LD, ATTR_CFG_GET_FLD(attr, load_filter));
 427	reg |= FIELD_PREP(PMSFCR_EL1_LDm, ATTR_CFG_GET_FLD(attr, load_filter_mask));
 428	reg |= FIELD_PREP(PMSFCR_EL1_ST, ATTR_CFG_GET_FLD(attr, store_filter));
 429	reg |= FIELD_PREP(PMSFCR_EL1_STm, ATTR_CFG_GET_FLD(attr, store_filter_mask));
 430	reg |= FIELD_PREP(PMSFCR_EL1_B, ATTR_CFG_GET_FLD(attr, branch_filter));
 431	reg |= FIELD_PREP(PMSFCR_EL1_Bm, ATTR_CFG_GET_FLD(attr, branch_filter_mask));
 432	reg |= FIELD_PREP(PMSFCR_EL1_SIMD, ATTR_CFG_GET_FLD(attr, simd_filter));
 433	reg |= FIELD_PREP(PMSFCR_EL1_SIMDm, ATTR_CFG_GET_FLD(attr, simd_filter_mask));
 434	reg |= FIELD_PREP(PMSFCR_EL1_FP, ATTR_CFG_GET_FLD(attr, float_filter));
 435	reg |= FIELD_PREP(PMSFCR_EL1_FPm, ATTR_CFG_GET_FLD(attr, float_filter_mask));
 436
 437	if (reg)
 438		reg |= PMSFCR_EL1_FT;
 439
 440	if (ATTR_CFG_GET_FLD(attr, event_filter))
 441		reg |= PMSFCR_EL1_FE;
 442
 443	if (ATTR_CFG_GET_FLD(attr, inv_event_filter))
 444		reg |= PMSFCR_EL1_FnE;
 445
 446	if (ATTR_CFG_GET_FLD(attr, inv_data_src_filter))
 447		reg |= PMSFCR_EL1_FDS;
 448
 449	if (ATTR_CFG_GET_FLD(attr, min_latency))
 450		reg |= PMSFCR_EL1_FL;
 451
 452	return reg;
 453}
 454
 455static u64 arm_spe_event_to_pmsevfr(struct perf_event *event)
 456{
 457	struct perf_event_attr *attr = &event->attr;
 458	return ATTR_CFG_GET_FLD(attr, event_filter);
 459}
 460
 461static u64 arm_spe_event_to_pmsnevfr(struct perf_event *event)
 462{
 463	struct perf_event_attr *attr = &event->attr;
 464	return ATTR_CFG_GET_FLD(attr, inv_event_filter);
 465}
 466
 467static u64 arm_spe_event_to_pmslatfr(struct perf_event *event)
 468{
 469	struct perf_event_attr *attr = &event->attr;
 470	return FIELD_PREP(PMSLATFR_EL1_MINLAT, ATTR_CFG_GET_FLD(attr, min_latency));
 471}
 472
 473static u64 arm_spe_event_to_pmsdsfr(struct perf_event *event)
 474{
 475	struct perf_event_attr *attr = &event->attr;
 476
 477	/*
 478	 * Data src filter is inverted so that the default value of 0 is
 479	 * equivalent to no filtering.
 480	 */
 481	return ~ATTR_CFG_GET_FLD(attr, inv_data_src_filter);
 482}
 483
 484static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len)
 485{
 486	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
 487	u64 head = PERF_IDX2OFF(handle->head, buf);
 488
 489	memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len);
 490	if (!buf->snapshot)
 491		perf_aux_output_skip(handle, len);
 492}
 493
 494static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle)
 495{
 496	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
 497	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
 498	u64 head = PERF_IDX2OFF(handle->head, buf);
 499	u64 limit = buf->nr_pages * PAGE_SIZE;
 500
 501	/*
 502	 * The trace format isn't parseable in reverse, so clamp
 503	 * the limit to half of the buffer size in snapshot mode
 504	 * so that the worst case is half a buffer of records, as
 505	 * opposed to a single record.
 506	 */
 507	if (head < limit >> 1)
 508		limit >>= 1;
 509
 510	/*
 511	 * If we're within max_record_sz of the limit, we must
 512	 * pad, move the head index and recompute the limit.
 513	 */
 514	if (limit - head < spe_pmu->max_record_sz) {
 515		arm_spe_pmu_pad_buf(handle, limit - head);
 516		handle->head = PERF_IDX2OFF(limit, buf);
 517		limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head;
 518	}
 519
 520	return limit;
 521}
 522
 523static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle)
 524{
 525	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
 526	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
 527	const u64 bufsize = buf->nr_pages * PAGE_SIZE;
 528	u64 limit = bufsize;
 529	u64 head, tail, wakeup;
 530
 531	/*
 532	 * The head can be misaligned for two reasons:
 533	 *
 534	 * 1. The hardware left PMBPTR pointing to the first byte after
 535	 *    a record when generating a buffer management event.
 536	 *
 537	 * 2. We used perf_aux_output_skip to consume handle->size bytes
 538	 *    and CIRC_SPACE was used to compute the size, which always
 539	 *    leaves one entry free.
 540	 *
 541	 * Deal with this by padding to the next alignment boundary and
 542	 * moving the head index. If we run out of buffer space, we'll
 543	 * reduce handle->size to zero and end up reporting truncation.
 544	 */
 545	head = PERF_IDX2OFF(handle->head, buf);
 546	if (!IS_ALIGNED(head, spe_pmu->align)) {
 547		unsigned long delta = roundup(head, spe_pmu->align) - head;
 548
 549		delta = min(delta, handle->size);
 550		arm_spe_pmu_pad_buf(handle, delta);
 551		head = PERF_IDX2OFF(handle->head, buf);
 552	}
 553
 554	/* If we've run out of free space, then nothing more to do */
 555	if (!handle->size)
 556		goto no_space;
 557
 558	/* Compute the tail and wakeup indices now that we've aligned head */
 559	tail = PERF_IDX2OFF(handle->head + handle->size, buf);
 560	wakeup = PERF_IDX2OFF(handle->wakeup, buf);
 561
 562	/*
 563	 * Avoid clobbering unconsumed data. We know we have space, so
 564	 * if we see head == tail we know that the buffer is empty. If
 565	 * head > tail, then there's nothing to clobber prior to
 566	 * wrapping.
 567	 */
 568	if (head < tail)
 569		limit = round_down(tail, PAGE_SIZE);
 570
 571	/*
 572	 * Wakeup may be arbitrarily far into the future. If it's not in
 573	 * the current generation, either we'll wrap before hitting it,
 574	 * or it's in the past and has been handled already.
 575	 *
 576	 * If there's a wakeup before we wrap, arrange to be woken up by
 577	 * the page boundary following it. Keep the tail boundary if
 578	 * that's lower.
 579	 */
 580	if (handle->wakeup < (handle->head + handle->size) && head <= wakeup)
 581		limit = min(limit, round_up(wakeup, PAGE_SIZE));
 582
 583	if (limit > head)
 584		return limit;
 585
 586	arm_spe_pmu_pad_buf(handle, handle->size);
 587no_space:
 588	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
 589	perf_aux_output_end(handle, 0);
 590	return 0;
 591}
 592
 593static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle)
 594{
 595	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
 596	struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu);
 597	u64 limit = __arm_spe_pmu_next_off(handle);
 598	u64 head = PERF_IDX2OFF(handle->head, buf);
 599
 600	/*
 601	 * If the head has come too close to the end of the buffer,
 602	 * then pad to the end and recompute the limit.
 603	 */
 604	if (limit && (limit - head < spe_pmu->max_record_sz)) {
 605		arm_spe_pmu_pad_buf(handle, limit - head);
 606		limit = __arm_spe_pmu_next_off(handle);
 607	}
 608
 609	return limit;
 610}
 611
 612static int arm_spe_perf_aux_output_begin(struct perf_output_handle *handle,
 613					 struct perf_event *event)
 614{
 615	u64 base, limit;
 616	struct arm_spe_pmu_buf *buf;
 617
 618	if (ATTR_CFG_GET_FLD(&event->attr, discard)) {
 619		limit = FIELD_PREP(PMBLIMITR_EL1_FM, PMBLIMITR_EL1_FM_DISCARD);
 620		limit |= PMBLIMITR_EL1_E;
 621		goto out_write_limit;
 622	}
 623
 624	/* Start a new aux session */
 625	buf = perf_aux_output_begin(handle, event);
 626	if (!buf) {
 627		/*
 628		 * We still need to clear the limit pointer, since the
 629		 * profiler might only be disabled by virtue of a fault.
 630		 */
 631		limit = 0;
 632		goto out_write_limit;
 633	}
 634
 635	limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle)
 636			      : arm_spe_pmu_next_off(handle);
 637	if (limit)
 638		limit |= PMBLIMITR_EL1_E;
 639
 640	limit += (u64)buf->base;
 641	base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf);
 642	write_sysreg_s(base, SYS_PMBPTR_EL1);
 643
 644out_write_limit:
 645	write_sysreg_s(limit, SYS_PMBLIMITR_EL1);
 646	return (limit & PMBLIMITR_EL1_E) ? 0 : -EIO;
 647}
 648
 649static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle)
 650{
 651	struct arm_spe_pmu_buf *buf = perf_get_aux(handle);
 652	u64 offset, size;
 653
 654	offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base;
 655	size = offset - PERF_IDX2OFF(handle->head, buf);
 656
 657	if (buf->snapshot)
 658		handle->head = offset;
 659
 660	perf_aux_output_end(handle, size);
 661}
 662
 663static void arm_spe_pmu_disable_and_drain_local(void)
 664{
 665	/* Disable profiling at EL0 and EL1 */
 666	write_sysreg_s(0, SYS_PMSCR_EL1);
 667	isb();
 668
 669	/* Drain any buffered data */
 670	psb_csync();
 671	dsb(nsh);
 672
 673	/* Disable the profiling buffer */
 674	write_sysreg_s(0, SYS_PMBLIMITR_EL1);
 675	isb();
 676}
 677
 678/* IRQ handling */
 679static enum arm_spe_pmu_buf_fault_action
 680arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle)
 681{
 682	const char *err_str;
 683	u64 pmbsr;
 684	enum arm_spe_pmu_buf_fault_action ret;
 685
 686	/*
 687	 * Ensure new profiling data is visible to the CPU and any external
 688	 * aborts have been resolved.
 689	 */
 690	psb_csync();
 691	dsb(nsh);
 692
 693	/* Ensure hardware updates to PMBPTR_EL1 are visible */
 694	isb();
 695
 696	/* Service required? */
 697	pmbsr = read_sysreg_s(SYS_PMBSR_EL1);
 698	if (!FIELD_GET(PMBSR_EL1_S, pmbsr))
 699		return SPE_PMU_BUF_FAULT_ACT_SPURIOUS;
 700
 701	/*
 702	 * If we've lost data, disable profiling and also set the PARTIAL
 703	 * flag to indicate that the last record is corrupted.
 704	 */
 705	if (FIELD_GET(PMBSR_EL1_DL, pmbsr))
 706		perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED |
 707					     PERF_AUX_FLAG_PARTIAL);
 708
 709	/* Report collisions to userspace so that it can up the period */
 710	if (FIELD_GET(PMBSR_EL1_COLL, pmbsr))
 711		perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION);
 712
 713	/* We only expect buffer management events */
 714	switch (FIELD_GET(PMBSR_EL1_EC, pmbsr)) {
 715	case PMBSR_EL1_EC_BUF:
 716		/* Handled below */
 717		break;
 718	case PMBSR_EL1_EC_FAULT_S1:
 719	case PMBSR_EL1_EC_FAULT_S2:
 720		err_str = "Unexpected buffer fault";
 721		goto out_err;
 722	default:
 723		err_str = "Unknown error code";
 724		goto out_err;
 725	}
 726
 727	/* Buffer management event */
 728	switch (FIELD_GET(PMBSR_EL1_BUF_BSC_MASK, pmbsr)) {
 729	case PMBSR_EL1_BUF_BSC_FULL:
 730		ret = SPE_PMU_BUF_FAULT_ACT_OK;
 731		goto out_stop;
 732	default:
 733		err_str = "Unknown buffer status code";
 734	}
 735
 736out_err:
 737	pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n",
 738			   err_str, smp_processor_id(), pmbsr,
 739			   read_sysreg_s(SYS_PMBPTR_EL1),
 740			   read_sysreg_s(SYS_PMBLIMITR_EL1));
 741	ret = SPE_PMU_BUF_FAULT_ACT_FATAL;
 742
 743out_stop:
 744	arm_spe_perf_aux_output_end(handle);
 745	return ret;
 746}
 747
 748static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev)
 749{
 750	struct perf_output_handle *handle = dev;
 751	struct perf_event *event = handle->event;
 752	enum arm_spe_pmu_buf_fault_action act;
 753
 754	if (!perf_get_aux(handle))
 755		return IRQ_NONE;
 756
 757	act = arm_spe_pmu_buf_get_fault_act(handle);
 758	if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS)
 759		return IRQ_NONE;
 760
 761	/*
 762	 * Ensure perf callbacks have completed, which may disable the
 763	 * profiling buffer in response to a TRUNCATION flag.
 764	 */
 765	irq_work_run();
 766
 767	switch (act) {
 768	case SPE_PMU_BUF_FAULT_ACT_FATAL:
 769		/*
 770		 * If a fatal exception occurred then leaving the profiling
 771		 * buffer enabled is a recipe waiting to happen. Since
 772		 * fatal faults don't always imply truncation, make sure
 773		 * that the profiling buffer is disabled explicitly before
 774		 * clearing the syndrome register.
 775		 */
 776		arm_spe_pmu_disable_and_drain_local();
 777		break;
 778	case SPE_PMU_BUF_FAULT_ACT_OK:
 779		/*
 780		 * We handled the fault (the buffer was full), so resume
 781		 * profiling as long as we didn't detect truncation.
 782		 * PMBPTR might be misaligned, but we'll burn that bridge
 783		 * when we get to it.
 784		 */
 785		if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) {
 786			if (arm_spe_perf_aux_output_begin(handle, event)) {
 787				arm_spe_pmu_stop(event, PERF_EF_UPDATE);
 788				break;
 789			}
 790			isb();
 791		}
 792		break;
 793	case SPE_PMU_BUF_FAULT_ACT_SPURIOUS:
 794		/* We've seen you before, but GCC has the memory of a sieve. */
 795		break;
 796	}
 797
 798	/* The buffer pointers are now sane, so resume profiling. */
 799	write_sysreg_s(0, SYS_PMBSR_EL1);
 800	return IRQ_HANDLED;
 801}
 802
 803/* Perf callbacks */
 804static int arm_spe_pmu_event_init(struct perf_event *event)
 805{
 806	u64 reg;
 807	struct perf_event_attr *attr = &event->attr;
 808	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
 809
 810	/* This is, of course, deeply driver-specific */
 811	if (attr->type != event->pmu->type)
 812		return -ENOENT;
 813
 814	if (event->cpu >= 0 &&
 815	    !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus))
 816		return -ENOENT;
 817
 818	if (arm_spe_event_to_pmsevfr(event) & spe_pmu->pmsevfr_res0)
 819		return -EOPNOTSUPP;
 820
 821	if (arm_spe_event_to_pmsnevfr(event) & spe_pmu->pmsevfr_res0)
 822		return -EOPNOTSUPP;
 823
 824	if (arm_spe_event_to_pmsdsfr(event) != U64_MAX &&
 825	    !(spe_pmu->features & SPE_PMU_FEAT_FDS))
 826		return -EOPNOTSUPP;
 827
 828	if (attr->exclude_idle)
 829		return -EOPNOTSUPP;
 830
 831	/*
 832	 * Feedback-directed frequency throttling doesn't work when we
 833	 * have a buffer of samples. We'd need to manually count the
 834	 * samples in the buffer when it fills up and adjust the event
 835	 * count to reflect that. Instead, just force the user to specify
 836	 * a sample period.
 837	 */
 838	if (attr->freq)
 839		return -EINVAL;
 840
 841	reg = arm_spe_event_to_pmsfcr(event);
 842	if ((FIELD_GET(PMSFCR_EL1_FE, reg)) &&
 843	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT))
 844		return -EOPNOTSUPP;
 845
 846	if ((FIELD_GET(PMSFCR_EL1_FnE, reg)) &&
 847	    !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT))
 848		return -EOPNOTSUPP;
 849
 850	if ((FIELD_GET(PMSFCR_EL1_FT, reg)) &&
 851	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP))
 852		return -EOPNOTSUPP;
 853
 854	if ((FIELD_GET(PMSFCR_EL1_FL, reg)) &&
 855	    !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT))
 856		return -EOPNOTSUPP;
 857
 858	if ((FIELD_GET(PMSFCR_EL1_LDm, reg) ||
 859	     FIELD_GET(PMSFCR_EL1_STm, reg) ||
 860	     FIELD_GET(PMSFCR_EL1_Bm, reg) ||
 861	     FIELD_GET(PMSFCR_EL1_SIMD, reg) ||
 862	     FIELD_GET(PMSFCR_EL1_SIMDm, reg) ||
 863	     FIELD_GET(PMSFCR_EL1_FP, reg) ||
 864	     FIELD_GET(PMSFCR_EL1_FPm, reg)) &&
 865	    !(spe_pmu->features & SPE_PMU_FEAT_EFT))
 866		return -EOPNOTSUPP;
 867
 868	if (ATTR_CFG_GET_FLD(&event->attr, discard) &&
 869	    !(spe_pmu->features & SPE_PMU_FEAT_DISCARD))
 870		return -EOPNOTSUPP;
 871
 872	set_spe_event_has_cx(event);
 873	reg = arm_spe_event_to_pmscr(event);
 874	if (reg & (PMSCR_EL1_PA | PMSCR_EL1_PCT))
 875		return perf_allow_kernel();
 876
 877	return 0;
 878}
 879
 880static void arm_spe_pmu_start(struct perf_event *event, int flags)
 881{
 882	u64 reg;
 883	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
 884	struct hw_perf_event *hwc = &event->hw;
 885	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);
 886
 887	hwc->state = 0;
 888	if (arm_spe_perf_aux_output_begin(handle, event)) {
 889		arm_spe_pmu_stop(event, 0);
 890		return;
 891	}
 892
 893	reg = arm_spe_event_to_pmsfcr(event);
 894	write_sysreg_s(reg, SYS_PMSFCR_EL1);
 895
 896	reg = arm_spe_event_to_pmsevfr(event);
 897	write_sysreg_s(reg, SYS_PMSEVFR_EL1);
 898
 899	if (spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT) {
 900		reg = arm_spe_event_to_pmsnevfr(event);
 901		write_sysreg_s(reg, SYS_PMSNEVFR_EL1);
 902	}
 903
 904	if (spe_pmu->features & SPE_PMU_FEAT_FDS) {
 905		reg = arm_spe_event_to_pmsdsfr(event);
 906		write_sysreg_s(reg, SYS_PMSDSFR_EL1);
 907	}
 908
 909	reg = arm_spe_event_to_pmslatfr(event);
 910	write_sysreg_s(reg, SYS_PMSLATFR_EL1);
 911
 912	if (flags & PERF_EF_RELOAD) {
 913		reg = arm_spe_event_to_pmsirr(event);
 914		write_sysreg_s(reg, SYS_PMSIRR_EL1);
 915		isb();
 916		reg = local64_read(&hwc->period_left);
 917		write_sysreg_s(reg, SYS_PMSICR_EL1);
 918	}
 919
 920	reg = arm_spe_event_to_pmscr(event);
 921	isb();
 922	write_sysreg_s(reg, SYS_PMSCR_EL1);
 923}
 924
 925static void arm_spe_pmu_stop(struct perf_event *event, int flags)
 926{
 927	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
 928	struct hw_perf_event *hwc = &event->hw;
 929	struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle);
 930
 931	/* If we're already stopped, then nothing to do */
 932	if (hwc->state & PERF_HES_STOPPED)
 933		return;
 934
 935	/* Stop all trace generation */
 936	arm_spe_pmu_disable_and_drain_local();
 937
 938	if (flags & PERF_EF_UPDATE) {
 939		/*
 940		 * If there's a fault pending then ensure we contain it
 941		 * to this buffer, since we might be on the context-switch
 942		 * path.
 943		 */
 944		if (perf_get_aux(handle)) {
 945			enum arm_spe_pmu_buf_fault_action act;
 946
 947			act = arm_spe_pmu_buf_get_fault_act(handle);
 948			if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS)
 949				arm_spe_perf_aux_output_end(handle);
 950			else
 951				write_sysreg_s(0, SYS_PMBSR_EL1);
 952		}
 953
 954		/*
 955		 * This may also contain ECOUNT, but nobody else should
 956		 * be looking at period_left, since we forbid frequency
 957		 * based sampling.
 958		 */
 959		local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1));
 960		hwc->state |= PERF_HES_UPTODATE;
 961	}
 962
 963	hwc->state |= PERF_HES_STOPPED;
 964}
 965
 966static int arm_spe_pmu_add(struct perf_event *event, int flags)
 967{
 968	int ret = 0;
 969	struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu);
 970	struct hw_perf_event *hwc = &event->hw;
 971	int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu;
 972
 973	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
 974		return -ENOENT;
 975
 976	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 977
 978	if (flags & PERF_EF_START) {
 979		arm_spe_pmu_start(event, PERF_EF_RELOAD);
 980		if (hwc->state & PERF_HES_STOPPED)
 981			ret = -EINVAL;
 982	}
 983
 984	return ret;
 985}
 986
 987static void arm_spe_pmu_del(struct perf_event *event, int flags)
 988{
 989	arm_spe_pmu_stop(event, PERF_EF_UPDATE);
 990}
 991
 992static void arm_spe_pmu_read(struct perf_event *event)
 993{
 994}
 995
 996static void *arm_spe_pmu_setup_aux(struct perf_event *event, void **pages,
 997				   int nr_pages, bool snapshot)
 998{
 999	int i, cpu = event->cpu;
1000	struct page **pglist;
1001	struct arm_spe_pmu_buf *buf;
1002
1003	/* We need at least two pages for this to work. */
1004	if (nr_pages < 2)
1005		return NULL;
1006
1007	/*
1008	 * We require an even number of pages for snapshot mode, so that
1009	 * we can effectively treat the buffer as consisting of two equal
1010	 * parts and give userspace a fighting chance of getting some
1011	 * useful data out of it.
1012	 */
1013	if (snapshot && (nr_pages & 1))
1014		return NULL;
1015
1016	if (cpu == -1)
1017		cpu = raw_smp_processor_id();
1018
1019	buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu));
1020	if (!buf)
1021		return NULL;
1022
1023	pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL);
1024	if (!pglist)
1025		goto out_free_buf;
1026
1027	for (i = 0; i < nr_pages; ++i)
1028		pglist[i] = virt_to_page(pages[i]);
1029
1030	buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL);
1031	if (!buf->base)
1032		goto out_free_pglist;
1033
1034	buf->nr_pages	= nr_pages;
1035	buf->snapshot	= snapshot;
1036
1037	kfree(pglist);
1038	return buf;
1039
1040out_free_pglist:
1041	kfree(pglist);
1042out_free_buf:
1043	kfree(buf);
1044	return NULL;
1045}
1046
1047static void arm_spe_pmu_free_aux(void *aux)
1048{
1049	struct arm_spe_pmu_buf *buf = aux;
1050
1051	vunmap(buf->base);
1052	kfree(buf);
1053}
1054
1055/* Initialisation and teardown functions */
1056static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu)
1057{
1058	static atomic_t pmu_idx = ATOMIC_INIT(-1);
1059
1060	int idx;
1061	char *name;
1062	struct device *dev = &spe_pmu->pdev->dev;
1063
1064	spe_pmu->pmu = (struct pmu) {
1065		.module = THIS_MODULE,
1066		.parent		= &spe_pmu->pdev->dev,
1067		.capabilities	= PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE,
1068		.attr_groups	= arm_spe_pmu_attr_groups,
1069		/*
1070		 * We hitch a ride on the software context here, so that
1071		 * we can support per-task profiling (which is not possible
1072		 * with the invalid context as it doesn't get sched callbacks).
1073		 * This requires that userspace either uses a dummy event for
1074		 * perf_event_open, since the aux buffer is not setup until
1075		 * a subsequent mmap, or creates the profiling event in a
1076		 * disabled state and explicitly PERF_EVENT_IOC_ENABLEs it
1077		 * once the buffer has been created.
1078		 */
1079		.task_ctx_nr	= perf_sw_context,
1080		.event_init	= arm_spe_pmu_event_init,
1081		.add		= arm_spe_pmu_add,
1082		.del		= arm_spe_pmu_del,
1083		.start		= arm_spe_pmu_start,
1084		.stop		= arm_spe_pmu_stop,
1085		.read		= arm_spe_pmu_read,
1086		.setup_aux	= arm_spe_pmu_setup_aux,
1087		.free_aux	= arm_spe_pmu_free_aux,
1088	};
1089
1090	idx = atomic_inc_return(&pmu_idx);
1091	name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx);
1092	if (!name) {
1093		dev_err(dev, "failed to allocate name for pmu %d\n", idx);
1094		return -ENOMEM;
1095	}
1096
1097	return perf_pmu_register(&spe_pmu->pmu, name, -1);
1098}
1099
1100static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu)
1101{
1102	perf_pmu_unregister(&spe_pmu->pmu);
1103}
1104
1105static void __arm_spe_pmu_dev_probe(void *info)
1106{
1107	int fld;
1108	u64 reg;
1109	struct arm_spe_pmu *spe_pmu = info;
1110	struct device *dev = &spe_pmu->pdev->dev;
1111
1112	fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1),
1113						   ID_AA64DFR0_EL1_PMSVer_SHIFT);
1114	if (!fld) {
1115		dev_err(dev,
1116			"unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n",
1117			fld, smp_processor_id());
1118		return;
1119	}
1120	spe_pmu->pmsver = (u16)fld;
1121
1122	/* Read PMBIDR first to determine whether or not we have access */
1123	reg = read_sysreg_s(SYS_PMBIDR_EL1);
1124	if (FIELD_GET(PMBIDR_EL1_P, reg)) {
1125		dev_err(dev,
1126			"profiling buffer owned by higher exception level\n");
1127		return;
1128	}
1129
1130	/* Minimum alignment. If it's out-of-range, then fail the probe */
1131	fld = FIELD_GET(PMBIDR_EL1_ALIGN, reg);
1132	spe_pmu->align = 1 << fld;
1133	if (spe_pmu->align > SZ_2K) {
1134		dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n",
1135			fld, smp_processor_id());
1136		return;
1137	}
1138
1139	/* It's now safe to read PMSIDR and figure out what we've got */
1140	reg = read_sysreg_s(SYS_PMSIDR_EL1);
1141	if (FIELD_GET(PMSIDR_EL1_FE, reg))
1142		spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT;
1143
1144	if (FIELD_GET(PMSIDR_EL1_FnE, reg))
1145		spe_pmu->features |= SPE_PMU_FEAT_INV_FILT_EVT;
1146
1147	if (FIELD_GET(PMSIDR_EL1_FT, reg))
1148		spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP;
1149
1150	if (FIELD_GET(PMSIDR_EL1_FL, reg))
1151		spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT;
1152
1153	if (FIELD_GET(PMSIDR_EL1_ARCHINST, reg))
1154		spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST;
1155
1156	if (FIELD_GET(PMSIDR_EL1_LDS, reg))
1157		spe_pmu->features |= SPE_PMU_FEAT_LDS;
1158
1159	if (FIELD_GET(PMSIDR_EL1_ERND, reg))
1160		spe_pmu->features |= SPE_PMU_FEAT_ERND;
1161
1162	if (spe_pmu->pmsver >= ID_AA64DFR0_EL1_PMSVer_V1P2)
1163		spe_pmu->features |= SPE_PMU_FEAT_DISCARD;
1164
1165	if (FIELD_GET(PMSIDR_EL1_EFT, reg))
1166		spe_pmu->features |= SPE_PMU_FEAT_EFT;
1167
1168	if (FIELD_GET(PMSIDR_EL1_FDS, reg))
1169		spe_pmu->features |= SPE_PMU_FEAT_FDS;
1170
1171	/* This field has a spaced out encoding, so just use a look-up */
1172	fld = FIELD_GET(PMSIDR_EL1_INTERVAL, reg);
1173	switch (fld) {
1174	case PMSIDR_EL1_INTERVAL_256:
1175		spe_pmu->min_period = 256;
1176		break;
1177	case PMSIDR_EL1_INTERVAL_512:
1178		spe_pmu->min_period = 512;
1179		break;
1180	case PMSIDR_EL1_INTERVAL_768:
1181		spe_pmu->min_period = 768;
1182		break;
1183	case PMSIDR_EL1_INTERVAL_1024:
1184		spe_pmu->min_period = 1024;
1185		break;
1186	case PMSIDR_EL1_INTERVAL_1536:
1187		spe_pmu->min_period = 1536;
1188		break;
1189	case PMSIDR_EL1_INTERVAL_2048:
1190		spe_pmu->min_period = 2048;
1191		break;
1192	case PMSIDR_EL1_INTERVAL_3072:
1193		spe_pmu->min_period = 3072;
1194		break;
1195	default:
1196		dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n",
1197			 fld);
1198		fallthrough;
1199	case PMSIDR_EL1_INTERVAL_4096:
1200		spe_pmu->min_period = 4096;
1201	}
1202
1203	/* Maximum record size. If it's out-of-range, then fail the probe */
1204	fld = FIELD_GET(PMSIDR_EL1_MAXSIZE, reg);
1205	spe_pmu->max_record_sz = 1 << fld;
1206	if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) {
1207		dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n",
1208			fld, smp_processor_id());
1209		return;
1210	}
1211
1212	fld = FIELD_GET(PMSIDR_EL1_COUNTSIZE, reg);
1213	switch (fld) {
1214	default:
1215		dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n",
1216			 fld);
1217		fallthrough;
1218	case PMSIDR_EL1_COUNTSIZE_12_BIT_SAT:
1219		spe_pmu->counter_sz = 12;
1220		break;
1221	case PMSIDR_EL1_COUNTSIZE_16_BIT_SAT:
1222		spe_pmu->counter_sz = 16;
1223	}
1224
1225	/* Write all 1s and then read back. Unsupported filter bits are RAZ/WI. */
1226	write_sysreg_s(U64_MAX, SYS_PMSEVFR_EL1);
1227	spe_pmu->pmsevfr_res0 = ~read_sysreg_s(SYS_PMSEVFR_EL1);
1228
1229	dev_info(dev,
1230		 "probed SPEv1.%d for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n",
1231		 spe_pmu->pmsver - 1, cpumask_pr_args(&spe_pmu->supported_cpus),
1232		 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);
1233
1234	spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED;
1235}
1236
1237static void __arm_spe_pmu_reset_local(void)
1238{
1239	/*
1240	 * This is probably overkill, as we have no idea where we're
1241	 * draining any buffered data to...
1242	 */
1243	arm_spe_pmu_disable_and_drain_local();
1244
1245	/* Reset the buffer base pointer */
1246	write_sysreg_s(0, SYS_PMBPTR_EL1);
1247	isb();
1248
1249	/* Clear any pending management interrupts */
1250	write_sysreg_s(0, SYS_PMBSR_EL1);
1251	isb();
1252}
1253
1254static void __arm_spe_pmu_setup_one(void *info)
1255{
1256	struct arm_spe_pmu *spe_pmu = info;
1257
1258	__arm_spe_pmu_reset_local();
1259	enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE);
1260}
1261
1262static void __arm_spe_pmu_stop_one(void *info)
1263{
1264	struct arm_spe_pmu *spe_pmu = info;
1265
1266	disable_percpu_irq(spe_pmu->irq);
1267	__arm_spe_pmu_reset_local();
1268}
1269
1270static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node)
1271{
1272	struct arm_spe_pmu *spe_pmu;
1273
1274	spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);
1275	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
1276		return 0;
1277
1278	__arm_spe_pmu_setup_one(spe_pmu);
1279	return 0;
1280}
1281
1282static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1283{
1284	struct arm_spe_pmu *spe_pmu;
1285
1286	spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node);
1287	if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus))
1288		return 0;
1289
1290	__arm_spe_pmu_stop_one(spe_pmu);
1291	return 0;
1292}
1293
1294static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu)
1295{
1296	int ret;
1297	cpumask_t *mask = &spe_pmu->supported_cpus;
1298
1299	/* Make sure we probe the hardware on a relevant CPU */
1300	ret = smp_call_function_any(mask,  __arm_spe_pmu_dev_probe, spe_pmu, 1);
1301	if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED))
1302		return -ENXIO;
1303
1304	/* Request our PPIs (note that the IRQ is still disabled) */
1305	ret = request_percpu_irq_affinity(spe_pmu->irq, arm_spe_pmu_irq_handler,
1306					  DRVNAME, mask, spe_pmu->handle);
1307	if (ret)
1308		return ret;
1309
1310	/*
1311	 * Register our hotplug notifier now so we don't miss any events.
1312	 * This will enable the IRQ for any supported CPUs that are already
1313	 * up.
1314	 */
1315	ret = cpuhp_state_add_instance(arm_spe_pmu_online,
1316				       &spe_pmu->hotplug_node);
1317	if (ret)
1318		free_percpu_irq(spe_pmu->irq, spe_pmu->handle);
1319
1320	return ret;
1321}
1322
1323static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu)
1324{
1325	cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node);
1326	free_percpu_irq(spe_pmu->irq, spe_pmu->handle);
1327}
1328
1329/* Driver and device probing */
1330static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu)
1331{
1332	struct platform_device *pdev = spe_pmu->pdev;
1333	const struct cpumask *affinity;
1334	int irq;
1335
1336	irq = platform_get_irq_affinity(pdev, 0, &affinity);
1337	if (irq < 0)
1338		return -ENXIO;
1339
1340	if (!irq_is_percpu(irq)) {
1341		dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq);
1342		return -EINVAL;
1343	}
1344
1345	cpumask_copy(&spe_pmu->supported_cpus, affinity);
1346
1347	spe_pmu->irq = irq;
1348	return 0;
1349}
1350
1351static const struct of_device_id arm_spe_pmu_of_match[] = {
1352	{ .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 },
1353	{ /* Sentinel */ },
1354};
1355MODULE_DEVICE_TABLE(of, arm_spe_pmu_of_match);
1356
1357static const struct platform_device_id arm_spe_match[] = {
1358	{ ARMV8_SPE_PDEV_NAME, 0},
1359	{ }
1360};
1361MODULE_DEVICE_TABLE(platform, arm_spe_match);
1362
1363static int arm_spe_pmu_device_probe(struct platform_device *pdev)
1364{
1365	int ret;
1366	struct arm_spe_pmu *spe_pmu;
1367	struct device *dev = &pdev->dev;
1368
1369	/*
1370	 * If kernelspace is unmapped when running at EL0, then the SPE
1371	 * buffer will fault and prematurely terminate the AUX session.
1372	 */
1373	if (arm64_kernel_unmapped_at_el0()) {
1374		dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command line\n");
1375		return -EPERM;
1376	}
1377
1378	spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL);
1379	if (!spe_pmu)
1380		return -ENOMEM;
1381
1382	spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));
1383	if (!spe_pmu->handle)
1384		return -ENOMEM;
1385
1386	spe_pmu->pdev = pdev;
1387	platform_set_drvdata(pdev, spe_pmu);
1388
1389	ret = arm_spe_pmu_irq_probe(spe_pmu);
1390	if (ret)
1391		goto out_free_handle;
1392
1393	ret = arm_spe_pmu_dev_init(spe_pmu);
1394	if (ret)
1395		goto out_free_handle;
1396
1397	ret = arm_spe_pmu_perf_init(spe_pmu);
1398	if (ret)
1399		goto out_teardown_dev;
1400
1401	return 0;
1402
1403out_teardown_dev:
1404	arm_spe_pmu_dev_teardown(spe_pmu);
1405out_free_handle:
1406	free_percpu(spe_pmu->handle);
1407	return ret;
1408}
1409
1410static void arm_spe_pmu_device_remove(struct platform_device *pdev)
1411{
1412	struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev);
1413
1414	arm_spe_pmu_perf_destroy(spe_pmu);
1415	arm_spe_pmu_dev_teardown(spe_pmu);
1416	free_percpu(spe_pmu->handle);
1417}
1418
1419static struct platform_driver arm_spe_pmu_driver = {
1420	.id_table = arm_spe_match,
1421	.driver	= {
1422		.name		= DRVNAME,
1423		.of_match_table	= of_match_ptr(arm_spe_pmu_of_match),
1424		.suppress_bind_attrs = true,
1425	},
1426	.probe	= arm_spe_pmu_device_probe,
1427	.remove = arm_spe_pmu_device_remove,
1428};
1429
1430static int __init arm_spe_pmu_init(void)
1431{
1432	int ret;
1433
1434	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME,
1435				      arm_spe_pmu_cpu_startup,
1436				      arm_spe_pmu_cpu_teardown);
1437	if (ret < 0)
1438		return ret;
1439	arm_spe_pmu_online = ret;
1440
1441	ret = platform_driver_register(&arm_spe_pmu_driver);
1442	if (ret)
1443		cpuhp_remove_multi_state(arm_spe_pmu_online);
1444
1445	return ret;
1446}
1447
1448static void __exit arm_spe_pmu_exit(void)
1449{
1450	platform_driver_unregister(&arm_spe_pmu_driver);
1451	cpuhp_remove_multi_state(arm_spe_pmu_online);
1452}
1453
1454module_init(arm_spe_pmu_init);
1455module_exit(arm_spe_pmu_exit);
1456
1457MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension");
1458MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>");
1459MODULE_LICENSE("GPL v2");