arm-cmn.c

   1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (C) 2016-2020 Arm Limited
   3// ARM CMN/CI interconnect PMU driver
   4
   5#include <linux/acpi.h>
   6#include <linux/bitfield.h>
   7#include <linux/bitops.h>
   8#include <linux/debugfs.h>
   9#include <linux/interrupt.h>
  10#include <linux/io.h>
  11#include <linux/io-64-nonatomic-lo-hi.h>
  12#include <linux/kernel.h>
  13#include <linux/list.h>
  14#include <linux/module.h>
  15#include <linux/of.h>
  16#include <linux/perf_event.h>
  17#include <linux/platform_device.h>
  18#include <linux/slab.h>
  19#include <linux/sort.h>
  20
  21/* Common register stuff */
  22#define CMN_NODE_INFO			0x0000
  23#define CMN_NI_NODE_TYPE		GENMASK_ULL(15, 0)
  24#define CMN_NI_NODE_ID			GENMASK_ULL(31, 16)
  25#define CMN_NI_LOGICAL_ID		GENMASK_ULL(47, 32)
  26
  27#define CMN_CHILD_INFO			0x0080
  28#define CMN_CI_CHILD_COUNT		GENMASK_ULL(15, 0)
  29#define CMN_CI_CHILD_PTR_OFFSET		GENMASK_ULL(31, 16)
  30
  31#define CMN_CHILD_NODE_ADDR		GENMASK(29, 0)
  32#define CMN_CHILD_NODE_EXTERNAL		BIT(31)
  33
  34#define CMN_MAX_DIMENSION		12
  35#define CMN_MAX_XPS			(CMN_MAX_DIMENSION * CMN_MAX_DIMENSION)
  36#define CMN_MAX_DTMS			(CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4)
  37
  38/* Currently XPs are the node type we can have most of; others top out at 128 */
  39#define CMN_MAX_NODES_PER_EVENT		CMN_MAX_XPS
  40
  41/* The CFG node has various info besides the discovery tree */
  42#define CMN_CFGM_PERIPH_ID_01		0x0008
  43#define CMN_CFGM_PID0_PART_0		GENMASK_ULL(7, 0)
  44#define CMN_CFGM_PID1_PART_1		GENMASK_ULL(35, 32)
  45#define CMN_CFGM_PERIPH_ID_23		0x0010
  46#define CMN_CFGM_PID2_REVISION		GENMASK_ULL(7, 4)
  47
  48#define CMN_CFGM_INFO_GLOBAL		0x0900
  49#define CMN_INFO_MULTIPLE_DTM_EN	BIT_ULL(63)
  50#define CMN_INFO_RSP_VC_NUM		GENMASK_ULL(53, 52)
  51#define CMN_INFO_DAT_VC_NUM		GENMASK_ULL(51, 50)
  52#define CMN_INFO_DEVICE_ISO_ENABLE	BIT_ULL(44)
  53
  54#define CMN_CFGM_INFO_GLOBAL_1		0x0908
  55#define CMN_INFO_SNP_VC_NUM		GENMASK_ULL(3, 2)
  56#define CMN_INFO_REQ_VC_NUM		GENMASK_ULL(1, 0)
  57
  58/* XPs also have some local topology info which has uses too */
  59#define CMN_MXP__CONNECT_INFO(p)	(0x0008 + 8 * (p))
  60#define CMN__CONNECT_INFO_DEVICE_TYPE	GENMASK_ULL(5, 0)
  61
  62#define CMN_MAX_PORTS			6
  63#define CI700_CONNECT_INFO_P2_5_OFFSET	0x10
  64
  65/* PMU registers occupy the 3rd 4KB page of each node's region */
  66#define CMN_PMU_OFFSET			0x2000
  67/* ...except when they don't :( */
  68#define CMN_S3_R1_DTM_OFFSET		0xa000
  69#define CMN_S3_PMU_OFFSET		0xd900
  70
  71/* For most nodes, this is all there is */
  72#define CMN_PMU_EVENT_SEL		0x000
  73#define CMN__PMU_CBUSY_SNTHROTTLE_SEL	GENMASK_ULL(44, 42)
  74#define CMN__PMU_SN_HOME_SEL		GENMASK_ULL(40, 39)
  75#define CMN__PMU_HBT_LBT_SEL		GENMASK_ULL(38, 37)
  76#define CMN__PMU_CLASS_OCCUP_ID		GENMASK_ULL(36, 35)
  77/* Technically this is 4 bits wide on DNs, but we only use 2 there anyway */
  78#define CMN__PMU_OCCUP1_ID		GENMASK_ULL(34, 32)
  79
  80/* Some types are designed to coexist with another device in the same node */
  81#define CMN_CCLA_PMU_EVENT_SEL		0x008
  82#define CMN_HNP_PMU_EVENT_SEL		0x008
  83
  84/* DTMs live in the PMU space of XP registers */
  85#define CMN_DTM_WPn(n)			(0x1A0 + (n) * 0x18)
  86#define CMN_DTM_WPn_CONFIG(n)		(CMN_DTM_WPn(n) + 0x00)
  87#define CMN_DTM_WPn_CONFIG_WP_CHN_NUM	GENMASK_ULL(20, 19)
  88#define CMN_DTM_WPn_CONFIG_WP_DEV_SEL2	GENMASK_ULL(18, 17)
  89#define CMN_DTM_WPn_CONFIG_WP_COMBINE	BIT(9)
  90#define CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE	BIT(8)
  91#define CMN600_WPn_CONFIG_WP_COMBINE	BIT(6)
  92#define CMN600_WPn_CONFIG_WP_EXCLUSIVE	BIT(5)
  93#define CMN_DTM_WPn_CONFIG_WP_GRP	GENMASK_ULL(5, 4)
  94#define CMN_DTM_WPn_CONFIG_WP_CHN_SEL	GENMASK_ULL(3, 1)
  95#define CMN_DTM_WPn_CONFIG_WP_DEV_SEL	BIT(0)
  96#define CMN_DTM_WPn_VAL(n)		(CMN_DTM_WPn(n) + 0x08)
  97#define CMN_DTM_WPn_MASK(n)		(CMN_DTM_WPn(n) + 0x10)
  98
  99#define CMN_DTM_PMU_CONFIG		0x210
 100#define CMN__PMEVCNT0_INPUT_SEL		GENMASK_ULL(37, 32)
 101#define CMN__PMEVCNT0_INPUT_SEL_WP	0x00
 102#define CMN__PMEVCNT0_INPUT_SEL_XP	0x04
 103#define CMN__PMEVCNT0_INPUT_SEL_DEV	0x10
 104#define CMN__PMEVCNT0_GLOBAL_NUM	GENMASK_ULL(18, 16)
 105#define CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(n)	((n) * 4)
 106#define CMN__PMEVCNT_PAIRED(n)		BIT(4 + (n))
 107#define CMN__PMEVCNT23_COMBINED		BIT(2)
 108#define CMN__PMEVCNT01_COMBINED		BIT(1)
 109#define CMN_DTM_PMU_CONFIG_PMU_EN	BIT(0)
 110
 111#define CMN_DTM_PMEVCNT			0x220
 112
 113#define CMN_DTM_PMEVCNTSR		0x240
 114
 115#define CMN650_DTM_UNIT_INFO		0x0910
 116#define CMN_DTM_UNIT_INFO		0x0960
 117#define CMN_DTM_UNIT_INFO_DTC_DOMAIN	GENMASK_ULL(1, 0)
 118
 119#define CMN_DTM_NUM_COUNTERS		4
 120/* Want more local counters? Why not replicate the whole DTM! Ugh... */
 121#define CMN_DTM_OFFSET(n)		((n) * 0x200)
 122
 123/* The DTC node is where the magic happens */
 124#define CMN_DT_DTC_CTL			0x0a00
 125#define CMN_DT_DTC_CTL_DT_EN		BIT(0)
 126#define CMN_DT_DTC_CTL_CG_DISABLE	BIT(10)
 127
 128/* DTC counters are paired in 64-bit registers on a 16-byte stride. Yuck */
 129#define _CMN_DT_CNT_REG(n)		((((n) / 2) * 4 + (n) % 2) * 4)
 130#define CMN_DT_PMEVCNT(dtc, n)		((dtc)->pmu_base + _CMN_DT_CNT_REG(n))
 131#define CMN_DT_PMCCNTR(dtc)		((dtc)->pmu_base + 0x40)
 132
 133#define CMN_DT_PMEVCNTSR(dtc, n)	((dtc)->pmu_base + 0x50 + _CMN_DT_CNT_REG(n))
 134#define CMN_DT_PMCCNTRSR(dtc)		((dtc)->pmu_base + 0x90)
 135
 136#define CMN_DT_PMCR(dtc)		((dtc)->pmu_base + 0x100)
 137#define CMN_DT_PMCR_PMU_EN		BIT(0)
 138#define CMN_DT_PMCR_CNTR_RST		BIT(5)
 139#define CMN_DT_PMCR_OVFL_INTR_EN	BIT(6)
 140
 141#define CMN_DT_PMOVSR(dtc)		((dtc)->pmu_base + 0x118)
 142#define CMN_DT_PMOVSR_CLR(dtc)		((dtc)->pmu_base + 0x120)
 143
 144#define CMN_DT_PMSSR(dtc)		((dtc)->pmu_base + 0x128)
 145#define CMN_DT_PMSSR_SS_STATUS(n)	BIT(n)
 146
 147#define CMN_DT_PMSRR(dtc)		((dtc)->pmu_base + 0x130)
 148#define CMN_DT_PMSRR_SS_REQ		BIT(0)
 149
 150#define CMN_DT_NUM_COUNTERS		8
 151#define CMN_MAX_DTCS			4
 152
 153/*
 154 * Even in the worst case a DTC counter can't wrap in fewer than 2^42 cycles,
 155 * so throwing away one bit to make overflow handling easy is no big deal.
 156 */
 157#define CMN_COUNTER_INIT		0x80000000
 158/* Similarly for the 40-bit cycle counter */
 159#define CMN_CC_INIT			0x8000000000ULL
 160
 161
 162/* Event attributes */
 163#define CMN_CONFIG_TYPE			GENMASK_ULL(15, 0)
 164#define CMN_CONFIG_EVENTID		GENMASK_ULL(26, 16)
 165#define CMN_CONFIG_OCCUPID		GENMASK_ULL(30, 27)
 166#define CMN_CONFIG_BYNODEID		BIT_ULL(31)
 167#define CMN_CONFIG_NODEID		GENMASK_ULL(47, 32)
 168
 169#define CMN_EVENT_TYPE(event)		FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config)
 170#define CMN_EVENT_EVENTID(event)	FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config)
 171#define CMN_EVENT_OCCUPID(event)	FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config)
 172#define CMN_EVENT_BYNODEID(event)	FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config)
 173#define CMN_EVENT_NODEID(event)		FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config)
 174
 175#define CMN_CONFIG_WP_COMBINE		GENMASK_ULL(30, 27)
 176#define CMN_CONFIG_WP_DEV_SEL		GENMASK_ULL(50, 48)
 177#define CMN_CONFIG_WP_CHN_SEL		GENMASK_ULL(55, 51)
 178#define CMN_CONFIG_WP_GRP		GENMASK_ULL(57, 56)
 179#define CMN_CONFIG_WP_EXCLUSIVE		BIT_ULL(58)
 180#define CMN_CONFIG1_WP_VAL		GENMASK_ULL(63, 0)
 181#define CMN_CONFIG2_WP_MASK		GENMASK_ULL(63, 0)
 182
 183#define CMN_EVENT_WP_COMBINE(event)	FIELD_GET(CMN_CONFIG_WP_COMBINE, (event)->attr.config)
 184#define CMN_EVENT_WP_DEV_SEL(event)	FIELD_GET(CMN_CONFIG_WP_DEV_SEL, (event)->attr.config)
 185#define CMN_EVENT_WP_CHN_SEL(event)	FIELD_GET(CMN_CONFIG_WP_CHN_SEL, (event)->attr.config)
 186#define CMN_EVENT_WP_GRP(event)		FIELD_GET(CMN_CONFIG_WP_GRP, (event)->attr.config)
 187#define CMN_EVENT_WP_EXCLUSIVE(event)	FIELD_GET(CMN_CONFIG_WP_EXCLUSIVE, (event)->attr.config)
 188#define CMN_EVENT_WP_VAL(event)		FIELD_GET(CMN_CONFIG1_WP_VAL, (event)->attr.config1)
 189#define CMN_EVENT_WP_MASK(event)	FIELD_GET(CMN_CONFIG2_WP_MASK, (event)->attr.config2)
 190
 191/* Made-up event IDs for watchpoint direction */
 192#define CMN_WP_UP			0
 193#define CMN_WP_DOWN			2
 194
 195
 196/* Internal values for encoding event support */
 197enum cmn_model {
 198	CMN600 = 1,
 199	CMN650 = 2,
 200	CMN700 = 4,
 201	CI700 = 8,
 202	CMNS3 = 16,
 203	/* ...and then we can use bitmap tricks for commonality */
 204	CMN_ANY = -1,
 205	NOT_CMN600 = -2,
 206	CMN_650ON = CMN650 | CMN700 | CMNS3,
 207};
 208
 209/* Actual part numbers and revision IDs defined by the hardware */
 210enum cmn_part {
 211	PART_CMN600 = 0x434,
 212	PART_CMN650 = 0x436,
 213	PART_CMN600AE = 0x438,
 214	PART_CMN700 = 0x43c,
 215	PART_CI700 = 0x43a,
 216	PART_CMN_S3 = 0x43e,
 217};
 218
 219/* CMN-600 r0px shouldn't exist in silicon, thankfully */
 220enum cmn_revision {
 221	REV_CMN600_R1P0,
 222	REV_CMN600_R1P1,
 223	REV_CMN600_R1P2,
 224	REV_CMN600_R1P3,
 225	REV_CMN600_R2P0,
 226	REV_CMN600_R3P0,
 227	REV_CMN600_R3P1,
 228	REV_CMN650_R0P0 = 0,
 229	REV_CMN650_R1P0,
 230	REV_CMN650_R1P1,
 231	REV_CMN650_R2P0,
 232	REV_CMN650_R1P2,
 233	REV_CMN700_R0P0 = 0,
 234	REV_CMN700_R1P0,
 235	REV_CMN700_R2P0,
 236	REV_CMN700_R3P0,
 237	REV_CMNS3_R0P0 = 0,
 238	REV_CMNS3_R0P1,
 239	REV_CMNS3_R1P0,
 240	REV_CI700_R0P0 = 0,
 241	REV_CI700_R1P0,
 242	REV_CI700_R2P0,
 243};
 244
 245enum cmn_node_type {
 246	CMN_TYPE_INVALID,
 247	CMN_TYPE_DVM,
 248	CMN_TYPE_CFG,
 249	CMN_TYPE_DTC,
 250	CMN_TYPE_HNI,
 251	CMN_TYPE_HNF,
 252	CMN_TYPE_XP,
 253	CMN_TYPE_SBSX,
 254	CMN_TYPE_MPAM_S,
 255	CMN_TYPE_MPAM_NS,
 256	CMN_TYPE_RNI,
 257	CMN_TYPE_RND = 0xd,
 258	CMN_TYPE_RNSAM = 0xf,
 259	CMN_TYPE_MTSX,
 260	CMN_TYPE_HNP,
 261	CMN_TYPE_CXRA = 0x100,
 262	CMN_TYPE_CXHA,
 263	CMN_TYPE_CXLA,
 264	CMN_TYPE_CCRA,
 265	CMN_TYPE_CCHA,
 266	CMN_TYPE_CCLA,
 267	CMN_TYPE_CCLA_RNI,
 268	CMN_TYPE_HNS = 0x200,
 269	CMN_TYPE_HNS_MPAM_S,
 270	CMN_TYPE_HNS_MPAM_NS,
 271	CMN_TYPE_APB = 0x1000,
 272	/* Not a real node type */
 273	CMN_TYPE_WP = 0x7770
 274};
 275
 276enum cmn_filter_select {
 277	SEL_NONE = -1,
 278	SEL_OCCUP1ID,
 279	SEL_CLASS_OCCUP_ID,
 280	SEL_CBUSY_SNTHROTTLE_SEL,
 281	SEL_HBT_LBT_SEL,
 282	SEL_SN_HOME_SEL,
 283	SEL_MAX
 284};
 285
 286struct arm_cmn_node {
 287	void __iomem *pmu_base;
 288	u16 id, logid;
 289	enum cmn_node_type type;
 290
 291	/* XP properties really, but replicated to children for convenience */
 292	u8 dtm;
 293	s8 dtc;
 294	u8 portid_bits:4;
 295	u8 deviceid_bits:4;
 296	/* DN/HN-F/CXHA */
 297	struct {
 298		u8 val : 4;
 299		u8 count : 4;
 300	} occupid[SEL_MAX];
 301	union {
 302		u8 event[4];
 303		__le32 event_sel;
 304		u16 event_w[4];
 305		__le64 event_sel_w;
 306	};
 307};
 308
 309struct arm_cmn_dtm {
 310	void __iomem *base;
 311	u32 pmu_config_low;
 312	union {
 313		u8 input_sel[4];
 314		__le32 pmu_config_high;
 315	};
 316	s8 wp_event[4];
 317};
 318
 319struct arm_cmn_dtc {
 320	void __iomem *base;
 321	void __iomem *pmu_base;
 322	int irq;
 323	s8 irq_friend;
 324	bool cc_active;
 325
 326	struct perf_event *counters[CMN_DT_NUM_COUNTERS];
 327	struct perf_event *cycles;
 328};
 329
 330#define CMN_STATE_DISABLED	BIT(0)
 331#define CMN_STATE_TXN		BIT(1)
 332
 333struct arm_cmn {
 334	struct device *dev;
 335	void __iomem *base;
 336	unsigned int state;
 337
 338	enum cmn_revision rev;
 339	enum cmn_part part;
 340	u8 mesh_x;
 341	u8 mesh_y;
 342	u16 num_xps;
 343	u16 num_dns;
 344	bool multi_dtm;
 345	u8 ports_used;
 346	struct {
 347		unsigned int rsp_vc_num : 2;
 348		unsigned int dat_vc_num : 2;
 349		unsigned int snp_vc_num : 2;
 350		unsigned int req_vc_num : 2;
 351	};
 352
 353	struct arm_cmn_node *xps;
 354	struct arm_cmn_node *dns;
 355
 356	struct arm_cmn_dtm *dtms;
 357	struct arm_cmn_dtc *dtc;
 358	unsigned int num_dtcs;
 359
 360	int cpu;
 361	struct hlist_node cpuhp_node;
 362
 363	struct pmu pmu;
 364	struct dentry *debug;
 365};
 366
 367#define to_cmn(p)	container_of(p, struct arm_cmn, pmu)
 368
 369static int arm_cmn_hp_state;
 370
 371struct arm_cmn_nodeid {
 372	u8 port;
 373	u8 dev;
 374};
 375
 376static int arm_cmn_xyidbits(const struct arm_cmn *cmn)
 377{
 378	return fls((cmn->mesh_x - 1) | (cmn->mesh_y - 1));
 379}
 380
 381static struct arm_cmn_nodeid arm_cmn_nid(const struct arm_cmn_node *dn)
 382{
 383	struct arm_cmn_nodeid nid;
 384
 385	nid.dev = dn->id & ((1U << dn->deviceid_bits) - 1);
 386	nid.port = (dn->id >> dn->deviceid_bits) & ((1U << dn->portid_bits) - 1);
 387	return nid;
 388}
 389
 390static struct arm_cmn_node *arm_cmn_node_to_xp(const struct arm_cmn *cmn,
 391					       const struct arm_cmn_node *dn)
 392{
 393	int id = dn->id >> (dn->portid_bits + dn->deviceid_bits);
 394	int bits = arm_cmn_xyidbits(cmn);
 395	int x = id >> bits;
 396	int y = id & ((1U << bits) - 1);
 397
 398	return cmn->xps + cmn->mesh_x * y + x;
 399}
 400static struct arm_cmn_node *arm_cmn_node(const struct arm_cmn *cmn,
 401					 enum cmn_node_type type)
 402{
 403	struct arm_cmn_node *dn;
 404
 405	for (dn = cmn->dns; dn->type; dn++)
 406		if (dn->type == type)
 407			return dn;
 408	return NULL;
 409}
 410
 411static enum cmn_model arm_cmn_model(const struct arm_cmn *cmn)
 412{
 413	switch (cmn->part) {
 414	case PART_CMN600:
 415		return CMN600;
 416	case PART_CMN650:
 417		return CMN650;
 418	case PART_CMN700:
 419		return CMN700;
 420	case PART_CI700:
 421		return CI700;
 422	case PART_CMN_S3:
 423		return CMNS3;
 424	default:
 425		return 0;
 426	};
 427}
 428
 429static int arm_cmn_pmu_offset(const struct arm_cmn *cmn, const struct arm_cmn_node *dn)
 430{
 431	if (cmn->part == PART_CMN_S3) {
 432		if (cmn->rev >= REV_CMNS3_R1P0 && dn->type == CMN_TYPE_XP)
 433			return CMN_S3_R1_DTM_OFFSET;
 434		return CMN_S3_PMU_OFFSET;
 435	}
 436	return CMN_PMU_OFFSET;
 437}
 438
 439static u32 arm_cmn_device_connect_info(const struct arm_cmn *cmn,
 440				       const struct arm_cmn_node *xp, int port)
 441{
 442	int offset = CMN_MXP__CONNECT_INFO(port) - arm_cmn_pmu_offset(cmn, xp);
 443
 444	if (port >= 2) {
 445		if (cmn->part == PART_CMN600 || cmn->part == PART_CMN650)
 446			return 0;
 447		/*
 448		 * CI-700 may have extra ports, but still has the
 449		 * mesh_port_connect_info registers in the way.
 450		 */
 451		if (cmn->part == PART_CI700)
 452			offset += CI700_CONNECT_INFO_P2_5_OFFSET;
 453	}
 454
 455	return readl_relaxed(xp->pmu_base + offset);
 456}
 457
 458static struct dentry *arm_cmn_debugfs;
 459
 460#ifdef CONFIG_DEBUG_FS
 461static const char *arm_cmn_device_type(u8 type)
 462{
 463	switch(FIELD_GET(CMN__CONNECT_INFO_DEVICE_TYPE, type)) {
 464		case 0x00: return "        |";
 465		case 0x01: return "  RN-I  |";
 466		case 0x02: return "  RN-D  |";
 467		case 0x04: return " RN-F_B |";
 468		case 0x05: return "RN-F_B_E|";
 469		case 0x06: return " RN-F_A |";
 470		case 0x07: return "RN-F_A_E|";
 471		case 0x08: return "  HN-T  |";
 472		case 0x09: return "  HN-I  |";
 473		case 0x0a: return "  HN-D  |";
 474		case 0x0b: return "  HN-P  |";
 475		case 0x0c: return "  SN-F  |";
 476		case 0x0d: return "  SBSX  |";
 477		case 0x0e: return "  HN-F  |";
 478		case 0x0f: return " SN-F_E |";
 479		case 0x10: return " SN-F_D |";
 480		case 0x11: return "  CXHA  |";
 481		case 0x12: return "  CXRA  |";
 482		case 0x13: return "  CXRH  |";
 483		case 0x14: return " RN-F_D |";
 484		case 0x15: return "RN-F_D_E|";
 485		case 0x16: return " RN-F_C |";
 486		case 0x17: return "RN-F_C_E|";
 487		case 0x18: return " RN-F_E |";
 488		case 0x19: return "RN-F_E_E|";
 489		case 0x1a: return "  HN-S  |";
 490		case 0x1b: return "  LCN   |";
 491		case 0x1c: return "  MTSX  |";
 492		case 0x1d: return "  HN-V  |";
 493		case 0x1e: return "  CCG   |";
 494		case 0x20: return " RN-F_F |";
 495		case 0x21: return "RN-F_F_E|";
 496		case 0x22: return " SN-F_F |";
 497		default:   return "  ????  |";
 498	}
 499}
 500
 501static void arm_cmn_show_logid(struct seq_file *s, const struct arm_cmn_node *xp, int p, int d)
 502{
 503	struct arm_cmn *cmn = s->private;
 504	struct arm_cmn_node *dn;
 505	u16 id = xp->id | d | (p << xp->deviceid_bits);
 506
 507	for (dn = cmn->dns; dn->type; dn++) {
 508		int pad = dn->logid < 10;
 509
 510		if (dn->type == CMN_TYPE_XP)
 511			continue;
 512		/* Ignore the extra components that will overlap on some ports */
 513		if (dn->type < CMN_TYPE_HNI)
 514			continue;
 515
 516		if (dn->id != id)
 517			continue;
 518
 519		seq_printf(s, " %*c#%-*d  |", pad + 1, ' ', 3 - pad, dn->logid);
 520		return;
 521	}
 522	seq_puts(s, "        |");
 523}
 524
 525static int arm_cmn_map_show(struct seq_file *s, void *data)
 526{
 527	struct arm_cmn *cmn = s->private;
 528	int x, y, p, pmax = fls(cmn->ports_used);
 529
 530	seq_puts(s, "     X");
 531	for (x = 0; x < cmn->mesh_x; x++)
 532		seq_printf(s, "    %-2d   ", x);
 533	seq_puts(s, "\nY P D+");
 534	y = cmn->mesh_y;
 535	while (y--) {
 536		int xp_base = cmn->mesh_x * y;
 537		struct arm_cmn_node *xp = cmn->xps + xp_base;
 538		u8 port[CMN_MAX_PORTS][CMN_MAX_DIMENSION];
 539
 540		for (x = 0; x < cmn->mesh_x; x++)
 541			seq_puts(s, "--------+");
 542
 543		seq_printf(s, "\n%-2d   |", y);
 544		for (x = 0; x < cmn->mesh_x; x++) {
 545			for (p = 0; p < CMN_MAX_PORTS; p++)
 546				port[p][x] = arm_cmn_device_connect_info(cmn, xp + x, p);
 547			seq_printf(s, " XP #%-3d|", xp_base + x);
 548		}
 549
 550		seq_puts(s, "\n     |");
 551		for (x = 0; x < cmn->mesh_x; x++) {
 552			s8 dtc = xp[x].dtc;
 553
 554			if (dtc < 0)
 555				seq_puts(s, " DTC ?? |");
 556			else
 557				seq_printf(s, " DTC %d  |", dtc);
 558		}
 559		seq_puts(s, "\n     |");
 560		for (x = 0; x < cmn->mesh_x; x++)
 561			seq_puts(s, "........|");
 562
 563		for (p = 0; p < pmax; p++) {
 564			seq_printf(s, "\n  %d  |", p);
 565			for (x = 0; x < cmn->mesh_x; x++)
 566				seq_puts(s, arm_cmn_device_type(port[p][x]));
 567			seq_puts(s, "\n    0|");
 568			for (x = 0; x < cmn->mesh_x; x++)
 569				arm_cmn_show_logid(s, xp + x, p, 0);
 570			seq_puts(s, "\n    1|");
 571			for (x = 0; x < cmn->mesh_x; x++)
 572				arm_cmn_show_logid(s, xp + x, p, 1);
 573		}
 574		seq_puts(s, "\n-----+");
 575	}
 576	for (x = 0; x < cmn->mesh_x; x++)
 577		seq_puts(s, "--------+");
 578	seq_puts(s, "\n");
 579	return 0;
 580}
 581DEFINE_SHOW_ATTRIBUTE(arm_cmn_map);
 582
 583static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id)
 584{
 585	const char *name  = "map";
 586
 587	if (id > 0)
 588		name = devm_kasprintf(cmn->dev, GFP_KERNEL, "map_%d", id);
 589	if (!name)
 590		return;
 591
 592	cmn->debug = debugfs_create_file(name, 0444, arm_cmn_debugfs, cmn, &arm_cmn_map_fops);
 593}
 594#else
 595static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) {}
 596#endif
 597
 598struct arm_cmn_hw_event {
 599	struct arm_cmn_node *dn;
 600	u64 dtm_idx[DIV_ROUND_UP(CMN_MAX_NODES_PER_EVENT * 2, 64)];
 601	s8 dtc_idx[CMN_MAX_DTCS];
 602	u8 num_dns;
 603	u8 dtm_offset;
 604
 605	/*
 606	 * WP config registers are divided to UP and DOWN events. We need to
 607	 * keep to track only one of them.
 608	 */
 609	DECLARE_BITMAP(wp_idx, CMN_MAX_XPS);
 610
 611	bool wide_sel;
 612	enum cmn_filter_select filter_sel;
 613};
 614static_assert(sizeof(struct arm_cmn_hw_event) <= offsetof(struct hw_perf_event, target));
 615
 616#define for_each_hw_dn(hw, dn, i) \
 617	for (i = 0, dn = hw->dn; i < hw->num_dns; i++, dn++)
 618
 619/* @i is the DTC number, @idx is the counter index on that DTC */
 620#define for_each_hw_dtc_idx(hw, i, idx) \
 621	for (int i = 0, idx; i < CMN_MAX_DTCS; i++) if ((idx = hw->dtc_idx[i]) >= 0)
 622
 623static struct arm_cmn_hw_event *to_cmn_hw(struct perf_event *event)
 624{
 625	return (struct arm_cmn_hw_event *)&event->hw;
 626}
 627
 628static void arm_cmn_set_index(u64 x[], unsigned int pos, unsigned int val)
 629{
 630	x[pos / 32] |= (u64)val << ((pos % 32) * 2);
 631}
 632
 633static unsigned int arm_cmn_get_index(u64 x[], unsigned int pos)
 634{
 635	return (x[pos / 32] >> ((pos % 32) * 2)) & 3;
 636}
 637
 638static void arm_cmn_set_wp_idx(unsigned long *wp_idx, unsigned int pos, bool val)
 639{
 640	if (val)
 641		set_bit(pos, wp_idx);
 642}
 643
 644static unsigned int arm_cmn_get_wp_idx(unsigned long *wp_idx, unsigned int pos)
 645{
 646	return test_bit(pos, wp_idx);
 647}
 648
 649struct arm_cmn_event_attr {
 650	struct device_attribute attr;
 651	enum cmn_model model;
 652	enum cmn_node_type type;
 653	enum cmn_filter_select fsel;
 654	u16 eventid;
 655	u8 occupid;
 656};
 657
 658struct arm_cmn_format_attr {
 659	struct device_attribute attr;
 660	u64 field;
 661	int config;
 662};
 663
 664#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _occupid, _fsel)\
 665	(&((struct arm_cmn_event_attr[]) {{				\
 666		.attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL),	\
 667		.model = _model,					\
 668		.type = _type,						\
 669		.eventid = _eventid,					\
 670		.occupid = _occupid,					\
 671		.fsel = _fsel,						\
 672	}})[0].attr.attr)
 673#define CMN_EVENT_ATTR(_model, _name, _type, _eventid)			\
 674	_CMN_EVENT_ATTR(_model, _name, _type, _eventid, 0, SEL_NONE)
 675
 676static ssize_t arm_cmn_event_show(struct device *dev,
 677				  struct device_attribute *attr, char *buf)
 678{
 679	struct arm_cmn_event_attr *eattr;
 680
 681	eattr = container_of(attr, typeof(*eattr), attr);
 682
 683	if (eattr->type == CMN_TYPE_DTC)
 684		return sysfs_emit(buf, "type=0x%x\n", eattr->type);
 685
 686	if (eattr->type == CMN_TYPE_WP)
 687		return sysfs_emit(buf,
 688				  "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n",
 689				  eattr->type, eattr->eventid);
 690
 691	if (eattr->fsel > SEL_NONE)
 692		return sysfs_emit(buf, "type=0x%x,eventid=0x%x,occupid=0x%x\n",
 693				  eattr->type, eattr->eventid, eattr->occupid);
 694
 695	return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type,
 696			  eattr->eventid);
 697}
 698
 699static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj,
 700					     struct attribute *attr,
 701					     int unused)
 702{
 703	struct device *dev = kobj_to_dev(kobj);
 704	struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));
 705	struct arm_cmn_event_attr *eattr;
 706	enum cmn_node_type type;
 707	u16 eventid;
 708
 709	eattr = container_of(attr, typeof(*eattr), attr.attr);
 710
 711	if (!(eattr->model & arm_cmn_model(cmn)))
 712		return 0;
 713
 714	type = eattr->type;
 715	eventid = eattr->eventid;
 716
 717	/* Watchpoints aren't nodes, so avoid confusion */
 718	if (type == CMN_TYPE_WP)
 719		return attr->mode;
 720
 721	/* Hide XP events for unused interfaces/channels */
 722	if (type == CMN_TYPE_XP) {
 723		unsigned int intf = (eventid >> 2) & 7;
 724		unsigned int chan = eventid >> 5;
 725
 726		if ((intf & 4) && !(cmn->ports_used & BIT(intf & 3)))
 727			return 0;
 728
 729		if (chan == 4 && cmn->part == PART_CMN600)
 730			return 0;
 731
 732		if ((chan == 5 && cmn->rsp_vc_num < 2) ||
 733		    (chan == 6 && cmn->dat_vc_num < 2) ||
 734		    (chan == 7 && cmn->req_vc_num < 2) ||
 735		    (chan == 8 && cmn->snp_vc_num < 2))
 736			return 0;
 737	}
 738
 739	/* Revision-specific differences */
 740	if (cmn->part == PART_CMN600) {
 741		if (cmn->rev < REV_CMN600_R1P3) {
 742			if (type == CMN_TYPE_CXRA && eventid > 0x10)
 743				return 0;
 744		}
 745		if (cmn->rev < REV_CMN600_R1P2) {
 746			if (type == CMN_TYPE_HNF && eventid == 0x1b)
 747				return 0;
 748			if (type == CMN_TYPE_CXRA || type == CMN_TYPE_CXHA)
 749				return 0;
 750		}
 751	} else if (cmn->part == PART_CMN650) {
 752		if (cmn->rev < REV_CMN650_R2P0 || cmn->rev == REV_CMN650_R1P2) {
 753			if (type == CMN_TYPE_HNF && eventid > 0x22)
 754				return 0;
 755			if (type == CMN_TYPE_SBSX && eventid == 0x17)
 756				return 0;
 757			if (type == CMN_TYPE_RNI && eventid > 0x10)
 758				return 0;
 759		}
 760	} else if (cmn->part == PART_CMN700) {
 761		if (cmn->rev < REV_CMN700_R2P0) {
 762			if (type == CMN_TYPE_HNF && eventid > 0x2c)
 763				return 0;
 764			if (type == CMN_TYPE_CCHA && eventid > 0x74)
 765				return 0;
 766			if (type == CMN_TYPE_CCLA && eventid > 0x27)
 767				return 0;
 768		}
 769		if (cmn->rev < REV_CMN700_R1P0) {
 770			if (type == CMN_TYPE_HNF && eventid > 0x2b)
 771				return 0;
 772		}
 773	}
 774
 775	if (!arm_cmn_node(cmn, type))
 776		return 0;
 777
 778	return attr->mode;
 779}
 780
 781#define _CMN_EVENT_DVM(_model, _name, _event, _occup, _fsel)	\
 782	_CMN_EVENT_ATTR(_model, dn_##_name, CMN_TYPE_DVM, _event, _occup, _fsel)
 783#define CMN_EVENT_DTC(_name)					\
 784	CMN_EVENT_ATTR(CMN_ANY, dtc_##_name, CMN_TYPE_DTC, 0)
 785#define CMN_EVENT_HNF(_model, _name, _event)			\
 786	CMN_EVENT_ATTR(_model, hnf_##_name, CMN_TYPE_HNF, _event)
 787#define CMN_EVENT_HNI(_name, _event)				\
 788	CMN_EVENT_ATTR(CMN_ANY, hni_##_name, CMN_TYPE_HNI, _event)
 789#define CMN_EVENT_HNP(_name, _event)				\
 790	CMN_EVENT_ATTR(CMN_ANY, hnp_##_name, CMN_TYPE_HNP, _event)
 791#define __CMN_EVENT_XP(_name, _event)				\
 792	CMN_EVENT_ATTR(CMN_ANY, mxp_##_name, CMN_TYPE_XP, _event)
 793#define CMN_EVENT_SBSX(_model, _name, _event)			\
 794	CMN_EVENT_ATTR(_model, sbsx_##_name, CMN_TYPE_SBSX, _event)
 795#define CMN_EVENT_RNID(_model, _name, _event)			\
 796	CMN_EVENT_ATTR(_model, rnid_##_name, CMN_TYPE_RNI, _event)
 797#define CMN_EVENT_MTSX(_name, _event)				\
 798	CMN_EVENT_ATTR(CMN_ANY, mtsx_##_name, CMN_TYPE_MTSX, _event)
 799#define CMN_EVENT_CXRA(_model, _name, _event)				\
 800	CMN_EVENT_ATTR(_model, cxra_##_name, CMN_TYPE_CXRA, _event)
 801#define CMN_EVENT_CXHA(_name, _event)				\
 802	CMN_EVENT_ATTR(CMN_ANY, cxha_##_name, CMN_TYPE_CXHA, _event)
 803#define CMN_EVENT_CCRA(_name, _event)				\
 804	CMN_EVENT_ATTR(CMN_ANY, ccra_##_name, CMN_TYPE_CCRA, _event)
 805#define CMN_EVENT_CCHA(_model, _name, _event)				\
 806	CMN_EVENT_ATTR(_model, ccha_##_name, CMN_TYPE_CCHA, _event)
 807#define CMN_EVENT_CCLA(_name, _event)				\
 808	CMN_EVENT_ATTR(CMN_ANY, ccla_##_name, CMN_TYPE_CCLA, _event)
 809#define CMN_EVENT_HNS(_name, _event)				\
 810	CMN_EVENT_ATTR(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event)
 811
 812#define CMN_EVENT_DVM(_model, _name, _event)			\
 813	_CMN_EVENT_DVM(_model, _name, _event, 0, SEL_NONE)
 814#define CMN_EVENT_DVM_OCC(_model, _name, _event)			\
 815	_CMN_EVENT_DVM(_model, _name##_all, _event, 0, SEL_OCCUP1ID),	\
 816	_CMN_EVENT_DVM(_model, _name##_dvmop, _event, 1, SEL_OCCUP1ID),	\
 817	_CMN_EVENT_DVM(_model, _name##_dvmsync, _event, 2, SEL_OCCUP1ID)
 818
 819#define CMN_EVENT_HN_OCC(_model, _name, _type, _event)		\
 820	_CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_OCCUP1ID), \
 821	_CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 1, SEL_OCCUP1ID), \
 822	_CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 2, SEL_OCCUP1ID), \
 823	_CMN_EVENT_ATTR(_model, _name##_atomic, _type, _event, 3, SEL_OCCUP1ID), \
 824	_CMN_EVENT_ATTR(_model, _name##_stash, _type, _event, 4, SEL_OCCUP1ID)
 825#define CMN_EVENT_HN_CLS(_model, _name, _type, _event)			\
 826	_CMN_EVENT_ATTR(_model, _name##_class0, _type, _event, 0, SEL_CLASS_OCCUP_ID), \
 827	_CMN_EVENT_ATTR(_model, _name##_class1, _type, _event, 1, SEL_CLASS_OCCUP_ID), \
 828	_CMN_EVENT_ATTR(_model, _name##_class2, _type, _event, 2, SEL_CLASS_OCCUP_ID), \
 829	_CMN_EVENT_ATTR(_model, _name##_class3, _type, _event, 3, SEL_CLASS_OCCUP_ID)
 830#define CMN_EVENT_HN_SNT(_model, _name, _type, _event)			\
 831	_CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_CBUSY_SNTHROTTLE_SEL), \
 832	_CMN_EVENT_ATTR(_model, _name##_group0_read, _type, _event, 1, SEL_CBUSY_SNTHROTTLE_SEL), \
 833	_CMN_EVENT_ATTR(_model, _name##_group0_write, _type, _event, 2, SEL_CBUSY_SNTHROTTLE_SEL), \
 834	_CMN_EVENT_ATTR(_model, _name##_group1_read, _type, _event, 3, SEL_CBUSY_SNTHROTTLE_SEL), \
 835	_CMN_EVENT_ATTR(_model, _name##_group1_write, _type, _event, 4, SEL_CBUSY_SNTHROTTLE_SEL), \
 836	_CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 5, SEL_CBUSY_SNTHROTTLE_SEL), \
 837	_CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 6, SEL_CBUSY_SNTHROTTLE_SEL)
 838
 839#define CMN_EVENT_HNF_OCC(_model, _name, _event)			\
 840	CMN_EVENT_HN_OCC(_model, hnf_##_name, CMN_TYPE_HNF, _event)
 841#define CMN_EVENT_HNF_CLS(_model, _name, _event)			\
 842	CMN_EVENT_HN_CLS(_model, hnf_##_name, CMN_TYPE_HNF, _event)
 843#define CMN_EVENT_HNF_SNT(_model, _name, _event)			\
 844	CMN_EVENT_HN_SNT(_model, hnf_##_name, CMN_TYPE_HNF, _event)
 845
 846#define CMN_EVENT_HNS_OCC(_name, _event)				\
 847	CMN_EVENT_HN_OCC(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event),	\
 848	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_rxsnp, CMN_TYPE_HNS, _event, 5, SEL_OCCUP1ID), \
 849	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 6, SEL_OCCUP1ID), \
 850	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 7, SEL_OCCUP1ID)
 851#define CMN_EVENT_HNS_CLS( _name, _event)				\
 852	CMN_EVENT_HN_CLS(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event)
 853#define CMN_EVENT_HNS_SNT(_name, _event)				\
 854	CMN_EVENT_HN_SNT(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event)
 855#define CMN_EVENT_HNS_HBT(_name, _event)				\
 856	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, 0, SEL_HBT_LBT_SEL), \
 857	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 1, SEL_HBT_LBT_SEL), \
 858	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 2, SEL_HBT_LBT_SEL)
 859#define CMN_EVENT_HNS_SNH(_name, _event)				\
 860	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, 0, SEL_SN_HOME_SEL), \
 861	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_sn, CMN_TYPE_HNS, _event, 1, SEL_SN_HOME_SEL), \
 862	_CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_home, CMN_TYPE_HNS, _event, 2, SEL_SN_HOME_SEL)
 863
 864#define _CMN_EVENT_XP_MESH(_name, _event)			\
 865	__CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)),		\
 866	__CMN_EVENT_XP(w_##_name, (_event) | (1 << 2)),		\
 867	__CMN_EVENT_XP(n_##_name, (_event) | (2 << 2)),		\
 868	__CMN_EVENT_XP(s_##_name, (_event) | (3 << 2))
 869
 870#define _CMN_EVENT_XP_PORT(_name, _event)			\
 871	__CMN_EVENT_XP(p0_##_name, (_event) | (4 << 2)),	\
 872	__CMN_EVENT_XP(p1_##_name, (_event) | (5 << 2)),	\
 873	__CMN_EVENT_XP(p2_##_name, (_event) | (6 << 2)),	\
 874	__CMN_EVENT_XP(p3_##_name, (_event) | (7 << 2))
 875
 876#define _CMN_EVENT_XP(_name, _event)				\
 877	_CMN_EVENT_XP_MESH(_name, _event),			\
 878	_CMN_EVENT_XP_PORT(_name, _event)
 879
 880/* Good thing there are only 3 fundamental XP events... */
 881#define CMN_EVENT_XP(_name, _event)				\
 882	_CMN_EVENT_XP(req_##_name, (_event) | (0 << 5)),	\
 883	_CMN_EVENT_XP(rsp_##_name, (_event) | (1 << 5)),	\
 884	_CMN_EVENT_XP(snp_##_name, (_event) | (2 << 5)),	\
 885	_CMN_EVENT_XP(dat_##_name, (_event) | (3 << 5)),	\
 886	_CMN_EVENT_XP(pub_##_name, (_event) | (4 << 5)),	\
 887	_CMN_EVENT_XP(rsp2_##_name, (_event) | (5 << 5)),	\
 888	_CMN_EVENT_XP(dat2_##_name, (_event) | (6 << 5)),	\
 889	_CMN_EVENT_XP(req2_##_name, (_event) | (7 << 5)),	\
 890	_CMN_EVENT_XP(snp2_##_name, (_event) | (8 << 5))
 891
 892#define CMN_EVENT_XP_DAT(_name, _event)				\
 893	_CMN_EVENT_XP_PORT(dat_##_name, (_event) | (3 << 5)),	\
 894	_CMN_EVENT_XP_PORT(dat2_##_name, (_event) | (6 << 5))
 895
 896
 897static struct attribute *arm_cmn_event_attrs[] = {
 898	CMN_EVENT_DTC(cycles),
 899
 900	/*
 901	 * DVM node events conflict with HN-I events in the equivalent PMU
 902	 * slot, but our lazy short-cut of using the DTM counter index for
 903	 * the PMU index as well happens to avoid that by construction.
 904	 */
 905	CMN_EVENT_DVM(CMN600, rxreq_dvmop,		0x01),
 906	CMN_EVENT_DVM(CMN600, rxreq_dvmsync,		0x02),
 907	CMN_EVENT_DVM(CMN600, rxreq_dvmop_vmid_filtered, 0x03),
 908	CMN_EVENT_DVM(CMN600, rxreq_retried,		0x04),
 909	CMN_EVENT_DVM_OCC(CMN600, rxreq_trk_occupancy,	0x05),
 910	CMN_EVENT_DVM(NOT_CMN600, dvmop_tlbi,		0x01),
 911	CMN_EVENT_DVM(NOT_CMN600, dvmop_bpi,		0x02),
 912	CMN_EVENT_DVM(NOT_CMN600, dvmop_pici,		0x03),
 913	CMN_EVENT_DVM(NOT_CMN600, dvmop_vici,		0x04),
 914	CMN_EVENT_DVM(NOT_CMN600, dvmsync,		0x05),
 915	CMN_EVENT_DVM(NOT_CMN600, vmid_filtered,	0x06),
 916	CMN_EVENT_DVM(NOT_CMN600, rndop_filtered,	0x07),
 917	CMN_EVENT_DVM(NOT_CMN600, retry,		0x08),
 918	CMN_EVENT_DVM(NOT_CMN600, txsnp_flitv,		0x09),
 919	CMN_EVENT_DVM(NOT_CMN600, txsnp_stall,		0x0a),
 920	CMN_EVENT_DVM(NOT_CMN600, trkfull,		0x0b),
 921	CMN_EVENT_DVM_OCC(NOT_CMN600, trk_occupancy,	0x0c),
 922	CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_cxha,	0x0d),
 923	CMN_EVENT_DVM_OCC(CMN700, trk_occupancy_pdn,	0x0e),
 924	CMN_EVENT_DVM(CMN700, trk_alloc,		0x0f),
 925	CMN_EVENT_DVM(CMN700, trk_cxha_alloc,		0x10),
 926	CMN_EVENT_DVM(CMN700, trk_pdn_alloc,		0x11),
 927	CMN_EVENT_DVM(CMN700, txsnp_stall_limit,	0x12),
 928	CMN_EVENT_DVM(CMN700, rxsnp_stall_starv,	0x13),
 929	CMN_EVENT_DVM(CMN700, txsnp_sync_stall_op,	0x14),
 930
 931	CMN_EVENT_HNF(CMN_ANY, cache_miss,		0x01),
 932	CMN_EVENT_HNF(CMN_ANY, slc_sf_cache_access,	0x02),
 933	CMN_EVENT_HNF(CMN_ANY, cache_fill,		0x03),
 934	CMN_EVENT_HNF(CMN_ANY, pocq_retry,		0x04),
 935	CMN_EVENT_HNF(CMN_ANY, pocq_reqs_recvd,		0x05),
 936	CMN_EVENT_HNF(CMN_ANY, sf_hit,			0x06),
 937	CMN_EVENT_HNF(CMN_ANY, sf_evictions,		0x07),
 938	CMN_EVENT_HNF(CMN_ANY, dir_snoops_sent,		0x08),
 939	CMN_EVENT_HNF(CMN_ANY, brd_snoops_sent,		0x09),
 940	CMN_EVENT_HNF(CMN_ANY, slc_eviction,		0x0a),
 941	CMN_EVENT_HNF(CMN_ANY, slc_fill_invalid_way,	0x0b),
 942	CMN_EVENT_HNF(CMN_ANY, mc_retries,		0x0c),
 943	CMN_EVENT_HNF(CMN_ANY, mc_reqs,			0x0d),
 944	CMN_EVENT_HNF(CMN_ANY, qos_hh_retry,		0x0e),
 945	CMN_EVENT_HNF_OCC(CMN_ANY, qos_pocq_occupancy,	0x0f),
 946	CMN_EVENT_HNF(CMN_ANY, pocq_addrhaz,		0x10),
 947	CMN_EVENT_HNF(CMN_ANY, pocq_atomic_addrhaz,	0x11),
 948	CMN_EVENT_HNF(CMN_ANY, ld_st_swp_adq_full,	0x12),
 949	CMN_EVENT_HNF(CMN_ANY, cmp_adq_full,		0x13),
 950	CMN_EVENT_HNF(CMN_ANY, txdat_stall,		0x14),
 951	CMN_EVENT_HNF(CMN_ANY, txrsp_stall,		0x15),
 952	CMN_EVENT_HNF(CMN_ANY, seq_full,		0x16),
 953	CMN_EVENT_HNF(CMN_ANY, seq_hit,			0x17),
 954	CMN_EVENT_HNF(CMN_ANY, snp_sent,		0x18),
 955	CMN_EVENT_HNF(CMN_ANY, sfbi_dir_snp_sent,	0x19),
 956	CMN_EVENT_HNF(CMN_ANY, sfbi_brd_snp_sent,	0x1a),
 957	CMN_EVENT_HNF(CMN_ANY, snp_sent_untrk,		0x1b),
 958	CMN_EVENT_HNF(CMN_ANY, intv_dirty,		0x1c),
 959	CMN_EVENT_HNF(CMN_ANY, stash_snp_sent,		0x1d),
 960	CMN_EVENT_HNF(CMN_ANY, stash_data_pull,		0x1e),
 961	CMN_EVENT_HNF(CMN_ANY, snp_fwded,		0x1f),
 962	CMN_EVENT_HNF(NOT_CMN600, atomic_fwd,		0x20),
 963	CMN_EVENT_HNF(NOT_CMN600, mpam_hardlim,		0x21),
 964	CMN_EVENT_HNF(NOT_CMN600, mpam_softlim,		0x22),
 965	CMN_EVENT_HNF(CMN_650ON, snp_sent_cluster,	0x23),
 966	CMN_EVENT_HNF(CMN_650ON, sf_imprecise_evict,	0x24),
 967	CMN_EVENT_HNF(CMN_650ON, sf_evict_shared_line,	0x25),
 968	CMN_EVENT_HNF_CLS(CMN700, pocq_class_occup,	0x26),
 969	CMN_EVENT_HNF_CLS(CMN700, pocq_class_retry,	0x27),
 970	CMN_EVENT_HNF_CLS(CMN700, class_mc_reqs,	0x28),
 971	CMN_EVENT_HNF_CLS(CMN700, class_cgnt_cmin,	0x29),
 972	CMN_EVENT_HNF_SNT(CMN700, sn_throttle,		0x2a),
 973	CMN_EVENT_HNF_SNT(CMN700, sn_throttle_min,	0x2b),
 974	CMN_EVENT_HNF(CMN700, sf_precise_to_imprecise,	0x2c),
 975	CMN_EVENT_HNF(CMN700, snp_intv_cln,		0x2d),
 976	CMN_EVENT_HNF(CMN700, nc_excl,			0x2e),
 977	CMN_EVENT_HNF(CMN700, excl_mon_ovfl,		0x2f),
 978
 979	CMN_EVENT_HNI(rrt_rd_occ_cnt_ovfl,		0x20),
 980	CMN_EVENT_HNI(rrt_wr_occ_cnt_ovfl,		0x21),
 981	CMN_EVENT_HNI(rdt_rd_occ_cnt_ovfl,		0x22),
 982	CMN_EVENT_HNI(rdt_wr_occ_cnt_ovfl,		0x23),
 983	CMN_EVENT_HNI(wdb_occ_cnt_ovfl,			0x24),
 984	CMN_EVENT_HNI(rrt_rd_alloc,			0x25),
 985	CMN_EVENT_HNI(rrt_wr_alloc,			0x26),
 986	CMN_EVENT_HNI(rdt_rd_alloc,			0x27),
 987	CMN_EVENT_HNI(rdt_wr_alloc,			0x28),
 988	CMN_EVENT_HNI(wdb_alloc,			0x29),
 989	CMN_EVENT_HNI(txrsp_retryack,			0x2a),
 990	CMN_EVENT_HNI(arvalid_no_arready,		0x2b),
 991	CMN_EVENT_HNI(arready_no_arvalid,		0x2c),
 992	CMN_EVENT_HNI(awvalid_no_awready,		0x2d),
 993	CMN_EVENT_HNI(awready_no_awvalid,		0x2e),
 994	CMN_EVENT_HNI(wvalid_no_wready,			0x2f),
 995	CMN_EVENT_HNI(txdat_stall,			0x30),
 996	CMN_EVENT_HNI(nonpcie_serialization,		0x31),
 997	CMN_EVENT_HNI(pcie_serialization,		0x32),
 998
 999	/*
1000	 * HN-P events squat on top of the HN-I similarly to DVM events, except
1001	 * for being crammed into the same physical node as well. And of course
1002	 * where would the fun be if the same events were in the same order...
1003	 */
1004	CMN_EVENT_HNP(rrt_wr_occ_cnt_ovfl,		0x01),
1005	CMN_EVENT_HNP(rdt_wr_occ_cnt_ovfl,		0x02),
1006	CMN_EVENT_HNP(wdb_occ_cnt_ovfl,			0x03),
1007	CMN_EVENT_HNP(rrt_wr_alloc,			0x04),
1008	CMN_EVENT_HNP(rdt_wr_alloc,			0x05),
1009	CMN_EVENT_HNP(wdb_alloc,			0x06),
1010	CMN_EVENT_HNP(awvalid_no_awready,		0x07),
1011	CMN_EVENT_HNP(awready_no_awvalid,		0x08),
1012	CMN_EVENT_HNP(wvalid_no_wready,			0x09),
1013	CMN_EVENT_HNP(rrt_rd_occ_cnt_ovfl,		0x11),
1014	CMN_EVENT_HNP(rdt_rd_occ_cnt_ovfl,		0x12),
1015	CMN_EVENT_HNP(rrt_rd_alloc,			0x13),
1016	CMN_EVENT_HNP(rdt_rd_alloc,			0x14),
1017	CMN_EVENT_HNP(arvalid_no_arready,		0x15),
1018	CMN_EVENT_HNP(arready_no_arvalid,		0x16),
1019
1020	CMN_EVENT_XP(txflit_valid,			0x01),
1021	CMN_EVENT_XP(txflit_stall,			0x02),
1022	CMN_EVENT_XP_DAT(partial_dat_flit,		0x03),
1023	/* We treat watchpoints as a special made-up class of XP events */
1024	CMN_EVENT_ATTR(CMN_ANY, watchpoint_up, CMN_TYPE_WP, CMN_WP_UP),
1025	CMN_EVENT_ATTR(CMN_ANY, watchpoint_down, CMN_TYPE_WP, CMN_WP_DOWN),
1026
1027	CMN_EVENT_SBSX(CMN_ANY, rd_req,			0x01),
1028	CMN_EVENT_SBSX(CMN_ANY, wr_req,			0x02),
1029	CMN_EVENT_SBSX(CMN_ANY, cmo_req,		0x03),
1030	CMN_EVENT_SBSX(CMN_ANY, txrsp_retryack,		0x04),
1031	CMN_EVENT_SBSX(CMN_ANY, txdat_flitv,		0x05),
1032	CMN_EVENT_SBSX(CMN_ANY, txrsp_flitv,		0x06),
1033	CMN_EVENT_SBSX(CMN_ANY, rd_req_trkr_occ_cnt_ovfl, 0x11),
1034	CMN_EVENT_SBSX(CMN_ANY, wr_req_trkr_occ_cnt_ovfl, 0x12),
1035	CMN_EVENT_SBSX(CMN_ANY, cmo_req_trkr_occ_cnt_ovfl, 0x13),
1036	CMN_EVENT_SBSX(CMN_ANY, wdb_occ_cnt_ovfl,	0x14),
1037	CMN_EVENT_SBSX(CMN_ANY, rd_axi_trkr_occ_cnt_ovfl, 0x15),
1038	CMN_EVENT_SBSX(CMN_ANY, cmo_axi_trkr_occ_cnt_ovfl, 0x16),
1039	CMN_EVENT_SBSX(NOT_CMN600, rdb_occ_cnt_ovfl,	0x17),
1040	CMN_EVENT_SBSX(CMN_ANY, arvalid_no_arready,	0x21),
1041	CMN_EVENT_SBSX(CMN_ANY, awvalid_no_awready,	0x22),
1042	CMN_EVENT_SBSX(CMN_ANY, wvalid_no_wready,	0x23),
1043	CMN_EVENT_SBSX(CMN_ANY, txdat_stall,		0x24),
1044	CMN_EVENT_SBSX(CMN_ANY, txrsp_stall,		0x25),
1045
1046	CMN_EVENT_RNID(CMN_ANY, s0_rdata_beats,		0x01),
1047	CMN_EVENT_RNID(CMN_ANY, s1_rdata_beats,		0x02),
1048	CMN_EVENT_RNID(CMN_ANY, s2_rdata_beats,		0x03),
1049	CMN_EVENT_RNID(CMN_ANY, rxdat_flits,		0x04),
1050	CMN_EVENT_RNID(CMN_ANY, txdat_flits,		0x05),
1051	CMN_EVENT_RNID(CMN_ANY, txreq_flits_total,	0x06),
1052	CMN_EVENT_RNID(CMN_ANY, txreq_flits_retried,	0x07),
1053	CMN_EVENT_RNID(CMN_ANY, rrt_occ_ovfl,		0x08),
1054	CMN_EVENT_RNID(CMN_ANY, wrt_occ_ovfl,		0x09),
1055	CMN_EVENT_RNID(CMN_ANY, txreq_flits_replayed,	0x0a),
1056	CMN_EVENT_RNID(CMN_ANY, wrcancel_sent,		0x0b),
1057	CMN_EVENT_RNID(CMN_ANY, s0_wdata_beats,		0x0c),
1058	CMN_EVENT_RNID(CMN_ANY, s1_wdata_beats,		0x0d),
1059	CMN_EVENT_RNID(CMN_ANY, s2_wdata_beats,		0x0e),
1060	CMN_EVENT_RNID(CMN_ANY, rrt_alloc,		0x0f),
1061	CMN_EVENT_RNID(CMN_ANY, wrt_alloc,		0x10),
1062	CMN_EVENT_RNID(CMN600, rdb_unord,		0x11),
1063	CMN_EVENT_RNID(CMN600, rdb_replay,		0x12),
1064	CMN_EVENT_RNID(CMN600, rdb_hybrid,		0x13),
1065	CMN_EVENT_RNID(CMN600, rdb_ord,			0x14),
1066	CMN_EVENT_RNID(NOT_CMN600, padb_occ_ovfl,	0x11),
1067	CMN_EVENT_RNID(NOT_CMN600, rpdb_occ_ovfl,	0x12),
1068	CMN_EVENT_RNID(NOT_CMN600, rrt_occup_ovfl_slice1, 0x13),
1069	CMN_EVENT_RNID(NOT_CMN600, rrt_occup_ovfl_slice2, 0x14),
1070	CMN_EVENT_RNID(NOT_CMN600, rrt_occup_ovfl_slice3, 0x15),
1071	CMN_EVENT_RNID(NOT_CMN600, wrt_throttled,	0x16),
1072	CMN_EVENT_RNID(CMN700, ldb_full,		0x17),
1073	CMN_EVENT_RNID(CMN700, rrt_rd_req_occup_ovfl_slice0, 0x18),
1074	CMN_EVENT_RNID(CMN700, rrt_rd_req_occup_ovfl_slice1, 0x19),
1075	CMN_EVENT_RNID(CMN700, rrt_rd_req_occup_ovfl_slice2, 0x1a),
1076	CMN_EVENT_RNID(CMN700, rrt_rd_req_occup_ovfl_slice3, 0x1b),
1077	CMN_EVENT_RNID(CMN700, rrt_burst_occup_ovfl_slice0, 0x1c),
1078	CMN_EVENT_RNID(CMN700, rrt_burst_occup_ovfl_slice1, 0x1d),
1079	CMN_EVENT_RNID(CMN700, rrt_burst_occup_ovfl_slice2, 0x1e),
1080	CMN_EVENT_RNID(CMN700, rrt_burst_occup_ovfl_slice3, 0x1f),
1081	CMN_EVENT_RNID(CMN700, rrt_burst_alloc,		0x20),
1082	CMN_EVENT_RNID(CMN700, awid_hash,		0x21),
1083	CMN_EVENT_RNID(CMN700, atomic_alloc,		0x22),
1084	CMN_EVENT_RNID(CMN700, atomic_occ_ovfl,		0x23),
1085
1086	CMN_EVENT_MTSX(tc_lookup,			0x01),
1087	CMN_EVENT_MTSX(tc_fill,				0x02),
1088	CMN_EVENT_MTSX(tc_miss,				0x03),
1089	CMN_EVENT_MTSX(tdb_forward,			0x04),
1090	CMN_EVENT_MTSX(tcq_hazard,			0x05),
1091	CMN_EVENT_MTSX(tcq_rd_alloc,			0x06),
1092	CMN_EVENT_MTSX(tcq_wr_alloc,			0x07),
1093	CMN_EVENT_MTSX(tcq_cmo_alloc,			0x08),
1094	CMN_EVENT_MTSX(axi_rd_req,			0x09),
1095	CMN_EVENT_MTSX(axi_wr_req,			0x0a),
1096	CMN_EVENT_MTSX(tcq_occ_cnt_ovfl,		0x0b),
1097	CMN_EVENT_MTSX(tdb_occ_cnt_ovfl,		0x0c),
1098
1099	CMN_EVENT_CXRA(CMN_ANY, rht_occ,		0x01),
1100	CMN_EVENT_CXRA(CMN_ANY, sht_occ,		0x02),
1101	CMN_EVENT_CXRA(CMN_ANY, rdb_occ,		0x03),
1102	CMN_EVENT_CXRA(CMN_ANY, wdb_occ,		0x04),
1103	CMN_EVENT_CXRA(CMN_ANY, ssb_occ,		0x05),
1104	CMN_EVENT_CXRA(CMN_ANY, snp_bcasts,		0x06),
1105	CMN_EVENT_CXRA(CMN_ANY, req_chains,		0x07),
1106	CMN_EVENT_CXRA(CMN_ANY, req_chain_avglen,	0x08),
1107	CMN_EVENT_CXRA(CMN_ANY, chirsp_stalls,		0x09),
1108	CMN_EVENT_CXRA(CMN_ANY, chidat_stalls,		0x0a),
1109	CMN_EVENT_CXRA(CMN_ANY, cxreq_pcrd_stalls_link0, 0x0b),
1110	CMN_EVENT_CXRA(CMN_ANY, cxreq_pcrd_stalls_link1, 0x0c),
1111	CMN_EVENT_CXRA(CMN_ANY, cxreq_pcrd_stalls_link2, 0x0d),
1112	CMN_EVENT_CXRA(CMN_ANY, cxdat_pcrd_stalls_link0, 0x0e),
1113	CMN_EVENT_CXRA(CMN_ANY, cxdat_pcrd_stalls_link1, 0x0f),
1114	CMN_EVENT_CXRA(CMN_ANY, cxdat_pcrd_stalls_link2, 0x10),
1115	CMN_EVENT_CXRA(CMN_ANY, external_chirsp_stalls,	0x11),
1116	CMN_EVENT_CXRA(CMN_ANY, external_chidat_stalls,	0x12),
1117	CMN_EVENT_CXRA(NOT_CMN600, cxmisc_pcrd_stalls_link0, 0x13),
1118	CMN_EVENT_CXRA(NOT_CMN600, cxmisc_pcrd_stalls_link1, 0x14),
1119	CMN_EVENT_CXRA(NOT_CMN600, cxmisc_pcrd_stalls_link2, 0x15),
1120
1121	CMN_EVENT_CXHA(rddatbyp,			0x21),
1122	CMN_EVENT_CXHA(chirsp_up_stall,			0x22),
1123	CMN_EVENT_CXHA(chidat_up_stall,			0x23),
1124	CMN_EVENT_CXHA(snppcrd_link0_stall,		0x24),
1125	CMN_EVENT_CXHA(snppcrd_link1_stall,		0x25),
1126	CMN_EVENT_CXHA(snppcrd_link2_stall,		0x26),
1127	CMN_EVENT_CXHA(reqtrk_occ,			0x27),
1128	CMN_EVENT_CXHA(rdb_occ,				0x28),
1129	CMN_EVENT_CXHA(rdbyp_occ,			0x29),
1130	CMN_EVENT_CXHA(wdb_occ,				0x2a),
1131	CMN_EVENT_CXHA(snptrk_occ,			0x2b),
1132	CMN_EVENT_CXHA(sdb_occ,				0x2c),
1133	CMN_EVENT_CXHA(snphaz_occ,			0x2d),
1134
1135	CMN_EVENT_CCRA(rht_occ,				0x41),
1136	CMN_EVENT_CCRA(sht_occ,				0x42),
1137	CMN_EVENT_CCRA(rdb_occ,				0x43),
1138	CMN_EVENT_CCRA(wdb_occ,				0x44),
1139	CMN_EVENT_CCRA(ssb_occ,				0x45),
1140	CMN_EVENT_CCRA(snp_bcasts,			0x46),
1141	CMN_EVENT_CCRA(req_chains,			0x47),
1142	CMN_EVENT_CCRA(req_chain_avglen,		0x48),
1143	CMN_EVENT_CCRA(chirsp_stalls,			0x49),
1144	CMN_EVENT_CCRA(chidat_stalls,			0x4a),
1145	CMN_EVENT_CCRA(cxreq_pcrd_stalls_link0,		0x4b),
1146	CMN_EVENT_CCRA(cxreq_pcrd_stalls_link1,		0x4c),
1147	CMN_EVENT_CCRA(cxreq_pcrd_stalls_link2,		0x4d),
1148	CMN_EVENT_CCRA(cxdat_pcrd_stalls_link0,		0x4e),
1149	CMN_EVENT_CCRA(cxdat_pcrd_stalls_link1,		0x4f),
1150	CMN_EVENT_CCRA(cxdat_pcrd_stalls_link2,		0x50),
1151	CMN_EVENT_CCRA(external_chirsp_stalls,		0x51),
1152	CMN_EVENT_CCRA(external_chidat_stalls,		0x52),
1153	CMN_EVENT_CCRA(cxmisc_pcrd_stalls_link0,	0x53),
1154	CMN_EVENT_CCRA(cxmisc_pcrd_stalls_link1,	0x54),
1155	CMN_EVENT_CCRA(cxmisc_pcrd_stalls_link2,	0x55),
1156	CMN_EVENT_CCRA(rht_alloc,			0x56),
1157	CMN_EVENT_CCRA(sht_alloc,			0x57),
1158	CMN_EVENT_CCRA(rdb_alloc,			0x58),
1159	CMN_EVENT_CCRA(wdb_alloc,			0x59),
1160	CMN_EVENT_CCRA(ssb_alloc,			0x5a),
1161
1162	CMN_EVENT_CCHA(CMN_ANY, rddatbyp,		0x61),
1163	CMN_EVENT_CCHA(CMN_ANY, chirsp_up_stall,	0x62),
1164	CMN_EVENT_CCHA(CMN_ANY, chidat_up_stall,	0x63),
1165	CMN_EVENT_CCHA(CMN_ANY, snppcrd_link0_stall,	0x64),
1166	CMN_EVENT_CCHA(CMN_ANY, snppcrd_link1_stall,	0x65),
1167	CMN_EVENT_CCHA(CMN_ANY, snppcrd_link2_stall,	0x66),
1168	CMN_EVENT_CCHA(CMN_ANY, reqtrk_occ,		0x67),
1169	CMN_EVENT_CCHA(CMN_ANY, rdb_occ,		0x68),
1170	CMN_EVENT_CCHA(CMN_ANY, rdbyp_occ,		0x69),
1171	CMN_EVENT_CCHA(CMN_ANY, wdb_occ,		0x6a),
1172	CMN_EVENT_CCHA(CMN_ANY, snptrk_occ,		0x6b),
1173	CMN_EVENT_CCHA(CMN_ANY, sdb_occ,		0x6c),
1174	CMN_EVENT_CCHA(CMN_ANY, snphaz_occ,		0x6d),
1175	CMN_EVENT_CCHA(CMN_ANY, reqtrk_alloc,		0x6e),
1176	CMN_EVENT_CCHA(CMN_ANY, rdb_alloc,		0x6f),
1177	CMN_EVENT_CCHA(CMN_ANY, rdbyp_alloc,		0x70),
1178	CMN_EVENT_CCHA(CMN_ANY, wdb_alloc,		0x71),
1179	CMN_EVENT_CCHA(CMN_ANY, snptrk_alloc,		0x72),
1180	CMN_EVENT_CCHA(CMN_ANY, db_alloc,		0x73),
1181	CMN_EVENT_CCHA(CMN_ANY, snphaz_alloc,		0x74),
1182	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_req_occ,		0x75),
1183	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_req_alloc,	0x76),
1184	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_pcie_req_occ,	0x77),
1185	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_pcie_req_alloc,	0x78),
1186	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_wr_req_occ,	0x79),
1187	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_wr_req_alloc,	0x7a),
1188	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_reg_req_occ,	0x7b),
1189	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_reg_req_alloc,	0x7c),
1190	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_rsvd_req_occ,	0x7d),
1191	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_rsvd_req_alloc,	0x7e),
1192	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_dat_occ,		0x7f),
1193	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_dat_alloc,	0x80),
1194	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_pcie_dat_occ,	0x81),
1195	CMN_EVENT_CCHA(CMN_ANY, pb_rhu_pcie_dat_alloc,	0x82),
1196	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_wr_dat_occ,	0x83),
1197	CMN_EVENT_CCHA(CMN_ANY, pb_pcie_wr_dat_alloc,	0x84),
1198	CMN_EVENT_CCHA(CMNS3, chirsp1_up_stall,		0x85),
1199
1200	CMN_EVENT_CCLA(rx_cxs,				0x21),
1201	CMN_EVENT_CCLA(tx_cxs,				0x22),
1202	CMN_EVENT_CCLA(rx_cxs_avg_size,			0x23),
1203	CMN_EVENT_CCLA(tx_cxs_avg_size,			0x24),
1204	CMN_EVENT_CCLA(tx_cxs_lcrd_backpressure,	0x25),
1205	CMN_EVENT_CCLA(link_crdbuf_occ,			0x26),
1206	CMN_EVENT_CCLA(link_crdbuf_alloc,		0x27),
1207	CMN_EVENT_CCLA(pfwd_rcvr_cxs,			0x28),
1208	CMN_EVENT_CCLA(pfwd_sndr_num_flits,		0x29),
1209	CMN_EVENT_CCLA(pfwd_sndr_stalls_static_crd,	0x2a),
1210	CMN_EVENT_CCLA(pfwd_sndr_stalls_dynmaic_crd,	0x2b),
1211
1212	CMN_EVENT_HNS_HBT(cache_miss,			0x01),
1213	CMN_EVENT_HNS_HBT(slc_sf_cache_access,		0x02),
1214	CMN_EVENT_HNS_HBT(cache_fill,			0x03),
1215	CMN_EVENT_HNS_HBT(pocq_retry,			0x04),
1216	CMN_EVENT_HNS_HBT(pocq_reqs_recvd,		0x05),
1217	CMN_EVENT_HNS_HBT(sf_hit,			0x06),
1218	CMN_EVENT_HNS_HBT(sf_evictions,			0x07),
1219	CMN_EVENT_HNS(dir_snoops_sent,			0x08),
1220	CMN_EVENT_HNS(brd_snoops_sent,			0x09),
1221	CMN_EVENT_HNS_HBT(slc_eviction,			0x0a),
1222	CMN_EVENT_HNS_HBT(slc_fill_invalid_way,		0x0b),
1223	CMN_EVENT_HNS(mc_retries_local,			0x0c),
1224	CMN_EVENT_HNS_SNH(mc_reqs_local,		0x0d),
1225	CMN_EVENT_HNS(qos_hh_retry,			0x0e),
1226	CMN_EVENT_HNS_OCC(qos_pocq_occupancy,		0x0f),
1227	CMN_EVENT_HNS(pocq_addrhaz,			0x10),
1228	CMN_EVENT_HNS(pocq_atomic_addrhaz,		0x11),
1229	CMN_EVENT_HNS(ld_st_swp_adq_full,		0x12),
1230	CMN_EVENT_HNS(cmp_adq_full,			0x13),
1231	CMN_EVENT_HNS(txdat_stall,			0x14),
1232	CMN_EVENT_HNS(txrsp_stall,			0x15),
1233	CMN_EVENT_HNS(seq_full,				0x16),
1234	CMN_EVENT_HNS(seq_hit,				0x17),
1235	CMN_EVENT_HNS(snp_sent,				0x18),
1236	CMN_EVENT_HNS(sfbi_dir_snp_sent,		0x19),
1237	CMN_EVENT_HNS(sfbi_brd_snp_sent,		0x1a),
1238	CMN_EVENT_HNS(intv_dirty,			0x1c),
1239	CMN_EVENT_HNS(stash_snp_sent,			0x1d),
1240	CMN_EVENT_HNS(stash_data_pull,			0x1e),
1241	CMN_EVENT_HNS(snp_fwded,			0x1f),
1242	CMN_EVENT_HNS(atomic_fwd,			0x20),
1243	CMN_EVENT_HNS(mpam_hardlim,			0x21),
1244	CMN_EVENT_HNS(mpam_softlim,			0x22),
1245	CMN_EVENT_HNS(snp_sent_cluster,			0x23),
1246	CMN_EVENT_HNS(sf_imprecise_evict,		0x24),
1247	CMN_EVENT_HNS(sf_evict_shared_line,		0x25),
1248	CMN_EVENT_HNS_CLS(pocq_class_occup,		0x26),
1249	CMN_EVENT_HNS_CLS(pocq_class_retry,		0x27),
1250	CMN_EVENT_HNS_CLS(class_mc_reqs_local,		0x28),
1251	CMN_EVENT_HNS_CLS(class_cgnt_cmin,		0x29),
1252	CMN_EVENT_HNS_SNT(sn_throttle,			0x2a),
1253	CMN_EVENT_HNS_SNT(sn_throttle_min,		0x2b),
1254	CMN_EVENT_HNS(sf_precise_to_imprecise,		0x2c),
1255	CMN_EVENT_HNS(snp_intv_cln,			0x2d),
1256	CMN_EVENT_HNS(nc_excl,				0x2e),
1257	CMN_EVENT_HNS(excl_mon_ovfl,			0x2f),
1258	CMN_EVENT_HNS(snp_req_recvd,			0x30),
1259	CMN_EVENT_HNS(snp_req_byp_pocq,			0x31),
1260	CMN_EVENT_HNS(dir_ccgha_snp_sent,		0x32),
1261	CMN_EVENT_HNS(brd_ccgha_snp_sent,		0x33),
1262	CMN_EVENT_HNS(ccgha_snp_stall,			0x34),
1263	CMN_EVENT_HNS(lbt_req_hardlim,			0x35),
1264	CMN_EVENT_HNS(hbt_req_hardlim,			0x36),
1265	CMN_EVENT_HNS(sf_reupdate,			0x37),
1266	CMN_EVENT_HNS(excl_sf_imprecise,		0x38),
1267	CMN_EVENT_HNS(snp_pocq_addrhaz,			0x39),
1268	CMN_EVENT_HNS(mc_retries_remote,		0x3a),
1269	CMN_EVENT_HNS_SNH(mc_reqs_remote,		0x3b),
1270	CMN_EVENT_HNS_CLS(class_mc_reqs_remote,		0x3c),
1271
1272	NULL
1273};
1274
1275static const struct attribute_group arm_cmn_event_attrs_group = {
1276	.name = "events",
1277	.attrs = arm_cmn_event_attrs,
1278	.is_visible = arm_cmn_event_attr_is_visible,
1279};
1280
1281static ssize_t arm_cmn_format_show(struct device *dev,
1282				   struct device_attribute *attr, char *buf)
1283{
1284	struct arm_cmn_format_attr *fmt = container_of(attr, typeof(*fmt), attr);
1285
1286	if (!fmt->config)
1287		return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field);
1288
1289	return sysfs_emit(buf, "config%d:%*pbl\n", fmt->config, 64, &fmt->field);
1290}
1291
1292#define _CMN_FORMAT_ATTR(_name, _cfg, _fld)				\
1293	(&((struct arm_cmn_format_attr[]) {{				\
1294		.attr = __ATTR(_name, 0444, arm_cmn_format_show, NULL),	\
1295		.config = _cfg,						\
1296		.field = _fld,						\
1297	}})[0].attr.attr)
1298#define CMN_FORMAT_ATTR(_name, _fld)	_CMN_FORMAT_ATTR(_name, 0, _fld)
1299
1300static struct attribute *arm_cmn_format_attrs[] = {
1301	CMN_FORMAT_ATTR(type, CMN_CONFIG_TYPE),
1302	CMN_FORMAT_ATTR(eventid, CMN_CONFIG_EVENTID),
1303	CMN_FORMAT_ATTR(occupid, CMN_CONFIG_OCCUPID),
1304	CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID),
1305	CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID),
1306
1307	CMN_FORMAT_ATTR(wp_dev_sel, CMN_CONFIG_WP_DEV_SEL),
1308	CMN_FORMAT_ATTR(wp_chn_sel, CMN_CONFIG_WP_CHN_SEL),
1309	CMN_FORMAT_ATTR(wp_grp, CMN_CONFIG_WP_GRP),
1310	CMN_FORMAT_ATTR(wp_exclusive, CMN_CONFIG_WP_EXCLUSIVE),
1311	CMN_FORMAT_ATTR(wp_combine, CMN_CONFIG_WP_COMBINE),
1312
1313	_CMN_FORMAT_ATTR(wp_val, 1, CMN_CONFIG1_WP_VAL),
1314	_CMN_FORMAT_ATTR(wp_mask, 2, CMN_CONFIG2_WP_MASK),
1315
1316	NULL
1317};
1318
1319static const struct attribute_group arm_cmn_format_attrs_group = {
1320	.name = "format",
1321	.attrs = arm_cmn_format_attrs,
1322};
1323
1324static ssize_t arm_cmn_cpumask_show(struct device *dev,
1325				    struct device_attribute *attr, char *buf)
1326{
1327	struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));
1328
1329	return cpumap_print_to_pagebuf(true, buf, cpumask_of(cmn->cpu));
1330}
1331
1332static struct device_attribute arm_cmn_cpumask_attr =
1333		__ATTR(cpumask, 0444, arm_cmn_cpumask_show, NULL);
1334
1335static ssize_t arm_cmn_identifier_show(struct device *dev,
1336				       struct device_attribute *attr, char *buf)
1337{
1338	struct arm_cmn *cmn = to_cmn(dev_get_drvdata(dev));
1339
1340	return sysfs_emit(buf, "%03x%02x\n", cmn->part, cmn->rev);
1341}
1342
1343static struct device_attribute arm_cmn_identifier_attr =
1344		__ATTR(identifier, 0444, arm_cmn_identifier_show, NULL);
1345
1346static struct attribute *arm_cmn_other_attrs[] = {
1347	&arm_cmn_cpumask_attr.attr,
1348	&arm_cmn_identifier_attr.attr,
1349	NULL,
1350};
1351
1352static const struct attribute_group arm_cmn_other_attrs_group = {
1353	.attrs = arm_cmn_other_attrs,
1354};
1355
1356static const struct attribute_group *arm_cmn_attr_groups[] = {
1357	&arm_cmn_event_attrs_group,
1358	&arm_cmn_format_attrs_group,
1359	&arm_cmn_other_attrs_group,
1360	NULL
1361};
1362
1363static int arm_cmn_find_free_wp_idx(struct arm_cmn_dtm *dtm,
1364				    struct perf_event *event)
1365{
1366	int wp_idx = CMN_EVENT_EVENTID(event);
1367
1368	if (dtm->wp_event[wp_idx] >= 0)
1369		if (dtm->wp_event[++wp_idx] >= 0)
1370			return -ENOSPC;
1371
1372	return wp_idx;
1373}
1374
1375static int arm_cmn_get_assigned_wp_idx(struct perf_event *event,
1376				       struct arm_cmn_hw_event *hw,
1377				       unsigned int pos)
1378{
1379	return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw->wp_idx, pos);
1380}
1381
1382static void arm_cmn_claim_wp_idx(struct arm_cmn_dtm *dtm,
1383				 struct perf_event *event,
1384				 unsigned int dtc, int wp_idx,
1385				 unsigned int pos)
1386{
1387	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1388
1389	dtm->wp_event[wp_idx] = hw->dtc_idx[dtc];
1390	arm_cmn_set_wp_idx(hw->wp_idx, pos, wp_idx - CMN_EVENT_EVENTID(event));
1391}
1392
1393static u32 arm_cmn_wp_config(struct perf_event *event, int wp_idx)
1394{
1395	u32 config;
1396	u32 dev = CMN_EVENT_WP_DEV_SEL(event);
1397	u32 chn = CMN_EVENT_WP_CHN_SEL(event);
1398	u32 grp = CMN_EVENT_WP_GRP(event);
1399	u32 exc = CMN_EVENT_WP_EXCLUSIVE(event);
1400	u32 combine = CMN_EVENT_WP_COMBINE(event);
1401	bool is_cmn600 = to_cmn(event->pmu)->part == PART_CMN600;
1402
1403	/* CMN-600 supports only primary and secondary matching groups */
1404	if (is_cmn600)
1405		grp &= 1;
1406
1407	config = FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL, dev) |
1408		 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_CHN_SEL, chn) |
1409		 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_GRP, grp) |
1410		 FIELD_PREP(CMN_DTM_WPn_CONFIG_WP_DEV_SEL2, dev >> 1);
1411	if (exc)
1412		config |= is_cmn600 ? CMN600_WPn_CONFIG_WP_EXCLUSIVE :
1413				      CMN_DTM_WPn_CONFIG_WP_EXCLUSIVE;
1414
1415	/*  wp_combine is available only on WP0 and WP2 */
1416	if (combine && !(wp_idx & 0x1))
1417		config |= is_cmn600 ? CMN600_WPn_CONFIG_WP_COMBINE :
1418				      CMN_DTM_WPn_CONFIG_WP_COMBINE;
1419	return config;
1420}
1421
1422static void arm_cmn_set_state(struct arm_cmn *cmn, u32 state)
1423{
1424	if (!cmn->state)
1425		writel_relaxed(0, CMN_DT_PMCR(&cmn->dtc[0]));
1426	cmn->state |= state;
1427}
1428
1429static void arm_cmn_clear_state(struct arm_cmn *cmn, u32 state)
1430{
1431	cmn->state &= ~state;
1432	if (!cmn->state)
1433		writel_relaxed(CMN_DT_PMCR_PMU_EN | CMN_DT_PMCR_OVFL_INTR_EN,
1434			       CMN_DT_PMCR(&cmn->dtc[0]));
1435}
1436
1437static void arm_cmn_pmu_enable(struct pmu *pmu)
1438{
1439	arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_DISABLED);
1440}
1441
1442static void arm_cmn_pmu_disable(struct pmu *pmu)
1443{
1444	arm_cmn_set_state(to_cmn(pmu), CMN_STATE_DISABLED);
1445}
1446
1447static u64 arm_cmn_read_dtm(struct arm_cmn *cmn, struct arm_cmn_hw_event *hw,
1448			    bool snapshot)
1449{
1450	struct arm_cmn_dtm *dtm = NULL;
1451	struct arm_cmn_node *dn;
1452	unsigned int i, offset, dtm_idx;
1453	u64 reg, count = 0;
1454
1455	offset = snapshot ? CMN_DTM_PMEVCNTSR : CMN_DTM_PMEVCNT;
1456	for_each_hw_dn(hw, dn, i) {
1457		if (dtm != &cmn->dtms[dn->dtm]) {
1458			dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset;
1459			reg = readq_relaxed(dtm->base + offset);
1460		}
1461		dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
1462		count += (u16)(reg >> (dtm_idx * 16));
1463	}
1464	return count;
1465}
1466
1467static u64 arm_cmn_read_cc(struct arm_cmn_dtc *dtc)
1468{
1469	void __iomem *pmccntr = CMN_DT_PMCCNTR(dtc);
1470	u64 val = readq_relaxed(pmccntr);
1471
1472	writeq_relaxed(CMN_CC_INIT, pmccntr);
1473	return (val - CMN_CC_INIT) & ((CMN_CC_INIT << 1) - 1);
1474}
1475
1476static u32 arm_cmn_read_counter(struct arm_cmn_dtc *dtc, int idx)
1477{
1478	void __iomem *pmevcnt = CMN_DT_PMEVCNT(dtc, idx);
1479	u32 val = readl_relaxed(pmevcnt);
1480
1481	writel_relaxed(CMN_COUNTER_INIT, pmevcnt);
1482	return val - CMN_COUNTER_INIT;
1483}
1484
1485static void arm_cmn_init_counter(struct perf_event *event)
1486{
1487	struct arm_cmn *cmn = to_cmn(event->pmu);
1488	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1489	u64 count;
1490
1491	for_each_hw_dtc_idx(hw, i, idx) {
1492		writel_relaxed(CMN_COUNTER_INIT, CMN_DT_PMEVCNT(&cmn->dtc[i], idx));
1493		cmn->dtc[i].counters[idx] = event;
1494	}
1495
1496	count = arm_cmn_read_dtm(cmn, hw, false);
1497	local64_set(&event->hw.prev_count, count);
1498}
1499
1500static void arm_cmn_event_read(struct perf_event *event)
1501{
1502	struct arm_cmn *cmn = to_cmn(event->pmu);
1503	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1504	u64 delta, new, prev;
1505	unsigned long flags;
1506
1507	if (CMN_EVENT_TYPE(event) == CMN_TYPE_DTC) {
1508		delta = arm_cmn_read_cc(cmn->dtc + hw->dtc_idx[0]);
1509		local64_add(delta, &event->count);
1510		return;
1511	}
1512	new = arm_cmn_read_dtm(cmn, hw, false);
1513	prev = local64_xchg(&event->hw.prev_count, new);
1514
1515	delta = new - prev;
1516
1517	local_irq_save(flags);
1518	for_each_hw_dtc_idx(hw, i, idx) {
1519		new = arm_cmn_read_counter(cmn->dtc + i, idx);
1520		delta += new << 16;
1521	}
1522	local_irq_restore(flags);
1523	local64_add(delta, &event->count);
1524}
1525
1526static int arm_cmn_set_event_sel_hi(struct arm_cmn_node *dn,
1527				    enum cmn_filter_select fsel, u8 occupid)
1528{
1529	u64 reg;
1530
1531	if (fsel == SEL_NONE)
1532		return 0;
1533
1534	if (!dn->occupid[fsel].count) {
1535		dn->occupid[fsel].val = occupid;
1536		reg = FIELD_PREP(CMN__PMU_CBUSY_SNTHROTTLE_SEL,
1537				 dn->occupid[SEL_CBUSY_SNTHROTTLE_SEL].val) |
1538		      FIELD_PREP(CMN__PMU_SN_HOME_SEL,
1539				 dn->occupid[SEL_SN_HOME_SEL].val) |
1540		      FIELD_PREP(CMN__PMU_HBT_LBT_SEL,
1541				 dn->occupid[SEL_HBT_LBT_SEL].val) |
1542		      FIELD_PREP(CMN__PMU_CLASS_OCCUP_ID,
1543				 dn->occupid[SEL_CLASS_OCCUP_ID].val) |
1544		      FIELD_PREP(CMN__PMU_OCCUP1_ID,
1545				 dn->occupid[SEL_OCCUP1ID].val);
1546		writel_relaxed(reg >> 32, dn->pmu_base + CMN_PMU_EVENT_SEL + 4);
1547	} else if (dn->occupid[fsel].val != occupid) {
1548		return -EBUSY;
1549	}
1550	dn->occupid[fsel].count++;
1551	return 0;
1552}
1553
1554static void arm_cmn_set_event_sel_lo(struct arm_cmn_node *dn, int dtm_idx,
1555				     int eventid, bool wide_sel)
1556{
1557	if (wide_sel) {
1558		dn->event_w[dtm_idx] = eventid;
1559		writeq_relaxed(le64_to_cpu(dn->event_sel_w), dn->pmu_base + CMN_PMU_EVENT_SEL);
1560	} else {
1561		dn->event[dtm_idx] = eventid;
1562		writel_relaxed(le32_to_cpu(dn->event_sel), dn->pmu_base + CMN_PMU_EVENT_SEL);
1563	}
1564}
1565
1566static void arm_cmn_event_start(struct perf_event *event, int flags)
1567{
1568	struct arm_cmn *cmn = to_cmn(event->pmu);
1569	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1570	struct arm_cmn_node *dn;
1571	enum cmn_node_type type = CMN_EVENT_TYPE(event);
1572	int i;
1573
1574	if (type == CMN_TYPE_DTC) {
1575		struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0];
1576
1577		writel_relaxed(CMN_DT_DTC_CTL_DT_EN | CMN_DT_DTC_CTL_CG_DISABLE,
1578			       dtc->base + CMN_DT_DTC_CTL);
1579		writeq_relaxed(CMN_CC_INIT, CMN_DT_PMCCNTR(dtc));
1580		dtc->cc_active = true;
1581	} else if (type == CMN_TYPE_WP) {
1582		u64 val = CMN_EVENT_WP_VAL(event);
1583		u64 mask = CMN_EVENT_WP_MASK(event);
1584
1585		for_each_hw_dn(hw, dn, i) {
1586			void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset);
1587			int wp_idx = arm_cmn_get_assigned_wp_idx(event, hw, i);
1588
1589			writeq_relaxed(val, base + CMN_DTM_WPn_VAL(wp_idx));
1590			writeq_relaxed(mask, base + CMN_DTM_WPn_MASK(wp_idx));
1591		}
1592	} else for_each_hw_dn(hw, dn, i) {
1593		int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
1594
1595		arm_cmn_set_event_sel_lo(dn, dtm_idx, CMN_EVENT_EVENTID(event),
1596					 hw->wide_sel);
1597	}
1598}
1599
1600static void arm_cmn_event_stop(struct perf_event *event, int flags)
1601{
1602	struct arm_cmn *cmn = to_cmn(event->pmu);
1603	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1604	struct arm_cmn_node *dn;
1605	enum cmn_node_type type = CMN_EVENT_TYPE(event);
1606	int i;
1607
1608	if (type == CMN_TYPE_DTC) {
1609		struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0];
1610
1611		dtc->cc_active = false;
1612		writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL);
1613	} else if (type == CMN_TYPE_WP) {
1614		for_each_hw_dn(hw, dn, i) {
1615			void __iomem *base = dn->pmu_base + CMN_DTM_OFFSET(hw->dtm_offset);
1616			int wp_idx = arm_cmn_get_assigned_wp_idx(event, hw, i);
1617
1618			writeq_relaxed(0, base + CMN_DTM_WPn_MASK(wp_idx));
1619			writeq_relaxed(~0ULL, base + CMN_DTM_WPn_VAL(wp_idx));
1620		}
1621	} else for_each_hw_dn(hw, dn, i) {
1622		int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
1623
1624		arm_cmn_set_event_sel_lo(dn, dtm_idx, 0, hw->wide_sel);
1625	}
1626
1627	arm_cmn_event_read(event);
1628}
1629
1630struct arm_cmn_val {
1631	u8 dtm_count[CMN_MAX_DTMS];
1632	u8 occupid[CMN_MAX_DTMS][SEL_MAX];
1633	u8 wp[CMN_MAX_DTMS][4];
1634	u8 wp_combine[CMN_MAX_DTMS][2];
1635	int dtc_count[CMN_MAX_DTCS];
1636	bool cycles;
1637};
1638
1639static int arm_cmn_val_find_free_wp_config(struct perf_event *event,
1640					  struct arm_cmn_val *val, int dtm)
1641{
1642	int wp_idx = CMN_EVENT_EVENTID(event);
1643
1644	if (val->wp[dtm][wp_idx])
1645		if (val->wp[dtm][++wp_idx])
1646			return -ENOSPC;
1647
1648	return wp_idx;
1649}
1650
1651static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val,
1652				  struct perf_event *event)
1653{
1654	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1655	struct arm_cmn_node *dn;
1656	enum cmn_node_type type;
1657	int i;
1658
1659	if (is_software_event(event))
1660		return;
1661
1662	type = CMN_EVENT_TYPE(event);
1663	if (type == CMN_TYPE_DTC) {
1664		val->cycles = true;
1665		return;
1666	}
1667
1668	for_each_hw_dtc_idx(hw, dtc, idx)
1669		val->dtc_count[dtc]++;
1670
1671	for_each_hw_dn(hw, dn, i) {
1672		int wp_idx, dtm = dn->dtm, sel = hw->filter_sel;
1673
1674		val->dtm_count[dtm]++;
1675
1676		if (sel > SEL_NONE)
1677			val->occupid[dtm][sel] = CMN_EVENT_OCCUPID(event) + 1;
1678
1679		if (type != CMN_TYPE_WP)
1680			continue;
1681
1682		wp_idx = arm_cmn_val_find_free_wp_config(event, val, dtm);
1683		val->wp[dtm][wp_idx] = 1;
1684		val->wp_combine[dtm][wp_idx >> 1] += !!CMN_EVENT_WP_COMBINE(event);
1685	}
1686}
1687
1688static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event)
1689{
1690	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1691	struct arm_cmn_node *dn;
1692	struct perf_event *sibling, *leader = event->group_leader;
1693	enum cmn_node_type type;
1694	struct arm_cmn_val *val;
1695	int i, ret = -EINVAL;
1696
1697	if (leader == event)
1698		return 0;
1699
1700	if (event->pmu != leader->pmu && !is_software_event(leader))
1701		return -EINVAL;
1702
1703	val = kzalloc(sizeof(*val), GFP_KERNEL);
1704	if (!val)
1705		return -ENOMEM;
1706
1707	arm_cmn_val_add_event(cmn, val, leader);
1708
1709	for_each_sibling_event(sibling, leader)
1710		arm_cmn_val_add_event(cmn, val, sibling);
1711
1712	type = CMN_EVENT_TYPE(event);
1713	if (type == CMN_TYPE_DTC) {
1714		ret = val->cycles ? -EINVAL : 0;
1715		goto done;
1716	}
1717
1718	for_each_hw_dtc_idx(hw, dtc, idx)
1719		if (val->dtc_count[dtc] == CMN_DT_NUM_COUNTERS)
1720			goto done;
1721
1722	for_each_hw_dn(hw, dn, i) {
1723		int wp_idx, dtm = dn->dtm, sel = hw->filter_sel;
1724
1725		if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS)
1726			goto done;
1727
1728		if (sel > SEL_NONE && val->occupid[dtm][sel] &&
1729		    val->occupid[dtm][sel] != CMN_EVENT_OCCUPID(event) + 1)
1730			goto done;
1731
1732		if (type != CMN_TYPE_WP)
1733			continue;
1734
1735		wp_idx = arm_cmn_val_find_free_wp_config(event, val, dtm);
1736		if (wp_idx < 0)
1737			goto done;
1738
1739		if (wp_idx & 1 &&
1740		    val->wp_combine[dtm][wp_idx >> 1] != !!CMN_EVENT_WP_COMBINE(event))
1741			goto done;
1742	}
1743
1744	ret = 0;
1745done:
1746	kfree(val);
1747	return ret;
1748}
1749
1750static enum cmn_filter_select arm_cmn_filter_sel(const struct arm_cmn *cmn,
1751						 enum cmn_node_type type,
1752						 unsigned int eventid)
1753{
1754	struct arm_cmn_event_attr *e;
1755	enum cmn_model model = arm_cmn_model(cmn);
1756
1757	for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) {
1758		e = container_of(arm_cmn_event_attrs[i], typeof(*e), attr.attr);
1759		if (e->model & model && e->type == type && e->eventid == eventid)
1760			return e->fsel;
1761	}
1762	return SEL_NONE;
1763}
1764
1765
1766static int arm_cmn_event_init(struct perf_event *event)
1767{
1768	struct arm_cmn *cmn = to_cmn(event->pmu);
1769	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1770	struct arm_cmn_node *dn;
1771	enum cmn_node_type type;
1772	bool bynodeid;
1773	u16 nodeid, eventid;
1774
1775	if (event->attr.type != event->pmu->type)
1776		return -ENOENT;
1777
1778	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
1779		return -EINVAL;
1780
1781	event->cpu = cmn->cpu;
1782	if (event->cpu < 0)
1783		return -EINVAL;
1784
1785	type = CMN_EVENT_TYPE(event);
1786	/* DTC events (i.e. cycles) already have everything they need */
1787	if (type == CMN_TYPE_DTC)
1788		return arm_cmn_validate_group(cmn, event);
1789
1790	eventid = CMN_EVENT_EVENTID(event);
1791	/* For watchpoints we need the actual XP node here */
1792	if (type == CMN_TYPE_WP) {
1793		type = CMN_TYPE_XP;
1794		/* ...and we need a "real" direction */
1795		if (eventid != CMN_WP_UP && eventid != CMN_WP_DOWN)
1796			return -EINVAL;
1797		/* ...but the DTM may depend on which port we're watching */
1798		if (cmn->multi_dtm)
1799			hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2;
1800	} else if (type == CMN_TYPE_XP &&
1801		   (cmn->part == PART_CMN700 || cmn->part == PART_CMN_S3)) {
1802		hw->wide_sel = true;
1803	} else if (type == CMN_TYPE_RND) {
1804		/* Secretly permit this as an alias for "rnid" events */
1805		type = CMN_TYPE_RNI;
1806	}
1807
1808	/* This is sufficiently annoying to recalculate, so cache it */
1809	hw->filter_sel = arm_cmn_filter_sel(cmn, type, eventid);
1810
1811	bynodeid = CMN_EVENT_BYNODEID(event);
1812	nodeid = CMN_EVENT_NODEID(event);
1813
1814	hw->dn = arm_cmn_node(cmn, type);
1815	if (!hw->dn)
1816		return -EINVAL;
1817
1818	memset(hw->dtc_idx, -1, sizeof(hw->dtc_idx));
1819	for (dn = hw->dn; dn->type == type; dn++) {
1820		if (bynodeid && dn->id != nodeid) {
1821			hw->dn++;
1822			continue;
1823		}
1824		hw->num_dns++;
1825		if (dn->dtc < 0)
1826			memset(hw->dtc_idx, 0, cmn->num_dtcs);
1827		else
1828			hw->dtc_idx[dn->dtc] = 0;
1829
1830		if (bynodeid)
1831			break;
1832	}
1833
1834	if (!hw->num_dns) {
1835		dev_dbg(cmn->dev, "invalid node 0x%x type 0x%x\n", nodeid, type);
1836		return -EINVAL;
1837	}
1838
1839	return arm_cmn_validate_group(cmn, event);
1840}
1841
1842static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event,
1843				int i)
1844{
1845	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1846	enum cmn_node_type type = CMN_EVENT_TYPE(event);
1847
1848	while (i--) {
1849		struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset;
1850		unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i);
1851
1852		if (type == CMN_TYPE_WP) {
1853			int wp_idx = arm_cmn_get_assigned_wp_idx(event, hw, i);
1854
1855			dtm->wp_event[wp_idx] = -1;
1856		}
1857
1858		if (hw->filter_sel > SEL_NONE)
1859			hw->dn[i].occupid[hw->filter_sel].count--;
1860
1861		dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx);
1862		writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG);
1863	}
1864	memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx));
1865	memset(hw->wp_idx, 0, sizeof(hw->wp_idx));
1866
1867	for_each_hw_dtc_idx(hw, j, idx)
1868		cmn->dtc[j].counters[idx] = NULL;
1869}
1870
1871static int arm_cmn_event_add(struct perf_event *event, int flags)
1872{
1873	struct arm_cmn *cmn = to_cmn(event->pmu);
1874	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1875	struct arm_cmn_node *dn;
1876	enum cmn_node_type type = CMN_EVENT_TYPE(event);
1877	unsigned int input_sel, i = 0;
1878
1879	if (type == CMN_TYPE_DTC) {
1880		while (cmn->dtc[i].cycles)
1881			if (++i == cmn->num_dtcs)
1882				return -ENOSPC;
1883
1884		cmn->dtc[i].cycles = event;
1885		hw->dtc_idx[0] = i;
1886
1887		if (flags & PERF_EF_START)
1888			arm_cmn_event_start(event, 0);
1889		return 0;
1890	}
1891
1892	/* Grab the global counters first... */
1893	for_each_hw_dtc_idx(hw, j, idx) {
1894		if (cmn->part == PART_CMN600 && j > 0) {
1895			idx = hw->dtc_idx[0];
1896		} else {
1897			idx = 0;
1898			while (cmn->dtc[j].counters[idx])
1899				if (++idx == CMN_DT_NUM_COUNTERS)
1900					return -ENOSPC;
1901		}
1902		hw->dtc_idx[j] = idx;
1903	}
1904
1905	/* ...then the local counters to feed them */
1906	for_each_hw_dn(hw, dn, i) {
1907		struct arm_cmn_dtm *dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset;
1908		unsigned int dtm_idx, shift, d = max_t(int, dn->dtc, 0);
1909		u64 reg;
1910
1911		dtm_idx = 0;
1912		while (dtm->pmu_config_low & CMN__PMEVCNT_PAIRED(dtm_idx))
1913			if (++dtm_idx == CMN_DTM_NUM_COUNTERS)
1914				goto free_dtms;
1915
1916		if (type == CMN_TYPE_XP) {
1917			input_sel = CMN__PMEVCNT0_INPUT_SEL_XP + dtm_idx;
1918		} else if (type == CMN_TYPE_WP) {
1919			int tmp, wp_idx;
1920			u32 cfg;
1921
1922			wp_idx = arm_cmn_find_free_wp_idx(dtm, event);
1923			if (wp_idx < 0)
1924				goto free_dtms;
1925
1926			cfg = arm_cmn_wp_config(event, wp_idx);
1927
1928			tmp = dtm->wp_event[wp_idx ^ 1];
1929			if (tmp >= 0 && CMN_EVENT_WP_COMBINE(event) !=
1930					CMN_EVENT_WP_COMBINE(cmn->dtc[d].counters[tmp]))
1931				goto free_dtms;
1932
1933			input_sel = CMN__PMEVCNT0_INPUT_SEL_WP + wp_idx;
1934
1935			arm_cmn_claim_wp_idx(dtm, event, d, wp_idx, i);
1936			writel_relaxed(cfg, dtm->base + CMN_DTM_WPn_CONFIG(wp_idx));
1937		} else {
1938			struct arm_cmn_nodeid nid = arm_cmn_nid(dn);
1939
1940			if (cmn->multi_dtm)
1941				nid.port %= 2;
1942
1943			input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx +
1944				    (nid.port << 4) + (nid.dev << 2);
1945
1946			if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_OCCUPID(event)))
1947				goto free_dtms;
1948		}
1949
1950		arm_cmn_set_index(hw->dtm_idx, i, dtm_idx);
1951
1952		dtm->input_sel[dtm_idx] = input_sel;
1953		shift = CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(dtm_idx);
1954		dtm->pmu_config_low &= ~(CMN__PMEVCNT0_GLOBAL_NUM << shift);
1955		dtm->pmu_config_low |= FIELD_PREP(CMN__PMEVCNT0_GLOBAL_NUM, hw->dtc_idx[d]) << shift;
1956		dtm->pmu_config_low |= CMN__PMEVCNT_PAIRED(dtm_idx);
1957		reg = (u64)le32_to_cpu(dtm->pmu_config_high) << 32 | dtm->pmu_config_low;
1958		writeq_relaxed(reg, dtm->base + CMN_DTM_PMU_CONFIG);
1959	}
1960
1961	/* Go go go! */
1962	arm_cmn_init_counter(event);
1963
1964	if (flags & PERF_EF_START)
1965		arm_cmn_event_start(event, 0);
1966
1967	return 0;
1968
1969free_dtms:
1970	arm_cmn_event_clear(cmn, event, i);
1971	return -ENOSPC;
1972}
1973
1974static void arm_cmn_event_del(struct perf_event *event, int flags)
1975{
1976	struct arm_cmn *cmn = to_cmn(event->pmu);
1977	struct arm_cmn_hw_event *hw = to_cmn_hw(event);
1978	enum cmn_node_type type = CMN_EVENT_TYPE(event);
1979
1980	arm_cmn_event_stop(event, PERF_EF_UPDATE);
1981
1982	if (type == CMN_TYPE_DTC)
1983		cmn->dtc[hw->dtc_idx[0]].cycles = NULL;
1984	else
1985		arm_cmn_event_clear(cmn, event, hw->num_dns);
1986}
1987
1988/*
1989 * We stop the PMU for both add and read, to avoid skew across DTM counters.
1990 * In theory we could use snapshots to read without stopping, but then it
1991 * becomes a lot trickier to deal with overlow and racing against interrupts,
1992 * plus it seems they don't work properly on some hardware anyway :(
1993 */
1994static void arm_cmn_start_txn(struct pmu *pmu, unsigned int flags)
1995{
1996	arm_cmn_set_state(to_cmn(pmu), CMN_STATE_TXN);
1997}
1998
1999static void arm_cmn_end_txn(struct pmu *pmu)
2000{
2001	arm_cmn_clear_state(to_cmn(pmu), CMN_STATE_TXN);
2002}
2003
2004static int arm_cmn_commit_txn(struct pmu *pmu)
2005{
2006	arm_cmn_end_txn(pmu);
2007	return 0;
2008}
2009
2010static void arm_cmn_migrate(struct arm_cmn *cmn, unsigned int cpu)
2011{
2012	unsigned int i;
2013
2014	perf_pmu_migrate_context(&cmn->pmu, cmn->cpu, cpu);
2015	for (i = 0; i < cmn->num_dtcs; i++)
2016		irq_set_affinity(cmn->dtc[i].irq, cpumask_of(cpu));
2017	cmn->cpu = cpu;
2018}
2019
2020static int arm_cmn_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
2021{
2022	struct arm_cmn *cmn;
2023	int node;
2024
2025	cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node);
2026	node = dev_to_node(cmn->dev);
2027	if (cpu_to_node(cmn->cpu) != node && cpu_to_node(cpu) == node)
2028		arm_cmn_migrate(cmn, cpu);
2029	return 0;
2030}
2031
2032static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
2033{
2034	struct arm_cmn *cmn;
2035	unsigned int target;
2036	int node;
2037
2038	cmn = hlist_entry_safe(cpuhp_node, struct arm_cmn, cpuhp_node);
2039	if (cpu != cmn->cpu)
2040		return 0;
2041
2042	node = dev_to_node(cmn->dev);
2043
2044	target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
2045	if (target >= nr_cpu_ids)
2046		target = cpumask_any_but(cpu_online_mask, cpu);
2047
2048	if (target < nr_cpu_ids)
2049		arm_cmn_migrate(cmn, target);
2050
2051	return 0;
2052}
2053
2054static irqreturn_t arm_cmn_handle_irq(int irq, void *dev_id)
2055{
2056	struct arm_cmn_dtc *dtc = dev_id;
2057	irqreturn_t ret = IRQ_NONE;
2058
2059	for (;;) {
2060		u32 status = readl_relaxed(CMN_DT_PMOVSR(dtc));
2061		u64 delta;
2062		int i;
2063
2064		for (i = 0; i < CMN_DT_NUM_COUNTERS; i++) {
2065			if (status & (1U << i)) {
2066				ret = IRQ_HANDLED;
2067				if (WARN_ON(!dtc->counters[i]))
2068					continue;
2069				delta = (u64)arm_cmn_read_counter(dtc, i) << 16;
2070				local64_add(delta, &dtc->counters[i]->count);
2071			}
2072		}
2073
2074		if (status & (1U << CMN_DT_NUM_COUNTERS)) {
2075			ret = IRQ_HANDLED;
2076			if (dtc->cc_active && !WARN_ON(!dtc->cycles)) {
2077				delta = arm_cmn_read_cc(dtc);
2078				local64_add(delta, &dtc->cycles->count);
2079			}
2080		}
2081
2082		writel_relaxed(status, CMN_DT_PMOVSR_CLR(dtc));
2083
2084		if (!dtc->irq_friend)
2085			return ret;
2086		dtc += dtc->irq_friend;
2087	}
2088}
2089
2090/* We can reasonably accommodate DTCs of the same CMN sharing IRQs */
2091static int arm_cmn_init_irqs(struct arm_cmn *cmn)
2092{
2093	int i, j, irq, err;
2094
2095	for (i = 0; i < cmn->num_dtcs; i++) {
2096		irq = cmn->dtc[i].irq;
2097		for (j = i; j--; ) {
2098			if (cmn->dtc[j].irq == irq) {
2099				cmn->dtc[j].irq_friend = i - j;
2100				goto next;
2101			}
2102		}
2103		err = devm_request_irq(cmn->dev, irq, arm_cmn_handle_irq,
2104				       IRQF_NOBALANCING | IRQF_NO_THREAD,
2105				       dev_name(cmn->dev), &cmn->dtc[i]);
2106		if (err)
2107			return err;
2108
2109		err = irq_set_affinity(irq, cpumask_of(cmn->cpu));
2110		if (err)
2111			return err;
2112	next:
2113		; /* isn't C great? */
2114	}
2115	return 0;
2116}
2117
2118static void arm_cmn_init_dtm(struct arm_cmn_dtm *dtm, struct arm_cmn_node *xp, int idx)
2119{
2120	int i;
2121
2122	dtm->base = xp->pmu_base + CMN_DTM_OFFSET(idx);
2123	dtm->pmu_config_low = CMN_DTM_PMU_CONFIG_PMU_EN;
2124	writeq_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG);
2125	for (i = 0; i < 4; i++) {
2126		dtm->wp_event[i] = -1;
2127		writeq_relaxed(0, dtm->base + CMN_DTM_WPn_MASK(i));
2128		writeq_relaxed(~0ULL, dtm->base + CMN_DTM_WPn_VAL(i));
2129	}
2130}
2131
2132static int arm_cmn_init_dtc(struct arm_cmn *cmn, struct arm_cmn_node *dn, int idx)
2133{
2134	struct arm_cmn_dtc *dtc = cmn->dtc + idx;
2135
2136	dtc->pmu_base = dn->pmu_base;
2137	dtc->base = dtc->pmu_base - arm_cmn_pmu_offset(cmn, dn);
2138	dtc->irq = platform_get_irq(to_platform_device(cmn->dev), idx);
2139	if (dtc->irq < 0)
2140		return dtc->irq;
2141
2142	writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL);
2143	writel_relaxed(CMN_DT_PMCR_PMU_EN | CMN_DT_PMCR_OVFL_INTR_EN, CMN_DT_PMCR(dtc));
2144	writeq_relaxed(0, CMN_DT_PMCCNTR(dtc));
2145	writel_relaxed(0x1ff, CMN_DT_PMOVSR_CLR(dtc));
2146
2147	return 0;
2148}
2149
2150static int arm_cmn_node_cmp(const void *a, const void *b)
2151{
2152	const struct arm_cmn_node *dna = a, *dnb = b;
2153	int cmp;
2154
2155	cmp = dna->type - dnb->type;
2156	if (!cmp)
2157		cmp = dna->logid - dnb->logid;
2158	return cmp;
2159}
2160
2161static int arm_cmn_init_dtcs(struct arm_cmn *cmn)
2162{
2163	struct arm_cmn_node *dn, *xp;
2164	int dtc_idx = 0;
2165
2166	cmn->dtc = devm_kcalloc(cmn->dev, cmn->num_dtcs, sizeof(cmn->dtc[0]), GFP_KERNEL);
2167	if (!cmn->dtc)
2168		return -ENOMEM;
2169
2170	sort(cmn->dns, cmn->num_dns, sizeof(cmn->dns[0]), arm_cmn_node_cmp, NULL);
2171
2172	cmn->xps = arm_cmn_node(cmn, CMN_TYPE_XP);
2173
2174	for (dn = cmn->dns; dn->type; dn++) {
2175		if (dn->type == CMN_TYPE_XP)
2176			continue;
2177
2178		xp = arm_cmn_node_to_xp(cmn, dn);
2179		dn->dtc = xp->dtc;
2180		dn->dtm = xp->dtm;
2181		if (cmn->multi_dtm)
2182			dn->dtm += arm_cmn_nid(dn).port / 2;
2183
2184		if (dn->type == CMN_TYPE_DTC) {
2185			int err = arm_cmn_init_dtc(cmn, dn, dtc_idx++);
2186
2187			if (err)
2188				return err;
2189		}
2190
2191		/* To the PMU, RN-Ds don't add anything over RN-Is, so smoosh them together */
2192		if (dn->type == CMN_TYPE_RND)
2193			dn->type = CMN_TYPE_RNI;
2194
2195		/* We split the RN-I off already, so let the CCLA part match CCLA events */
2196		if (dn->type == CMN_TYPE_CCLA_RNI)
2197			dn->type = CMN_TYPE_CCLA;
2198	}
2199
2200	arm_cmn_set_state(cmn, CMN_STATE_DISABLED);
2201
2202	return 0;
2203}
2204
2205static unsigned int arm_cmn_dtc_domain(struct arm_cmn *cmn, void __iomem *xp_region)
2206{
2207	int offset = CMN_DTM_UNIT_INFO;
2208
2209	if (cmn->part == PART_CMN650 || cmn->part == PART_CI700)
2210		offset = CMN650_DTM_UNIT_INFO;
2211
2212	return FIELD_GET(CMN_DTM_UNIT_INFO_DTC_DOMAIN, readl_relaxed(xp_region + offset));
2213}
2214
2215static void arm_cmn_init_node_info(struct arm_cmn *cmn, u32 offset, struct arm_cmn_node *node)
2216{
2217	int level;
2218	u64 reg = readq_relaxed(cmn->base + offset + CMN_NODE_INFO);
2219
2220	node->type = FIELD_GET(CMN_NI_NODE_TYPE, reg);
2221	node->id = FIELD_GET(CMN_NI_NODE_ID, reg);
2222	node->logid = FIELD_GET(CMN_NI_LOGICAL_ID, reg);
2223
2224	node->pmu_base = cmn->base + offset + arm_cmn_pmu_offset(cmn, node);
2225
2226	if (node->type == CMN_TYPE_CFG)
2227		level = 0;
2228	else if (node->type == CMN_TYPE_XP)
2229		level = 1;
2230	else
2231		level = 2;
2232
2233	dev_dbg(cmn->dev, "node%*c%#06hx%*ctype:%-#6x id:%-4hd off:%#x\n",
2234			(level * 2) + 1, ' ', node->id, 5 - (level * 2), ' ',
2235			node->type, node->logid, offset);
2236}
2237
2238static enum cmn_node_type arm_cmn_subtype(enum cmn_node_type type)
2239{
2240	switch (type) {
2241	case CMN_TYPE_HNP:
2242		return CMN_TYPE_HNI;
2243	case CMN_TYPE_CCLA_RNI:
2244		return CMN_TYPE_RNI;
2245	default:
2246		return CMN_TYPE_INVALID;
2247	}
2248}
2249
2250static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset)
2251{
2252	void __iomem *cfg_region, __iomem *xp_region;
2253	struct arm_cmn_node cfg, *dn;
2254	struct arm_cmn_dtm *dtm;
2255	enum cmn_part part;
2256	u16 child_count, child_poff;
2257	u64 reg;
2258	int i, j;
2259	size_t sz;
2260
2261	arm_cmn_init_node_info(cmn, rgn_offset, &cfg);
2262	if (cfg.type != CMN_TYPE_CFG)
2263		return -ENODEV;
2264
2265	cfg_region = cmn->base + rgn_offset;
2266
2267	reg = readq_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_01);
2268	part = FIELD_GET(CMN_CFGM_PID0_PART_0, reg);
2269	part |= FIELD_GET(CMN_CFGM_PID1_PART_1, reg) << 8;
2270	/* 600AE is close enough that it's not really worth more complexity */
2271	if (part == PART_CMN600AE)
2272		part = PART_CMN600;
2273	if (cmn->part && cmn->part != part)
2274		dev_warn(cmn->dev,
2275			 "Firmware binding mismatch: expected part number 0x%x, found 0x%x\n",
2276			 cmn->part, part);
2277	cmn->part = part;
2278	if (!arm_cmn_model(cmn))
2279		dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part);
2280
2281	reg = readl_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_23);
2282	cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg);
2283
2284	/*
2285	 * With the device isolation feature, if firmware has neglected to enable
2286	 * an XP port then we risk locking up if we try to access anything behind
2287	 * it; however we also have no way to tell from Non-Secure whether any
2288	 * given port is disabled or not, so the only way to win is not to play...
2289	 */
2290	reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL);
2291	if (reg & CMN_INFO_DEVICE_ISO_ENABLE) {
2292		dev_err(cmn->dev, "Device isolation enabled, not continuing due to risk of lockup\n");
2293		return -ENODEV;
2294	}
2295	cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN;
2296	cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg);
2297	cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg);
2298
2299	reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL_1);
2300	cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg);
2301	cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg);
2302
2303	reg = readq_relaxed(cfg_region + CMN_CHILD_INFO);
2304	child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg);
2305	child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg);
2306
2307	cmn->num_xps = child_count;
2308	cmn->num_dns = cmn->num_xps;
2309
2310	/* Pass 1: visit the XPs, enumerate their children */
2311	cfg_region += child_poff;
2312	for (i = 0; i < cmn->num_xps; i++) {
2313		reg = readq_relaxed(cfg_region + i * 8);
2314		xp_region = cmn->base + (reg & CMN_CHILD_NODE_ADDR);
2315
2316		reg = readq_relaxed(xp_region + CMN_CHILD_INFO);
2317		cmn->num_dns += FIELD_GET(CMN_CI_CHILD_COUNT, reg);
2318	}
2319
2320	/*
2321	 * Some nodes effectively have two separate types, which we'll handle
2322	 * by creating one of each internally. For a (very) safe initial upper
2323	 * bound, account for double the number of non-XP nodes.
2324	 */
2325	dn = devm_kcalloc(cmn->dev, cmn->num_dns * 2 - cmn->num_xps,
2326			  sizeof(*dn), GFP_KERNEL);
2327	if (!dn)
2328		return -ENOMEM;
2329
2330	/* Initial safe upper bound on DTMs for any possible mesh layout */
2331	i = cmn->num_xps;
2332	if (cmn->multi_dtm)
2333		i += cmn->num_xps + 1;
2334	dtm = devm_kcalloc(cmn->dev, i, sizeof(*dtm), GFP_KERNEL);
2335	if (!dtm)
2336		return -ENOMEM;
2337
2338	/* Pass 2: now we can actually populate the nodes */
2339	cmn->dns = dn;
2340	cmn->dtms = dtm;
2341	for (i = 0; i < cmn->num_xps; i++) {
2342		struct arm_cmn_node *xp = dn++;
2343		unsigned int xp_ports = 0;
2344
2345		reg = readq_relaxed(cfg_region + i * 8);
2346		xp_region = cmn->base + (reg & CMN_CHILD_NODE_ADDR);
2347		arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, xp);
2348		/*
2349		 * Thanks to the order in which XP logical IDs seem to be
2350		 * assigned, we can handily infer the mesh X dimension by
2351		 * looking out for the XP at (0,1) without needing to know
2352		 * the exact node ID format, which we can later derive.
2353		 */
2354		if (xp->id == (1 << 3))
2355			cmn->mesh_x = xp->logid;
2356
2357		if (cmn->part == PART_CMN600)
2358			xp->dtc = -1;
2359		else
2360			xp->dtc = arm_cmn_dtc_domain(cmn, xp_region);
2361
2362		xp->dtm = dtm - cmn->dtms;
2363		arm_cmn_init_dtm(dtm++, xp, 0);
2364		/*
2365		 * Keeping track of connected ports will let us filter out
2366		 * unnecessary XP events easily, and also infer the per-XP
2367		 * part of the node ID format.
2368		 */
2369		for (int p = 0; p < CMN_MAX_PORTS; p++)
2370			if (arm_cmn_device_connect_info(cmn, xp, p))
2371				xp_ports |= BIT(p);
2372
2373		if (cmn->num_xps == 1) {
2374			xp->portid_bits = 3;
2375			xp->deviceid_bits = 2;
2376		} else if (xp_ports > 0x3) {
2377			xp->portid_bits = 2;
2378			xp->deviceid_bits = 1;
2379		} else {
2380			xp->portid_bits = 1;
2381			xp->deviceid_bits = 2;
2382		}
2383
2384		if (cmn->multi_dtm && (xp_ports > 0x3))
2385			arm_cmn_init_dtm(dtm++, xp, 1);
2386		if (cmn->multi_dtm && (xp_ports > 0xf))
2387			arm_cmn_init_dtm(dtm++, xp, 2);
2388
2389		cmn->ports_used |= xp_ports;
2390
2391		reg = readq_relaxed(xp_region + CMN_CHILD_INFO);
2392		child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg);
2393		child_poff = FIELD_GET(CMN_CI_CHILD_PTR_OFFSET, reg);
2394
2395		for (j = 0; j < child_count; j++) {
2396			reg = readq_relaxed(xp_region + child_poff + j * 8);
2397			/*
2398			 * Don't even try to touch anything external, since in general
2399			 * we haven't a clue how to power up arbitrary CHI requesters.
2400			 * As of CMN-600r1 these could only be RN-SAMs or CXLAs,
2401			 * neither of which have any PMU events anyway.
2402			 * (Actually, CXLAs do seem to have grown some events in r1p2,
2403			 * but they don't go to regular XP DTMs, and they depend on
2404			 * secure configuration which we can't easily deal with)
2405			 */
2406			if (reg & CMN_CHILD_NODE_EXTERNAL) {
2407				dev_dbg(cmn->dev, "ignoring external node %llx\n", reg);
2408				continue;
2409			}
2410			/*
2411			 * AmpereOneX erratum AC04_MESH_1 makes some XPs report a bogus
2412			 * child count larger than the number of valid child pointers.
2413			 * A child offset of 0 can only occur on CMN-600; otherwise it
2414			 * would imply the root node being its own grandchild, which
2415			 * we can safely dismiss in general.
2416			 */
2417			if (reg == 0 && cmn->part != PART_CMN600) {
2418				dev_dbg(cmn->dev, "bogus child pointer?\n");
2419				continue;
2420			}
2421
2422			arm_cmn_init_node_info(cmn, reg & CMN_CHILD_NODE_ADDR, dn);
2423			dn->portid_bits = xp->portid_bits;
2424			dn->deviceid_bits = xp->deviceid_bits;
2425			/*
2426			 * Logical IDs are assigned from 0 per node type, so as
2427			 * soon as we see one bigger than expected, we can assume
2428			 * there are more than we can cope with.
2429			 */
2430			if (dn->logid > CMN_MAX_NODES_PER_EVENT) {
2431				dev_err(cmn->dev, "Node ID invalid for supported CMN versions: %d\n", dn->logid);
2432				return -ENODEV;
2433			}
2434
2435			switch (dn->type) {
2436			case CMN_TYPE_DTC:
2437				cmn->num_dtcs++;
2438				dn++;
2439				break;
2440			/* These guys have PMU events */
2441			case CMN_TYPE_DVM:
2442			case CMN_TYPE_HNI:
2443			case CMN_TYPE_HNF:
2444			case CMN_TYPE_SBSX:
2445			case CMN_TYPE_RNI:
2446			case CMN_TYPE_RND:
2447			case CMN_TYPE_MTSX:
2448			case CMN_TYPE_CXRA:
2449			case CMN_TYPE_CXHA:
2450			case CMN_TYPE_CCRA:
2451			case CMN_TYPE_CCHA:
2452			case CMN_TYPE_HNS:
2453				dn++;
2454				break;
2455			case CMN_TYPE_CCLA:
2456				dn->pmu_base += CMN_CCLA_PMU_EVENT_SEL;
2457				dn++;
2458				break;
2459			/* Nothing to see here */
2460			case CMN_TYPE_MPAM_S:
2461			case CMN_TYPE_MPAM_NS:
2462			case CMN_TYPE_RNSAM:
2463			case CMN_TYPE_CXLA:
2464			case CMN_TYPE_HNS_MPAM_S:
2465			case CMN_TYPE_HNS_MPAM_NS:
2466			case CMN_TYPE_APB:
2467				break;
2468			/*
2469			 * Split "optimised" combination nodes into separate
2470			 * types for the different event sets. Offsetting the
2471			 * base address lets us handle the second pmu_event_sel
2472			 * register via the normal mechanism later.
2473			 */
2474			case CMN_TYPE_HNP:
2475			case CMN_TYPE_CCLA_RNI:
2476				dn[1] = dn[0];
2477				dn[0].pmu_base += CMN_CCLA_PMU_EVENT_SEL;
2478				dn[1].type = arm_cmn_subtype(dn->type);
2479				dn += 2;
2480				break;
2481			/* Something has gone horribly wrong */
2482			default:
2483				dev_err(cmn->dev, "Device node type invalid for supported CMN versions: 0x%x\n", dn->type);
2484				return -ENODEV;
2485			}
2486		}
2487	}
2488
2489	/* Correct for any nodes we added or skipped */
2490	cmn->num_dns = dn - cmn->dns;
2491
2492	/* Cheeky +1 to help terminate pointer-based iteration later */
2493	sz = (void *)(dn + 1) - (void *)cmn->dns;
2494	dn = devm_krealloc(cmn->dev, cmn->dns, sz, GFP_KERNEL);
2495	if (dn)
2496		cmn->dns = dn;
2497
2498	sz = (void *)dtm - (void *)cmn->dtms;
2499	dtm = devm_krealloc(cmn->dev, cmn->dtms, sz, GFP_KERNEL);
2500	if (dtm)
2501		cmn->dtms = dtm;
2502
2503	/*
2504	 * If mesh_x wasn't set during discovery then we never saw
2505	 * an XP at (0,1), thus we must have an Nx1 configuration.
2506	 */
2507	if (!cmn->mesh_x)
2508		cmn->mesh_x = cmn->num_xps;
2509	cmn->mesh_y = cmn->num_xps / cmn->mesh_x;
2510
2511	if (max(cmn->mesh_x, cmn->mesh_y) > CMN_MAX_DIMENSION) {
2512		dev_err(cmn->dev, "Mesh size invalid for supported CMN versions: %dx%d\n", cmn->mesh_x, cmn->mesh_y);
2513		return -ENODEV;
2514	}
2515	/* 1x1 config plays havoc with XP event encodings */
2516	if (cmn->num_xps == 1)
2517		dev_warn(cmn->dev, "1x1 config not fully supported, translate XP events manually\n");
2518
2519	dev_dbg(cmn->dev, "periph_id part 0x%03x revision %d\n", cmn->part, cmn->rev);
2520	reg = cmn->ports_used;
2521	dev_dbg(cmn->dev, "mesh %dx%d, ID width %d, ports %6pbl%s\n",
2522		cmn->mesh_x, cmn->mesh_y, arm_cmn_xyidbits(cmn), &reg,
2523		cmn->multi_dtm ? ", multi-DTM" : "");
2524
2525	return 0;
2526}
2527
2528static int arm_cmn600_acpi_probe(struct platform_device *pdev, struct arm_cmn *cmn)
2529{
2530	struct resource *cfg, *root;
2531
2532	cfg = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2533	if (!cfg)
2534		return -EINVAL;
2535
2536	root = platform_get_resource(pdev, IORESOURCE_MEM, 1);
2537	if (!root)
2538		return -EINVAL;
2539
2540	if (!resource_contains(cfg, root))
2541		swap(cfg, root);
2542	/*
2543	 * Note that devm_ioremap_resource() is dumb and won't let the platform
2544	 * device claim cfg when the ACPI companion device has already claimed
2545	 * root within it. But since they *are* already both claimed in the
2546	 * appropriate name, we don't really need to do it again here anyway.
2547	 */
2548	cmn->base = devm_ioremap(cmn->dev, cfg->start, resource_size(cfg));
2549	if (!cmn->base)
2550		return -ENOMEM;
2551
2552	return root->start - cfg->start;
2553}
2554
2555static int arm_cmn600_of_probe(struct device_node *np)
2556{
2557	u32 rootnode;
2558
2559	return of_property_read_u32(np, "arm,root-node", &rootnode) ?: rootnode;
2560}
2561
2562static int arm_cmn_probe(struct platform_device *pdev)
2563{
2564	struct arm_cmn *cmn;
2565	const char *name;
2566	static atomic_t id;
2567	int err, rootnode, this_id;
2568
2569	cmn = devm_kzalloc(&pdev->dev, sizeof(*cmn), GFP_KERNEL);
2570	if (!cmn)
2571		return -ENOMEM;
2572
2573	cmn->dev = &pdev->dev;
2574	cmn->part = (unsigned long)device_get_match_data(cmn->dev);
2575	cmn->cpu = cpumask_local_spread(0, dev_to_node(cmn->dev));
2576	platform_set_drvdata(pdev, cmn);
2577
2578	if (cmn->part == PART_CMN600 && has_acpi_companion(cmn->dev)) {
2579		rootnode = arm_cmn600_acpi_probe(pdev, cmn);
2580	} else {
2581		rootnode = 0;
2582		cmn->base = devm_platform_ioremap_resource(pdev, 0);
2583		if (IS_ERR(cmn->base))
2584			return PTR_ERR(cmn->base);
2585		if (cmn->part == PART_CMN600)
2586			rootnode = arm_cmn600_of_probe(pdev->dev.of_node);
2587	}
2588	if (rootnode < 0)
2589		return rootnode;
2590
2591	err = arm_cmn_discover(cmn, rootnode);
2592	if (err)
2593		return err;
2594
2595	err = arm_cmn_init_dtcs(cmn);
2596	if (err)
2597		return err;
2598
2599	err = arm_cmn_init_irqs(cmn);
2600	if (err)
2601		return err;
2602
2603	cmn->pmu = (struct pmu) {
2604		.module = THIS_MODULE,
2605		.parent = cmn->dev,
2606		.attr_groups = arm_cmn_attr_groups,
2607		.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
2608		.task_ctx_nr = perf_invalid_context,
2609		.pmu_enable = arm_cmn_pmu_enable,
2610		.pmu_disable = arm_cmn_pmu_disable,
2611		.event_init = arm_cmn_event_init,
2612		.add = arm_cmn_event_add,
2613		.del = arm_cmn_event_del,
2614		.start = arm_cmn_event_start,
2615		.stop = arm_cmn_event_stop,
2616		.read = arm_cmn_event_read,
2617		.start_txn = arm_cmn_start_txn,
2618		.commit_txn = arm_cmn_commit_txn,
2619		.cancel_txn = arm_cmn_end_txn,
2620	};
2621
2622	this_id = atomic_fetch_inc(&id);
2623	name = devm_kasprintf(cmn->dev, GFP_KERNEL, "arm_cmn_%d", this_id);
2624	if (!name)
2625		return -ENOMEM;
2626
2627	err = cpuhp_state_add_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
2628	if (err)
2629		return err;
2630
2631	err = perf_pmu_register(&cmn->pmu, name, -1);
2632	if (err)
2633		cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node);
2634	else
2635		arm_cmn_debugfs_init(cmn, this_id);
2636
2637	return err;
2638}
2639
2640static void arm_cmn_remove(struct platform_device *pdev)
2641{
2642	struct arm_cmn *cmn = platform_get_drvdata(pdev);
2643
2644	writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL);
2645
2646	perf_pmu_unregister(&cmn->pmu);
2647	cpuhp_state_remove_instance_nocalls(arm_cmn_hp_state, &cmn->cpuhp_node);
2648	debugfs_remove(cmn->debug);
2649}
2650
2651#ifdef CONFIG_OF
2652static const struct of_device_id arm_cmn_of_match[] = {
2653	{ .compatible = "arm,cmn-600", .data = (void *)PART_CMN600 },
2654	{ .compatible = "arm,cmn-650" },
2655	{ .compatible = "arm,cmn-700" },
2656	{ .compatible = "arm,cmn-s3" },
2657	{ .compatible = "arm,ci-700" },
2658	{}
2659};
2660MODULE_DEVICE_TABLE(of, arm_cmn_of_match);
2661#endif
2662
2663#ifdef CONFIG_ACPI
2664static const struct acpi_device_id arm_cmn_acpi_match[] = {
2665	{ "ARMHC600", PART_CMN600 },
2666	{ "ARMHC650" },
2667	{ "ARMHC700" },
2668	{ "ARMHC003" },
2669	{}
2670};
2671MODULE_DEVICE_TABLE(acpi, arm_cmn_acpi_match);
2672#endif
2673
2674static struct platform_driver arm_cmn_driver = {
2675	.driver = {
2676		.name = "arm-cmn",
2677		.of_match_table = of_match_ptr(arm_cmn_of_match),
2678		.acpi_match_table = ACPI_PTR(arm_cmn_acpi_match),
2679		.suppress_bind_attrs = true,
2680	},
2681	.probe = arm_cmn_probe,
2682	.remove = arm_cmn_remove,
2683};
2684
2685static int __init arm_cmn_init(void)
2686{
2687	int ret;
2688
2689	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
2690				      "perf/arm/cmn:online",
2691				      arm_cmn_pmu_online_cpu,
2692				      arm_cmn_pmu_offline_cpu);
2693	if (ret < 0)
2694		return ret;
2695
2696	arm_cmn_hp_state = ret;
2697	arm_cmn_debugfs = debugfs_create_dir("arm-cmn", NULL);
2698
2699	ret = platform_driver_register(&arm_cmn_driver);
2700	if (ret) {
2701		cpuhp_remove_multi_state(arm_cmn_hp_state);
2702		debugfs_remove(arm_cmn_debugfs);
2703	}
2704	return ret;
2705}
2706
2707static void __exit arm_cmn_exit(void)
2708{
2709	platform_driver_unregister(&arm_cmn_driver);
2710	cpuhp_remove_multi_state(arm_cmn_hp_state);
2711	debugfs_remove(arm_cmn_debugfs);
2712}
2713
2714module_init(arm_cmn_init);
2715module_exit(arm_cmn_exit);
2716
2717MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
2718MODULE_DESCRIPTION("Arm CMN/CI interconnect PMU driver");
2719MODULE_LICENSE("GPL v2");