hns3_pmu.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * This driver adds support for HNS3 PMU iEP device. Related perf events are
   4 * bandwidth, latency, packet rate, interrupt rate etc.
   5 *
   6 * Copyright (C) 2022 HiSilicon Limited
   7 */
   8#include <linux/bitfield.h>
   9#include <linux/bitmap.h>
  10#include <linux/bug.h>
  11#include <linux/cpuhotplug.h>
  12#include <linux/cpumask.h>
  13#include <linux/delay.h>
  14#include <linux/device.h>
  15#include <linux/err.h>
  16#include <linux/interrupt.h>
  17#include <linux/iopoll.h>
  18#include <linux/io-64-nonatomic-hi-lo.h>
  19#include <linux/irq.h>
  20#include <linux/kernel.h>
  21#include <linux/list.h>
  22#include <linux/module.h>
  23#include <linux/pci.h>
  24#include <linux/pci-epf.h>
  25#include <linux/perf_event.h>
  26#include <linux/smp.h>
  27
  28/* registers offset address */
  29#define HNS3_PMU_REG_GLOBAL_CTRL		0x0000
  30#define HNS3_PMU_REG_CLOCK_FREQ			0x0020
  31#define HNS3_PMU_REG_BDF			0x0fe0
  32#define HNS3_PMU_REG_VERSION			0x0fe4
  33#define HNS3_PMU_REG_DEVICE_ID			0x0fe8
  34
  35#define HNS3_PMU_REG_EVENT_OFFSET		0x1000
  36#define HNS3_PMU_REG_EVENT_SIZE			0x1000
  37#define HNS3_PMU_REG_EVENT_CTRL_LOW		0x00
  38#define HNS3_PMU_REG_EVENT_CTRL_HIGH		0x04
  39#define HNS3_PMU_REG_EVENT_INTR_STATUS		0x08
  40#define HNS3_PMU_REG_EVENT_INTR_MASK		0x0c
  41#define HNS3_PMU_REG_EVENT_COUNTER		0x10
  42#define HNS3_PMU_REG_EVENT_EXT_COUNTER		0x18
  43#define HNS3_PMU_REG_EVENT_QID_CTRL		0x28
  44#define HNS3_PMU_REG_EVENT_QID_PARA		0x2c
  45
  46#define HNS3_PMU_FILTER_SUPPORT_GLOBAL		BIT(0)
  47#define HNS3_PMU_FILTER_SUPPORT_PORT		BIT(1)
  48#define HNS3_PMU_FILTER_SUPPORT_PORT_TC		BIT(2)
  49#define HNS3_PMU_FILTER_SUPPORT_FUNC		BIT(3)
  50#define HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE	BIT(4)
  51#define HNS3_PMU_FILTER_SUPPORT_FUNC_INTR	BIT(5)
  52
  53#define HNS3_PMU_FILTER_ALL_TC			0xf
  54#define HNS3_PMU_FILTER_ALL_QUEUE		0xffff
  55
  56#define HNS3_PMU_CTRL_SUBEVENT_S		4
  57#define HNS3_PMU_CTRL_FILTER_MODE_S		24
  58
  59#define HNS3_PMU_GLOBAL_START			BIT(0)
  60
  61#define HNS3_PMU_EVENT_STATUS_RESET		BIT(11)
  62#define HNS3_PMU_EVENT_EN			BIT(12)
  63#define HNS3_PMU_EVENT_OVERFLOW_RESTART		BIT(15)
  64
  65#define HNS3_PMU_QID_PARA_FUNC_S		0
  66#define HNS3_PMU_QID_PARA_QUEUE_S		16
  67
  68#define HNS3_PMU_QID_CTRL_REQ_ENABLE		BIT(0)
  69#define HNS3_PMU_QID_CTRL_DONE			BIT(1)
  70#define HNS3_PMU_QID_CTRL_MISS			BIT(2)
  71
  72#define HNS3_PMU_INTR_MASK_OVERFLOW		BIT(1)
  73
  74#define HNS3_PMU_MAX_HW_EVENTS			8
  75
  76/*
  77 * Each hardware event contains two registers (counter and ext_counter) for
  78 * bandwidth, packet rate, latency and interrupt rate. These two registers will
  79 * be triggered to run at the same when a hardware event is enabled. The meaning
  80 * of counter and ext_counter of different event type are different, their
  81 * meaning show as follow:
  82 *
  83 * +----------------+------------------+---------------+
  84 * |   event type   |     counter      |  ext_counter  |
  85 * +----------------+------------------+---------------+
  86 * | bandwidth      | byte number      | cycle number  |
  87 * +----------------+------------------+---------------+
  88 * | packet rate    | packet number    | cycle number  |
  89 * +----------------+------------------+---------------+
  90 * | latency        | cycle number     | packet number |
  91 * +----------------+------------------+---------------+
  92 * | interrupt rate | interrupt number | cycle number  |
  93 * +----------------+------------------+---------------+
  94 *
  95 * The cycle number indicates increment of counter of hardware timer, the
  96 * frequency of hardware timer can be read from hw_clk_freq file.
  97 *
  98 * Performance of each hardware event is calculated by: counter / ext_counter.
  99 *
 100 * Since processing of data is preferred to be done in userspace, we expose
 101 * ext_counter as a separate event for userspace and use bit 16 to indicate it.
 102 * For example, event 0x00001 and 0x10001 are actually one event for hardware
 103 * because bit 0-15 are same. If the bit 16 of one event is 0 means to read
 104 * counter register, otherwise means to read ext_counter register.
 105 */
 106/* bandwidth events */
 107#define HNS3_PMU_EVT_BW_SSU_EGU_BYTE_NUM		0x00001
 108#define HNS3_PMU_EVT_BW_SSU_EGU_TIME			0x10001
 109#define HNS3_PMU_EVT_BW_SSU_RPU_BYTE_NUM		0x00002
 110#define HNS3_PMU_EVT_BW_SSU_RPU_TIME			0x10002
 111#define HNS3_PMU_EVT_BW_SSU_ROCE_BYTE_NUM		0x00003
 112#define HNS3_PMU_EVT_BW_SSU_ROCE_TIME			0x10003
 113#define HNS3_PMU_EVT_BW_ROCE_SSU_BYTE_NUM		0x00004
 114#define HNS3_PMU_EVT_BW_ROCE_SSU_TIME			0x10004
 115#define HNS3_PMU_EVT_BW_TPU_SSU_BYTE_NUM		0x00005
 116#define HNS3_PMU_EVT_BW_TPU_SSU_TIME			0x10005
 117#define HNS3_PMU_EVT_BW_RPU_RCBRX_BYTE_NUM		0x00006
 118#define HNS3_PMU_EVT_BW_RPU_RCBRX_TIME			0x10006
 119#define HNS3_PMU_EVT_BW_RCBTX_TXSCH_BYTE_NUM		0x00008
 120#define HNS3_PMU_EVT_BW_RCBTX_TXSCH_TIME		0x10008
 121#define HNS3_PMU_EVT_BW_WR_FBD_BYTE_NUM			0x00009
 122#define HNS3_PMU_EVT_BW_WR_FBD_TIME			0x10009
 123#define HNS3_PMU_EVT_BW_WR_EBD_BYTE_NUM			0x0000a
 124#define HNS3_PMU_EVT_BW_WR_EBD_TIME			0x1000a
 125#define HNS3_PMU_EVT_BW_RD_FBD_BYTE_NUM			0x0000b
 126#define HNS3_PMU_EVT_BW_RD_FBD_TIME			0x1000b
 127#define HNS3_PMU_EVT_BW_RD_EBD_BYTE_NUM			0x0000c
 128#define HNS3_PMU_EVT_BW_RD_EBD_TIME			0x1000c
 129#define HNS3_PMU_EVT_BW_RD_PAY_M0_BYTE_NUM		0x0000d
 130#define HNS3_PMU_EVT_BW_RD_PAY_M0_TIME			0x1000d
 131#define HNS3_PMU_EVT_BW_RD_PAY_M1_BYTE_NUM		0x0000e
 132#define HNS3_PMU_EVT_BW_RD_PAY_M1_TIME			0x1000e
 133#define HNS3_PMU_EVT_BW_WR_PAY_M0_BYTE_NUM		0x0000f
 134#define HNS3_PMU_EVT_BW_WR_PAY_M0_TIME			0x1000f
 135#define HNS3_PMU_EVT_BW_WR_PAY_M1_BYTE_NUM		0x00010
 136#define HNS3_PMU_EVT_BW_WR_PAY_M1_TIME			0x10010
 137
 138/* packet rate events */
 139#define HNS3_PMU_EVT_PPS_IGU_SSU_PACKET_NUM		0x00100
 140#define HNS3_PMU_EVT_PPS_IGU_SSU_TIME			0x10100
 141#define HNS3_PMU_EVT_PPS_SSU_EGU_PACKET_NUM		0x00101
 142#define HNS3_PMU_EVT_PPS_SSU_EGU_TIME			0x10101
 143#define HNS3_PMU_EVT_PPS_SSU_RPU_PACKET_NUM		0x00102
 144#define HNS3_PMU_EVT_PPS_SSU_RPU_TIME			0x10102
 145#define HNS3_PMU_EVT_PPS_SSU_ROCE_PACKET_NUM		0x00103
 146#define HNS3_PMU_EVT_PPS_SSU_ROCE_TIME			0x10103
 147#define HNS3_PMU_EVT_PPS_ROCE_SSU_PACKET_NUM		0x00104
 148#define HNS3_PMU_EVT_PPS_ROCE_SSU_TIME			0x10104
 149#define HNS3_PMU_EVT_PPS_TPU_SSU_PACKET_NUM		0x00105
 150#define HNS3_PMU_EVT_PPS_TPU_SSU_TIME			0x10105
 151#define HNS3_PMU_EVT_PPS_RPU_RCBRX_PACKET_NUM		0x00106
 152#define HNS3_PMU_EVT_PPS_RPU_RCBRX_TIME			0x10106
 153#define HNS3_PMU_EVT_PPS_RCBTX_TPU_PACKET_NUM		0x00107
 154#define HNS3_PMU_EVT_PPS_RCBTX_TPU_TIME			0x10107
 155#define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_PACKET_NUM		0x00108
 156#define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_TIME		0x10108
 157#define HNS3_PMU_EVT_PPS_WR_FBD_PACKET_NUM		0x00109
 158#define HNS3_PMU_EVT_PPS_WR_FBD_TIME			0x10109
 159#define HNS3_PMU_EVT_PPS_WR_EBD_PACKET_NUM		0x0010a
 160#define HNS3_PMU_EVT_PPS_WR_EBD_TIME			0x1010a
 161#define HNS3_PMU_EVT_PPS_RD_FBD_PACKET_NUM		0x0010b
 162#define HNS3_PMU_EVT_PPS_RD_FBD_TIME			0x1010b
 163#define HNS3_PMU_EVT_PPS_RD_EBD_PACKET_NUM		0x0010c
 164#define HNS3_PMU_EVT_PPS_RD_EBD_TIME			0x1010c
 165#define HNS3_PMU_EVT_PPS_RD_PAY_M0_PACKET_NUM		0x0010d
 166#define HNS3_PMU_EVT_PPS_RD_PAY_M0_TIME			0x1010d
 167#define HNS3_PMU_EVT_PPS_RD_PAY_M1_PACKET_NUM		0x0010e
 168#define HNS3_PMU_EVT_PPS_RD_PAY_M1_TIME			0x1010e
 169#define HNS3_PMU_EVT_PPS_WR_PAY_M0_PACKET_NUM		0x0010f
 170#define HNS3_PMU_EVT_PPS_WR_PAY_M0_TIME			0x1010f
 171#define HNS3_PMU_EVT_PPS_WR_PAY_M1_PACKET_NUM		0x00110
 172#define HNS3_PMU_EVT_PPS_WR_PAY_M1_TIME			0x10110
 173#define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_PACKET_NUM	0x00111
 174#define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_TIME		0x10111
 175#define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_PACKET_NUM	0x00112
 176#define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_TIME		0x10112
 177
 178/* latency events */
 179#define HNS3_PMU_EVT_DLY_TX_PUSH_TIME			0x00202
 180#define HNS3_PMU_EVT_DLY_TX_PUSH_PACKET_NUM		0x10202
 181#define HNS3_PMU_EVT_DLY_TX_TIME			0x00204
 182#define HNS3_PMU_EVT_DLY_TX_PACKET_NUM			0x10204
 183#define HNS3_PMU_EVT_DLY_SSU_TX_NIC_TIME		0x00206
 184#define HNS3_PMU_EVT_DLY_SSU_TX_NIC_PACKET_NUM		0x10206
 185#define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_TIME		0x00207
 186#define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_PACKET_NUM		0x10207
 187#define HNS3_PMU_EVT_DLY_SSU_RX_NIC_TIME		0x00208
 188#define HNS3_PMU_EVT_DLY_SSU_RX_NIC_PACKET_NUM		0x10208
 189#define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_TIME		0x00209
 190#define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_PACKET_NUM		0x10209
 191#define HNS3_PMU_EVT_DLY_RPU_TIME			0x0020e
 192#define HNS3_PMU_EVT_DLY_RPU_PACKET_NUM			0x1020e
 193#define HNS3_PMU_EVT_DLY_TPU_TIME			0x0020f
 194#define HNS3_PMU_EVT_DLY_TPU_PACKET_NUM			0x1020f
 195#define HNS3_PMU_EVT_DLY_RPE_TIME			0x00210
 196#define HNS3_PMU_EVT_DLY_RPE_PACKET_NUM			0x10210
 197#define HNS3_PMU_EVT_DLY_TPE_TIME			0x00211
 198#define HNS3_PMU_EVT_DLY_TPE_PACKET_NUM			0x10211
 199#define HNS3_PMU_EVT_DLY_TPE_PUSH_TIME			0x00212
 200#define HNS3_PMU_EVT_DLY_TPE_PUSH_PACKET_NUM		0x10212
 201#define HNS3_PMU_EVT_DLY_WR_FBD_TIME			0x00213
 202#define HNS3_PMU_EVT_DLY_WR_FBD_PACKET_NUM		0x10213
 203#define HNS3_PMU_EVT_DLY_WR_EBD_TIME			0x00214
 204#define HNS3_PMU_EVT_DLY_WR_EBD_PACKET_NUM		0x10214
 205#define HNS3_PMU_EVT_DLY_RD_FBD_TIME			0x00215
 206#define HNS3_PMU_EVT_DLY_RD_FBD_PACKET_NUM		0x10215
 207#define HNS3_PMU_EVT_DLY_RD_EBD_TIME			0x00216
 208#define HNS3_PMU_EVT_DLY_RD_EBD_PACKET_NUM		0x10216
 209#define HNS3_PMU_EVT_DLY_RD_PAY_M0_TIME			0x00217
 210#define HNS3_PMU_EVT_DLY_RD_PAY_M0_PACKET_NUM		0x10217
 211#define HNS3_PMU_EVT_DLY_RD_PAY_M1_TIME			0x00218
 212#define HNS3_PMU_EVT_DLY_RD_PAY_M1_PACKET_NUM		0x10218
 213#define HNS3_PMU_EVT_DLY_WR_PAY_M0_TIME			0x00219
 214#define HNS3_PMU_EVT_DLY_WR_PAY_M0_PACKET_NUM		0x10219
 215#define HNS3_PMU_EVT_DLY_WR_PAY_M1_TIME			0x0021a
 216#define HNS3_PMU_EVT_DLY_WR_PAY_M1_PACKET_NUM		0x1021a
 217#define HNS3_PMU_EVT_DLY_MSIX_WRITE_TIME		0x0021c
 218#define HNS3_PMU_EVT_DLY_MSIX_WRITE_PACKET_NUM		0x1021c
 219
 220/* interrupt rate events */
 221#define HNS3_PMU_EVT_PPS_MSIX_NIC_INTR_NUM		0x00300
 222#define HNS3_PMU_EVT_PPS_MSIX_NIC_TIME			0x10300
 223
 224/* filter mode supported by each bandwidth event */
 225#define HNS3_PMU_FILTER_BW_SSU_EGU		0x07
 226#define HNS3_PMU_FILTER_BW_SSU_RPU		0x1f
 227#define HNS3_PMU_FILTER_BW_SSU_ROCE		0x0f
 228#define HNS3_PMU_FILTER_BW_ROCE_SSU		0x0f
 229#define HNS3_PMU_FILTER_BW_TPU_SSU		0x1f
 230#define HNS3_PMU_FILTER_BW_RPU_RCBRX		0x11
 231#define HNS3_PMU_FILTER_BW_RCBTX_TXSCH		0x11
 232#define HNS3_PMU_FILTER_BW_WR_FBD		0x1b
 233#define HNS3_PMU_FILTER_BW_WR_EBD		0x11
 234#define HNS3_PMU_FILTER_BW_RD_FBD		0x01
 235#define HNS3_PMU_FILTER_BW_RD_EBD		0x1b
 236#define HNS3_PMU_FILTER_BW_RD_PAY_M0		0x01
 237#define HNS3_PMU_FILTER_BW_RD_PAY_M1		0x01
 238#define HNS3_PMU_FILTER_BW_WR_PAY_M0		0x01
 239#define HNS3_PMU_FILTER_BW_WR_PAY_M1		0x01
 240
 241/* filter mode supported by each packet rate event */
 242#define HNS3_PMU_FILTER_PPS_IGU_SSU		0x07
 243#define HNS3_PMU_FILTER_PPS_SSU_EGU		0x07
 244#define HNS3_PMU_FILTER_PPS_SSU_RPU		0x1f
 245#define HNS3_PMU_FILTER_PPS_SSU_ROCE		0x0f
 246#define HNS3_PMU_FILTER_PPS_ROCE_SSU		0x0f
 247#define HNS3_PMU_FILTER_PPS_TPU_SSU		0x1f
 248#define HNS3_PMU_FILTER_PPS_RPU_RCBRX		0x11
 249#define HNS3_PMU_FILTER_PPS_RCBTX_TPU		0x1f
 250#define HNS3_PMU_FILTER_PPS_RCBTX_TXSCH		0x11
 251#define HNS3_PMU_FILTER_PPS_WR_FBD		0x1b
 252#define HNS3_PMU_FILTER_PPS_WR_EBD		0x11
 253#define HNS3_PMU_FILTER_PPS_RD_FBD		0x01
 254#define HNS3_PMU_FILTER_PPS_RD_EBD		0x1b
 255#define HNS3_PMU_FILTER_PPS_RD_PAY_M0		0x01
 256#define HNS3_PMU_FILTER_PPS_RD_PAY_M1		0x01
 257#define HNS3_PMU_FILTER_PPS_WR_PAY_M0		0x01
 258#define HNS3_PMU_FILTER_PPS_WR_PAY_M1		0x01
 259#define HNS3_PMU_FILTER_PPS_NICROH_TX_PRE	0x01
 260#define HNS3_PMU_FILTER_PPS_NICROH_RX_PRE	0x01
 261
 262/* filter mode supported by each latency event */
 263#define HNS3_PMU_FILTER_DLY_TX_PUSH		0x01
 264#define HNS3_PMU_FILTER_DLY_TX			0x01
 265#define HNS3_PMU_FILTER_DLY_SSU_TX_NIC		0x07
 266#define HNS3_PMU_FILTER_DLY_SSU_TX_ROCE		0x07
 267#define HNS3_PMU_FILTER_DLY_SSU_RX_NIC		0x07
 268#define HNS3_PMU_FILTER_DLY_SSU_RX_ROCE		0x07
 269#define HNS3_PMU_FILTER_DLY_RPU			0x11
 270#define HNS3_PMU_FILTER_DLY_TPU			0x1f
 271#define HNS3_PMU_FILTER_DLY_RPE			0x01
 272#define HNS3_PMU_FILTER_DLY_TPE			0x0b
 273#define HNS3_PMU_FILTER_DLY_TPE_PUSH		0x1b
 274#define HNS3_PMU_FILTER_DLY_WR_FBD		0x1b
 275#define HNS3_PMU_FILTER_DLY_WR_EBD		0x11
 276#define HNS3_PMU_FILTER_DLY_RD_FBD		0x01
 277#define HNS3_PMU_FILTER_DLY_RD_EBD		0x1b
 278#define HNS3_PMU_FILTER_DLY_RD_PAY_M0		0x01
 279#define HNS3_PMU_FILTER_DLY_RD_PAY_M1		0x01
 280#define HNS3_PMU_FILTER_DLY_WR_PAY_M0		0x01
 281#define HNS3_PMU_FILTER_DLY_WR_PAY_M1		0x01
 282#define HNS3_PMU_FILTER_DLY_MSIX_WRITE		0x01
 283
 284/* filter mode supported by each interrupt rate event */
 285#define HNS3_PMU_FILTER_INTR_MSIX_NIC		0x01
 286
 287enum hns3_pmu_hw_filter_mode {
 288	HNS3_PMU_HW_FILTER_GLOBAL,
 289	HNS3_PMU_HW_FILTER_PORT,
 290	HNS3_PMU_HW_FILTER_PORT_TC,
 291	HNS3_PMU_HW_FILTER_FUNC,
 292	HNS3_PMU_HW_FILTER_FUNC_QUEUE,
 293	HNS3_PMU_HW_FILTER_FUNC_INTR,
 294};
 295
 296struct hns3_pmu_event_attr {
 297	u32 event;
 298	u16 filter_support;
 299};
 300
 301struct hns3_pmu {
 302	struct perf_event *hw_events[HNS3_PMU_MAX_HW_EVENTS];
 303	struct hlist_node node;
 304	struct pci_dev *pdev;
 305	struct pmu pmu;
 306	void __iomem *base;
 307	int irq;
 308	int on_cpu;
 309	u32 identifier;
 310	u32 hw_clk_freq; /* hardware clock frequency of PMU */
 311	/* maximum and minimum bdf allowed by PMU */
 312	u16 bdf_min;
 313	u16 bdf_max;
 314};
 315
 316#define to_hns3_pmu(p)  (container_of((p), struct hns3_pmu, pmu))
 317
 318#define GET_PCI_DEVFN(bdf)  ((bdf) & 0xff)
 319
 320#define FILTER_CONDITION_PORT(port) ((1 << (port)) & 0xff)
 321#define FILTER_CONDITION_PORT_TC(port, tc) (((port) << 3) | ((tc) & 0x07))
 322#define FILTER_CONDITION_FUNC_INTR(func, intr) (((intr) << 8) | (func))
 323
 324#define HNS3_PMU_FILTER_ATTR(_name, _config, _start, _end)               \
 325	static inline u64 hns3_pmu_get_##_name(struct perf_event *event) \
 326	{                                                                \
 327		return FIELD_GET(GENMASK_ULL(_end, _start),              \
 328				 event->attr._config);                   \
 329	}
 330
 331HNS3_PMU_FILTER_ATTR(subevent, config, 0, 7);
 332HNS3_PMU_FILTER_ATTR(event_type, config, 8, 15);
 333HNS3_PMU_FILTER_ATTR(ext_counter_used, config, 16, 16);
 334HNS3_PMU_FILTER_ATTR(port, config1, 0, 3);
 335HNS3_PMU_FILTER_ATTR(tc, config1, 4, 7);
 336HNS3_PMU_FILTER_ATTR(bdf, config1, 8, 23);
 337HNS3_PMU_FILTER_ATTR(queue, config1, 24, 39);
 338HNS3_PMU_FILTER_ATTR(intr, config1, 40, 51);
 339HNS3_PMU_FILTER_ATTR(global, config1, 52, 52);
 340
 341#define HNS3_BW_EVT_BYTE_NUM(_name)	(&(struct hns3_pmu_event_attr) {\
 342	HNS3_PMU_EVT_BW_##_name##_BYTE_NUM,				\
 343	HNS3_PMU_FILTER_BW_##_name})
 344#define HNS3_BW_EVT_TIME(_name)		(&(struct hns3_pmu_event_attr) {\
 345	HNS3_PMU_EVT_BW_##_name##_TIME,					\
 346	HNS3_PMU_FILTER_BW_##_name})
 347#define HNS3_PPS_EVT_PACKET_NUM(_name)	(&(struct hns3_pmu_event_attr) {\
 348	HNS3_PMU_EVT_PPS_##_name##_PACKET_NUM,				\
 349	HNS3_PMU_FILTER_PPS_##_name})
 350#define HNS3_PPS_EVT_TIME(_name)	(&(struct hns3_pmu_event_attr) {\
 351	HNS3_PMU_EVT_PPS_##_name##_TIME,				\
 352	HNS3_PMU_FILTER_PPS_##_name})
 353#define HNS3_DLY_EVT_TIME(_name)	(&(struct hns3_pmu_event_attr) {\
 354	HNS3_PMU_EVT_DLY_##_name##_TIME,				\
 355	HNS3_PMU_FILTER_DLY_##_name})
 356#define HNS3_DLY_EVT_PACKET_NUM(_name)	(&(struct hns3_pmu_event_attr) {\
 357	HNS3_PMU_EVT_DLY_##_name##_PACKET_NUM,				\
 358	HNS3_PMU_FILTER_DLY_##_name})
 359#define HNS3_INTR_EVT_INTR_NUM(_name)	(&(struct hns3_pmu_event_attr) {\
 360	HNS3_PMU_EVT_PPS_##_name##_INTR_NUM,				\
 361	HNS3_PMU_FILTER_INTR_##_name})
 362#define HNS3_INTR_EVT_TIME(_name)	(&(struct hns3_pmu_event_attr) {\
 363	HNS3_PMU_EVT_PPS_##_name##_TIME,				\
 364	HNS3_PMU_FILTER_INTR_##_name})
 365
 366static ssize_t hns3_pmu_event_show(struct device *dev,
 367				   struct device_attribute *attr, char *buf)
 368{
 369	struct hns3_pmu_event_attr *event;
 370	struct dev_ext_attribute *eattr;
 371
 372	eattr = container_of(attr, struct dev_ext_attribute, attr);
 373	event = eattr->var;
 374
 375	return sysfs_emit(buf, "config=0x%x\n", event->event);
 376}
 377
 378static ssize_t hns3_pmu_filter_mode_show(struct device *dev,
 379					 struct device_attribute *attr,
 380					 char *buf)
 381{
 382	struct hns3_pmu_event_attr *event;
 383	struct dev_ext_attribute *eattr;
 384	int len;
 385
 386	eattr = container_of(attr, struct dev_ext_attribute, attr);
 387	event = eattr->var;
 388
 389	len = sysfs_emit_at(buf, 0, "filter mode supported: ");
 390	if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL)
 391		len += sysfs_emit_at(buf, len, "global ");
 392	if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT)
 393		len += sysfs_emit_at(buf, len, "port ");
 394	if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC)
 395		len += sysfs_emit_at(buf, len, "port-tc ");
 396	if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC)
 397		len += sysfs_emit_at(buf, len, "func ");
 398	if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE)
 399		len += sysfs_emit_at(buf, len, "func-queue ");
 400	if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR)
 401		len += sysfs_emit_at(buf, len, "func-intr ");
 402
 403	len += sysfs_emit_at(buf, len, "\n");
 404
 405	return len;
 406}
 407
 408#define HNS3_PMU_ATTR(_name, _func, _config)				\
 409	(&((struct dev_ext_attribute[]) {				\
 410		{ __ATTR(_name, 0444, _func, NULL), (void *)_config }	\
 411	})[0].attr.attr)
 412
 413#define HNS3_PMU_FORMAT_ATTR(_name, _format) \
 414	HNS3_PMU_ATTR(_name, device_show_string, _format)
 415#define HNS3_PMU_EVENT_ATTR(_name, _event) \
 416	HNS3_PMU_ATTR(_name, hns3_pmu_event_show, (void *)_event)
 417#define HNS3_PMU_FLT_MODE_ATTR(_name, _event) \
 418	HNS3_PMU_ATTR(_name, hns3_pmu_filter_mode_show, (void *)_event)
 419
 420#define HNS3_PMU_BW_EVT_PAIR(_name, _macro) \
 421	HNS3_PMU_EVENT_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \
 422	HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro))
 423#define HNS3_PMU_PPS_EVT_PAIR(_name, _macro) \
 424	HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \
 425	HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro))
 426#define HNS3_PMU_DLY_EVT_PAIR(_name, _macro) \
 427	HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \
 428	HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro))
 429#define HNS3_PMU_INTR_EVT_PAIR(_name, _macro) \
 430	HNS3_PMU_EVENT_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \
 431	HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro))
 432
 433#define HNS3_PMU_BW_FLT_MODE_PAIR(_name, _macro) \
 434	HNS3_PMU_FLT_MODE_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \
 435	HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro))
 436#define HNS3_PMU_PPS_FLT_MODE_PAIR(_name, _macro) \
 437	HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \
 438	HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro))
 439#define HNS3_PMU_DLY_FLT_MODE_PAIR(_name, _macro) \
 440	HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \
 441	HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro))
 442#define HNS3_PMU_INTR_FLT_MODE_PAIR(_name, _macro) \
 443	HNS3_PMU_FLT_MODE_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \
 444	HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro))
 445
 446static u8 hns3_pmu_hw_filter_modes[] = {
 447	HNS3_PMU_HW_FILTER_GLOBAL,
 448	HNS3_PMU_HW_FILTER_PORT,
 449	HNS3_PMU_HW_FILTER_PORT_TC,
 450	HNS3_PMU_HW_FILTER_FUNC,
 451	HNS3_PMU_HW_FILTER_FUNC_QUEUE,
 452	HNS3_PMU_HW_FILTER_FUNC_INTR,
 453};
 454
 455#define HNS3_PMU_SET_HW_FILTER(_hwc, _mode) \
 456	((_hwc)->addr_filters = (void *)&hns3_pmu_hw_filter_modes[(_mode)])
 457
 458static ssize_t identifier_show(struct device *dev,
 459			       struct device_attribute *attr, char *buf)
 460{
 461	struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
 462
 463	return sysfs_emit(buf, "0x%x\n", hns3_pmu->identifier);
 464}
 465static DEVICE_ATTR_RO(identifier);
 466
 467static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
 468			    char *buf)
 469{
 470	struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
 471
 472	return sysfs_emit(buf, "%d\n", hns3_pmu->on_cpu);
 473}
 474static DEVICE_ATTR_RO(cpumask);
 475
 476static ssize_t bdf_min_show(struct device *dev, struct device_attribute *attr,
 477			    char *buf)
 478{
 479	struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
 480	u16 bdf = hns3_pmu->bdf_min;
 481
 482	return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf),
 483			  PCI_SLOT(bdf), PCI_FUNC(bdf));
 484}
 485static DEVICE_ATTR_RO(bdf_min);
 486
 487static ssize_t bdf_max_show(struct device *dev, struct device_attribute *attr,
 488			    char *buf)
 489{
 490	struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
 491	u16 bdf = hns3_pmu->bdf_max;
 492
 493	return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf),
 494			  PCI_SLOT(bdf), PCI_FUNC(bdf));
 495}
 496static DEVICE_ATTR_RO(bdf_max);
 497
 498static ssize_t hw_clk_freq_show(struct device *dev,
 499				struct device_attribute *attr, char *buf)
 500{
 501	struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
 502
 503	return sysfs_emit(buf, "%u\n", hns3_pmu->hw_clk_freq);
 504}
 505static DEVICE_ATTR_RO(hw_clk_freq);
 506
 507static struct attribute *hns3_pmu_events_attr[] = {
 508	/* bandwidth events */
 509	HNS3_PMU_BW_EVT_PAIR(bw_ssu_egu, SSU_EGU),
 510	HNS3_PMU_BW_EVT_PAIR(bw_ssu_rpu, SSU_RPU),
 511	HNS3_PMU_BW_EVT_PAIR(bw_ssu_roce, SSU_ROCE),
 512	HNS3_PMU_BW_EVT_PAIR(bw_roce_ssu, ROCE_SSU),
 513	HNS3_PMU_BW_EVT_PAIR(bw_tpu_ssu, TPU_SSU),
 514	HNS3_PMU_BW_EVT_PAIR(bw_rpu_rcbrx, RPU_RCBRX),
 515	HNS3_PMU_BW_EVT_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH),
 516	HNS3_PMU_BW_EVT_PAIR(bw_wr_fbd, WR_FBD),
 517	HNS3_PMU_BW_EVT_PAIR(bw_wr_ebd, WR_EBD),
 518	HNS3_PMU_BW_EVT_PAIR(bw_rd_fbd, RD_FBD),
 519	HNS3_PMU_BW_EVT_PAIR(bw_rd_ebd, RD_EBD),
 520	HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m0, RD_PAY_M0),
 521	HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m1, RD_PAY_M1),
 522	HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m0, WR_PAY_M0),
 523	HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m1, WR_PAY_M1),
 524
 525	/* packet rate events */
 526	HNS3_PMU_PPS_EVT_PAIR(pps_igu_ssu, IGU_SSU),
 527	HNS3_PMU_PPS_EVT_PAIR(pps_ssu_egu, SSU_EGU),
 528	HNS3_PMU_PPS_EVT_PAIR(pps_ssu_rpu, SSU_RPU),
 529	HNS3_PMU_PPS_EVT_PAIR(pps_ssu_roce, SSU_ROCE),
 530	HNS3_PMU_PPS_EVT_PAIR(pps_roce_ssu, ROCE_SSU),
 531	HNS3_PMU_PPS_EVT_PAIR(pps_tpu_ssu, TPU_SSU),
 532	HNS3_PMU_PPS_EVT_PAIR(pps_rpu_rcbrx, RPU_RCBRX),
 533	HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_tpu, RCBTX_TPU),
 534	HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH),
 535	HNS3_PMU_PPS_EVT_PAIR(pps_wr_fbd, WR_FBD),
 536	HNS3_PMU_PPS_EVT_PAIR(pps_wr_ebd, WR_EBD),
 537	HNS3_PMU_PPS_EVT_PAIR(pps_rd_fbd, RD_FBD),
 538	HNS3_PMU_PPS_EVT_PAIR(pps_rd_ebd, RD_EBD),
 539	HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m0, RD_PAY_M0),
 540	HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m1, RD_PAY_M1),
 541	HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m0, WR_PAY_M0),
 542	HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m1, WR_PAY_M1),
 543	HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE),
 544	HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE),
 545
 546	/* latency events */
 547	HNS3_PMU_DLY_EVT_PAIR(dly_tx_push_to_mac, TX_PUSH),
 548	HNS3_PMU_DLY_EVT_PAIR(dly_tx_normal_to_mac, TX),
 549	HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC),
 550	HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE),
 551	HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC),
 552	HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE),
 553	HNS3_PMU_DLY_EVT_PAIR(dly_rpu, RPU),
 554	HNS3_PMU_DLY_EVT_PAIR(dly_tpu, TPU),
 555	HNS3_PMU_DLY_EVT_PAIR(dly_rpe, RPE),
 556	HNS3_PMU_DLY_EVT_PAIR(dly_tpe_normal, TPE),
 557	HNS3_PMU_DLY_EVT_PAIR(dly_tpe_push, TPE_PUSH),
 558	HNS3_PMU_DLY_EVT_PAIR(dly_wr_fbd, WR_FBD),
 559	HNS3_PMU_DLY_EVT_PAIR(dly_wr_ebd, WR_EBD),
 560	HNS3_PMU_DLY_EVT_PAIR(dly_rd_fbd, RD_FBD),
 561	HNS3_PMU_DLY_EVT_PAIR(dly_rd_ebd, RD_EBD),
 562	HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m0, RD_PAY_M0),
 563	HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m1, RD_PAY_M1),
 564	HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m0, WR_PAY_M0),
 565	HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m1, WR_PAY_M1),
 566	HNS3_PMU_DLY_EVT_PAIR(dly_msix_write, MSIX_WRITE),
 567
 568	/* interrupt rate events */
 569	HNS3_PMU_INTR_EVT_PAIR(pps_intr_msix_nic, MSIX_NIC),
 570
 571	NULL
 572};
 573
 574static struct attribute *hns3_pmu_filter_mode_attr[] = {
 575	/* bandwidth events */
 576	HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_egu, SSU_EGU),
 577	HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_rpu, SSU_RPU),
 578	HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_roce, SSU_ROCE),
 579	HNS3_PMU_BW_FLT_MODE_PAIR(bw_roce_ssu, ROCE_SSU),
 580	HNS3_PMU_BW_FLT_MODE_PAIR(bw_tpu_ssu, TPU_SSU),
 581	HNS3_PMU_BW_FLT_MODE_PAIR(bw_rpu_rcbrx, RPU_RCBRX),
 582	HNS3_PMU_BW_FLT_MODE_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH),
 583	HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_fbd, WR_FBD),
 584	HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_ebd, WR_EBD),
 585	HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_fbd, RD_FBD),
 586	HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_ebd, RD_EBD),
 587	HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m0, RD_PAY_M0),
 588	HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m1, RD_PAY_M1),
 589	HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m0, WR_PAY_M0),
 590	HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m1, WR_PAY_M1),
 591
 592	/* packet rate events */
 593	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_igu_ssu, IGU_SSU),
 594	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_egu, SSU_EGU),
 595	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_rpu, SSU_RPU),
 596	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_roce, SSU_ROCE),
 597	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_roce_ssu, ROCE_SSU),
 598	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_tpu_ssu, TPU_SSU),
 599	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rpu_rcbrx, RPU_RCBRX),
 600	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_tpu, RCBTX_TPU),
 601	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH),
 602	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_fbd, WR_FBD),
 603	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_ebd, WR_EBD),
 604	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_fbd, RD_FBD),
 605	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_ebd, RD_EBD),
 606	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m0, RD_PAY_M0),
 607	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m1, RD_PAY_M1),
 608	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m0, WR_PAY_M0),
 609	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m1, WR_PAY_M1),
 610	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE),
 611	HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE),
 612
 613	/* latency events */
 614	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_push_to_mac, TX_PUSH),
 615	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_normal_to_mac, TX),
 616	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC),
 617	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE),
 618	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC),
 619	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE),
 620	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpu, RPU),
 621	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpu, TPU),
 622	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpe, RPE),
 623	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_normal, TPE),
 624	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_push, TPE_PUSH),
 625	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_fbd, WR_FBD),
 626	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_ebd, WR_EBD),
 627	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_fbd, RD_FBD),
 628	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_ebd, RD_EBD),
 629	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m0, RD_PAY_M0),
 630	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m1, RD_PAY_M1),
 631	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m0, WR_PAY_M0),
 632	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m1, WR_PAY_M1),
 633	HNS3_PMU_DLY_FLT_MODE_PAIR(dly_msix_write, MSIX_WRITE),
 634
 635	/* interrupt rate events */
 636	HNS3_PMU_INTR_FLT_MODE_PAIR(pps_intr_msix_nic, MSIX_NIC),
 637
 638	NULL
 639};
 640
 641static struct attribute_group hns3_pmu_events_group = {
 642	.name = "events",
 643	.attrs = hns3_pmu_events_attr,
 644};
 645
 646static struct attribute_group hns3_pmu_filter_mode_group = {
 647	.name = "filtermode",
 648	.attrs = hns3_pmu_filter_mode_attr,
 649};
 650
 651static struct attribute *hns3_pmu_format_attr[] = {
 652	HNS3_PMU_FORMAT_ATTR(subevent, "config:0-7"),
 653	HNS3_PMU_FORMAT_ATTR(event_type, "config:8-15"),
 654	HNS3_PMU_FORMAT_ATTR(ext_counter_used, "config:16"),
 655	HNS3_PMU_FORMAT_ATTR(port, "config1:0-3"),
 656	HNS3_PMU_FORMAT_ATTR(tc, "config1:4-7"),
 657	HNS3_PMU_FORMAT_ATTR(bdf, "config1:8-23"),
 658	HNS3_PMU_FORMAT_ATTR(queue, "config1:24-39"),
 659	HNS3_PMU_FORMAT_ATTR(intr, "config1:40-51"),
 660	HNS3_PMU_FORMAT_ATTR(global, "config1:52"),
 661	NULL
 662};
 663
 664static struct attribute_group hns3_pmu_format_group = {
 665	.name = "format",
 666	.attrs = hns3_pmu_format_attr,
 667};
 668
 669static struct attribute *hns3_pmu_cpumask_attrs[] = {
 670	&dev_attr_cpumask.attr,
 671	NULL
 672};
 673
 674static struct attribute_group hns3_pmu_cpumask_attr_group = {
 675	.attrs = hns3_pmu_cpumask_attrs,
 676};
 677
 678static struct attribute *hns3_pmu_identifier_attrs[] = {
 679	&dev_attr_identifier.attr,
 680	NULL
 681};
 682
 683static struct attribute_group hns3_pmu_identifier_attr_group = {
 684	.attrs = hns3_pmu_identifier_attrs,
 685};
 686
 687static struct attribute *hns3_pmu_bdf_range_attrs[] = {
 688	&dev_attr_bdf_min.attr,
 689	&dev_attr_bdf_max.attr,
 690	NULL
 691};
 692
 693static struct attribute_group hns3_pmu_bdf_range_attr_group = {
 694	.attrs = hns3_pmu_bdf_range_attrs,
 695};
 696
 697static struct attribute *hns3_pmu_hw_clk_freq_attrs[] = {
 698	&dev_attr_hw_clk_freq.attr,
 699	NULL
 700};
 701
 702static struct attribute_group hns3_pmu_hw_clk_freq_attr_group = {
 703	.attrs = hns3_pmu_hw_clk_freq_attrs,
 704};
 705
 706static const struct attribute_group *hns3_pmu_attr_groups[] = {
 707	&hns3_pmu_events_group,
 708	&hns3_pmu_filter_mode_group,
 709	&hns3_pmu_format_group,
 710	&hns3_pmu_cpumask_attr_group,
 711	&hns3_pmu_identifier_attr_group,
 712	&hns3_pmu_bdf_range_attr_group,
 713	&hns3_pmu_hw_clk_freq_attr_group,
 714	NULL
 715};
 716
 717static u32 hns3_pmu_get_event(struct perf_event *event)
 718{
 719	return hns3_pmu_get_ext_counter_used(event) << 16 |
 720	       hns3_pmu_get_event_type(event) << 8 |
 721	       hns3_pmu_get_subevent(event);
 722}
 723
 724static u32 hns3_pmu_get_real_event(struct perf_event *event)
 725{
 726	return hns3_pmu_get_event_type(event) << 8 |
 727	       hns3_pmu_get_subevent(event);
 728}
 729
 730static u32 hns3_pmu_get_offset(u32 offset, u32 idx)
 731{
 732	return offset + HNS3_PMU_REG_EVENT_OFFSET +
 733	       HNS3_PMU_REG_EVENT_SIZE * idx;
 734}
 735
 736static u32 hns3_pmu_readl(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx)
 737{
 738	u32 offset = hns3_pmu_get_offset(reg_offset, idx);
 739
 740	return readl(hns3_pmu->base + offset);
 741}
 742
 743static void hns3_pmu_writel(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx,
 744			    u32 val)
 745{
 746	u32 offset = hns3_pmu_get_offset(reg_offset, idx);
 747
 748	writel(val, hns3_pmu->base + offset);
 749}
 750
 751static u64 hns3_pmu_readq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx)
 752{
 753	u32 offset = hns3_pmu_get_offset(reg_offset, idx);
 754
 755	return readq(hns3_pmu->base + offset);
 756}
 757
 758static void hns3_pmu_writeq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx,
 759			    u64 val)
 760{
 761	u32 offset = hns3_pmu_get_offset(reg_offset, idx);
 762
 763	writeq(val, hns3_pmu->base + offset);
 764}
 765
 766static bool hns3_pmu_cmp_event(struct perf_event *target,
 767			       struct perf_event *event)
 768{
 769	return hns3_pmu_get_real_event(target) == hns3_pmu_get_real_event(event);
 770}
 771
 772static int hns3_pmu_find_related_event_idx(struct hns3_pmu *hns3_pmu,
 773					   struct perf_event *event)
 774{
 775	struct perf_event *sibling;
 776	int hw_event_used = 0;
 777	int idx;
 778
 779	for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
 780		sibling = hns3_pmu->hw_events[idx];
 781		if (!sibling)
 782			continue;
 783
 784		hw_event_used++;
 785
 786		if (!hns3_pmu_cmp_event(sibling, event))
 787			continue;
 788
 789		/* Related events is used in group */
 790		if (sibling->group_leader == event->group_leader)
 791			return idx;
 792	}
 793
 794	/* No related event and all hardware events are used up */
 795	if (hw_event_used >= HNS3_PMU_MAX_HW_EVENTS)
 796		return -EBUSY;
 797
 798	/* No related event and there is extra hardware events can be use */
 799	return -ENOENT;
 800}
 801
 802static int hns3_pmu_get_event_idx(struct hns3_pmu *hns3_pmu)
 803{
 804	int idx;
 805
 806	for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
 807		if (!hns3_pmu->hw_events[idx])
 808			return idx;
 809	}
 810
 811	return -EBUSY;
 812}
 813
 814static bool hns3_pmu_valid_bdf(struct hns3_pmu *hns3_pmu, u16 bdf)
 815{
 816	struct pci_dev *pdev;
 817
 818	if (bdf < hns3_pmu->bdf_min || bdf > hns3_pmu->bdf_max) {
 819		pci_err(hns3_pmu->pdev, "Invalid EP device: %#x!\n", bdf);
 820		return false;
 821	}
 822
 823	pdev = pci_get_domain_bus_and_slot(pci_domain_nr(hns3_pmu->pdev->bus),
 824					   PCI_BUS_NUM(bdf),
 825					   GET_PCI_DEVFN(bdf));
 826	if (!pdev) {
 827		pci_err(hns3_pmu->pdev, "Nonexistent EP device: %#x!\n", bdf);
 828		return false;
 829	}
 830
 831	pci_dev_put(pdev);
 832	return true;
 833}
 834
 835static void hns3_pmu_set_qid_para(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf,
 836				  u16 queue)
 837{
 838	u32 val;
 839
 840	val = GET_PCI_DEVFN(bdf);
 841	val |= (u32)queue << HNS3_PMU_QID_PARA_QUEUE_S;
 842	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_PARA, idx, val);
 843}
 844
 845static bool hns3_pmu_qid_req_start(struct hns3_pmu *hns3_pmu, u32 idx)
 846{
 847	bool queue_id_valid = false;
 848	u32 reg_qid_ctrl, val;
 849	int err;
 850
 851	/* enable queue id request */
 852	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx,
 853			HNS3_PMU_QID_CTRL_REQ_ENABLE);
 854
 855	reg_qid_ctrl = hns3_pmu_get_offset(HNS3_PMU_REG_EVENT_QID_CTRL, idx);
 856	err = readl_poll_timeout(hns3_pmu->base + reg_qid_ctrl, val,
 857				 val & HNS3_PMU_QID_CTRL_DONE, 1, 1000);
 858	if (err == -ETIMEDOUT) {
 859		pci_err(hns3_pmu->pdev, "QID request timeout!\n");
 860		goto out;
 861	}
 862
 863	queue_id_valid = !(val & HNS3_PMU_QID_CTRL_MISS);
 864
 865out:
 866	/* disable qid request and clear status */
 867	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx, 0);
 868
 869	return queue_id_valid;
 870}
 871
 872static bool hns3_pmu_valid_queue(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf,
 873				 u16 queue)
 874{
 875	hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue);
 876
 877	return hns3_pmu_qid_req_start(hns3_pmu, idx);
 878}
 879
 880static struct hns3_pmu_event_attr *hns3_pmu_get_pmu_event(u32 event)
 881{
 882	struct hns3_pmu_event_attr *pmu_event;
 883	struct dev_ext_attribute *eattr;
 884	struct device_attribute *dattr;
 885	struct attribute *attr;
 886	u32 i;
 887
 888	for (i = 0; i < ARRAY_SIZE(hns3_pmu_events_attr) - 1; i++) {
 889		attr = hns3_pmu_events_attr[i];
 890		dattr = container_of(attr, struct device_attribute, attr);
 891		eattr = container_of(dattr, struct dev_ext_attribute, attr);
 892		pmu_event = eattr->var;
 893
 894		if (event == pmu_event->event)
 895			return pmu_event;
 896	}
 897
 898	return NULL;
 899}
 900
 901static int hns3_pmu_set_func_mode(struct perf_event *event,
 902				  struct hns3_pmu *hns3_pmu)
 903{
 904	struct hw_perf_event *hwc = &event->hw;
 905	u16 bdf = hns3_pmu_get_bdf(event);
 906
 907	if (!hns3_pmu_valid_bdf(hns3_pmu, bdf))
 908		return -ENOENT;
 909
 910	HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC);
 911
 912	return 0;
 913}
 914
 915static int hns3_pmu_set_func_queue_mode(struct perf_event *event,
 916					struct hns3_pmu *hns3_pmu)
 917{
 918	u16 queue_id = hns3_pmu_get_queue(event);
 919	struct hw_perf_event *hwc = &event->hw;
 920	u16 bdf = hns3_pmu_get_bdf(event);
 921
 922	if (!hns3_pmu_valid_bdf(hns3_pmu, bdf))
 923		return -ENOENT;
 924
 925	if (!hns3_pmu_valid_queue(hns3_pmu, hwc->idx, bdf, queue_id)) {
 926		pci_err(hns3_pmu->pdev, "Invalid queue: %u\n", queue_id);
 927		return -ENOENT;
 928	}
 929
 930	HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_QUEUE);
 931
 932	return 0;
 933}
 934
 935static bool
 936hns3_pmu_is_enabled_global_mode(struct perf_event *event,
 937				struct hns3_pmu_event_attr *pmu_event)
 938{
 939	u8 global = hns3_pmu_get_global(event);
 940
 941	if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL))
 942		return false;
 943
 944	return global;
 945}
 946
 947static bool hns3_pmu_is_enabled_func_mode(struct perf_event *event,
 948					  struct hns3_pmu_event_attr *pmu_event)
 949{
 950	u16 queue_id = hns3_pmu_get_queue(event);
 951	u16 bdf = hns3_pmu_get_bdf(event);
 952
 953	if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC))
 954		return false;
 955	else if (queue_id != HNS3_PMU_FILTER_ALL_QUEUE)
 956		return false;
 957
 958	return bdf;
 959}
 960
 961static bool
 962hns3_pmu_is_enabled_func_queue_mode(struct perf_event *event,
 963				    struct hns3_pmu_event_attr *pmu_event)
 964{
 965	u16 queue_id = hns3_pmu_get_queue(event);
 966	u16 bdf = hns3_pmu_get_bdf(event);
 967
 968	if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE))
 969		return false;
 970	else if (queue_id == HNS3_PMU_FILTER_ALL_QUEUE)
 971		return false;
 972
 973	return bdf;
 974}
 975
 976static bool hns3_pmu_is_enabled_port_mode(struct perf_event *event,
 977					  struct hns3_pmu_event_attr *pmu_event)
 978{
 979	u8 tc_id = hns3_pmu_get_tc(event);
 980
 981	if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT))
 982		return false;
 983
 984	return tc_id == HNS3_PMU_FILTER_ALL_TC;
 985}
 986
 987static bool
 988hns3_pmu_is_enabled_port_tc_mode(struct perf_event *event,
 989				 struct hns3_pmu_event_attr *pmu_event)
 990{
 991	u8 tc_id = hns3_pmu_get_tc(event);
 992
 993	if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC))
 994		return false;
 995
 996	return tc_id != HNS3_PMU_FILTER_ALL_TC;
 997}
 998
 999static bool
1000hns3_pmu_is_enabled_func_intr_mode(struct perf_event *event,
1001				   struct hns3_pmu *hns3_pmu,
1002				   struct hns3_pmu_event_attr *pmu_event)
1003{
1004	u16 bdf = hns3_pmu_get_bdf(event);
1005
1006	if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR))
1007		return false;
1008
1009	return hns3_pmu_valid_bdf(hns3_pmu, bdf);
1010}
1011
1012static int hns3_pmu_select_filter_mode(struct perf_event *event,
1013				       struct hns3_pmu *hns3_pmu)
1014{
1015	u32 event_id = hns3_pmu_get_event(event);
1016	struct hw_perf_event *hwc = &event->hw;
1017	struct hns3_pmu_event_attr *pmu_event;
1018
1019	pmu_event = hns3_pmu_get_pmu_event(event_id);
1020	if (!pmu_event) {
1021		pci_err(hns3_pmu->pdev, "Invalid pmu event\n");
1022		return -ENOENT;
1023	}
1024
1025	if (hns3_pmu_is_enabled_global_mode(event, pmu_event)) {
1026		HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_GLOBAL);
1027		return 0;
1028	}
1029
1030	if (hns3_pmu_is_enabled_func_mode(event, pmu_event))
1031		return hns3_pmu_set_func_mode(event, hns3_pmu);
1032
1033	if (hns3_pmu_is_enabled_func_queue_mode(event, pmu_event))
1034		return hns3_pmu_set_func_queue_mode(event, hns3_pmu);
1035
1036	if (hns3_pmu_is_enabled_port_mode(event, pmu_event)) {
1037		HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT);
1038		return 0;
1039	}
1040
1041	if (hns3_pmu_is_enabled_port_tc_mode(event, pmu_event)) {
1042		HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT_TC);
1043		return 0;
1044	}
1045
1046	if (hns3_pmu_is_enabled_func_intr_mode(event, hns3_pmu, pmu_event)) {
1047		HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_INTR);
1048		return 0;
1049	}
1050
1051	return -ENOENT;
1052}
1053
1054static bool hns3_pmu_validate_event_group(struct perf_event *event)
1055{
1056	struct perf_event *sibling, *leader = event->group_leader;
1057	struct perf_event *event_group[HNS3_PMU_MAX_HW_EVENTS];
1058	int counters = 1;
1059	int num;
1060
1061	event_group[0] = leader;
1062	if (!is_software_event(leader)) {
1063		if (leader->pmu != event->pmu)
1064			return false;
1065
1066		if (leader != event && !hns3_pmu_cmp_event(leader, event))
1067			event_group[counters++] = event;
1068	}
1069
1070	for_each_sibling_event(sibling, event->group_leader) {
1071		if (is_software_event(sibling))
1072			continue;
1073
1074		if (sibling->pmu != event->pmu)
1075			return false;
1076
1077		for (num = 0; num < counters; num++) {
1078			/*
1079			 * If we find a related event, then it's a valid group
1080			 * since we don't need to allocate a new counter for it.
1081			 */
1082			if (hns3_pmu_cmp_event(event_group[num], sibling))
1083				break;
1084		}
1085
1086		/*
1087		 * Otherwise it's a new event but if there's no available counter,
1088		 * fail the check since we cannot schedule all the events in
1089		 * the group simultaneously.
1090		 */
1091		if (num == HNS3_PMU_MAX_HW_EVENTS)
1092			return false;
1093
1094		if (num == counters)
1095			event_group[counters++] = sibling;
1096	}
1097
1098	return true;
1099}
1100
1101static u32 hns3_pmu_get_filter_condition(struct perf_event *event)
1102{
1103	struct hw_perf_event *hwc = &event->hw;
1104	u16 intr_id = hns3_pmu_get_intr(event);
1105	u8 port_id = hns3_pmu_get_port(event);
1106	u16 bdf = hns3_pmu_get_bdf(event);
1107	u8 tc_id = hns3_pmu_get_tc(event);
1108	u8 filter_mode;
1109
1110	filter_mode = *(u8 *)hwc->addr_filters;
1111	switch (filter_mode) {
1112	case HNS3_PMU_HW_FILTER_PORT:
1113		return FILTER_CONDITION_PORT(port_id);
1114	case HNS3_PMU_HW_FILTER_PORT_TC:
1115		return FILTER_CONDITION_PORT_TC(port_id, tc_id);
1116	case HNS3_PMU_HW_FILTER_FUNC:
1117	case HNS3_PMU_HW_FILTER_FUNC_QUEUE:
1118		return GET_PCI_DEVFN(bdf);
1119	case HNS3_PMU_HW_FILTER_FUNC_INTR:
1120		return FILTER_CONDITION_FUNC_INTR(GET_PCI_DEVFN(bdf), intr_id);
1121	default:
1122		break;
1123	}
1124
1125	return 0;
1126}
1127
1128static void hns3_pmu_config_filter(struct perf_event *event)
1129{
1130	struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1131	u8 event_type = hns3_pmu_get_event_type(event);
1132	u8 subevent_id = hns3_pmu_get_subevent(event);
1133	u16 queue_id = hns3_pmu_get_queue(event);
1134	struct hw_perf_event *hwc = &event->hw;
1135	u8 filter_mode = *(u8 *)hwc->addr_filters;
1136	u16 bdf = hns3_pmu_get_bdf(event);
1137	u32 idx = hwc->idx;
1138	u32 val;
1139
1140	val = event_type;
1141	val |= subevent_id << HNS3_PMU_CTRL_SUBEVENT_S;
1142	val |= filter_mode << HNS3_PMU_CTRL_FILTER_MODE_S;
1143	val |= HNS3_PMU_EVENT_OVERFLOW_RESTART;
1144	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1145
1146	val = hns3_pmu_get_filter_condition(event);
1147	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_HIGH, idx, val);
1148
1149	if (filter_mode == HNS3_PMU_HW_FILTER_FUNC_QUEUE)
1150		hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue_id);
1151}
1152
1153static void hns3_pmu_enable_counter(struct hns3_pmu *hns3_pmu,
1154				    struct hw_perf_event *hwc)
1155{
1156	u32 idx = hwc->idx;
1157	u32 val;
1158
1159	val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
1160	val |= HNS3_PMU_EVENT_EN;
1161	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1162}
1163
1164static void hns3_pmu_disable_counter(struct hns3_pmu *hns3_pmu,
1165				     struct hw_perf_event *hwc)
1166{
1167	u32 idx = hwc->idx;
1168	u32 val;
1169
1170	val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
1171	val &= ~HNS3_PMU_EVENT_EN;
1172	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1173}
1174
1175static void hns3_pmu_enable_intr(struct hns3_pmu *hns3_pmu,
1176				 struct hw_perf_event *hwc)
1177{
1178	u32 idx = hwc->idx;
1179	u32 val;
1180
1181	val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx);
1182	val &= ~HNS3_PMU_INTR_MASK_OVERFLOW;
1183	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val);
1184}
1185
1186static void hns3_pmu_disable_intr(struct hns3_pmu *hns3_pmu,
1187				  struct hw_perf_event *hwc)
1188{
1189	u32 idx = hwc->idx;
1190	u32 val;
1191
1192	val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx);
1193	val |= HNS3_PMU_INTR_MASK_OVERFLOW;
1194	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val);
1195}
1196
1197static void hns3_pmu_clear_intr_status(struct hns3_pmu *hns3_pmu, u32 idx)
1198{
1199	u32 val;
1200
1201	val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
1202	val |= HNS3_PMU_EVENT_STATUS_RESET;
1203	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1204
1205	val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
1206	val &= ~HNS3_PMU_EVENT_STATUS_RESET;
1207	hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1208}
1209
1210static u64 hns3_pmu_read_counter(struct perf_event *event)
1211{
1212	struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1213
1214	return hns3_pmu_readq(hns3_pmu, event->hw.event_base, event->hw.idx);
1215}
1216
1217static void hns3_pmu_write_counter(struct perf_event *event, u64 value)
1218{
1219	struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1220	u32 idx = event->hw.idx;
1221
1222	hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_COUNTER, idx, value);
1223	hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_EXT_COUNTER, idx, value);
1224}
1225
1226static void hns3_pmu_init_counter(struct perf_event *event)
1227{
1228	struct hw_perf_event *hwc = &event->hw;
1229
1230	local64_set(&hwc->prev_count, 0);
1231	hns3_pmu_write_counter(event, 0);
1232}
1233
1234static int hns3_pmu_event_init(struct perf_event *event)
1235{
1236	struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1237	struct hw_perf_event *hwc = &event->hw;
1238	int idx;
1239	int ret;
1240
1241	if (event->attr.type != event->pmu->type)
1242		return -ENOENT;
1243
1244	/* Sampling is not supported */
1245	if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
1246		return -EOPNOTSUPP;
1247
1248	event->cpu = hns3_pmu->on_cpu;
1249
1250	idx = hns3_pmu_get_event_idx(hns3_pmu);
1251	if (idx < 0) {
1252		pci_err(hns3_pmu->pdev, "Up to %u events are supported!\n",
1253			HNS3_PMU_MAX_HW_EVENTS);
1254		return -EBUSY;
1255	}
1256
1257	hwc->idx = idx;
1258
1259	ret = hns3_pmu_select_filter_mode(event, hns3_pmu);
1260	if (ret) {
1261		pci_err(hns3_pmu->pdev, "Invalid filter, ret = %d.\n", ret);
1262		return ret;
1263	}
1264
1265	if (!hns3_pmu_validate_event_group(event)) {
1266		pci_err(hns3_pmu->pdev, "Invalid event group.\n");
1267		return -EINVAL;
1268	}
1269
1270	if (hns3_pmu_get_ext_counter_used(event))
1271		hwc->event_base = HNS3_PMU_REG_EVENT_EXT_COUNTER;
1272	else
1273		hwc->event_base = HNS3_PMU_REG_EVENT_COUNTER;
1274
1275	return 0;
1276}
1277
1278static void hns3_pmu_read(struct perf_event *event)
1279{
1280	struct hw_perf_event *hwc = &event->hw;
1281	u64 new_cnt, prev_cnt, delta;
1282
1283	do {
1284		prev_cnt = local64_read(&hwc->prev_count);
1285		new_cnt = hns3_pmu_read_counter(event);
1286	} while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) !=
1287		 prev_cnt);
1288
1289	delta = new_cnt - prev_cnt;
1290	local64_add(delta, &event->count);
1291}
1292
1293static void hns3_pmu_start(struct perf_event *event, int flags)
1294{
1295	struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1296	struct hw_perf_event *hwc = &event->hw;
1297
1298	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
1299		return;
1300
1301	WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
1302	hwc->state = 0;
1303
1304	hns3_pmu_config_filter(event);
1305	hns3_pmu_init_counter(event);
1306	hns3_pmu_enable_intr(hns3_pmu, hwc);
1307	hns3_pmu_enable_counter(hns3_pmu, hwc);
1308
1309	perf_event_update_userpage(event);
1310}
1311
1312static void hns3_pmu_stop(struct perf_event *event, int flags)
1313{
1314	struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1315	struct hw_perf_event *hwc = &event->hw;
1316
1317	hns3_pmu_disable_counter(hns3_pmu, hwc);
1318	hns3_pmu_disable_intr(hns3_pmu, hwc);
1319
1320	WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1321	hwc->state |= PERF_HES_STOPPED;
1322
1323	if (hwc->state & PERF_HES_UPTODATE)
1324		return;
1325
1326	/* Read hardware counter and update the perf counter statistics */
1327	hns3_pmu_read(event);
1328	hwc->state |= PERF_HES_UPTODATE;
1329}
1330
1331static int hns3_pmu_add(struct perf_event *event, int flags)
1332{
1333	struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1334	struct hw_perf_event *hwc = &event->hw;
1335	int idx;
1336
1337	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1338
1339	/* Check all working events to find a related event. */
1340	idx = hns3_pmu_find_related_event_idx(hns3_pmu, event);
1341	if (idx < 0 && idx != -ENOENT)
1342		return idx;
1343
1344	/* Current event shares an enabled hardware event with related event */
1345	if (idx >= 0 && idx < HNS3_PMU_MAX_HW_EVENTS) {
1346		hwc->idx = idx;
1347		goto start_count;
1348	}
1349
1350	idx = hns3_pmu_get_event_idx(hns3_pmu);
1351	if (idx < 0)
1352		return idx;
1353
1354	hwc->idx = idx;
1355	hns3_pmu->hw_events[idx] = event;
1356
1357start_count:
1358	if (flags & PERF_EF_START)
1359		hns3_pmu_start(event, PERF_EF_RELOAD);
1360
1361	return 0;
1362}
1363
1364static void hns3_pmu_del(struct perf_event *event, int flags)
1365{
1366	struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1367	struct hw_perf_event *hwc = &event->hw;
1368
1369	hns3_pmu_stop(event, PERF_EF_UPDATE);
1370	hns3_pmu->hw_events[hwc->idx] = NULL;
1371	perf_event_update_userpage(event);
1372}
1373
1374static void hns3_pmu_enable(struct pmu *pmu)
1375{
1376	struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu);
1377	u32 val;
1378
1379	val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
1380	val |= HNS3_PMU_GLOBAL_START;
1381	writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
1382}
1383
1384static void hns3_pmu_disable(struct pmu *pmu)
1385{
1386	struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu);
1387	u32 val;
1388
1389	val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
1390	val &= ~HNS3_PMU_GLOBAL_START;
1391	writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
1392}
1393
1394static int hns3_pmu_alloc_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu)
1395{
1396	u16 device_id;
1397	char *name;
1398	u32 val;
1399
1400	hns3_pmu->base = pcim_iomap_table(pdev)[BAR_2];
1401	if (!hns3_pmu->base) {
1402		pci_err(pdev, "ioremap failed\n");
1403		return -ENOMEM;
1404	}
1405
1406	hns3_pmu->hw_clk_freq = readl(hns3_pmu->base + HNS3_PMU_REG_CLOCK_FREQ);
1407
1408	val = readl(hns3_pmu->base + HNS3_PMU_REG_BDF);
1409	hns3_pmu->bdf_min = val & 0xffff;
1410	hns3_pmu->bdf_max = val >> 16;
1411
1412	val = readl(hns3_pmu->base + HNS3_PMU_REG_DEVICE_ID);
1413	device_id = val & 0xffff;
1414	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hns3_pmu_sicl_%u", device_id);
1415	if (!name)
1416		return -ENOMEM;
1417
1418	hns3_pmu->pdev = pdev;
1419	hns3_pmu->on_cpu = -1;
1420	hns3_pmu->identifier = readl(hns3_pmu->base + HNS3_PMU_REG_VERSION);
1421	hns3_pmu->pmu = (struct pmu) {
1422		.name		= name,
1423		.module		= THIS_MODULE,
1424		.parent		= &pdev->dev,
1425		.event_init	= hns3_pmu_event_init,
1426		.pmu_enable	= hns3_pmu_enable,
1427		.pmu_disable	= hns3_pmu_disable,
1428		.add		= hns3_pmu_add,
1429		.del		= hns3_pmu_del,
1430		.start		= hns3_pmu_start,
1431		.stop		= hns3_pmu_stop,
1432		.read		= hns3_pmu_read,
1433		.task_ctx_nr	= perf_invalid_context,
1434		.attr_groups	= hns3_pmu_attr_groups,
1435		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
1436	};
1437
1438	return 0;
1439}
1440
1441static irqreturn_t hns3_pmu_irq(int irq, void *data)
1442{
1443	struct hns3_pmu *hns3_pmu = data;
1444	u32 intr_status, idx;
1445
1446	for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
1447		intr_status = hns3_pmu_readl(hns3_pmu,
1448					     HNS3_PMU_REG_EVENT_INTR_STATUS,
1449					     idx);
1450
1451		/*
1452		 * As each counter will restart from 0 when it is overflowed,
1453		 * extra processing is no need, just clear interrupt status.
1454		 */
1455		if (intr_status)
1456			hns3_pmu_clear_intr_status(hns3_pmu, idx);
1457	}
1458
1459	return IRQ_HANDLED;
1460}
1461
1462static int hns3_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
1463{
1464	struct hns3_pmu *hns3_pmu;
1465
1466	hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node);
1467	if (!hns3_pmu)
1468		return -ENODEV;
1469
1470	if (hns3_pmu->on_cpu == -1) {
1471		hns3_pmu->on_cpu = cpu;
1472		irq_set_affinity(hns3_pmu->irq, cpumask_of(cpu));
1473	}
1474
1475	return 0;
1476}
1477
1478static int hns3_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
1479{
1480	struct hns3_pmu *hns3_pmu;
1481	unsigned int target;
1482
1483	hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node);
1484	if (!hns3_pmu)
1485		return -ENODEV;
1486
1487	/* Nothing to do if this CPU doesn't own the PMU */
1488	if (hns3_pmu->on_cpu != cpu)
1489		return 0;
1490
1491	/* Choose a new CPU from all online cpus */
1492	target = cpumask_any_but(cpu_online_mask, cpu);
1493	if (target >= nr_cpu_ids)
1494		return 0;
1495
1496	perf_pmu_migrate_context(&hns3_pmu->pmu, cpu, target);
1497	hns3_pmu->on_cpu = target;
1498	irq_set_affinity(hns3_pmu->irq, cpumask_of(target));
1499
1500	return 0;
1501}
1502
1503static void hns3_pmu_free_irq(void *data)
1504{
1505	struct pci_dev *pdev = data;
1506
1507	pci_free_irq_vectors(pdev);
1508}
1509
1510static int hns3_pmu_irq_register(struct pci_dev *pdev,
1511				 struct hns3_pmu *hns3_pmu)
1512{
1513	int irq, ret;
1514
1515	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
1516	if (ret < 0) {
1517		pci_err(pdev, "failed to enable MSI vectors, ret = %d.\n", ret);
1518		return ret;
1519	}
1520
1521	ret = devm_add_action_or_reset(&pdev->dev, hns3_pmu_free_irq, pdev);
1522	if (ret) {
1523		pci_err(pdev, "failed to add free irq action, ret = %d.\n", ret);
1524		return ret;
1525	}
1526
1527	irq = pci_irq_vector(pdev, 0);
1528	ret = devm_request_irq(&pdev->dev, irq, hns3_pmu_irq, 0,
1529			       hns3_pmu->pmu.name, hns3_pmu);
1530	if (ret) {
1531		pci_err(pdev, "failed to register irq, ret = %d.\n", ret);
1532		return ret;
1533	}
1534
1535	hns3_pmu->irq = irq;
1536
1537	return 0;
1538}
1539
1540static int hns3_pmu_init_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu)
1541{
1542	int ret;
1543
1544	ret = hns3_pmu_alloc_pmu(pdev, hns3_pmu);
1545	if (ret)
1546		return ret;
1547
1548	ret = hns3_pmu_irq_register(pdev, hns3_pmu);
1549	if (ret)
1550		return ret;
1551
1552	ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
1553				       &hns3_pmu->node);
1554	if (ret) {
1555		pci_err(pdev, "failed to register hotplug, ret = %d.\n", ret);
1556		return ret;
1557	}
1558
1559	ret = perf_pmu_register(&hns3_pmu->pmu, hns3_pmu->pmu.name, -1);
1560	if (ret) {
1561		pci_err(pdev, "failed to register perf PMU, ret = %d.\n", ret);
1562		cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
1563						    &hns3_pmu->node);
1564	}
1565
1566	return ret;
1567}
1568
1569static void hns3_pmu_uninit_pmu(struct pci_dev *pdev)
1570{
1571	struct hns3_pmu *hns3_pmu = pci_get_drvdata(pdev);
1572
1573	perf_pmu_unregister(&hns3_pmu->pmu);
1574	cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
1575					    &hns3_pmu->node);
1576}
1577
1578static int hns3_pmu_init_dev(struct pci_dev *pdev)
1579{
1580	int ret;
1581
1582	ret = pcim_enable_device(pdev);
1583	if (ret) {
1584		pci_err(pdev, "failed to enable pci device, ret = %d.\n", ret);
1585		return ret;
1586	}
1587
1588	ret = pcim_iomap_regions(pdev, BIT(BAR_2), "hns3_pmu");
1589	if (ret < 0) {
1590		pci_err(pdev, "failed to request pci region, ret = %d.\n", ret);
1591		return ret;
1592	}
1593
1594	pci_set_master(pdev);
1595
1596	return 0;
1597}
1598
1599static int hns3_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1600{
1601	struct hns3_pmu *hns3_pmu;
1602	int ret;
1603
1604	hns3_pmu = devm_kzalloc(&pdev->dev, sizeof(*hns3_pmu), GFP_KERNEL);
1605	if (!hns3_pmu)
1606		return -ENOMEM;
1607
1608	ret = hns3_pmu_init_dev(pdev);
1609	if (ret)
1610		return ret;
1611
1612	ret = hns3_pmu_init_pmu(pdev, hns3_pmu);
1613	if (ret) {
1614		pci_clear_master(pdev);
1615		return ret;
1616	}
1617
1618	pci_set_drvdata(pdev, hns3_pmu);
1619
1620	return ret;
1621}
1622
1623static void hns3_pmu_remove(struct pci_dev *pdev)
1624{
1625	hns3_pmu_uninit_pmu(pdev);
1626	pci_clear_master(pdev);
1627	pci_set_drvdata(pdev, NULL);
1628}
1629
1630static const struct pci_device_id hns3_pmu_ids[] = {
1631	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa22b) },
1632	{ 0, }
1633};
1634MODULE_DEVICE_TABLE(pci, hns3_pmu_ids);
1635
1636static struct pci_driver hns3_pmu_driver = {
1637	.name = "hns3_pmu",
1638	.id_table = hns3_pmu_ids,
1639	.probe = hns3_pmu_probe,
1640	.remove = hns3_pmu_remove,
1641};
1642
1643static int __init hns3_pmu_module_init(void)
1644{
1645	int ret;
1646
1647	ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
1648				      "AP_PERF_ARM_HNS3_PMU_ONLINE",
1649				      hns3_pmu_online_cpu,
1650				      hns3_pmu_offline_cpu);
1651	if (ret) {
1652		pr_err("failed to setup HNS3 PMU hotplug, ret = %d.\n", ret);
1653		return ret;
1654	}
1655
1656	ret = pci_register_driver(&hns3_pmu_driver);
1657	if (ret) {
1658		pr_err("failed to register pci driver, ret = %d.\n", ret);
1659		cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE);
1660	}
1661
1662	return ret;
1663}
1664module_init(hns3_pmu_module_init);
1665
1666static void __exit hns3_pmu_module_exit(void)
1667{
1668	pci_unregister_driver(&hns3_pmu_driver);
1669	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE);
1670}
1671module_exit(hns3_pmu_module_exit);
1672
1673MODULE_DESCRIPTION("HNS3 PMU driver");
1674MODULE_LICENSE("GPL v2");