dwc_pcie_pmu.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Synopsys DesignWare PCIe PMU driver
  4 *
  5 * Copyright (C) 2021-2023 Alibaba Inc.
  6 */
  7
  8#include <linux/bitfield.h>
  9#include <linux/bitops.h>
 10#include <linux/cpuhotplug.h>
 11#include <linux/cpumask.h>
 12#include <linux/device.h>
 13#include <linux/errno.h>
 14#include <linux/kernel.h>
 15#include <linux/list.h>
 16#include <linux/pcie-dwc.h>
 17#include <linux/perf_event.h>
 18#include <linux/pci.h>
 19#include <linux/platform_device.h>
 20#include <linux/smp.h>
 21#include <linux/sysfs.h>
 22#include <linux/types.h>
 23
 24#define DWC_PCIE_EVENT_CNT_CTL			0x8
 25
 26/*
 27 * Event Counter Data Select includes two parts:
 28 * - 27-24: Group number(4-bit: 0..0x7)
 29 * - 23-16: Event number(8-bit: 0..0x13) within the Group
 30 *
 31 * Put them together as in TRM.
 32 */
 33#define DWC_PCIE_CNT_EVENT_SEL			GENMASK(27, 16)
 34#define DWC_PCIE_CNT_LANE_SEL			GENMASK(11, 8)
 35#define DWC_PCIE_CNT_STATUS			BIT(7)
 36#define DWC_PCIE_CNT_ENABLE			GENMASK(4, 2)
 37#define DWC_PCIE_PER_EVENT_OFF			0x1
 38#define DWC_PCIE_PER_EVENT_ON			0x3
 39#define DWC_PCIE_EVENT_CLEAR			GENMASK(1, 0)
 40#define DWC_PCIE_EVENT_PER_CLEAR		0x1
 41
 42/* Event Selection Field has two subfields */
 43#define DWC_PCIE_CNT_EVENT_SEL_GROUP		GENMASK(11, 8)
 44#define DWC_PCIE_CNT_EVENT_SEL_EVID		GENMASK(7, 0)
 45
 46#define DWC_PCIE_EVENT_CNT_DATA			0xC
 47
 48#define DWC_PCIE_TIME_BASED_ANAL_CTL		0x10
 49#define DWC_PCIE_TIME_BASED_REPORT_SEL		GENMASK(31, 24)
 50#define DWC_PCIE_TIME_BASED_DURATION_SEL	GENMASK(15, 8)
 51#define DWC_PCIE_DURATION_MANUAL_CTL		0x0
 52#define DWC_PCIE_DURATION_1MS			0x1
 53#define DWC_PCIE_DURATION_10MS			0x2
 54#define DWC_PCIE_DURATION_100MS			0x3
 55#define DWC_PCIE_DURATION_1S			0x4
 56#define DWC_PCIE_DURATION_2S			0x5
 57#define DWC_PCIE_DURATION_4S			0x6
 58#define DWC_PCIE_DURATION_4US			0xFF
 59#define DWC_PCIE_TIME_BASED_TIMER_START		BIT(0)
 60#define DWC_PCIE_TIME_BASED_CNT_ENABLE		0x1
 61
 62#define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW	0x14
 63#define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH	0x18
 64
 65/* Event attributes */
 66#define DWC_PCIE_CONFIG_EVENTID			GENMASK(15, 0)
 67#define DWC_PCIE_CONFIG_TYPE			GENMASK(19, 16)
 68#define DWC_PCIE_CONFIG_LANE			GENMASK(27, 20)
 69
 70#define DWC_PCIE_EVENT_ID(event)	FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config)
 71#define DWC_PCIE_EVENT_TYPE(event)	FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config)
 72#define DWC_PCIE_EVENT_LANE(event)	FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config)
 73
 74enum dwc_pcie_event_type {
 75	DWC_PCIE_TIME_BASE_EVENT,
 76	DWC_PCIE_LANE_EVENT,
 77	DWC_PCIE_EVENT_TYPE_MAX,
 78};
 79
 80#define DWC_PCIE_LANE_GROUP_6 6
 81#define DWC_PCIE_LANE_GROUP_7 7
 82#define DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP 256
 83
 84#define DWC_PCIE_LANE_EVENT_MAX_PERIOD		GENMASK_ULL(31, 0)
 85#define DWC_PCIE_MAX_PERIOD			GENMASK_ULL(63, 0)
 86
 87struct dwc_pcie_pmu {
 88	struct pmu		pmu;
 89	struct pci_dev		*pdev;		/* Root Port device */
 90	u16			ras_des_offset;
 91	u32			nr_lanes;
 92
 93	/* Groups #6 and #7 */
 94	DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP);
 95	struct perf_event	*time_based_event;
 96
 97	struct hlist_node	cpuhp_node;
 98	int			on_cpu;
 99};
100
101#define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu))
102
103static int dwc_pcie_pmu_hp_state;
104static struct list_head dwc_pcie_dev_info_head =
105				LIST_HEAD_INIT(dwc_pcie_dev_info_head);
106static bool notify;
107
108struct dwc_pcie_dev_info {
109	struct platform_device *plat_dev;
110	struct pci_dev *pdev;
111	struct list_head dev_node;
112};
113
114static ssize_t cpumask_show(struct device *dev,
115					 struct device_attribute *attr,
116					 char *buf)
117{
118	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev));
119
120	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pcie_pmu->on_cpu));
121}
122static DEVICE_ATTR_RO(cpumask);
123
124static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = {
125	&dev_attr_cpumask.attr,
126	NULL
127};
128
129static struct attribute_group dwc_pcie_cpumask_attr_group = {
130	.attrs = dwc_pcie_pmu_cpumask_attrs,
131};
132
133struct dwc_pcie_format_attr {
134	struct device_attribute attr;
135	u64 field;
136	int config;
137};
138
139PMU_FORMAT_ATTR(eventid, "config:0-15");
140PMU_FORMAT_ATTR(type, "config:16-19");
141PMU_FORMAT_ATTR(lane, "config:20-27");
142
143static struct attribute *dwc_pcie_format_attrs[] = {
144	&format_attr_type.attr,
145	&format_attr_eventid.attr,
146	&format_attr_lane.attr,
147	NULL,
148};
149
150static struct attribute_group dwc_pcie_format_attrs_group = {
151	.name = "format",
152	.attrs = dwc_pcie_format_attrs,
153};
154
155struct dwc_pcie_event_attr {
156	struct device_attribute attr;
157	enum dwc_pcie_event_type type;
158	u16 eventid;
159	u8 lane;
160};
161
162static ssize_t dwc_pcie_event_show(struct device *dev,
163				struct device_attribute *attr, char *buf)
164{
165	struct dwc_pcie_event_attr *eattr;
166
167	eattr = container_of(attr, typeof(*eattr), attr);
168
169	if (eattr->type == DWC_PCIE_LANE_EVENT)
170		return sysfs_emit(buf, "eventid=0x%x,type=0x%x,lane=?\n",
171				  eattr->eventid, eattr->type);
172	else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT)
173		return sysfs_emit(buf, "eventid=0x%x,type=0x%x\n",
174				  eattr->eventid, eattr->type);
175
176	return 0;
177}
178
179#define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane)		\
180	(&((struct dwc_pcie_event_attr[]) {{				\
181		.attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL),	\
182		.type = _type,						\
183		.eventid = _eventid,					\
184		.lane = _lane,						\
185	}})[0].attr.attr)
186
187#define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid)		\
188	DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0)
189#define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid)			\
190	DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0)
191
192static struct attribute *dwc_pcie_pmu_time_event_attrs[] = {
193	/* Group #0 */
194	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00),
195	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01),
196	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02),
197	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03),
198	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04),
199	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05),
200	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06),
201	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07),
202	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x08),
203	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x09),
204
205	/* Group #1 */
206	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_pcie_tlp_data_payload, 0x20),
207	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_pcie_tlp_data_payload, 0x21),
208	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(tx_ccix_tlp_data_payload, 0x22),
209	DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(rx_ccix_tlp_data_payload, 0x23),
210
211	/*
212	 * Leave it to the user to specify the lane ID to avoid generating
213	 * a list of hundreds of events.
214	 */
215	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600),
216	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601),
217	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602),
218	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603),
219	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nullified_tlp, 0x604),
220	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nullified_tlp, 0x605),
221	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tlp, 0x606),
222	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700),
223	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701),
224	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702),
225	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703),
226	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704),
227	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705),
228	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706),
229	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707),
230	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708),
231	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709),
232	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A),
233	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B),
234	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C),
235	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F),
236	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710),
237	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711),
238	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712),
239	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713),
240	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714),
241	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715),
242	DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716),
243	DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717),
244	NULL
245};
246
247static const struct attribute_group dwc_pcie_event_attrs_group = {
248	.name = "events",
249	.attrs = dwc_pcie_pmu_time_event_attrs,
250};
251
252static const struct attribute_group *dwc_pcie_attr_groups[] = {
253	&dwc_pcie_event_attrs_group,
254	&dwc_pcie_format_attrs_group,
255	&dwc_pcie_cpumask_attr_group,
256	NULL
257};
258
259static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu,
260					   struct perf_event *event,
261					   bool enable)
262{
263	struct pci_dev *pdev = pcie_pmu->pdev;
264	u16 ras_des_offset = pcie_pmu->ras_des_offset;
265	int event_id = DWC_PCIE_EVENT_ID(event);
266	int lane = DWC_PCIE_EVENT_LANE(event);
267	u32 ctrl;
268
269	ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
270		FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
271		FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
272
273	if (enable)
274		ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
275	else
276		ctrl |= FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF);
277
278	pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
279			       ctrl);
280}
281
282static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu,
283					  bool enable)
284{
285	struct pci_dev *pdev = pcie_pmu->pdev;
286	u16 ras_des_offset = pcie_pmu->ras_des_offset;
287
288	pci_clear_and_set_config_dword(pdev,
289				       ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL,
290				       DWC_PCIE_TIME_BASED_TIMER_START, enable);
291}
292
293static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event)
294{
295	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
296	struct pci_dev *pdev = pcie_pmu->pdev;
297	int event_id = DWC_PCIE_EVENT_ID(event);
298	int lane = DWC_PCIE_EVENT_LANE(event);
299	u16 ras_des_offset = pcie_pmu->ras_des_offset;
300	u32 val, ctrl;
301
302	ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
303		FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
304		FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON);
305	pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
306			       ctrl);
307	pci_read_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, &val);
308
309	ctrl |= FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
310	pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
311			       ctrl);
312
313	return val;
314}
315
316static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event)
317{
318	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
319	struct pci_dev *pdev = pcie_pmu->pdev;
320	int event_id = DWC_PCIE_EVENT_ID(event);
321	u16 ras_des_offset = pcie_pmu->ras_des_offset;
322	u32 lo, hi, ss;
323	u64 val;
324
325	/*
326	 * The 64-bit value of the data counter is spread across two
327	 * registers that are not synchronized. In order to read them
328	 * atomically, ensure that the high 32 bits match before and after
329	 * reading the low 32 bits.
330	 */
331	pci_read_config_dword(pdev,
332		ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, &hi);
333	do {
334		/* snapshot the high 32 bits */
335		ss = hi;
336
337		pci_read_config_dword(
338			pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW,
339			&lo);
340		pci_read_config_dword(
341			pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH,
342			&hi);
343	} while (hi != ss);
344
345	val = ((u64)hi << 32) | lo;
346	/*
347	 * The Group#1 event measures the amount of data processed in 16-byte
348	 * units. Simplify the end-user interface by multiplying the counter
349	 * at the point of read.
350	 */
351	if (event_id >= 0x20 && event_id <= 0x23)
352		val *= 16;
353
354	return val;
355}
356
357static void dwc_pcie_pmu_event_update(struct perf_event *event)
358{
359	struct hw_perf_event *hwc = &event->hw;
360	enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
361	u64 delta, prev, now;
362
363	if (type == DWC_PCIE_LANE_EVENT) {
364		now = dwc_pcie_pmu_read_lane_event_counter(event) &
365			DWC_PCIE_LANE_EVENT_MAX_PERIOD;
366		local64_add(now, &event->count);
367		return;
368	}
369
370	do {
371		prev = local64_read(&hwc->prev_count);
372		now = dwc_pcie_pmu_read_time_based_counter(event);
373
374	} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
375
376	delta = (now - prev) & DWC_PCIE_MAX_PERIOD;
377	local64_add(delta, &event->count);
378}
379
380static int dwc_pcie_pmu_validate_add_lane_event(struct perf_event *event,
381						unsigned long val_lane_events[])
382{
383	int event_id, event_nr, group;
384
385	event_id = DWC_PCIE_EVENT_ID(event);
386	event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
387	group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id);
388
389	if (group != DWC_PCIE_LANE_GROUP_6 && group != DWC_PCIE_LANE_GROUP_7)
390		return -EINVAL;
391
392	group -= DWC_PCIE_LANE_GROUP_6;
393
394	if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
395			     val_lane_events))
396		return -EINVAL;
397
398	return 0;
399}
400
401static int dwc_pcie_pmu_validate_group(struct perf_event *event)
402{
403	struct perf_event *sibling, *leader = event->group_leader;
404	DECLARE_BITMAP(val_lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP);
405	bool time_event = false;
406	int type;
407
408	type = DWC_PCIE_EVENT_TYPE(leader);
409	if (type == DWC_PCIE_TIME_BASE_EVENT)
410		time_event = true;
411	else
412		if (dwc_pcie_pmu_validate_add_lane_event(leader, val_lane_events))
413			return -ENOSPC;
414
415	for_each_sibling_event(sibling, leader) {
416		type = DWC_PCIE_EVENT_TYPE(sibling);
417		if (type == DWC_PCIE_TIME_BASE_EVENT) {
418			if (time_event)
419				return -ENOSPC;
420
421			time_event = true;
422			continue;
423		}
424
425		if (dwc_pcie_pmu_validate_add_lane_event(sibling, val_lane_events))
426			return -ENOSPC;
427	}
428
429	return 0;
430}
431
432static int dwc_pcie_pmu_event_init(struct perf_event *event)
433{
434	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
435	enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
436	struct perf_event *sibling;
437	u32 lane;
438
439	if (event->attr.type != event->pmu->type)
440		return -ENOENT;
441
442	/* We don't support sampling */
443	if (is_sampling_event(event))
444		return -EINVAL;
445
446	/* We cannot support task bound events */
447	if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK)
448		return -EINVAL;
449
450	for_each_sibling_event(sibling, event->group_leader) {
451		if (sibling->pmu != event->pmu && !is_software_event(sibling))
452			return -EINVAL;
453	}
454
455	if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX)
456		return -EINVAL;
457
458	if (type == DWC_PCIE_LANE_EVENT) {
459		lane = DWC_PCIE_EVENT_LANE(event);
460		if (lane < 0 || lane >= pcie_pmu->nr_lanes)
461			return -EINVAL;
462	}
463
464	if (dwc_pcie_pmu_validate_group(event))
465		return -ENOSPC;
466
467	event->cpu = pcie_pmu->on_cpu;
468
469	return 0;
470}
471
472static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags)
473{
474	struct hw_perf_event *hwc = &event->hw;
475	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
476	enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
477
478	hwc->state = 0;
479	local64_set(&hwc->prev_count, 0);
480
481	if (type == DWC_PCIE_LANE_EVENT)
482		dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true);
483	else if (type == DWC_PCIE_TIME_BASE_EVENT)
484		dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true);
485}
486
487static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags)
488{
489	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
490	enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
491	struct hw_perf_event *hwc = &event->hw;
492
493	if (event->hw.state & PERF_HES_STOPPED)
494		return;
495
496	dwc_pcie_pmu_event_update(event);
497
498	if (type == DWC_PCIE_LANE_EVENT)
499		dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false);
500	else if (type == DWC_PCIE_TIME_BASE_EVENT)
501		dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false);
502
503	hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
504}
505
506static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags)
507{
508	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
509	struct pci_dev *pdev = pcie_pmu->pdev;
510	struct hw_perf_event *hwc = &event->hw;
511	enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
512	int event_id = DWC_PCIE_EVENT_ID(event);
513	int lane = DWC_PCIE_EVENT_LANE(event);
514	u16 ras_des_offset = pcie_pmu->ras_des_offset;
515	u32 ctrl;
516
517	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
518
519	if (type == DWC_PCIE_LANE_EVENT) {
520		int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
521		int group = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) -
522			DWC_PCIE_LANE_GROUP_6;
523
524		if (test_and_set_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
525				     pcie_pmu->lane_events))
526			return -ENOSPC;
527
528		/* EVENT_COUNTER_DATA_REG needs clear manually */
529		ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) |
530			FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) |
531			FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) |
532			FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR);
533		pci_write_config_dword(pdev, ras_des_offset + DWC_PCIE_EVENT_CNT_CTL,
534				       ctrl);
535	} else if (type == DWC_PCIE_TIME_BASE_EVENT) {
536		if (pcie_pmu->time_based_event)
537			return -ENOSPC;
538
539		pcie_pmu->time_based_event = event;
540
541		/*
542		 * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely
543		 * use it with any manually controlled duration. And it is
544		 * cleared when next measurement starts.
545		 */
546		ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) |
547			FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL,
548				   DWC_PCIE_DURATION_MANUAL_CTL) |
549			DWC_PCIE_TIME_BASED_CNT_ENABLE;
550		pci_write_config_dword(
551			pdev, ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, ctrl);
552	}
553
554	if (flags & PERF_EF_START)
555		dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD);
556
557	perf_event_update_userpage(event);
558
559	return 0;
560}
561
562static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags)
563{
564	struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu);
565	enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event);
566
567	dwc_pcie_pmu_event_stop(event, flags | PERF_EF_UPDATE);
568	perf_event_update_userpage(event);
569
570	if (type == DWC_PCIE_TIME_BASE_EVENT) {
571		pcie_pmu->time_based_event = NULL;
572	} else {
573		int event_id = DWC_PCIE_EVENT_ID(event);
574		int event_nr = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_EVID, event_id);
575		int group    = FIELD_GET(DWC_PCIE_CNT_EVENT_SEL_GROUP, event_id) -
576			DWC_PCIE_LANE_GROUP_6;
577
578		clear_bit(group * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP + event_nr,
579			  pcie_pmu->lane_events);
580	}
581}
582
583static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node)
584{
585	cpuhp_state_remove_instance_nocalls(dwc_pcie_pmu_hp_state, hotplug_node);
586}
587
588/*
589 * Find the binded DES capability device info of a PCI device.
590 * @pdev: The PCI device.
591 */
592static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev)
593{
594	struct dwc_pcie_dev_info *dev_info;
595
596	list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node)
597		if (dev_info->pdev == pdev)
598			return dev_info;
599
600	return NULL;
601}
602
603static void dwc_pcie_unregister_pmu(void *data)
604{
605	struct dwc_pcie_pmu *pcie_pmu = data;
606
607	perf_pmu_unregister(&pcie_pmu->pmu);
608}
609
610static u16 dwc_pcie_des_cap(struct pci_dev *pdev)
611{
612	const struct dwc_pcie_vsec_id *vid;
613	u16 vsec;
614	u32 val;
615
616	if (!pci_is_pcie(pdev) || !(pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT))
617		return 0;
618
619	for (vid = dwc_pcie_rasdes_vsec_ids; vid->vendor_id; vid++) {
620		vsec = pci_find_vsec_capability(pdev, vid->vendor_id,
621						vid->vsec_id);
622		if (vsec) {
623			pci_read_config_dword(pdev, vsec + PCI_VNDR_HEADER,
624					      &val);
625			if (PCI_VNDR_HEADER_REV(val) == vid->vsec_rev) {
626				pci_dbg(pdev, "Detected PCIe Vendor-Specific Extended Capability RAS DES\n");
627				return vsec;
628			}
629		}
630	}
631	return 0;
632}
633
634static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info)
635{
636	platform_device_unregister(dev_info->plat_dev);
637	list_del(&dev_info->dev_node);
638	kfree(dev_info);
639}
640
641static int dwc_pcie_register_dev(struct pci_dev *pdev)
642{
643	struct platform_device *plat_dev;
644	struct dwc_pcie_dev_info *dev_info;
645	u32 sbdf;
646
647	sbdf = (pci_domain_nr(pdev->bus) << 16) | PCI_DEVID(pdev->bus->number, pdev->devfn);
648	plat_dev = platform_device_register_simple("dwc_pcie_pmu", sbdf, NULL, 0);
649	if (IS_ERR(plat_dev))
650		return PTR_ERR(plat_dev);
651
652	dev_info = kzalloc(sizeof(*dev_info), GFP_KERNEL);
653	if (!dev_info) {
654		platform_device_unregister(plat_dev);
655		return -ENOMEM;
656	}
657
658	/* Cache platform device to handle pci device hotplug */
659	dev_info->plat_dev = plat_dev;
660	dev_info->pdev = pdev;
661	list_add(&dev_info->dev_node, &dwc_pcie_dev_info_head);
662
663	return 0;
664}
665
666static int dwc_pcie_pmu_notifier(struct notifier_block *nb,
667				     unsigned long action, void *data)
668{
669	struct device *dev = data;
670	struct pci_dev *pdev = to_pci_dev(dev);
671	struct dwc_pcie_dev_info *dev_info;
672
673	switch (action) {
674	case BUS_NOTIFY_ADD_DEVICE:
675		if (!dwc_pcie_des_cap(pdev))
676			return NOTIFY_DONE;
677		if (dwc_pcie_register_dev(pdev))
678			return NOTIFY_BAD;
679		break;
680	case BUS_NOTIFY_DEL_DEVICE:
681		dev_info = dwc_pcie_find_dev_info(pdev);
682		if (!dev_info)
683			return NOTIFY_DONE;
684		dwc_pcie_unregister_dev(dev_info);
685		break;
686	}
687
688	return NOTIFY_OK;
689}
690
691static struct notifier_block dwc_pcie_pmu_nb = {
692	.notifier_call = dwc_pcie_pmu_notifier,
693};
694
695static int dwc_pcie_pmu_probe(struct platform_device *plat_dev)
696{
697	struct pci_dev *pdev;
698	struct dwc_pcie_pmu *pcie_pmu;
699	char *name;
700	u32 sbdf;
701	u16 vsec;
702	int ret;
703
704	sbdf = plat_dev->id;
705	pdev = pci_get_domain_bus_and_slot(sbdf >> 16, PCI_BUS_NUM(sbdf & 0xffff),
706					   sbdf & 0xff);
707	if (!pdev) {
708		pr_err("No pdev found for the sbdf 0x%x\n", sbdf);
709		return -ENODEV;
710	}
711
712	vsec = dwc_pcie_des_cap(pdev);
713	if (!vsec)
714		return -ENODEV;
715
716	pci_dev_put(pdev);
717	name = devm_kasprintf(&plat_dev->dev, GFP_KERNEL, "dwc_rootport_%x", sbdf);
718	if (!name)
719		return -ENOMEM;
720
721	pcie_pmu = devm_kzalloc(&plat_dev->dev, sizeof(*pcie_pmu), GFP_KERNEL);
722	if (!pcie_pmu)
723		return -ENOMEM;
724
725	pcie_pmu->pdev = pdev;
726	pcie_pmu->ras_des_offset = vsec;
727	pcie_pmu->nr_lanes = pcie_get_width_cap(pdev);
728	pcie_pmu->on_cpu = -1;
729	pcie_pmu->pmu = (struct pmu){
730		.name		= name,
731		.parent		= &plat_dev->dev,
732		.module		= THIS_MODULE,
733		.attr_groups	= dwc_pcie_attr_groups,
734		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
735		.task_ctx_nr	= perf_invalid_context,
736		.event_init	= dwc_pcie_pmu_event_init,
737		.add		= dwc_pcie_pmu_event_add,
738		.del		= dwc_pcie_pmu_event_del,
739		.start		= dwc_pcie_pmu_event_start,
740		.stop		= dwc_pcie_pmu_event_stop,
741		.read		= dwc_pcie_pmu_event_update,
742	};
743
744	/* Add this instance to the list used by the offline callback */
745	ret = cpuhp_state_add_instance(dwc_pcie_pmu_hp_state,
746				       &pcie_pmu->cpuhp_node);
747	if (ret) {
748		pci_err(pdev, "Error %d registering hotplug @%x\n", ret, sbdf);
749		return ret;
750	}
751
752	/* Unwind when platform driver removes */
753	ret = devm_add_action_or_reset(&plat_dev->dev,
754				       dwc_pcie_pmu_remove_cpuhp_instance,
755				       &pcie_pmu->cpuhp_node);
756	if (ret)
757		return ret;
758
759	ret = perf_pmu_register(&pcie_pmu->pmu, name, -1);
760	if (ret) {
761		pci_err(pdev, "Error %d registering PMU @%x\n", ret, sbdf);
762		return ret;
763	}
764	ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu,
765				       pcie_pmu);
766	if (ret)
767		return ret;
768
769	return 0;
770}
771
772static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
773{
774	struct dwc_pcie_pmu *pcie_pmu;
775
776	pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
777	if (pcie_pmu->on_cpu == -1)
778		pcie_pmu->on_cpu = cpumask_local_spread(
779			0, dev_to_node(&pcie_pmu->pdev->dev));
780
781	return 0;
782}
783
784static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
785{
786	struct dwc_pcie_pmu *pcie_pmu;
787	struct pci_dev *pdev;
788	unsigned int target;
789	int node;
790
791	pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node);
792	/* Nothing to do if this CPU doesn't own the PMU */
793	if (cpu != pcie_pmu->on_cpu)
794		return 0;
795
796	pcie_pmu->on_cpu = -1;
797	pdev = pcie_pmu->pdev;
798	node = dev_to_node(&pdev->dev);
799
800	target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
801	if (target >= nr_cpu_ids)
802		target = cpumask_any_but(cpu_online_mask, cpu);
803
804	if (target >= nr_cpu_ids) {
805		pci_err(pdev, "There is no CPU to set\n");
806		return 0;
807	}
808
809	/* This PMU does NOT support interrupt, just migrate context. */
810	perf_pmu_migrate_context(&pcie_pmu->pmu, cpu, target);
811	pcie_pmu->on_cpu = target;
812
813	return 0;
814}
815
816static struct platform_driver dwc_pcie_pmu_driver = {
817	.probe = dwc_pcie_pmu_probe,
818	.driver = {.name = "dwc_pcie_pmu",},
819};
820
821static void dwc_pcie_cleanup_devices(void)
822{
823	struct dwc_pcie_dev_info *dev_info, *tmp;
824
825	list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) {
826		dwc_pcie_unregister_dev(dev_info);
827	}
828}
829
830static int __init dwc_pcie_pmu_init(void)
831{
832	struct pci_dev *pdev = NULL;
833	int ret;
834
835	for_each_pci_dev(pdev) {
836		if (!dwc_pcie_des_cap(pdev))
837			continue;
838
839		ret = dwc_pcie_register_dev(pdev);
840		if (ret) {
841			pci_dev_put(pdev);
842			goto err_cleanup;
843		}
844	}
845
846	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
847				      "perf/dwc_pcie_pmu:online",
848				      dwc_pcie_pmu_online_cpu,
849				      dwc_pcie_pmu_offline_cpu);
850	if (ret < 0)
851		goto err_cleanup;
852
853	dwc_pcie_pmu_hp_state = ret;
854
855	ret = platform_driver_register(&dwc_pcie_pmu_driver);
856	if (ret)
857		goto err_remove_cpuhp;
858
859	ret = bus_register_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
860	if (ret)
861		goto err_unregister_driver;
862	notify = true;
863
864	return 0;
865
866err_unregister_driver:
867	platform_driver_unregister(&dwc_pcie_pmu_driver);
868err_remove_cpuhp:
869	cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
870err_cleanup:
871	dwc_pcie_cleanup_devices();
872	return ret;
873}
874
875static void __exit dwc_pcie_pmu_exit(void)
876{
877	if (notify)
878		bus_unregister_notifier(&pci_bus_type, &dwc_pcie_pmu_nb);
879	dwc_pcie_cleanup_devices();
880	platform_driver_unregister(&dwc_pcie_pmu_driver);
881	cpuhp_remove_multi_state(dwc_pcie_pmu_hp_state);
882}
883
884module_init(dwc_pcie_pmu_init);
885module_exit(dwc_pcie_pmu_exit);
886
887MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller");
888MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
889MODULE_LICENSE("GPL v2");