v3d_perfmon.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2021 Raspberry Pi
  4 */
  5
  6#include "v3d_drv.h"
  7#include "v3d_regs.h"
  8
  9#define V3D_PERFMONID_MIN	1
 10#define V3D_PERFMONID_MAX	U32_MAX
 11
 12static const struct v3d_perf_counter_desc v3d_v42_performance_counters[] = {
 13	{"FEP", "FEP-valid-primitives-no-rendered-pixels", "[FEP] Valid primitives that result in no rendered pixels, for all rendered tiles"},
 14	{"FEP", "FEP-valid-primitives-rendered-pixels", "[FEP] Valid primitives for all rendered tiles (primitives may be counted in more than one tile)"},
 15	{"FEP", "FEP-clipped-quads", "[FEP] Early-Z/Near/Far clipped quads"},
 16	{"FEP", "FEP-valid-quads", "[FEP] Valid quads"},
 17	{"TLB", "TLB-quads-not-passing-stencil-test", "[TLB] Quads with no pixels passing the stencil test"},
 18	{"TLB", "TLB-quads-not-passing-z-and-stencil-test", "[TLB] Quads with no pixels passing the Z and stencil tests"},
 19	{"TLB", "TLB-quads-passing-z-and-stencil-test", "[TLB] Quads with any pixels passing the Z and stencil tests"},
 20	{"TLB", "TLB-quads-with-zero-coverage", "[TLB] Quads with all pixels having zero coverage"},
 21	{"TLB", "TLB-quads-with-non-zero-coverage", "[TLB] Quads with any pixels having non-zero coverage"},
 22	{"TLB", "TLB-quads-written-to-color-buffer", "[TLB] Quads with valid pixels written to colour buffer"},
 23	{"PTB", "PTB-primitives-discarded-outside-viewport", "[PTB] Primitives discarded by being outside the viewport"},
 24	{"PTB", "PTB-primitives-need-clipping", "[PTB] Primitives that need clipping"},
 25	{"PTB", "PTB-primitives-discarded-reversed", "[PTB] Primitives that are discarded because they are reversed"},
 26	{"QPU", "QPU-total-idle-clk-cycles", "[QPU] Total idle clock cycles for all QPUs"},
 27	{"QPU", "QPU-total-active-clk-cycles-vertex-coord-shading", "[QPU] Total active clock cycles for all QPUs doing vertex/coordinate/user shading (counts only when QPU is not stalled)"},
 28	{"QPU", "QPU-total-active-clk-cycles-fragment-shading", "[QPU] Total active clock cycles for all QPUs doing fragment shading (counts only when QPU is not stalled)"},
 29	{"QPU", "QPU-total-clk-cycles-executing-valid-instr", "[QPU] Total clock cycles for all QPUs executing valid instructions"},
 30	{"QPU", "QPU-total-clk-cycles-waiting-TMU", "[QPU] Total clock cycles for all QPUs stalled waiting for TMUs only (counter won't increment if QPU also stalling for another reason)"},
 31	{"QPU", "QPU-total-clk-cycles-waiting-scoreboard", "[QPU] Total clock cycles for all QPUs stalled waiting for Scoreboard only (counter won't increment if QPU also stalling for another reason)"},
 32	{"QPU", "QPU-total-clk-cycles-waiting-varyings", "[QPU] Total clock cycles for all QPUs stalled waiting for Varyings only (counter won't increment if QPU also stalling for another reason)"},
 33	{"QPU", "QPU-total-instr-cache-hit", "[QPU] Total instruction cache hits for all slices"},
 34	{"QPU", "QPU-total-instr-cache-miss", "[QPU] Total instruction cache misses for all slices"},
 35	{"QPU", "QPU-total-uniform-cache-hit", "[QPU] Total uniforms cache hits for all slices"},
 36	{"QPU", "QPU-total-uniform-cache-miss", "[QPU] Total uniforms cache misses for all slices"},
 37	{"TMU", "TMU-total-text-quads-access", "[TMU] Total texture cache accesses"},
 38	{"TMU", "TMU-total-text-cache-miss", "[TMU] Total texture cache misses (number of fetches from memory/L2cache)"},
 39	{"VPM", "VPM-total-clk-cycles-VDW-stalled", "[VPM] Total clock cycles VDW is stalled waiting for VPM access"},
 40	{"VPM", "VPM-total-clk-cycles-VCD-stalled", "[VPM] Total clock cycles VCD is stalled waiting for VPM access"},
 41	{"CLE", "CLE-bin-thread-active-cycles", "[CLE] Bin thread active cycles"},
 42	{"CLE", "CLE-render-thread-active-cycles", "[CLE] Render thread active cycles"},
 43	{"L2T", "L2T-total-cache-hit", "[L2T] Total Level 2 cache hits"},
 44	{"L2T", "L2T-total-cache-miss", "[L2T] Total Level 2 cache misses"},
 45	{"CORE", "cycle-count", "[CORE] Cycle counter"},
 46	{"QPU", "QPU-total-clk-cycles-waiting-vertex-coord-shading", "[QPU] Total stalled clock cycles for all QPUs doing vertex/coordinate/user shading"},
 47	{"QPU", "QPU-total-clk-cycles-waiting-fragment-shading", "[QPU] Total stalled clock cycles for all QPUs doing fragment shading"},
 48	{"PTB", "PTB-primitives-binned", "[PTB] Total primitives binned"},
 49	{"AXI", "AXI-writes-seen-watch-0", "[AXI] Writes seen by watch 0"},
 50	{"AXI", "AXI-reads-seen-watch-0", "[AXI] Reads seen by watch 0"},
 51	{"AXI", "AXI-writes-stalled-seen-watch-0", "[AXI] Write stalls seen by watch 0"},
 52	{"AXI", "AXI-reads-stalled-seen-watch-0", "[AXI] Read stalls seen by watch 0"},
 53	{"AXI", "AXI-write-bytes-seen-watch-0", "[AXI] Total bytes written seen by watch 0"},
 54	{"AXI", "AXI-read-bytes-seen-watch-0", "[AXI] Total bytes read seen by watch 0"},
 55	{"AXI", "AXI-writes-seen-watch-1", "[AXI] Writes seen by watch 1"},
 56	{"AXI", "AXI-reads-seen-watch-1", "[AXI] Reads seen by watch 1"},
 57	{"AXI", "AXI-writes-stalled-seen-watch-1", "[AXI] Write stalls seen by watch 1"},
 58	{"AXI", "AXI-reads-stalled-seen-watch-1", "[AXI] Read stalls seen by watch 1"},
 59	{"AXI", "AXI-write-bytes-seen-watch-1", "[AXI] Total bytes written seen by watch 1"},
 60	{"AXI", "AXI-read-bytes-seen-watch-1", "[AXI] Total bytes read seen by watch 1"},
 61	{"TLB", "TLB-partial-quads-written-to-color-buffer", "[TLB] Partial quads written to the colour buffer"},
 62	{"TMU", "TMU-total-config-access", "[TMU] Total config accesses"},
 63	{"L2T", "L2T-no-id-stalled", "[L2T] No ID stall"},
 64	{"L2T", "L2T-command-queue-stalled", "[L2T] Command queue full stall"},
 65	{"L2T", "L2T-TMU-writes", "[L2T] TMU write accesses"},
 66	{"TMU", "TMU-active-cycles", "[TMU] Active cycles"},
 67	{"TMU", "TMU-stalled-cycles", "[TMU] Stalled cycles"},
 68	{"CLE", "CLE-thread-active-cycles", "[CLE] Bin or render thread active cycles"},
 69	{"L2T", "L2T-TMU-reads", "[L2T] TMU read accesses"},
 70	{"L2T", "L2T-CLE-reads", "[L2T] CLE read accesses"},
 71	{"L2T", "L2T-VCD-reads", "[L2T] VCD read accesses"},
 72	{"L2T", "L2T-TMU-config-reads", "[L2T] TMU CFG read accesses"},
 73	{"L2T", "L2T-SLC0-reads", "[L2T] SLC0 read accesses"},
 74	{"L2T", "L2T-SLC1-reads", "[L2T] SLC1 read accesses"},
 75	{"L2T", "L2T-SLC2-reads", "[L2T] SLC2 read accesses"},
 76	{"L2T", "L2T-TMU-write-miss", "[L2T] TMU write misses"},
 77	{"L2T", "L2T-TMU-read-miss", "[L2T] TMU read misses"},
 78	{"L2T", "L2T-CLE-read-miss", "[L2T] CLE read misses"},
 79	{"L2T", "L2T-VCD-read-miss", "[L2T] VCD read misses"},
 80	{"L2T", "L2T-TMU-config-read-miss", "[L2T] TMU CFG read misses"},
 81	{"L2T", "L2T-SLC0-read-miss", "[L2T] SLC0 read misses"},
 82	{"L2T", "L2T-SLC1-read-miss", "[L2T] SLC1 read misses"},
 83	{"L2T", "L2T-SLC2-read-miss", "[L2T] SLC2 read misses"},
 84	{"CORE", "core-memory-writes", "[CORE] Total memory writes"},
 85	{"L2T", "L2T-memory-writes", "[L2T] Total memory writes"},
 86	{"PTB", "PTB-memory-writes", "[PTB] Total memory writes"},
 87	{"TLB", "TLB-memory-writes", "[TLB] Total memory writes"},
 88	{"CORE", "core-memory-reads", "[CORE] Total memory reads"},
 89	{"L2T", "L2T-memory-reads", "[L2T] Total memory reads"},
 90	{"PTB", "PTB-memory-reads", "[PTB] Total memory reads"},
 91	{"PSE", "PSE-memory-reads", "[PSE] Total memory reads"},
 92	{"TLB", "TLB-memory-reads", "[TLB] Total memory reads"},
 93	{"GMP", "GMP-memory-reads", "[GMP] Total memory reads"},
 94	{"PTB", "PTB-memory-words-writes", "[PTB] Total memory words written"},
 95	{"TLB", "TLB-memory-words-writes", "[TLB] Total memory words written"},
 96	{"PSE", "PSE-memory-words-reads", "[PSE] Total memory words read"},
 97	{"TLB", "TLB-memory-words-reads", "[TLB] Total memory words read"},
 98	{"TMU", "TMU-MRU-hits", "[TMU] Total MRU hits"},
 99	{"CORE", "compute-active-cycles", "[CORE] Compute active cycles"},
100};
101
102static const struct v3d_perf_counter_desc v3d_v71_performance_counters[] = {
103	{"CORE", "cycle-count", "[CORE] Cycle counter"},
104	{"CORE", "core-active", "[CORE] Bin/Render/Compute active cycles"},
105	{"CLE", "CLE-bin-thread-active-cycles", "[CLE] Bin thread active cycles"},
106	{"CLE", "CLE-render-thread-active-cycles", "[CLE] Render thread active cycles"},
107	{"CORE", "compute-active-cycles", "[CORE] Compute active cycles"},
108	{"FEP", "FEP-valid-primitives-no-rendered-pixels", "[FEP] Valid primitives that result in no rendered pixels, for all rendered tiles"},
109	{"FEP", "FEP-valid-primitives-rendered-pixels", "[FEP] Valid primitives for all rendered tiles (primitives may be counted in more than one tile)"},
110	{"FEP", "FEP-clipped-quads", "[FEP] Early-Z/Near/Far clipped quads"},
111	{"FEP", "FEP-valid-quads", "[FEP] Valid quads"},
112	{"TLB", "TLB-quads-not-passing-stencil-test", "[TLB] Quads with no pixels passing the stencil test"},
113	{"TLB", "TLB-quads-not-passing-z-and-stencil-test", "[TLB] Quads with no pixels passing the Z and stencil tests"},
114	{"TLB", "TLB-quads-passing-z-and-stencil-test", "[TLB] Quads with any pixels passing the Z and stencil tests"},
115	{"TLB", "TLB-quads-written-to-color-buffer", "[TLB] Quads with valid pixels written to colour buffer"},
116	{"TLB", "TLB-partial-quads-written-to-color-buffer", "[TLB] Partial quads written to the colour buffer"},
117	{"PTB", "PTB-primitives-need-clipping", "[PTB] Primitives that need clipping"},
118	{"PTB", "PTB-primitives-discarded-outside-viewport", "[PTB] Primitives discarded by being outside the viewport"},
119	{"PTB", "PTB-primitives-binned", "[PTB] Total primitives binned"},
120	{"PTB", "PTB-primitives-discarded-reversed", "[PTB] Primitives that are discarded because they are reversed"},
121	{"QPU", "QPU-total-instr-cache-hit", "[QPU] Total instruction cache hits for all slices"},
122	{"QPU", "QPU-total-instr-cache-miss", "[QPU] Total instruction cache misses for all slices"},
123	{"QPU", "QPU-total-uniform-cache-hit", "[QPU] Total uniforms cache hits for all slices"},
124	{"QPU", "QPU-total-uniform-cache-miss", "[QPU] Total uniforms cache misses for all slices"},
125	{"TMU", "TMU-active-cycles", "[TMU] Active cycles"},
126	{"TMU", "TMU-stalled-cycles", "[TMU] Stalled cycles"},
127	{"TMU", "TMU-total-text-quads-access", "[TMU] Total texture cache accesses"},
128	{"TMU", "TMU-cache-x4-active-cycles", "[TMU] Cache active cycles for x4 access"},
129	{"TMU", "TMU-cache-x4-stalled-cycles", "[TMU] Cache stalled cycles for x4 access"},
130	{"TMU", "TMU-total-text-quads-x4-access", "[TMU] Total texture cache x4 access"},
131	{"L2T", "L2T-total-cache-hit", "[L2T] Total Level 2 cache hits"},
132	{"L2T", "L2T-total-cache-miss", "[L2T] Total Level 2 cache misses"},
133	{"L2T", "L2T-local", "[L2T] Local mode access"},
134	{"L2T", "L2T-writeback", "[L2T] Writeback"},
135	{"L2T", "L2T-zero", "[L2T] Zero"},
136	{"L2T", "L2T-merge", "[L2T] Merge"},
137	{"L2T", "L2T-fill", "[L2T] Fill"},
138	{"L2T", "L2T-stalls-no-wid", "[L2T] Stalls because no WID available"},
139	{"L2T", "L2T-stalls-no-rid", "[L2T] Stalls because no RID available"},
140	{"L2T", "L2T-stalls-queue-full", "[L2T] Stalls because internal queue full"},
141	{"L2T", "L2T-stalls-wrightback", "[L2T] Stalls because writeback in flight"},
142	{"L2T", "L2T-stalls-mem", "[L2T] Stalls because AXI blocks read"},
143	{"L2T", "L2T-stalls-fill", "[L2T] Stalls because fill pending for victim cache-line"},
144	{"L2T", "L2T-hitq", "[L2T] Sent request via hit queue"},
145	{"L2T", "L2T-hitq-full", "[L2T] Sent request via main queue because hit queue is full"},
146	{"L2T", "L2T-stalls-read-data", "[L2T] Stalls because waiting for data from SDRAM"},
147	{"L2T", "L2T-TMU-read-hits", "[L2T] TMU read hits"},
148	{"L2T", "L2T-TMU-read-miss", "[L2T] TMU read misses"},
149	{"L2T", "L2T-VCD-read-hits", "[L2T] VCD read hits"},
150	{"L2T", "L2T-VCD-read-miss", "[L2T] VCD read misses"},
151	{"L2T", "L2T-SLC-read-hits", "[L2T] SLC read hits (all slices)"},
152	{"L2T", "L2T-SLC-read-miss", "[L2T] SLC read misses (all slices)"},
153	{"AXI", "AXI-writes-seen-watch-0", "[AXI] Writes seen by watch 0"},
154	{"AXI", "AXI-reads-seen-watch-0", "[AXI] Reads seen by watch 0"},
155	{"AXI", "AXI-writes-stalled-seen-watch-0", "[AXI] Write stalls seen by watch 0"},
156	{"AXI", "AXI-reads-stalled-seen-watch-0", "[AXI] Read stalls seen by watch 0"},
157	{"AXI", "AXI-write-bytes-seen-watch-0", "[AXI] Total bytes written seen by watch 0"},
158	{"AXI", "AXI-read-bytes-seen-watch-0", "[AXI] Total bytes read seen by watch 0"},
159	{"AXI", "AXI-writes-seen-watch-1", "[AXI] Writes seen by watch 1"},
160	{"AXI", "AXI-reads-seen-watch-1", "[AXI] Reads seen by watch 1"},
161	{"AXI", "AXI-writes-stalled-seen-watch-1", "[AXI] Write stalls seen by watch 1"},
162	{"AXI", "AXI-reads-stalled-seen-watch-1", "[AXI] Read stalls seen by watch 1"},
163	{"AXI", "AXI-write-bytes-seen-watch-1", "[AXI] Total bytes written seen by watch 1"},
164	{"AXI", "AXI-read-bytes-seen-watch-1", "[AXI] Total bytes read seen by watch 1"},
165	{"CORE", "core-memory-writes", "[CORE] Total memory writes"},
166	{"L2T", "L2T-memory-writes", "[L2T] Total memory writes"},
167	{"PTB", "PTB-memory-writes", "[PTB] Total memory writes"},
168	{"TLB", "TLB-memory-writes", "[TLB] Total memory writes"},
169	{"CORE", "core-memory-reads", "[CORE] Total memory reads"},
170	{"L2T", "L2T-memory-reads", "[L2T] Total memory reads"},
171	{"PTB", "PTB-memory-reads", "[PTB] Total memory reads"},
172	{"PSE", "PSE-memory-reads", "[PSE] Total memory reads"},
173	{"TLB", "TLB-memory-reads", "[TLB] Total memory reads"},
174	{"PTB", "PTB-memory-words-writes", "[PTB] Total memory words written"},
175	{"TLB", "TLB-memory-words-writes", "[TLB] Total memory words written"},
176	{"PSE", "PSE-memory-words-reads", "[PSE] Total memory words read"},
177	{"TLB", "TLB-memory-words-reads", "[TLB] Total memory words read"},
178	{"AXI", "AXI-read-trans", "[AXI] Read transaction count"},
179	{"AXI", "AXI-write-trans", "[AXI] Write transaction count"},
180	{"AXI", "AXI-read-wait-cycles", "[AXI] Read total wait cycles"},
181	{"AXI", "AXI-write-wait-cycles", "[AXI] Write total wait cycles"},
182	{"AXI", "AXI-max-outstanding-reads", "[AXI] Maximum outstanding read transactions"},
183	{"AXI", "AXI-max-outstanding-writes", "[AXI] Maximum outstanding write transactions"},
184	{"QPU", "QPU-wait-bubble", "[QPU] Pipeline bubble in qcycles due all threads waiting"},
185	{"QPU", "QPU-ic-miss-bubble", "[QPU] Pipeline bubble in qcycles due instruction-cache miss"},
186	{"QPU", "QPU-active", "[QPU] Executed shader instruction"},
187	{"QPU", "QPU-total-active-clk-cycles-fragment-shading", "[QPU] Total active clock cycles for all QPUs doing fragment shading (counts only when QPU is not stalled)"},
188	{"QPU", "QPU-stalls", "[QPU] Stalled qcycles executing shader instruction"},
189	{"QPU", "QPU-total-clk-cycles-waiting-fragment-shading", "[QPU] Total stalled clock cycles for all QPUs doing fragment shading"},
190	{"QPU", "QPU-stalls-TMU", "[QPU] Stalled qcycles waiting for TMU"},
191	{"QPU", "QPU-stalls-TLB", "[QPU] Stalled qcycles waiting for TLB"},
192	{"QPU", "QPU-stalls-VPM", "[QPU] Stalled qcycles waiting for VPM"},
193	{"QPU", "QPU-stalls-uniforms", "[QPU] Stalled qcycles waiting for uniforms"},
194	{"QPU", "QPU-stalls-SFU", "[QPU] Stalled qcycles waiting for SFU"},
195	{"QPU", "QPU-stalls-other", "[QPU] Stalled qcycles waiting for any other reason (vary/W/Z)"},
196};
197
198void v3d_perfmon_init(struct v3d_dev *v3d)
199{
200	const struct v3d_perf_counter_desc *counters = NULL;
201	unsigned int max = 0;
202
203	if (v3d->ver >= V3D_GEN_71) {
204		counters = v3d_v71_performance_counters;
205		max = ARRAY_SIZE(v3d_v71_performance_counters);
206	} else if (v3d->ver >= V3D_GEN_42) {
207		counters = v3d_v42_performance_counters;
208		max = ARRAY_SIZE(v3d_v42_performance_counters);
209	}
210
211	v3d->perfmon_info.max_counters = max;
212	v3d->perfmon_info.counters = counters;
213}
214
215void v3d_perfmon_get(struct v3d_perfmon *perfmon)
216{
217	if (perfmon)
218		refcount_inc(&perfmon->refcnt);
219}
220
221void v3d_perfmon_put(struct v3d_perfmon *perfmon)
222{
223	if (perfmon && refcount_dec_and_test(&perfmon->refcnt)) {
224		mutex_destroy(&perfmon->lock);
225		kfree(perfmon);
226	}
227}
228
229void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
230{
231	unsigned int i;
232	u32 mask;
233	u8 ncounters;
234
235	if (WARN_ON_ONCE(!perfmon || v3d->active_perfmon))
236		return;
237
238	ncounters = perfmon->ncounters;
239	mask = GENMASK(ncounters - 1, 0);
240
241	for (i = 0; i < ncounters; i++) {
242		u32 source = i / 4;
243		u32 channel = V3D_SET_FIELD_VER(perfmon->counters[i], V3D_PCTR_S0,
244						v3d->ver);
245
246		i++;
247		channel |= V3D_SET_FIELD_VER(i < ncounters ? perfmon->counters[i] : 0,
248					     V3D_PCTR_S1, v3d->ver);
249		i++;
250		channel |= V3D_SET_FIELD_VER(i < ncounters ? perfmon->counters[i] : 0,
251					     V3D_PCTR_S2, v3d->ver);
252		i++;
253		channel |= V3D_SET_FIELD_VER(i < ncounters ? perfmon->counters[i] : 0,
254					     V3D_PCTR_S3, v3d->ver);
255		V3D_CORE_WRITE(0, V3D_V4_PCTR_0_SRC_X(source), channel);
256	}
257
258	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, mask);
259	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_CLR, mask);
260	V3D_CORE_WRITE(0, V3D_PCTR_0_OVERFLOW, mask);
261
262	v3d->active_perfmon = perfmon;
263}
264
265void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
266		      bool capture)
267{
268	unsigned int i;
269
270	if (!perfmon || !v3d->active_perfmon)
271		return;
272
273	mutex_lock(&perfmon->lock);
274	if (perfmon != v3d->active_perfmon) {
275		mutex_unlock(&perfmon->lock);
276		return;
277	}
278
279	if (capture)
280		for (i = 0; i < perfmon->ncounters; i++)
281			perfmon->values[i] += V3D_CORE_READ(0, V3D_PCTR_0_PCTRX(i));
282
283	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, 0);
284
285	v3d->active_perfmon = NULL;
286	mutex_unlock(&perfmon->lock);
287}
288
289struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id)
290{
291	struct v3d_perfmon *perfmon;
292
293	mutex_lock(&v3d_priv->perfmon.lock);
294	perfmon = idr_find(&v3d_priv->perfmon.idr, id);
295	v3d_perfmon_get(perfmon);
296	mutex_unlock(&v3d_priv->perfmon.lock);
297
298	return perfmon;
299}
300
301void v3d_perfmon_open_file(struct v3d_file_priv *v3d_priv)
302{
303	mutex_init(&v3d_priv->perfmon.lock);
304	idr_init_base(&v3d_priv->perfmon.idr, 1);
305}
306
307static int v3d_perfmon_idr_del(int id, void *elem, void *data)
308{
309	struct v3d_perfmon *perfmon = elem;
310	struct v3d_dev *v3d = (struct v3d_dev *)data;
311
312	/* If the active perfmon is being destroyed, stop it first */
313	if (perfmon == v3d->active_perfmon)
314		v3d_perfmon_stop(v3d, perfmon, false);
315
316	/* If the global perfmon is being destroyed, set it to NULL */
317	cmpxchg(&v3d->global_perfmon, perfmon, NULL);
318
319	v3d_perfmon_put(perfmon);
320
321	return 0;
322}
323
324void v3d_perfmon_close_file(struct v3d_file_priv *v3d_priv)
325{
326	struct v3d_dev *v3d = v3d_priv->v3d;
327
328	mutex_lock(&v3d_priv->perfmon.lock);
329	idr_for_each(&v3d_priv->perfmon.idr, v3d_perfmon_idr_del, v3d);
330	idr_destroy(&v3d_priv->perfmon.idr);
331	mutex_unlock(&v3d_priv->perfmon.lock);
332	mutex_destroy(&v3d_priv->perfmon.lock);
333}
334
335int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data,
336			     struct drm_file *file_priv)
337{
338	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
339	struct drm_v3d_perfmon_create *req = data;
340	struct v3d_dev *v3d = v3d_priv->v3d;
341	struct v3d_perfmon *perfmon;
342	unsigned int i;
343	int ret;
344
345	/* Number of monitored counters cannot exceed HW limits. */
346	if (req->ncounters > DRM_V3D_MAX_PERF_COUNTERS ||
347	    !req->ncounters)
348		return -EINVAL;
349
350	/* Make sure all counters are valid. */
351	for (i = 0; i < req->ncounters; i++) {
352		if (req->counters[i] >= v3d->perfmon_info.max_counters)
353			return -EINVAL;
354	}
355
356	perfmon = kzalloc(struct_size(perfmon, values, req->ncounters),
357			  GFP_KERNEL);
358	if (!perfmon)
359		return -ENOMEM;
360
361	for (i = 0; i < req->ncounters; i++)
362		perfmon->counters[i] = req->counters[i];
363
364	perfmon->ncounters = req->ncounters;
365
366	refcount_set(&perfmon->refcnt, 1);
367	mutex_init(&perfmon->lock);
368
369	mutex_lock(&v3d_priv->perfmon.lock);
370	ret = idr_alloc(&v3d_priv->perfmon.idr, perfmon, V3D_PERFMONID_MIN,
371			V3D_PERFMONID_MAX, GFP_KERNEL);
372	mutex_unlock(&v3d_priv->perfmon.lock);
373
374	if (ret < 0) {
375		mutex_destroy(&perfmon->lock);
376		kfree(perfmon);
377		return ret;
378	}
379
380	req->id = ret;
381
382	return 0;
383}
384
385int v3d_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
386			      struct drm_file *file_priv)
387{
388	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
389	struct drm_v3d_perfmon_destroy *req = data;
390	struct v3d_dev *v3d = v3d_priv->v3d;
391	struct v3d_perfmon *perfmon;
392
393	mutex_lock(&v3d_priv->perfmon.lock);
394	perfmon = idr_remove(&v3d_priv->perfmon.idr, req->id);
395	mutex_unlock(&v3d_priv->perfmon.lock);
396
397	if (!perfmon)
398		return -EINVAL;
399
400	/* If the active perfmon is being destroyed, stop it first */
401	if (perfmon == v3d->active_perfmon)
402		v3d_perfmon_stop(v3d, perfmon, false);
403
404	/* If the global perfmon is being destroyed, set it to NULL */
405	cmpxchg(&v3d->global_perfmon, perfmon, NULL);
406
407	v3d_perfmon_put(perfmon);
408
409	return 0;
410}
411
412int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
413				 struct drm_file *file_priv)
414{
415	struct v3d_dev *v3d = to_v3d_dev(dev);
416	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
417	struct drm_v3d_perfmon_get_values *req = data;
418	struct v3d_perfmon *perfmon;
419	int ret = 0;
420
421	if (req->pad != 0)
422		return -EINVAL;
423
424	perfmon = v3d_perfmon_find(v3d_priv, req->id);
425	if (!perfmon)
426		return -EINVAL;
427
428	v3d_perfmon_stop(v3d, perfmon, true);
429
430	if (copy_to_user(u64_to_user_ptr(req->values_ptr), perfmon->values,
431			 perfmon->ncounters * sizeof(u64)))
432		ret = -EFAULT;
433
434	v3d_perfmon_put(perfmon);
435
436	return ret;
437}
438
439int v3d_perfmon_get_counter_ioctl(struct drm_device *dev, void *data,
440				  struct drm_file *file_priv)
441{
442	struct drm_v3d_perfmon_get_counter *req = data;
443	struct v3d_dev *v3d = to_v3d_dev(dev);
444	const struct v3d_perf_counter_desc *counter;
445
446	for (int i = 0; i < ARRAY_SIZE(req->reserved); i++) {
447		if (req->reserved[i] != 0)
448			return -EINVAL;
449	}
450
451	if (!v3d->perfmon_info.max_counters)
452		return -EOPNOTSUPP;
453
454	/* Make sure that the counter ID is valid */
455	if (req->counter >= v3d->perfmon_info.max_counters)
456		return -EINVAL;
457
458	counter = &v3d->perfmon_info.counters[req->counter];
459
460	strscpy(req->name, counter->name, sizeof(req->name));
461	strscpy(req->category, counter->category, sizeof(req->category));
462	strscpy(req->description, counter->description, sizeof(req->description));
463
464	return 0;
465}
466
467int v3d_perfmon_set_global_ioctl(struct drm_device *dev, void *data,
468				 struct drm_file *file_priv)
469{
470	struct v3d_file_priv *v3d_priv = file_priv->driver_priv;
471	struct drm_v3d_perfmon_set_global *req = data;
472	struct v3d_dev *v3d = to_v3d_dev(dev);
473	struct v3d_perfmon *perfmon;
474
475	if (req->flags & ~DRM_V3D_PERFMON_CLEAR_GLOBAL)
476		return -EINVAL;
477
478	perfmon = v3d_perfmon_find(v3d_priv, req->id);
479	if (!perfmon)
480		return -EINVAL;
481
482	/* If the request is to clear the global performance monitor */
483	if (req->flags & DRM_V3D_PERFMON_CLEAR_GLOBAL) {
484		if (!v3d->global_perfmon)
485			return -EINVAL;
486
487		xchg(&v3d->global_perfmon, NULL);
488
489		return 0;
490	}
491
492	if (cmpxchg(&v3d->global_perfmon, NULL, perfmon))
493		return -EBUSY;
494
495	return 0;
496}