a5xx_preempt.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright (c) 2017 The Linux Foundation. All rights reserved.
  3 */
  4
  5#include "msm_gem.h"
  6#include "a5xx_gpu.h"
  7
  8/*
  9 * Try to transition the preemption state from old to new. Return
 10 * true on success or false if the original state wasn't 'old'
 11 */
 12static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
 13		enum preempt_state old, enum preempt_state new)
 14{
 15	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
 16		old, new);
 17
 18	return (cur == old);
 19}
 20
 21/*
 22 * Force the preemption state to the specified state.  This is used in cases
 23 * where the current state is known and won't change
 24 */
 25static inline void set_preempt_state(struct a5xx_gpu *gpu,
 26		enum preempt_state new)
 27{
 28	/*
 29	 * preempt_state may be read by other cores trying to trigger a
 30	 * preemption or in the interrupt handler so barriers are needed
 31	 * before...
 32	 */
 33	smp_mb__before_atomic();
 34	atomic_set(&gpu->preempt_state, new);
 35	/* ... and after*/
 36	smp_mb__after_atomic();
 37}
 38
 39/* Write the most recent wptr for the given ring into the hardware */
 40static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
 41{
 42	unsigned long flags;
 43	uint32_t wptr;
 44
 45	if (!ring)
 46		return;
 47
 48	spin_lock_irqsave(&ring->preempt_lock, flags);
 49	wptr = get_wptr(ring);
 50	spin_unlock_irqrestore(&ring->preempt_lock, flags);
 51
 52	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
 53}
 54
 55/* Return the highest priority ringbuffer with something in it */
 56static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
 57{
 58	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 59	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 60	unsigned long flags;
 61	int i;
 62
 63	for (i = 0; i < gpu->nr_rings; i++) {
 64		bool empty;
 65		struct msm_ringbuffer *ring = gpu->rb[i];
 66
 67		spin_lock_irqsave(&ring->preempt_lock, flags);
 68		empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));
 69		if (!empty && ring == a5xx_gpu->cur_ring)
 70			empty = ring->memptrs->fence == a5xx_gpu->last_seqno[i];
 71		spin_unlock_irqrestore(&ring->preempt_lock, flags);
 72
 73		if (!empty)
 74			return ring;
 75	}
 76
 77	return NULL;
 78}
 79
 80static void a5xx_preempt_timer(struct timer_list *t)
 81{
 82	struct a5xx_gpu *a5xx_gpu = timer_container_of(a5xx_gpu, t,
 83						       preempt_timer);
 84	struct msm_gpu *gpu = &a5xx_gpu->base.base;
 85	struct drm_device *dev = gpu->dev;
 86
 87	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
 88		return;
 89
 90	DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
 91	kthread_queue_work(gpu->worker, &gpu->recover_work);
 92}
 93
 94/* Try to trigger a preemption switch */
 95void a5xx_preempt_trigger(struct msm_gpu *gpu)
 96{
 97	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 98	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
 99	unsigned long flags;
100	struct msm_ringbuffer *ring;
101
102	if (gpu->nr_rings == 1)
103		return;
104
105	/*
106	 * Serialize preemption start to ensure that we always make
107	 * decision on latest state. Otherwise we can get stuck in
108	 * lower priority or empty ring.
109	 */
110	spin_lock_irqsave(&a5xx_gpu->preempt_start_lock, flags);
111
112	/*
113	 * Try to start preemption by moving from NONE to START. If
114	 * unsuccessful, a preemption is already in flight
115	 */
116	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
117		goto out;
118
119	/* Get the next ring to preempt to */
120	ring = get_next_ring(gpu);
121
122	/*
123	 * If no ring is populated or the highest priority ring is the current
124	 * one do nothing except to update the wptr to the latest and greatest
125	 */
126	if (!ring || (a5xx_gpu->cur_ring == ring)) {
127		/*
128		 * Its possible that while a preemption request is in progress
129		 * from an irq context, a user context trying to submit might
130		 * fail to update the write pointer, because it determines
131		 * that the preempt state is not PREEMPT_NONE.
132		 *
133		 * Close the race by introducing an intermediate
134		 * state PREEMPT_ABORT to let the submit path
135		 * know that the ringbuffer is not going to change
136		 * and can safely update the write pointer.
137		 */
138
139		set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
140		update_wptr(gpu, a5xx_gpu->cur_ring);
141		set_preempt_state(a5xx_gpu, PREEMPT_NONE);
142		goto out;
143	}
144
145	spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags);
146
147	/* Make sure the wptr doesn't update while we're in motion */
148	spin_lock_irqsave(&ring->preempt_lock, flags);
149	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
150	spin_unlock_irqrestore(&ring->preempt_lock, flags);
151
152	/* Set the address of the incoming preemption record */
153	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
154		a5xx_gpu->preempt_iova[ring->id]);
155
156	a5xx_gpu->next_ring = ring;
157
158	/* Start a timer to catch a stuck preemption */
159	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
160
161	/* Set the preemption state to triggered */
162	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
163
164	/* Make sure everything is written before hitting the button */
165	wmb();
166
167	/* And actually start the preemption */
168	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
169	return;
170
171out:
172	spin_unlock_irqrestore(&a5xx_gpu->preempt_start_lock, flags);
173}
174
175void a5xx_preempt_irq(struct msm_gpu *gpu)
176{
177	uint32_t status;
178	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
179	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
180	struct drm_device *dev = gpu->dev;
181
182	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
183		return;
184
185	/* Delete the preemption watchdog timer */
186	timer_delete(&a5xx_gpu->preempt_timer);
187
188	/*
189	 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
190	 * firing the interrupt, but there is a non zero chance of a hardware
191	 * condition or a software race that could set it again before we have a
192	 * chance to finish. If that happens, log and go for recovery
193	 */
194	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
195	if (unlikely(status)) {
196		set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
197		DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
198			gpu->name);
199		kthread_queue_work(gpu->worker, &gpu->recover_work);
200		return;
201	}
202
203	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
204	a5xx_gpu->next_ring = NULL;
205
206	update_wptr(gpu, a5xx_gpu->cur_ring);
207
208	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
209
210	/*
211	 * Try to trigger preemption again in case there was a submit or
212	 * retire during ring switch
213	 */
214	a5xx_preempt_trigger(gpu);
215}
216
217void a5xx_preempt_hw_init(struct msm_gpu *gpu)
218{
219	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
220	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
221	int i;
222
223	/* Always come up on rb 0 */
224	a5xx_gpu->cur_ring = gpu->rb[0];
225
226	/* No preemption if we only have one ring */
227	if (gpu->nr_rings == 1)
228		return;
229
230	for (i = 0; i < gpu->nr_rings; i++) {
231		a5xx_gpu->preempt[i]->data = 0;
232		a5xx_gpu->preempt[i]->info = 0;
233		a5xx_gpu->preempt[i]->wptr = 0;
234		a5xx_gpu->preempt[i]->rptr = 0;
235		a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
236		a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]);
237	}
238
239	/* Write a 0 to signal that we aren't switching pagetables */
240	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO, 0);
241
242	/* Reset the preemption state */
243	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
244}
245
246static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
247		struct msm_ringbuffer *ring)
248{
249	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
250	struct msm_gpu *gpu = &adreno_gpu->base;
251	struct a5xx_preempt_record *ptr;
252	void *counters;
253	struct drm_gem_object *bo = NULL, *counters_bo = NULL;
254	u64 iova = 0, counters_iova = 0;
255
256	ptr = msm_gem_kernel_new(gpu->dev,
257		A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
258		MSM_BO_WC | MSM_BO_MAP_PRIV, gpu->vm, &bo, &iova);
259
260	if (IS_ERR(ptr))
261		return PTR_ERR(ptr);
262
263	/* The buffer to store counters needs to be unprivileged */
264	counters = msm_gem_kernel_new(gpu->dev,
265		A5XX_PREEMPT_COUNTER_SIZE,
266		MSM_BO_WC, gpu->vm, &counters_bo, &counters_iova);
267	if (IS_ERR(counters)) {
268		msm_gem_kernel_put(bo, gpu->vm);
269		return PTR_ERR(counters);
270	}
271
272	msm_gem_object_set_name(bo, "preempt");
273	msm_gem_object_set_name(counters_bo, "preempt_counters");
274
275	a5xx_gpu->preempt_bo[ring->id] = bo;
276	a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo;
277	a5xx_gpu->preempt_iova[ring->id] = iova;
278	a5xx_gpu->preempt[ring->id] = ptr;
279
280	/* Set up the defaults on the preemption record */
281
282	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
283	ptr->info = 0;
284	ptr->data = 0;
285	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE;
286
287	ptr->counter = counters_iova;
288
289	return 0;
290}
291
292void a5xx_preempt_fini(struct msm_gpu *gpu)
293{
294	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
295	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
296	int i;
297
298	for (i = 0; i < gpu->nr_rings; i++) {
299		msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->vm);
300		msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i], gpu->vm);
301	}
302}
303
304void a5xx_preempt_init(struct msm_gpu *gpu)
305{
306	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
307	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
308	int i;
309
310	/* No preemption if we only have one ring */
311	if (gpu->nr_rings <= 1)
312		return;
313
314	for (i = 0; i < gpu->nr_rings; i++) {
315		if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
316			/*
317			 * On any failure our adventure is over. Clean up and
318			 * set nr_rings to 1 to force preemption off
319			 */
320			a5xx_preempt_fini(gpu);
321			gpu->nr_rings = 1;
322
323			return;
324		}
325	}
326
327	spin_lock_init(&a5xx_gpu->preempt_start_lock);
328	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
329}