nouveau_fence.c

  1/*
  2 * Copyright (C) 2007 Ben Skeggs.
  3 * All Rights Reserved.
  4 *
  5 * Permission is hereby granted, free of charge, to any person obtaining
  6 * a copy of this software and associated documentation files (the
  7 * "Software"), to deal in the Software without restriction, including
  8 * without limitation the rights to use, copy, modify, merge, publish,
  9 * distribute, sublicense, and/or sell copies of the Software, and to
 10 * permit persons to whom the Software is furnished to do so, subject to
 11 * the following conditions:
 12 *
 13 * The above copyright notice and this permission notice (including the
 14 * next paragraph) shall be included in all copies or substantial
 15 * portions of the Software.
 16 *
 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 24 *
 25 */
 26
 27#include <linux/ktime.h>
 28#include <linux/hrtimer.h>
 29#include <linux/sched/signal.h>
 30#include <trace/events/dma_fence.h>
 31
 32#include <nvif/if0020.h>
 33
 34#include "nouveau_drv.h"
 35#include "nouveau_dma.h"
 36#include "nouveau_fence.h"
 37
 38static const struct dma_fence_ops nouveau_fence_ops_uevent;
 39static const struct dma_fence_ops nouveau_fence_ops_legacy;
 40
 41static inline struct nouveau_fence_chan *
 42nouveau_fctx(struct nouveau_fence *fence)
 43{
 44	return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
 45}
 46
 47static bool
 48nouveau_fence_signal(struct nouveau_fence *fence)
 49{
 50	bool drop = false;
 51
 52	dma_fence_signal_locked(&fence->base);
 53	list_del(&fence->head);
 54	rcu_assign_pointer(fence->channel, NULL);
 55
 56	if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
 57		struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 58
 59		if (!--fctx->notify_ref)
 60			drop = true;
 61	}
 62
 63	dma_fence_put(&fence->base);
 64	return drop;
 65}
 66
 67static struct nouveau_fence *
 68nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm)
 69{
 70	if (fence->ops != &nouveau_fence_ops_legacy &&
 71	    fence->ops != &nouveau_fence_ops_uevent)
 72		return NULL;
 73
 74	return to_nouveau_fence(fence);
 75}
 76
 77void
 78nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error)
 79{
 80	struct nouveau_fence *fence, *tmp;
 81	unsigned long flags;
 82
 83	spin_lock_irqsave(&fctx->lock, flags);
 84	list_for_each_entry_safe(fence, tmp, &fctx->pending, head) {
 85		if (error && !dma_fence_is_signaled_locked(&fence->base))
 86			dma_fence_set_error(&fence->base, error);
 87
 88		if (nouveau_fence_signal(fence))
 89			nvif_event_block(&fctx->event);
 90	}
 91	fctx->killed = 1;
 92	spin_unlock_irqrestore(&fctx->lock, flags);
 93}
 94
 95void
 96nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
 97{
 98	cancel_work_sync(&fctx->uevent_work);
 99	nouveau_fence_context_kill(fctx, 0);
100	nvif_event_dtor(&fctx->event);
101	fctx->dead = 1;
102
103	/*
104	 * Ensure that all accesses to fence->channel complete before freeing
105	 * the channel.
106	 */
107	synchronize_rcu();
108}
109
110static void
111nouveau_fence_context_put(struct kref *fence_ref)
112{
113	kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
114}
115
116void
117nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
118{
119	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
120}
121
122static void
123nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
124{
125	struct nouveau_fence *fence, *tmp;
126	bool drop = false;
127	u32 seq = fctx->read(chan);
128
129	list_for_each_entry_safe(fence, tmp, &fctx->pending, head) {
130		if ((int)(seq - fence->base.seqno) < 0)
131			break;
132
133		if (nouveau_fence_signal(fence))
134			drop = true;
135	}
136
137	if (drop)
138		nvif_event_block(&fctx->event);
139}
140
141static void
142nouveau_fence_uevent_work(struct work_struct *work)
143{
144	struct nouveau_fence_chan *fctx = container_of(work, struct nouveau_fence_chan,
145						       uevent_work);
146	struct nouveau_channel *chan;
147	struct nouveau_fence *fence;
148	unsigned long flags;
149
150	spin_lock_irqsave(&fctx->lock, flags);
151	fence = list_first_entry_or_null(&fctx->pending, typeof(*fence), head);
152	if (fence) {
153		chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
154		nouveau_fence_update(chan, fctx);
155	}
156	spin_unlock_irqrestore(&fctx->lock, flags);
157}
158
159static int
160nouveau_fence_wait_uevent_handler(struct nvif_event *event, void *repv, u32 repc)
161{
162	struct nouveau_fence_chan *fctx = container_of(event, typeof(*fctx), event);
163	schedule_work(&fctx->uevent_work);
164	return NVIF_EVENT_KEEP;
165}
166
167void
168nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
169{
170	struct nouveau_cli *cli = chan->cli;
171	struct nouveau_drm *drm = cli->drm;
172	struct nouveau_fence_priv *priv = (void*)drm->fence;
173	DEFINE_RAW_FLEX(struct nvif_event_v0, args, data,
174			sizeof(struct nvif_chan_event_v0));
175	struct nvif_chan_event_v0 *host =
176				(struct nvif_chan_event_v0 *)args->data;
177	int ret;
178
179	INIT_WORK(&fctx->uevent_work, nouveau_fence_uevent_work);
180	INIT_LIST_HEAD(&fctx->flip);
181	INIT_LIST_HEAD(&fctx->pending);
182	spin_lock_init(&fctx->lock);
183	fctx->context = drm->runl[chan->runlist].context_base + chan->chid;
184
185	if (chan == drm->cechan)
186		strscpy(fctx->name, "copy engine channel");
187	else if (chan == drm->channel)
188		strscpy(fctx->name, "generic kernel channel");
189	else
190		strscpy(fctx->name, cli->name);
191
192	kref_init(&fctx->fence_ref);
193	if (!priv->uevent)
194		return;
195
196	host->version = 0;
197	host->type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR;
198
199	ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr", (chan->runlist << 16) | chan->chid,
200			      nouveau_fence_wait_uevent_handler, false,
201			      args, __struct_size(args), &fctx->event);
202
203	WARN_ON(ret);
204}
205
206int
207nouveau_fence_emit(struct nouveau_fence *fence)
208{
209	struct nouveau_channel *chan = unrcu_pointer(fence->channel);
210	struct nouveau_fence_chan *fctx = chan->fence;
211	struct nouveau_fence_priv *priv = (void*)chan->cli->drm->fence;
212	int ret;
213
214	fence->timeout  = jiffies + (15 * HZ);
215
216	if (priv->uevent)
217		dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
218			       &fctx->lock, fctx->context, ++fctx->sequence);
219	else
220		dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
221			       &fctx->lock, fctx->context, ++fctx->sequence);
222	kref_get(&fctx->fence_ref);
223
224	ret = fctx->emit(fence);
225	if (!ret) {
226		dma_fence_get(&fence->base);
227		spin_lock_irq(&fctx->lock);
228
229		if (unlikely(fctx->killed)) {
230			spin_unlock_irq(&fctx->lock);
231			dma_fence_put(&fence->base);
232			return -ENODEV;
233		}
234
235		nouveau_fence_update(chan, fctx);
236		list_add_tail(&fence->head, &fctx->pending);
237		spin_unlock_irq(&fctx->lock);
238	}
239
240	return ret;
241}
242
243bool
244nouveau_fence_done(struct nouveau_fence *fence)
245{
246	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
247	struct nouveau_channel *chan;
248	unsigned long flags;
249
250	if (dma_fence_is_signaled(&fence->base))
251		return true;
252
253	spin_lock_irqsave(&fctx->lock, flags);
254	chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
255	if (chan)
256		nouveau_fence_update(chan, fctx);
257	spin_unlock_irqrestore(&fctx->lock, flags);
258
259	return dma_fence_is_signaled(&fence->base);
260}
261
262static long
263nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
264{
265	struct nouveau_fence *fence = to_nouveau_fence(f);
266	unsigned long sleep_time = NSEC_PER_MSEC / 1000;
267	unsigned long t = jiffies, timeout = t + wait;
268
269	while (!nouveau_fence_done(fence)) {
270		ktime_t kt;
271
272		t = jiffies;
273
274		if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
275			__set_current_state(TASK_RUNNING);
276			return 0;
277		}
278
279		__set_current_state(intr ? TASK_INTERRUPTIBLE :
280					   TASK_UNINTERRUPTIBLE);
281
282		kt = sleep_time;
283		schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
284		sleep_time *= 2;
285		if (sleep_time > NSEC_PER_MSEC)
286			sleep_time = NSEC_PER_MSEC;
287
288		if (intr && signal_pending(current))
289			return -ERESTARTSYS;
290	}
291
292	__set_current_state(TASK_RUNNING);
293
294	return timeout - t;
295}
296
297static int
298nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
299{
300	int ret = 0;
301
302	while (!nouveau_fence_done(fence)) {
303		if (time_after_eq(jiffies, fence->timeout)) {
304			ret = -EBUSY;
305			break;
306		}
307
308		__set_current_state(intr ?
309				    TASK_INTERRUPTIBLE :
310				    TASK_UNINTERRUPTIBLE);
311
312		if (intr && signal_pending(current)) {
313			ret = -ERESTARTSYS;
314			break;
315		}
316	}
317
318	__set_current_state(TASK_RUNNING);
319	return ret;
320}
321
322int
323nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
324{
325	long ret;
326
327	if (!lazy)
328		return nouveau_fence_wait_busy(fence, intr);
329
330	ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
331	if (ret < 0)
332		return ret;
333	else if (!ret)
334		return -EBUSY;
335	else
336		return 0;
337}
338
339int
340nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan,
341		   bool exclusive, bool intr)
342{
343	struct nouveau_fence_chan *fctx = chan->fence;
344	struct dma_resv *resv = nvbo->bo.base.resv;
345	int i, ret;
346
347	ret = dma_resv_reserve_fences(resv, 1);
348	if (ret)
349		return ret;
350
351	/* Waiting for the writes first causes performance regressions
352	 * under some circumstances. So manually wait for the reads first.
353	 */
354	for (i = 0; i < 2; ++i) {
355		struct dma_resv_iter cursor;
356		struct dma_fence *fence;
357
358		dma_resv_for_each_fence(&cursor, resv,
359					dma_resv_usage_rw(exclusive),
360					fence) {
361			enum dma_resv_usage usage;
362			struct nouveau_fence *f;
363
364			usage = dma_resv_iter_usage(&cursor);
365			if (i == 0 && usage == DMA_RESV_USAGE_WRITE)
366				continue;
367
368			f = nouveau_local_fence(fence, chan->cli->drm);
369			if (f) {
370				struct nouveau_channel *prev;
371				bool must_wait = true;
372				bool local;
373
374				rcu_read_lock();
375				prev = rcu_dereference(f->channel);
376				local = prev && prev->cli->drm == chan->cli->drm;
377				if (local && (prev == chan ||
378					      fctx->sync(f, prev, chan) == 0))
379					must_wait = false;
380				rcu_read_unlock();
381				if (!must_wait)
382					continue;
383			}
384
385			ret = dma_fence_wait(fence, intr);
386			if (ret)
387				return ret;
388		}
389	}
390
391	return 0;
392}
393
394void
395nouveau_fence_unref(struct nouveau_fence **pfence)
396{
397	if (*pfence)
398		dma_fence_put(&(*pfence)->base);
399	*pfence = NULL;
400}
401
402int
403nouveau_fence_create(struct nouveau_fence **pfence,
404		     struct nouveau_channel *chan)
405{
406	struct nouveau_fence *fence;
407
408	if (unlikely(!chan->fence))
409		return -ENODEV;
410
411	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
412	if (!fence)
413		return -ENOMEM;
414
415	fence->channel = chan;
416
417	*pfence = fence;
418	return 0;
419}
420
421int
422nouveau_fence_new(struct nouveau_fence **pfence,
423		  struct nouveau_channel *chan)
424{
425	int ret = 0;
426
427	ret = nouveau_fence_create(pfence, chan);
428	if (ret)
429		return ret;
430
431	ret = nouveau_fence_emit(*pfence);
432	if (ret)
433		nouveau_fence_unref(pfence);
434
435	return ret;
436}
437
438static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
439{
440	return "nouveau";
441}
442
443static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
444{
445	struct nouveau_fence *fence = to_nouveau_fence(f);
446	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
447
448	return !fctx->dead ? fctx->name : "dead channel";
449}
450
451/*
452 * In an ideal world, read would not assume the channel context is still alive.
453 * This function may be called from another device, running into free memory as a
454 * result. The drm node should still be there, so we can derive the index from
455 * the fence context.
456 */
457static bool nouveau_fence_is_signaled(struct dma_fence *f)
458{
459	struct nouveau_fence *fence = to_nouveau_fence(f);
460	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
461	struct nouveau_channel *chan;
462	bool ret = false;
463
464	rcu_read_lock();
465	chan = rcu_dereference(fence->channel);
466	if (chan)
467		ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
468	rcu_read_unlock();
469
470	return ret;
471}
472
473static bool nouveau_fence_no_signaling(struct dma_fence *f)
474{
475	struct nouveau_fence *fence = to_nouveau_fence(f);
476
477	/*
478	 * caller should have a reference on the fence,
479	 * else fence could get freed here
480	 */
481	WARN_ON(kref_read(&fence->base.refcount) <= 1);
482
483	/*
484	 * This needs uevents to work correctly, but dma_fence_add_callback relies on
485	 * being able to enable signaling. It will still get signaled eventually,
486	 * just not right away.
487	 */
488	if (nouveau_fence_is_signaled(f)) {
489		list_del(&fence->head);
490
491		dma_fence_put(&fence->base);
492		return false;
493	}
494
495	return true;
496}
497
498static void nouveau_fence_release(struct dma_fence *f)
499{
500	struct nouveau_fence *fence = to_nouveau_fence(f);
501	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
502
503	kref_put(&fctx->fence_ref, nouveau_fence_context_put);
504	dma_fence_free(&fence->base);
505}
506
507static const struct dma_fence_ops nouveau_fence_ops_legacy = {
508	.get_driver_name = nouveau_fence_get_get_driver_name,
509	.get_timeline_name = nouveau_fence_get_timeline_name,
510	.enable_signaling = nouveau_fence_no_signaling,
511	.signaled = nouveau_fence_is_signaled,
512	.wait = nouveau_fence_wait_legacy,
513	.release = nouveau_fence_release
514};
515
516static bool nouveau_fence_enable_signaling(struct dma_fence *f)
517{
518	struct nouveau_fence *fence = to_nouveau_fence(f);
519	struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
520	bool ret;
521
522	if (!fctx->notify_ref++)
523		nvif_event_allow(&fctx->event);
524
525	ret = nouveau_fence_no_signaling(f);
526	if (ret)
527		set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
528	else if (!--fctx->notify_ref)
529		nvif_event_block(&fctx->event);
530
531	return ret;
532}
533
534static const struct dma_fence_ops nouveau_fence_ops_uevent = {
535	.get_driver_name = nouveau_fence_get_get_driver_name,
536	.get_timeline_name = nouveau_fence_get_timeline_name,
537	.enable_signaling = nouveau_fence_enable_signaling,
538	.signaled = nouveau_fence_is_signaled,
539	.release = nouveau_fence_release
540};