i915_gem_wait.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2016 Intel Corporation
  4 */
  5
  6#include <linux/dma-fence-array.h>
  7#include <linux/dma-fence-chain.h>
  8#include <linux/jiffies.h>
  9
 10#include "gt/intel_engine.h"
 11#include "gt/intel_rps.h"
 12
 13#include "i915_gem_ioctls.h"
 14#include "i915_gem_object.h"
 15
 16static long
 17i915_gem_object_wait_fence(struct dma_fence *fence,
 18			   unsigned int flags,
 19			   long timeout)
 20{
 21	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
 22
 23	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 24		return timeout;
 25
 26	if (dma_fence_is_i915(fence))
 27		return i915_request_wait_timeout(to_request(fence), flags, timeout);
 28
 29	return dma_fence_wait_timeout(fence,
 30				      flags & I915_WAIT_INTERRUPTIBLE,
 31				      timeout);
 32}
 33
 34static void
 35i915_gem_object_boost(struct dma_resv *resv, unsigned int flags)
 36{
 37	struct dma_resv_iter cursor;
 38	struct dma_fence *fence;
 39
 40	/*
 41	 * Prescan all fences for potential boosting before we begin waiting.
 42	 *
 43	 * When we wait, we wait on outstanding fences serially. If the
 44	 * dma-resv contains a sequence such as 1:1, 1:2 instead of a reduced
 45	 * form 1:2, then as we look at each wait in turn we see that each
 46	 * request is currently executing and not worthy of boosting. But if
 47	 * we only happen to look at the final fence in the sequence (because
 48	 * of request coalescing or splitting between read/write arrays by
 49	 * the iterator), then we would boost. As such our decision to boost
 50	 * or not is delicately balanced on the order we wait on fences.
 51	 *
 52	 * So instead of looking for boosts sequentially, look for all boosts
 53	 * upfront and then wait on the outstanding fences.
 54	 */
 55
 56	dma_resv_iter_begin(&cursor, resv,
 57			    dma_resv_usage_rw(flags & I915_WAIT_ALL));
 58	dma_resv_for_each_fence_unlocked(&cursor, fence)
 59		if (dma_fence_is_i915(fence) &&
 60		    !i915_request_started(to_request(fence)))
 61			intel_rps_boost(to_request(fence));
 62	dma_resv_iter_end(&cursor);
 63}
 64
 65static long
 66i915_gem_object_wait_reservation(struct dma_resv *resv,
 67				 unsigned int flags,
 68				 long timeout)
 69{
 70	struct dma_resv_iter cursor;
 71	struct dma_fence *fence;
 72	long ret = timeout ?: 1;
 73
 74	i915_gem_object_boost(resv, flags);
 75
 76	dma_resv_iter_begin(&cursor, resv,
 77			    dma_resv_usage_rw(flags & I915_WAIT_ALL));
 78	dma_resv_for_each_fence_unlocked(&cursor, fence) {
 79		ret = i915_gem_object_wait_fence(fence, flags, timeout);
 80		if (ret <= 0)
 81			break;
 82
 83		if (timeout)
 84			timeout = ret;
 85	}
 86	dma_resv_iter_end(&cursor);
 87
 88	return ret;
 89}
 90
 91static void fence_set_priority(struct dma_fence *fence,
 92			       const struct i915_sched_attr *attr)
 93{
 94	struct i915_request *rq;
 95	struct intel_engine_cs *engine;
 96
 97	if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
 98		return;
 99
100	rq = to_request(fence);
101	engine = rq->engine;
102
103	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
104	if (engine->sched_engine->schedule)
105		engine->sched_engine->schedule(rq, attr);
106	rcu_read_unlock();
107}
108
109void i915_gem_fence_wait_priority(struct dma_fence *fence,
110				  const struct i915_sched_attr *attr)
111{
112	if (dma_fence_is_signaled(fence))
113		return;
114
115	local_bh_disable();
116
117	/* Recurse once into a fence-array */
118	if (dma_fence_is_array(fence)) {
119		struct dma_fence_array *array = to_dma_fence_array(fence);
120		int i;
121
122		for (i = 0; i < array->num_fences; i++)
123			fence_set_priority(array->fences[i], attr);
124	} else if (dma_fence_is_chain(fence)) {
125		struct dma_fence *iter;
126
127		/* The chain is ordered; if we boost the last, we boost all */
128		dma_fence_chain_for_each(iter, fence) {
129			fence_set_priority(to_dma_fence_chain(iter)->fence,
130					   attr);
131			break;
132		}
133		dma_fence_put(iter);
134	} else {
135		fence_set_priority(fence, attr);
136	}
137
138	local_bh_enable(); /* kick the tasklets if queues were reprioritised */
139}
140
141void i915_gem_fence_wait_priority_display(struct dma_fence *fence)
142{
143	struct i915_sched_attr attr = { .priority = I915_PRIORITY_DISPLAY };
144
145	i915_gem_fence_wait_priority(fence, &attr);
146}
147
148int
149i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
150			      unsigned int flags,
151			      const struct i915_sched_attr *attr)
152{
153	struct dma_resv_iter cursor;
154	struct dma_fence *fence;
155
156	dma_resv_iter_begin(&cursor, obj->base.resv,
157			    dma_resv_usage_rw(flags & I915_WAIT_ALL));
158	dma_resv_for_each_fence_unlocked(&cursor, fence)
159		i915_gem_fence_wait_priority(fence, attr);
160	dma_resv_iter_end(&cursor);
161	return 0;
162}
163
164/**
165 * i915_gem_object_wait - Waits for rendering to the object to be completed
166 * @obj: i915 gem object
167 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
168 * @timeout: how long to wait
169 */
170int
171i915_gem_object_wait(struct drm_i915_gem_object *obj,
172		     unsigned int flags,
173		     long timeout)
174{
175	might_sleep();
176	GEM_BUG_ON(timeout < 0);
177
178	timeout = i915_gem_object_wait_reservation(obj->base.resv,
179						   flags, timeout);
180
181	if (timeout < 0)
182		return timeout;
183
184	return !timeout ? -ETIME : 0;
185}
186
187static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
188{
189	/* nsecs_to_jiffies64() does not guard against overflow */
190	if ((NSEC_PER_SEC % HZ) != 0 &&
191	    div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
192		return MAX_JIFFY_OFFSET;
193
194	return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
195}
196
197static unsigned long to_wait_timeout(s64 timeout_ns)
198{
199	if (timeout_ns < 0)
200		return MAX_SCHEDULE_TIMEOUT;
201
202	if (timeout_ns == 0)
203		return 0;
204
205	return nsecs_to_jiffies_timeout(timeout_ns);
206}
207
208/**
209 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
210 * @dev: drm device pointer
211 * @data: ioctl data blob
212 * @file: drm file pointer
213 *
214 * Returns 0 if successful, else an error is returned with the remaining time in
215 * the timeout parameter.
216 *  -ETIME: object is still busy after timeout
217 *  -ERESTARTSYS: signal interrupted the wait
218 *  -ENONENT: object doesn't exist
219 * Also possible, but rare:
220 *  -EAGAIN: incomplete, restart syscall
221 *  -ENOMEM: damn
222 *  -ENODEV: Internal IRQ fail
223 *  -E?: The add request failed
224 *
225 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
226 * non-zero timeout parameter the wait ioctl will wait for the given number of
227 * nanoseconds on an object becoming unbusy. Since the wait occurs without
228 * holding a global or exclusive lock the object may become re-busied before
229 * this function completes. A similar but shorter * race condition exists
230 * in the busy ioctl
231 */
232int
233i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
234{
235	struct drm_i915_gem_wait *args = data;
236	struct drm_i915_gem_object *obj;
237	ktime_t start;
238	long ret;
239
240	if (args->flags != 0)
241		return -EINVAL;
242
243	obj = i915_gem_object_lookup(file, args->bo_handle);
244	if (!obj)
245		return -ENOENT;
246
247	start = ktime_get();
248
249	ret = i915_gem_object_wait(obj,
250				   I915_WAIT_INTERRUPTIBLE |
251				   I915_WAIT_PRIORITY |
252				   I915_WAIT_ALL,
253				   to_wait_timeout(args->timeout_ns));
254
255	if (args->timeout_ns > 0) {
256		args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
257		if (args->timeout_ns < 0)
258			args->timeout_ns = 0;
259
260		/*
261		 * Apparently ktime isn't accurate enough and occasionally has a
262		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
263		 * things up to make the test happy. We allow up to 1 jiffy.
264		 *
265		 * This is a regression from the timespec->ktime conversion.
266		 */
267		if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
268			args->timeout_ns = 0;
269
270		/* Asked to wait beyond the jiffy/scheduler precision? */
271		if (ret == -ETIME && args->timeout_ns)
272			ret = -EAGAIN;
273	}
274
275	i915_gem_object_put(obj);
276	return ret;
277}
278
279/**
280 * i915_gem_object_wait_migration - Sync an accelerated migration operation
281 * @obj: The migrating object.
282 * @flags: waiting flags. Currently supports only I915_WAIT_INTERRUPTIBLE.
283 *
284 * Wait for any pending async migration operation on the object,
285 * whether it's explicitly (i915_gem_object_migrate()) or implicitly
286 * (swapin, initial clearing) initiated.
287 *
288 * Return: 0 if successful, -ERESTARTSYS if a signal was hit during waiting.
289 */
290int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj,
291				   unsigned int flags)
292{
293	might_sleep();
294
295	return i915_gem_object_wait_moving_fence(obj, !!(flags & I915_WAIT_INTERRUPTIBLE));
296}