xe_exec.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2022 Intel Corporation
  4 */
  5
  6#include "xe_exec.h"
  7
  8#include <drm/drm_device.h>
  9#include <drm/drm_exec.h>
 10#include <drm/drm_file.h>
 11#include <uapi/drm/xe_drm.h>
 12#include <linux/delay.h>
 13
 14#include "xe_bo.h"
 15#include "xe_device.h"
 16#include "xe_exec_queue.h"
 17#include "xe_hw_engine_group.h"
 18#include "xe_macros.h"
 19#include "xe_pm.h"
 20#include "xe_ring_ops_types.h"
 21#include "xe_sched_job.h"
 22#include "xe_sync.h"
 23#include "xe_svm.h"
 24#include "xe_trace.h"
 25#include "xe_vm.h"
 26
 27/**
 28 * DOC: Execbuf (User GPU command submission)
 29 *
 30 * Execs have historically been rather complicated in DRM drivers (at least in
 31 * the i915) because a few things:
 32 *
 33 * - Passing in a list BO which are read / written to creating implicit syncs
 34 * - Binding at exec time
 35 * - Flow controlling the ring at exec time
 36 *
 37 * In Xe we avoid all of this complication by not allowing a BO list to be
 38 * passed into an exec, using the dma-buf implicit sync uAPI, have binds as
 39 * separate operations, and using the DRM scheduler to flow control the ring.
 40 * Let's deep dive on each of these.
 41 *
 42 * We can get away from a BO list by forcing the user to use in / out fences on
 43 * every exec rather than the kernel tracking dependencies of BO (e.g. if the
 44 * user knows an exec writes to a BO and reads from the BO in the next exec, it
 45 * is the user's responsibility to pass in / out fence between the two execs).
 46 *
 47 * We do not allow a user to trigger a bind at exec time rather we have a VM
 48 * bind IOCTL which uses the same in / out fence interface as exec. In that
 49 * sense, a VM bind is basically the same operation as an exec from the user
 50 * perspective. e.g. If an exec depends on a VM bind use the in / out fence
 51 * interface (struct drm_xe_sync) to synchronize like syncing between two
 52 * dependent execs.
 53 *
 54 * Although a user cannot trigger a bind, we still have to rebind userptrs in
 55 * the VM that have been invalidated since the last exec, likewise we also have
 56 * to rebind BOs that have been evicted by the kernel. We schedule these rebinds
 57 * behind any pending kernel operations on any external BOs in VM or any BOs
 58 * private to the VM. This is accomplished by the rebinds waiting on BOs
 59 * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs
 60 * slots (inflight execs are in the DMA_RESV_USAGE_BOOKKEEP for private BOs and
 61 * for external BOs).
 62 *
 63 * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute
 64 * mode VMs we use preempt fences and a rebind worker (TODO: add link).
 65 *
 66 * There is no need to flow control the ring in the exec as we write the ring at
 67 * submission time and set the DRM scheduler max job limit SIZE_OF_RING /
 68 * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the
 69 * ring is available.
 70 *
 71 * All of this results in a rather simple exec implementation.
 72 *
 73 * Flow
 74 * ~~~~
 75 *
 76 * .. code-block::
 77 *
 78 *	Parse input arguments
 79 *	Wait for any async VM bind passed as in-fences to start
 80 *	<----------------------------------------------------------------------|
 81 *	Lock global VM lock in read mode                                       |
 82 *	Pin userptrs (also finds userptr invalidated since last exec)          |
 83 *	Lock exec (VM dma-resv lock, external BOs dma-resv locks)              |
 84 *	Validate BOs that have been evicted                                    |
 85 *	Create job                                                             |
 86 *	Rebind invalidated userptrs + evicted BOs (non-compute-mode)           |
 87 *	Add rebind fence dependency to job                                     |
 88 *	Add job VM dma-resv bookkeeping slot (non-compute mode)                |
 89 *	Add job to external BOs dma-resv write slots (non-compute mode)        |
 90 *	Check if any userptrs invalidated since pin ------ Drop locks ---------|
 91 *	Install in / out fences for job
 92 *	Submit job
 93 *	Unlock all
 94 */
 95
 96/*
 97 * Add validation and rebinding to the drm_exec locking loop, since both can
 98 * trigger eviction which may require sleeping dma_resv locks.
 99 */
100static int xe_exec_fn(struct drm_gpuvm_exec *vm_exec)
101{
102	struct xe_vm *vm = container_of(vm_exec->vm, struct xe_vm, gpuvm);
103	int ret;
104
105	/* The fence slot added here is intended for the exec sched job. */
106	xe_vm_set_validation_exec(vm, &vm_exec->exec);
107	ret = xe_vm_validate_rebind(vm, &vm_exec->exec, 1);
108	xe_vm_set_validation_exec(vm, NULL);
109	return ret;
110}
111
112int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
113{
114	struct xe_device *xe = to_xe_device(dev);
115	struct xe_file *xef = to_xe_file(file);
116	struct drm_xe_exec *args = data;
117	struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs);
118	u64 __user *addresses_user = u64_to_user_ptr(args->address);
119	struct xe_exec_queue *q;
120	struct xe_sync_entry *syncs = NULL;
121	u64 addresses[XE_HW_ENGINE_MAX_INSTANCE];
122	struct drm_gpuvm_exec vm_exec = {.extra.fn = xe_exec_fn};
123	struct drm_exec *exec = &vm_exec.exec;
124	u32 i, num_syncs, num_ufence = 0;
125	struct xe_validation_ctx ctx;
126	struct xe_sched_job *job;
127	struct xe_vm *vm;
128	bool write_locked;
129	int err = 0;
130	struct xe_hw_engine_group *group;
131	enum xe_hw_engine_group_execution_mode mode, previous_mode;
132
133	if (XE_IOCTL_DBG(xe, args->extensions) ||
134	    XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) ||
135	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]) ||
136	    XE_IOCTL_DBG(xe, args->num_syncs > DRM_XE_MAX_SYNCS))
137		return -EINVAL;
138
139	q = xe_exec_queue_lookup(xef, args->exec_queue_id);
140	if (XE_IOCTL_DBG(xe, !q))
141		return -ENOENT;
142
143	if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) {
144		err = -EINVAL;
145		goto err_exec_queue;
146	}
147
148	if (XE_IOCTL_DBG(xe, args->num_batch_buffer &&
149			 q->width != args->num_batch_buffer)) {
150		err = -EINVAL;
151		goto err_exec_queue;
152	}
153
154	if (XE_IOCTL_DBG(xe, q->ops->reset_status(q))) {
155		err = -ECANCELED;
156		goto err_exec_queue;
157	}
158
159	if (atomic_read(&q->job_cnt) >= XE_MAX_JOB_COUNT_PER_EXEC_QUEUE) {
160		trace_xe_exec_queue_reach_max_job_count(q, XE_MAX_JOB_COUNT_PER_EXEC_QUEUE);
161		err = -EAGAIN;
162		goto err_exec_queue;
163	}
164
165	if (args->num_syncs) {
166		syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL);
167		if (!syncs) {
168			err = -ENOMEM;
169			goto err_exec_queue;
170		}
171	}
172
173	vm = q->vm;
174
175	for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) {
176		err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs],
177					  &syncs_user[num_syncs], NULL, 0,
178					  SYNC_PARSE_FLAG_EXEC |
179					  (xe_vm_in_lr_mode(vm) ?
180					   SYNC_PARSE_FLAG_LR_MODE : 0));
181		if (err)
182			goto err_syncs;
183
184		if (xe_sync_is_ufence(&syncs[num_syncs]))
185			num_ufence++;
186	}
187
188	if (XE_IOCTL_DBG(xe, num_ufence > 1)) {
189		err = -EINVAL;
190		goto err_syncs;
191	}
192
193	if (args->num_batch_buffer && xe_exec_queue_is_parallel(q)) {
194		err = copy_from_user(addresses, addresses_user,
195				     sizeof(u64) * q->width);
196		if (err) {
197			err = -EFAULT;
198			goto err_syncs;
199		}
200	}
201
202	group = q->hwe->hw_engine_group;
203	mode = xe_hw_engine_group_find_exec_mode(q);
204
205	if (mode == EXEC_MODE_DMA_FENCE) {
206		err = xe_hw_engine_group_get_mode(group, mode, &previous_mode);
207		if (err)
208			goto err_syncs;
209	}
210
211retry:
212	if (!xe_vm_in_lr_mode(vm) && xe_vm_userptr_check_repin(vm)) {
213		err = down_write_killable(&vm->lock);
214		write_locked = true;
215	} else {
216		/* We don't allow execs while the VM is in error state */
217		err = down_read_interruptible(&vm->lock);
218		write_locked = false;
219	}
220	if (err)
221		goto err_hw_exec_mode;
222
223	if (write_locked) {
224		err = xe_vm_userptr_pin(vm);
225		downgrade_write(&vm->lock);
226		write_locked = false;
227		if (err)
228			goto err_unlock_list;
229	}
230
231	if (!args->num_batch_buffer) {
232		err = xe_vm_lock(vm, true);
233		if (err)
234			goto err_unlock_list;
235
236		if (!xe_vm_in_lr_mode(vm)) {
237			struct dma_fence *fence;
238
239			fence = xe_sync_in_fence_get(syncs, num_syncs, q, vm);
240			if (IS_ERR(fence)) {
241				err = PTR_ERR(fence);
242				xe_vm_unlock(vm);
243				goto err_unlock_list;
244			}
245			for (i = 0; i < num_syncs; i++)
246				xe_sync_entry_signal(&syncs[i], fence);
247			xe_exec_queue_last_fence_set(q, vm, fence);
248			dma_fence_put(fence);
249		}
250
251		xe_vm_unlock(vm);
252		goto err_unlock_list;
253	}
254
255	/*
256	 * It's OK to block interruptible here with the vm lock held, since
257	 * on task freezing during suspend / hibernate, the call will
258	 * return -ERESTARTSYS and the IOCTL will be rerun.
259	 */
260	err = xe_pm_block_on_suspend(xe);
261	if (err)
262		goto err_unlock_list;
263
264	if (!xe_vm_in_lr_mode(vm)) {
265		vm_exec.vm = &vm->gpuvm;
266		vm_exec.flags = DRM_EXEC_INTERRUPTIBLE_WAIT;
267		err = xe_validation_exec_lock(&ctx, &vm_exec, &xe->val);
268		if (err)
269			goto err_unlock_list;
270	}
271
272	if (xe_vm_is_closed_or_banned(q->vm)) {
273		drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n");
274		err = -ECANCELED;
275		goto err_exec;
276	}
277
278	if (xe_exec_queue_uses_pxp(q)) {
279		err = xe_vm_validate_protected(q->vm);
280		if (err)
281			goto err_exec;
282	}
283
284	job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ?
285				  addresses : &args->address);
286	if (IS_ERR(job)) {
287		err = PTR_ERR(job);
288		goto err_exec;
289	}
290
291	/* Wait behind rebinds */
292	if (!xe_vm_in_lr_mode(vm)) {
293		err = xe_sched_job_add_deps(job,
294					    xe_vm_resv(vm),
295					    DMA_RESV_USAGE_KERNEL);
296		if (err)
297			goto err_put_job;
298	}
299
300	for (i = 0; i < num_syncs && !err; i++)
301		err = xe_sync_entry_add_deps(&syncs[i], job);
302	if (err)
303		goto err_put_job;
304
305	if (!xe_vm_in_lr_mode(vm)) {
306		err = xe_svm_notifier_lock_interruptible(vm);
307		if (err)
308			goto err_put_job;
309
310		err = __xe_vm_userptr_needs_repin(vm);
311		if (err)
312			goto err_repin;
313	}
314
315	/*
316	 * Point of no return, if we error after this point just set an error on
317	 * the job and let the DRM scheduler / backend clean up the job.
318	 */
319	xe_sched_job_arm(job);
320	if (!xe_vm_in_lr_mode(vm))
321		drm_gpuvm_resv_add_fence(&vm->gpuvm, exec, &job->drm.s_fence->finished,
322					 DMA_RESV_USAGE_BOOKKEEP,
323					 DMA_RESV_USAGE_BOOKKEEP);
324
325	for (i = 0; i < num_syncs; i++) {
326		xe_sync_entry_signal(&syncs[i], &job->drm.s_fence->finished);
327		xe_sched_job_init_user_fence(job, &syncs[i]);
328	}
329
330	if (!xe_vm_in_lr_mode(vm))
331		xe_exec_queue_last_fence_set(q, vm, &job->drm.s_fence->finished);
332	xe_sched_job_push(job);
333	xe_vm_reactivate_rebind(vm);
334
335	if (!err && !xe_vm_in_lr_mode(vm)) {
336		spin_lock(&xe->ttm.lru_lock);
337		ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
338		spin_unlock(&xe->ttm.lru_lock);
339	}
340
341	if (mode == EXEC_MODE_LR)
342		xe_hw_engine_group_resume_faulting_lr_jobs(group);
343
344err_repin:
345	if (!xe_vm_in_lr_mode(vm))
346		xe_svm_notifier_unlock(vm);
347err_put_job:
348	if (err)
349		xe_sched_job_put(job);
350err_exec:
351	if (!xe_vm_in_lr_mode(vm))
352		xe_validation_ctx_fini(&ctx);
353err_unlock_list:
354	up_read(&vm->lock);
355	if (err == -EAGAIN)
356		goto retry;
357err_hw_exec_mode:
358	if (mode == EXEC_MODE_DMA_FENCE)
359		xe_hw_engine_group_put(group);
360err_syncs:
361	while (num_syncs--)
362		xe_sync_entry_cleanup(&syncs[num_syncs]);
363	kfree(syncs);
364err_exec_queue:
365	xe_exec_queue_put(q);
366
367	return err;
368}