xe_sync.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2021 Intel Corporation
  4 */
  5
  6#include "xe_sync.h"
  7
  8#include <linux/dma-fence-array.h>
  9#include <linux/kthread.h>
 10#include <linux/sched/mm.h>
 11#include <linux/uaccess.h>
 12
 13#include <drm/drm_print.h>
 14#include <drm/drm_syncobj.h>
 15#include <uapi/drm/xe_drm.h>
 16
 17#include "xe_device.h"
 18#include "xe_exec_queue.h"
 19#include "xe_macros.h"
 20#include "xe_sched_job_types.h"
 21
 22struct xe_user_fence {
 23	struct xe_device *xe;
 24	struct kref refcount;
 25	struct dma_fence_cb cb;
 26	struct work_struct worker;
 27	struct mm_struct *mm;
 28	u64 __user *addr;
 29	u64 value;
 30	int signalled;
 31};
 32
 33static void user_fence_destroy(struct kref *kref)
 34{
 35	struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,
 36						 refcount);
 37
 38	mmdrop(ufence->mm);
 39	kfree(ufence);
 40}
 41
 42static void user_fence_get(struct xe_user_fence *ufence)
 43{
 44	kref_get(&ufence->refcount);
 45}
 46
 47static void user_fence_put(struct xe_user_fence *ufence)
 48{
 49	kref_put(&ufence->refcount, user_fence_destroy);
 50}
 51
 52static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr,
 53					       u64 value)
 54{
 55	struct xe_user_fence *ufence;
 56	u64 __user *ptr = u64_to_user_ptr(addr);
 57	u64 __maybe_unused prefetch_val;
 58
 59	if (get_user(prefetch_val, ptr))
 60		return ERR_PTR(-EFAULT);
 61
 62	ufence = kzalloc(sizeof(*ufence), GFP_KERNEL);
 63	if (!ufence)
 64		return ERR_PTR(-ENOMEM);
 65
 66	ufence->xe = xe;
 67	kref_init(&ufence->refcount);
 68	ufence->addr = ptr;
 69	ufence->value = value;
 70	ufence->mm = current->mm;
 71	mmgrab(ufence->mm);
 72
 73	return ufence;
 74}
 75
 76static void user_fence_worker(struct work_struct *w)
 77{
 78	struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);
 79
 80	WRITE_ONCE(ufence->signalled, 1);
 81	if (mmget_not_zero(ufence->mm)) {
 82		kthread_use_mm(ufence->mm);
 83		if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))
 84			XE_WARN_ON("Copy to user failed");
 85		kthread_unuse_mm(ufence->mm);
 86		mmput(ufence->mm);
 87	} else {
 88		drm_dbg(&ufence->xe->drm, "mmget_not_zero() failed, ufence wasn't signaled\n");
 89	}
 90
 91	/*
 92	 * Wake up waiters only after updating the ufence state, allowing the UMD
 93	 * to safely reuse the same ufence without encountering -EBUSY errors.
 94	 */
 95	wake_up_all(&ufence->xe->ufence_wq);
 96	user_fence_put(ufence);
 97}
 98
 99static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence)
100{
101	INIT_WORK(&ufence->worker, user_fence_worker);
102	queue_work(ufence->xe->ordered_wq, &ufence->worker);
103	dma_fence_put(fence);
104}
105
106static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
107{
108	struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);
109
110	kick_ufence(ufence, fence);
111}
112
113int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
114			struct xe_sync_entry *sync,
115			struct drm_xe_sync __user *sync_user,
116			struct drm_syncobj *ufence_syncobj,
117			u64 ufence_timeline_value,
118			unsigned int flags)
119{
120	struct drm_xe_sync sync_in;
121	int err;
122	bool exec = flags & SYNC_PARSE_FLAG_EXEC;
123	bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;
124	bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;
125	bool signal;
126
127	if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))
128		return -EFAULT;
129
130	if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) ||
131	    XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))
132		return -EINVAL;
133
134	signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;
135	switch (sync_in.type) {
136	case DRM_XE_SYNC_TYPE_SYNCOBJ:
137		if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
138			return -EOPNOTSUPP;
139
140		if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
141			return -EINVAL;
142
143		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
144		if (XE_IOCTL_DBG(xe, !sync->syncobj))
145			return -ENOENT;
146
147		if (!signal) {
148			sync->fence = drm_syncobj_fence_get(sync->syncobj);
149			if (XE_IOCTL_DBG(xe, !sync->fence)) {
150				err = -EINVAL;
151				goto free_sync;
152			}
153		}
154		break;
155
156	case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:
157		if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
158			return -EOPNOTSUPP;
159
160		if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
161			return -EINVAL;
162
163		if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))
164			return -EINVAL;
165
166		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
167		if (XE_IOCTL_DBG(xe, !sync->syncobj))
168			return -ENOENT;
169
170		if (signal) {
171			sync->chain_fence = dma_fence_chain_alloc();
172			if (!sync->chain_fence) {
173				err = -ENOMEM;
174				goto free_sync;
175			}
176		} else {
177			sync->fence = drm_syncobj_fence_get(sync->syncobj);
178			if (XE_IOCTL_DBG(xe, !sync->fence)) {
179				err = -EINVAL;
180				goto free_sync;
181			}
182
183			err = dma_fence_chain_find_seqno(&sync->fence,
184							 sync_in.timeline_value);
185			if (err)
186				goto free_sync;
187		}
188		break;
189
190	case DRM_XE_SYNC_TYPE_USER_FENCE:
191		if (XE_IOCTL_DBG(xe, disallow_user_fence))
192			return -EOPNOTSUPP;
193
194		if (XE_IOCTL_DBG(xe, !signal))
195			return -EOPNOTSUPP;
196
197		if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))
198			return -EINVAL;
199
200		if (exec) {
201			sync->addr = sync_in.addr;
202		} else {
203			sync->ufence_timeline_value = ufence_timeline_value;
204			sync->ufence = user_fence_create(xe, sync_in.addr,
205							 sync_in.timeline_value);
206			if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence)))
207				return PTR_ERR(sync->ufence);
208			sync->ufence_chain_fence = dma_fence_chain_alloc();
209			if (!sync->ufence_chain_fence) {
210				err = -ENOMEM;
211				goto free_sync;
212			}
213			sync->ufence_syncobj = ufence_syncobj;
214		}
215
216		break;
217
218	default:
219		return -EINVAL;
220	}
221
222	sync->type = sync_in.type;
223	sync->flags = sync_in.flags;
224	sync->timeline_value = sync_in.timeline_value;
225
226	return 0;
227
228free_sync:
229	xe_sync_entry_cleanup(sync);
230	return err;
231}
232ALLOW_ERROR_INJECTION(xe_sync_entry_parse, ERRNO);
233
234int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
235{
236	if (sync->fence)
237		return  drm_sched_job_add_dependency(&job->drm,
238						     dma_fence_get(sync->fence));
239
240	return 0;
241}
242
243void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)
244{
245	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
246		return;
247
248	if (sync->chain_fence) {
249		drm_syncobj_add_point(sync->syncobj, sync->chain_fence,
250				      fence, sync->timeline_value);
251		/*
252		 * The chain's ownership is transferred to the
253		 * timeline.
254		 */
255		sync->chain_fence = NULL;
256	} else if (sync->syncobj) {
257		drm_syncobj_replace_fence(sync->syncobj, fence);
258	} else if (sync->ufence) {
259		int err;
260
261		drm_syncobj_add_point(sync->ufence_syncobj,
262				      sync->ufence_chain_fence,
263				      fence, sync->ufence_timeline_value);
264		sync->ufence_chain_fence = NULL;
265
266		fence = drm_syncobj_fence_get(sync->ufence_syncobj);
267		user_fence_get(sync->ufence);
268		err = dma_fence_add_callback(fence, &sync->ufence->cb,
269					     user_fence_cb);
270		if (err == -ENOENT) {
271			kick_ufence(sync->ufence, fence);
272		} else if (err) {
273			XE_WARN_ON("failed to add user fence");
274			user_fence_put(sync->ufence);
275			dma_fence_put(fence);
276		}
277	}
278}
279
280void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
281{
282	if (sync->syncobj)
283		drm_syncobj_put(sync->syncobj);
284	dma_fence_put(sync->fence);
285	dma_fence_chain_free(sync->chain_fence);
286	dma_fence_chain_free(sync->ufence_chain_fence);
287	if (!IS_ERR_OR_NULL(sync->ufence))
288		user_fence_put(sync->ufence);
289}
290
291/**
292 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
293 * @sync: input syncs
294 * @num_sync: number of syncs
295 * @q: exec queue
296 * @vm: VM
297 *
298 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
299 * and return a composite fence of all in-fences + last fence. If no in-fences
300 * return last fence on  input exec queue. Caller must drop reference to
301 * returned fence.
302 *
303 * Return: fence on success, ERR_PTR(-ENOMEM) on failure
304 */
305struct dma_fence *
306xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
307		     struct xe_exec_queue *q, struct xe_vm *vm)
308{
309	struct dma_fence **fences = NULL;
310	struct dma_fence_array *cf = NULL;
311	struct dma_fence *fence;
312	int i, num_fence = 0, current_fence = 0;
313
314	lockdep_assert_held(&vm->lock);
315
316	/* Reject in fences */
317	for (i = 0; i < num_sync; ++i)
318		if (sync[i].fence)
319			return ERR_PTR(-EOPNOTSUPP);
320
321	if (q->flags & EXEC_QUEUE_FLAG_VM) {
322		struct xe_exec_queue *__q;
323		struct xe_tile *tile;
324		u8 id;
325
326		for_each_tile(tile, vm->xe, id)
327			num_fence += (1 + XE_MAX_GT_PER_TILE);
328
329		fences = kmalloc_array(num_fence, sizeof(*fences),
330				       GFP_KERNEL);
331		if (!fences)
332			return ERR_PTR(-ENOMEM);
333
334		fences[current_fence++] =
335			xe_exec_queue_last_fence_get(q, vm);
336		for_each_tlb_inval(i)
337			fences[current_fence++] =
338				xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
339		list_for_each_entry(__q, &q->multi_gt_list,
340				    multi_gt_link) {
341			fences[current_fence++] =
342				xe_exec_queue_last_fence_get(__q, vm);
343			for_each_tlb_inval(i)
344				fences[current_fence++] =
345					xe_exec_queue_tlb_inval_last_fence_get(__q, vm, i);
346		}
347
348		xe_assert(vm->xe, current_fence == num_fence);
349		cf = dma_fence_array_create(num_fence, fences,
350					    dma_fence_context_alloc(1),
351					    1, false);
352		if (!cf)
353			goto err_out;
354
355		return &cf->base;
356	}
357
358	fence = xe_exec_queue_last_fence_get(q, vm);
359	return fence;
360
361err_out:
362	while (current_fence)
363		dma_fence_put(fences[--current_fence]);
364	kfree(fences);
365
366	return ERR_PTR(-ENOMEM);
367}
368
369/**
370 * __xe_sync_ufence_get() - Get user fence from user fence
371 * @ufence: input user fence
372 *
373 * Get a user fence reference from user fence
374 *
375 * Return: xe_user_fence pointer with reference
376 */
377struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence)
378{
379	user_fence_get(ufence);
380
381	return ufence;
382}
383
384/**
385 * xe_sync_ufence_get() - Get user fence from sync
386 * @sync: input sync
387 *
388 * Get a user fence reference from sync.
389 *
390 * Return: xe_user_fence pointer with reference
391 */
392struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)
393{
394	user_fence_get(sync->ufence);
395
396	return sync->ufence;
397}
398
399/**
400 * xe_sync_ufence_put() - Put user fence reference
401 * @ufence: user fence reference
402 *
403 */
404void xe_sync_ufence_put(struct xe_user_fence *ufence)
405{
406	user_fence_put(ufence);
407}
408
409/**
410 * xe_sync_ufence_get_status() - Get user fence status
411 * @ufence: user fence
412 *
413 * Return: 1 if signalled, 0 not signalled, <0 on error
414 */
415int xe_sync_ufence_get_status(struct xe_user_fence *ufence)
416{
417	return READ_ONCE(ufence->signalled);
418}