amdgpu_userq_fence.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright 2023 Advanced Micro Devices, Inc.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice shall be included in
  13 * all copies or substantial portions of the Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 * OTHER DEALINGS IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <linux/kref.h>
  26#include <linux/slab.h>
  27#include <linux/dma-fence-unwrap.h>
  28
  29#include <drm/drm_exec.h>
  30#include <drm/drm_syncobj.h>
  31
  32#include "amdgpu.h"
  33#include "amdgpu_userq_fence.h"
  34
  35static const struct dma_fence_ops amdgpu_userq_fence_ops;
  36static struct kmem_cache *amdgpu_userq_fence_slab;
  37
  38#define AMDGPU_USERQ_MAX_HANDLES	(1U << 16)
  39
  40int amdgpu_userq_fence_slab_init(void)
  41{
  42	amdgpu_userq_fence_slab = kmem_cache_create("amdgpu_userq_fence",
  43						    sizeof(struct amdgpu_userq_fence),
  44						    0,
  45						    SLAB_HWCACHE_ALIGN,
  46						    NULL);
  47	if (!amdgpu_userq_fence_slab)
  48		return -ENOMEM;
  49
  50	return 0;
  51}
  52
  53void amdgpu_userq_fence_slab_fini(void)
  54{
  55	rcu_barrier();
  56	kmem_cache_destroy(amdgpu_userq_fence_slab);
  57}
  58
  59static inline struct amdgpu_userq_fence *to_amdgpu_userq_fence(struct dma_fence *f)
  60{
  61	if (!f || f->ops != &amdgpu_userq_fence_ops)
  62		return NULL;
  63
  64	return container_of(f, struct amdgpu_userq_fence, base);
  65}
  66
  67static u64 amdgpu_userq_fence_read(struct amdgpu_userq_fence_driver *fence_drv)
  68{
  69	return le64_to_cpu(*fence_drv->cpu_addr);
  70}
  71
  72static void
  73amdgpu_userq_fence_write(struct amdgpu_userq_fence_driver *fence_drv,
  74			 u64 seq)
  75{
  76	if (fence_drv->cpu_addr)
  77		*fence_drv->cpu_addr = cpu_to_le64(seq);
  78}
  79
  80int amdgpu_userq_fence_driver_alloc(struct amdgpu_device *adev,
  81				    struct amdgpu_usermode_queue *userq)
  82{
  83	struct amdgpu_userq_fence_driver *fence_drv;
  84	unsigned long flags;
  85	int r;
  86
  87	fence_drv = kzalloc(sizeof(*fence_drv), GFP_KERNEL);
  88	if (!fence_drv)
  89		return -ENOMEM;
  90
  91	/* Acquire seq64 memory */
  92	r = amdgpu_seq64_alloc(adev, &fence_drv->va, &fence_drv->gpu_addr,
  93			       &fence_drv->cpu_addr);
  94	if (r)
  95		goto free_fence_drv;
  96
  97	memset(fence_drv->cpu_addr, 0, sizeof(u64));
  98
  99	kref_init(&fence_drv->refcount);
 100	INIT_LIST_HEAD(&fence_drv->fences);
 101	spin_lock_init(&fence_drv->fence_list_lock);
 102
 103	fence_drv->adev = adev;
 104	fence_drv->context = dma_fence_context_alloc(1);
 105	get_task_comm(fence_drv->timeline_name, current);
 106
 107	xa_lock_irqsave(&adev->userq_xa, flags);
 108	r = xa_err(__xa_store(&adev->userq_xa, userq->doorbell_index,
 109			      fence_drv, GFP_KERNEL));
 110	xa_unlock_irqrestore(&adev->userq_xa, flags);
 111	if (r)
 112		goto free_seq64;
 113
 114	userq->fence_drv = fence_drv;
 115
 116	return 0;
 117
 118free_seq64:
 119	amdgpu_seq64_free(adev, fence_drv->va);
 120free_fence_drv:
 121	kfree(fence_drv);
 122
 123	return r;
 124}
 125
 126static void amdgpu_userq_walk_and_drop_fence_drv(struct xarray *xa)
 127{
 128	struct amdgpu_userq_fence_driver *fence_drv;
 129	unsigned long index;
 130
 131	if (xa_empty(xa))
 132		return;
 133
 134	xa_lock(xa);
 135	xa_for_each(xa, index, fence_drv) {
 136		__xa_erase(xa, index);
 137		amdgpu_userq_fence_driver_put(fence_drv);
 138	}
 139
 140	xa_unlock(xa);
 141}
 142
 143void
 144amdgpu_userq_fence_driver_free(struct amdgpu_usermode_queue *userq)
 145{
 146	dma_fence_put(userq->last_fence);
 147
 148	amdgpu_userq_walk_and_drop_fence_drv(&userq->fence_drv_xa);
 149	xa_destroy(&userq->fence_drv_xa);
 150	/* Drop the fence_drv reference held by user queue */
 151	amdgpu_userq_fence_driver_put(userq->fence_drv);
 152}
 153
 154void amdgpu_userq_fence_driver_process(struct amdgpu_userq_fence_driver *fence_drv)
 155{
 156	struct amdgpu_userq_fence *userq_fence, *tmp;
 157	struct dma_fence *fence;
 158	unsigned long flags;
 159	u64 rptr;
 160	int i;
 161
 162	if (!fence_drv)
 163		return;
 164
 165	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
 166	rptr = amdgpu_userq_fence_read(fence_drv);
 167
 168	list_for_each_entry_safe(userq_fence, tmp, &fence_drv->fences, link) {
 169		fence = &userq_fence->base;
 170
 171		if (rptr < fence->seqno)
 172			break;
 173
 174		dma_fence_signal(fence);
 175
 176		for (i = 0; i < userq_fence->fence_drv_array_count; i++)
 177			amdgpu_userq_fence_driver_put(userq_fence->fence_drv_array[i]);
 178
 179		list_del(&userq_fence->link);
 180		dma_fence_put(fence);
 181	}
 182	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
 183}
 184
 185void amdgpu_userq_fence_driver_destroy(struct kref *ref)
 186{
 187	struct amdgpu_userq_fence_driver *fence_drv = container_of(ref,
 188					 struct amdgpu_userq_fence_driver,
 189					 refcount);
 190	struct amdgpu_userq_fence_driver *xa_fence_drv;
 191	struct amdgpu_device *adev = fence_drv->adev;
 192	struct amdgpu_userq_fence *fence, *tmp;
 193	struct xarray *xa = &adev->userq_xa;
 194	unsigned long index, flags;
 195	struct dma_fence *f;
 196
 197	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
 198	list_for_each_entry_safe(fence, tmp, &fence_drv->fences, link) {
 199		f = &fence->base;
 200
 201		if (!dma_fence_is_signaled(f)) {
 202			dma_fence_set_error(f, -ECANCELED);
 203			dma_fence_signal(f);
 204		}
 205
 206		list_del(&fence->link);
 207		dma_fence_put(f);
 208	}
 209	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
 210
 211	xa_lock_irqsave(xa, flags);
 212	xa_for_each(xa, index, xa_fence_drv)
 213		if (xa_fence_drv == fence_drv)
 214			__xa_erase(xa, index);
 215	xa_unlock_irqrestore(xa, flags);
 216
 217	/* Free seq64 memory */
 218	amdgpu_seq64_free(adev, fence_drv->va);
 219	kfree(fence_drv);
 220}
 221
 222void amdgpu_userq_fence_driver_get(struct amdgpu_userq_fence_driver *fence_drv)
 223{
 224	kref_get(&fence_drv->refcount);
 225}
 226
 227void amdgpu_userq_fence_driver_put(struct amdgpu_userq_fence_driver *fence_drv)
 228{
 229	kref_put(&fence_drv->refcount, amdgpu_userq_fence_driver_destroy);
 230}
 231
 232static int amdgpu_userq_fence_alloc(struct amdgpu_userq_fence **userq_fence)
 233{
 234	*userq_fence = kmem_cache_alloc(amdgpu_userq_fence_slab, GFP_ATOMIC);
 235	return *userq_fence ? 0 : -ENOMEM;
 236}
 237
 238static int amdgpu_userq_fence_create(struct amdgpu_usermode_queue *userq,
 239				     struct amdgpu_userq_fence *userq_fence,
 240				     u64 seq, struct dma_fence **f)
 241{
 242	struct amdgpu_userq_fence_driver *fence_drv;
 243	struct dma_fence *fence;
 244	unsigned long flags;
 245
 246	fence_drv = userq->fence_drv;
 247	if (!fence_drv)
 248		return -EINVAL;
 249
 250	spin_lock_init(&userq_fence->lock);
 251	INIT_LIST_HEAD(&userq_fence->link);
 252	fence = &userq_fence->base;
 253	userq_fence->fence_drv = fence_drv;
 254
 255	dma_fence_init64(fence, &amdgpu_userq_fence_ops, &userq_fence->lock,
 256			 fence_drv->context, seq);
 257
 258	amdgpu_userq_fence_driver_get(fence_drv);
 259	dma_fence_get(fence);
 260
 261	if (!xa_empty(&userq->fence_drv_xa)) {
 262		struct amdgpu_userq_fence_driver *stored_fence_drv;
 263		unsigned long index, count = 0;
 264		int i = 0;
 265
 266		xa_lock(&userq->fence_drv_xa);
 267		xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv)
 268			count++;
 269
 270		userq_fence->fence_drv_array =
 271			kvmalloc_array(count,
 272				       sizeof(struct amdgpu_userq_fence_driver *),
 273				       GFP_ATOMIC);
 274
 275		if (userq_fence->fence_drv_array) {
 276			xa_for_each(&userq->fence_drv_xa, index, stored_fence_drv) {
 277				userq_fence->fence_drv_array[i] = stored_fence_drv;
 278				__xa_erase(&userq->fence_drv_xa, index);
 279				i++;
 280			}
 281		}
 282
 283		userq_fence->fence_drv_array_count = i;
 284		xa_unlock(&userq->fence_drv_xa);
 285	} else {
 286		userq_fence->fence_drv_array = NULL;
 287		userq_fence->fence_drv_array_count = 0;
 288	}
 289
 290	/* Check if hardware has already processed the job */
 291	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
 292	if (!dma_fence_is_signaled(fence))
 293		list_add_tail(&userq_fence->link, &fence_drv->fences);
 294	else
 295		dma_fence_put(fence);
 296
 297	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
 298
 299	*f = fence;
 300
 301	return 0;
 302}
 303
 304static const char *amdgpu_userq_fence_get_driver_name(struct dma_fence *f)
 305{
 306	return "amdgpu_userq_fence";
 307}
 308
 309static const char *amdgpu_userq_fence_get_timeline_name(struct dma_fence *f)
 310{
 311	struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
 312
 313	return fence->fence_drv->timeline_name;
 314}
 315
 316static bool amdgpu_userq_fence_signaled(struct dma_fence *f)
 317{
 318	struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
 319	struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
 320	u64 rptr, wptr;
 321
 322	rptr = amdgpu_userq_fence_read(fence_drv);
 323	wptr = fence->base.seqno;
 324
 325	if (rptr >= wptr)
 326		return true;
 327
 328	return false;
 329}
 330
 331static void amdgpu_userq_fence_free(struct rcu_head *rcu)
 332{
 333	struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
 334	struct amdgpu_userq_fence *userq_fence = to_amdgpu_userq_fence(fence);
 335	struct amdgpu_userq_fence_driver *fence_drv = userq_fence->fence_drv;
 336
 337	/* Release the fence driver reference */
 338	amdgpu_userq_fence_driver_put(fence_drv);
 339
 340	kvfree(userq_fence->fence_drv_array);
 341	kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
 342}
 343
 344static void amdgpu_userq_fence_release(struct dma_fence *f)
 345{
 346	call_rcu(&f->rcu, amdgpu_userq_fence_free);
 347}
 348
 349static const struct dma_fence_ops amdgpu_userq_fence_ops = {
 350	.get_driver_name = amdgpu_userq_fence_get_driver_name,
 351	.get_timeline_name = amdgpu_userq_fence_get_timeline_name,
 352	.signaled = amdgpu_userq_fence_signaled,
 353	.release = amdgpu_userq_fence_release,
 354};
 355
 356/**
 357 * amdgpu_userq_fence_read_wptr - Read the userq wptr value
 358 *
 359 * @queue: user mode queue structure pointer
 360 * @wptr: write pointer value
 361 *
 362 * Read the wptr value from userq's MQD. The userq signal IOCTL
 363 * creates a dma_fence for the shared buffers that expects the
 364 * RPTR value written to seq64 memory >= WPTR.
 365 *
 366 * Returns wptr value on success, error on failure.
 367 */
 368static int amdgpu_userq_fence_read_wptr(struct amdgpu_usermode_queue *queue,
 369					u64 *wptr)
 370{
 371	struct amdgpu_bo_va_mapping *mapping;
 372	struct amdgpu_bo *bo;
 373	u64 addr, *ptr;
 374	int r;
 375
 376	r = amdgpu_bo_reserve(queue->vm->root.bo, false);
 377	if (r)
 378		return r;
 379
 380	addr = queue->userq_prop->wptr_gpu_addr;
 381	addr &= AMDGPU_GMC_HOLE_MASK;
 382
 383	mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT);
 384	if (!mapping) {
 385		amdgpu_bo_unreserve(queue->vm->root.bo);
 386		DRM_ERROR("Failed to lookup amdgpu_bo_va_mapping\n");
 387		return -EINVAL;
 388	}
 389
 390	bo = amdgpu_bo_ref(mapping->bo_va->base.bo);
 391	amdgpu_bo_unreserve(queue->vm->root.bo);
 392	r = amdgpu_bo_reserve(bo, true);
 393	if (r) {
 394		amdgpu_bo_unref(&bo);
 395		DRM_ERROR("Failed to reserve userqueue wptr bo");
 396		return r;
 397	}
 398
 399	r = amdgpu_bo_kmap(bo, (void **)&ptr);
 400	if (r) {
 401		DRM_ERROR("Failed mapping the userqueue wptr bo");
 402		goto map_error;
 403	}
 404
 405	*wptr = le64_to_cpu(*ptr);
 406
 407	amdgpu_bo_kunmap(bo);
 408	amdgpu_bo_unreserve(bo);
 409	amdgpu_bo_unref(&bo);
 410
 411	return 0;
 412
 413map_error:
 414	amdgpu_bo_unreserve(bo);
 415	amdgpu_bo_unref(&bo);
 416
 417	return r;
 418}
 419
 420static void amdgpu_userq_fence_cleanup(struct dma_fence *fence)
 421{
 422	dma_fence_put(fence);
 423}
 424
 425static void
 426amdgpu_userq_fence_driver_set_error(struct amdgpu_userq_fence *fence,
 427				    int error)
 428{
 429	struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
 430	unsigned long flags;
 431	struct dma_fence *f;
 432
 433	spin_lock_irqsave(&fence_drv->fence_list_lock, flags);
 434
 435	f = rcu_dereference_protected(&fence->base,
 436				      lockdep_is_held(&fence_drv->fence_list_lock));
 437	if (f && !dma_fence_is_signaled_locked(f))
 438		dma_fence_set_error(f, error);
 439	spin_unlock_irqrestore(&fence_drv->fence_list_lock, flags);
 440}
 441
 442void
 443amdgpu_userq_fence_driver_force_completion(struct amdgpu_usermode_queue *userq)
 444{
 445	struct dma_fence *f = userq->last_fence;
 446
 447	if (f) {
 448		struct amdgpu_userq_fence *fence = to_amdgpu_userq_fence(f);
 449		struct amdgpu_userq_fence_driver *fence_drv = fence->fence_drv;
 450		u64 wptr = fence->base.seqno;
 451
 452		amdgpu_userq_fence_driver_set_error(fence, -ECANCELED);
 453		amdgpu_userq_fence_write(fence_drv, wptr);
 454		amdgpu_userq_fence_driver_process(fence_drv);
 455
 456	}
 457}
 458
 459int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data,
 460			      struct drm_file *filp)
 461{
 462	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 463	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
 464	struct drm_amdgpu_userq_signal *args = data;
 465	struct drm_gem_object **gobj_write = NULL;
 466	struct drm_gem_object **gobj_read = NULL;
 467	struct amdgpu_usermode_queue *queue;
 468	struct amdgpu_userq_fence *userq_fence;
 469	struct drm_syncobj **syncobj = NULL;
 470	u32 *bo_handles_write, num_write_bo_handles;
 471	u32 *syncobj_handles, num_syncobj_handles;
 472	u32 *bo_handles_read, num_read_bo_handles;
 473	int r, i, entry, rentry, wentry;
 474	struct dma_fence *fence;
 475	struct drm_exec exec;
 476	u64 wptr;
 477
 478	if (!amdgpu_userq_enabled(dev))
 479		return -ENOTSUPP;
 480
 481	if (args->num_syncobj_handles > AMDGPU_USERQ_MAX_HANDLES ||
 482	    args->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
 483	    args->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
 484		return -EINVAL;
 485
 486	num_syncobj_handles = args->num_syncobj_handles;
 487	syncobj_handles = memdup_user(u64_to_user_ptr(args->syncobj_handles),
 488				      size_mul(sizeof(u32), num_syncobj_handles));
 489	if (IS_ERR(syncobj_handles))
 490		return PTR_ERR(syncobj_handles);
 491
 492	/* Array of pointers to the looked up syncobjs */
 493	syncobj = kmalloc_array(num_syncobj_handles, sizeof(*syncobj), GFP_KERNEL);
 494	if (!syncobj) {
 495		r = -ENOMEM;
 496		goto free_syncobj_handles;
 497	}
 498
 499	for (entry = 0; entry < num_syncobj_handles; entry++) {
 500		syncobj[entry] = drm_syncobj_find(filp, syncobj_handles[entry]);
 501		if (!syncobj[entry]) {
 502			r = -ENOENT;
 503			goto free_syncobj;
 504		}
 505	}
 506
 507	num_read_bo_handles = args->num_bo_read_handles;
 508	bo_handles_read = memdup_user(u64_to_user_ptr(args->bo_read_handles),
 509				      sizeof(u32) * num_read_bo_handles);
 510	if (IS_ERR(bo_handles_read)) {
 511		r = PTR_ERR(bo_handles_read);
 512		goto free_syncobj;
 513	}
 514
 515	/* Array of pointers to the GEM read objects */
 516	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL);
 517	if (!gobj_read) {
 518		r = -ENOMEM;
 519		goto free_bo_handles_read;
 520	}
 521
 522	for (rentry = 0; rentry < num_read_bo_handles; rentry++) {
 523		gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]);
 524		if (!gobj_read[rentry]) {
 525			r = -ENOENT;
 526			goto put_gobj_read;
 527		}
 528	}
 529
 530	num_write_bo_handles = args->num_bo_write_handles;
 531	bo_handles_write = memdup_user(u64_to_user_ptr(args->bo_write_handles),
 532				       sizeof(u32) * num_write_bo_handles);
 533	if (IS_ERR(bo_handles_write)) {
 534		r = PTR_ERR(bo_handles_write);
 535		goto put_gobj_read;
 536	}
 537
 538	/* Array of pointers to the GEM write objects */
 539	gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL);
 540	if (!gobj_write) {
 541		r = -ENOMEM;
 542		goto free_bo_handles_write;
 543	}
 544
 545	for (wentry = 0; wentry < num_write_bo_handles; wentry++) {
 546		gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]);
 547		if (!gobj_write[wentry]) {
 548			r = -ENOENT;
 549			goto put_gobj_write;
 550		}
 551	}
 552
 553	/* Retrieve the user queue */
 554	queue = xa_load(&userq_mgr->userq_mgr_xa, args->queue_id);
 555	if (!queue) {
 556		r = -ENOENT;
 557		goto put_gobj_write;
 558	}
 559
 560	r = amdgpu_userq_fence_read_wptr(queue, &wptr);
 561	if (r)
 562		goto put_gobj_write;
 563
 564	r = amdgpu_userq_fence_alloc(&userq_fence);
 565	if (r)
 566		goto put_gobj_write;
 567
 568	/* We are here means UQ is active, make sure the eviction fence is valid */
 569	amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
 570
 571	/* Create a new fence */
 572	r = amdgpu_userq_fence_create(queue, userq_fence, wptr, &fence);
 573	if (r) {
 574		mutex_unlock(&userq_mgr->userq_mutex);
 575		kmem_cache_free(amdgpu_userq_fence_slab, userq_fence);
 576		goto put_gobj_write;
 577	}
 578
 579	dma_fence_put(queue->last_fence);
 580	queue->last_fence = dma_fence_get(fence);
 581	mutex_unlock(&userq_mgr->userq_mutex);
 582
 583	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
 584		      (num_read_bo_handles + num_write_bo_handles));
 585
 586	/* Lock all BOs with retry handling */
 587	drm_exec_until_all_locked(&exec) {
 588		r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
 589		drm_exec_retry_on_contention(&exec);
 590		if (r) {
 591			amdgpu_userq_fence_cleanup(fence);
 592			goto exec_fini;
 593		}
 594
 595		r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
 596		drm_exec_retry_on_contention(&exec);
 597		if (r) {
 598			amdgpu_userq_fence_cleanup(fence);
 599			goto exec_fini;
 600		}
 601	}
 602
 603	for (i = 0; i < num_read_bo_handles; i++) {
 604		if (!gobj_read || !gobj_read[i]->resv)
 605			continue;
 606
 607		dma_resv_add_fence(gobj_read[i]->resv, fence,
 608				   DMA_RESV_USAGE_READ);
 609	}
 610
 611	for (i = 0; i < num_write_bo_handles; i++) {
 612		if (!gobj_write || !gobj_write[i]->resv)
 613			continue;
 614
 615		dma_resv_add_fence(gobj_write[i]->resv, fence,
 616				   DMA_RESV_USAGE_WRITE);
 617	}
 618
 619	/* Add the created fence to syncobj/BO's */
 620	for (i = 0; i < num_syncobj_handles; i++)
 621		drm_syncobj_replace_fence(syncobj[i], fence);
 622
 623	/* drop the reference acquired in fence creation function */
 624	dma_fence_put(fence);
 625
 626exec_fini:
 627	drm_exec_fini(&exec);
 628put_gobj_write:
 629	while (wentry-- > 0)
 630		drm_gem_object_put(gobj_write[wentry]);
 631	kfree(gobj_write);
 632free_bo_handles_write:
 633	kfree(bo_handles_write);
 634put_gobj_read:
 635	while (rentry-- > 0)
 636		drm_gem_object_put(gobj_read[rentry]);
 637	kfree(gobj_read);
 638free_bo_handles_read:
 639	kfree(bo_handles_read);
 640free_syncobj:
 641	while (entry-- > 0)
 642		if (syncobj[entry])
 643			drm_syncobj_put(syncobj[entry]);
 644	kfree(syncobj);
 645free_syncobj_handles:
 646	kfree(syncobj_handles);
 647
 648	return r;
 649}
 650
 651int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data,
 652			    struct drm_file *filp)
 653{
 654	u32 *syncobj_handles, *timeline_points, *timeline_handles, *bo_handles_read, *bo_handles_write;
 655	u32 num_syncobj, num_read_bo_handles, num_write_bo_handles;
 656	struct drm_amdgpu_userq_fence_info *fence_info = NULL;
 657	struct drm_amdgpu_userq_wait *wait_info = data;
 658	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 659	struct amdgpu_userq_mgr *userq_mgr = &fpriv->userq_mgr;
 660	struct amdgpu_usermode_queue *waitq;
 661	struct drm_gem_object **gobj_write;
 662	struct drm_gem_object **gobj_read;
 663	struct dma_fence **fences = NULL;
 664	u16 num_points, num_fences = 0;
 665	int r, i, rentry, wentry, cnt;
 666	struct drm_exec exec;
 667
 668	if (!amdgpu_userq_enabled(dev))
 669		return -ENOTSUPP;
 670
 671	if (wait_info->num_syncobj_handles > AMDGPU_USERQ_MAX_HANDLES ||
 672	    wait_info->num_bo_write_handles > AMDGPU_USERQ_MAX_HANDLES ||
 673	    wait_info->num_bo_read_handles > AMDGPU_USERQ_MAX_HANDLES)
 674		return -EINVAL;
 675
 676	num_read_bo_handles = wait_info->num_bo_read_handles;
 677	bo_handles_read = memdup_user(u64_to_user_ptr(wait_info->bo_read_handles),
 678				      size_mul(sizeof(u32), num_read_bo_handles));
 679	if (IS_ERR(bo_handles_read))
 680		return PTR_ERR(bo_handles_read);
 681
 682	num_write_bo_handles = wait_info->num_bo_write_handles;
 683	bo_handles_write = memdup_user(u64_to_user_ptr(wait_info->bo_write_handles),
 684				       size_mul(sizeof(u32), num_write_bo_handles));
 685	if (IS_ERR(bo_handles_write)) {
 686		r = PTR_ERR(bo_handles_write);
 687		goto free_bo_handles_read;
 688	}
 689
 690	num_syncobj = wait_info->num_syncobj_handles;
 691	syncobj_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_handles),
 692				      size_mul(sizeof(u32), num_syncobj));
 693	if (IS_ERR(syncobj_handles)) {
 694		r = PTR_ERR(syncobj_handles);
 695		goto free_bo_handles_write;
 696	}
 697
 698	num_points = wait_info->num_syncobj_timeline_handles;
 699	timeline_handles = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_handles),
 700				       sizeof(u32) * num_points);
 701	if (IS_ERR(timeline_handles)) {
 702		r = PTR_ERR(timeline_handles);
 703		goto free_syncobj_handles;
 704	}
 705
 706	timeline_points = memdup_user(u64_to_user_ptr(wait_info->syncobj_timeline_points),
 707				      sizeof(u32) * num_points);
 708	if (IS_ERR(timeline_points)) {
 709		r = PTR_ERR(timeline_points);
 710		goto free_timeline_handles;
 711	}
 712
 713	gobj_read = kmalloc_array(num_read_bo_handles, sizeof(*gobj_read), GFP_KERNEL);
 714	if (!gobj_read) {
 715		r = -ENOMEM;
 716		goto free_timeline_points;
 717	}
 718
 719	for (rentry = 0; rentry < num_read_bo_handles; rentry++) {
 720		gobj_read[rentry] = drm_gem_object_lookup(filp, bo_handles_read[rentry]);
 721		if (!gobj_read[rentry]) {
 722			r = -ENOENT;
 723			goto put_gobj_read;
 724		}
 725	}
 726
 727	gobj_write = kmalloc_array(num_write_bo_handles, sizeof(*gobj_write), GFP_KERNEL);
 728	if (!gobj_write) {
 729		r = -ENOMEM;
 730		goto put_gobj_read;
 731	}
 732
 733	for (wentry = 0; wentry < num_write_bo_handles; wentry++) {
 734		gobj_write[wentry] = drm_gem_object_lookup(filp, bo_handles_write[wentry]);
 735		if (!gobj_write[wentry]) {
 736			r = -ENOENT;
 737			goto put_gobj_write;
 738		}
 739	}
 740
 741	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT,
 742		      (num_read_bo_handles + num_write_bo_handles));
 743
 744	/* Lock all BOs with retry handling */
 745	drm_exec_until_all_locked(&exec) {
 746		r = drm_exec_prepare_array(&exec, gobj_read, num_read_bo_handles, 1);
 747		drm_exec_retry_on_contention(&exec);
 748		if (r) {
 749			drm_exec_fini(&exec);
 750			goto put_gobj_write;
 751		}
 752
 753		r = drm_exec_prepare_array(&exec, gobj_write, num_write_bo_handles, 1);
 754		drm_exec_retry_on_contention(&exec);
 755		if (r) {
 756			drm_exec_fini(&exec);
 757			goto put_gobj_write;
 758		}
 759	}
 760
 761	if (!wait_info->num_fences) {
 762		if (num_points) {
 763			struct dma_fence_unwrap iter;
 764			struct dma_fence *fence;
 765			struct dma_fence *f;
 766
 767			for (i = 0; i < num_points; i++) {
 768				r = drm_syncobj_find_fence(filp, timeline_handles[i],
 769							   timeline_points[i],
 770							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
 771							   &fence);
 772				if (r)
 773					goto exec_fini;
 774
 775				dma_fence_unwrap_for_each(f, &iter, fence)
 776					num_fences++;
 777
 778				dma_fence_put(fence);
 779			}
 780		}
 781
 782		/* Count syncobj's fence */
 783		for (i = 0; i < num_syncobj; i++) {
 784			struct dma_fence *fence;
 785
 786			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
 787						   0,
 788						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
 789						   &fence);
 790			if (r)
 791				goto exec_fini;
 792
 793			num_fences++;
 794			dma_fence_put(fence);
 795		}
 796
 797		/* Count GEM objects fence */
 798		for (i = 0; i < num_read_bo_handles; i++) {
 799			struct dma_resv_iter resv_cursor;
 800			struct dma_fence *fence;
 801
 802			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
 803						DMA_RESV_USAGE_READ, fence)
 804				num_fences++;
 805		}
 806
 807		for (i = 0; i < num_write_bo_handles; i++) {
 808			struct dma_resv_iter resv_cursor;
 809			struct dma_fence *fence;
 810
 811			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
 812						DMA_RESV_USAGE_WRITE, fence)
 813				num_fences++;
 814		}
 815
 816		/*
 817		 * Passing num_fences = 0 means that userspace doesn't want to
 818		 * retrieve userq_fence_info. If num_fences = 0 we skip filling
 819		 * userq_fence_info and return the actual number of fences on
 820		 * args->num_fences.
 821		 */
 822		wait_info->num_fences = num_fences;
 823	} else {
 824		/* Array of fence info */
 825		fence_info = kmalloc_array(wait_info->num_fences, sizeof(*fence_info), GFP_KERNEL);
 826		if (!fence_info) {
 827			r = -ENOMEM;
 828			goto exec_fini;
 829		}
 830
 831		/* Array of fences */
 832		fences = kmalloc_array(wait_info->num_fences, sizeof(*fences), GFP_KERNEL);
 833		if (!fences) {
 834			r = -ENOMEM;
 835			goto free_fence_info;
 836		}
 837
 838		/* Retrieve GEM read objects fence */
 839		for (i = 0; i < num_read_bo_handles; i++) {
 840			struct dma_resv_iter resv_cursor;
 841			struct dma_fence *fence;
 842
 843			dma_resv_for_each_fence(&resv_cursor, gobj_read[i]->resv,
 844						DMA_RESV_USAGE_READ, fence) {
 845				if (num_fences >= wait_info->num_fences) {
 846					r = -EINVAL;
 847					goto free_fences;
 848				}
 849
 850				fences[num_fences++] = fence;
 851				dma_fence_get(fence);
 852			}
 853		}
 854
 855		/* Retrieve GEM write objects fence */
 856		for (i = 0; i < num_write_bo_handles; i++) {
 857			struct dma_resv_iter resv_cursor;
 858			struct dma_fence *fence;
 859
 860			dma_resv_for_each_fence(&resv_cursor, gobj_write[i]->resv,
 861						DMA_RESV_USAGE_WRITE, fence) {
 862				if (num_fences >= wait_info->num_fences) {
 863					r = -EINVAL;
 864					goto free_fences;
 865				}
 866
 867				fences[num_fences++] = fence;
 868				dma_fence_get(fence);
 869			}
 870		}
 871
 872		if (num_points) {
 873			struct dma_fence_unwrap iter;
 874			struct dma_fence *fence;
 875			struct dma_fence *f;
 876
 877			for (i = 0; i < num_points; i++) {
 878				r = drm_syncobj_find_fence(filp, timeline_handles[i],
 879							   timeline_points[i],
 880							   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
 881							   &fence);
 882				if (r)
 883					goto free_fences;
 884
 885				dma_fence_unwrap_for_each(f, &iter, fence) {
 886					if (num_fences >= wait_info->num_fences) {
 887						r = -EINVAL;
 888						dma_fence_put(fence);
 889						goto free_fences;
 890					}
 891
 892					dma_fence_get(f);
 893					fences[num_fences++] = f;
 894				}
 895
 896				dma_fence_put(fence);
 897			}
 898		}
 899
 900		/* Retrieve syncobj's fence */
 901		for (i = 0; i < num_syncobj; i++) {
 902			struct dma_fence *fence;
 903
 904			r = drm_syncobj_find_fence(filp, syncobj_handles[i],
 905						   0,
 906						   DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT,
 907						   &fence);
 908			if (r)
 909				goto free_fences;
 910
 911			if (num_fences >= wait_info->num_fences) {
 912				r = -EINVAL;
 913				dma_fence_put(fence);
 914				goto free_fences;
 915			}
 916
 917			fences[num_fences++] = fence;
 918		}
 919
 920		/*
 921		 * Keep only the latest fences to reduce the number of values
 922		 * given back to userspace.
 923		 */
 924		num_fences = dma_fence_dedup_array(fences, num_fences);
 925
 926		waitq = xa_load(&userq_mgr->userq_mgr_xa, wait_info->waitq_id);
 927		if (!waitq) {
 928			r = -EINVAL;
 929			goto free_fences;
 930		}
 931
 932		for (i = 0, cnt = 0; i < num_fences; i++) {
 933			struct amdgpu_userq_fence_driver *fence_drv;
 934			struct amdgpu_userq_fence *userq_fence;
 935			u32 index;
 936
 937			userq_fence = to_amdgpu_userq_fence(fences[i]);
 938			if (!userq_fence) {
 939				/*
 940				 * Just waiting on other driver fences should
 941				 * be good for now
 942				 */
 943				r = dma_fence_wait(fences[i], true);
 944				if (r) {
 945					dma_fence_put(fences[i]);
 946					goto free_fences;
 947				}
 948
 949				dma_fence_put(fences[i]);
 950				continue;
 951			}
 952
 953			fence_drv = userq_fence->fence_drv;
 954			/*
 955			 * We need to make sure the user queue release their reference
 956			 * to the fence drivers at some point before queue destruction.
 957			 * Otherwise, we would gather those references until we don't
 958			 * have any more space left and crash.
 959			 */
 960			r = xa_alloc(&waitq->fence_drv_xa, &index, fence_drv,
 961				     xa_limit_32b, GFP_KERNEL);
 962			if (r)
 963				goto free_fences;
 964
 965			amdgpu_userq_fence_driver_get(fence_drv);
 966
 967			/* Store drm syncobj's gpu va address and value */
 968			fence_info[cnt].va = fence_drv->va;
 969			fence_info[cnt].value = fences[i]->seqno;
 970
 971			dma_fence_put(fences[i]);
 972			/* Increment the actual userq fence count */
 973			cnt++;
 974		}
 975
 976		wait_info->num_fences = cnt;
 977		/* Copy userq fence info to user space */
 978		if (copy_to_user(u64_to_user_ptr(wait_info->out_fences),
 979				 fence_info, wait_info->num_fences * sizeof(*fence_info))) {
 980			r = -EFAULT;
 981			goto free_fences;
 982		}
 983
 984		kfree(fences);
 985		kfree(fence_info);
 986	}
 987
 988	drm_exec_fini(&exec);
 989	for (i = 0; i < num_read_bo_handles; i++)
 990		drm_gem_object_put(gobj_read[i]);
 991	kfree(gobj_read);
 992
 993	for (i = 0; i < num_write_bo_handles; i++)
 994		drm_gem_object_put(gobj_write[i]);
 995	kfree(gobj_write);
 996
 997	kfree(timeline_points);
 998	kfree(timeline_handles);
 999	kfree(syncobj_handles);
1000	kfree(bo_handles_write);
1001	kfree(bo_handles_read);
1002
1003	return 0;
1004
1005free_fences:
1006	while (num_fences-- > 0)
1007		dma_fence_put(fences[num_fences]);
1008	kfree(fences);
1009free_fence_info:
1010	kfree(fence_info);
1011exec_fini:
1012	drm_exec_fini(&exec);
1013put_gobj_write:
1014	while (wentry-- > 0)
1015		drm_gem_object_put(gobj_write[wentry]);
1016	kfree(gobj_write);
1017put_gobj_read:
1018	while (rentry-- > 0)
1019		drm_gem_object_put(gobj_read[rentry]);
1020	kfree(gobj_read);
1021free_timeline_points:
1022	kfree(timeline_points);
1023free_timeline_handles:
1024	kfree(timeline_handles);
1025free_syncobj_handles:
1026	kfree(syncobj_handles);
1027free_bo_handles_write:
1028	kfree(bo_handles_write);
1029free_bo_handles_read:
1030	kfree(bo_handles_read);
1031
1032	return r;
1033}