amdgpu_userq.c

   1// SPDX-License-Identifier: MIT
   2/*
   3 * Copyright 2023 Advanced Micro Devices, Inc.
   4 *
   5 * Permission is hereby granted, free of charge, to any person obtaining a
   6 * copy of this software and associated documentation files (the "Software"),
   7 * to deal in the Software without restriction, including without limitation
   8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9 * and/or sell copies of the Software, and to permit persons to whom the
  10 * Software is furnished to do so, subject to the following conditions:
  11 *
  12 * The above copyright notice and this permission notice shall be included in
  13 * all copies or substantial portions of the Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21 * OTHER DEALINGS IN THE SOFTWARE.
  22 *
  23 */
  24
  25#include <drm/drm_auth.h>
  26#include <drm/drm_exec.h>
  27#include <linux/pm_runtime.h>
  28#include <drm/drm_drv.h>
  29
  30#include "amdgpu.h"
  31#include "amdgpu_reset.h"
  32#include "amdgpu_vm.h"
  33#include "amdgpu_userq.h"
  34#include "amdgpu_hmm.h"
  35#include "amdgpu_userq_fence.h"
  36
  37u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev)
  38{
  39	int i;
  40	u32 userq_ip_mask = 0;
  41
  42	for (i = 0; i < AMDGPU_HW_IP_NUM; i++) {
  43		if (adev->userq_funcs[i])
  44			userq_ip_mask |= (1 << i);
  45	}
  46
  47	return userq_ip_mask;
  48}
  49
  50static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,
  51				enum amdgpu_ring_type ring_type, int reset_type)
  52{
  53
  54	if (ring_type < 0 || ring_type >= AMDGPU_RING_TYPE_MAX)
  55		return false;
  56
  57	switch (ring_type) {
  58	case AMDGPU_RING_TYPE_GFX:
  59		if (adev->gfx.gfx_supported_reset & reset_type)
  60			return true;
  61		break;
  62	case AMDGPU_RING_TYPE_COMPUTE:
  63		if (adev->gfx.compute_supported_reset & reset_type)
  64			return true;
  65		break;
  66	case AMDGPU_RING_TYPE_SDMA:
  67		if (adev->sdma.supported_reset & reset_type)
  68			return true;
  69		break;
  70	case AMDGPU_RING_TYPE_VCN_DEC:
  71	case AMDGPU_RING_TYPE_VCN_ENC:
  72		if (adev->vcn.supported_reset & reset_type)
  73			return true;
  74		break;
  75	case AMDGPU_RING_TYPE_VCN_JPEG:
  76		if (adev->jpeg.supported_reset & reset_type)
  77			return true;
  78		break;
  79	default:
  80		break;
  81	}
  82	return false;
  83}
  84
  85static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev)
  86{
  87	if (amdgpu_device_should_recover_gpu(adev)) {
  88		amdgpu_reset_domain_schedule(adev->reset_domain,
  89					     &adev->userq_reset_work);
  90		/* Wait for the reset job to complete */
  91		flush_work(&adev->userq_reset_work);
  92	}
  93}
  94
  95static int
  96amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
  97{
  98	struct amdgpu_device *adev = uq_mgr->adev;
  99	const int queue_types[] = {
 100		AMDGPU_RING_TYPE_COMPUTE,
 101		AMDGPU_RING_TYPE_GFX,
 102		AMDGPU_RING_TYPE_SDMA
 103	};
 104	const int num_queue_types = ARRAY_SIZE(queue_types);
 105	bool gpu_reset = false;
 106	int r = 0;
 107	int i;
 108
 109	/* Warning if current process mutex is not held */
 110	WARN_ON(!mutex_is_locked(&uq_mgr->userq_mutex));
 111
 112	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
 113		dev_err(adev->dev, "userq reset disabled by debug mask\n");
 114		return 0;
 115	}
 116
 117	/*
 118	 * If GPU recovery feature is disabled system-wide,
 119	 * skip all reset detection logic
 120	 */
 121	if (!amdgpu_gpu_recovery)
 122		return 0;
 123
 124	/*
 125	 * Iterate through all queue types to detect and reset problematic queues
 126	 * Process each queue type in the defined order
 127	 */
 128	for (i = 0; i < num_queue_types; i++) {
 129		int ring_type = queue_types[i];
 130		const struct amdgpu_userq_funcs *funcs = adev->userq_funcs[ring_type];
 131
 132		if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, AMDGPU_RESET_TYPE_PER_QUEUE))
 133				continue;
 134
 135		if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 &&
 136		    funcs && funcs->detect_and_reset) {
 137			r = funcs->detect_and_reset(adev, ring_type);
 138			if (r) {
 139				gpu_reset = true;
 140				break;
 141			}
 142		}
 143	}
 144
 145	if (gpu_reset)
 146		amdgpu_userq_gpu_reset(adev);
 147
 148	return r;
 149}
 150
 151static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,
 152					   struct amdgpu_bo_va_mapping *va_map, u64 addr)
 153{
 154	struct amdgpu_userq_va_cursor *va_cursor;
 155	struct userq_va_list;
 156
 157	va_cursor = kzalloc(sizeof(*va_cursor), GFP_KERNEL);
 158	if (!va_cursor)
 159		return -ENOMEM;
 160
 161	INIT_LIST_HEAD(&va_cursor->list);
 162	va_cursor->gpu_addr = addr;
 163	atomic_set(&va_map->bo_va->userq_va_mapped, 1);
 164	list_add(&va_cursor->list, &queue->userq_va_list);
 165
 166	return 0;
 167}
 168
 169int amdgpu_userq_input_va_validate(struct amdgpu_usermode_queue *queue,
 170				   u64 addr, u64 expected_size)
 171{
 172	struct amdgpu_bo_va_mapping *va_map;
 173	struct amdgpu_vm *vm = queue->vm;
 174	u64 user_addr;
 175	u64 size;
 176	int r = 0;
 177
 178	user_addr = (addr & AMDGPU_GMC_HOLE_MASK) >> AMDGPU_GPU_PAGE_SHIFT;
 179	size = expected_size >> AMDGPU_GPU_PAGE_SHIFT;
 180
 181	r = amdgpu_bo_reserve(vm->root.bo, false);
 182	if (r)
 183		return r;
 184
 185	va_map = amdgpu_vm_bo_lookup_mapping(vm, user_addr);
 186	if (!va_map) {
 187		r = -EINVAL;
 188		goto out_err;
 189	}
 190	/* Only validate the userq whether resident in the VM mapping range */
 191	if (user_addr >= va_map->start  &&
 192	    va_map->last - user_addr + 1 >= size) {
 193		amdgpu_userq_buffer_va_list_add(queue, va_map, user_addr);
 194		amdgpu_bo_unreserve(vm->root.bo);
 195		return 0;
 196	}
 197
 198	r = -EINVAL;
 199out_err:
 200	amdgpu_bo_unreserve(vm->root.bo);
 201	return r;
 202}
 203
 204static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
 205{
 206	struct amdgpu_bo_va_mapping *mapping;
 207	bool r;
 208
 209	if (amdgpu_bo_reserve(vm->root.bo, false))
 210		return false;
 211
 212	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
 213	if (!IS_ERR_OR_NULL(mapping) && atomic_read(&mapping->bo_va->userq_va_mapped))
 214		r = true;
 215	else
 216		r = false;
 217	amdgpu_bo_unreserve(vm->root.bo);
 218
 219	return r;
 220}
 221
 222static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue)
 223{
 224	struct amdgpu_userq_va_cursor *va_cursor, *tmp;
 225	int r = 0;
 226
 227	list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) {
 228		r += amdgpu_userq_buffer_va_mapped(queue->vm, va_cursor->gpu_addr);
 229		dev_dbg(queue->userq_mgr->adev->dev,
 230			"validate the userq mapping:%p va:%llx r:%d\n",
 231			queue, va_cursor->gpu_addr, r);
 232	}
 233
 234	if (r != 0)
 235		return true;
 236
 237	return false;
 238}
 239
 240static void amdgpu_userq_buffer_va_list_del(struct amdgpu_bo_va_mapping *mapping,
 241					    struct amdgpu_userq_va_cursor *va_cursor)
 242{
 243	atomic_set(&mapping->bo_va->userq_va_mapped, 0);
 244	list_del(&va_cursor->list);
 245	kfree(va_cursor);
 246}
 247
 248static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev,
 249						struct amdgpu_usermode_queue *queue)
 250{
 251	struct amdgpu_userq_va_cursor *va_cursor, *tmp;
 252	struct amdgpu_bo_va_mapping *mapping;
 253	int r;
 254
 255	r = amdgpu_bo_reserve(queue->vm->root.bo, false);
 256	if (r)
 257		return r;
 258
 259	list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) {
 260		mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, va_cursor->gpu_addr);
 261		if (!mapping) {
 262			r = -EINVAL;
 263			goto err;
 264		}
 265		dev_dbg(adev->dev, "delete the userq:%p va:%llx\n",
 266			queue, va_cursor->gpu_addr);
 267		amdgpu_userq_buffer_va_list_del(mapping, va_cursor);
 268	}
 269err:
 270	amdgpu_bo_unreserve(queue->vm->root.bo);
 271	return r;
 272}
 273
 274static int
 275amdgpu_userq_preempt_helper(struct amdgpu_userq_mgr *uq_mgr,
 276			  struct amdgpu_usermode_queue *queue)
 277{
 278	struct amdgpu_device *adev = uq_mgr->adev;
 279	const struct amdgpu_userq_funcs *userq_funcs =
 280		adev->userq_funcs[queue->queue_type];
 281	bool found_hung_queue = false;
 282	int r = 0;
 283
 284	if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
 285		r = userq_funcs->preempt(uq_mgr, queue);
 286		if (r) {
 287			queue->state = AMDGPU_USERQ_STATE_HUNG;
 288			found_hung_queue = true;
 289		} else {
 290			queue->state = AMDGPU_USERQ_STATE_PREEMPTED;
 291		}
 292	}
 293
 294	if (found_hung_queue)
 295		amdgpu_userq_detect_and_reset_queues(uq_mgr);
 296
 297	return r;
 298}
 299
 300static int
 301amdgpu_userq_restore_helper(struct amdgpu_userq_mgr *uq_mgr,
 302			struct amdgpu_usermode_queue *queue)
 303{
 304	struct amdgpu_device *adev = uq_mgr->adev;
 305	const struct amdgpu_userq_funcs *userq_funcs =
 306		adev->userq_funcs[queue->queue_type];
 307	int r = 0;
 308
 309	if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) {
 310		r = userq_funcs->restore(uq_mgr, queue);
 311		if (r) {
 312			queue->state = AMDGPU_USERQ_STATE_HUNG;
 313		} else {
 314			queue->state = AMDGPU_USERQ_STATE_MAPPED;
 315		}
 316	}
 317
 318	return r;
 319}
 320
 321static int
 322amdgpu_userq_unmap_helper(struct amdgpu_userq_mgr *uq_mgr,
 323			  struct amdgpu_usermode_queue *queue)
 324{
 325	struct amdgpu_device *adev = uq_mgr->adev;
 326	const struct amdgpu_userq_funcs *userq_funcs =
 327		adev->userq_funcs[queue->queue_type];
 328	bool found_hung_queue = false;
 329	int r = 0;
 330
 331	if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) ||
 332		(queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) {
 333		r = userq_funcs->unmap(uq_mgr, queue);
 334		if (r) {
 335			queue->state = AMDGPU_USERQ_STATE_HUNG;
 336			found_hung_queue = true;
 337		} else {
 338			queue->state = AMDGPU_USERQ_STATE_UNMAPPED;
 339		}
 340	}
 341
 342	if (found_hung_queue)
 343		amdgpu_userq_detect_and_reset_queues(uq_mgr);
 344
 345	return r;
 346}
 347
 348static int
 349amdgpu_userq_map_helper(struct amdgpu_userq_mgr *uq_mgr,
 350			struct amdgpu_usermode_queue *queue)
 351{
 352	struct amdgpu_device *adev = uq_mgr->adev;
 353	const struct amdgpu_userq_funcs *userq_funcs =
 354		adev->userq_funcs[queue->queue_type];
 355	int r = 0;
 356
 357	if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) {
 358		r = userq_funcs->map(uq_mgr, queue);
 359		if (r) {
 360			queue->state = AMDGPU_USERQ_STATE_HUNG;
 361			amdgpu_userq_detect_and_reset_queues(uq_mgr);
 362		} else {
 363			queue->state = AMDGPU_USERQ_STATE_MAPPED;
 364		}
 365	}
 366
 367	return r;
 368}
 369
 370static int
 371amdgpu_userq_wait_for_last_fence(struct amdgpu_userq_mgr *uq_mgr,
 372				 struct amdgpu_usermode_queue *queue)
 373{
 374	struct dma_fence *f = queue->last_fence;
 375	int ret = 0;
 376
 377	if (f && !dma_fence_is_signaled(f)) {
 378		ret = dma_fence_wait_timeout(f, true, MAX_SCHEDULE_TIMEOUT);
 379		if (ret <= 0) {
 380			drm_file_err(uq_mgr->file, "Timed out waiting for fence=%llu:%llu\n",
 381				     f->context, f->seqno);
 382			queue->state = AMDGPU_USERQ_STATE_HUNG;
 383			return -ETIME;
 384		}
 385	}
 386
 387	return ret;
 388}
 389
 390static void
 391amdgpu_userq_cleanup(struct amdgpu_userq_mgr *uq_mgr,
 392		     struct amdgpu_usermode_queue *queue,
 393		     int queue_id)
 394{
 395	struct amdgpu_device *adev = uq_mgr->adev;
 396	const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type];
 397
 398	/* Wait for mode-1 reset to complete */
 399	down_read(&adev->reset_domain->sem);
 400
 401	/* Drop the userq reference. */
 402	amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
 403	uq_funcs->mqd_destroy(uq_mgr, queue);
 404	amdgpu_userq_fence_driver_free(queue);
 405	/* Use interrupt-safe locking since IRQ handlers may access these XArrays */
 406	xa_erase_irq(&uq_mgr->userq_mgr_xa, (unsigned long)queue_id);
 407	xa_erase_irq(&adev->userq_doorbell_xa, queue->doorbell_index);
 408	queue->userq_mgr = NULL;
 409	list_del(&queue->userq_va_list);
 410	kfree(queue);
 411
 412	up_read(&adev->reset_domain->sem);
 413}
 414
 415static struct amdgpu_usermode_queue *
 416amdgpu_userq_find(struct amdgpu_userq_mgr *uq_mgr, int qid)
 417{
 418	return xa_load(&uq_mgr->userq_mgr_xa, qid);
 419}
 420
 421void
 422amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *uq_mgr,
 423			     struct amdgpu_eviction_fence_mgr *evf_mgr)
 424{
 425	struct amdgpu_eviction_fence *ev_fence;
 426
 427retry:
 428	/* Flush any pending resume work to create ev_fence */
 429	flush_delayed_work(&uq_mgr->resume_work);
 430
 431	mutex_lock(&uq_mgr->userq_mutex);
 432	spin_lock(&evf_mgr->ev_fence_lock);
 433	ev_fence = evf_mgr->ev_fence;
 434	spin_unlock(&evf_mgr->ev_fence_lock);
 435	if (!ev_fence || dma_fence_is_signaled(&ev_fence->base)) {
 436		mutex_unlock(&uq_mgr->userq_mutex);
 437		/*
 438		 * Looks like there was no pending resume work,
 439		 * add one now to create a valid eviction fence
 440		 */
 441		schedule_delayed_work(&uq_mgr->resume_work, 0);
 442		goto retry;
 443	}
 444}
 445
 446int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
 447			       struct amdgpu_userq_obj *userq_obj,
 448			       int size)
 449{
 450	struct amdgpu_device *adev = uq_mgr->adev;
 451	struct amdgpu_bo_param bp;
 452	int r;
 453
 454	memset(&bp, 0, sizeof(bp));
 455	bp.byte_align = PAGE_SIZE;
 456	bp.domain = AMDGPU_GEM_DOMAIN_GTT;
 457	bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
 458		   AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 459	bp.type = ttm_bo_type_kernel;
 460	bp.size = size;
 461	bp.resv = NULL;
 462	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
 463
 464	r = amdgpu_bo_create(adev, &bp, &userq_obj->obj);
 465	if (r) {
 466		drm_file_err(uq_mgr->file, "Failed to allocate BO for userqueue (%d)", r);
 467		return r;
 468	}
 469
 470	r = amdgpu_bo_reserve(userq_obj->obj, true);
 471	if (r) {
 472		drm_file_err(uq_mgr->file, "Failed to reserve BO to map (%d)", r);
 473		goto free_obj;
 474	}
 475
 476	r = amdgpu_ttm_alloc_gart(&(userq_obj->obj)->tbo);
 477	if (r) {
 478		drm_file_err(uq_mgr->file, "Failed to alloc GART for userqueue object (%d)", r);
 479		goto unresv;
 480	}
 481
 482	r = amdgpu_bo_kmap(userq_obj->obj, &userq_obj->cpu_ptr);
 483	if (r) {
 484		drm_file_err(uq_mgr->file, "Failed to map BO for userqueue (%d)", r);
 485		goto unresv;
 486	}
 487
 488	userq_obj->gpu_addr = amdgpu_bo_gpu_offset(userq_obj->obj);
 489	amdgpu_bo_unreserve(userq_obj->obj);
 490	memset(userq_obj->cpu_ptr, 0, size);
 491	return 0;
 492
 493unresv:
 494	amdgpu_bo_unreserve(userq_obj->obj);
 495
 496free_obj:
 497	amdgpu_bo_unref(&userq_obj->obj);
 498	return r;
 499}
 500
 501void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr,
 502				 struct amdgpu_userq_obj *userq_obj)
 503{
 504	amdgpu_bo_kunmap(userq_obj->obj);
 505	amdgpu_bo_unref(&userq_obj->obj);
 506}
 507
 508uint64_t
 509amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr,
 510				struct amdgpu_db_info *db_info,
 511				struct drm_file *filp)
 512{
 513	uint64_t index;
 514	struct drm_gem_object *gobj;
 515	struct amdgpu_userq_obj *db_obj = db_info->db_obj;
 516	int r, db_size;
 517
 518	gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle);
 519	if (gobj == NULL) {
 520		drm_file_err(uq_mgr->file, "Can't find GEM object for doorbell\n");
 521		return -EINVAL;
 522	}
 523
 524	db_obj->obj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
 525	drm_gem_object_put(gobj);
 526
 527	r = amdgpu_bo_reserve(db_obj->obj, true);
 528	if (r) {
 529		drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n");
 530		goto unref_bo;
 531	}
 532
 533	/* Pin the BO before generating the index, unpin in queue destroy */
 534	r = amdgpu_bo_pin(db_obj->obj, AMDGPU_GEM_DOMAIN_DOORBELL);
 535	if (r) {
 536		drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin doorbell object\n");
 537		goto unresv_bo;
 538	}
 539
 540	switch (db_info->queue_type) {
 541	case AMDGPU_HW_IP_GFX:
 542	case AMDGPU_HW_IP_COMPUTE:
 543	case AMDGPU_HW_IP_DMA:
 544		db_size = sizeof(u64);
 545		break;
 546	default:
 547		drm_file_err(uq_mgr->file, "[Usermode queues] IP %d not support\n",
 548			     db_info->queue_type);
 549		r = -EINVAL;
 550		goto unpin_bo;
 551	}
 552
 553	/* Validate doorbell_offset is within the doorbell BO */
 554	if ((u64)db_info->doorbell_offset * db_size + db_size >
 555	    amdgpu_bo_size(db_obj->obj)) {
 556		r = -EINVAL;
 557		goto unpin_bo;
 558	}
 559
 560	index = amdgpu_doorbell_index_on_bar(uq_mgr->adev, db_obj->obj,
 561					     db_info->doorbell_offset, db_size);
 562	drm_dbg_driver(adev_to_drm(uq_mgr->adev),
 563		       "[Usermode queues] doorbell index=%lld\n", index);
 564	amdgpu_bo_unreserve(db_obj->obj);
 565	return index;
 566
 567unpin_bo:
 568	amdgpu_bo_unpin(db_obj->obj);
 569unresv_bo:
 570	amdgpu_bo_unreserve(db_obj->obj);
 571unref_bo:
 572	amdgpu_bo_unref(&db_obj->obj);
 573	return r;
 574}
 575
 576static int
 577amdgpu_userq_destroy(struct drm_file *filp, int queue_id)
 578{
 579	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 580	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
 581	struct amdgpu_device *adev = uq_mgr->adev;
 582	struct amdgpu_usermode_queue *queue;
 583	int r = 0;
 584
 585	cancel_delayed_work_sync(&uq_mgr->resume_work);
 586	mutex_lock(&uq_mgr->userq_mutex);
 587
 588	queue = amdgpu_userq_find(uq_mgr, queue_id);
 589	if (!queue) {
 590		drm_dbg_driver(adev_to_drm(uq_mgr->adev), "Invalid queue id to destroy\n");
 591		mutex_unlock(&uq_mgr->userq_mutex);
 592		return -EINVAL;
 593	}
 594	amdgpu_userq_wait_for_last_fence(uq_mgr, queue);
 595	r = amdgpu_bo_reserve(queue->db_obj.obj, true);
 596	if (!r) {
 597		amdgpu_bo_unpin(queue->db_obj.obj);
 598		amdgpu_bo_unreserve(queue->db_obj.obj);
 599	}
 600	amdgpu_bo_unref(&queue->db_obj.obj);
 601	atomic_dec(&uq_mgr->userq_count[queue->queue_type]);
 602#if defined(CONFIG_DEBUG_FS)
 603	debugfs_remove_recursive(queue->debugfs_queue);
 604#endif
 605	amdgpu_userq_detect_and_reset_queues(uq_mgr);
 606	r = amdgpu_userq_unmap_helper(uq_mgr, queue);
 607	/*TODO: It requires a reset for userq hw unmap error*/
 608	if (unlikely(r != AMDGPU_USERQ_STATE_UNMAPPED)) {
 609		drm_warn(adev_to_drm(uq_mgr->adev), "trying to destroy a HW mapping userq\n");
 610		queue->state = AMDGPU_USERQ_STATE_HUNG;
 611	}
 612	amdgpu_userq_cleanup(uq_mgr, queue, queue_id);
 613	mutex_unlock(&uq_mgr->userq_mutex);
 614
 615	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 616
 617	return r;
 618}
 619
 620static int amdgpu_userq_priority_permit(struct drm_file *filp,
 621					int priority)
 622{
 623	if (priority < AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH)
 624		return 0;
 625
 626	if (capable(CAP_SYS_NICE))
 627		return 0;
 628
 629	if (drm_is_current_master(filp))
 630		return 0;
 631
 632	return -EACCES;
 633}
 634
 635#if defined(CONFIG_DEBUG_FS)
 636static int amdgpu_mqd_info_read(struct seq_file *m, void *unused)
 637{
 638	struct amdgpu_usermode_queue *queue = m->private;
 639	struct amdgpu_bo *bo;
 640	int r;
 641
 642	if (!queue || !queue->mqd.obj)
 643		return -EINVAL;
 644
 645	bo = amdgpu_bo_ref(queue->mqd.obj);
 646	r = amdgpu_bo_reserve(bo, true);
 647	if (r) {
 648		amdgpu_bo_unref(&bo);
 649		return -EINVAL;
 650	}
 651
 652	seq_printf(m, "queue_type: %d\n", queue->queue_type);
 653	seq_printf(m, "mqd_gpu_address: 0x%llx\n", amdgpu_bo_gpu_offset(queue->mqd.obj));
 654
 655	amdgpu_bo_unreserve(bo);
 656	amdgpu_bo_unref(&bo);
 657
 658	return 0;
 659}
 660
 661static int amdgpu_mqd_info_open(struct inode *inode, struct file *file)
 662{
 663	return single_open(file, amdgpu_mqd_info_read, inode->i_private);
 664}
 665
 666static const struct file_operations amdgpu_mqd_info_fops = {
 667	.owner = THIS_MODULE,
 668	.open = amdgpu_mqd_info_open,
 669	.read = seq_read,
 670	.llseek = seq_lseek,
 671	.release = single_release,
 672};
 673#endif
 674
 675static int
 676amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 677{
 678	struct amdgpu_fpriv *fpriv = filp->driver_priv;
 679	struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
 680	struct amdgpu_device *adev = uq_mgr->adev;
 681	const struct amdgpu_userq_funcs *uq_funcs;
 682	struct amdgpu_usermode_queue *queue;
 683	struct amdgpu_db_info db_info;
 684	char *queue_name;
 685	bool skip_map_queue;
 686	u32 qid;
 687	uint64_t index;
 688	int r = 0;
 689	int priority =
 690		(args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >>
 691		AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
 692
 693	r = amdgpu_userq_priority_permit(filp, priority);
 694	if (r)
 695		return r;
 696
 697	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 698	if (r < 0) {
 699		drm_file_err(uq_mgr->file, "pm_runtime_get_sync() failed for userqueue create\n");
 700		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 701		return r;
 702	}
 703
 704	/*
 705	 * There could be a situation that we are creating a new queue while
 706	 * the other queues under this UQ_mgr are suspended. So if there is any
 707	 * resume work pending, wait for it to get done.
 708	 *
 709	 * This will also make sure we have a valid eviction fence ready to be used.
 710	 */
 711	amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
 712
 713	uq_funcs = adev->userq_funcs[args->in.ip_type];
 714	if (!uq_funcs) {
 715		drm_file_err(uq_mgr->file, "Usermode queue is not supported for this IP (%u)\n",
 716			     args->in.ip_type);
 717		r = -EINVAL;
 718		goto unlock;
 719	}
 720
 721	queue = kzalloc(sizeof(struct amdgpu_usermode_queue), GFP_KERNEL);
 722	if (!queue) {
 723		drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n");
 724		r = -ENOMEM;
 725		goto unlock;
 726	}
 727
 728	INIT_LIST_HEAD(&queue->userq_va_list);
 729	queue->doorbell_handle = args->in.doorbell_handle;
 730	queue->queue_type = args->in.ip_type;
 731	queue->vm = &fpriv->vm;
 732	queue->priority = priority;
 733
 734	db_info.queue_type = queue->queue_type;
 735	db_info.doorbell_handle = queue->doorbell_handle;
 736	db_info.db_obj = &queue->db_obj;
 737	db_info.doorbell_offset = args->in.doorbell_offset;
 738
 739	/* Validate the userq virtual address.*/
 740	if (amdgpu_userq_input_va_validate(queue, args->in.queue_va, args->in.queue_size) ||
 741	    amdgpu_userq_input_va_validate(queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) ||
 742	    amdgpu_userq_input_va_validate(queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) {
 743		r = -EINVAL;
 744		kfree(queue);
 745		goto unlock;
 746	}
 747
 748	/* Convert relative doorbell offset into absolute doorbell index */
 749	index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp);
 750	if (index == (uint64_t)-EINVAL) {
 751		drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");
 752		kfree(queue);
 753		r = -EINVAL;
 754		goto unlock;
 755	}
 756
 757	queue->doorbell_index = index;
 758	xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
 759	r = amdgpu_userq_fence_driver_alloc(adev, queue);
 760	if (r) {
 761		drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n");
 762		goto unlock;
 763	}
 764
 765	r = uq_funcs->mqd_create(uq_mgr, &args->in, queue);
 766	if (r) {
 767		drm_file_err(uq_mgr->file, "Failed to create Queue\n");
 768		amdgpu_userq_fence_driver_free(queue);
 769		kfree(queue);
 770		goto unlock;
 771	}
 772
 773	/* Wait for mode-1 reset to complete */
 774	down_read(&adev->reset_domain->sem);
 775	r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL));
 776	if (r) {
 777		kfree(queue);
 778		up_read(&adev->reset_domain->sem);
 779		goto unlock;
 780	}
 781
 782	r = xa_alloc(&uq_mgr->userq_mgr_xa, &qid, queue, XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
 783	if (r) {
 784		drm_file_err(uq_mgr->file, "Failed to allocate a queue id\n");
 785		amdgpu_userq_fence_driver_free(queue);
 786		uq_funcs->mqd_destroy(uq_mgr, queue);
 787		kfree(queue);
 788		r = -ENOMEM;
 789		up_read(&adev->reset_domain->sem);
 790		goto unlock;
 791	}
 792	up_read(&adev->reset_domain->sem);
 793	queue->userq_mgr = uq_mgr;
 794
 795	/* don't map the queue if scheduling is halted */
 796	if (adev->userq_halt_for_enforce_isolation &&
 797	    ((queue->queue_type == AMDGPU_HW_IP_GFX) ||
 798	     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)))
 799		skip_map_queue = true;
 800	else
 801		skip_map_queue = false;
 802	if (!skip_map_queue) {
 803		r = amdgpu_userq_map_helper(uq_mgr, queue);
 804		if (r) {
 805			drm_file_err(uq_mgr->file, "Failed to map Queue\n");
 806			xa_erase(&uq_mgr->userq_mgr_xa, qid);
 807			amdgpu_userq_fence_driver_free(queue);
 808			uq_funcs->mqd_destroy(uq_mgr, queue);
 809			kfree(queue);
 810			goto unlock;
 811		}
 812	}
 813
 814	queue_name = kasprintf(GFP_KERNEL, "queue-%d", qid);
 815	if (!queue_name) {
 816		r = -ENOMEM;
 817		goto unlock;
 818	}
 819
 820#if defined(CONFIG_DEBUG_FS)
 821	/* Queue dentry per client to hold MQD information   */
 822	queue->debugfs_queue = debugfs_create_dir(queue_name, filp->debugfs_client);
 823	debugfs_create_file("mqd_info", 0444, queue->debugfs_queue, queue, &amdgpu_mqd_info_fops);
 824#endif
 825	kfree(queue_name);
 826
 827	args->out.queue_id = qid;
 828	atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
 829
 830unlock:
 831	mutex_unlock(&uq_mgr->userq_mutex);
 832
 833	return r;
 834}
 835
 836static int amdgpu_userq_input_args_validate(struct drm_device *dev,
 837					union drm_amdgpu_userq *args,
 838					struct drm_file *filp)
 839{
 840	struct amdgpu_device *adev = drm_to_adev(dev);
 841
 842	switch (args->in.op) {
 843	case AMDGPU_USERQ_OP_CREATE:
 844		if (args->in.flags & ~(AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK |
 845				       AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE))
 846			return -EINVAL;
 847		/* Usermode queues are only supported for GFX IP as of now */
 848		if (args->in.ip_type != AMDGPU_HW_IP_GFX &&
 849		    args->in.ip_type != AMDGPU_HW_IP_DMA &&
 850		    args->in.ip_type != AMDGPU_HW_IP_COMPUTE) {
 851			drm_file_err(filp, "Usermode queue doesn't support IP type %u\n",
 852				     args->in.ip_type);
 853			return -EINVAL;
 854		}
 855
 856		if ((args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE) &&
 857		    (args->in.ip_type != AMDGPU_HW_IP_GFX) &&
 858		    (args->in.ip_type != AMDGPU_HW_IP_COMPUTE) &&
 859		    !amdgpu_is_tmz(adev)) {
 860			drm_file_err(filp, "Secure only supported on GFX/Compute queues\n");
 861			return -EINVAL;
 862		}
 863
 864		if (args->in.queue_va == AMDGPU_BO_INVALID_OFFSET ||
 865		    args->in.queue_va == 0 ||
 866		    args->in.queue_size == 0) {
 867			drm_file_err(filp, "invalidate userq queue va or size\n");
 868			return -EINVAL;
 869		}
 870
 871		if (!is_power_of_2(args->in.queue_size)) {
 872			drm_file_err(filp, "Queue size must be a power of 2\n");
 873			return -EINVAL;
 874		}
 875
 876		if (args->in.queue_size < AMDGPU_GPU_PAGE_SIZE) {
 877			drm_file_err(filp, "Queue size smaller than AMDGPU_GPU_PAGE_SIZE\n");
 878			return -EINVAL;
 879		}
 880
 881		if (!args->in.wptr_va || !args->in.rptr_va) {
 882			drm_file_err(filp, "invalidate userq queue rptr or wptr\n");
 883			return -EINVAL;
 884		}
 885		break;
 886	case AMDGPU_USERQ_OP_FREE:
 887		if (args->in.ip_type ||
 888		    args->in.doorbell_handle ||
 889		    args->in.doorbell_offset ||
 890		    args->in.flags ||
 891		    args->in.queue_va ||
 892		    args->in.queue_size ||
 893		    args->in.rptr_va ||
 894		    args->in.wptr_va ||
 895		    args->in.mqd ||
 896		    args->in.mqd_size)
 897			return -EINVAL;
 898		break;
 899	default:
 900		return -EINVAL;
 901	}
 902
 903	return 0;
 904}
 905
 906bool amdgpu_userq_enabled(struct drm_device *dev)
 907{
 908	struct amdgpu_device *adev = drm_to_adev(dev);
 909	int i;
 910
 911	for (i = 0; i < AMDGPU_HW_IP_NUM; i++) {
 912		if (adev->userq_funcs[i])
 913			return true;
 914	}
 915
 916	return false;
 917}
 918
 919int amdgpu_userq_ioctl(struct drm_device *dev, void *data,
 920		       struct drm_file *filp)
 921{
 922	union drm_amdgpu_userq *args = data;
 923	int r;
 924
 925	if (!amdgpu_userq_enabled(dev))
 926		return -ENOTSUPP;
 927
 928	if (amdgpu_userq_input_args_validate(dev, args, filp) < 0)
 929		return -EINVAL;
 930
 931	switch (args->in.op) {
 932	case AMDGPU_USERQ_OP_CREATE:
 933		r = amdgpu_userq_create(filp, args);
 934		if (r)
 935			drm_file_err(filp, "Failed to create usermode queue\n");
 936		break;
 937
 938	case AMDGPU_USERQ_OP_FREE:
 939		r = amdgpu_userq_destroy(filp, args->in.queue_id);
 940		if (r)
 941			drm_file_err(filp, "Failed to destroy usermode queue\n");
 942		break;
 943
 944	default:
 945		drm_dbg_driver(dev, "Invalid user queue op specified: %d\n", args->in.op);
 946		return -EINVAL;
 947	}
 948
 949	return r;
 950}
 951
 952static int
 953amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr)
 954{
 955	struct amdgpu_usermode_queue *queue;
 956	unsigned long queue_id;
 957	int ret = 0, r;
 958
 959	/* Resume all the queues for this process */
 960	xa_for_each(&uq_mgr->userq_mgr_xa, queue_id, queue) {
 961
 962		if (!amdgpu_userq_buffer_vas_mapped(queue)) {
 963			drm_file_err(uq_mgr->file,
 964				     "trying restore queue without va mapping\n");
 965			queue->state = AMDGPU_USERQ_STATE_INVALID_VA;
 966			continue;
 967		}
 968
 969		r = amdgpu_userq_restore_helper(uq_mgr, queue);
 970		if (r)
 971			ret = r;
 972	}
 973
 974	if (ret)
 975		drm_file_err(uq_mgr->file, "Failed to map all the queues\n");
 976	return ret;
 977}
 978
 979static int amdgpu_userq_validate_vm(void *param, struct amdgpu_bo *bo)
 980{
 981	struct ttm_operation_ctx ctx = { false, false };
 982
 983	amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
 984	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 985}
 986
 987/* Handle all BOs on the invalidated list, validate them and update the PTs */
 988static int
 989amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 990			 struct amdgpu_vm *vm)
 991{
 992	struct ttm_operation_ctx ctx = { false, false };
 993	struct amdgpu_bo_va *bo_va;
 994	struct amdgpu_bo *bo;
 995	int ret;
 996
 997	spin_lock(&vm->status_lock);
 998	while (!list_empty(&vm->invalidated)) {
 999		bo_va = list_first_entry(&vm->invalidated,
1000					 struct amdgpu_bo_va,
1001					 base.vm_status);
1002		spin_unlock(&vm->status_lock);
1003
1004		bo = bo_va->base.bo;
1005		ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
1006		if (unlikely(ret))
1007			return ret;
1008
1009		amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
1010		ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1011		if (ret)
1012			return ret;
1013
1014		/* This moves the bo_va to the done list */
1015		ret = amdgpu_vm_bo_update(adev, bo_va, false);
1016		if (ret)
1017			return ret;
1018
1019		spin_lock(&vm->status_lock);
1020	}
1021	spin_unlock(&vm->status_lock);
1022
1023	return 0;
1024}
1025
1026/* Make sure the whole VM is ready to be used */
1027static int
1028amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
1029{
1030	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
1031	bool invalidated = false, new_addition = false;
1032	struct ttm_operation_ctx ctx = { true, false };
1033	struct amdgpu_device *adev = uq_mgr->adev;
1034	struct amdgpu_hmm_range *range;
1035	struct amdgpu_vm *vm = &fpriv->vm;
1036	unsigned long key, tmp_key;
1037	struct amdgpu_bo_va *bo_va;
1038	struct amdgpu_bo *bo;
1039	struct drm_exec exec;
1040	struct xarray xa;
1041	int ret;
1042
1043	xa_init(&xa);
1044
1045retry_lock:
1046	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
1047	drm_exec_until_all_locked(&exec) {
1048		ret = amdgpu_vm_lock_pd(vm, &exec, 1);
1049		drm_exec_retry_on_contention(&exec);
1050		if (unlikely(ret))
1051			goto unlock_all;
1052
1053		ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
1054		drm_exec_retry_on_contention(&exec);
1055		if (unlikely(ret))
1056			goto unlock_all;
1057
1058		/* This validates PDs, PTs and per VM BOs */
1059		ret = amdgpu_vm_validate(adev, vm, NULL,
1060					 amdgpu_userq_validate_vm,
1061					 NULL);
1062		if (unlikely(ret))
1063			goto unlock_all;
1064
1065		/* This locks and validates the remaining evicted BOs */
1066		ret = amdgpu_userq_bo_validate(adev, &exec, vm);
1067		drm_exec_retry_on_contention(&exec);
1068		if (unlikely(ret))
1069			goto unlock_all;
1070	}
1071
1072	if (invalidated) {
1073		xa_for_each(&xa, tmp_key, range) {
1074			bo = range->bo;
1075			amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
1076			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1077			if (ret)
1078				goto unlock_all;
1079
1080			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range);
1081
1082			amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
1083			ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1084			if (ret)
1085				goto unlock_all;
1086		}
1087		invalidated = false;
1088	}
1089
1090	ret = amdgpu_vm_handle_moved(adev, vm, NULL);
1091	if (ret)
1092		goto unlock_all;
1093
1094	key = 0;
1095	/* Validate User Ptr BOs */
1096	list_for_each_entry(bo_va, &vm->done, base.vm_status) {
1097		bo = bo_va->base.bo;
1098		if (!bo)
1099			continue;
1100
1101		if (!amdgpu_ttm_tt_is_userptr(bo->tbo.ttm))
1102			continue;
1103
1104		range = xa_load(&xa, key);
1105		if (range && range->bo != bo) {
1106			xa_erase(&xa, key);
1107			amdgpu_hmm_range_free(range);
1108			range = NULL;
1109		}
1110
1111		if (!range) {
1112			range = amdgpu_hmm_range_alloc(bo);
1113			if (!range) {
1114				ret = -ENOMEM;
1115				goto unlock_all;
1116			}
1117
1118			xa_store(&xa, key, range, GFP_KERNEL);
1119			new_addition = true;
1120		}
1121		key++;
1122	}
1123
1124	if (new_addition) {
1125		drm_exec_fini(&exec);
1126		xa_for_each(&xa, tmp_key, range) {
1127			if (!range)
1128				continue;
1129			bo = range->bo;
1130			ret = amdgpu_ttm_tt_get_user_pages(bo, range);
1131			if (ret)
1132				goto unlock_all;
1133		}
1134
1135		invalidated = true;
1136		new_addition = false;
1137		goto retry_lock;
1138	}
1139
1140	ret = amdgpu_vm_update_pdes(adev, vm, false);
1141	if (ret)
1142		goto unlock_all;
1143
1144	/*
1145	 * We need to wait for all VM updates to finish before restarting the
1146	 * queues. Using the done list like that is now ok since everything is
1147	 * locked in place.
1148	 */
1149	list_for_each_entry(bo_va, &vm->done, base.vm_status)
1150		dma_fence_wait(bo_va->last_pt_update, false);
1151	dma_fence_wait(vm->last_update, false);
1152
1153	ret = amdgpu_eviction_fence_replace_fence(&fpriv->evf_mgr, &exec);
1154	if (ret)
1155		drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n");
1156
1157unlock_all:
1158	drm_exec_fini(&exec);
1159	xa_for_each(&xa, tmp_key, range) {
1160		if (!range)
1161			continue;
1162		bo = range->bo;
1163		amdgpu_hmm_range_free(range);
1164	}
1165	xa_destroy(&xa);
1166	return ret;
1167}
1168
1169static void amdgpu_userq_restore_worker(struct work_struct *work)
1170{
1171	struct amdgpu_userq_mgr *uq_mgr = work_to_uq_mgr(work, resume_work.work);
1172	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
1173	int ret;
1174
1175	flush_delayed_work(&fpriv->evf_mgr.suspend_work);
1176
1177	mutex_lock(&uq_mgr->userq_mutex);
1178
1179	ret = amdgpu_userq_vm_validate(uq_mgr);
1180	if (ret) {
1181		drm_file_err(uq_mgr->file, "Failed to validate BOs to restore\n");
1182		goto unlock;
1183	}
1184
1185	ret = amdgpu_userq_restore_all(uq_mgr);
1186	if (ret) {
1187		drm_file_err(uq_mgr->file, "Failed to restore all queues\n");
1188		goto unlock;
1189	}
1190
1191unlock:
1192	mutex_unlock(&uq_mgr->userq_mutex);
1193}
1194
1195static int
1196amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
1197{
1198	struct amdgpu_usermode_queue *queue;
1199	unsigned long queue_id;
1200	int ret = 0, r;
1201
1202	amdgpu_userq_detect_and_reset_queues(uq_mgr);
1203	/* Try to unmap all the queues in this process ctx */
1204	xa_for_each(&uq_mgr->userq_mgr_xa, queue_id, queue) {
1205		r = amdgpu_userq_preempt_helper(uq_mgr, queue);
1206		if (r)
1207			ret = r;
1208	}
1209
1210	if (ret)
1211		drm_file_err(uq_mgr->file, "Couldn't unmap all the queues\n");
1212	return ret;
1213}
1214
1215void amdgpu_userq_reset_work(struct work_struct *work)
1216{
1217	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
1218						  userq_reset_work);
1219	struct amdgpu_reset_context reset_context;
1220
1221	memset(&reset_context, 0, sizeof(reset_context));
1222
1223	reset_context.method = AMD_RESET_METHOD_NONE;
1224	reset_context.reset_req_dev = adev;
1225	reset_context.src = AMDGPU_RESET_SRC_USERQ;
1226	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1227	/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
1228
1229	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
1230}
1231
1232static int
1233amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
1234{
1235	struct amdgpu_usermode_queue *queue;
1236	unsigned long queue_id;
1237	int ret;
1238
1239	xa_for_each(&uq_mgr->userq_mgr_xa, queue_id, queue) {
1240		struct dma_fence *f = queue->last_fence;
1241
1242		if (!f || dma_fence_is_signaled(f))
1243			continue;
1244		ret = dma_fence_wait_timeout(f, true, msecs_to_jiffies(100));
1245		if (ret <= 0) {
1246			drm_file_err(uq_mgr->file, "Timed out waiting for fence=%llu:%llu\n",
1247				     f->context, f->seqno);
1248			return -ETIMEDOUT;
1249		}
1250	}
1251
1252	return 0;
1253}
1254
1255void
1256amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr,
1257		   struct amdgpu_eviction_fence *ev_fence)
1258{
1259	struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr);
1260	struct amdgpu_eviction_fence_mgr *evf_mgr = &fpriv->evf_mgr;
1261	struct amdgpu_device *adev = uq_mgr->adev;
1262	int ret;
1263
1264	/* Wait for any pending userqueue fence work to finish */
1265	ret = amdgpu_userq_wait_for_signal(uq_mgr);
1266	if (ret)
1267		dev_err(adev->dev, "Not evicting userqueue, timeout waiting for work\n");
1268
1269	ret = amdgpu_userq_evict_all(uq_mgr);
1270	if (ret)
1271		dev_err(adev->dev, "Failed to evict userqueue\n");
1272
1273	/* Signal current eviction fence */
1274	amdgpu_eviction_fence_signal(evf_mgr, ev_fence);
1275
1276	if (evf_mgr->fd_closing) {
1277		cancel_delayed_work_sync(&uq_mgr->resume_work);
1278		return;
1279	}
1280
1281	/* Schedule a resume work */
1282	schedule_delayed_work(&uq_mgr->resume_work, 0);
1283}
1284
1285int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
1286			  struct amdgpu_device *adev)
1287{
1288	mutex_init(&userq_mgr->userq_mutex);
1289	xa_init_flags(&userq_mgr->userq_mgr_xa, XA_FLAGS_ALLOC);
1290	userq_mgr->adev = adev;
1291	userq_mgr->file = file_priv;
1292
1293	INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
1294	return 0;
1295}
1296
1297void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
1298{
1299	struct amdgpu_usermode_queue *queue;
1300	unsigned long queue_id;
1301
1302	cancel_delayed_work_sync(&userq_mgr->resume_work);
1303
1304	mutex_lock(&userq_mgr->userq_mutex);
1305	amdgpu_userq_detect_and_reset_queues(userq_mgr);
1306	xa_for_each(&userq_mgr->userq_mgr_xa, queue_id, queue) {
1307		amdgpu_userq_wait_for_last_fence(userq_mgr, queue);
1308		amdgpu_userq_unmap_helper(userq_mgr, queue);
1309		amdgpu_userq_cleanup(userq_mgr, queue, queue_id);
1310	}
1311
1312	xa_destroy(&userq_mgr->userq_mgr_xa);
1313	mutex_unlock(&userq_mgr->userq_mutex);
1314	mutex_destroy(&userq_mgr->userq_mutex);
1315}
1316
1317int amdgpu_userq_suspend(struct amdgpu_device *adev)
1318{
1319	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1320	struct amdgpu_usermode_queue *queue;
1321	struct amdgpu_userq_mgr *uqm;
1322	unsigned long queue_id;
1323	int r;
1324
1325	if (!ip_mask)
1326		return 0;
1327
1328	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1329		uqm = queue->userq_mgr;
1330		cancel_delayed_work_sync(&uqm->resume_work);
1331		guard(mutex)(&uqm->userq_mutex);
1332		amdgpu_userq_detect_and_reset_queues(uqm);
1333		if (adev->in_s0ix)
1334			r = amdgpu_userq_preempt_helper(uqm, queue);
1335		else
1336			r = amdgpu_userq_unmap_helper(uqm, queue);
1337		if (r)
1338			return r;
1339	}
1340	return 0;
1341}
1342
1343int amdgpu_userq_resume(struct amdgpu_device *adev)
1344{
1345	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1346	struct amdgpu_usermode_queue *queue;
1347	struct amdgpu_userq_mgr *uqm;
1348	unsigned long queue_id;
1349	int r;
1350
1351	if (!ip_mask)
1352		return 0;
1353
1354	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1355		uqm = queue->userq_mgr;
1356		guard(mutex)(&uqm->userq_mutex);
1357		if (adev->in_s0ix)
1358			r = amdgpu_userq_restore_helper(uqm, queue);
1359		else
1360			r = amdgpu_userq_map_helper(uqm, queue);
1361		if (r)
1362			return r;
1363	}
1364
1365	return 0;
1366}
1367
1368int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
1369						  u32 idx)
1370{
1371	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1372	struct amdgpu_usermode_queue *queue;
1373	struct amdgpu_userq_mgr *uqm;
1374	unsigned long queue_id;
1375	int ret = 0, r;
1376
1377	/* only need to stop gfx/compute */
1378	if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE))))
1379		return 0;
1380
1381	if (adev->userq_halt_for_enforce_isolation)
1382		dev_warn(adev->dev, "userq scheduling already stopped!\n");
1383	adev->userq_halt_for_enforce_isolation = true;
1384	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1385		uqm = queue->userq_mgr;
1386		cancel_delayed_work_sync(&uqm->resume_work);
1387		mutex_lock(&uqm->userq_mutex);
1388		if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
1389		     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
1390		    (queue->xcp_id == idx)) {
1391			amdgpu_userq_detect_and_reset_queues(uqm);
1392			r = amdgpu_userq_preempt_helper(uqm, queue);
1393			if (r)
1394				ret = r;
1395		}
1396		mutex_unlock(&uqm->userq_mutex);
1397	}
1398
1399	return ret;
1400}
1401
1402int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev,
1403						   u32 idx)
1404{
1405	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1406	struct amdgpu_usermode_queue *queue;
1407	struct amdgpu_userq_mgr *uqm;
1408	unsigned long queue_id;
1409	int ret = 0, r;
1410
1411	/* only need to stop gfx/compute */
1412	if (!(ip_mask & ((1 << AMDGPU_HW_IP_GFX) | (1 << AMDGPU_HW_IP_COMPUTE))))
1413		return 0;
1414
1415	if (!adev->userq_halt_for_enforce_isolation)
1416		dev_warn(adev->dev, "userq scheduling already started!\n");
1417	adev->userq_halt_for_enforce_isolation = false;
1418	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1419		uqm = queue->userq_mgr;
1420		mutex_lock(&uqm->userq_mutex);
1421			if (((queue->queue_type == AMDGPU_HW_IP_GFX) ||
1422			     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)) &&
1423			    (queue->xcp_id == idx)) {
1424				r = amdgpu_userq_restore_helper(uqm, queue);
1425				if (r)
1426					ret = r;
1427			}
1428		mutex_unlock(&uqm->userq_mutex);
1429	}
1430
1431	return ret;
1432}
1433
1434int amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
1435				       struct amdgpu_bo_va_mapping *mapping,
1436				       uint64_t saddr)
1437{
1438	u32 ip_mask = amdgpu_userq_get_supported_ip_mask(adev);
1439	struct amdgpu_bo_va *bo_va = mapping->bo_va;
1440	struct dma_resv *resv = bo_va->base.bo->tbo.base.resv;
1441	int ret = 0;
1442
1443	if (!ip_mask)
1444		return 0;
1445
1446	dev_warn_once(adev->dev, "now unmapping a vital queue va:%llx\n", saddr);
1447	/**
1448	 * The userq VA mapping reservation should include the eviction fence,
1449	 * if the eviction fence can't signal successfully during unmapping,
1450	 * then driver will warn to flag this improper unmap of the userq VA.
1451	 * Note: The eviction fence may be attached to different BOs, and this
1452	 * unmap is only for one kind of userq VAs, so at this point suppose
1453	 * the eviction fence is always unsignaled.
1454	 */
1455	if (!dma_resv_test_signaled(resv, DMA_RESV_USAGE_BOOKKEEP)) {
1456		ret = dma_resv_wait_timeout(resv, DMA_RESV_USAGE_BOOKKEEP, true,
1457					    MAX_SCHEDULE_TIMEOUT);
1458		if (ret <= 0)
1459			return -EBUSY;
1460	}
1461
1462	return 0;
1463}
1464
1465void amdgpu_userq_pre_reset(struct amdgpu_device *adev)
1466{
1467	const struct amdgpu_userq_funcs *userq_funcs;
1468	struct amdgpu_usermode_queue *queue;
1469	struct amdgpu_userq_mgr *uqm;
1470	unsigned long queue_id;
1471
1472	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1473		uqm = queue->userq_mgr;
1474		cancel_delayed_work_sync(&uqm->resume_work);
1475		if (queue->state == AMDGPU_USERQ_STATE_MAPPED) {
1476			amdgpu_userq_wait_for_last_fence(uqm, queue);
1477			userq_funcs = adev->userq_funcs[queue->queue_type];
1478			userq_funcs->unmap(uqm, queue);
1479			/* just mark all queues as hung at this point.
1480			 * if unmap succeeds, we could map again
1481			 * in amdgpu_userq_post_reset() if vram is not lost
1482			 */
1483			queue->state = AMDGPU_USERQ_STATE_HUNG;
1484			amdgpu_userq_fence_driver_force_completion(queue);
1485		}
1486	}
1487}
1488
1489int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost)
1490{
1491	/* if any queue state is AMDGPU_USERQ_STATE_UNMAPPED
1492	 * at this point, we should be able to map it again
1493	 * and continue if vram is not lost.
1494	 */
1495	struct amdgpu_userq_mgr *uqm;
1496	struct amdgpu_usermode_queue *queue;
1497	const struct amdgpu_userq_funcs *userq_funcs;
1498	unsigned long queue_id;
1499	int r = 0;
1500
1501	xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
1502		uqm = queue->userq_mgr;
1503		if (queue->state == AMDGPU_USERQ_STATE_HUNG && !vram_lost) {
1504			userq_funcs = adev->userq_funcs[queue->queue_type];
1505			/* Re-map queue */
1506			r = userq_funcs->map(uqm, queue);
1507			if (r) {
1508				dev_err(adev->dev, "Failed to remap queue %ld\n", queue_id);
1509				continue;
1510			}
1511			queue->state = AMDGPU_USERQ_STATE_MAPPED;
1512		}
1513	}
1514
1515	return r;
1516}