mes_userqueue.c

  1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright 2024 Advanced Micro Devices, Inc.
  4 *
  5 * Permission is hereby granted, free of charge, to any person obtaining a
  6 * copy of this software and associated documentation files (the "Software"),
  7 * to deal in the Software without restriction, including without limitation
  8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9 * and/or sell copies of the Software, and to permit persons to whom the
 10 * Software is furnished to do so, subject to the following conditions:
 11 *
 12 * The above copyright notice and this permission notice shall be included in
 13 * all copies or substantial portions of the Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 21 * OTHER DEALINGS IN THE SOFTWARE.
 22 *
 23 */
 24#include <drm/drm_drv.h>
 25#include "amdgpu.h"
 26#include "amdgpu_gfx.h"
 27#include "mes_userqueue.h"
 28#include "amdgpu_userq_fence.h"
 29
 30#define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE
 31#define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE
 32
 33static int
 34mes_userq_map_gtt_bo_to_gart(struct amdgpu_bo *bo)
 35{
 36	int ret;
 37
 38	ret = amdgpu_bo_reserve(bo, true);
 39	if (ret) {
 40		DRM_ERROR("Failed to reserve bo. ret %d\n", ret);
 41		goto err_reserve_bo_failed;
 42	}
 43
 44	ret = amdgpu_ttm_alloc_gart(&bo->tbo);
 45	if (ret) {
 46		DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret);
 47		goto err_map_bo_gart_failed;
 48	}
 49
 50	amdgpu_bo_unreserve(bo);
 51	bo = amdgpu_bo_ref(bo);
 52
 53	return 0;
 54
 55err_map_bo_gart_failed:
 56	amdgpu_bo_unreserve(bo);
 57err_reserve_bo_failed:
 58	return ret;
 59}
 60
 61static int
 62mes_userq_create_wptr_mapping(struct amdgpu_userq_mgr *uq_mgr,
 63			      struct amdgpu_usermode_queue *queue,
 64			      uint64_t wptr)
 65{
 66	struct amdgpu_bo_va_mapping *wptr_mapping;
 67	struct amdgpu_vm *wptr_vm;
 68	struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj;
 69	int ret;
 70
 71	wptr_vm = queue->vm;
 72	ret = amdgpu_bo_reserve(wptr_vm->root.bo, false);
 73	if (ret)
 74		return ret;
 75
 76	wptr &= AMDGPU_GMC_HOLE_MASK;
 77	wptr_mapping = amdgpu_vm_bo_lookup_mapping(wptr_vm, wptr >> PAGE_SHIFT);
 78	amdgpu_bo_unreserve(wptr_vm->root.bo);
 79	if (!wptr_mapping) {
 80		DRM_ERROR("Failed to lookup wptr bo\n");
 81		return -EINVAL;
 82	}
 83
 84	wptr_obj->obj = wptr_mapping->bo_va->base.bo;
 85	if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) {
 86		DRM_ERROR("Requested GART mapping for wptr bo larger than one page\n");
 87		return -EINVAL;
 88	}
 89
 90	ret = mes_userq_map_gtt_bo_to_gart(wptr_obj->obj);
 91	if (ret) {
 92		DRM_ERROR("Failed to map wptr bo to GART\n");
 93		return ret;
 94	}
 95
 96	queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset_no_check(wptr_obj->obj);
 97	return 0;
 98}
 99
100static int convert_to_mes_priority(int priority)
101{
102	switch (priority) {
103	case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_LOW:
104	default:
105		return AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
106	case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_LOW:
107		return AMDGPU_MES_PRIORITY_LEVEL_LOW;
108	case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_NORMAL_HIGH:
109		return AMDGPU_MES_PRIORITY_LEVEL_MEDIUM;
110	case AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_HIGH:
111		return AMDGPU_MES_PRIORITY_LEVEL_HIGH;
112	}
113}
114
115static int mes_userq_map(struct amdgpu_userq_mgr *uq_mgr,
116			 struct amdgpu_usermode_queue *queue)
117{
118	struct amdgpu_device *adev = uq_mgr->adev;
119	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
120	struct amdgpu_mqd_prop *userq_props = queue->userq_prop;
121	struct mes_add_queue_input queue_input;
122	int r;
123
124	memset(&queue_input, 0x0, sizeof(struct mes_add_queue_input));
125
126	queue_input.process_va_start = 0;
127	queue_input.process_va_end = adev->vm_manager.max_pfn - 1;
128
129	/* set process quantum to 10 ms and gang quantum to 1 ms as default */
130	queue_input.process_quantum = 100000;
131	queue_input.gang_quantum = 10000;
132	queue_input.paging = false;
133
134	queue_input.process_context_addr = ctx->gpu_addr;
135	queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
136	queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL;
137	queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority);
138
139	queue_input.process_id = queue->vm->pasid;
140	queue_input.queue_type = queue->queue_type;
141	queue_input.mqd_addr = queue->mqd.gpu_addr;
142	queue_input.wptr_addr = userq_props->wptr_gpu_addr;
143	queue_input.queue_size = userq_props->queue_size >> 2;
144	queue_input.doorbell_offset = userq_props->doorbell_index;
145	queue_input.page_table_base_addr = amdgpu_gmc_pd_addr(queue->vm->root.bo);
146	queue_input.wptr_mc_addr = queue->wptr_obj.gpu_addr;
147
148	amdgpu_mes_lock(&adev->mes);
149	r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input);
150	amdgpu_mes_unlock(&adev->mes);
151	if (r) {
152		DRM_ERROR("Failed to map queue in HW, err (%d)\n", r);
153		return r;
154	}
155
156	DRM_DEBUG_DRIVER("Queue (doorbell:%d) mapped successfully\n", userq_props->doorbell_index);
157	return 0;
158}
159
160static int mes_userq_unmap(struct amdgpu_userq_mgr *uq_mgr,
161			   struct amdgpu_usermode_queue *queue)
162{
163	struct amdgpu_device *adev = uq_mgr->adev;
164	struct mes_remove_queue_input queue_input;
165	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
166	int r;
167
168	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
169	queue_input.doorbell_offset = queue->doorbell_index;
170	queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
171
172	amdgpu_mes_lock(&adev->mes);
173	r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input);
174	amdgpu_mes_unlock(&adev->mes);
175	if (r)
176		DRM_ERROR("Failed to unmap queue in HW, err (%d)\n", r);
177	return r;
178}
179
180static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr,
181				      struct amdgpu_usermode_queue *queue,
182				      struct drm_amdgpu_userq_in *mqd_user)
183{
184	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
185	int r, size;
186
187	/*
188	 * The FW expects at least one page space allocated for
189	 * process ctx and gang ctx each. Create an object
190	 * for the same.
191	 */
192	size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ;
193	r = amdgpu_userq_create_object(uq_mgr, ctx, size);
194	if (r) {
195		DRM_ERROR("Failed to allocate ctx space bo for userqueue, err:%d\n", r);
196		return r;
197	}
198
199	return 0;
200}
201
202static int mes_userq_detect_and_reset(struct amdgpu_device *adev,
203				      int queue_type)
204{
205	int db_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev);
206	struct mes_detect_and_reset_queue_input input;
207	struct amdgpu_usermode_queue *queue;
208	unsigned int hung_db_num = 0;
209	unsigned long queue_id;
210	u32 db_array[8];
211	bool found_hung_queue = false;
212	int r, i;
213
214	if (db_array_size > 8) {
215		dev_err(adev->dev, "DB array size (%d vs 8) too small\n",
216			db_array_size);
217		return -EINVAL;
218	}
219
220	memset(&input, 0x0, sizeof(struct mes_detect_and_reset_queue_input));
221
222	input.queue_type = queue_type;
223
224	amdgpu_mes_lock(&adev->mes);
225	r = amdgpu_mes_detect_and_reset_hung_queues(adev, queue_type, false,
226						    &hung_db_num, db_array);
227	amdgpu_mes_unlock(&adev->mes);
228	if (r) {
229		dev_err(adev->dev, "Failed to detect and reset queues, err (%d)\n", r);
230	} else if (hung_db_num) {
231		xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) {
232			if (queue->queue_type == queue_type) {
233				for (i = 0; i < hung_db_num; i++) {
234					if (queue->doorbell_index == db_array[i]) {
235						queue->state = AMDGPU_USERQ_STATE_HUNG;
236						found_hung_queue = true;
237						atomic_inc(&adev->gpu_reset_counter);
238						amdgpu_userq_fence_driver_force_completion(queue);
239						drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, NULL);
240					}
241				}
242			}
243		}
244	}
245
246	if (found_hung_queue) {
247		/* Resume scheduling after hang recovery */
248		r = amdgpu_mes_resume(adev);
249	}
250
251	return r;
252}
253
254static int mes_userq_mqd_create(struct amdgpu_userq_mgr *uq_mgr,
255				struct drm_amdgpu_userq_in *args_in,
256				struct amdgpu_usermode_queue *queue)
257{
258	struct amdgpu_device *adev = uq_mgr->adev;
259	struct amdgpu_mqd *mqd_hw_default = &adev->mqds[queue->queue_type];
260	struct drm_amdgpu_userq_in *mqd_user = args_in;
261	struct amdgpu_mqd_prop *userq_props;
262	int r;
263
264	/* Structure to initialize MQD for userqueue using generic MQD init function */
265	userq_props = kzalloc(sizeof(struct amdgpu_mqd_prop), GFP_KERNEL);
266	if (!userq_props) {
267		DRM_ERROR("Failed to allocate memory for userq_props\n");
268		return -ENOMEM;
269	}
270
271	r = amdgpu_userq_create_object(uq_mgr, &queue->mqd, mqd_hw_default->mqd_size);
272	if (r) {
273		DRM_ERROR("Failed to create MQD object for userqueue\n");
274		goto free_props;
275	}
276
277	/* Initialize the MQD BO with user given values */
278	userq_props->wptr_gpu_addr = mqd_user->wptr_va;
279	userq_props->rptr_gpu_addr = mqd_user->rptr_va;
280	userq_props->queue_size = mqd_user->queue_size;
281	userq_props->hqd_base_gpu_addr = mqd_user->queue_va;
282	userq_props->mqd_gpu_addr = queue->mqd.gpu_addr;
283	userq_props->use_doorbell = true;
284	userq_props->doorbell_index = queue->doorbell_index;
285	userq_props->fence_address = queue->fence_drv->gpu_addr;
286
287	if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) {
288		struct drm_amdgpu_userq_mqd_compute_gfx11 *compute_mqd;
289
290		if (mqd_user->mqd_size != sizeof(*compute_mqd)) {
291			DRM_ERROR("Invalid compute IP MQD size\n");
292			r = -EINVAL;
293			goto free_mqd;
294		}
295
296		compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
297		if (IS_ERR(compute_mqd)) {
298			DRM_ERROR("Failed to read user MQD\n");
299			r = -ENOMEM;
300			goto free_mqd;
301		}
302
303		r = amdgpu_userq_input_va_validate(queue, compute_mqd->eop_va,
304						   2048);
305		if (r)
306			goto free_mqd;
307
308		userq_props->eop_gpu_addr = compute_mqd->eop_va;
309		userq_props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL;
310		userq_props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM;
311		userq_props->hqd_active = false;
312		userq_props->tmz_queue =
313			mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
314		kfree(compute_mqd);
315	} else if (queue->queue_type == AMDGPU_HW_IP_GFX) {
316		struct drm_amdgpu_userq_mqd_gfx11 *mqd_gfx_v11;
317		struct amdgpu_gfx_shadow_info shadow_info;
318
319		if (adev->gfx.funcs->get_gfx_shadow_info) {
320			adev->gfx.funcs->get_gfx_shadow_info(adev, &shadow_info, true);
321		} else {
322			r = -EINVAL;
323			goto free_mqd;
324		}
325
326		if (mqd_user->mqd_size != sizeof(*mqd_gfx_v11) || !mqd_user->mqd) {
327			DRM_ERROR("Invalid GFX MQD\n");
328			r = -EINVAL;
329			goto free_mqd;
330		}
331
332		mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
333		if (IS_ERR(mqd_gfx_v11)) {
334			DRM_ERROR("Failed to read user MQD\n");
335			r = -ENOMEM;
336			goto free_mqd;
337		}
338
339		userq_props->shadow_addr = mqd_gfx_v11->shadow_va;
340		userq_props->csa_addr = mqd_gfx_v11->csa_va;
341		userq_props->tmz_queue =
342			mqd_user->flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_SECURE;
343
344		r = amdgpu_userq_input_va_validate(queue, mqd_gfx_v11->shadow_va,
345						   shadow_info.shadow_size);
346		if (r)
347			goto free_mqd;
348		r = amdgpu_userq_input_va_validate(queue, mqd_gfx_v11->csa_va,
349						   shadow_info.csa_size);
350		if (r)
351			goto free_mqd;
352
353		kfree(mqd_gfx_v11);
354	} else if (queue->queue_type == AMDGPU_HW_IP_DMA) {
355		struct drm_amdgpu_userq_mqd_sdma_gfx11 *mqd_sdma_v11;
356
357		if (mqd_user->mqd_size != sizeof(*mqd_sdma_v11) || !mqd_user->mqd) {
358			DRM_ERROR("Invalid SDMA MQD\n");
359			r = -EINVAL;
360			goto free_mqd;
361		}
362
363		mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size);
364		if (IS_ERR(mqd_sdma_v11)) {
365			DRM_ERROR("Failed to read sdma user MQD\n");
366			r = -ENOMEM;
367			goto free_mqd;
368		}
369		r = amdgpu_userq_input_va_validate(queue, mqd_sdma_v11->csa_va,
370						   32);
371		if (r)
372			goto free_mqd;
373
374		userq_props->csa_addr = mqd_sdma_v11->csa_va;
375		kfree(mqd_sdma_v11);
376	}
377
378	queue->userq_prop = userq_props;
379
380	r = mqd_hw_default->init_mqd(adev, (void *)queue->mqd.cpu_ptr, userq_props);
381	if (r) {
382		DRM_ERROR("Failed to initialize MQD for userqueue\n");
383		goto free_mqd;
384	}
385
386	/* Create BO for FW operations */
387	r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user);
388	if (r) {
389		DRM_ERROR("Failed to allocate BO for userqueue (%d)", r);
390		goto free_mqd;
391	}
392
393	/* FW expects WPTR BOs to be mapped into GART */
394	r = mes_userq_create_wptr_mapping(uq_mgr, queue, userq_props->wptr_gpu_addr);
395	if (r) {
396		DRM_ERROR("Failed to create WPTR mapping\n");
397		goto free_ctx;
398	}
399
400	return 0;
401
402free_ctx:
403	amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj);
404
405free_mqd:
406	amdgpu_userq_destroy_object(uq_mgr, &queue->mqd);
407
408free_props:
409	kfree(userq_props);
410
411	return r;
412}
413
414static void
415mes_userq_mqd_destroy(struct amdgpu_userq_mgr *uq_mgr,
416		      struct amdgpu_usermode_queue *queue)
417{
418	amdgpu_userq_destroy_object(uq_mgr, &queue->fw_obj);
419	kfree(queue->userq_prop);
420	amdgpu_userq_destroy_object(uq_mgr, &queue->mqd);
421}
422
423static int mes_userq_preempt(struct amdgpu_userq_mgr *uq_mgr,
424				struct amdgpu_usermode_queue *queue)
425{
426	struct amdgpu_device *adev = uq_mgr->adev;
427	struct mes_suspend_gang_input queue_input;
428	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
429	signed long timeout = 2100000; /* 2100 ms */
430	u64 fence_gpu_addr;
431	u32 fence_offset;
432	u64 *fence_ptr;
433	int i, r;
434
435	if (queue->state != AMDGPU_USERQ_STATE_MAPPED)
436		return 0;
437	r = amdgpu_device_wb_get(adev, &fence_offset);
438	if (r)
439		return r;
440
441	fence_gpu_addr = adev->wb.gpu_addr + (fence_offset * 4);
442	fence_ptr = (u64 *)&adev->wb.wb[fence_offset];
443	*fence_ptr = 0;
444
445	memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input));
446	queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
447	queue_input.suspend_fence_addr = fence_gpu_addr;
448	queue_input.suspend_fence_value = 1;
449	amdgpu_mes_lock(&adev->mes);
450	r = adev->mes.funcs->suspend_gang(&adev->mes, &queue_input);
451	amdgpu_mes_unlock(&adev->mes);
452	if (r) {
453		DRM_ERROR("Failed to suspend gang: %d\n", r);
454		goto out;
455	}
456
457	for (i = 0; i < timeout; i++) {
458		if (*fence_ptr == 1)
459			goto out;
460		udelay(1);
461	}
462	r = -ETIMEDOUT;
463
464out:
465	amdgpu_device_wb_free(adev, fence_offset);
466	return r;
467}
468
469static int mes_userq_restore(struct amdgpu_userq_mgr *uq_mgr,
470				struct amdgpu_usermode_queue *queue)
471{
472	struct amdgpu_device *adev = uq_mgr->adev;
473	struct mes_resume_gang_input queue_input;
474	struct amdgpu_userq_obj *ctx = &queue->fw_obj;
475	int r;
476
477	if (queue->state == AMDGPU_USERQ_STATE_HUNG)
478		return -EINVAL;
479	if (queue->state != AMDGPU_USERQ_STATE_PREEMPTED)
480		return 0;
481
482	memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input));
483	queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ;
484
485	amdgpu_mes_lock(&adev->mes);
486	r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input);
487	amdgpu_mes_unlock(&adev->mes);
488	if (r)
489		dev_err(adev->dev, "Failed to resume queue, err (%d)\n", r);
490	return r;
491}
492
493const struct amdgpu_userq_funcs userq_mes_funcs = {
494	.mqd_create = mes_userq_mqd_create,
495	.mqd_destroy = mes_userq_mqd_destroy,
496	.unmap = mes_userq_unmap,
497	.map = mes_userq_map,
498	.detect_and_reset = mes_userq_detect_and_reset,
499	.preempt = mes_userq_preempt,
500	.restore = mes_userq_restore,
501};