amdxdna_ctx.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
  4 */
  5
  6#include <drm/amdxdna_accel.h>
  7#include <drm/drm_device.h>
  8#include <drm/drm_drv.h>
  9#include <drm/drm_file.h>
 10#include <drm/drm_gem.h>
 11#include <drm/drm_gem_shmem_helper.h>
 12#include <drm/drm_print.h>
 13#include <drm/gpu_scheduler.h>
 14#include <linux/xarray.h>
 15#include <trace/events/amdxdna.h>
 16
 17#include "amdxdna_ctx.h"
 18#include "amdxdna_gem.h"
 19#include "amdxdna_pci_drv.h"
 20#include "amdxdna_pm.h"
 21
 22#define MAX_HWCTX_ID		255
 23#define MAX_ARG_COUNT		4095
 24
 25struct amdxdna_fence {
 26	struct dma_fence	base;
 27	spinlock_t		lock; /* for base */
 28	struct amdxdna_hwctx	*hwctx;
 29};
 30
 31static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence)
 32{
 33	return KBUILD_MODNAME;
 34}
 35
 36static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence)
 37{
 38	struct amdxdna_fence *xdna_fence;
 39
 40	xdna_fence = container_of(fence, struct amdxdna_fence, base);
 41
 42	return xdna_fence->hwctx->name;
 43}
 44
 45static const struct dma_fence_ops fence_ops = {
 46	.get_driver_name = amdxdna_fence_get_driver_name,
 47	.get_timeline_name = amdxdna_fence_get_timeline_name,
 48};
 49
 50static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx)
 51{
 52	struct amdxdna_fence *fence;
 53
 54	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 55	if (!fence)
 56		return NULL;
 57
 58	fence->hwctx = hwctx;
 59	spin_lock_init(&fence->lock);
 60	dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0);
 61	return &fence->base;
 62}
 63
 64static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx,
 65				      struct srcu_struct *ss)
 66{
 67	struct amdxdna_dev *xdna = hwctx->client->xdna;
 68
 69	synchronize_srcu(ss);
 70
 71	/* At this point, user is not able to submit new commands */
 72	xdna->dev_info->ops->hwctx_fini(hwctx);
 73
 74	kfree(hwctx->name);
 75	kfree(hwctx);
 76}
 77
 78int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg,
 79		       int (*walk)(struct amdxdna_hwctx *hwctx, void *arg))
 80{
 81	struct amdxdna_hwctx *hwctx;
 82	unsigned long hwctx_id;
 83	int ret = 0, idx;
 84
 85	idx = srcu_read_lock(&client->hwctx_srcu);
 86	amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
 87		ret = walk(hwctx, arg);
 88		if (ret)
 89			break;
 90	}
 91	srcu_read_unlock(&client->hwctx_srcu, idx);
 92
 93	return ret;
 94}
 95
 96void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size)
 97{
 98	struct amdxdna_cmd *cmd = abo->mem.kva;
 99	u32 num_masks, count;
100
101	if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
102		num_masks = 0;
103	else
104		num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
105
106	if (size) {
107		count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header);
108		if (unlikely(count <= num_masks ||
109			     count * sizeof(u32) +
110			     offsetof(struct amdxdna_cmd, data[0]) >
111			     abo->mem.size)) {
112			*size = 0;
113			return NULL;
114		}
115		*size = (count - num_masks) * sizeof(u32);
116	}
117	return &cmd->data[num_masks];
118}
119
120u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo)
121{
122	struct amdxdna_cmd *cmd = abo->mem.kva;
123	u32 num_masks, i;
124	u32 *cu_mask;
125
126	if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
127		return INVALID_CU_IDX;
128
129	num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
130	cu_mask = cmd->data;
131	for (i = 0; i < num_masks; i++) {
132		if (cu_mask[i])
133			return ffs(cu_mask[i]) - 1;
134	}
135
136	return INVALID_CU_IDX;
137}
138
139int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo,
140			  struct amdxdna_sched_job *job, u32 cmd_idx,
141			  enum ert_cmd_state error_state)
142{
143	struct amdxdna_client *client = job->hwctx->client;
144	struct amdxdna_cmd *cmd = abo->mem.kva;
145	struct amdxdna_cmd_chain *cc = NULL;
146
147	cmd->header &= ~AMDXDNA_CMD_STATE;
148	cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state);
149
150	if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) {
151		cc = amdxdna_cmd_get_payload(abo, NULL);
152		cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0;
153		abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_CMD);
154		if (!abo)
155			return -EINVAL;
156		cmd = abo->mem.kva;
157	}
158
159	memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd));
160	if (cc)
161		amdxdna_gem_put_obj(abo);
162
163	return 0;
164}
165
166/*
167 * This should be called in close() and remove(). DO NOT call in other syscalls.
168 * This guarantee that when hwctx and resources will be released, if user
169 * doesn't call amdxdna_drm_destroy_hwctx_ioctl.
170 */
171void amdxdna_hwctx_remove_all(struct amdxdna_client *client)
172{
173	struct amdxdna_hwctx *hwctx;
174	unsigned long hwctx_id;
175
176	amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
177		XDNA_DBG(client->xdna, "PID %d close HW context %d",
178			 client->pid, hwctx->id);
179		xa_erase(&client->hwctx_xa, hwctx->id);
180		amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
181	}
182}
183
184int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
185{
186	struct amdxdna_client *client = filp->driver_priv;
187	struct amdxdna_drm_create_hwctx *args = data;
188	struct amdxdna_dev *xdna = to_xdna_dev(dev);
189	struct amdxdna_hwctx *hwctx;
190	int ret, idx;
191
192	if (args->ext || args->ext_flags)
193		return -EINVAL;
194
195	hwctx = kzalloc(sizeof(*hwctx), GFP_KERNEL);
196	if (!hwctx)
197		return -ENOMEM;
198
199	if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) {
200		XDNA_ERR(xdna, "Access QoS info failed");
201		kfree(hwctx);
202		return -EFAULT;
203	}
204
205	hwctx->client = client;
206	hwctx->fw_ctx_id = -1;
207	hwctx->num_tiles = args->num_tiles;
208	hwctx->mem_size = args->mem_size;
209	hwctx->max_opc = args->max_opc;
210
211	guard(mutex)(&xdna->dev_lock);
212
213	if (!drm_dev_enter(dev, &idx)) {
214		ret = -ENODEV;
215		goto free_hwctx;
216	}
217
218	ret = xdna->dev_info->ops->hwctx_init(hwctx);
219	if (ret) {
220		XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret);
221		goto dev_exit;
222	}
223
224	hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id);
225	if (!hwctx->name) {
226		ret = -ENOMEM;
227		goto fini_hwctx;
228	}
229
230	ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx,
231			      XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID),
232			      &client->next_hwctxid, GFP_KERNEL);
233	if (ret < 0) {
234		XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret);
235		goto free_name;
236	}
237
238	args->handle = hwctx->id;
239	args->syncobj_handle = hwctx->syncobj_hdl;
240
241	atomic64_set(&hwctx->job_submit_cnt, 0);
242	atomic64_set(&hwctx->job_free_cnt, 0);
243	XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret);
244	drm_dev_exit(idx);
245	return 0;
246
247free_name:
248	kfree(hwctx->name);
249fini_hwctx:
250	xdna->dev_info->ops->hwctx_fini(hwctx);
251dev_exit:
252	drm_dev_exit(idx);
253free_hwctx:
254	kfree(hwctx);
255	return ret;
256}
257
258int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
259{
260	struct amdxdna_client *client = filp->driver_priv;
261	struct amdxdna_drm_destroy_hwctx *args = data;
262	struct amdxdna_dev *xdna = to_xdna_dev(dev);
263	struct amdxdna_hwctx *hwctx;
264	int ret = 0, idx;
265
266	if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
267		return -EINVAL;
268
269	if (!drm_dev_enter(dev, &idx))
270		return -ENODEV;
271
272	mutex_lock(&xdna->dev_lock);
273	hwctx = xa_erase(&client->hwctx_xa, args->handle);
274	if (!hwctx) {
275		ret = -EINVAL;
276		XDNA_DBG(xdna, "PID %d HW context %d not exist",
277			 client->pid, args->handle);
278		goto out;
279	}
280
281	/*
282	 * The pushed jobs are handled by DRM scheduler during destroy.
283	 * SRCU to synchronize with exec command ioctls.
284	 */
285	amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
286
287	XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle);
288out:
289	mutex_unlock(&xdna->dev_lock);
290	drm_dev_exit(idx);
291	return ret;
292}
293
294int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
295{
296	struct amdxdna_client *client = filp->driver_priv;
297	struct amdxdna_drm_config_hwctx *args = data;
298	struct amdxdna_dev *xdna = to_xdna_dev(dev);
299	struct amdxdna_hwctx *hwctx;
300	u32 buf_size;
301	void *buf;
302	int ret;
303	u64 val;
304
305	if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
306		return -EINVAL;
307
308	if (!xdna->dev_info->ops->hwctx_config)
309		return -EOPNOTSUPP;
310
311	val = args->param_val;
312	buf_size = args->param_val_size;
313
314	switch (args->param_type) {
315	case DRM_AMDXDNA_HWCTX_CONFIG_CU:
316		/* For those types that param_val is pointer */
317		if (buf_size > PAGE_SIZE) {
318			XDNA_ERR(xdna, "Config CU param buffer too large");
319			return -E2BIG;
320		}
321
322		/* Hwctx needs to keep buf */
323		buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
324		if (!buf)
325			return -ENOMEM;
326
327		if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) {
328			kfree(buf);
329			return -EFAULT;
330		}
331
332		break;
333	case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF:
334	case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF:
335		/* For those types that param_val is a value */
336		buf = NULL;
337		buf_size = 0;
338		break;
339	default:
340		XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type);
341		return -EINVAL;
342	}
343
344	guard(mutex)(&xdna->dev_lock);
345	hwctx = xa_load(&client->hwctx_xa, args->handle);
346	if (!hwctx) {
347		XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle);
348		ret = -EINVAL;
349		goto free_buf;
350	}
351
352	ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size);
353
354free_buf:
355	kfree(buf);
356	return ret;
357}
358
359int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl)
360{
361	struct amdxdna_dev *xdna = client->xdna;
362	struct amdxdna_hwctx *hwctx;
363	struct amdxdna_gem_obj *abo;
364	struct drm_gem_object *gobj;
365	int ret;
366
367	if (!xdna->dev_info->ops->hwctx_sync_debug_bo)
368		return -EOPNOTSUPP;
369
370	gobj = drm_gem_object_lookup(client->filp, debug_bo_hdl);
371	if (!gobj)
372		return -EINVAL;
373
374	abo = to_xdna_obj(gobj);
375	guard(mutex)(&xdna->dev_lock);
376	hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx);
377	if (!hwctx) {
378		ret = -EINVAL;
379		goto put_obj;
380	}
381
382	ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl);
383
384put_obj:
385	drm_gem_object_put(gobj);
386	return ret;
387}
388
389static void
390amdxdna_arg_bos_put(struct amdxdna_sched_job *job)
391{
392	int i;
393
394	for (i = 0; i < job->bo_cnt; i++) {
395		if (!job->bos[i])
396			break;
397		drm_gem_object_put(job->bos[i]);
398	}
399}
400
401static int
402amdxdna_arg_bos_lookup(struct amdxdna_client *client,
403		       struct amdxdna_sched_job *job,
404		       u32 *bo_hdls, u32 bo_cnt)
405{
406	struct drm_gem_object *gobj;
407	int i, ret;
408
409	job->bo_cnt = bo_cnt;
410	for (i = 0; i < job->bo_cnt; i++) {
411		struct amdxdna_gem_obj *abo;
412
413		gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]);
414		if (!gobj) {
415			ret = -ENOENT;
416			goto put_shmem_bo;
417		}
418		abo = to_xdna_obj(gobj);
419
420		mutex_lock(&abo->lock);
421		if (abo->pinned) {
422			mutex_unlock(&abo->lock);
423			job->bos[i] = gobj;
424			continue;
425		}
426
427		ret = amdxdna_gem_pin_nolock(abo);
428		if (ret) {
429			mutex_unlock(&abo->lock);
430			drm_gem_object_put(gobj);
431			goto put_shmem_bo;
432		}
433		abo->pinned = true;
434		mutex_unlock(&abo->lock);
435
436		job->bos[i] = gobj;
437	}
438
439	return 0;
440
441put_shmem_bo:
442	amdxdna_arg_bos_put(job);
443	return ret;
444}
445
446void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job)
447{
448	trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release");
449	amdxdna_pm_suspend_put(job->hwctx->client->xdna);
450	amdxdna_arg_bos_put(job);
451	amdxdna_gem_put_obj(job->cmd_bo);
452	dma_fence_put(job->fence);
453}
454
455int amdxdna_cmd_submit(struct amdxdna_client *client,
456		       struct amdxdna_drv_cmd *drv_cmd,
457		       u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt,
458		       u32 hwctx_hdl, u64 *seq)
459{
460	struct amdxdna_dev *xdna = client->xdna;
461	struct amdxdna_sched_job *job;
462	struct amdxdna_hwctx *hwctx;
463	int ret, idx;
464
465	XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt);
466	job = kzalloc(struct_size(job, bos, arg_bo_cnt), GFP_KERNEL);
467	if (!job)
468		return -ENOMEM;
469
470	job->drv_cmd = drv_cmd;
471
472	if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) {
473		job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_CMD);
474		if (!job->cmd_bo) {
475			XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl);
476			ret = -EINVAL;
477			goto free_job;
478		}
479	}
480
481	ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt);
482	if (ret) {
483		XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret);
484		goto cmd_put;
485	}
486
487	ret = amdxdna_pm_resume_get(xdna);
488	if (ret) {
489		XDNA_ERR(xdna, "Resume failed, ret %d", ret);
490		goto put_bos;
491	}
492
493	idx = srcu_read_lock(&client->hwctx_srcu);
494	hwctx = xa_load(&client->hwctx_xa, hwctx_hdl);
495	if (!hwctx) {
496		XDNA_DBG(xdna, "PID %d failed to get hwctx %d",
497			 client->pid, hwctx_hdl);
498		ret = -EINVAL;
499		goto unlock_srcu;
500	}
501
502
503	job->hwctx = hwctx;
504	job->mm = current->mm;
505
506	job->fence = amdxdna_fence_create(hwctx);
507	if (!job->fence) {
508		XDNA_ERR(xdna, "Failed to create fence");
509		ret = -ENOMEM;
510		goto unlock_srcu;
511	}
512	kref_init(&job->refcnt);
513
514	ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq);
515	if (ret)
516		goto put_fence;
517
518	/*
519	 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated
520	 * resource after synchronize_srcu(). The submitted jobs should be
521	 * handled by the queue, for example DRM scheduler, in device layer.
522	 * For here we can unlock SRCU.
523	 */
524	srcu_read_unlock(&client->hwctx_srcu, idx);
525	trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed");
526
527	return 0;
528
529put_fence:
530	dma_fence_put(job->fence);
531unlock_srcu:
532	srcu_read_unlock(&client->hwctx_srcu, idx);
533	amdxdna_pm_suspend_put(xdna);
534put_bos:
535	amdxdna_arg_bos_put(job);
536cmd_put:
537	amdxdna_gem_put_obj(job->cmd_bo);
538free_job:
539	kfree(job);
540	return ret;
541}
542
543/*
544 * The submit command ioctl submits a command to firmware. One firmware command
545 * may contain multiple command BOs for processing as a whole.
546 * The command sequence number is returned which can be used for wait command ioctl.
547 */
548static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client,
549				      struct amdxdna_drm_exec_cmd *args)
550{
551	struct amdxdna_dev *xdna = client->xdna;
552	u32 *arg_bo_hdls = NULL;
553	u32 cmd_bo_hdl;
554	int ret;
555
556	if (args->arg_count > MAX_ARG_COUNT) {
557		XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count);
558		return -EINVAL;
559	}
560
561	/* Only support single command for now. */
562	if (args->cmd_count != 1) {
563		XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count);
564		return -EINVAL;
565	}
566
567	cmd_bo_hdl = (u32)args->cmd_handles;
568	if (args->arg_count) {
569		arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL);
570		if (!arg_bo_hdls)
571			return -ENOMEM;
572		ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args),
573				     args->arg_count * sizeof(u32));
574		if (ret) {
575			ret = -EFAULT;
576			goto free_cmd_bo_hdls;
577		}
578	}
579
580	ret = amdxdna_cmd_submit(client, NULL, cmd_bo_hdl, arg_bo_hdls,
581				 args->arg_count, args->hwctx, &args->seq);
582	if (ret)
583		XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret);
584
585free_cmd_bo_hdls:
586	kfree(arg_bo_hdls);
587	if (!ret)
588		XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq);
589	return ret;
590}
591
592int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
593{
594	struct amdxdna_client *client = filp->driver_priv;
595	struct amdxdna_drm_exec_cmd *args = data;
596
597	if (args->ext || args->ext_flags)
598		return -EINVAL;
599
600	switch (args->type) {
601	case AMDXDNA_CMD_SUBMIT_EXEC_BUF:
602		return amdxdna_drm_submit_execbuf(client, args);
603	}
604
605	XDNA_ERR(client->xdna, "Invalid command type %d", args->type);
606	return -EINVAL;
607}