amdxdna_pci_drv.c

  1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
  4 */
  5
  6#include <drm/amdxdna_accel.h>
  7#include <drm/drm_accel.h>
  8#include <drm/drm_drv.h>
  9#include <drm/drm_gem.h>
 10#include <drm/drm_gem_shmem_helper.h>
 11#include <drm/drm_ioctl.h>
 12#include <drm/drm_managed.h>
 13#include <drm/gpu_scheduler.h>
 14#include <linux/iommu.h>
 15#include <linux/pci.h>
 16
 17#include "amdxdna_ctx.h"
 18#include "amdxdna_gem.h"
 19#include "amdxdna_pci_drv.h"
 20#include "amdxdna_pm.h"
 21
 22MODULE_FIRMWARE("amdnpu/1502_00/npu.sbin");
 23MODULE_FIRMWARE("amdnpu/17f0_10/npu.sbin");
 24MODULE_FIRMWARE("amdnpu/17f0_11/npu.sbin");
 25MODULE_FIRMWARE("amdnpu/17f0_20/npu.sbin");
 26
 27/*
 28 * 0.0: Initial version
 29 * 0.1: Support getting all hardware contexts by DRM_IOCTL_AMDXDNA_GET_ARRAY
 30 * 0.2: Support getting last error hardware error
 31 * 0.3: Support firmware debug buffer
 32 * 0.4: Support getting resource information
 33 * 0.5: Support getting telemetry data
 34 * 0.6: Support preemption
 35 */
 36#define AMDXDNA_DRIVER_MAJOR		0
 37#define AMDXDNA_DRIVER_MINOR		6
 38
 39/*
 40 * Bind the driver base on (vendor_id, device_id) pair and later use the
 41 * (device_id, rev_id) pair as a key to select the devices. The devices with
 42 * same device_id have very similar interface to host driver.
 43 */
 44static const struct pci_device_id pci_ids[] = {
 45	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) },
 46	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) },
 47	{0}
 48};
 49
 50MODULE_DEVICE_TABLE(pci, pci_ids);
 51
 52static const struct amdxdna_device_id amdxdna_ids[] = {
 53	{ 0x1502, 0x0,  &dev_npu1_info },
 54	{ 0x17f0, 0x0,  &dev_npu2_info },
 55	{ 0x17f0, 0x10, &dev_npu4_info },
 56	{ 0x17f0, 0x11, &dev_npu5_info },
 57	{ 0x17f0, 0x20, &dev_npu6_info },
 58	{0}
 59};
 60
 61static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
 62{
 63	struct amdxdna_dev *xdna = to_xdna_dev(ddev);
 64	struct amdxdna_client *client;
 65	int ret;
 66
 67	client = kzalloc(sizeof(*client), GFP_KERNEL);
 68	if (!client)
 69		return -ENOMEM;
 70
 71	client->pid = pid_nr(rcu_access_pointer(filp->pid));
 72	client->xdna = xdna;
 73
 74	client->sva = iommu_sva_bind_device(xdna->ddev.dev, current->mm);
 75	if (IS_ERR(client->sva)) {
 76		ret = PTR_ERR(client->sva);
 77		XDNA_ERR(xdna, "SVA bind device failed, ret %d", ret);
 78		goto failed;
 79	}
 80	client->pasid = iommu_sva_get_pasid(client->sva);
 81	if (client->pasid == IOMMU_PASID_INVALID) {
 82		XDNA_ERR(xdna, "SVA get pasid failed");
 83		ret = -ENODEV;
 84		goto unbind_sva;
 85	}
 86	client->mm = current->mm;
 87	mmgrab(client->mm);
 88	init_srcu_struct(&client->hwctx_srcu);
 89	xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC);
 90	mutex_init(&client->mm_lock);
 91
 92	mutex_lock(&xdna->dev_lock);
 93	list_add_tail(&client->node, &xdna->client_list);
 94	mutex_unlock(&xdna->dev_lock);
 95
 96	filp->driver_priv = client;
 97	client->filp = filp;
 98
 99	XDNA_DBG(xdna, "pid %d opened", client->pid);
100	return 0;
101
102unbind_sva:
103	iommu_sva_unbind_device(client->sva);
104failed:
105	kfree(client);
106
107	return ret;
108}
109
110static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp)
111{
112	struct amdxdna_client *client = filp->driver_priv;
113	struct amdxdna_dev *xdna = to_xdna_dev(ddev);
114
115	XDNA_DBG(xdna, "closing pid %d", client->pid);
116
117	xa_destroy(&client->hwctx_xa);
118	cleanup_srcu_struct(&client->hwctx_srcu);
119	mutex_destroy(&client->mm_lock);
120	if (client->dev_heap)
121		drm_gem_object_put(to_gobj(client->dev_heap));
122
123	iommu_sva_unbind_device(client->sva);
124	mmdrop(client->mm);
125
126	XDNA_DBG(xdna, "pid %d closed", client->pid);
127	kfree(client);
128}
129
130static int amdxdna_flush(struct file *f, fl_owner_t id)
131{
132	struct drm_file *filp = f->private_data;
133	struct amdxdna_client *client = filp->driver_priv;
134	struct amdxdna_dev *xdna = client->xdna;
135	int idx;
136
137	XDNA_DBG(xdna, "PID %d flushing...", client->pid);
138	if (!drm_dev_enter(&xdna->ddev, &idx))
139		return 0;
140
141	mutex_lock(&xdna->dev_lock);
142	list_del_init(&client->node);
143	amdxdna_hwctx_remove_all(client);
144	mutex_unlock(&xdna->dev_lock);
145
146	drm_dev_exit(idx);
147	return 0;
148}
149
150static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
151{
152	struct amdxdna_client *client = filp->driver_priv;
153	struct amdxdna_dev *xdna = to_xdna_dev(dev);
154	struct amdxdna_drm_get_info *args = data;
155	int ret;
156
157	if (!xdna->dev_info->ops->get_aie_info)
158		return -EOPNOTSUPP;
159
160	XDNA_DBG(xdna, "Request parameter %u", args->param);
161	mutex_lock(&xdna->dev_lock);
162	ret = xdna->dev_info->ops->get_aie_info(client, args);
163	mutex_unlock(&xdna->dev_lock);
164	return ret;
165}
166
167static int amdxdna_drm_get_array_ioctl(struct drm_device *dev, void *data,
168				       struct drm_file *filp)
169{
170	struct amdxdna_client *client = filp->driver_priv;
171	struct amdxdna_dev *xdna = to_xdna_dev(dev);
172	struct amdxdna_drm_get_array *args = data;
173
174	if (!xdna->dev_info->ops->get_array)
175		return -EOPNOTSUPP;
176
177	if (args->pad || !args->num_element || !args->element_size)
178		return -EINVAL;
179
180	guard(mutex)(&xdna->dev_lock);
181	return xdna->dev_info->ops->get_array(client, args);
182}
183
184static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
185{
186	struct amdxdna_client *client = filp->driver_priv;
187	struct amdxdna_dev *xdna = to_xdna_dev(dev);
188	struct amdxdna_drm_set_state *args = data;
189	int ret;
190
191	if (!xdna->dev_info->ops->set_aie_state)
192		return -EOPNOTSUPP;
193
194	XDNA_DBG(xdna, "Request parameter %u", args->param);
195	mutex_lock(&xdna->dev_lock);
196	ret = xdna->dev_info->ops->set_aie_state(client, args);
197	mutex_unlock(&xdna->dev_lock);
198
199	return ret;
200}
201
202static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = {
203	/* Context */
204	DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0),
205	DRM_IOCTL_DEF_DRV(AMDXDNA_DESTROY_HWCTX, amdxdna_drm_destroy_hwctx_ioctl, 0),
206	DRM_IOCTL_DEF_DRV(AMDXDNA_CONFIG_HWCTX, amdxdna_drm_config_hwctx_ioctl, 0),
207	/* BO */
208	DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_BO, amdxdna_drm_create_bo_ioctl, 0),
209	DRM_IOCTL_DEF_DRV(AMDXDNA_GET_BO_INFO, amdxdna_drm_get_bo_info_ioctl, 0),
210	DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0),
211	/* Execution */
212	DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0),
213	/* AIE hardware */
214	DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0),
215	DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0),
216	DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY),
217};
218
219static const struct file_operations amdxdna_fops = {
220	.owner		= THIS_MODULE,
221	.open		= accel_open,
222	.release	= drm_release,
223	.flush		= amdxdna_flush,
224	.unlocked_ioctl	= drm_ioctl,
225	.compat_ioctl	= drm_compat_ioctl,
226	.poll		= drm_poll,
227	.read		= drm_read,
228	.llseek		= noop_llseek,
229	.mmap		= drm_gem_mmap,
230	.fop_flags	= FOP_UNSIGNED_OFFSET,
231};
232
233const struct drm_driver amdxdna_drm_drv = {
234	.driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL |
235		DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE,
236	.fops = &amdxdna_fops,
237	.name = "amdxdna_accel_driver",
238	.desc = "AMD XDNA DRM implementation",
239	.major = AMDXDNA_DRIVER_MAJOR,
240	.minor = AMDXDNA_DRIVER_MINOR,
241	.open = amdxdna_drm_open,
242	.postclose = amdxdna_drm_close,
243	.ioctls = amdxdna_drm_ioctls,
244	.num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls),
245
246	.gem_create_object = amdxdna_gem_create_object_cb,
247	.gem_prime_import = amdxdna_gem_prime_import,
248};
249
250static const struct amdxdna_dev_info *
251amdxdna_get_dev_info(struct pci_dev *pdev)
252{
253	int i;
254
255	for (i = 0; i < ARRAY_SIZE(amdxdna_ids); i++) {
256		if (pdev->device == amdxdna_ids[i].device &&
257		    pdev->revision == amdxdna_ids[i].revision)
258			return amdxdna_ids[i].dev_info;
259	}
260	return NULL;
261}
262
263static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
264{
265	struct device *dev = &pdev->dev;
266	struct amdxdna_dev *xdna;
267	int ret;
268
269	xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev);
270	if (IS_ERR(xdna))
271		return PTR_ERR(xdna);
272
273	xdna->dev_info = amdxdna_get_dev_info(pdev);
274	if (!xdna->dev_info)
275		return -ENODEV;
276
277	drmm_mutex_init(&xdna->ddev, &xdna->dev_lock);
278	init_rwsem(&xdna->notifier_lock);
279	INIT_LIST_HEAD(&xdna->client_list);
280	pci_set_drvdata(pdev, xdna);
281
282	if (IS_ENABLED(CONFIG_LOCKDEP)) {
283		fs_reclaim_acquire(GFP_KERNEL);
284		might_lock(&xdna->notifier_lock);
285		fs_reclaim_release(GFP_KERNEL);
286	}
287
288	xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM);
289	if (!xdna->notifier_wq)
290		return -ENOMEM;
291
292	mutex_lock(&xdna->dev_lock);
293	ret = xdna->dev_info->ops->init(xdna);
294	mutex_unlock(&xdna->dev_lock);
295	if (ret) {
296		XDNA_ERR(xdna, "Hardware init failed, ret %d", ret);
297		goto destroy_notifier_wq;
298	}
299
300	ret = amdxdna_sysfs_init(xdna);
301	if (ret) {
302		XDNA_ERR(xdna, "Create amdxdna attrs failed: %d", ret);
303		goto failed_dev_fini;
304	}
305
306	ret = drm_dev_register(&xdna->ddev, 0);
307	if (ret) {
308		XDNA_ERR(xdna, "DRM register failed, ret %d", ret);
309		goto failed_sysfs_fini;
310	}
311
312	return 0;
313
314failed_sysfs_fini:
315	amdxdna_sysfs_fini(xdna);
316failed_dev_fini:
317	mutex_lock(&xdna->dev_lock);
318	xdna->dev_info->ops->fini(xdna);
319	mutex_unlock(&xdna->dev_lock);
320destroy_notifier_wq:
321	destroy_workqueue(xdna->notifier_wq);
322	return ret;
323}
324
325static void amdxdna_remove(struct pci_dev *pdev)
326{
327	struct amdxdna_dev *xdna = pci_get_drvdata(pdev);
328	struct amdxdna_client *client;
329
330	destroy_workqueue(xdna->notifier_wq);
331
332	drm_dev_unplug(&xdna->ddev);
333	amdxdna_sysfs_fini(xdna);
334
335	mutex_lock(&xdna->dev_lock);
336	client = list_first_entry_or_null(&xdna->client_list,
337					  struct amdxdna_client, node);
338	while (client) {
339		list_del_init(&client->node);
340		amdxdna_hwctx_remove_all(client);
341
342		client = list_first_entry_or_null(&xdna->client_list,
343						  struct amdxdna_client, node);
344	}
345
346	xdna->dev_info->ops->fini(xdna);
347	mutex_unlock(&xdna->dev_lock);
348}
349
350static const struct dev_pm_ops amdxdna_pm_ops = {
351	SYSTEM_SLEEP_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume)
352	RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL)
353};
354
355static struct pci_driver amdxdna_pci_driver = {
356	.name = KBUILD_MODNAME,
357	.id_table = pci_ids,
358	.probe = amdxdna_probe,
359	.remove = amdxdna_remove,
360	.driver.pm = &amdxdna_pm_ops,
361};
362
363module_pci_driver(amdxdna_pci_driver);
364
365MODULE_LICENSE("GPL");
366MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>");
367MODULE_DESCRIPTION("amdxdna driver");