rocket_drv.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright 2024-2025 Tomeu Vizoso <tomeu@tomeuvizoso.net> */
  3
  4#include <drm/drm_accel.h>
  5#include <drm/drm_drv.h>
  6#include <drm/drm_gem.h>
  7#include <drm/drm_ioctl.h>
  8#include <drm/rocket_accel.h>
  9#include <linux/clk.h>
 10#include <linux/err.h>
 11#include <linux/iommu.h>
 12#include <linux/of.h>
 13#include <linux/platform_device.h>
 14#include <linux/pm_runtime.h>
 15
 16#include "rocket_device.h"
 17#include "rocket_drv.h"
 18#include "rocket_gem.h"
 19#include "rocket_job.h"
 20
 21/*
 22 * Facade device, used to expose a single DRM device to userspace, that
 23 * schedules jobs to any RKNN cores in the system.
 24 */
 25static struct platform_device *drm_dev;
 26static struct rocket_device *rdev;
 27
 28static void
 29rocket_iommu_domain_destroy(struct kref *kref)
 30{
 31	struct rocket_iommu_domain *domain = container_of(kref, struct rocket_iommu_domain, kref);
 32
 33	iommu_domain_free(domain->domain);
 34	domain->domain = NULL;
 35	kfree(domain);
 36}
 37
 38static struct rocket_iommu_domain*
 39rocket_iommu_domain_create(struct device *dev)
 40{
 41	struct rocket_iommu_domain *domain = kmalloc(sizeof(*domain), GFP_KERNEL);
 42	void *err;
 43
 44	if (!domain)
 45		return ERR_PTR(-ENOMEM);
 46
 47	domain->domain = iommu_paging_domain_alloc(dev);
 48	if (IS_ERR(domain->domain)) {
 49		err = ERR_CAST(domain->domain);
 50		kfree(domain);
 51		return err;
 52	}
 53	kref_init(&domain->kref);
 54
 55	return domain;
 56}
 57
 58struct rocket_iommu_domain *
 59rocket_iommu_domain_get(struct rocket_file_priv *rocket_priv)
 60{
 61	kref_get(&rocket_priv->domain->kref);
 62	return rocket_priv->domain;
 63}
 64
 65void
 66rocket_iommu_domain_put(struct rocket_iommu_domain *domain)
 67{
 68	kref_put(&domain->kref, rocket_iommu_domain_destroy);
 69}
 70
 71static int
 72rocket_open(struct drm_device *dev, struct drm_file *file)
 73{
 74	struct rocket_device *rdev = to_rocket_device(dev);
 75	struct rocket_file_priv *rocket_priv;
 76	u64 start, end;
 77	int ret;
 78
 79	if (!try_module_get(THIS_MODULE))
 80		return -EINVAL;
 81
 82	rocket_priv = kzalloc(sizeof(*rocket_priv), GFP_KERNEL);
 83	if (!rocket_priv) {
 84		ret = -ENOMEM;
 85		goto err_put_mod;
 86	}
 87
 88	rocket_priv->rdev = rdev;
 89	rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev);
 90	if (IS_ERR(rocket_priv->domain)) {
 91		ret = PTR_ERR(rocket_priv->domain);
 92		goto err_free;
 93	}
 94
 95	file->driver_priv = rocket_priv;
 96
 97	start = rocket_priv->domain->domain->geometry.aperture_start;
 98	end = rocket_priv->domain->domain->geometry.aperture_end;
 99	drm_mm_init(&rocket_priv->mm, start, end - start + 1);
100	mutex_init(&rocket_priv->mm_lock);
101
102	ret = rocket_job_open(rocket_priv);
103	if (ret)
104		goto err_mm_takedown;
105
106	return 0;
107
108err_mm_takedown:
109	mutex_destroy(&rocket_priv->mm_lock);
110	drm_mm_takedown(&rocket_priv->mm);
111	rocket_iommu_domain_put(rocket_priv->domain);
112err_free:
113	kfree(rocket_priv);
114err_put_mod:
115	module_put(THIS_MODULE);
116	return ret;
117}
118
119static void
120rocket_postclose(struct drm_device *dev, struct drm_file *file)
121{
122	struct rocket_file_priv *rocket_priv = file->driver_priv;
123
124	rocket_job_close(rocket_priv);
125	mutex_destroy(&rocket_priv->mm_lock);
126	drm_mm_takedown(&rocket_priv->mm);
127	rocket_iommu_domain_put(rocket_priv->domain);
128	kfree(rocket_priv);
129	module_put(THIS_MODULE);
130}
131
132static const struct drm_ioctl_desc rocket_drm_driver_ioctls[] = {
133#define ROCKET_IOCTL(n, func) \
134	DRM_IOCTL_DEF_DRV(ROCKET_##n, rocket_ioctl_##func, 0)
135
136	ROCKET_IOCTL(CREATE_BO, create_bo),
137	ROCKET_IOCTL(SUBMIT, submit),
138	ROCKET_IOCTL(PREP_BO, prep_bo),
139	ROCKET_IOCTL(FINI_BO, fini_bo),
140};
141
142DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops);
143
144/*
145 * Rocket driver version:
146 * - 1.0 - initial interface
147 */
148static const struct drm_driver rocket_drm_driver = {
149	.driver_features	= DRIVER_COMPUTE_ACCEL | DRIVER_GEM,
150	.open			= rocket_open,
151	.postclose		= rocket_postclose,
152	.gem_create_object	= rocket_gem_create_object,
153	.ioctls			= rocket_drm_driver_ioctls,
154	.num_ioctls		= ARRAY_SIZE(rocket_drm_driver_ioctls),
155	.fops			= &rocket_accel_driver_fops,
156	.name			= "rocket",
157	.desc			= "rocket DRM",
158};
159
160static int rocket_probe(struct platform_device *pdev)
161{
162	int ret;
163
164	if (rdev == NULL) {
165		/* First core probing, initialize DRM device. */
166		rdev = rocket_device_init(drm_dev, &rocket_drm_driver);
167		if (IS_ERR(rdev)) {
168			dev_err(&pdev->dev, "failed to initialize rocket device\n");
169			return PTR_ERR(rdev);
170		}
171	}
172
173	unsigned int core = rdev->num_cores;
174
175	dev_set_drvdata(&pdev->dev, rdev);
176
177	rdev->cores[core].rdev = rdev;
178	rdev->cores[core].dev = &pdev->dev;
179	rdev->cores[core].index = core;
180
181	rdev->num_cores++;
182
183	ret = rocket_core_init(&rdev->cores[core]);
184	if (ret) {
185		rdev->num_cores--;
186
187		if (rdev->num_cores == 0) {
188			rocket_device_fini(rdev);
189			rdev = NULL;
190		}
191	}
192
193	return ret;
194}
195
196static void rocket_remove(struct platform_device *pdev)
197{
198	struct device *dev = &pdev->dev;
199
200	for (unsigned int core = 0; core < rdev->num_cores; core++) {
201		if (rdev->cores[core].dev == dev) {
202			rocket_core_fini(&rdev->cores[core]);
203			rdev->num_cores--;
204			break;
205		}
206	}
207
208	if (rdev->num_cores == 0) {
209		/* Last core removed, deinitialize DRM device. */
210		rocket_device_fini(rdev);
211		rdev = NULL;
212	}
213}
214
215static const struct of_device_id dt_match[] = {
216	{ .compatible = "rockchip,rk3588-rknn-core" },
217	{}
218};
219MODULE_DEVICE_TABLE(of, dt_match);
220
221static int find_core_for_dev(struct device *dev)
222{
223	struct rocket_device *rdev = dev_get_drvdata(dev);
224
225	for (unsigned int core = 0; core < rdev->num_cores; core++) {
226		if (dev == rdev->cores[core].dev)
227			return core;
228	}
229
230	return -1;
231}
232
233static int rocket_device_runtime_resume(struct device *dev)
234{
235	struct rocket_device *rdev = dev_get_drvdata(dev);
236	int core = find_core_for_dev(dev);
237	int err = 0;
238
239	if (core < 0)
240		return -ENODEV;
241
242	err = clk_bulk_prepare_enable(ARRAY_SIZE(rdev->cores[core].clks), rdev->cores[core].clks);
243	if (err) {
244		dev_err(dev, "failed to enable (%d) clocks for core %d\n", err, core);
245		return err;
246	}
247
248	return 0;
249}
250
251static int rocket_device_runtime_suspend(struct device *dev)
252{
253	struct rocket_device *rdev = dev_get_drvdata(dev);
254	int core = find_core_for_dev(dev);
255
256	if (core < 0)
257		return -ENODEV;
258
259	if (!rocket_job_is_idle(&rdev->cores[core]))
260		return -EBUSY;
261
262	clk_bulk_disable_unprepare(ARRAY_SIZE(rdev->cores[core].clks), rdev->cores[core].clks);
263
264	return 0;
265}
266
267EXPORT_GPL_DEV_PM_OPS(rocket_pm_ops) = {
268	RUNTIME_PM_OPS(rocket_device_runtime_suspend, rocket_device_runtime_resume, NULL)
269	SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
270};
271
272static struct platform_driver rocket_driver = {
273	.probe = rocket_probe,
274	.remove = rocket_remove,
275	.driver	 = {
276		.name = "rocket",
277		.pm = pm_ptr(&rocket_pm_ops),
278		.of_match_table = dt_match,
279	},
280};
281
282static int __init rocket_register(void)
283{
284	drm_dev = platform_device_register_simple("rknn", -1, NULL, 0);
285	if (IS_ERR(drm_dev))
286		return PTR_ERR(drm_dev);
287
288	return platform_driver_register(&rocket_driver);
289}
290
291static void __exit rocket_unregister(void)
292{
293	platform_driver_unregister(&rocket_driver);
294
295	platform_device_unregister(drm_dev);
296}
297
298module_init(rocket_register);
299module_exit(rocket_unregister);
300
301MODULE_LICENSE("GPL");
302MODULE_DESCRIPTION("DRM driver for the Rockchip NPU IP");
303MODULE_AUTHOR("Tomeu Vizoso");