ethosu_drv.c

  1// SPDX-License-Identifier: GPL-2.0-only or MIT
  2// Copyright (C) 2025 Arm, Ltd.
  3
  4#include <linux/bitfield.h>
  5#include <linux/clk.h>
  6#include <linux/genalloc.h>
  7#include <linux/io.h>
  8#include <linux/iopoll.h>
  9#include <linux/module.h>
 10#include <linux/mod_devicetable.h>
 11#include <linux/platform_device.h>
 12#include <linux/pm_runtime.h>
 13
 14#include <drm/drm_drv.h>
 15#include <drm/drm_ioctl.h>
 16#include <drm/drm_utils.h>
 17#include <drm/drm_gem.h>
 18#include <drm/drm_accel.h>
 19#include <drm/ethosu_accel.h>
 20
 21#include "ethosu_drv.h"
 22#include "ethosu_device.h"
 23#include "ethosu_gem.h"
 24#include "ethosu_job.h"
 25
 26static int ethosu_ioctl_dev_query(struct drm_device *ddev, void *data,
 27				  struct drm_file *file)
 28{
 29	struct ethosu_device *ethosudev = to_ethosu_device(ddev);
 30	struct drm_ethosu_dev_query *args = data;
 31
 32	if (!args->pointer) {
 33		switch (args->type) {
 34		case DRM_ETHOSU_DEV_QUERY_NPU_INFO:
 35			args->size = sizeof(ethosudev->npu_info);
 36			return 0;
 37		default:
 38			return -EINVAL;
 39		}
 40	}
 41
 42	switch (args->type) {
 43	case DRM_ETHOSU_DEV_QUERY_NPU_INFO:
 44		if (args->size < offsetofend(struct drm_ethosu_npu_info, sram_size))
 45			return -EINVAL;
 46		return copy_struct_to_user(u64_to_user_ptr(args->pointer),
 47					   args->size,
 48					   &ethosudev->npu_info,
 49					   sizeof(ethosudev->npu_info), NULL);
 50	default:
 51		return -EINVAL;
 52	}
 53}
 54
 55#define ETHOSU_BO_FLAGS		DRM_ETHOSU_BO_NO_MMAP
 56
 57static int ethosu_ioctl_bo_create(struct drm_device *ddev, void *data,
 58				  struct drm_file *file)
 59{
 60	struct drm_ethosu_bo_create *args = data;
 61	int cookie, ret;
 62
 63	if (!drm_dev_enter(ddev, &cookie))
 64		return -ENODEV;
 65
 66	if (!args->size || (args->flags & ~ETHOSU_BO_FLAGS)) {
 67		ret = -EINVAL;
 68		goto out_dev_exit;
 69	}
 70
 71	ret = ethosu_gem_create_with_handle(file, ddev, &args->size,
 72					    args->flags, &args->handle);
 73
 74out_dev_exit:
 75	drm_dev_exit(cookie);
 76	return ret;
 77}
 78
 79static int ethosu_ioctl_bo_wait(struct drm_device *ddev, void *data,
 80				struct drm_file *file)
 81{
 82	struct drm_ethosu_bo_wait *args = data;
 83	int cookie, ret;
 84	unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
 85
 86	if (args->pad)
 87		return -EINVAL;
 88
 89	if (!drm_dev_enter(ddev, &cookie))
 90		return -ENODEV;
 91
 92	ret = drm_gem_dma_resv_wait(file, args->handle, true, timeout);
 93
 94	drm_dev_exit(cookie);
 95	return ret;
 96}
 97
 98static int ethosu_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data,
 99				       struct drm_file *file)
100{
101	struct drm_ethosu_bo_mmap_offset *args = data;
102	struct drm_gem_object *obj;
103
104	if (args->pad)
105		return -EINVAL;
106
107	obj = drm_gem_object_lookup(file, args->handle);
108	if (!obj)
109		return -ENOENT;
110
111	args->offset = drm_vma_node_offset_addr(&obj->vma_node);
112	drm_gem_object_put(obj);
113	return 0;
114}
115
116static int ethosu_ioctl_cmdstream_bo_create(struct drm_device *ddev, void *data,
117					    struct drm_file *file)
118{
119	struct drm_ethosu_cmdstream_bo_create *args = data;
120	int cookie, ret;
121
122	if (!drm_dev_enter(ddev, &cookie))
123		return -ENODEV;
124
125	if (!args->size || !args->data || args->pad || args->flags) {
126		ret = -EINVAL;
127		goto out_dev_exit;
128	}
129
130	args->flags |= DRM_ETHOSU_BO_NO_MMAP;
131
132	ret = ethosu_gem_cmdstream_create(file, ddev, args->size, args->data,
133					  args->flags, &args->handle);
134
135out_dev_exit:
136	drm_dev_exit(cookie);
137	return ret;
138}
139
140static int ethosu_open(struct drm_device *ddev, struct drm_file *file)
141{
142	int ret = 0;
143
144	if (!try_module_get(THIS_MODULE))
145		return -EINVAL;
146
147	struct ethosu_file_priv __free(kfree) *priv = kzalloc(sizeof(*priv), GFP_KERNEL);
148	if (!priv) {
149		ret = -ENOMEM;
150		goto err_put_mod;
151	}
152	priv->edev = to_ethosu_device(ddev);
153
154	ret = ethosu_job_open(priv);
155	if (ret)
156		goto err_put_mod;
157
158	file->driver_priv = no_free_ptr(priv);
159	return 0;
160
161err_put_mod:
162	module_put(THIS_MODULE);
163	return ret;
164}
165
166static void ethosu_postclose(struct drm_device *ddev, struct drm_file *file)
167{
168	ethosu_job_close(file->driver_priv);
169	kfree(file->driver_priv);
170	module_put(THIS_MODULE);
171}
172
173static const struct drm_ioctl_desc ethosu_drm_driver_ioctls[] = {
174#define ETHOSU_IOCTL(n, func, flags) \
175	DRM_IOCTL_DEF_DRV(ETHOSU_##n, ethosu_ioctl_##func, flags)
176
177	ETHOSU_IOCTL(DEV_QUERY, dev_query, 0),
178	ETHOSU_IOCTL(BO_CREATE, bo_create, 0),
179	ETHOSU_IOCTL(BO_WAIT, bo_wait, 0),
180	ETHOSU_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, 0),
181	ETHOSU_IOCTL(CMDSTREAM_BO_CREATE, cmdstream_bo_create, 0),
182	ETHOSU_IOCTL(SUBMIT, submit, 0),
183};
184
185DEFINE_DRM_ACCEL_FOPS(ethosu_drm_driver_fops);
186
187/*
188 * Ethosu driver version:
189 * - 1.0 - initial interface
190 */
191static const struct drm_driver ethosu_drm_driver = {
192	.driver_features = DRIVER_COMPUTE_ACCEL | DRIVER_GEM,
193	.open = ethosu_open,
194	.postclose = ethosu_postclose,
195	.ioctls = ethosu_drm_driver_ioctls,
196	.num_ioctls = ARRAY_SIZE(ethosu_drm_driver_ioctls),
197	.fops = &ethosu_drm_driver_fops,
198	.name = "ethosu",
199	.desc = "Arm Ethos-U Accel driver",
200	.major = 1,
201	.minor = 0,
202
203	.gem_create_object = ethosu_gem_create_object,
204};
205
206#define U65_DRAM_AXI_LIMIT_CFG	0x1f3f0002
207#define U65_SRAM_AXI_LIMIT_CFG	0x1f3f00b0
208#define U85_AXI_EXT_CFG		0x00021f3f
209#define U85_AXI_SRAM_CFG	0x00021f3f
210#define U85_MEM_ATTR0_CFG	0x00000000
211#define U85_MEM_ATTR2_CFG	0x000000b7
212
213static int ethosu_reset(struct ethosu_device *ethosudev)
214{
215	int ret;
216	u32 reg;
217
218	writel_relaxed(RESET_PENDING_CSL, ethosudev->regs + NPU_REG_RESET);
219	ret = readl_poll_timeout(ethosudev->regs + NPU_REG_STATUS, reg,
220				 !FIELD_GET(STATUS_RESET_STATUS, reg),
221				 USEC_PER_MSEC, USEC_PER_SEC);
222	if (ret)
223		return ret;
224
225	if (!FIELD_GET(PROT_ACTIVE_CSL, readl_relaxed(ethosudev->regs + NPU_REG_PROT))) {
226		dev_warn(ethosudev->base.dev, "Could not reset to non-secure mode (PROT = %x)\n",
227			 readl_relaxed(ethosudev->regs + NPU_REG_PROT));
228	}
229
230	/*
231	 * Assign region 2 (SRAM) to AXI M0 (AXILIMIT0),
232	 * everything else to AXI M1 (AXILIMIT2)
233	 */
234	writel_relaxed(0x0000aa8a, ethosudev->regs + NPU_REG_REGIONCFG);
235	if (ethosu_is_u65(ethosudev)) {
236		writel_relaxed(U65_SRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT0);
237		writel_relaxed(U65_DRAM_AXI_LIMIT_CFG, ethosudev->regs + NPU_REG_AXILIMIT2);
238	} else {
239		writel_relaxed(U85_AXI_SRAM_CFG, ethosudev->regs + NPU_REG_AXI_SRAM);
240		writel_relaxed(U85_AXI_EXT_CFG, ethosudev->regs + NPU_REG_AXI_EXT);
241		writel_relaxed(U85_MEM_ATTR0_CFG, ethosudev->regs + NPU_REG_MEM_ATTR0);	// SRAM
242		writel_relaxed(U85_MEM_ATTR2_CFG, ethosudev->regs + NPU_REG_MEM_ATTR2);	// DRAM
243	}
244
245	if (ethosudev->sram)
246		memset_io(ethosudev->sram, 0, ethosudev->npu_info.sram_size);
247
248	return 0;
249}
250
251static int ethosu_device_resume(struct device *dev)
252{
253	struct ethosu_device *ethosudev = dev_get_drvdata(dev);
254	int ret;
255
256	ret = clk_bulk_prepare_enable(ethosudev->num_clks, ethosudev->clks);
257	if (ret)
258		return ret;
259
260	ret = ethosu_reset(ethosudev);
261	if (!ret)
262		return 0;
263
264	clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks);
265	return ret;
266}
267
268static int ethosu_device_suspend(struct device *dev)
269{
270	struct ethosu_device *ethosudev = dev_get_drvdata(dev);
271
272	clk_bulk_disable_unprepare(ethosudev->num_clks, ethosudev->clks);
273	return 0;
274}
275
276static int ethosu_sram_init(struct ethosu_device *ethosudev)
277{
278	ethosudev->npu_info.sram_size = 0;
279
280	ethosudev->srampool = of_gen_pool_get(ethosudev->base.dev->of_node, "sram", 0);
281	if (!ethosudev->srampool)
282		return 0;
283
284	ethosudev->npu_info.sram_size = gen_pool_size(ethosudev->srampool);
285
286	ethosudev->sram = (void __iomem *)gen_pool_dma_alloc(ethosudev->srampool,
287							     ethosudev->npu_info.sram_size,
288							     &ethosudev->sramphys);
289	if (!ethosudev->sram) {
290		dev_err(ethosudev->base.dev, "failed to allocate from SRAM pool\n");
291		return -ENOMEM;
292	}
293
294	return 0;
295}
296
297static int ethosu_init(struct ethosu_device *ethosudev)
298{
299	int ret;
300	u32 id, config;
301
302	ret = ethosu_device_resume(ethosudev->base.dev);
303	if (ret)
304		return ret;
305
306	pm_runtime_set_autosuspend_delay(ethosudev->base.dev, 50);
307	pm_runtime_use_autosuspend(ethosudev->base.dev);
308	ret = devm_pm_runtime_set_active_enabled(ethosudev->base.dev);
309	if (ret)
310		return ret;
311	pm_runtime_get_noresume(ethosudev->base.dev);
312
313	ethosudev->npu_info.id = id = readl_relaxed(ethosudev->regs + NPU_REG_ID);
314	ethosudev->npu_info.config = config = readl_relaxed(ethosudev->regs + NPU_REG_CONFIG);
315
316	ethosu_sram_init(ethosudev);
317
318	dev_info(ethosudev->base.dev,
319		 "Ethos-U NPU, arch v%ld.%ld.%ld, rev r%ldp%ld, cmd stream ver%ld, %d MACs, %dKB SRAM\n",
320		 FIELD_GET(ID_ARCH_MAJOR_MASK, id),
321		 FIELD_GET(ID_ARCH_MINOR_MASK, id),
322		 FIELD_GET(ID_ARCH_PATCH_MASK, id),
323		 FIELD_GET(ID_VER_MAJOR_MASK, id),
324		 FIELD_GET(ID_VER_MINOR_MASK, id),
325		 FIELD_GET(CONFIG_CMD_STREAM_VER_MASK, config),
326		 1 << FIELD_GET(CONFIG_MACS_PER_CC_MASK, config),
327		 ethosudev->npu_info.sram_size / 1024);
328
329	return 0;
330}
331
332static int ethosu_probe(struct platform_device *pdev)
333{
334	int ret;
335	struct ethosu_device *ethosudev;
336
337	ethosudev = devm_drm_dev_alloc(&pdev->dev, &ethosu_drm_driver,
338				       struct ethosu_device, base);
339	if (IS_ERR(ethosudev))
340		return -ENOMEM;
341	platform_set_drvdata(pdev, ethosudev);
342
343	dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
344
345	ethosudev->regs = devm_platform_ioremap_resource(pdev, 0);
346
347	ethosudev->num_clks = devm_clk_bulk_get_all(&pdev->dev, &ethosudev->clks);
348	if (ethosudev->num_clks < 0)
349		return ethosudev->num_clks;
350
351	ret = ethosu_job_init(ethosudev);
352	if (ret)
353		return ret;
354
355	ret = ethosu_init(ethosudev);
356	if (ret)
357		return ret;
358
359	ret = drm_dev_register(&ethosudev->base, 0);
360	if (ret)
361		pm_runtime_dont_use_autosuspend(ethosudev->base.dev);
362
363	pm_runtime_put_autosuspend(ethosudev->base.dev);
364	return ret;
365}
366
367static void ethosu_remove(struct platform_device *pdev)
368{
369	struct ethosu_device *ethosudev = dev_get_drvdata(&pdev->dev);
370
371	drm_dev_unregister(&ethosudev->base);
372	ethosu_job_fini(ethosudev);
373	if (ethosudev->sram)
374		gen_pool_free(ethosudev->srampool, (unsigned long)ethosudev->sram,
375			      ethosudev->npu_info.sram_size);
376}
377
378static const struct of_device_id dt_match[] = {
379	{ .compatible = "arm,ethos-u65" },
380	{ .compatible = "arm,ethos-u85" },
381	{}
382};
383MODULE_DEVICE_TABLE(of, dt_match);
384
385static DEFINE_RUNTIME_DEV_PM_OPS(ethosu_pm_ops,
386				 ethosu_device_suspend,
387				 ethosu_device_resume,
388				 NULL);
389
390static struct platform_driver ethosu_driver = {
391	.probe = ethosu_probe,
392	.remove = ethosu_remove,
393	.driver = {
394		.name = "ethosu",
395		.pm = pm_ptr(&ethosu_pm_ops),
396		.of_match_table = dt_match,
397	},
398};
399module_platform_driver(ethosu_driver);
400
401MODULE_AUTHOR("Rob Herring <robh@kernel.org>");
402MODULE_DESCRIPTION("Arm Ethos-U Accel Driver");
403MODULE_LICENSE("Dual MIT/GPL");