ethosu_gem.c

  1// SPDX-License-Identifier: GPL-2.0-only or MIT
  2/* Copyright 2025 Arm, Ltd. */
  3
  4#include <linux/err.h>
  5#include <linux/slab.h>
  6
  7#include <drm/ethosu_accel.h>
  8
  9#include "ethosu_device.h"
 10#include "ethosu_gem.h"
 11
 12static void ethosu_gem_free_object(struct drm_gem_object *obj)
 13{
 14	struct ethosu_gem_object *bo = to_ethosu_bo(obj);
 15
 16	kfree(bo->info);
 17	drm_gem_free_mmap_offset(&bo->base.base);
 18	drm_gem_dma_free(&bo->base);
 19}
 20
 21static int ethosu_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 22{
 23	struct ethosu_gem_object *bo = to_ethosu_bo(obj);
 24
 25	/* Don't allow mmap on objects that have the NO_MMAP flag set. */
 26	if (bo->flags & DRM_ETHOSU_BO_NO_MMAP)
 27		return -EINVAL;
 28
 29	return drm_gem_dma_object_mmap(obj, vma);
 30}
 31
 32static const struct drm_gem_object_funcs ethosu_gem_funcs = {
 33	.free = ethosu_gem_free_object,
 34	.print_info = drm_gem_dma_object_print_info,
 35	.get_sg_table = drm_gem_dma_object_get_sg_table,
 36	.vmap = drm_gem_dma_object_vmap,
 37	.mmap = ethosu_gem_mmap,
 38	.vm_ops = &drm_gem_dma_vm_ops,
 39};
 40
 41/**
 42 * ethosu_gem_create_object - Implementation of driver->gem_create_object.
 43 * @ddev: DRM device
 44 * @size: Size in bytes of the memory the object will reference
 45 *
 46 * This lets the GEM helpers allocate object structs for us, and keep
 47 * our BO stats correct.
 48 */
 49struct drm_gem_object *ethosu_gem_create_object(struct drm_device *ddev, size_t size)
 50{
 51	struct ethosu_gem_object *obj;
 52
 53	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
 54	if (!obj)
 55		return ERR_PTR(-ENOMEM);
 56
 57	obj->base.base.funcs = &ethosu_gem_funcs;
 58	return &obj->base.base;
 59}
 60
 61/**
 62 * ethosu_gem_create_with_handle() - Create a GEM object and attach it to a handle.
 63 * @file: DRM file.
 64 * @ddev: DRM device.
 65 * @size: Size of the GEM object to allocate.
 66 * @flags: Combination of drm_ethosu_bo_flags flags.
 67 * @handle: Pointer holding the handle pointing to the new GEM object.
 68 *
 69 * Return: Zero on success
 70 */
 71int ethosu_gem_create_with_handle(struct drm_file *file,
 72				  struct drm_device *ddev,
 73				  u64 *size, u32 flags, u32 *handle)
 74{
 75	struct drm_gem_dma_object *mem;
 76	struct ethosu_gem_object *bo;
 77	int ret;
 78
 79	mem = drm_gem_dma_create(ddev, *size);
 80	if (IS_ERR(mem))
 81		return PTR_ERR(mem);
 82
 83	bo = to_ethosu_bo(&mem->base);
 84	bo->flags = flags;
 85
 86	/*
 87	 * Allocate an id of idr table where the obj is registered
 88	 * and handle has the id what user can see.
 89	 */
 90	ret = drm_gem_handle_create(file, &mem->base, handle);
 91	if (!ret)
 92		*size = bo->base.base.size;
 93
 94	/* drop reference from allocate - handle holds it now. */
 95	drm_gem_object_put(&mem->base);
 96
 97	return ret;
 98}
 99
100struct dma {
101	s8 region;
102	u64 len;
103	u64 offset;
104	s64 stride[2];
105};
106
107struct dma_state {
108	u16 size0;
109	u16 size1;
110	s8 mode;
111	struct dma src;
112	struct dma dst;
113};
114
115struct buffer {
116	u64 base;
117	u32 length;
118	s8 region;
119};
120
121struct feat_matrix {
122	u64 base[4];
123	s64 stride_x;
124	s64 stride_y;
125	s64 stride_c;
126	s8 region;
127	u8 broadcast;
128	u16 stride_kernel;
129	u16 precision;
130	u16 depth;
131	u16 width;
132	u16 width0;
133	u16 height[3];
134	u8 pad_top;
135	u8 pad_left;
136	u8 pad_bottom;
137	u8 pad_right;
138};
139
140struct cmd_state {
141	struct dma_state dma;
142	struct buffer scale[2];
143	struct buffer weight[4];
144	struct feat_matrix ofm;
145	struct feat_matrix ifm;
146	struct feat_matrix ifm2;
147};
148
149static void cmd_state_init(struct cmd_state *st)
150{
151	/* Initialize to all 1s to detect missing setup */
152	memset(st, 0xff, sizeof(*st));
153}
154
155static u64 cmd_to_addr(u32 *cmd)
156{
157	return (((u64)cmd[0] & 0xff0000) << 16) | cmd[1];
158}
159
160static u64 dma_length(struct ethosu_validated_cmdstream_info *info,
161		      struct dma_state *dma_st, struct dma *dma)
162{
163	s8 mode = dma_st->mode;
164	u64 len = dma->len;
165
166	if (mode >= 1) {
167		len += dma->stride[0];
168		len *= dma_st->size0;
169	}
170	if (mode == 2) {
171		len += dma->stride[1];
172		len *= dma_st->size1;
173	}
174	if (dma->region >= 0)
175		info->region_size[dma->region] = max(info->region_size[dma->region],
176						     len + dma->offset);
177
178	return len;
179}
180
181static u64 feat_matrix_length(struct ethosu_validated_cmdstream_info *info,
182			      struct feat_matrix *fm,
183			      u32 x, u32 y, u32 c)
184{
185	u32 element_size, storage = fm->precision >> 14;
186	int tile = 0;
187	u64 addr;
188
189	if (fm->region < 0)
190		return U64_MAX;
191
192	switch (storage) {
193	case 0:
194		if (x >= fm->width0 + 1) {
195			x -= fm->width0 + 1;
196			tile += 1;
197		}
198		if (y >= fm->height[tile] + 1) {
199			y -= fm->height[tile] + 1;
200			tile += 2;
201		}
202		break;
203	case 1:
204		if (y >= fm->height[1] + 1) {
205			y -= fm->height[1] + 1;
206			tile = 2;
207		} else if (y >= fm->height[0] + 1) {
208			y -= fm->height[0] + 1;
209			tile = 1;
210		}
211		break;
212	}
213	if (fm->base[tile] == U64_MAX)
214		return U64_MAX;
215
216	addr = fm->base[tile] + y * fm->stride_y;
217
218	switch ((fm->precision >> 6) & 0x3) { // format
219	case 0: //nhwc:
220		addr += x * fm->stride_x + c;
221		break;
222	case 1: //nhcwb16:
223		element_size = BIT((fm->precision >> 1) & 0x3);
224
225		addr += (c / 16) * fm->stride_c + (16 * x + (c & 0xf)) * element_size;
226		break;
227	}
228
229	info->region_size[fm->region] = max(info->region_size[fm->region], addr + 1);
230
231	return addr;
232}
233
234static int calc_sizes(struct drm_device *ddev,
235		      struct ethosu_validated_cmdstream_info *info,
236		      u16 op, struct cmd_state *st,
237		      bool ifm, bool ifm2, bool weight, bool scale)
238{
239	u64 len;
240
241	if (ifm) {
242		if (st->ifm.stride_kernel == U16_MAX)
243			return -EINVAL;
244		u32 stride_y = ((st->ifm.stride_kernel >> 8) & 0x2) +
245			((st->ifm.stride_kernel >> 1) & 0x1) + 1;
246		u32 stride_x = ((st->ifm.stride_kernel >> 5) & 0x2) +
247			(st->ifm.stride_kernel & 0x1) + 1;
248		u32 ifm_height = st->ofm.height[2] * stride_y +
249			st->ifm.height[2] - (st->ifm.pad_top + st->ifm.pad_bottom);
250		u32 ifm_width  = st->ofm.width * stride_x +
251			st->ifm.width - (st->ifm.pad_left + st->ifm.pad_right);
252
253		len = feat_matrix_length(info, &st->ifm, ifm_width,
254					 ifm_height, st->ifm.depth);
255		dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n",
256			op, st->ifm.region, st->ifm.base[0], len);
257		if (len == U64_MAX)
258			return -EINVAL;
259	}
260
261	if (ifm2) {
262		len = feat_matrix_length(info, &st->ifm2, st->ifm.depth,
263					 0, st->ofm.depth);
264		dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n",
265			op, st->ifm2.region, st->ifm2.base[0], len);
266		if (len == U64_MAX)
267			return -EINVAL;
268	}
269
270	if (weight) {
271		dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n",
272			op, st->weight[0].region, st->weight[0].base,
273			st->weight[0].base + st->weight[0].length - 1);
274		if (st->weight[0].region < 0 || st->weight[0].base == U64_MAX ||
275		    st->weight[0].length == U32_MAX)
276			return -EINVAL;
277		info->region_size[st->weight[0].region] =
278			max(info->region_size[st->weight[0].region],
279			    st->weight[0].base + st->weight[0].length);
280	}
281
282	if (scale) {
283		dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n",
284			op, st->scale[0].region, st->scale[0].base,
285			st->scale[0].base + st->scale[0].length - 1);
286		if (st->scale[0].region < 0 || st->scale[0].base == U64_MAX ||
287		    st->scale[0].length == U32_MAX)
288			return -EINVAL;
289		info->region_size[st->scale[0].region] =
290			max(info->region_size[st->scale[0].region],
291			    st->scale[0].base + st->scale[0].length);
292	}
293
294	len = feat_matrix_length(info, &st->ofm, st->ofm.width,
295				 st->ofm.height[2], st->ofm.depth);
296	dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n",
297		op, st->ofm.region, st->ofm.base[0], len);
298	if (len == U64_MAX)
299		return -EINVAL;
300	info->output_region[st->ofm.region] = true;
301
302	return 0;
303}
304
305static int calc_sizes_elemwise(struct drm_device *ddev,
306			       struct ethosu_validated_cmdstream_info *info,
307			       u16 op, struct cmd_state *st,
308			       bool ifm, bool ifm2)
309{
310	u32 height, width, depth;
311	u64 len;
312
313	if (ifm) {
314		height = st->ifm.broadcast & 0x1 ? 0 : st->ofm.height[2];
315		width = st->ifm.broadcast & 0x2 ? 0 : st->ofm.width;
316		depth = st->ifm.broadcast & 0x4 ? 0 : st->ofm.depth;
317
318		len = feat_matrix_length(info, &st->ifm, width,
319					 height, depth);
320		dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n",
321			op, st->ifm.region, st->ifm.base[0], len);
322		if (len == U64_MAX)
323			return -EINVAL;
324	}
325
326	if (ifm2) {
327		height = st->ifm2.broadcast & 0x1 ? 0 : st->ofm.height[2];
328		width = st->ifm2.broadcast & 0x2 ? 0 : st->ofm.width;
329		depth = st->ifm2.broadcast & 0x4 ? 0 : st->ofm.depth;
330
331		len = feat_matrix_length(info, &st->ifm2, width,
332					 height, depth);
333		dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n",
334			op, st->ifm2.region, st->ifm2.base[0], len);
335		if (len == U64_MAX)
336			return -EINVAL;
337	}
338
339	len = feat_matrix_length(info, &st->ofm, st->ofm.width,
340				 st->ofm.height[2], st->ofm.depth);
341	dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n",
342		op, st->ofm.region, st->ofm.base[0], len);
343	if (len == U64_MAX)
344		return -EINVAL;
345	info->output_region[st->ofm.region] = true;
346
347	return 0;
348}
349
350static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev,
351						  u32 __user *ucmds,
352						  struct ethosu_gem_object *bo,
353						  u32 size)
354{
355	struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc(sizeof(*info), GFP_KERNEL);
356	struct ethosu_device *edev = to_ethosu_device(ddev);
357	u32 *bocmds = bo->base.vaddr;
358	struct cmd_state st;
359	int i, ret;
360
361	if (!info)
362		return -ENOMEM;
363	info->cmd_size = size;
364
365	cmd_state_init(&st);
366
367	for (i = 0; i < size / 4; i++) {
368		bool use_ifm, use_ifm2, use_scale;
369		u64 dstlen, srclen;
370		u16 cmd, param;
371		u32 cmds[2];
372		u64 addr;
373
374		if (get_user(cmds[0], ucmds++))
375			return -EFAULT;
376
377		bocmds[i] = cmds[0];
378
379		cmd = cmds[0];
380		param = cmds[0] >> 16;
381
382		if (cmd & 0x4000) {
383			if (get_user(cmds[1], ucmds++))
384				return -EFAULT;
385
386			i++;
387			bocmds[i] = cmds[1];
388			addr = cmd_to_addr(cmds);
389		}
390
391		switch (cmd) {
392		case NPU_OP_DMA_START:
393			srclen = dma_length(info, &st.dma, &st.dma.src);
394			dstlen = dma_length(info, &st.dma, &st.dma.dst);
395
396			if (st.dma.dst.region >= 0)
397				info->output_region[st.dma.dst.region] = true;
398			dev_dbg(ddev->dev, "cmd: DMA SRC:%d:0x%llx+0x%llx DST:%d:0x%llx+0x%llx\n",
399				st.dma.src.region, st.dma.src.offset, srclen,
400				st.dma.dst.region, st.dma.dst.offset, dstlen);
401			break;
402		case NPU_OP_CONV:
403		case NPU_OP_DEPTHWISE:
404			use_ifm2 = param & 0x1;  // weights_ifm2
405			use_scale = !(st.ofm.precision & 0x100);
406			ret = calc_sizes(ddev, info, cmd, &st, true, use_ifm2,
407					 !use_ifm2, use_scale);
408			if (ret)
409				return ret;
410			break;
411		case NPU_OP_POOL:
412			use_ifm = param != 0x4;  // pooling mode
413			use_scale = !(st.ofm.precision & 0x100);
414			ret = calc_sizes(ddev, info, cmd, &st, use_ifm, false,
415					 false, use_scale);
416			if (ret)
417				return ret;
418			break;
419		case NPU_OP_ELEMENTWISE:
420			use_scale = ethosu_is_u65(edev) ?
421				    (st.ifm2.broadcast & 0x80) :
422				    (st.ifm2.broadcast == 8);
423			use_ifm2 = !(use_scale || (param == 5) ||
424				(param == 6) || (param == 7) || (param == 0x24));
425			use_ifm = st.ifm.broadcast != 8;
426			ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2);
427			if (ret)
428				return ret;
429			break;
430		case NPU_OP_RESIZE: // U85 only
431			WARN_ON(1); // TODO
432			break;
433		case NPU_SET_KERNEL_WIDTH_M1:
434			st.ifm.width = param;
435			break;
436		case NPU_SET_KERNEL_HEIGHT_M1:
437			st.ifm.height[2] = param;
438			break;
439		case NPU_SET_KERNEL_STRIDE:
440			st.ifm.stride_kernel = param;
441			break;
442		case NPU_SET_IFM_PAD_TOP:
443			st.ifm.pad_top = param & 0x7f;
444			break;
445		case NPU_SET_IFM_PAD_LEFT:
446			st.ifm.pad_left = param & 0x7f;
447			break;
448		case NPU_SET_IFM_PAD_RIGHT:
449			st.ifm.pad_right = param & 0xff;
450			break;
451		case NPU_SET_IFM_PAD_BOTTOM:
452			st.ifm.pad_bottom = param & 0xff;
453			break;
454		case NPU_SET_IFM_DEPTH_M1:
455			st.ifm.depth = param;
456			break;
457		case NPU_SET_IFM_PRECISION:
458			st.ifm.precision = param;
459			break;
460		case NPU_SET_IFM_BROADCAST:
461			st.ifm.broadcast = param;
462			break;
463		case NPU_SET_IFM_REGION:
464			st.ifm.region = param & 0x7f;
465			break;
466		case NPU_SET_IFM_WIDTH0_M1:
467			st.ifm.width0 = param;
468			break;
469		case NPU_SET_IFM_HEIGHT0_M1:
470			st.ifm.height[0] = param;
471			break;
472		case NPU_SET_IFM_HEIGHT1_M1:
473			st.ifm.height[1] = param;
474			break;
475		case NPU_SET_IFM_BASE0:
476		case NPU_SET_IFM_BASE1:
477		case NPU_SET_IFM_BASE2:
478		case NPU_SET_IFM_BASE3:
479			st.ifm.base[cmd & 0x3] = addr;
480			break;
481		case NPU_SET_IFM_STRIDE_X:
482			st.ifm.stride_x = addr;
483			break;
484		case NPU_SET_IFM_STRIDE_Y:
485			st.ifm.stride_y = addr;
486			break;
487		case NPU_SET_IFM_STRIDE_C:
488			st.ifm.stride_c = addr;
489			break;
490
491		case NPU_SET_OFM_WIDTH_M1:
492			st.ofm.width = param;
493			break;
494		case NPU_SET_OFM_HEIGHT_M1:
495			st.ofm.height[2] = param;
496			break;
497		case NPU_SET_OFM_DEPTH_M1:
498			st.ofm.depth = param;
499			break;
500		case NPU_SET_OFM_PRECISION:
501			st.ofm.precision = param;
502			break;
503		case NPU_SET_OFM_REGION:
504			st.ofm.region = param & 0x7;
505			break;
506		case NPU_SET_OFM_WIDTH0_M1:
507			st.ofm.width0 = param;
508			break;
509		case NPU_SET_OFM_HEIGHT0_M1:
510			st.ofm.height[0] = param;
511			break;
512		case NPU_SET_OFM_HEIGHT1_M1:
513			st.ofm.height[1] = param;
514			break;
515		case NPU_SET_OFM_BASE0:
516		case NPU_SET_OFM_BASE1:
517		case NPU_SET_OFM_BASE2:
518		case NPU_SET_OFM_BASE3:
519			st.ofm.base[cmd & 0x3] = addr;
520			break;
521		case NPU_SET_OFM_STRIDE_X:
522			st.ofm.stride_x = addr;
523			break;
524		case NPU_SET_OFM_STRIDE_Y:
525			st.ofm.stride_y = addr;
526			break;
527		case NPU_SET_OFM_STRIDE_C:
528			st.ofm.stride_c = addr;
529			break;
530
531		case NPU_SET_IFM2_BROADCAST:
532			st.ifm2.broadcast = param;
533			break;
534		case NPU_SET_IFM2_PRECISION:
535			st.ifm2.precision = param;
536			break;
537		case NPU_SET_IFM2_REGION:
538			st.ifm2.region = param & 0x7;
539			break;
540		case NPU_SET_IFM2_WIDTH0_M1:
541			st.ifm2.width0 = param;
542			break;
543		case NPU_SET_IFM2_HEIGHT0_M1:
544			st.ifm2.height[0] = param;
545			break;
546		case NPU_SET_IFM2_HEIGHT1_M1:
547			st.ifm2.height[1] = param;
548			break;
549		case NPU_SET_IFM2_BASE0:
550		case NPU_SET_IFM2_BASE1:
551		case NPU_SET_IFM2_BASE2:
552		case NPU_SET_IFM2_BASE3:
553			st.ifm2.base[cmd & 0x3] = addr;
554			break;
555		case NPU_SET_IFM2_STRIDE_X:
556			st.ifm2.stride_x = addr;
557			break;
558		case NPU_SET_IFM2_STRIDE_Y:
559			st.ifm2.stride_y = addr;
560			break;
561		case NPU_SET_IFM2_STRIDE_C:
562			st.ifm2.stride_c = addr;
563			break;
564
565		case NPU_SET_WEIGHT_REGION:
566			st.weight[0].region = param & 0x7;
567			break;
568		case NPU_SET_SCALE_REGION:
569			st.scale[0].region = param & 0x7;
570			break;
571		case NPU_SET_WEIGHT_BASE:
572			st.weight[0].base = addr;
573			break;
574		case NPU_SET_WEIGHT_LENGTH:
575			st.weight[0].length = cmds[1];
576			break;
577		case NPU_SET_SCALE_BASE:
578			st.scale[0].base = addr;
579			break;
580		case NPU_SET_SCALE_LENGTH:
581			st.scale[0].length = cmds[1];
582			break;
583		case NPU_SET_WEIGHT1_BASE:
584			st.weight[1].base = addr;
585			break;
586		case NPU_SET_WEIGHT1_LENGTH:
587			st.weight[1].length = cmds[1];
588			break;
589		case NPU_SET_SCALE1_BASE: // NPU_SET_WEIGHT2_BASE (U85)
590			if (ethosu_is_u65(edev))
591				st.scale[1].base = addr;
592			else
593				st.weight[2].base = addr;
594			break;
595		case NPU_SET_SCALE1_LENGTH: // NPU_SET_WEIGHT2_LENGTH (U85)
596			if (ethosu_is_u65(edev))
597				st.scale[1].length = cmds[1];
598			else
599				st.weight[1].length = cmds[1];
600			break;
601		case NPU_SET_WEIGHT3_BASE:
602			st.weight[3].base = addr;
603			break;
604		case NPU_SET_WEIGHT3_LENGTH:
605			st.weight[3].length = cmds[1];
606			break;
607
608		case NPU_SET_DMA0_SRC_REGION:
609			if (param & 0x100)
610				st.dma.src.region = -1;
611			else
612				st.dma.src.region = param & 0x7;
613			st.dma.mode = (param >> 9) & 0x3;
614			break;
615		case NPU_SET_DMA0_DST_REGION:
616			if (param & 0x100)
617				st.dma.dst.region = -1;
618			else
619				st.dma.dst.region = param & 0x7;
620			break;
621		case NPU_SET_DMA0_SIZE0:
622			st.dma.size0 = param;
623			break;
624		case NPU_SET_DMA0_SIZE1:
625			st.dma.size1 = param;
626			break;
627		case NPU_SET_DMA0_SRC_STRIDE0:
628			st.dma.src.stride[0] = ((s64)addr << 24) >> 24;
629			break;
630		case NPU_SET_DMA0_SRC_STRIDE1:
631			st.dma.src.stride[1] = ((s64)addr << 24) >> 24;
632			break;
633		case NPU_SET_DMA0_DST_STRIDE0:
634			st.dma.dst.stride[0] = ((s64)addr << 24) >> 24;
635			break;
636		case NPU_SET_DMA0_DST_STRIDE1:
637			st.dma.dst.stride[1] = ((s64)addr << 24) >> 24;
638			break;
639		case NPU_SET_DMA0_SRC:
640			st.dma.src.offset = addr;
641			break;
642		case NPU_SET_DMA0_DST:
643			st.dma.dst.offset = addr;
644			break;
645		case NPU_SET_DMA0_LEN:
646			st.dma.src.len = st.dma.dst.len = addr;
647			break;
648		default:
649			break;
650		}
651	}
652
653	for (i = 0; i < NPU_BASEP_REGION_MAX; i++) {
654		if (!info->region_size[i])
655			continue;
656		dev_dbg(ddev->dev, "region %d max size: 0x%llx\n",
657			i, info->region_size[i]);
658	}
659
660	bo->info = no_free_ptr(info);
661	return 0;
662}
663
664/**
665 * ethosu_gem_cmdstream_create() - Create a GEM object and attach it to a handle.
666 * @file: DRM file.
667 * @ddev: DRM device.
668 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared.
669 * @size: Size of the GEM object to allocate.
670 * @flags: Combination of drm_ethosu_bo_flags flags.
671 * @handle: Pointer holding the handle pointing to the new GEM object.
672 *
673 * Return: Zero on success
674 */
675int ethosu_gem_cmdstream_create(struct drm_file *file,
676				struct drm_device *ddev,
677				u32 size, u64 data, u32 flags, u32 *handle)
678{
679	int ret;
680	struct drm_gem_dma_object *mem;
681	struct ethosu_gem_object *bo;
682
683	mem = drm_gem_dma_create(ddev, size);
684	if (IS_ERR(mem))
685		return PTR_ERR(mem);
686
687	bo = to_ethosu_bo(&mem->base);
688	bo->flags = flags;
689
690	ret = ethosu_gem_cmdstream_copy_and_validate(ddev,
691						     (void __user *)(uintptr_t)data,
692						     bo, size);
693	if (ret)
694		goto fail;
695
696	/*
697	 * Allocate an id of idr table where the obj is registered
698	 * and handle has the id what user can see.
699	 */
700	ret = drm_gem_handle_create(file, &mem->base, handle);
701
702fail:
703	/* drop reference from allocate - handle holds it now. */
704	drm_gem_object_put(&mem->base);
705
706	return ret;
707}