msm_gem_submit.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Copyright (C) 2013 Red Hat
  4 * Author: Rob Clark <robdclark@gmail.com>
  5 */
  6
  7#include <linux/dma-fence-unwrap.h>
  8#include <linux/file.h>
  9#include <linux/sync_file.h>
 10#include <linux/uaccess.h>
 11
 12#include <drm/drm_drv.h>
 13#include <drm/drm_file.h>
 14#include <drm/drm_syncobj.h>
 15
 16#include "msm_drv.h"
 17#include "msm_gpu.h"
 18#include "msm_gem.h"
 19#include "msm_gpu_trace.h"
 20#include "msm_syncobj.h"
 21
 22/* For userspace errors, use DRM_UT_DRIVER.. so that userspace can enable
 23 * error msgs for debugging, but we don't spam dmesg by default
 24 */
 25#define SUBMIT_ERROR(err, submit, fmt, ...) \
 26	UERR(err, (submit)->dev, fmt, ##__VA_ARGS__)
 27
 28/*
 29 * Cmdstream submission:
 30 */
 31
 32static struct msm_gem_submit *submit_create(struct drm_device *dev,
 33		struct msm_gpu *gpu,
 34		struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
 35		uint32_t nr_cmds, u64 drm_client_id)
 36{
 37	static atomic_t ident = ATOMIC_INIT(0);
 38	struct msm_gem_submit *submit;
 39	uint64_t sz;
 40	int ret;
 41
 42	sz = struct_size(submit, bos, nr_bos) +
 43			((u64)nr_cmds * sizeof(submit->cmd[0]));
 44
 45	if (sz > SIZE_MAX)
 46		return ERR_PTR(-ENOMEM);
 47
 48	submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN);
 49	if (!submit)
 50		return ERR_PTR(-ENOMEM);
 51
 52	submit->hw_fence = msm_fence_alloc();
 53	if (IS_ERR(submit->hw_fence)) {
 54		ret = PTR_ERR(submit->hw_fence);
 55		kfree(submit);
 56		return ERR_PTR(ret);
 57	}
 58
 59	ret = drm_sched_job_init(&submit->base, queue->entity, 1, queue,
 60				 drm_client_id);
 61	if (ret) {
 62		kfree(submit->hw_fence);
 63		kfree(submit);
 64		return ERR_PTR(ret);
 65	}
 66
 67	kref_init(&submit->ref);
 68	submit->dev = dev;
 69	submit->vm = msm_context_vm(dev, queue->ctx);
 70	submit->gpu = gpu;
 71	submit->cmd = (void *)&submit->bos[nr_bos];
 72	submit->queue = queue;
 73	submit->pid = get_pid(task_pid(current));
 74	submit->ring = gpu->rb[queue->ring_nr];
 75	submit->fault_dumped = false;
 76
 77	/* Get a unique identifier for the submission for logging purposes */
 78	submit->ident = atomic_inc_return(&ident) - 1;
 79
 80	INIT_LIST_HEAD(&submit->node);
 81
 82	return submit;
 83}
 84
 85void __msm_gem_submit_destroy(struct kref *kref)
 86{
 87	struct msm_gem_submit *submit =
 88			container_of(kref, struct msm_gem_submit, ref);
 89	unsigned i;
 90
 91	/*
 92	 * In error paths, we could unref the submit without calling
 93	 * drm_sched_entity_push_job(), so msm_job_free() will never
 94	 * get called.  Since drm_sched_job_cleanup() will NULL out
 95	 * s_fence, we can use that to detect this case.
 96	 */
 97	if (submit->base.s_fence)
 98		drm_sched_job_cleanup(&submit->base);
 99
100	if (submit->fence_id) {
101		spin_lock(&submit->queue->idr_lock);
102		idr_remove(&submit->queue->fence_idr, submit->fence_id);
103		spin_unlock(&submit->queue->idr_lock);
104	}
105
106	dma_fence_put(submit->user_fence);
107
108	/*
109	 * If the submit is freed before msm_job_run(), then hw_fence is
110	 * just some pre-allocated memory, not a reference counted fence.
111	 * Once the job runs and the hw_fence is initialized, it will
112	 * have a refcount of at least one, since the submit holds a ref
113	 * to the hw_fence.
114	 */
115	if (kref_read(&submit->hw_fence->refcount) == 0) {
116		kfree(submit->hw_fence);
117	} else {
118		dma_fence_put(submit->hw_fence);
119	}
120
121	put_pid(submit->pid);
122	msm_submitqueue_put(submit->queue);
123
124	for (i = 0; i < submit->nr_cmds; i++)
125		kfree(submit->cmd[i].relocs);
126
127	kfree(submit);
128}
129
130static int submit_lookup_objects(struct msm_gem_submit *submit,
131		struct drm_msm_gem_submit *args, struct drm_file *file)
132{
133	unsigned i;
134	int ret = 0;
135
136	for (i = 0; i < args->nr_bos; i++) {
137		struct drm_msm_gem_submit_bo submit_bo;
138		void __user *userptr =
139			u64_to_user_ptr(args->bos + (i * sizeof(submit_bo)));
140
141		/* make sure we don't have garbage flags, in case we hit
142		 * error path before flags is initialized:
143		 */
144		submit->bos[i].flags = 0;
145
146		if (copy_from_user(&submit_bo, userptr, sizeof(submit_bo))) {
147			ret = -EFAULT;
148			i = 0;
149			goto out;
150		}
151
152/* at least one of READ and/or WRITE flags should be set: */
153#define MANDATORY_FLAGS (MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
154
155		if ((submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) ||
156			!(submit_bo.flags & MANDATORY_FLAGS)) {
157			ret = SUBMIT_ERROR(EINVAL, submit, "invalid flags: %x\n", submit_bo.flags);
158			i = 0;
159			goto out;
160		}
161
162		submit->bos[i].handle = submit_bo.handle;
163		submit->bos[i].flags = submit_bo.flags;
164	}
165
166	spin_lock(&file->table_lock);
167
168	for (i = 0; i < args->nr_bos; i++) {
169		struct drm_gem_object *obj;
170
171		/* normally use drm_gem_object_lookup(), but for bulk lookup
172		 * all under single table_lock just hit object_idr directly:
173		 */
174		obj = idr_find(&file->object_idr, submit->bos[i].handle);
175		if (!obj) {
176			ret = SUBMIT_ERROR(EINVAL, submit, "invalid handle %u at index %u\n", submit->bos[i].handle, i);
177			goto out_unlock;
178		}
179
180		drm_gem_object_get(obj);
181
182		submit->bos[i].obj = obj;
183	}
184
185out_unlock:
186	spin_unlock(&file->table_lock);
187
188out:
189	submit->nr_bos = i;
190
191	return ret;
192}
193
194static int submit_lookup_cmds(struct msm_gem_submit *submit,
195		struct drm_msm_gem_submit *args, struct drm_file *file)
196{
197	struct msm_context *ctx = file->driver_priv;
198	unsigned i;
199	size_t sz;
200	int ret = 0;
201
202	for (i = 0; i < args->nr_cmds; i++) {
203		struct drm_msm_gem_submit_cmd submit_cmd;
204		void __user *userptr =
205			u64_to_user_ptr(args->cmds + (i * sizeof(submit_cmd)));
206
207		ret = copy_from_user(&submit_cmd, userptr, sizeof(submit_cmd));
208		if (ret) {
209			ret = -EFAULT;
210			goto out;
211		}
212
213		/* validate input from userspace: */
214		switch (submit_cmd.type) {
215		case MSM_SUBMIT_CMD_BUF:
216		case MSM_SUBMIT_CMD_IB_TARGET_BUF:
217		case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
218			break;
219		default:
220			return SUBMIT_ERROR(EINVAL, submit, "invalid type: %08x\n", submit_cmd.type);
221		}
222
223		if (submit_cmd.size % 4) {
224			ret = SUBMIT_ERROR(EINVAL, submit, "non-aligned cmdstream buffer size: %u\n",
225					   submit_cmd.size);
226			goto out;
227		}
228
229		if (msm_context_is_vmbind(ctx)) {
230			if (submit_cmd.nr_relocs) {
231				ret = SUBMIT_ERROR(EINVAL, submit, "nr_relocs must be zero");
232				goto out;
233			}
234
235			if (submit_cmd.submit_idx || submit_cmd.submit_offset) {
236				ret = SUBMIT_ERROR(EINVAL, submit, "submit_idx/offset must be zero");
237				goto out;
238			}
239
240			submit->cmd[i].iova = submit_cmd.iova;
241		}
242
243		submit->cmd[i].type = submit_cmd.type;
244		submit->cmd[i].size = submit_cmd.size / 4;
245		submit->cmd[i].offset = submit_cmd.submit_offset / 4;
246		submit->cmd[i].idx  = submit_cmd.submit_idx;
247		submit->cmd[i].nr_relocs = submit_cmd.nr_relocs;
248
249		userptr = u64_to_user_ptr(submit_cmd.relocs);
250
251		sz = array_size(submit_cmd.nr_relocs,
252				sizeof(struct drm_msm_gem_submit_reloc));
253		/* check for overflow: */
254		if (sz == SIZE_MAX) {
255			ret = -ENOMEM;
256			goto out;
257		}
258		submit->cmd[i].relocs = kmalloc(sz, GFP_KERNEL | __GFP_NOWARN);
259		if (!submit->cmd[i].relocs) {
260			ret = -ENOMEM;
261			goto out;
262		}
263		ret = copy_from_user(submit->cmd[i].relocs, userptr, sz);
264		if (ret) {
265			ret = -EFAULT;
266			goto out;
267		}
268	}
269
270out:
271	return ret;
272}
273
274static int submit_lock_objects_vmbind(struct msm_gem_submit *submit)
275{
276	unsigned flags = DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES;
277	struct drm_exec *exec = &submit->exec;
278	int ret = 0;
279
280	drm_exec_init(&submit->exec, flags, submit->nr_bos);
281
282	drm_exec_until_all_locked (&submit->exec) {
283		ret = drm_gpuvm_prepare_vm(submit->vm, exec, 1);
284		drm_exec_retry_on_contention(exec);
285		if (ret)
286			break;
287
288		ret = drm_gpuvm_prepare_objects(submit->vm, exec, 1);
289		drm_exec_retry_on_contention(exec);
290		if (ret)
291			break;
292	}
293
294	return ret;
295}
296
297/* This is where we make sure all the bo's are reserved and pin'd: */
298static int submit_lock_objects(struct msm_gem_submit *submit)
299{
300	unsigned flags = DRM_EXEC_INTERRUPTIBLE_WAIT;
301	int ret = 0;
302
303	if (msm_context_is_vmbind(submit->queue->ctx))
304		return submit_lock_objects_vmbind(submit);
305
306	drm_exec_init(&submit->exec, flags, submit->nr_bos);
307
308	drm_exec_until_all_locked (&submit->exec) {
309		ret = drm_exec_lock_obj(&submit->exec,
310					drm_gpuvm_resv_obj(submit->vm));
311		drm_exec_retry_on_contention(&submit->exec);
312		if (ret)
313			break;
314		for (unsigned i = 0; i < submit->nr_bos; i++) {
315			struct drm_gem_object *obj = submit->bos[i].obj;
316			ret = drm_exec_prepare_obj(&submit->exec, obj, 1);
317			drm_exec_retry_on_contention(&submit->exec);
318			if (ret)
319				break;
320		}
321	}
322
323	return ret;
324}
325
326static int submit_fence_sync(struct msm_gem_submit *submit)
327{
328	int i, ret = 0;
329
330	for (i = 0; i < submit->nr_bos; i++) {
331		struct drm_gem_object *obj = submit->bos[i].obj;
332		bool write = submit->bos[i].flags & MSM_SUBMIT_BO_WRITE;
333
334		/* Otherwise userspace can ask for implicit sync to be
335		 * disabled on specific buffers.  This is useful for internal
336		 * usermode driver managed buffers, suballocation, etc.
337		 */
338		if (submit->bos[i].flags & MSM_SUBMIT_BO_NO_IMPLICIT)
339			continue;
340
341		ret = drm_sched_job_add_implicit_dependencies(&submit->base,
342							      obj,
343							      write);
344		if (ret)
345			break;
346	}
347
348	return ret;
349}
350
351static int submit_pin_objects(struct msm_gem_submit *submit)
352{
353	struct msm_drm_private *priv = submit->dev->dev_private;
354	int i, ret = 0;
355
356	for (i = 0; i < submit->nr_bos; i++) {
357		struct drm_gem_object *obj = submit->bos[i].obj;
358		struct drm_gpuva *vma;
359
360		/* if locking succeeded, pin bo: */
361		vma = msm_gem_get_vma_locked(obj, submit->vm);
362		if (IS_ERR(vma)) {
363			ret = PTR_ERR(vma);
364			break;
365		}
366
367		ret = msm_gem_pin_vma_locked(obj, vma);
368		if (ret)
369			break;
370
371		submit->bos[i].vm_bo = drm_gpuvm_bo_get(vma->vm_bo);
372		submit->bos[i].iova = vma->va.addr;
373	}
374
375	/*
376	 * A second loop while holding the LRU lock (a) avoids acquiring/dropping
377	 * the LRU lock for each individual bo, while (b) avoiding holding the
378	 * LRU lock while calling msm_gem_pin_vma_locked() (which could trigger
379	 * get_pages() which could trigger reclaim.. and if we held the LRU lock
380	 * could trigger deadlock with the shrinker).
381	 */
382	mutex_lock(&priv->lru.lock);
383	for (i = 0; i < submit->nr_bos; i++) {
384		msm_gem_pin_obj_locked(submit->bos[i].obj);
385	}
386	mutex_unlock(&priv->lru.lock);
387
388	submit->bos_pinned = true;
389
390	return ret;
391}
392
393static void submit_unpin_objects(struct msm_gem_submit *submit)
394{
395	if (!submit->bos_pinned)
396		return;
397
398	for (int i = 0; i < submit->nr_bos; i++) {
399		struct drm_gem_object *obj = submit->bos[i].obj;
400
401		msm_gem_unpin_locked(obj);
402	}
403
404	submit->bos_pinned = false;
405}
406
407static void submit_attach_object_fences(struct msm_gem_submit *submit)
408{
409	struct msm_gem_vm *vm = to_msm_vm(submit->vm);
410	struct dma_fence *last_fence;
411
412	if (msm_context_is_vmbind(submit->queue->ctx)) {
413		drm_gpuvm_resv_add_fence(submit->vm, &submit->exec,
414					 submit->user_fence,
415					 DMA_RESV_USAGE_BOOKKEEP,
416					 DMA_RESV_USAGE_BOOKKEEP);
417
418		last_fence = vm->last_fence;
419		vm->last_fence = dma_fence_unwrap_merge(submit->user_fence, last_fence);
420		dma_fence_put(last_fence);
421
422		return;
423	}
424
425	for (unsigned i = 0; i < submit->nr_bos; i++) {
426		struct drm_gem_object *obj = submit->bos[i].obj;
427
428		if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
429			dma_resv_add_fence(obj->resv, submit->user_fence,
430					   DMA_RESV_USAGE_WRITE);
431		else if (submit->bos[i].flags & MSM_SUBMIT_BO_READ)
432			dma_resv_add_fence(obj->resv, submit->user_fence,
433					   DMA_RESV_USAGE_READ);
434	}
435}
436
437static int submit_bo(struct msm_gem_submit *submit, uint32_t idx,
438		struct drm_gem_object **obj, uint64_t *iova)
439{
440	if (idx >= submit->nr_bos) {
441		return SUBMIT_ERROR(EINVAL, submit, "invalid buffer index: %u (out of %u)\n",
442				    idx, submit->nr_bos);
443	}
444
445	if (obj)
446		*obj = submit->bos[idx].obj;
447	if (iova)
448		*iova = submit->bos[idx].iova;
449
450	return 0;
451}
452
453/* process the reloc's and patch up the cmdstream as needed: */
454static int submit_reloc(struct msm_gem_submit *submit, struct drm_gem_object *obj,
455		uint32_t offset, uint32_t nr_relocs, struct drm_msm_gem_submit_reloc *relocs)
456{
457	uint32_t i, last_offset = 0;
458	uint32_t *ptr;
459	int ret = 0;
460
461	if (offset % 4)
462		return SUBMIT_ERROR(EINVAL, submit, "non-aligned cmdstream buffer: %u\n", offset);
463
464	/* For now, just map the entire thing.  Eventually we probably
465	 * to do it page-by-page, w/ kmap() if not vmap()d..
466	 */
467	ptr = msm_gem_get_vaddr_locked(obj);
468
469	if (IS_ERR(ptr)) {
470		ret = PTR_ERR(ptr);
471		DBG("failed to map: %d", ret);
472		return ret;
473	}
474
475	for (i = 0; i < nr_relocs; i++) {
476		struct drm_msm_gem_submit_reloc submit_reloc = relocs[i];
477		uint32_t off;
478		uint64_t iova;
479
480		if (submit_reloc.submit_offset % 4) {
481			ret = SUBMIT_ERROR(EINVAL, submit, "non-aligned reloc offset: %u\n",
482					   submit_reloc.submit_offset);
483			goto out;
484		}
485
486		/* offset in dwords: */
487		off = submit_reloc.submit_offset / 4;
488
489		if ((off >= (obj->size / 4)) ||
490				(off < last_offset)) {
491			ret = SUBMIT_ERROR(EINVAL, submit, "invalid offset %u at reloc %u\n", off, i);
492			goto out;
493		}
494
495		ret = submit_bo(submit, submit_reloc.reloc_idx, NULL, &iova);
496		if (ret)
497			goto out;
498
499		iova += submit_reloc.reloc_offset;
500
501		if (submit_reloc.shift < 0)
502			iova >>= -submit_reloc.shift;
503		else
504			iova <<= submit_reloc.shift;
505
506		ptr[off] = iova | submit_reloc.or;
507
508		last_offset = off;
509	}
510
511out:
512	msm_gem_put_vaddr_locked(obj);
513
514	return ret;
515}
516
517/* Cleanup submit at end of ioctl.  In the error case, this also drops
518 * references, unpins, and drops active refcnt.  In the non-error case,
519 * this is done when the submit is retired.
520 */
521static void submit_cleanup(struct msm_gem_submit *submit, bool error)
522{
523	if (error)
524		submit_unpin_objects(submit);
525
526	if (submit->exec.objects)
527		drm_exec_fini(&submit->exec);
528
529	/* if job wasn't enqueued to scheduler, early retirement: */
530	if (error)
531		msm_submit_retire(submit);
532}
533
534void msm_submit_retire(struct msm_gem_submit *submit)
535{
536	int i;
537
538	for (i = 0; i < submit->nr_bos; i++) {
539		struct drm_gem_object *obj = submit->bos[i].obj;
540		struct drm_gpuvm_bo *vm_bo = submit->bos[i].vm_bo;
541
542		msm_gem_lock(obj);
543		drm_gpuvm_bo_put(vm_bo);
544		msm_gem_unlock(obj);
545		drm_gem_object_put(obj);
546	}
547}
548
549int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
550		struct drm_file *file)
551{
552	struct msm_drm_private *priv = dev->dev_private;
553	struct drm_msm_gem_submit *args = data;
554	struct msm_context *ctx = file->driver_priv;
555	struct msm_gem_submit *submit = NULL;
556	struct msm_gpu *gpu = priv->gpu;
557	struct msm_gpu_submitqueue *queue;
558	struct msm_ringbuffer *ring;
559	struct msm_syncobj_post_dep *post_deps = NULL;
560	struct drm_syncobj **syncobjs_to_reset = NULL;
561	struct sync_file *sync_file = NULL;
562	unsigned cmds_to_parse;
563	int out_fence_fd = -1;
564	unsigned i;
565	int ret;
566
567	if (!gpu)
568		return -ENXIO;
569
570	if (args->pad)
571		return -EINVAL;
572
573	if (to_msm_vm(ctx->vm)->unusable)
574		return UERR(EPIPE, dev, "context is unusable");
575
576	/* for now, we just have 3d pipe.. eventually this would need to
577	 * be more clever to dispatch to appropriate gpu module:
578	 */
579	if (MSM_PIPE_ID(args->flags) != MSM_PIPE_3D0)
580		return UERR(EINVAL, dev, "invalid pipe");
581
582	if (MSM_PIPE_FLAGS(args->flags) & ~MSM_SUBMIT_FLAGS)
583		return UERR(EINVAL, dev, "invalid flags");
584
585	if (args->flags & MSM_SUBMIT_SUDO) {
586		if (!IS_ENABLED(CONFIG_DRM_MSM_GPU_SUDO) ||
587		    !capable(CAP_SYS_RAWIO))
588			return -EINVAL;
589	}
590
591	queue = msm_submitqueue_get(ctx, args->queueid);
592	if (!queue)
593		return -ENOENT;
594
595	if (queue->flags & MSM_SUBMITQUEUE_VM_BIND) {
596		ret = UERR(EINVAL, dev, "Invalid queue type");
597		goto out_post_unlock;
598	}
599
600	ring = gpu->rb[queue->ring_nr];
601
602	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
603		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
604		if (out_fence_fd < 0) {
605			ret = out_fence_fd;
606			goto out_post_unlock;
607		}
608	}
609
610	submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds,
611			       file->client_id);
612	if (IS_ERR(submit)) {
613		ret = PTR_ERR(submit);
614		goto out_post_unlock;
615	}
616
617	trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident,
618		args->nr_bos, args->nr_cmds);
619
620	ret = mutex_lock_interruptible(&queue->lock);
621	if (ret)
622		goto out_post_unlock;
623
624	if (args->flags & MSM_SUBMIT_SUDO)
625		submit->in_rb = true;
626
627	if (args->flags & MSM_SUBMIT_FENCE_FD_IN) {
628		struct dma_fence *in_fence;
629
630		in_fence = sync_file_get_fence(args->fence_fd);
631
632		if (!in_fence) {
633			ret = UERR(EINVAL, dev, "invalid in-fence");
634			goto out_unlock;
635		}
636
637		ret = drm_sched_job_add_dependency(&submit->base, in_fence);
638		if (ret)
639			goto out_unlock;
640	}
641
642	if (args->flags & MSM_SUBMIT_SYNCOBJ_IN) {
643		syncobjs_to_reset = msm_syncobj_parse_deps(dev, &submit->base,
644							   file, args->in_syncobjs,
645							   args->nr_in_syncobjs,
646							   args->syncobj_stride);
647		if (IS_ERR(syncobjs_to_reset)) {
648			ret = PTR_ERR(syncobjs_to_reset);
649			goto out_unlock;
650		}
651	}
652
653	if (args->flags & MSM_SUBMIT_SYNCOBJ_OUT) {
654		post_deps = msm_syncobj_parse_post_deps(dev, file,
655							args->out_syncobjs,
656							args->nr_out_syncobjs,
657							args->syncobj_stride);
658		if (IS_ERR(post_deps)) {
659			ret = PTR_ERR(post_deps);
660			goto out_unlock;
661		}
662	}
663
664	ret = submit_lookup_objects(submit, args, file);
665	if (ret)
666		goto out;
667
668	ret = submit_lookup_cmds(submit, args, file);
669	if (ret)
670		goto out;
671
672	/* copy_*_user while holding a ww ticket upsets lockdep */
673	ret = submit_lock_objects(submit);
674	if (ret)
675		goto out;
676
677	if (!(args->flags & MSM_SUBMIT_NO_IMPLICIT)) {
678		ret = submit_fence_sync(submit);
679		if (ret)
680			goto out;
681	}
682
683	ret = submit_pin_objects(submit);
684	if (ret)
685		goto out;
686
687	cmds_to_parse = msm_context_is_vmbind(ctx) ? 0 : args->nr_cmds;
688
689	for (i = 0; i < cmds_to_parse; i++) {
690		struct drm_gem_object *obj;
691		uint64_t iova;
692
693		ret = submit_bo(submit, submit->cmd[i].idx, &obj, &iova);
694		if (ret)
695			goto out;
696
697		if (!submit->cmd[i].size ||
698		    (size_add(submit->cmd[i].size, submit->cmd[i].offset) > obj->size / 4)) {
699			ret = UERR(EINVAL, dev, "invalid cmdstream size: %u\n",
700				   submit->cmd[i].size * 4);
701			goto out;
702		}
703
704		submit->cmd[i].iova = iova + (submit->cmd[i].offset * 4);
705
706		if (likely(!submit->cmd[i].nr_relocs))
707			continue;
708
709		if (!gpu->allow_relocs) {
710			ret = UERR(EINVAL, dev, "relocs not allowed\n");
711			goto out;
712		}
713
714		ret = submit_reloc(submit, obj, submit->cmd[i].offset * 4,
715				submit->cmd[i].nr_relocs, submit->cmd[i].relocs);
716		if (ret)
717			goto out;
718	}
719
720	submit->nr_cmds = args->nr_cmds;
721
722	idr_preload(GFP_KERNEL);
723
724	spin_lock(&queue->idr_lock);
725
726	/*
727	 * If using userspace provided seqno fence, validate that the id
728	 * is available before arming sched job.  Since access to fence_idr
729	 * is serialized on the queue lock, the slot should be still avail
730	 * after the job is armed
731	 */
732	if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) &&
733			(!args->fence || idr_find(&queue->fence_idr, args->fence))) {
734		spin_unlock(&queue->idr_lock);
735		idr_preload_end();
736		ret = UERR(EINVAL, dev, "invalid in-fence-sn");
737		goto out;
738	}
739
740	drm_sched_job_arm(&submit->base);
741
742	submit->user_fence = dma_fence_get(&submit->base.s_fence->finished);
743
744	if (args->flags & MSM_SUBMIT_FENCE_SN_IN) {
745		/*
746		 * Userspace has assigned the seqno fence that it wants
747		 * us to use.  It is an error to pick a fence sequence
748		 * number that is not available.
749		 */
750		submit->fence_id = args->fence;
751		ret = idr_alloc_u32(&queue->fence_idr, submit->user_fence,
752				    &submit->fence_id, submit->fence_id,
753				    GFP_NOWAIT);
754		/*
755		 * We've already validated that the fence_id slot is valid,
756		 * so if idr_alloc_u32 failed, it is a kernel bug
757		 */
758		WARN_ON(ret);
759	} else {
760		/*
761		 * Allocate an id which can be used by WAIT_FENCE ioctl to map
762		 * back to the underlying fence.
763		 */
764		submit->fence_id = idr_alloc_cyclic(&queue->fence_idr,
765						    submit->user_fence, 1,
766						    INT_MAX, GFP_NOWAIT);
767	}
768
769	spin_unlock(&queue->idr_lock);
770	idr_preload_end();
771
772	if (submit->fence_id < 0) {
773		ret = submit->fence_id;
774		submit->fence_id = 0;
775	}
776
777	if (ret == 0 && args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
778		sync_file = sync_file_create(submit->user_fence);
779		if (!sync_file)
780			ret = -ENOMEM;
781	}
782
783	if (ret)
784		goto out;
785
786	submit_attach_object_fences(submit);
787
788	if (msm_context_is_vmbind(ctx)) {
789		/*
790		 * If we are not using VM_BIND, submit_pin_vmas() will validate
791		 * just the BOs attached to the submit.  In that case we don't
792		 * need to validate the _entire_ vm, because userspace tracked
793		 * what BOs are associated with the submit.
794		 */
795		ret = drm_gpuvm_validate(submit->vm, &submit->exec);
796		if (ret)
797			goto out;
798	}
799
800	/* The scheduler owns a ref now: */
801	msm_gem_submit_get(submit);
802
803	msm_rd_dump_submit(priv->rd, submit, NULL);
804
805	drm_sched_entity_push_job(&submit->base);
806
807	args->fence = submit->fence_id;
808	queue->last_fence = submit->fence_id;
809
810	msm_syncobj_reset(syncobjs_to_reset, args->nr_in_syncobjs);
811	msm_syncobj_process_post_deps(post_deps, args->nr_out_syncobjs, submit->user_fence);
812
813out:
814	submit_cleanup(submit, !!ret);
815out_unlock:
816	mutex_unlock(&queue->lock);
817out_post_unlock:
818	if (ret) {
819		if (out_fence_fd >= 0)
820			put_unused_fd(out_fence_fd);
821		if (sync_file)
822			fput(sync_file->file);
823	} else if (sync_file) {
824		fd_install(out_fence_fd, sync_file->file);
825		args->fence_fd = out_fence_fd;
826	}
827
828	if (!IS_ERR_OR_NULL(submit)) {
829		msm_gem_submit_put(submit);
830	} else {
831		/*
832		 * If the submit hasn't yet taken ownership of the queue
833		 * then we need to drop the reference ourself:
834		 */
835		msm_submitqueue_put(queue);
836	}
837	if (!IS_ERR_OR_NULL(post_deps)) {
838		for (i = 0; i < args->nr_out_syncobjs; ++i) {
839			kfree(post_deps[i].chain);
840			drm_syncobj_put(post_deps[i].syncobj);
841		}
842		kfree(post_deps);
843	}
844
845	if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
846		for (i = 0; i < args->nr_in_syncobjs; ++i) {
847			if (syncobjs_to_reset[i])
848				drm_syncobj_put(syncobjs_to_reset[i]);
849		}
850		kfree(syncobjs_to_reset);
851	}
852
853	return ret;
854}