nouveau_sched.c

  1// SPDX-License-Identifier: MIT
  2
  3#include <linux/slab.h>
  4#include <drm/gpu_scheduler.h>
  5#include <drm/drm_syncobj.h>
  6
  7#include "nouveau_drv.h"
  8#include "nouveau_gem.h"
  9#include "nouveau_mem.h"
 10#include "nouveau_dma.h"
 11#include "nouveau_exec.h"
 12#include "nouveau_abi16.h"
 13#include "nouveau_sched.h"
 14
 15#define NOUVEAU_SCHED_JOB_TIMEOUT_MS		10000
 16
 17/* Starts at 0, since the DRM scheduler interprets those parameters as (initial)
 18 * index to the run-queue array.
 19 */
 20enum nouveau_sched_priority {
 21	NOUVEAU_SCHED_PRIORITY_SINGLE = DRM_SCHED_PRIORITY_KERNEL,
 22	NOUVEAU_SCHED_PRIORITY_COUNT,
 23};
 24
 25int
 26nouveau_job_init(struct nouveau_job *job,
 27		 struct nouveau_job_args *args)
 28{
 29	struct nouveau_sched *sched = args->sched;
 30	int ret;
 31
 32	INIT_LIST_HEAD(&job->entry);
 33
 34	job->file_priv = args->file_priv;
 35	job->cli = nouveau_cli(args->file_priv);
 36	job->sched = sched;
 37
 38	job->sync = args->sync;
 39	job->resv_usage = args->resv_usage;
 40
 41	job->ops = args->ops;
 42
 43	job->in_sync.count = args->in_sync.count;
 44	if (job->in_sync.count) {
 45		if (job->sync)
 46			return -EINVAL;
 47
 48		job->in_sync.data = kmemdup(args->in_sync.s,
 49					 sizeof(*args->in_sync.s) *
 50					 args->in_sync.count,
 51					 GFP_KERNEL);
 52		if (!job->in_sync.data)
 53			return -ENOMEM;
 54	}
 55
 56	job->out_sync.count = args->out_sync.count;
 57	if (job->out_sync.count) {
 58		if (job->sync) {
 59			ret = -EINVAL;
 60			goto err_free_in_sync;
 61		}
 62
 63		job->out_sync.data = kmemdup(args->out_sync.s,
 64					  sizeof(*args->out_sync.s) *
 65					  args->out_sync.count,
 66					  GFP_KERNEL);
 67		if (!job->out_sync.data) {
 68			ret = -ENOMEM;
 69			goto err_free_in_sync;
 70		}
 71
 72		job->out_sync.objs = kcalloc(job->out_sync.count,
 73					     sizeof(*job->out_sync.objs),
 74					     GFP_KERNEL);
 75		if (!job->out_sync.objs) {
 76			ret = -ENOMEM;
 77			goto err_free_out_sync;
 78		}
 79
 80		job->out_sync.chains = kcalloc(job->out_sync.count,
 81					       sizeof(*job->out_sync.chains),
 82					       GFP_KERNEL);
 83		if (!job->out_sync.chains) {
 84			ret = -ENOMEM;
 85			goto err_free_objs;
 86		}
 87	}
 88
 89	ret = drm_sched_job_init(&job->base, &sched->entity,
 90				 args->credits, NULL,
 91				 job->file_priv->client_id);
 92	if (ret)
 93		goto err_free_chains;
 94
 95	job->state = NOUVEAU_JOB_INITIALIZED;
 96
 97	return 0;
 98
 99err_free_chains:
100	kfree(job->out_sync.chains);
101err_free_objs:
102	kfree(job->out_sync.objs);
103err_free_out_sync:
104	kfree(job->out_sync.data);
105err_free_in_sync:
106	kfree(job->in_sync.data);
107return ret;
108}
109
110void
111nouveau_job_fini(struct nouveau_job *job)
112{
113	dma_fence_put(job->done_fence);
114	drm_sched_job_cleanup(&job->base);
115
116	job->ops->free(job);
117}
118
119void
120nouveau_job_done(struct nouveau_job *job)
121{
122	struct nouveau_sched *sched = job->sched;
123
124	spin_lock(&sched->job.list.lock);
125	list_del(&job->entry);
126	spin_unlock(&sched->job.list.lock);
127
128	wake_up(&sched->job.wq);
129}
130
131void
132nouveau_job_free(struct nouveau_job *job)
133{
134	kfree(job->in_sync.data);
135	kfree(job->out_sync.data);
136	kfree(job->out_sync.objs);
137	kfree(job->out_sync.chains);
138}
139
140static int
141sync_find_fence(struct nouveau_job *job,
142		struct drm_nouveau_sync *sync,
143		struct dma_fence **fence)
144{
145	u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK;
146	u64 point = 0;
147	int ret;
148
149	if (stype != DRM_NOUVEAU_SYNC_SYNCOBJ &&
150	    stype != DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ)
151		return -EOPNOTSUPP;
152
153	if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ)
154		point = sync->timeline_value;
155
156	ret = drm_syncobj_find_fence(job->file_priv,
157				     sync->handle, point,
158				     0 /* flags */, fence);
159	if (ret)
160		return ret;
161
162	return 0;
163}
164
165static int
166nouveau_job_add_deps(struct nouveau_job *job)
167{
168	struct dma_fence *in_fence = NULL;
169	int ret, i;
170
171	for (i = 0; i < job->in_sync.count; i++) {
172		struct drm_nouveau_sync *sync = &job->in_sync.data[i];
173
174		ret = sync_find_fence(job, sync, &in_fence);
175		if (ret) {
176			NV_PRINTK(warn, job->cli,
177				  "Failed to find syncobj (-> in): handle=%d\n",
178				  sync->handle);
179			return ret;
180		}
181
182		ret = drm_sched_job_add_dependency(&job->base, in_fence);
183		if (ret)
184			return ret;
185	}
186
187	return 0;
188}
189
190static void
191nouveau_job_fence_attach_cleanup(struct nouveau_job *job)
192{
193	int i;
194
195	for (i = 0; i < job->out_sync.count; i++) {
196		struct drm_syncobj *obj = job->out_sync.objs[i];
197		struct dma_fence_chain *chain = job->out_sync.chains[i];
198
199		if (obj)
200			drm_syncobj_put(obj);
201
202		if (chain)
203			dma_fence_chain_free(chain);
204	}
205}
206
207static int
208nouveau_job_fence_attach_prepare(struct nouveau_job *job)
209{
210	int i, ret;
211
212	for (i = 0; i < job->out_sync.count; i++) {
213		struct drm_nouveau_sync *sync = &job->out_sync.data[i];
214		struct drm_syncobj **pobj = &job->out_sync.objs[i];
215		struct dma_fence_chain **pchain = &job->out_sync.chains[i];
216		u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK;
217
218		if (stype != DRM_NOUVEAU_SYNC_SYNCOBJ &&
219		    stype != DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) {
220			ret = -EINVAL;
221			goto err_sync_cleanup;
222		}
223
224		*pobj = drm_syncobj_find(job->file_priv, sync->handle);
225		if (!*pobj) {
226			NV_PRINTK(warn, job->cli,
227				  "Failed to find syncobj (-> out): handle=%d\n",
228				  sync->handle);
229			ret = -ENOENT;
230			goto err_sync_cleanup;
231		}
232
233		if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) {
234			*pchain = dma_fence_chain_alloc();
235			if (!*pchain) {
236				ret = -ENOMEM;
237				goto err_sync_cleanup;
238			}
239		}
240	}
241
242	return 0;
243
244err_sync_cleanup:
245	nouveau_job_fence_attach_cleanup(job);
246	return ret;
247}
248
249static void
250nouveau_job_fence_attach(struct nouveau_job *job)
251{
252	struct dma_fence *fence = job->done_fence;
253	int i;
254
255	for (i = 0; i < job->out_sync.count; i++) {
256		struct drm_nouveau_sync *sync = &job->out_sync.data[i];
257		struct drm_syncobj **pobj = &job->out_sync.objs[i];
258		struct dma_fence_chain **pchain = &job->out_sync.chains[i];
259		u32 stype = sync->flags & DRM_NOUVEAU_SYNC_TYPE_MASK;
260
261		if (stype == DRM_NOUVEAU_SYNC_TIMELINE_SYNCOBJ) {
262			drm_syncobj_add_point(*pobj, *pchain, fence,
263					      sync->timeline_value);
264		} else {
265			drm_syncobj_replace_fence(*pobj, fence);
266		}
267
268		drm_syncobj_put(*pobj);
269		*pobj = NULL;
270		*pchain = NULL;
271	}
272}
273
274int
275nouveau_job_submit(struct nouveau_job *job)
276{
277	struct nouveau_sched *sched = job->sched;
278	struct dma_fence *done_fence = NULL;
279	struct drm_gpuvm_exec vm_exec = {
280		.vm = &nouveau_cli_uvmm(job->cli)->base,
281		.flags = DRM_EXEC_IGNORE_DUPLICATES,
282		.num_fences = 1,
283	};
284	int ret;
285
286	ret = nouveau_job_add_deps(job);
287	if (ret)
288		goto err;
289
290	ret = nouveau_job_fence_attach_prepare(job);
291	if (ret)
292		goto err;
293
294	/* Make sure the job appears on the sched_entity's queue in the same
295	 * order as it was submitted.
296	 */
297	mutex_lock(&sched->mutex);
298
299	/* Guarantee we won't fail after the submit() callback returned
300	 * successfully.
301	 */
302	if (job->ops->submit) {
303		ret = job->ops->submit(job, &vm_exec);
304		if (ret)
305			goto err_cleanup;
306	}
307
308	/* Submit was successful; add the job to the schedulers job list. */
309	spin_lock(&sched->job.list.lock);
310	list_add(&job->entry, &sched->job.list.head);
311	spin_unlock(&sched->job.list.lock);
312
313	drm_sched_job_arm(&job->base);
314	job->done_fence = dma_fence_get(&job->base.s_fence->finished);
315	if (job->sync)
316		done_fence = dma_fence_get(job->done_fence);
317
318	if (job->ops->armed_submit)
319		job->ops->armed_submit(job, &vm_exec);
320
321	nouveau_job_fence_attach(job);
322
323	/* Set job state before pushing the job to the scheduler,
324	 * such that we do not overwrite the job state set in run().
325	 */
326	job->state = NOUVEAU_JOB_SUBMIT_SUCCESS;
327
328	drm_sched_entity_push_job(&job->base);
329
330	mutex_unlock(&sched->mutex);
331
332	if (done_fence) {
333		dma_fence_wait(done_fence, true);
334		dma_fence_put(done_fence);
335	}
336
337	return 0;
338
339err_cleanup:
340	mutex_unlock(&sched->mutex);
341	nouveau_job_fence_attach_cleanup(job);
342err:
343	job->state = NOUVEAU_JOB_SUBMIT_FAILED;
344	return ret;
345}
346
347static struct dma_fence *
348nouveau_job_run(struct nouveau_job *job)
349{
350	struct dma_fence *fence;
351
352	fence = job->ops->run(job);
353	if (IS_ERR(fence))
354		job->state = NOUVEAU_JOB_RUN_FAILED;
355	else
356		job->state = NOUVEAU_JOB_RUN_SUCCESS;
357
358	return fence;
359}
360
361static struct dma_fence *
362nouveau_sched_run_job(struct drm_sched_job *sched_job)
363{
364	struct nouveau_job *job = to_nouveau_job(sched_job);
365
366	return nouveau_job_run(job);
367}
368
369static enum drm_gpu_sched_stat
370nouveau_sched_timedout_job(struct drm_sched_job *sched_job)
371{
372	struct drm_gpu_scheduler *sched = sched_job->sched;
373	struct nouveau_job *job = to_nouveau_job(sched_job);
374	enum drm_gpu_sched_stat stat = DRM_GPU_SCHED_STAT_RESET;
375
376	drm_sched_stop(sched, sched_job);
377
378	if (job->ops->timeout)
379		stat = job->ops->timeout(job);
380	else
381		NV_PRINTK(warn, job->cli, "Generic job timeout.\n");
382
383	drm_sched_start(sched, 0);
384
385	return stat;
386}
387
388static void
389nouveau_sched_free_job(struct drm_sched_job *sched_job)
390{
391	struct nouveau_job *job = to_nouveau_job(sched_job);
392
393	nouveau_job_fini(job);
394}
395
396static const struct drm_sched_backend_ops nouveau_sched_ops = {
397	.run_job = nouveau_sched_run_job,
398	.timedout_job = nouveau_sched_timedout_job,
399	.free_job = nouveau_sched_free_job,
400};
401
402static int
403nouveau_sched_init(struct nouveau_sched *sched, struct nouveau_drm *drm,
404		   struct workqueue_struct *wq, u32 credit_limit)
405{
406	struct drm_gpu_scheduler *drm_sched = &sched->base;
407	struct drm_sched_entity *entity = &sched->entity;
408	struct drm_sched_init_args args = {
409		.ops = &nouveau_sched_ops,
410		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
411		.credit_limit = credit_limit,
412		.timeout = msecs_to_jiffies(NOUVEAU_SCHED_JOB_TIMEOUT_MS),
413		.name = "nouveau_sched",
414		.dev = drm->dev->dev
415	};
416	int ret;
417
418	if (!wq) {
419		wq = alloc_workqueue("nouveau_sched_wq_%d", 0, WQ_MAX_ACTIVE,
420				     current->pid);
421		if (!wq)
422			return -ENOMEM;
423
424		sched->wq = wq;
425	}
426
427	args.submit_wq = wq,
428
429	ret = drm_sched_init(drm_sched, &args);
430	if (ret)
431		goto fail_wq;
432
433	/* Using DRM_SCHED_PRIORITY_KERNEL, since that's what we're required to use
434	 * when we want to have a single run-queue only.
435	 *
436	 * It's not documented, but one will find out when trying to use any
437	 * other priority running into faults, because the scheduler uses the
438	 * priority as array index.
439	 *
440	 * Can't use NOUVEAU_SCHED_PRIORITY_SINGLE either, because it's not
441	 * matching the enum type used in drm_sched_entity_init().
442	 */
443	ret = drm_sched_entity_init(entity, DRM_SCHED_PRIORITY_KERNEL,
444				    &drm_sched, 1, NULL);
445	if (ret)
446		goto fail_sched;
447
448	mutex_init(&sched->mutex);
449	spin_lock_init(&sched->job.list.lock);
450	INIT_LIST_HEAD(&sched->job.list.head);
451	init_waitqueue_head(&sched->job.wq);
452
453	return 0;
454
455fail_sched:
456	drm_sched_fini(drm_sched);
457fail_wq:
458	if (sched->wq)
459		destroy_workqueue(sched->wq);
460	return ret;
461}
462
463int
464nouveau_sched_create(struct nouveau_sched **psched, struct nouveau_drm *drm,
465		     struct workqueue_struct *wq, u32 credit_limit)
466{
467	struct nouveau_sched *sched;
468	int ret;
469
470	sched = kzalloc(sizeof(*sched), GFP_KERNEL);
471	if (!sched)
472		return -ENOMEM;
473
474	ret = nouveau_sched_init(sched, drm, wq, credit_limit);
475	if (ret) {
476		kfree(sched);
477		return ret;
478	}
479
480	*psched = sched;
481
482	return 0;
483}
484
485static bool
486nouveau_sched_job_list_empty(struct nouveau_sched *sched)
487{
488	bool empty;
489
490	spin_lock(&sched->job.list.lock);
491	empty = list_empty(&sched->job.list.head);
492	spin_unlock(&sched->job.list.lock);
493
494	return empty;
495}
496
497static void
498nouveau_sched_fini(struct nouveau_sched *sched)
499{
500	struct drm_gpu_scheduler *drm_sched = &sched->base;
501	struct drm_sched_entity *entity = &sched->entity;
502
503	wait_event(sched->job.wq, nouveau_sched_job_list_empty(sched));
504
505	drm_sched_entity_fini(entity);
506	drm_sched_fini(drm_sched);
507
508	/* Destroy workqueue after scheduler tear down, otherwise it might still
509	 * be in use.
510	 */
511	if (sched->wq)
512		destroy_workqueue(sched->wq);
513}
514
515void
516nouveau_sched_destroy(struct nouveau_sched **psched)
517{
518	struct nouveau_sched *sched = *psched;
519
520	nouveau_sched_fini(sched);
521	kfree(sched);
522
523	*psched = NULL;
524}