cdma.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Tegra host1x Command DMA
  4 *
  5 * Copyright (c) 2010-2013, NVIDIA Corporation.
  6 */
  7
  8
  9#include <asm/cacheflush.h>
 10#include <linux/device.h>
 11#include <linux/dma-mapping.h>
 12#include <linux/host1x.h>
 13#include <linux/interrupt.h>
 14#include <linux/kernel.h>
 15#include <linux/kfifo.h>
 16#include <linux/slab.h>
 17#include <trace/events/host1x.h>
 18
 19#include "cdma.h"
 20#include "channel.h"
 21#include "dev.h"
 22#include "debug.h"
 23#include "job.h"
 24
 25/*
 26 * push_buffer
 27 *
 28 * The push buffer is a circular array of words to be fetched by command DMA.
 29 * Note that it works slightly differently to the sync queue; fence == pos
 30 * means that the push buffer is full, not empty.
 31 */
 32
 33/*
 34 * Typically the commands written into the push buffer are a pair of words. We
 35 * use slots to represent each of these pairs and to simplify things. Note the
 36 * strange number of slots allocated here. 512 slots will fit exactly within a
 37 * single memory page. We also need one additional word at the end of the push
 38 * buffer for the RESTART opcode that will instruct the CDMA to jump back to
 39 * the beginning of the push buffer. With 512 slots, this means that we'll use
 40 * 2 memory pages and waste 4092 bytes of the second page that will never be
 41 * used.
 42 */
 43#define HOST1X_PUSHBUFFER_SLOTS	511
 44
 45/*
 46 * Clean up push buffer resources
 47 */
 48static void host1x_pushbuffer_destroy(struct push_buffer *pb)
 49{
 50	struct host1x_cdma *cdma = pb_to_cdma(pb);
 51	struct host1x *host1x = cdma_to_host1x(cdma);
 52
 53	if (!pb->mapped)
 54		return;
 55
 56	if (host1x->domain) {
 57		iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
 58		free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
 59	}
 60
 61	dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
 62
 63	pb->mapped = NULL;
 64	pb->phys = 0;
 65}
 66
 67/*
 68 * Init push buffer resources
 69 */
 70static int host1x_pushbuffer_init(struct push_buffer *pb)
 71{
 72	struct host1x_cdma *cdma = pb_to_cdma(pb);
 73	struct host1x *host1x = cdma_to_host1x(cdma);
 74	struct iova *alloc;
 75	u32 size;
 76	int err;
 77
 78	pb->mapped = NULL;
 79	pb->phys = 0;
 80	pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;
 81
 82	size = pb->size + 4;
 83
 84	/* initialize buffer pointers */
 85	pb->fence = pb->size - 8;
 86	pb->pos = 0;
 87
 88	if (host1x->domain) {
 89		unsigned long shift;
 90
 91		size = iova_align(&host1x->iova, size);
 92
 93		pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
 94					  GFP_KERNEL);
 95		if (!pb->mapped)
 96			return -ENOMEM;
 97
 98		shift = iova_shift(&host1x->iova);
 99		alloc = alloc_iova(&host1x->iova, size >> shift,
100				   host1x->iova_end >> shift, true);
101		if (!alloc) {
102			err = -ENOMEM;
103			goto iommu_free_mem;
104		}
105
106		pb->dma = iova_dma_addr(&host1x->iova, alloc);
107		err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
108				IOMMU_READ, GFP_KERNEL);
109		if (err)
110			goto iommu_free_iova;
111	} else {
112		pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
113					  GFP_KERNEL);
114		if (!pb->mapped)
115			return -ENOMEM;
116
117		pb->dma = pb->phys;
118	}
119
120	pb->alloc_size = size;
121
122	host1x_hw_pushbuffer_init(host1x, pb);
123
124	return 0;
125
126iommu_free_iova:
127	__free_iova(&host1x->iova, alloc);
128iommu_free_mem:
129	dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
130
131	return err;
132}
133
134/*
135 * Push two words to the push buffer
136 * Caller must ensure push buffer is not full
137 */
138static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
139{
140	u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
141
142	WARN_ON(pb->pos == pb->fence);
143	*(p++) = op1;
144	*(p++) = op2;
145	pb->pos += 8;
146
147	if (pb->pos >= pb->size)
148		pb->pos -= pb->size;
149}
150
151/*
152 * Pop a number of two word slots from the push buffer
153 * Caller must ensure push buffer is not empty
154 */
155static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
156{
157	/* Advance the next write position */
158	pb->fence += slots * 8;
159
160	if (pb->fence >= pb->size)
161		pb->fence -= pb->size;
162}
163
164/*
165 * Return the number of two word slots free in the push buffer
166 */
167static u32 host1x_pushbuffer_space(struct push_buffer *pb)
168{
169	unsigned int fence = pb->fence;
170
171	if (pb->fence < pb->pos)
172		fence += pb->size;
173
174	return (fence - pb->pos) / 8;
175}
176
177/*
178 * Sleep (if necessary) until the requested event happens
179 *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
180 *     - Returns 1
181 *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
182 *     - Return the amount of space (> 0)
183 * Must be called with the cdma lock held.
184 */
185unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
186				     enum cdma_event event)
187{
188	for (;;) {
189		struct push_buffer *pb = &cdma->push_buffer;
190		unsigned int space;
191
192		switch (event) {
193		case CDMA_EVENT_SYNC_QUEUE_EMPTY:
194			space = list_empty(&cdma->sync_queue) ? 1 : 0;
195			break;
196
197		case CDMA_EVENT_PUSH_BUFFER_SPACE:
198			space = host1x_pushbuffer_space(pb);
199			break;
200
201		default:
202			WARN_ON(1);
203			return -EINVAL;
204		}
205
206		if (space)
207			return space;
208
209		trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
210				       event);
211
212		/* If somebody has managed to already start waiting, yield */
213		if (cdma->event != CDMA_EVENT_NONE) {
214			mutex_unlock(&cdma->lock);
215			schedule();
216			mutex_lock(&cdma->lock);
217			continue;
218		}
219
220		cdma->event = event;
221
222		mutex_unlock(&cdma->lock);
223		wait_for_completion(&cdma->complete);
224		mutex_lock(&cdma->lock);
225	}
226
227	return 0;
228}
229
230/*
231 * Sleep (if necessary) until the push buffer has enough free space.
232 *
233 * Must be called with the cdma lock held.
234 */
235static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
236					     struct host1x_cdma *cdma,
237					     unsigned int needed)
238{
239	while (true) {
240		struct push_buffer *pb = &cdma->push_buffer;
241		unsigned int space;
242
243		space = host1x_pushbuffer_space(pb);
244		if (space >= needed)
245			break;
246
247		trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
248				       CDMA_EVENT_PUSH_BUFFER_SPACE);
249
250		/* If somebody has managed to already start waiting, yield */
251		if (cdma->event != CDMA_EVENT_NONE) {
252			mutex_unlock(&cdma->lock);
253			schedule();
254			mutex_lock(&cdma->lock);
255			continue;
256		}
257
258		cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;
259
260		mutex_unlock(&cdma->lock);
261		wait_for_completion(&cdma->complete);
262		mutex_lock(&cdma->lock);
263	}
264
265	return 0;
266}
267/*
268 * Start timer that tracks the time spent by the job.
269 * Must be called with the cdma lock held.
270 */
271static void cdma_start_timer_locked(struct host1x_cdma *cdma,
272				    struct host1x_job *job)
273{
274	if (cdma->timeout.client) {
275		/* timer already started */
276		return;
277	}
278
279	cdma->timeout.client = job->client;
280	cdma->timeout.syncpt = job->syncpt;
281	cdma->timeout.syncpt_val = job->syncpt_end;
282	cdma->timeout.start_ktime = ktime_get();
283
284	schedule_delayed_work(&cdma->timeout.wq,
285			      msecs_to_jiffies(job->timeout));
286}
287
288/*
289 * Stop timer when a buffer submission completes.
290 * Must be called with the cdma lock held.
291 */
292static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
293{
294	cancel_delayed_work(&cdma->timeout.wq);
295	cdma->timeout.client = NULL;
296}
297
298/*
299 * For all sync queue entries that have already finished according to the
300 * current sync point registers:
301 *  - unpin & unref their mems
302 *  - pop their push buffer slots
303 *  - remove them from the sync queue
304 * This is normally called from the host code's worker thread, but can be
305 * called manually if necessary.
306 * Must be called with the cdma lock held.
307 */
308static void update_cdma_locked(struct host1x_cdma *cdma)
309{
310	bool signal = false;
311	struct host1x_job *job, *n;
312
313	/*
314	 * Walk the sync queue, reading the sync point registers as necessary,
315	 * to consume as many sync queue entries as possible without blocking
316	 */
317	list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
318		struct host1x_syncpt *sp = job->syncpt;
319
320		/* Check whether this syncpt has completed, and bail if not */
321		if (!host1x_syncpt_is_expired(sp, job->syncpt_end) &&
322		    !job->cancelled) {
323			/* Start timer on next pending syncpt */
324			if (job->timeout)
325				cdma_start_timer_locked(cdma, job);
326
327			break;
328		}
329
330		/* Cancel timeout, when a buffer completes */
331		if (cdma->timeout.client)
332			stop_cdma_timer_locked(cdma);
333
334		/* Unpin the memory */
335		host1x_job_unpin(job);
336
337		/* Pop push buffer slots */
338		if (job->num_slots) {
339			struct push_buffer *pb = &cdma->push_buffer;
340
341			host1x_pushbuffer_pop(pb, job->num_slots);
342
343			if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
344				signal = true;
345		}
346
347		list_del(&job->list);
348		host1x_job_put(job);
349	}
350
351	if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
352	    list_empty(&cdma->sync_queue))
353		signal = true;
354
355	if (signal) {
356		cdma->event = CDMA_EVENT_NONE;
357		complete(&cdma->complete);
358	}
359}
360
361void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
362				   struct device *dev)
363{
364	struct host1x *host1x = cdma_to_host1x(cdma);
365	u32 restart_addr, syncpt_incrs, syncpt_val;
366	struct host1x_job *job, *next_job = NULL;
367
368	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
369
370	dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
371		__func__, syncpt_val);
372
373	/*
374	 * Move the sync_queue read pointer to the first entry that hasn't
375	 * completed based on the current HW syncpt value. It's likely there
376	 * won't be any (i.e. we're still at the head), but covers the case
377	 * where a syncpt incr happens just prior/during the teardown.
378	 */
379
380	dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
381		__func__);
382
383	list_for_each_entry(job, &cdma->sync_queue, list) {
384		if (syncpt_val < job->syncpt_end) {
385
386			if (!list_is_last(&job->list, &cdma->sync_queue))
387				next_job = list_next_entry(job, list);
388
389			goto syncpt_incr;
390		}
391
392		host1x_job_dump(dev, job);
393	}
394
395	/* all jobs have been completed */
396	job = NULL;
397
398syncpt_incr:
399
400	/*
401	 * Increment with CPU the remaining syncpts of a partially executed job.
402	 *
403	 * CDMA will continue execution starting with the next job or will get
404	 * into idle state.
405	 */
406	if (next_job)
407		restart_addr = next_job->first_get;
408	else
409		restart_addr = cdma->last_pos;
410
411	if (!job)
412		goto resume;
413
414	/* do CPU increments for the remaining syncpts */
415	if (job->syncpt_recovery) {
416		dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
417			__func__);
418
419		/* won't need a timeout when replayed */
420		job->timeout = 0;
421
422		syncpt_incrs = job->syncpt_end - syncpt_val;
423		dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
424
425		host1x_job_dump(dev, job);
426
427		/* safe to use CPU to incr syncpts */
428		host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
429						syncpt_incrs, job->syncpt_end,
430						job->num_slots);
431
432		dev_dbg(dev, "%s: finished sync_queue modification\n",
433			__func__);
434	} else {
435		struct host1x_job *failed_job = job;
436
437		host1x_job_dump(dev, job);
438
439		host1x_syncpt_set_locked(job->syncpt);
440		failed_job->cancelled = true;
441
442		list_for_each_entry_continue(job, &cdma->sync_queue, list) {
443			unsigned int i;
444
445			if (job->syncpt != failed_job->syncpt)
446				continue;
447
448			for (i = 0; i < job->num_slots; i++) {
449				unsigned int slot = (job->first_get/8 + i) %
450						    HOST1X_PUSHBUFFER_SLOTS;
451				u32 *mapped = cdma->push_buffer.mapped;
452
453				/*
454				 * Overwrite opcodes with 0 word writes
455				 * to offset 0xbad. This does nothing but
456				 * has a easily detected signature in debug
457				 * traces.
458				 *
459				 * On systems with MLOCK enforcement enabled,
460				 * the above 0 word writes would fall foul of
461				 * the enforcement. As such, in the first slot
462				 * put a RESTART_W opcode to the beginning
463				 * of the next job. We don't use this for older
464				 * chips since those only support the RESTART
465				 * opcode with inconvenient alignment requirements.
466				 */
467				if (i == 0 && host1x->info->has_wide_gather) {
468					unsigned int next_job = (job->first_get/8 + job->num_slots)
469						% HOST1X_PUSHBUFFER_SLOTS;
470					mapped[2*slot+0] = (0xd << 28) | (next_job * 2);
471					mapped[2*slot+1] = 0x0;
472				} else {
473					mapped[2*slot+0] = 0x1bad0000;
474					mapped[2*slot+1] = 0x1bad0000;
475				}
476			}
477
478			job->cancelled = true;
479		}
480
481		wmb();
482
483		update_cdma_locked(cdma);
484	}
485
486resume:
487	/* roll back DMAGET and start up channel again */
488	host1x_hw_cdma_resume(host1x, cdma, restart_addr);
489}
490
491static void cdma_update_work(struct work_struct *work)
492{
493	struct host1x_cdma *cdma = container_of(work, struct host1x_cdma, update_work);
494
495	mutex_lock(&cdma->lock);
496	update_cdma_locked(cdma);
497	mutex_unlock(&cdma->lock);
498}
499
500/*
501 * Create a cdma
502 */
503int host1x_cdma_init(struct host1x_cdma *cdma)
504{
505	int err;
506
507	mutex_init(&cdma->lock);
508	init_completion(&cdma->complete);
509	INIT_WORK(&cdma->update_work, cdma_update_work);
510
511	INIT_LIST_HEAD(&cdma->sync_queue);
512
513	cdma->event = CDMA_EVENT_NONE;
514	cdma->running = false;
515	cdma->torndown = false;
516
517	err = host1x_pushbuffer_init(&cdma->push_buffer);
518	if (err)
519		return err;
520
521	return 0;
522}
523
524/*
525 * Destroy a cdma
526 */
527int host1x_cdma_deinit(struct host1x_cdma *cdma)
528{
529	struct push_buffer *pb = &cdma->push_buffer;
530	struct host1x *host1x = cdma_to_host1x(cdma);
531
532	if (cdma->running) {
533		pr_warn("%s: CDMA still running\n", __func__);
534		return -EBUSY;
535	}
536
537	host1x_pushbuffer_destroy(pb);
538	host1x_hw_cdma_timeout_destroy(host1x, cdma);
539
540	return 0;
541}
542
543/*
544 * Begin a cdma submit
545 */
546int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
547{
548	struct host1x *host1x = cdma_to_host1x(cdma);
549
550	mutex_lock(&cdma->lock);
551
552	/*
553	 * Check if syncpoint was locked due to previous job timeout.
554	 * This needs to be done within the cdma lock to avoid a race
555	 * with the timeout handler.
556	 */
557	if (job->syncpt->locked) {
558		mutex_unlock(&cdma->lock);
559		return -EPERM;
560	}
561
562	if (job->timeout) {
563		/* init state on first submit with timeout value */
564		if (!cdma->timeout.initialized) {
565			int err;
566
567			err = host1x_hw_cdma_timeout_init(host1x, cdma);
568			if (err) {
569				mutex_unlock(&cdma->lock);
570				return err;
571			}
572		}
573	}
574
575	if (!cdma->running)
576		host1x_hw_cdma_start(host1x, cdma);
577
578	cdma->slots_free = 0;
579	cdma->slots_used = 0;
580	cdma->first_get = cdma->push_buffer.pos;
581
582	trace_host1x_cdma_begin(dev_name(job->channel->dev));
583	return 0;
584}
585
586/*
587 * Push two words into a push buffer slot
588 * Blocks as necessary if the push buffer is full.
589 */
590void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
591{
592	struct push_buffer *pb = &cdma->push_buffer;
593	u32 slots_free = cdma->slots_free;
594
595	if (host1x_debug_trace_cmdbuf)
596		trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
597				       op1, op2);
598
599	if (slots_free == 0)
600		slots_free = host1x_cdma_wait_locked(cdma,
601						CDMA_EVENT_PUSH_BUFFER_SPACE);
602
603	cdma->slots_free = slots_free - 1;
604	cdma->slots_used++;
605	host1x_pushbuffer_push(pb, op1, op2);
606}
607
608/*
609 * Push four words into two consecutive push buffer slots. Note that extra
610 * care needs to be taken not to split the two slots across the end of the
611 * push buffer. Otherwise the RESTART opcode at the end of the push buffer
612 * that ensures processing will restart at the beginning will break up the
613 * four words.
614 *
615 * Blocks as necessary if the push buffer is full.
616 */
617void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
618			   u32 op3, u32 op4)
619{
620	struct host1x_channel *channel = cdma_to_channel(cdma);
621	struct host1x *host1x = cdma_to_host1x(cdma);
622	struct push_buffer *pb = &cdma->push_buffer;
623	unsigned int space, needed = 2, extra = 0;
624
625	if (host1x_debug_trace_cmdbuf)
626		trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
627					    op3, op4);
628
629	/* compute number of extra slots needed for padding */
630	if (pb->pos + 16 > pb->size) {
631		extra = (pb->size - pb->pos) / 8;
632		needed += extra;
633	}
634
635	host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
636	space = host1x_pushbuffer_space(pb);
637
638	cdma->slots_free = space - needed;
639	cdma->slots_used += needed;
640
641	if (extra > 0) {
642		/*
643		 * If there isn't enough space at the tail of the pushbuffer,
644		 * insert a RESTART(0) here to go back to the beginning.
645		 * The code above adjusted the indexes appropriately.
646		 */
647		host1x_pushbuffer_push(pb, (0x5 << 28), 0xdead0000);
648	}
649
650	host1x_pushbuffer_push(pb, op1, op2);
651	host1x_pushbuffer_push(pb, op3, op4);
652}
653
654/*
655 * End a cdma submit
656 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
657 * from the pushbuffer. The handles for a submit must all be pinned at the same
658 * time, but they can be unpinned in smaller chunks.
659 */
660void host1x_cdma_end(struct host1x_cdma *cdma,
661		     struct host1x_job *job)
662{
663	struct host1x *host1x = cdma_to_host1x(cdma);
664	bool idle = list_empty(&cdma->sync_queue);
665
666	host1x_hw_cdma_flush(host1x, cdma);
667
668	job->first_get = cdma->first_get;
669	job->num_slots = cdma->slots_used;
670	host1x_job_get(job);
671	list_add_tail(&job->list, &cdma->sync_queue);
672
673	/* start timer on idle -> active transitions */
674	if (job->timeout && idle)
675		cdma_start_timer_locked(cdma, job);
676
677	trace_host1x_cdma_end(dev_name(job->channel->dev));
678	mutex_unlock(&cdma->lock);
679}
680
681/*
682 * Update cdma state according to current sync point values
683 */
684void host1x_cdma_update(struct host1x_cdma *cdma)
685{
686	schedule_work(&cdma->update_work);
687}