mtk-cqdma.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (c) 2018-2019 MediaTek Inc.
  3
  4/*
  5 * Driver for MediaTek Command-Queue DMA Controller
  6 *
  7 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
  8 *
  9 */
 10
 11#include <linux/bitops.h>
 12#include <linux/clk.h>
 13#include <linux/dmaengine.h>
 14#include <linux/dma-mapping.h>
 15#include <linux/err.h>
 16#include <linux/iopoll.h>
 17#include <linux/interrupt.h>
 18#include <linux/list.h>
 19#include <linux/module.h>
 20#include <linux/of.h>
 21#include <linux/of_dma.h>
 22#include <linux/platform_device.h>
 23#include <linux/pm_runtime.h>
 24#include <linux/refcount.h>
 25#include <linux/slab.h>
 26
 27#include "../virt-dma.h"
 28
 29#define MTK_CQDMA_USEC_POLL		10
 30#define MTK_CQDMA_TIMEOUT_POLL		1000
 31#define MTK_CQDMA_DMA_BUSWIDTHS		BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
 32#define MTK_CQDMA_ALIGN_SIZE		1
 33
 34/* The default number of virtual channel */
 35#define MTK_CQDMA_NR_VCHANS		32
 36
 37/* The default number of physical channel */
 38#define MTK_CQDMA_NR_PCHANS		3
 39
 40/* Registers for underlying dma manipulation */
 41#define MTK_CQDMA_INT_FLAG		0x0
 42#define MTK_CQDMA_INT_EN		0x4
 43#define MTK_CQDMA_EN			0x8
 44#define MTK_CQDMA_RESET			0xc
 45#define MTK_CQDMA_FLUSH			0x14
 46#define MTK_CQDMA_SRC			0x1c
 47#define MTK_CQDMA_DST			0x20
 48#define MTK_CQDMA_LEN1			0x24
 49#define MTK_CQDMA_LEN2			0x28
 50#define MTK_CQDMA_SRC2			0x60
 51#define MTK_CQDMA_DST2			0x64
 52
 53/* Registers setting */
 54#define MTK_CQDMA_EN_BIT		BIT(0)
 55#define MTK_CQDMA_INT_FLAG_BIT		BIT(0)
 56#define MTK_CQDMA_INT_EN_BIT		BIT(0)
 57#define MTK_CQDMA_FLUSH_BIT		BIT(0)
 58
 59#define MTK_CQDMA_WARM_RST_BIT		BIT(0)
 60#define MTK_CQDMA_HARD_RST_BIT		BIT(1)
 61
 62#define MTK_CQDMA_MAX_LEN		GENMASK(27, 0)
 63#define MTK_CQDMA_ADDR_LIMIT		GENMASK(31, 0)
 64#define MTK_CQDMA_ADDR2_SHFIT		(32)
 65
 66/**
 67 * struct mtk_cqdma_vdesc - The struct holding info describing virtual
 68 *                         descriptor (CVD)
 69 * @vd:                    An instance for struct virt_dma_desc
 70 * @len:                   The total data size device wants to move
 71 * @residue:               The remaining data size device will move
 72 * @dest:                  The destination address device wants to move to
 73 * @src:                   The source address device wants to move from
 74 * @ch:                    The pointer to the corresponding dma channel
 75 * @node:                  The lise_head struct to build link-list for VDs
 76 * @parent:                The pointer to the parent CVD
 77 */
 78struct mtk_cqdma_vdesc {
 79	struct virt_dma_desc vd;
 80	size_t len;
 81	size_t residue;
 82	dma_addr_t dest;
 83	dma_addr_t src;
 84	struct dma_chan *ch;
 85
 86	struct list_head node;
 87	struct mtk_cqdma_vdesc *parent;
 88};
 89
 90/**
 91 * struct mtk_cqdma_pchan - The struct holding info describing physical
 92 *                         channel (PC)
 93 * @queue:                 Queue for the PDs issued to this PC
 94 * @base:                  The mapped register I/O base of this PC
 95 * @irq:                   The IRQ that this PC are using
 96 * @refcnt:                Track how many VCs are using this PC
 97 * @tasklet:               Tasklet for this PC
 98 * @lock:                  Lock protect agaisting multiple VCs access PC
 99 */
100struct mtk_cqdma_pchan {
101	struct list_head queue;
102	void __iomem *base;
103	u32 irq;
104
105	refcount_t refcnt;
106
107	struct tasklet_struct tasklet;
108
109	/* lock to protect PC */
110	spinlock_t lock;
111};
112
113/**
114 * struct mtk_cqdma_vchan - The struct holding info describing virtual
115 *                         channel (VC)
116 * @vc:                    An instance for struct virt_dma_chan
117 * @pc:                    The pointer to the underlying PC
118 * @issue_completion:	   The wait for all issued descriptors completited
119 * @issue_synchronize:	   Bool indicating channel synchronization starts
120 */
121struct mtk_cqdma_vchan {
122	struct virt_dma_chan vc;
123	struct mtk_cqdma_pchan *pc;
124	struct completion issue_completion;
125	bool issue_synchronize;
126};
127
128/**
129 * struct mtk_cqdma_device - The struct holding info describing CQDMA
130 *                          device
131 * @ddev:                   An instance for struct dma_device
132 * @clk:                    The clock that device internal is using
133 * @dma_requests:           The number of VCs the device supports to
134 * @dma_channels:           The number of PCs the device supports to
135 * @vc:                     The pointer to all available VCs
136 * @pc:                     The pointer to all the underlying PCs
137 */
138struct mtk_cqdma_device {
139	struct dma_device ddev;
140	struct clk *clk;
141
142	u32 dma_requests;
143	u32 dma_channels;
144	struct mtk_cqdma_vchan *vc;
145	struct mtk_cqdma_pchan **pc;
146};
147
148static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
149{
150	return container_of(chan->device, struct mtk_cqdma_device, ddev);
151}
152
153static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
154{
155	return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
156}
157
158static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
159{
160	return container_of(vd, struct mtk_cqdma_vdesc, vd);
161}
162
163static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
164{
165	return cqdma->ddev.dev;
166}
167
168static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
169{
170	return readl(pc->base + reg);
171}
172
173static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
174{
175	writel_relaxed(val, pc->base + reg);
176}
177
178static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
179			u32 mask, u32 set)
180{
181	u32 val;
182
183	val = mtk_dma_read(pc, reg);
184	val &= ~mask;
185	val |= set;
186	mtk_dma_write(pc, reg, val);
187}
188
189static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
190{
191	mtk_dma_rmw(pc, reg, 0, val);
192}
193
194static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
195{
196	mtk_dma_rmw(pc, reg, val, 0);
197}
198
199static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
200{
201	kfree(to_cqdma_vdesc(vd));
202}
203
204static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
205{
206	u32 status = 0;
207
208	if (!atomic)
209		return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
210					  status,
211					  !(status & MTK_CQDMA_EN_BIT),
212					  MTK_CQDMA_USEC_POLL,
213					  MTK_CQDMA_TIMEOUT_POLL);
214
215	return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
216					 status,
217					 !(status & MTK_CQDMA_EN_BIT),
218					 MTK_CQDMA_USEC_POLL,
219					 MTK_CQDMA_TIMEOUT_POLL);
220}
221
222static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
223{
224	mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
225	mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
226
227	return mtk_cqdma_poll_engine_done(pc, true);
228}
229
230static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
231			    struct mtk_cqdma_vdesc *cvd)
232{
233	/* wait for the previous transaction done */
234	if (mtk_cqdma_poll_engine_done(pc, true) < 0)
235		dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");
236
237	/* warm reset the dma engine for the new transaction */
238	mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
239	if (mtk_cqdma_poll_engine_done(pc, true) < 0)
240		dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");
241
242	/* setup the source */
243	mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
244#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
245	mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
246#else
247	mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
248#endif
249
250	/* setup the destination */
251	mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
252#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
253	mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
254#else
255	mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
256#endif
257
258	/* setup the length */
259	mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);
260
261	/* start dma engine */
262	mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
263}
264
265static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
266{
267	struct virt_dma_desc *vd, *vd2;
268	struct mtk_cqdma_pchan *pc = cvc->pc;
269	struct mtk_cqdma_vdesc *cvd;
270	bool trigger_engine = false;
271
272	lockdep_assert_held(&cvc->vc.lock);
273	lockdep_assert_held(&pc->lock);
274
275	list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
276		/* need to trigger dma engine if PC's queue is empty */
277		if (list_empty(&pc->queue))
278			trigger_engine = true;
279
280		cvd = to_cqdma_vdesc(vd);
281
282		/* add VD into PC's queue */
283		list_add_tail(&cvd->node, &pc->queue);
284
285		/* start the dma engine */
286		if (trigger_engine)
287			mtk_cqdma_start(pc, cvd);
288
289		/* remove VD from list desc_issued */
290		list_del(&vd->node);
291	}
292}
293
294/*
295 * return true if this VC is active,
296 * meaning that there are VDs under processing by the PC
297 */
298static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
299{
300	struct mtk_cqdma_vdesc *cvd;
301
302	list_for_each_entry(cvd, &cvc->pc->queue, node)
303		if (cvc == to_cqdma_vchan(cvd->ch))
304			return true;
305
306	return false;
307}
308
309/*
310 * return the pointer of the CVD that is just consumed by the PC
311 */
312static struct mtk_cqdma_vdesc
313*mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
314{
315	struct mtk_cqdma_vchan *cvc;
316	struct mtk_cqdma_vdesc *cvd, *ret = NULL;
317
318	/* consume a CVD from PC's queue */
319	cvd = list_first_entry_or_null(&pc->queue,
320				       struct mtk_cqdma_vdesc, node);
321	if (unlikely(!cvd || !cvd->parent))
322		return NULL;
323
324	cvc = to_cqdma_vchan(cvd->ch);
325	ret = cvd;
326
327	/* update residue of the parent CVD */
328	cvd->parent->residue -= cvd->len;
329
330	/* delete CVD from PC's queue */
331	list_del(&cvd->node);
332
333	spin_lock(&cvc->vc.lock);
334
335	/* check whether all the child CVDs completed */
336	if (!cvd->parent->residue) {
337		/* add the parent VD into list desc_completed */
338		vchan_cookie_complete(&cvd->parent->vd);
339
340		/* setup completion if this VC is under synchronization */
341		if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
342			complete(&cvc->issue_completion);
343			cvc->issue_synchronize = false;
344		}
345	}
346
347	spin_unlock(&cvc->vc.lock);
348
349	/* start transaction for next CVD in the queue */
350	cvd = list_first_entry_or_null(&pc->queue,
351				       struct mtk_cqdma_vdesc, node);
352	if (cvd)
353		mtk_cqdma_start(pc, cvd);
354
355	return ret;
356}
357
358static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t)
359{
360	struct mtk_cqdma_pchan *pc = from_tasklet(pc, t, tasklet);
361	struct mtk_cqdma_vdesc *cvd = NULL;
362	unsigned long flags;
363
364	spin_lock_irqsave(&pc->lock, flags);
365	/* consume the queue */
366	cvd = mtk_cqdma_consume_work_queue(pc);
367	spin_unlock_irqrestore(&pc->lock, flags);
368
369	/* submit the next CVD */
370	if (cvd) {
371		dma_run_dependencies(&cvd->vd.tx);
372
373		/*
374		 * free child CVD after completion.
375		 * the parent CVD would be freed with desc_free by user.
376		 */
377		if (cvd->parent != cvd)
378			kfree(cvd);
379	}
380
381	/* re-enable interrupt before leaving tasklet */
382	enable_irq(pc->irq);
383}
384
385static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
386{
387	struct mtk_cqdma_device *cqdma = devid;
388	irqreturn_t ret = IRQ_NONE;
389	bool schedule_tasklet = false;
390	u32 i;
391
392	/* clear interrupt flags for each PC */
393	for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
394		spin_lock(&cqdma->pc[i]->lock);
395		if (mtk_dma_read(cqdma->pc[i],
396				 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
397			/* clear interrupt */
398			mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
399				    MTK_CQDMA_INT_FLAG_BIT);
400
401			schedule_tasklet = true;
402			ret = IRQ_HANDLED;
403		}
404		spin_unlock(&cqdma->pc[i]->lock);
405
406		if (schedule_tasklet) {
407			/* disable interrupt */
408			disable_irq_nosync(cqdma->pc[i]->irq);
409
410			/* schedule the tasklet to handle the transactions */
411			tasklet_schedule(&cqdma->pc[i]->tasklet);
412		}
413	}
414
415	return ret;
416}
417
418static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
419							dma_cookie_t cookie)
420{
421	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
422	struct virt_dma_desc *vd;
423
424	list_for_each_entry(vd, &cvc->pc->queue, node)
425		if (vd->tx.cookie == cookie) {
426			return vd;
427		}
428
429	list_for_each_entry(vd, &cvc->vc.desc_issued, node)
430		if (vd->tx.cookie == cookie)
431			return vd;
432
433	return NULL;
434}
435
436static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
437					   dma_cookie_t cookie,
438					   struct dma_tx_state *txstate)
439{
440	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
441	struct mtk_cqdma_vdesc *cvd;
442	struct virt_dma_desc *vd;
443	enum dma_status ret;
444	unsigned long flags;
445	size_t bytes = 0;
446
447	ret = dma_cookie_status(c, cookie, txstate);
448	if (ret == DMA_COMPLETE || !txstate)
449		return ret;
450
451	spin_lock_irqsave(&cvc->pc->lock, flags);
452	spin_lock(&cvc->vc.lock);
453	vd = mtk_cqdma_find_active_desc(c, cookie);
454	spin_unlock(&cvc->vc.lock);
455	spin_unlock_irqrestore(&cvc->pc->lock, flags);
456
457	if (vd) {
458		cvd = to_cqdma_vdesc(vd);
459		bytes = cvd->residue;
460	}
461
462	dma_set_residue(txstate, bytes);
463
464	return ret;
465}
466
467static void mtk_cqdma_issue_pending(struct dma_chan *c)
468{
469	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
470	unsigned long pc_flags;
471	unsigned long vc_flags;
472
473	/* acquire PC's lock before VS's lock for lock dependency in tasklet */
474	spin_lock_irqsave(&cvc->pc->lock, pc_flags);
475	spin_lock_irqsave(&cvc->vc.lock, vc_flags);
476
477	if (vchan_issue_pending(&cvc->vc))
478		mtk_cqdma_issue_vchan_pending(cvc);
479
480	spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
481	spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
482}
483
484static struct dma_async_tx_descriptor *
485mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
486			  dma_addr_t src, size_t len, unsigned long flags)
487{
488	struct mtk_cqdma_vdesc **cvd;
489	struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
490	size_t i, tlen, nr_vd;
491
492	/*
493	 * In the case that trsanction length is larger than the
494	 * DMA engine supports, a single memcpy transaction needs
495	 * to be separated into several DMA transactions.
496	 * Each DMA transaction would be described by a CVD,
497	 * and the first one is referred as the parent CVD,
498	 * while the others are child CVDs.
499	 * The parent CVD's tx descriptor is the only tx descriptor
500	 * returned to the DMA user, and it should not be completed
501	 * until all the child CVDs completed.
502	 */
503	nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
504	cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
505	if (!cvd)
506		return NULL;
507
508	for (i = 0; i < nr_vd; ++i) {
509		cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
510		if (!cvd[i]) {
511			for (; i > 0; --i)
512				kfree(cvd[i - 1]);
513			return NULL;
514		}
515
516		/* setup dma channel */
517		cvd[i]->ch = c;
518
519		/* setup source, destination, and length */
520		tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
521		cvd[i]->len = tlen;
522		cvd[i]->src = src;
523		cvd[i]->dest = dest;
524
525		/* setup tx descriptor */
526		tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
527		tx->next = NULL;
528
529		if (!i) {
530			cvd[0]->residue = len;
531		} else {
532			prev_tx->next = tx;
533			cvd[i]->residue = tlen;
534		}
535
536		cvd[i]->parent = cvd[0];
537
538		/* update the src, dest, len, prev_tx for the next CVD */
539		src += tlen;
540		dest += tlen;
541		len -= tlen;
542		prev_tx = tx;
543	}
544
545	return &cvd[0]->vd.tx;
546}
547
548static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
549{
550	struct virt_dma_chan *vc = to_virt_chan(c);
551	unsigned long flags;
552	LIST_HEAD(head);
553
554	/*
555	 * set desc_allocated, desc_submitted,
556	 * and desc_issued as the candicates to be freed
557	 */
558	spin_lock_irqsave(&vc->lock, flags);
559	list_splice_tail_init(&vc->desc_allocated, &head);
560	list_splice_tail_init(&vc->desc_submitted, &head);
561	list_splice_tail_init(&vc->desc_issued, &head);
562	spin_unlock_irqrestore(&vc->lock, flags);
563
564	/* free descriptor lists */
565	vchan_dma_desc_free_list(vc, &head);
566}
567
568static void mtk_cqdma_free_active_desc(struct dma_chan *c)
569{
570	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
571	bool sync_needed = false;
572	unsigned long pc_flags;
573	unsigned long vc_flags;
574
575	/* acquire PC's lock first due to lock dependency in dma ISR */
576	spin_lock_irqsave(&cvc->pc->lock, pc_flags);
577	spin_lock_irqsave(&cvc->vc.lock, vc_flags);
578
579	/* synchronization is required if this VC is active */
580	if (mtk_cqdma_is_vchan_active(cvc)) {
581		cvc->issue_synchronize = true;
582		sync_needed = true;
583	}
584
585	spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
586	spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
587
588	/* waiting for the completion of this VC */
589	if (sync_needed)
590		wait_for_completion(&cvc->issue_completion);
591
592	/* free all descriptors in list desc_completed */
593	vchan_synchronize(&cvc->vc);
594
595	WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
596		  "Desc pending still in list desc_completed\n");
597}
598
599static int mtk_cqdma_terminate_all(struct dma_chan *c)
600{
601	/* free descriptors not processed yet by hardware */
602	mtk_cqdma_free_inactive_desc(c);
603
604	/* free descriptors being processed by hardware */
605	mtk_cqdma_free_active_desc(c);
606
607	return 0;
608}
609
610static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
611{
612	struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
613	struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
614	struct mtk_cqdma_pchan *pc = NULL;
615	u32 i, min_refcnt = U32_MAX, refcnt;
616	unsigned long flags;
617
618	/* allocate PC with the minimum refcount */
619	for (i = 0; i < cqdma->dma_channels; ++i) {
620		refcnt = refcount_read(&cqdma->pc[i]->refcnt);
621		if (refcnt < min_refcnt) {
622			pc = cqdma->pc[i];
623			min_refcnt = refcnt;
624		}
625	}
626
627	if (!pc)
628		return -ENOSPC;
629
630	spin_lock_irqsave(&pc->lock, flags);
631
632	if (!refcount_read(&pc->refcnt)) {
633		/* allocate PC when the refcount is zero */
634		mtk_cqdma_hard_reset(pc);
635
636		/* enable interrupt for this PC */
637		mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
638
639		/*
640		 * refcount_inc would complain increment on 0; use-after-free.
641		 * Thus, we need to explicitly set it as 1 initially.
642		 */
643		refcount_set(&pc->refcnt, 1);
644	} else {
645		refcount_inc(&pc->refcnt);
646	}
647
648	spin_unlock_irqrestore(&pc->lock, flags);
649
650	vc->pc = pc;
651
652	return 0;
653}
654
655static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
656{
657	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
658	unsigned long flags;
659
660	/* free all descriptors in all lists on the VC */
661	mtk_cqdma_terminate_all(c);
662
663	spin_lock_irqsave(&cvc->pc->lock, flags);
664
665	/* PC is not freed until there is no VC mapped to it */
666	if (refcount_dec_and_test(&cvc->pc->refcnt)) {
667		/* start the flush operation and stop the engine */
668		mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
669
670		/* wait for the completion of flush operation */
671		if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
672			dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");
673
674		/* clear the flush bit and interrupt flag */
675		mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
676		mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
677			    MTK_CQDMA_INT_FLAG_BIT);
678
679		/* disable interrupt for this PC */
680		mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
681	}
682
683	spin_unlock_irqrestore(&cvc->pc->lock, flags);
684}
685
686static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
687{
688	unsigned long flags;
689	int err;
690	u32 i;
691
692	pm_runtime_enable(cqdma2dev(cqdma));
693	pm_runtime_get_sync(cqdma2dev(cqdma));
694
695	err = clk_prepare_enable(cqdma->clk);
696
697	if (err) {
698		pm_runtime_put_sync(cqdma2dev(cqdma));
699		pm_runtime_disable(cqdma2dev(cqdma));
700		return err;
701	}
702
703	/* reset all PCs */
704	for (i = 0; i < cqdma->dma_channels; ++i) {
705		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
706		if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
707			dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
708			spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
709
710			clk_disable_unprepare(cqdma->clk);
711			pm_runtime_put_sync(cqdma2dev(cqdma));
712			pm_runtime_disable(cqdma2dev(cqdma));
713			return -EINVAL;
714		}
715		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
716	}
717
718	return 0;
719}
720
721static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
722{
723	unsigned long flags;
724	u32 i;
725
726	/* reset all PCs */
727	for (i = 0; i < cqdma->dma_channels; ++i) {
728		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
729		if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
730			dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
731		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
732	}
733
734	clk_disable_unprepare(cqdma->clk);
735
736	pm_runtime_put_sync(cqdma2dev(cqdma));
737	pm_runtime_disable(cqdma2dev(cqdma));
738}
739
740static const struct of_device_id mtk_cqdma_match[] = {
741	{ .compatible = "mediatek,mt6765-cqdma" },
742	{ /* sentinel */ }
743};
744MODULE_DEVICE_TABLE(of, mtk_cqdma_match);
745
746static int mtk_cqdma_probe(struct platform_device *pdev)
747{
748	struct mtk_cqdma_device *cqdma;
749	struct mtk_cqdma_vchan *vc;
750	struct dma_device *dd;
751	int err;
752	u32 i;
753
754	cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
755	if (!cqdma)
756		return -ENOMEM;
757
758	dd = &cqdma->ddev;
759
760	cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
761	if (IS_ERR(cqdma->clk)) {
762		dev_err(&pdev->dev, "No clock for %s\n",
763			dev_name(&pdev->dev));
764		return PTR_ERR(cqdma->clk);
765	}
766
767	dma_cap_set(DMA_MEMCPY, dd->cap_mask);
768
769	dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
770	dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
771	dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
772	dd->device_tx_status = mtk_cqdma_tx_status;
773	dd->device_issue_pending = mtk_cqdma_issue_pending;
774	dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
775	dd->device_terminate_all = mtk_cqdma_terminate_all;
776	dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
777	dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
778	dd->directions = BIT(DMA_MEM_TO_MEM);
779	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
780	dd->dev = &pdev->dev;
781	INIT_LIST_HEAD(&dd->channels);
782
783	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
784						      "dma-requests",
785						      &cqdma->dma_requests)) {
786		dev_info(&pdev->dev,
787			 "Using %u as missing dma-requests property\n",
788			 MTK_CQDMA_NR_VCHANS);
789
790		cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
791	}
792
793	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
794						      "dma-channels",
795						      &cqdma->dma_channels)) {
796		dev_info(&pdev->dev,
797			 "Using %u as missing dma-channels property\n",
798			 MTK_CQDMA_NR_PCHANS);
799
800		cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
801	}
802
803	cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
804				 sizeof(*cqdma->pc), GFP_KERNEL);
805	if (!cqdma->pc)
806		return -ENOMEM;
807
808	/* initialization for PCs */
809	for (i = 0; i < cqdma->dma_channels; ++i) {
810		cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
811					    sizeof(**cqdma->pc), GFP_KERNEL);
812		if (!cqdma->pc[i])
813			return -ENOMEM;
814
815		INIT_LIST_HEAD(&cqdma->pc[i]->queue);
816		spin_lock_init(&cqdma->pc[i]->lock);
817		refcount_set(&cqdma->pc[i]->refcnt, 0);
818		cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
819		if (IS_ERR(cqdma->pc[i]->base))
820			return PTR_ERR(cqdma->pc[i]->base);
821
822		/* allocate IRQ resource */
823		err = platform_get_irq(pdev, i);
824		if (err < 0)
825			return err;
826		cqdma->pc[i]->irq = err;
827
828		err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
829				       mtk_cqdma_irq, 0, dev_name(&pdev->dev),
830				       cqdma);
831		if (err) {
832			dev_err(&pdev->dev,
833				"request_irq failed with err %d\n", err);
834			return -EINVAL;
835		}
836	}
837
838	/* allocate resource for VCs */
839	cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
840				 sizeof(*cqdma->vc), GFP_KERNEL);
841	if (!cqdma->vc)
842		return -ENOMEM;
843
844	for (i = 0; i < cqdma->dma_requests; i++) {
845		vc = &cqdma->vc[i];
846		vc->vc.desc_free = mtk_cqdma_vdesc_free;
847		vchan_init(&vc->vc, dd);
848		init_completion(&vc->issue_completion);
849	}
850
851	err = dma_async_device_register(dd);
852	if (err)
853		return err;
854
855	err = of_dma_controller_register(pdev->dev.of_node,
856					 of_dma_xlate_by_chan_id, cqdma);
857	if (err) {
858		dev_err(&pdev->dev,
859			"MediaTek CQDMA OF registration failed %d\n", err);
860		goto err_unregister;
861	}
862
863	err = mtk_cqdma_hw_init(cqdma);
864	if (err) {
865		dev_err(&pdev->dev,
866			"MediaTek CQDMA HW initialization failed %d\n", err);
867		goto err_unregister;
868	}
869
870	platform_set_drvdata(pdev, cqdma);
871
872	/* initialize tasklet for each PC */
873	for (i = 0; i < cqdma->dma_channels; ++i)
874		tasklet_setup(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb);
875
876	dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");
877
878	return 0;
879
880err_unregister:
881	dma_async_device_unregister(dd);
882
883	return err;
884}
885
886static void mtk_cqdma_remove(struct platform_device *pdev)
887{
888	struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
889	struct mtk_cqdma_vchan *vc;
890	unsigned long flags;
891	int i;
892
893	/* kill VC task */
894	for (i = 0; i < cqdma->dma_requests; i++) {
895		vc = &cqdma->vc[i];
896
897		list_del(&vc->vc.chan.device_node);
898		tasklet_kill(&vc->vc.task);
899	}
900
901	/* disable interrupt */
902	for (i = 0; i < cqdma->dma_channels; i++) {
903		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
904		mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
905			    MTK_CQDMA_INT_EN_BIT);
906		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
907
908		/* Waits for any pending IRQ handlers to complete */
909		synchronize_irq(cqdma->pc[i]->irq);
910
911		tasklet_kill(&cqdma->pc[i]->tasklet);
912	}
913
914	/* disable hardware */
915	mtk_cqdma_hw_deinit(cqdma);
916
917	dma_async_device_unregister(&cqdma->ddev);
918	of_dma_controller_free(pdev->dev.of_node);
919}
920
921static struct platform_driver mtk_cqdma_driver = {
922	.probe = mtk_cqdma_probe,
923	.remove = mtk_cqdma_remove,
924	.driver = {
925		.name           = KBUILD_MODNAME,
926		.of_match_table = mtk_cqdma_match,
927	},
928};
929module_platform_driver(mtk_cqdma_driver);
930
931MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
932MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
933MODULE_LICENSE("GPL v2");