arm-dma350.c

  1// SPDX-License-Identifier: GPL-2.0
  2// Copyright (C) 2024-2025 Arm Limited
  3// Arm DMA-350 driver
  4
  5#include <linux/bitfield.h>
  6#include <linux/dmaengine.h>
  7#include <linux/dma-mapping.h>
  8#include <linux/io.h>
  9#include <linux/of.h>
 10#include <linux/module.h>
 11#include <linux/platform_device.h>
 12
 13#include "dmaengine.h"
 14#include "virt-dma.h"
 15
 16#define DMAINFO			0x0f00
 17
 18#define DMA_BUILDCFG0		0xb0
 19#define DMA_CFG_DATA_WIDTH	GENMASK(18, 16)
 20#define DMA_CFG_ADDR_WIDTH	GENMASK(15, 10)
 21#define DMA_CFG_NUM_CHANNELS	GENMASK(9, 4)
 22
 23#define DMA_BUILDCFG1		0xb4
 24#define DMA_CFG_NUM_TRIGGER_IN	GENMASK(8, 0)
 25
 26#define IIDR			0xc8
 27#define IIDR_PRODUCTID		GENMASK(31, 20)
 28#define IIDR_VARIANT		GENMASK(19, 16)
 29#define IIDR_REVISION		GENMASK(15, 12)
 30#define IIDR_IMPLEMENTER	GENMASK(11, 0)
 31
 32#define PRODUCTID_DMA350	0x3a0
 33#define IMPLEMENTER_ARM		0x43b
 34
 35#define DMACH(n)		(0x1000 + 0x0100 * (n))
 36
 37#define CH_CMD			0x00
 38#define CH_CMD_RESUME		BIT(5)
 39#define CH_CMD_PAUSE		BIT(4)
 40#define CH_CMD_STOP		BIT(3)
 41#define CH_CMD_DISABLE		BIT(2)
 42#define CH_CMD_CLEAR		BIT(1)
 43#define CH_CMD_ENABLE		BIT(0)
 44
 45#define CH_STATUS		0x04
 46#define CH_STAT_RESUMEWAIT	BIT(21)
 47#define CH_STAT_PAUSED		BIT(20)
 48#define CH_STAT_STOPPED		BIT(19)
 49#define CH_STAT_DISABLED	BIT(18)
 50#define CH_STAT_ERR		BIT(17)
 51#define CH_STAT_DONE		BIT(16)
 52#define CH_STAT_INTR_ERR	BIT(1)
 53#define CH_STAT_INTR_DONE	BIT(0)
 54
 55#define CH_INTREN		0x08
 56#define CH_INTREN_ERR		BIT(1)
 57#define CH_INTREN_DONE		BIT(0)
 58
 59#define CH_CTRL			0x0c
 60#define CH_CTRL_USEDESTRIGIN	BIT(26)
 61#define CH_CTRL_USESRCTRIGIN	BIT(26)
 62#define CH_CTRL_DONETYPE	GENMASK(23, 21)
 63#define CH_CTRL_REGRELOADTYPE	GENMASK(20, 18)
 64#define CH_CTRL_XTYPE		GENMASK(11, 9)
 65#define CH_CTRL_TRANSIZE	GENMASK(2, 0)
 66
 67#define CH_SRCADDR		0x10
 68#define CH_SRCADDRHI		0x14
 69#define CH_DESADDR		0x18
 70#define CH_DESADDRHI		0x1c
 71#define CH_XSIZE		0x20
 72#define CH_XSIZEHI		0x24
 73#define CH_SRCTRANSCFG		0x28
 74#define CH_DESTRANSCFG		0x2c
 75#define CH_CFG_MAXBURSTLEN	GENMASK(19, 16)
 76#define CH_CFG_PRIVATTR		BIT(11)
 77#define CH_CFG_SHAREATTR	GENMASK(9, 8)
 78#define CH_CFG_MEMATTR		GENMASK(7, 0)
 79
 80#define TRANSCFG_DEVICE					\
 81	FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) |		\
 82	FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) |	\
 83	FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_DEVICE)
 84#define TRANSCFG_NC					\
 85	FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) |		\
 86	FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) |	\
 87	FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_NC)
 88#define TRANSCFG_WB					\
 89	FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) |		\
 90	FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_ISH) |	\
 91	FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_WB)
 92
 93#define CH_XADDRINC		0x30
 94#define CH_XY_DES		GENMASK(31, 16)
 95#define CH_XY_SRC		GENMASK(15, 0)
 96
 97#define CH_FILLVAL		0x38
 98#define CH_SRCTRIGINCFG		0x4c
 99#define CH_DESTRIGINCFG		0x50
100#define CH_LINKATTR		0x70
101#define CH_LINK_SHAREATTR	GENMASK(9, 8)
102#define CH_LINK_MEMATTR		GENMASK(7, 0)
103
104#define CH_AUTOCFG		0x74
105#define CH_LINKADDR		0x78
106#define CH_LINKADDR_EN		BIT(0)
107
108#define CH_LINKADDRHI		0x7c
109#define CH_ERRINFO		0x90
110#define CH_ERRINFO_AXIRDPOISERR BIT(18)
111#define CH_ERRINFO_AXIWRRESPERR BIT(17)
112#define CH_ERRINFO_AXIRDRESPERR BIT(16)
113
114#define CH_BUILDCFG0		0xf8
115#define CH_CFG_INC_WIDTH	GENMASK(29, 26)
116#define CH_CFG_DATA_WIDTH	GENMASK(24, 22)
117#define CH_CFG_DATA_BUF_SIZE	GENMASK(7, 0)
118
119#define CH_BUILDCFG1		0xfc
120#define CH_CFG_HAS_CMDLINK	BIT(8)
121#define CH_CFG_HAS_TRIGSEL	BIT(7)
122#define CH_CFG_HAS_TRIGIN	BIT(5)
123#define CH_CFG_HAS_WRAP		BIT(1)
124
125
126#define LINK_REGCLEAR		BIT(0)
127#define LINK_INTREN		BIT(2)
128#define LINK_CTRL		BIT(3)
129#define LINK_SRCADDR		BIT(4)
130#define LINK_SRCADDRHI		BIT(5)
131#define LINK_DESADDR		BIT(6)
132#define LINK_DESADDRHI		BIT(7)
133#define LINK_XSIZE		BIT(8)
134#define LINK_XSIZEHI		BIT(9)
135#define LINK_SRCTRANSCFG	BIT(10)
136#define LINK_DESTRANSCFG	BIT(11)
137#define LINK_XADDRINC		BIT(12)
138#define LINK_FILLVAL		BIT(14)
139#define LINK_SRCTRIGINCFG	BIT(19)
140#define LINK_DESTRIGINCFG	BIT(20)
141#define LINK_AUTOCFG		BIT(29)
142#define LINK_LINKADDR		BIT(30)
143#define LINK_LINKADDRHI		BIT(31)
144
145
146enum ch_ctrl_donetype {
147	CH_CTRL_DONETYPE_NONE = 0,
148	CH_CTRL_DONETYPE_CMD = 1,
149	CH_CTRL_DONETYPE_CYCLE = 3
150};
151
152enum ch_ctrl_xtype {
153	CH_CTRL_XTYPE_DISABLE = 0,
154	CH_CTRL_XTYPE_CONTINUE = 1,
155	CH_CTRL_XTYPE_WRAP = 2,
156	CH_CTRL_XTYPE_FILL = 3
157};
158
159enum ch_cfg_shareattr {
160	SHAREATTR_NSH = 0,
161	SHAREATTR_OSH = 2,
162	SHAREATTR_ISH = 3
163};
164
165enum ch_cfg_memattr {
166	MEMATTR_DEVICE = 0x00,
167	MEMATTR_NC = 0x44,
168	MEMATTR_WB = 0xff
169};
170
171struct d350_desc {
172	struct virt_dma_desc vd;
173	u32 command[16];
174	u16 xsize;
175	u16 xsizehi;
176	u8 tsz;
177};
178
179struct d350_chan {
180	struct virt_dma_chan vc;
181	struct d350_desc *desc;
182	void __iomem *base;
183	int irq;
184	enum dma_status status;
185	dma_cookie_t cookie;
186	u32 residue;
187	u8 tsz;
188	bool has_trig;
189	bool has_wrap;
190	bool coherent;
191};
192
193struct d350 {
194	struct dma_device dma;
195	int nchan;
196	int nreq;
197	struct d350_chan channels[] __counted_by(nchan);
198};
199
200static inline struct d350_chan *to_d350_chan(struct dma_chan *chan)
201{
202	return container_of(chan, struct d350_chan, vc.chan);
203}
204
205static inline struct d350_desc *to_d350_desc(struct virt_dma_desc *vd)
206{
207	return container_of(vd, struct d350_desc, vd);
208}
209
210static void d350_desc_free(struct virt_dma_desc *vd)
211{
212	kfree(to_d350_desc(vd));
213}
214
215static struct dma_async_tx_descriptor *d350_prep_memcpy(struct dma_chan *chan,
216		dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags)
217{
218	struct d350_chan *dch = to_d350_chan(chan);
219	struct d350_desc *desc;
220	u32 *cmd;
221
222	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
223	if (!desc)
224		return NULL;
225
226	desc->tsz = __ffs(len | dest | src | (1 << dch->tsz));
227	desc->xsize = lower_16_bits(len >> desc->tsz);
228	desc->xsizehi = upper_16_bits(len >> desc->tsz);
229
230	cmd = desc->command;
231	cmd[0] = LINK_CTRL | LINK_SRCADDR | LINK_SRCADDRHI | LINK_DESADDR |
232		 LINK_DESADDRHI | LINK_XSIZE | LINK_XSIZEHI | LINK_SRCTRANSCFG |
233		 LINK_DESTRANSCFG | LINK_XADDRINC | LINK_LINKADDR;
234
235	cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) |
236		 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_CONTINUE) |
237		 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD);
238
239	cmd[2] = lower_32_bits(src);
240	cmd[3] = upper_32_bits(src);
241	cmd[4] = lower_32_bits(dest);
242	cmd[5] = upper_32_bits(dest);
243	cmd[6] = FIELD_PREP(CH_XY_SRC, desc->xsize) | FIELD_PREP(CH_XY_DES, desc->xsize);
244	cmd[7] = FIELD_PREP(CH_XY_SRC, desc->xsizehi) | FIELD_PREP(CH_XY_DES, desc->xsizehi);
245	cmd[8] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
246	cmd[9] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
247	cmd[10] = FIELD_PREP(CH_XY_SRC, 1) | FIELD_PREP(CH_XY_DES, 1);
248	cmd[11] = 0;
249
250	return vchan_tx_prep(&dch->vc, &desc->vd, flags);
251}
252
253static struct dma_async_tx_descriptor *d350_prep_memset(struct dma_chan *chan,
254		dma_addr_t dest, int value, size_t len, unsigned long flags)
255{
256	struct d350_chan *dch = to_d350_chan(chan);
257	struct d350_desc *desc;
258	u32 *cmd;
259
260	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
261	if (!desc)
262		return NULL;
263
264	desc->tsz = __ffs(len | dest | (1 << dch->tsz));
265	desc->xsize = lower_16_bits(len >> desc->tsz);
266	desc->xsizehi = upper_16_bits(len >> desc->tsz);
267
268	cmd = desc->command;
269	cmd[0] = LINK_CTRL | LINK_DESADDR | LINK_DESADDRHI |
270		 LINK_XSIZE | LINK_XSIZEHI | LINK_DESTRANSCFG |
271		 LINK_XADDRINC | LINK_FILLVAL | LINK_LINKADDR;
272
273	cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) |
274		 FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_FILL) |
275		 FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD);
276
277	cmd[2] = lower_32_bits(dest);
278	cmd[3] = upper_32_bits(dest);
279	cmd[4] = FIELD_PREP(CH_XY_DES, desc->xsize);
280	cmd[5] = FIELD_PREP(CH_XY_DES, desc->xsizehi);
281	cmd[6] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC;
282	cmd[7] = FIELD_PREP(CH_XY_DES, 1);
283	cmd[8] = (u8)value * 0x01010101;
284	cmd[9] = 0;
285
286	return vchan_tx_prep(&dch->vc, &desc->vd, flags);
287}
288
289static int d350_pause(struct dma_chan *chan)
290{
291	struct d350_chan *dch = to_d350_chan(chan);
292	unsigned long flags;
293
294	spin_lock_irqsave(&dch->vc.lock, flags);
295	if (dch->status == DMA_IN_PROGRESS) {
296		writel_relaxed(CH_CMD_PAUSE, dch->base + CH_CMD);
297		dch->status = DMA_PAUSED;
298	}
299	spin_unlock_irqrestore(&dch->vc.lock, flags);
300
301	return 0;
302}
303
304static int d350_resume(struct dma_chan *chan)
305{
306	struct d350_chan *dch = to_d350_chan(chan);
307	unsigned long flags;
308
309	spin_lock_irqsave(&dch->vc.lock, flags);
310	if (dch->status == DMA_PAUSED) {
311		writel_relaxed(CH_CMD_RESUME, dch->base + CH_CMD);
312		dch->status = DMA_IN_PROGRESS;
313	}
314	spin_unlock_irqrestore(&dch->vc.lock, flags);
315
316	return 0;
317}
318
319static u32 d350_get_residue(struct d350_chan *dch)
320{
321	u32 res, xsize, xsizehi, hi_new;
322	int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */
323
324	hi_new = readl_relaxed(dch->base + CH_XSIZEHI);
325	do {
326		xsizehi = hi_new;
327		xsize = readl_relaxed(dch->base + CH_XSIZE);
328		hi_new = readl_relaxed(dch->base + CH_XSIZEHI);
329	} while (xsizehi != hi_new && --retries);
330
331	res = FIELD_GET(CH_XY_DES, xsize);
332	res |= FIELD_GET(CH_XY_DES, xsizehi) << 16;
333
334	return res << dch->desc->tsz;
335}
336
337static int d350_terminate_all(struct dma_chan *chan)
338{
339	struct d350_chan *dch = to_d350_chan(chan);
340	unsigned long flags;
341	LIST_HEAD(list);
342
343	spin_lock_irqsave(&dch->vc.lock, flags);
344	writel_relaxed(CH_CMD_STOP, dch->base + CH_CMD);
345	if (dch->desc) {
346		if (dch->status != DMA_ERROR)
347			vchan_terminate_vdesc(&dch->desc->vd);
348		dch->desc = NULL;
349		dch->status = DMA_COMPLETE;
350	}
351	vchan_get_all_descriptors(&dch->vc, &list);
352	list_splice_tail(&list, &dch->vc.desc_terminated);
353	spin_unlock_irqrestore(&dch->vc.lock, flags);
354
355	return 0;
356}
357
358static void d350_synchronize(struct dma_chan *chan)
359{
360	struct d350_chan *dch = to_d350_chan(chan);
361
362	vchan_synchronize(&dch->vc);
363}
364
365static u32 d350_desc_bytes(struct d350_desc *desc)
366{
367	return ((u32)desc->xsizehi << 16 | desc->xsize) << desc->tsz;
368}
369
370static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
371				      struct dma_tx_state *state)
372{
373	struct d350_chan *dch = to_d350_chan(chan);
374	struct virt_dma_desc *vd;
375	enum dma_status status;
376	unsigned long flags;
377	u32 residue = 0;
378
379	status = dma_cookie_status(chan, cookie, state);
380
381	spin_lock_irqsave(&dch->vc.lock, flags);
382	if (cookie == dch->cookie) {
383		status = dch->status;
384		if (status == DMA_IN_PROGRESS || status == DMA_PAUSED)
385			dch->residue = d350_get_residue(dch);
386		residue = dch->residue;
387	} else if ((vd = vchan_find_desc(&dch->vc, cookie))) {
388		residue = d350_desc_bytes(to_d350_desc(vd));
389	} else if (status == DMA_IN_PROGRESS) {
390		/* Somebody else terminated it? */
391		status = DMA_ERROR;
392	}
393	spin_unlock_irqrestore(&dch->vc.lock, flags);
394
395	dma_set_residue(state, residue);
396	return status;
397}
398
399static void d350_start_next(struct d350_chan *dch)
400{
401	u32 hdr, *reg;
402
403	dch->desc = to_d350_desc(vchan_next_desc(&dch->vc));
404	if (!dch->desc)
405		return;
406
407	list_del(&dch->desc->vd.node);
408	dch->status = DMA_IN_PROGRESS;
409	dch->cookie = dch->desc->vd.tx.cookie;
410	dch->residue = d350_desc_bytes(dch->desc);
411
412	hdr = dch->desc->command[0];
413	reg = &dch->desc->command[1];
414
415	if (hdr & LINK_INTREN)
416		writel_relaxed(*reg++, dch->base + CH_INTREN);
417	if (hdr & LINK_CTRL)
418		writel_relaxed(*reg++, dch->base + CH_CTRL);
419	if (hdr & LINK_SRCADDR)
420		writel_relaxed(*reg++, dch->base + CH_SRCADDR);
421	if (hdr & LINK_SRCADDRHI)
422		writel_relaxed(*reg++, dch->base + CH_SRCADDRHI);
423	if (hdr & LINK_DESADDR)
424		writel_relaxed(*reg++, dch->base + CH_DESADDR);
425	if (hdr & LINK_DESADDRHI)
426		writel_relaxed(*reg++, dch->base + CH_DESADDRHI);
427	if (hdr & LINK_XSIZE)
428		writel_relaxed(*reg++, dch->base + CH_XSIZE);
429	if (hdr & LINK_XSIZEHI)
430		writel_relaxed(*reg++, dch->base + CH_XSIZEHI);
431	if (hdr & LINK_SRCTRANSCFG)
432		writel_relaxed(*reg++, dch->base + CH_SRCTRANSCFG);
433	if (hdr & LINK_DESTRANSCFG)
434		writel_relaxed(*reg++, dch->base + CH_DESTRANSCFG);
435	if (hdr & LINK_XADDRINC)
436		writel_relaxed(*reg++, dch->base + CH_XADDRINC);
437	if (hdr & LINK_FILLVAL)
438		writel_relaxed(*reg++, dch->base + CH_FILLVAL);
439	if (hdr & LINK_SRCTRIGINCFG)
440		writel_relaxed(*reg++, dch->base + CH_SRCTRIGINCFG);
441	if (hdr & LINK_DESTRIGINCFG)
442		writel_relaxed(*reg++, dch->base + CH_DESTRIGINCFG);
443	if (hdr & LINK_AUTOCFG)
444		writel_relaxed(*reg++, dch->base + CH_AUTOCFG);
445	if (hdr & LINK_LINKADDR)
446		writel_relaxed(*reg++, dch->base + CH_LINKADDR);
447	if (hdr & LINK_LINKADDRHI)
448		writel_relaxed(*reg++, dch->base + CH_LINKADDRHI);
449
450	writel(CH_CMD_ENABLE, dch->base + CH_CMD);
451}
452
453static void d350_issue_pending(struct dma_chan *chan)
454{
455	struct d350_chan *dch = to_d350_chan(chan);
456	unsigned long flags;
457
458	spin_lock_irqsave(&dch->vc.lock, flags);
459	if (vchan_issue_pending(&dch->vc) && !dch->desc)
460		d350_start_next(dch);
461	spin_unlock_irqrestore(&dch->vc.lock, flags);
462}
463
464static irqreturn_t d350_irq(int irq, void *data)
465{
466	struct d350_chan *dch = data;
467	struct device *dev = dch->vc.chan.device->dev;
468	struct virt_dma_desc *vd = &dch->desc->vd;
469	u32 ch_status;
470
471	ch_status = readl(dch->base + CH_STATUS);
472	if (!ch_status)
473		return IRQ_NONE;
474
475	if (ch_status & CH_STAT_INTR_ERR) {
476		u32 errinfo = readl_relaxed(dch->base + CH_ERRINFO);
477
478		if (errinfo & (CH_ERRINFO_AXIRDPOISERR | CH_ERRINFO_AXIRDRESPERR))
479			vd->tx_result.result = DMA_TRANS_READ_FAILED;
480		else if (errinfo & CH_ERRINFO_AXIWRRESPERR)
481			vd->tx_result.result = DMA_TRANS_WRITE_FAILED;
482		else
483			vd->tx_result.result = DMA_TRANS_ABORTED;
484
485		vd->tx_result.residue = d350_get_residue(dch);
486	} else if (!(ch_status & CH_STAT_INTR_DONE)) {
487		dev_warn(dev, "Unexpected IRQ source? 0x%08x\n", ch_status);
488	}
489	writel_relaxed(ch_status, dch->base + CH_STATUS);
490
491	spin_lock(&dch->vc.lock);
492	vchan_cookie_complete(vd);
493	if (ch_status & CH_STAT_INTR_DONE) {
494		dch->status = DMA_COMPLETE;
495		dch->residue = 0;
496		d350_start_next(dch);
497	} else {
498		dch->status = DMA_ERROR;
499		dch->residue = vd->tx_result.residue;
500	}
501	spin_unlock(&dch->vc.lock);
502
503	return IRQ_HANDLED;
504}
505
506static int d350_alloc_chan_resources(struct dma_chan *chan)
507{
508	struct d350_chan *dch = to_d350_chan(chan);
509	int ret = request_irq(dch->irq, d350_irq, IRQF_SHARED,
510			      dev_name(&dch->vc.chan.dev->device), dch);
511	if (!ret)
512		writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN);
513
514	return ret;
515}
516
517static void d350_free_chan_resources(struct dma_chan *chan)
518{
519	struct d350_chan *dch = to_d350_chan(chan);
520
521	writel_relaxed(0, dch->base + CH_INTREN);
522	free_irq(dch->irq, dch);
523	vchan_free_chan_resources(&dch->vc);
524}
525
526static int d350_probe(struct platform_device *pdev)
527{
528	struct device *dev = &pdev->dev;
529	struct d350 *dmac;
530	void __iomem *base;
531	u32 reg;
532	int ret, nchan, dw, aw, r, p;
533	bool coherent, memset;
534
535	base = devm_platform_ioremap_resource(pdev, 0);
536	if (IS_ERR(base))
537		return PTR_ERR(base);
538
539	reg = readl_relaxed(base + DMAINFO + IIDR);
540	r = FIELD_GET(IIDR_VARIANT, reg);
541	p = FIELD_GET(IIDR_REVISION, reg);
542	if (FIELD_GET(IIDR_IMPLEMENTER, reg) != IMPLEMENTER_ARM ||
543	    FIELD_GET(IIDR_PRODUCTID, reg) != PRODUCTID_DMA350)
544		return dev_err_probe(dev, -ENODEV, "Not a DMA-350!");
545
546	reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG0);
547	nchan = FIELD_GET(DMA_CFG_NUM_CHANNELS, reg) + 1;
548	dw = 1 << FIELD_GET(DMA_CFG_DATA_WIDTH, reg);
549	aw = FIELD_GET(DMA_CFG_ADDR_WIDTH, reg) + 1;
550
551	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aw));
552	coherent = device_get_dma_attr(dev) == DEV_DMA_COHERENT;
553
554	dmac = devm_kzalloc(dev, struct_size(dmac, channels, nchan), GFP_KERNEL);
555	if (!dmac)
556		return -ENOMEM;
557
558	dmac->nchan = nchan;
559
560	reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG1);
561	dmac->nreq = FIELD_GET(DMA_CFG_NUM_TRIGGER_IN, reg);
562
563	dev_dbg(dev, "DMA-350 r%dp%d with %d channels, %d requests\n", r, p, dmac->nchan, dmac->nreq);
564
565	dmac->dma.dev = dev;
566	for (int i = min(dw, 16); i > 0; i /= 2) {
567		dmac->dma.src_addr_widths |= BIT(i);
568		dmac->dma.dst_addr_widths |= BIT(i);
569	}
570	dmac->dma.directions = BIT(DMA_MEM_TO_MEM);
571	dmac->dma.descriptor_reuse = true;
572	dmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
573	dmac->dma.device_alloc_chan_resources = d350_alloc_chan_resources;
574	dmac->dma.device_free_chan_resources = d350_free_chan_resources;
575	dma_cap_set(DMA_MEMCPY, dmac->dma.cap_mask);
576	dmac->dma.device_prep_dma_memcpy = d350_prep_memcpy;
577	dmac->dma.device_pause = d350_pause;
578	dmac->dma.device_resume = d350_resume;
579	dmac->dma.device_terminate_all = d350_terminate_all;
580	dmac->dma.device_synchronize = d350_synchronize;
581	dmac->dma.device_tx_status = d350_tx_status;
582	dmac->dma.device_issue_pending = d350_issue_pending;
583	INIT_LIST_HEAD(&dmac->dma.channels);
584
585	/* Would be nice to have per-channel caps for this... */
586	memset = true;
587	for (int i = 0; i < nchan; i++) {
588		struct d350_chan *dch = &dmac->channels[i];
589
590		dch->base = base + DMACH(i);
591		writel_relaxed(CH_CMD_CLEAR, dch->base + CH_CMD);
592
593		reg = readl_relaxed(dch->base + CH_BUILDCFG1);
594		if (!(FIELD_GET(CH_CFG_HAS_CMDLINK, reg))) {
595			dev_warn(dev, "No command link support on channel %d\n", i);
596			continue;
597		}
598		dch->irq = platform_get_irq(pdev, i);
599		if (dch->irq < 0)
600			return dev_err_probe(dev, dch->irq,
601					     "Failed to get IRQ for channel %d\n", i);
602
603		dch->has_wrap = FIELD_GET(CH_CFG_HAS_WRAP, reg);
604		dch->has_trig = FIELD_GET(CH_CFG_HAS_TRIGIN, reg) &
605				FIELD_GET(CH_CFG_HAS_TRIGSEL, reg);
606
607		/* Fill is a special case of Wrap */
608		memset &= dch->has_wrap;
609
610		reg = readl_relaxed(dch->base + CH_BUILDCFG0);
611		dch->tsz = FIELD_GET(CH_CFG_DATA_WIDTH, reg);
612
613		reg = FIELD_PREP(CH_LINK_SHAREATTR, coherent ? SHAREATTR_ISH : SHAREATTR_OSH);
614		reg |= FIELD_PREP(CH_LINK_MEMATTR, coherent ? MEMATTR_WB : MEMATTR_NC);
615		writel_relaxed(reg, dch->base + CH_LINKATTR);
616
617		dch->vc.desc_free = d350_desc_free;
618		vchan_init(&dch->vc, &dmac->dma);
619	}
620
621	if (memset) {
622		dma_cap_set(DMA_MEMSET, dmac->dma.cap_mask);
623		dmac->dma.device_prep_dma_memset = d350_prep_memset;
624	}
625
626	platform_set_drvdata(pdev, dmac);
627
628	ret = dma_async_device_register(&dmac->dma);
629	if (ret)
630		return dev_err_probe(dev, ret, "Failed to register DMA device\n");
631
632	return 0;
633}
634
635static void d350_remove(struct platform_device *pdev)
636{
637	struct d350 *dmac = platform_get_drvdata(pdev);
638
639	dma_async_device_unregister(&dmac->dma);
640}
641
642static const struct of_device_id d350_of_match[] __maybe_unused = {
643	{ .compatible = "arm,dma-350" },
644	{}
645};
646MODULE_DEVICE_TABLE(of, d350_of_match);
647
648static struct platform_driver d350_driver = {
649	.driver = {
650		.name = "arm-dma350",
651		.of_match_table = of_match_ptr(d350_of_match),
652	},
653	.probe = d350_probe,
654	.remove = d350_remove,
655};
656module_platform_driver(d350_driver);
657
658MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
659MODULE_DESCRIPTION("Arm DMA-350 driver");
660MODULE_LICENSE("GPL v2");