sun6i-dma.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) 2013-2014 Allwinner Tech Co., Ltd
   4 * Author: Sugar <shuge@allwinnertech.com>
   5 *
   6 * Copyright (C) 2014 Maxime Ripard
   7 * Maxime Ripard <maxime.ripard@free-electrons.com>
   8 */
   9
  10#include <linux/clk.h>
  11#include <linux/delay.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/dmaengine.h>
  14#include <linux/dmapool.h>
  15#include <linux/interrupt.h>
  16#include <linux/module.h>
  17#include <linux/of.h>
  18#include <linux/of_dma.h>
  19#include <linux/platform_device.h>
  20#include <linux/reset.h>
  21#include <linux/slab.h>
  22#include <linux/string_choices.h>
  23#include <linux/types.h>
  24
  25#include "virt-dma.h"
  26
  27/*
  28 * Common registers
  29 */
  30#define DMA_IRQ_EN(x)		((x) * 0x04)
  31#define DMA_IRQ_HALF			BIT(0)
  32#define DMA_IRQ_PKG			BIT(1)
  33#define DMA_IRQ_QUEUE			BIT(2)
  34
  35#define DMA_IRQ_CHAN_NR			8
  36#define DMA_IRQ_CHAN_WIDTH		4
  37
  38
  39#define DMA_IRQ_STAT(x)		((x) * 0x04 + 0x10)
  40
  41#define DMA_STAT		0x30
  42
  43/* Offset between DMA_IRQ_EN and DMA_IRQ_STAT limits number of channels */
  44#define DMA_MAX_CHANNELS	(DMA_IRQ_CHAN_NR * 0x10 / 4)
  45
  46/*
  47 * sun8i specific registers
  48 */
  49#define SUN8I_DMA_GATE		0x20
  50#define SUN8I_DMA_GATE_ENABLE	0x4
  51
  52#define SUNXI_H3_SECURE_REG		0x20
  53#define SUNXI_H3_DMA_GATE		0x28
  54#define SUNXI_H3_DMA_GATE_ENABLE	0x4
  55/*
  56 * Channels specific registers
  57 */
  58#define DMA_CHAN_ENABLE		0x00
  59#define DMA_CHAN_ENABLE_START		BIT(0)
  60#define DMA_CHAN_ENABLE_STOP		0
  61
  62#define DMA_CHAN_PAUSE		0x04
  63#define DMA_CHAN_PAUSE_PAUSE		BIT(1)
  64#define DMA_CHAN_PAUSE_RESUME		0
  65
  66#define DMA_CHAN_LLI_ADDR	0x08
  67
  68#define DMA_CHAN_CUR_CFG	0x0c
  69#define DMA_CHAN_MAX_DRQ_A31		0x1f
  70#define DMA_CHAN_MAX_DRQ_H6		0x3f
  71#define DMA_CHAN_CFG_SRC_DRQ_A31(x)	((x) & DMA_CHAN_MAX_DRQ_A31)
  72#define DMA_CHAN_CFG_SRC_DRQ_H6(x)	((x) & DMA_CHAN_MAX_DRQ_H6)
  73#define DMA_CHAN_CFG_SRC_MODE_A31(x)	(((x) & 0x1) << 5)
  74#define DMA_CHAN_CFG_SRC_MODE_H6(x)	(((x) & 0x1) << 8)
  75#define DMA_CHAN_CFG_SRC_BURST_A31(x)	(((x) & 0x3) << 7)
  76#define DMA_CHAN_CFG_SRC_BURST_H3(x)	(((x) & 0x3) << 6)
  77#define DMA_CHAN_CFG_SRC_WIDTH(x)	(((x) & 0x3) << 9)
  78
  79#define DMA_CHAN_CFG_DST_DRQ_A31(x)	(DMA_CHAN_CFG_SRC_DRQ_A31(x) << 16)
  80#define DMA_CHAN_CFG_DST_DRQ_H6(x)	(DMA_CHAN_CFG_SRC_DRQ_H6(x) << 16)
  81#define DMA_CHAN_CFG_DST_MODE_A31(x)	(DMA_CHAN_CFG_SRC_MODE_A31(x) << 16)
  82#define DMA_CHAN_CFG_DST_MODE_H6(x)	(DMA_CHAN_CFG_SRC_MODE_H6(x) << 16)
  83#define DMA_CHAN_CFG_DST_BURST_A31(x)	(DMA_CHAN_CFG_SRC_BURST_A31(x) << 16)
  84#define DMA_CHAN_CFG_DST_BURST_H3(x)	(DMA_CHAN_CFG_SRC_BURST_H3(x) << 16)
  85#define DMA_CHAN_CFG_DST_WIDTH(x)	(DMA_CHAN_CFG_SRC_WIDTH(x) << 16)
  86
  87#define DMA_CHAN_CUR_SRC	0x10
  88
  89#define DMA_CHAN_CUR_DST	0x14
  90
  91#define DMA_CHAN_CUR_CNT	0x18
  92
  93#define DMA_CHAN_CUR_PARA	0x1c
  94
  95/*
  96 * LLI address mangling
  97 *
  98 * The LLI link physical address is also mangled, but we avoid dealing
  99 * with that by allocating LLIs from the DMA32 zone.
 100 */
 101#define SRC_HIGH_ADDR(x)		(((x) & 0x3U) << 16)
 102#define DST_HIGH_ADDR(x)		(((x) & 0x3U) << 18)
 103
 104/*
 105 * Various hardware related defines
 106 */
 107#define LLI_LAST_ITEM	0xfffff800
 108#define NORMAL_WAIT	8
 109#define DRQ_SDRAM	1
 110#define LINEAR_MODE     0
 111#define IO_MODE         1
 112
 113/* forward declaration */
 114struct sun6i_dma_dev;
 115
 116/*
 117 * Hardware channels / ports representation
 118 *
 119 * The hardware is used in several SoCs, with differing numbers
 120 * of channels and endpoints. This structure ties those numbers
 121 * to a certain compatible string.
 122 */
 123struct sun6i_dma_config {
 124	u32 nr_max_channels;
 125	u32 nr_max_requests;
 126	u32 nr_max_vchans;
 127	/*
 128	 * In the datasheets/user manuals of newer Allwinner SoCs, a special
 129	 * bit (bit 2 at register 0x20) is present.
 130	 * It's named "DMA MCLK interface circuit auto gating bit" in the
 131	 * documents, and the footnote of this register says that this bit
 132	 * should be set up when initializing the DMA controller.
 133	 * Allwinner A23/A33 user manuals do not have this bit documented,
 134	 * however these SoCs really have and need this bit, as seen in the
 135	 * BSP kernel source code.
 136	 */
 137	void (*clock_autogate_enable)(struct sun6i_dma_dev *);
 138	void (*set_burst_length)(u32 *p_cfg, s8 src_burst, s8 dst_burst);
 139	void (*set_drq)(u32 *p_cfg, s8 src_drq, s8 dst_drq);
 140	void (*set_mode)(u32 *p_cfg, s8 src_mode, s8 dst_mode);
 141	u32 src_burst_lengths;
 142	u32 dst_burst_lengths;
 143	u32 src_addr_widths;
 144	u32 dst_addr_widths;
 145	bool has_high_addr;
 146	bool has_mbus_clk;
 147};
 148
 149/*
 150 * Hardware representation of the LLI
 151 *
 152 * The hardware will be fed the physical address of this structure,
 153 * and read its content in order to start the transfer.
 154 */
 155struct sun6i_dma_lli {
 156	u32			cfg;
 157	u32			src;
 158	u32			dst;
 159	u32			len;
 160	u32			para;
 161	u32			p_lli_next;
 162
 163	/*
 164	 * This field is not used by the DMA controller, but will be
 165	 * used by the CPU to go through the list (mostly for dumping
 166	 * or freeing it).
 167	 */
 168	struct sun6i_dma_lli	*v_lli_next;
 169};
 170
 171
 172struct sun6i_desc {
 173	struct virt_dma_desc	vd;
 174	dma_addr_t		p_lli;
 175	struct sun6i_dma_lli	*v_lli;
 176};
 177
 178struct sun6i_pchan {
 179	u32			idx;
 180	void __iomem		*base;
 181	struct sun6i_vchan	*vchan;
 182	struct sun6i_desc	*desc;
 183	struct sun6i_desc	*done;
 184};
 185
 186struct sun6i_vchan {
 187	struct virt_dma_chan	vc;
 188	struct list_head	node;
 189	struct dma_slave_config	cfg;
 190	struct sun6i_pchan	*phy;
 191	u8			port;
 192	u8			irq_type;
 193	bool			cyclic;
 194};
 195
 196struct sun6i_dma_dev {
 197	struct dma_device	slave;
 198	void __iomem		*base;
 199	struct clk		*clk;
 200	struct clk		*clk_mbus;
 201	int			irq;
 202	spinlock_t		lock;
 203	struct reset_control	*rstc;
 204	struct tasklet_struct	task;
 205	atomic_t		tasklet_shutdown;
 206	struct list_head	pending;
 207	struct dma_pool		*pool;
 208	struct sun6i_pchan	*pchans;
 209	struct sun6i_vchan	*vchans;
 210	const struct sun6i_dma_config *cfg;
 211	u32			num_pchans;
 212	u32			num_vchans;
 213	u32			max_request;
 214};
 215
 216static struct device *chan2dev(struct dma_chan *chan)
 217{
 218	return &chan->dev->device;
 219}
 220
 221static inline struct sun6i_dma_dev *to_sun6i_dma_dev(struct dma_device *d)
 222{
 223	return container_of(d, struct sun6i_dma_dev, slave);
 224}
 225
 226static inline struct sun6i_vchan *to_sun6i_vchan(struct dma_chan *chan)
 227{
 228	return container_of(chan, struct sun6i_vchan, vc.chan);
 229}
 230
 231static inline struct sun6i_desc *
 232to_sun6i_desc(struct dma_async_tx_descriptor *tx)
 233{
 234	return container_of(tx, struct sun6i_desc, vd.tx);
 235}
 236
 237static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev)
 238{
 239	dev_dbg(sdev->slave.dev, "Common register:\n"
 240		"\tmask0(%04x): 0x%08x\n"
 241		"\tmask1(%04x): 0x%08x\n"
 242		"\tpend0(%04x): 0x%08x\n"
 243		"\tpend1(%04x): 0x%08x\n"
 244		"\tstats(%04x): 0x%08x\n",
 245		DMA_IRQ_EN(0), readl(sdev->base + DMA_IRQ_EN(0)),
 246		DMA_IRQ_EN(1), readl(sdev->base + DMA_IRQ_EN(1)),
 247		DMA_IRQ_STAT(0), readl(sdev->base + DMA_IRQ_STAT(0)),
 248		DMA_IRQ_STAT(1), readl(sdev->base + DMA_IRQ_STAT(1)),
 249		DMA_STAT, readl(sdev->base + DMA_STAT));
 250}
 251
 252static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev,
 253					    struct sun6i_pchan *pchan)
 254{
 255	dev_dbg(sdev->slave.dev, "Chan %d reg:\n"
 256		"\t___en(%04x): \t0x%08x\n"
 257		"\tpause(%04x): \t0x%08x\n"
 258		"\tstart(%04x): \t0x%08x\n"
 259		"\t__cfg(%04x): \t0x%08x\n"
 260		"\t__src(%04x): \t0x%08x\n"
 261		"\t__dst(%04x): \t0x%08x\n"
 262		"\tcount(%04x): \t0x%08x\n"
 263		"\t_para(%04x): \t0x%08x\n\n",
 264		pchan->idx,
 265		DMA_CHAN_ENABLE,
 266		readl(pchan->base + DMA_CHAN_ENABLE),
 267		DMA_CHAN_PAUSE,
 268		readl(pchan->base + DMA_CHAN_PAUSE),
 269		DMA_CHAN_LLI_ADDR,
 270		readl(pchan->base + DMA_CHAN_LLI_ADDR),
 271		DMA_CHAN_CUR_CFG,
 272		readl(pchan->base + DMA_CHAN_CUR_CFG),
 273		DMA_CHAN_CUR_SRC,
 274		readl(pchan->base + DMA_CHAN_CUR_SRC),
 275		DMA_CHAN_CUR_DST,
 276		readl(pchan->base + DMA_CHAN_CUR_DST),
 277		DMA_CHAN_CUR_CNT,
 278		readl(pchan->base + DMA_CHAN_CUR_CNT),
 279		DMA_CHAN_CUR_PARA,
 280		readl(pchan->base + DMA_CHAN_CUR_PARA));
 281}
 282
 283static inline s8 convert_burst(u32 maxburst)
 284{
 285	switch (maxburst) {
 286	case 1:
 287		return 0;
 288	case 4:
 289		return 1;
 290	case 8:
 291		return 2;
 292	case 16:
 293		return 3;
 294	default:
 295		return -EINVAL;
 296	}
 297}
 298
 299static inline s8 convert_buswidth(enum dma_slave_buswidth addr_width)
 300{
 301	return ilog2(addr_width);
 302}
 303
 304static void sun6i_enable_clock_autogate_a23(struct sun6i_dma_dev *sdev)
 305{
 306	writel(SUN8I_DMA_GATE_ENABLE, sdev->base + SUN8I_DMA_GATE);
 307}
 308
 309static void sun6i_enable_clock_autogate_h3(struct sun6i_dma_dev *sdev)
 310{
 311	writel(SUNXI_H3_DMA_GATE_ENABLE, sdev->base + SUNXI_H3_DMA_GATE);
 312}
 313
 314static void sun6i_set_burst_length_a31(u32 *p_cfg, s8 src_burst, s8 dst_burst)
 315{
 316	*p_cfg |= DMA_CHAN_CFG_SRC_BURST_A31(src_burst) |
 317		  DMA_CHAN_CFG_DST_BURST_A31(dst_burst);
 318}
 319
 320static void sun6i_set_burst_length_h3(u32 *p_cfg, s8 src_burst, s8 dst_burst)
 321{
 322	*p_cfg |= DMA_CHAN_CFG_SRC_BURST_H3(src_burst) |
 323		  DMA_CHAN_CFG_DST_BURST_H3(dst_burst);
 324}
 325
 326static void sun6i_set_drq_a31(u32 *p_cfg, s8 src_drq, s8 dst_drq)
 327{
 328	*p_cfg |= DMA_CHAN_CFG_SRC_DRQ_A31(src_drq) |
 329		  DMA_CHAN_CFG_DST_DRQ_A31(dst_drq);
 330}
 331
 332static void sun6i_set_drq_h6(u32 *p_cfg, s8 src_drq, s8 dst_drq)
 333{
 334	*p_cfg |= DMA_CHAN_CFG_SRC_DRQ_H6(src_drq) |
 335		  DMA_CHAN_CFG_DST_DRQ_H6(dst_drq);
 336}
 337
 338static void sun6i_set_mode_a31(u32 *p_cfg, s8 src_mode, s8 dst_mode)
 339{
 340	*p_cfg |= DMA_CHAN_CFG_SRC_MODE_A31(src_mode) |
 341		  DMA_CHAN_CFG_DST_MODE_A31(dst_mode);
 342}
 343
 344static void sun6i_set_mode_h6(u32 *p_cfg, s8 src_mode, s8 dst_mode)
 345{
 346	*p_cfg |= DMA_CHAN_CFG_SRC_MODE_H6(src_mode) |
 347		  DMA_CHAN_CFG_DST_MODE_H6(dst_mode);
 348}
 349
 350static size_t sun6i_get_chan_size(struct sun6i_pchan *pchan)
 351{
 352	struct sun6i_desc *txd = pchan->desc;
 353	struct sun6i_dma_lli *lli;
 354	size_t bytes;
 355	dma_addr_t pos;
 356
 357	pos = readl(pchan->base + DMA_CHAN_LLI_ADDR);
 358	bytes = readl(pchan->base + DMA_CHAN_CUR_CNT);
 359
 360	if (pos == LLI_LAST_ITEM)
 361		return bytes;
 362
 363	for (lli = txd->v_lli; lli; lli = lli->v_lli_next) {
 364		if (lli->p_lli_next == pos) {
 365			for (lli = lli->v_lli_next; lli; lli = lli->v_lli_next)
 366				bytes += lli->len;
 367			break;
 368		}
 369	}
 370
 371	return bytes;
 372}
 373
 374static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev,
 375			       struct sun6i_dma_lli *next,
 376			       dma_addr_t next_phy,
 377			       struct sun6i_desc *txd)
 378{
 379	if ((!prev && !txd) || !next)
 380		return NULL;
 381
 382	if (!prev) {
 383		txd->p_lli = next_phy;
 384		txd->v_lli = next;
 385	} else {
 386		prev->p_lli_next = next_phy;
 387		prev->v_lli_next = next;
 388	}
 389
 390	next->p_lli_next = LLI_LAST_ITEM;
 391	next->v_lli_next = NULL;
 392
 393	return next;
 394}
 395
 396static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan,
 397				      struct sun6i_dma_lli *v_lli,
 398				      dma_addr_t p_lli)
 399{
 400	dev_dbg(chan2dev(&vchan->vc.chan),
 401		"\n\tdesc:\tp - %pad v - 0x%p\n"
 402		"\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n"
 403		"\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n",
 404		&p_lli, v_lli,
 405		v_lli->cfg, v_lli->src, v_lli->dst,
 406		v_lli->len, v_lli->para, v_lli->p_lli_next);
 407}
 408
 409static void sun6i_dma_free_desc(struct virt_dma_desc *vd)
 410{
 411	struct sun6i_desc *txd = to_sun6i_desc(&vd->tx);
 412	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vd->tx.chan->device);
 413	struct sun6i_dma_lli *v_lli, *v_next;
 414	dma_addr_t p_lli, p_next;
 415
 416	if (unlikely(!txd))
 417		return;
 418
 419	p_lli = txd->p_lli;
 420	v_lli = txd->v_lli;
 421
 422	while (v_lli) {
 423		v_next = v_lli->v_lli_next;
 424		p_next = v_lli->p_lli_next;
 425
 426		dma_pool_free(sdev->pool, v_lli, p_lli);
 427
 428		v_lli = v_next;
 429		p_lli = p_next;
 430	}
 431
 432	kfree(txd);
 433}
 434
 435static int sun6i_dma_start_desc(struct sun6i_vchan *vchan)
 436{
 437	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(vchan->vc.chan.device);
 438	struct virt_dma_desc *desc = vchan_next_desc(&vchan->vc);
 439	struct sun6i_pchan *pchan = vchan->phy;
 440	u32 irq_val, irq_reg, irq_offset;
 441
 442	if (!pchan)
 443		return -EAGAIN;
 444
 445	if (!desc) {
 446		pchan->desc = NULL;
 447		pchan->done = NULL;
 448		return -EAGAIN;
 449	}
 450
 451	list_del(&desc->node);
 452
 453	pchan->desc = to_sun6i_desc(&desc->tx);
 454	pchan->done = NULL;
 455
 456	sun6i_dma_dump_lli(vchan, pchan->desc->v_lli, pchan->desc->p_lli);
 457
 458	irq_reg = pchan->idx / DMA_IRQ_CHAN_NR;
 459	irq_offset = pchan->idx % DMA_IRQ_CHAN_NR;
 460
 461	vchan->irq_type = vchan->cyclic ? DMA_IRQ_PKG : DMA_IRQ_QUEUE;
 462
 463	irq_val = readl(sdev->base + DMA_IRQ_EN(irq_reg));
 464	irq_val &= ~((DMA_IRQ_HALF | DMA_IRQ_PKG | DMA_IRQ_QUEUE) <<
 465			(irq_offset * DMA_IRQ_CHAN_WIDTH));
 466	irq_val |= vchan->irq_type << (irq_offset * DMA_IRQ_CHAN_WIDTH);
 467	writel(irq_val, sdev->base + DMA_IRQ_EN(irq_reg));
 468
 469	writel(pchan->desc->p_lli, pchan->base + DMA_CHAN_LLI_ADDR);
 470	writel(DMA_CHAN_ENABLE_START, pchan->base + DMA_CHAN_ENABLE);
 471
 472	sun6i_dma_dump_com_regs(sdev);
 473	sun6i_dma_dump_chan_regs(sdev, pchan);
 474
 475	return 0;
 476}
 477
 478static void sun6i_dma_tasklet(struct tasklet_struct *t)
 479{
 480	struct sun6i_dma_dev *sdev = from_tasklet(sdev, t, task);
 481	struct sun6i_vchan *vchan;
 482	struct sun6i_pchan *pchan;
 483	unsigned int pchan_alloc = 0;
 484	unsigned int pchan_idx;
 485
 486	list_for_each_entry(vchan, &sdev->slave.channels, vc.chan.device_node) {
 487		spin_lock_irq(&vchan->vc.lock);
 488
 489		pchan = vchan->phy;
 490
 491		if (pchan && pchan->done) {
 492			if (sun6i_dma_start_desc(vchan)) {
 493				/*
 494				 * No current txd associated with this channel
 495				 */
 496				dev_dbg(sdev->slave.dev, "pchan %u: free\n",
 497					pchan->idx);
 498
 499				/* Mark this channel free */
 500				vchan->phy = NULL;
 501				pchan->vchan = NULL;
 502			}
 503		}
 504		spin_unlock_irq(&vchan->vc.lock);
 505	}
 506
 507	spin_lock_irq(&sdev->lock);
 508	for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) {
 509		pchan = &sdev->pchans[pchan_idx];
 510
 511		if (pchan->vchan || list_empty(&sdev->pending))
 512			continue;
 513
 514		vchan = list_first_entry(&sdev->pending,
 515					 struct sun6i_vchan, node);
 516
 517		/* Remove from pending channels */
 518		list_del_init(&vchan->node);
 519		pchan_alloc |= BIT(pchan_idx);
 520
 521		/* Mark this channel allocated */
 522		pchan->vchan = vchan;
 523		vchan->phy = pchan;
 524		dev_dbg(sdev->slave.dev, "pchan %u: alloc vchan %p\n",
 525			pchan->idx, &vchan->vc);
 526	}
 527	spin_unlock_irq(&sdev->lock);
 528
 529	for (pchan_idx = 0; pchan_idx < sdev->num_pchans; pchan_idx++) {
 530		if (!(pchan_alloc & BIT(pchan_idx)))
 531			continue;
 532
 533		pchan = sdev->pchans + pchan_idx;
 534		vchan = pchan->vchan;
 535		if (vchan) {
 536			spin_lock_irq(&vchan->vc.lock);
 537			sun6i_dma_start_desc(vchan);
 538			spin_unlock_irq(&vchan->vc.lock);
 539		}
 540	}
 541}
 542
 543static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id)
 544{
 545	struct sun6i_dma_dev *sdev = dev_id;
 546	struct sun6i_vchan *vchan;
 547	struct sun6i_pchan *pchan;
 548	int i, j, ret = IRQ_NONE;
 549	u32 status;
 550
 551	for (i = 0; i < sdev->num_pchans / DMA_IRQ_CHAN_NR; i++) {
 552		status = readl(sdev->base + DMA_IRQ_STAT(i));
 553		if (!status)
 554			continue;
 555
 556		dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n",
 557			str_high_low(i), status);
 558
 559		writel(status, sdev->base + DMA_IRQ_STAT(i));
 560
 561		for (j = 0; (j < DMA_IRQ_CHAN_NR) && status; j++) {
 562			pchan = sdev->pchans + j;
 563			vchan = pchan->vchan;
 564			if (vchan && (status & vchan->irq_type)) {
 565				if (vchan->cyclic) {
 566					vchan_cyclic_callback(&pchan->desc->vd);
 567				} else {
 568					spin_lock(&vchan->vc.lock);
 569					vchan_cookie_complete(&pchan->desc->vd);
 570					pchan->done = pchan->desc;
 571					spin_unlock(&vchan->vc.lock);
 572				}
 573			}
 574
 575			status = status >> DMA_IRQ_CHAN_WIDTH;
 576		}
 577
 578		if (!atomic_read(&sdev->tasklet_shutdown))
 579			tasklet_schedule(&sdev->task);
 580		ret = IRQ_HANDLED;
 581	}
 582
 583	return ret;
 584}
 585
 586static u32 find_burst_size(const u32 burst_lengths, u32 maxburst)
 587{
 588	if (!maxburst)
 589		return 1;
 590
 591	if (BIT(maxburst) & burst_lengths)
 592		return maxburst;
 593
 594	/* Hardware only does power-of-two bursts. */
 595	for (u32 burst = rounddown_pow_of_two(maxburst); burst > 0; burst /= 2)
 596		if (BIT(burst) & burst_lengths)
 597			return burst;
 598
 599	return 1;
 600}
 601
 602static int set_config(struct sun6i_dma_dev *sdev,
 603			struct dma_slave_config *sconfig,
 604			enum dma_transfer_direction direction,
 605			u32 *p_cfg)
 606{
 607	enum dma_slave_buswidth src_addr_width, dst_addr_width;
 608	u32 src_maxburst, dst_maxburst;
 609	s8 src_width, dst_width, src_burst, dst_burst;
 610
 611	src_addr_width = sconfig->src_addr_width;
 612	dst_addr_width = sconfig->dst_addr_width;
 613	src_maxburst = sconfig->src_maxburst;
 614	dst_maxburst = sconfig->dst_maxburst;
 615
 616	switch (direction) {
 617	case DMA_MEM_TO_DEV:
 618		if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
 619			src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 620		src_maxburst = src_maxburst ? src_maxburst : 8;
 621		break;
 622	case DMA_DEV_TO_MEM:
 623		if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
 624			dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 625		dst_maxburst = dst_maxburst ? dst_maxburst : 8;
 626		break;
 627	default:
 628		return -EINVAL;
 629	}
 630
 631	if (!(BIT(src_addr_width) & sdev->slave.src_addr_widths))
 632		return -EINVAL;
 633	if (!(BIT(dst_addr_width) & sdev->slave.dst_addr_widths))
 634		return -EINVAL;
 635
 636	src_width = convert_buswidth(src_addr_width);
 637	dst_width = convert_buswidth(dst_addr_width);
 638	src_burst = find_burst_size(sdev->cfg->src_burst_lengths, src_maxburst);
 639	dst_burst = find_burst_size(sdev->cfg->dst_burst_lengths, dst_maxburst);
 640	dst_burst = convert_burst(dst_burst);
 641	src_burst = convert_burst(src_burst);
 642
 643	*p_cfg = DMA_CHAN_CFG_SRC_WIDTH(src_width) |
 644		DMA_CHAN_CFG_DST_WIDTH(dst_width);
 645
 646	sdev->cfg->set_burst_length(p_cfg, src_burst, dst_burst);
 647
 648	return 0;
 649}
 650
 651static inline void sun6i_dma_set_addr(struct sun6i_dma_dev *sdev,
 652				      struct sun6i_dma_lli *v_lli,
 653				      dma_addr_t src, dma_addr_t dst)
 654{
 655	v_lli->src = lower_32_bits(src);
 656	v_lli->dst = lower_32_bits(dst);
 657
 658	if (sdev->cfg->has_high_addr)
 659		v_lli->para |= SRC_HIGH_ADDR(upper_32_bits(src)) |
 660			       DST_HIGH_ADDR(upper_32_bits(dst));
 661}
 662
 663static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy(
 664		struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 665		size_t len, unsigned long flags)
 666{
 667	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 668	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 669	struct sun6i_dma_lli *v_lli;
 670	struct sun6i_desc *txd;
 671	dma_addr_t p_lli;
 672	s8 burst, width;
 673
 674	dev_dbg(chan2dev(chan),
 675		"%s; chan: %d, dest: %pad, src: %pad, len: %zu. flags: 0x%08lx\n",
 676		__func__, vchan->vc.chan.chan_id, &dest, &src, len, flags);
 677
 678	if (!len)
 679		return NULL;
 680
 681	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 682	if (!txd)
 683		return NULL;
 684
 685	v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli);
 686	if (!v_lli) {
 687		dev_err(sdev->slave.dev, "Failed to alloc lli memory\n");
 688		goto err_txd_free;
 689	}
 690
 691	v_lli->len = len;
 692	v_lli->para = NORMAL_WAIT;
 693	sun6i_dma_set_addr(sdev, v_lli, src, dest);
 694
 695	burst = convert_burst(8);
 696	width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES);
 697	v_lli->cfg = DMA_CHAN_CFG_SRC_WIDTH(width) |
 698		DMA_CHAN_CFG_DST_WIDTH(width);
 699
 700	sdev->cfg->set_burst_length(&v_lli->cfg, burst, burst);
 701	sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, DRQ_SDRAM);
 702	sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, LINEAR_MODE);
 703
 704	sun6i_dma_lli_add(NULL, v_lli, p_lli, txd);
 705
 706	sun6i_dma_dump_lli(vchan, v_lli, p_lli);
 707
 708	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 709
 710err_txd_free:
 711	kfree(txd);
 712	return NULL;
 713}
 714
 715static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg(
 716		struct dma_chan *chan, struct scatterlist *sgl,
 717		unsigned int sg_len, enum dma_transfer_direction dir,
 718		unsigned long flags, void *context)
 719{
 720	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 721	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 722	struct dma_slave_config *sconfig = &vchan->cfg;
 723	struct sun6i_dma_lli *v_lli, *prev = NULL;
 724	struct sun6i_desc *txd;
 725	struct scatterlist *sg;
 726	dma_addr_t p_lli;
 727	u32 lli_cfg;
 728	int i, ret;
 729
 730	if (!sgl)
 731		return NULL;
 732
 733	ret = set_config(sdev, sconfig, dir, &lli_cfg);
 734	if (ret) {
 735		dev_err(chan2dev(chan), "Invalid DMA configuration\n");
 736		return NULL;
 737	}
 738
 739	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 740	if (!txd)
 741		return NULL;
 742
 743	for_each_sg(sgl, sg, sg_len, i) {
 744		v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli);
 745		if (!v_lli)
 746			goto err_lli_free;
 747
 748		v_lli->len = sg_dma_len(sg);
 749		v_lli->para = NORMAL_WAIT;
 750
 751		if (dir == DMA_MEM_TO_DEV) {
 752			sun6i_dma_set_addr(sdev, v_lli,
 753					   sg_dma_address(sg),
 754					   sconfig->dst_addr);
 755			v_lli->cfg = lli_cfg;
 756			sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
 757			sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
 758
 759			dev_dbg(chan2dev(chan),
 760				"%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
 761				__func__, vchan->vc.chan.chan_id,
 762				&sconfig->dst_addr, &sg_dma_address(sg),
 763				sg_dma_len(sg), flags);
 764
 765		} else {
 766			sun6i_dma_set_addr(sdev, v_lli,
 767					   sconfig->src_addr,
 768					   sg_dma_address(sg));
 769			v_lli->cfg = lli_cfg;
 770			sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
 771			sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
 772
 773			dev_dbg(chan2dev(chan),
 774				"%s; chan: %d, dest: %pad, src: %pad, len: %u. flags: 0x%08lx\n",
 775				__func__, vchan->vc.chan.chan_id,
 776				&sg_dma_address(sg), &sconfig->src_addr,
 777				sg_dma_len(sg), flags);
 778		}
 779
 780		prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd);
 781	}
 782
 783	dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli);
 784	for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
 785	     p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
 786		sun6i_dma_dump_lli(vchan, v_lli, p_lli);
 787
 788	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 789
 790err_lli_free:
 791	for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
 792	     p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
 793		dma_pool_free(sdev->pool, v_lli, p_lli);
 794	kfree(txd);
 795	return NULL;
 796}
 797
 798static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic(
 799					struct dma_chan *chan,
 800					dma_addr_t buf_addr,
 801					size_t buf_len,
 802					size_t period_len,
 803					enum dma_transfer_direction dir,
 804					unsigned long flags)
 805{
 806	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 807	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 808	struct dma_slave_config *sconfig = &vchan->cfg;
 809	struct sun6i_dma_lli *v_lli, *prev = NULL;
 810	struct sun6i_desc *txd;
 811	dma_addr_t p_lli;
 812	u32 lli_cfg;
 813	unsigned int i, periods = buf_len / period_len;
 814	int ret;
 815
 816	ret = set_config(sdev, sconfig, dir, &lli_cfg);
 817	if (ret) {
 818		dev_err(chan2dev(chan), "Invalid DMA configuration\n");
 819		return NULL;
 820	}
 821
 822	txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
 823	if (!txd)
 824		return NULL;
 825
 826	for (i = 0; i < periods; i++) {
 827		v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli);
 828		if (!v_lli) {
 829			dev_err(sdev->slave.dev, "Failed to alloc lli memory\n");
 830			goto err_lli_free;
 831		}
 832
 833		v_lli->len = period_len;
 834		v_lli->para = NORMAL_WAIT;
 835
 836		if (dir == DMA_MEM_TO_DEV) {
 837			sun6i_dma_set_addr(sdev, v_lli,
 838					   buf_addr + period_len * i,
 839					   sconfig->dst_addr);
 840			v_lli->cfg = lli_cfg;
 841			sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port);
 842			sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE);
 843		} else {
 844			sun6i_dma_set_addr(sdev, v_lli,
 845					   sconfig->src_addr,
 846					   buf_addr + period_len * i);
 847			v_lli->cfg = lli_cfg;
 848			sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM);
 849			sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE);
 850		}
 851
 852		prev = sun6i_dma_lli_add(prev, v_lli, p_lli, txd);
 853	}
 854
 855	prev->p_lli_next = txd->p_lli;		/* cyclic list */
 856
 857	vchan->cyclic = true;
 858
 859	return vchan_tx_prep(&vchan->vc, &txd->vd, flags);
 860
 861err_lli_free:
 862	for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli;
 863	     p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next)
 864		dma_pool_free(sdev->pool, v_lli, p_lli);
 865	kfree(txd);
 866	return NULL;
 867}
 868
 869static int sun6i_dma_config(struct dma_chan *chan,
 870			    struct dma_slave_config *config)
 871{
 872	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 873
 874	memcpy(&vchan->cfg, config, sizeof(*config));
 875
 876	return 0;
 877}
 878
 879static int sun6i_dma_pause(struct dma_chan *chan)
 880{
 881	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 882	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 883	struct sun6i_pchan *pchan = vchan->phy;
 884
 885	dev_dbg(chan2dev(chan), "vchan %p: pause\n", &vchan->vc);
 886
 887	if (pchan) {
 888		writel(DMA_CHAN_PAUSE_PAUSE,
 889		       pchan->base + DMA_CHAN_PAUSE);
 890	} else {
 891		spin_lock(&sdev->lock);
 892		list_del_init(&vchan->node);
 893		spin_unlock(&sdev->lock);
 894	}
 895
 896	return 0;
 897}
 898
 899static int sun6i_dma_resume(struct dma_chan *chan)
 900{
 901	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 902	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 903	struct sun6i_pchan *pchan = vchan->phy;
 904	unsigned long flags;
 905
 906	dev_dbg(chan2dev(chan), "vchan %p: resume\n", &vchan->vc);
 907
 908	spin_lock_irqsave(&vchan->vc.lock, flags);
 909
 910	if (pchan) {
 911		writel(DMA_CHAN_PAUSE_RESUME,
 912		       pchan->base + DMA_CHAN_PAUSE);
 913	} else if (!list_empty(&vchan->vc.desc_issued)) {
 914		spin_lock(&sdev->lock);
 915		list_add_tail(&vchan->node, &sdev->pending);
 916		spin_unlock(&sdev->lock);
 917	}
 918
 919	spin_unlock_irqrestore(&vchan->vc.lock, flags);
 920
 921	return 0;
 922}
 923
 924static int sun6i_dma_terminate_all(struct dma_chan *chan)
 925{
 926	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
 927	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 928	struct sun6i_pchan *pchan = vchan->phy;
 929	unsigned long flags;
 930	LIST_HEAD(head);
 931
 932	spin_lock(&sdev->lock);
 933	list_del_init(&vchan->node);
 934	spin_unlock(&sdev->lock);
 935
 936	spin_lock_irqsave(&vchan->vc.lock, flags);
 937
 938	if (vchan->cyclic) {
 939		vchan->cyclic = false;
 940		if (pchan && pchan->desc) {
 941			struct virt_dma_desc *vd = &pchan->desc->vd;
 942			struct virt_dma_chan *vc = &vchan->vc;
 943
 944			list_add_tail(&vd->node, &vc->desc_completed);
 945		}
 946	}
 947
 948	vchan_get_all_descriptors(&vchan->vc, &head);
 949
 950	if (pchan) {
 951		writel(DMA_CHAN_ENABLE_STOP, pchan->base + DMA_CHAN_ENABLE);
 952		writel(DMA_CHAN_PAUSE_RESUME, pchan->base + DMA_CHAN_PAUSE);
 953
 954		vchan->phy = NULL;
 955		pchan->vchan = NULL;
 956		pchan->desc = NULL;
 957		pchan->done = NULL;
 958	}
 959
 960	spin_unlock_irqrestore(&vchan->vc.lock, flags);
 961
 962	vchan_dma_desc_free_list(&vchan->vc, &head);
 963
 964	return 0;
 965}
 966
 967static enum dma_status sun6i_dma_tx_status(struct dma_chan *chan,
 968					   dma_cookie_t cookie,
 969					   struct dma_tx_state *state)
 970{
 971	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
 972	struct sun6i_pchan *pchan = vchan->phy;
 973	struct sun6i_dma_lli *lli;
 974	struct virt_dma_desc *vd;
 975	struct sun6i_desc *txd;
 976	enum dma_status ret;
 977	unsigned long flags;
 978	size_t bytes = 0;
 979
 980	ret = dma_cookie_status(chan, cookie, state);
 981	if (ret == DMA_COMPLETE || !state)
 982		return ret;
 983
 984	spin_lock_irqsave(&vchan->vc.lock, flags);
 985
 986	vd = vchan_find_desc(&vchan->vc, cookie);
 987	txd = to_sun6i_desc(&vd->tx);
 988
 989	if (vd) {
 990		for (lli = txd->v_lli; lli != NULL; lli = lli->v_lli_next)
 991			bytes += lli->len;
 992	} else if (!pchan || !pchan->desc) {
 993		bytes = 0;
 994	} else {
 995		bytes = sun6i_get_chan_size(pchan);
 996	}
 997
 998	spin_unlock_irqrestore(&vchan->vc.lock, flags);
 999
1000	dma_set_residue(state, bytes);
1001
1002	return ret;
1003}
1004
1005static void sun6i_dma_issue_pending(struct dma_chan *chan)
1006{
1007	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
1008	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
1009	unsigned long flags;
1010
1011	spin_lock_irqsave(&vchan->vc.lock, flags);
1012
1013	if (vchan_issue_pending(&vchan->vc)) {
1014		spin_lock(&sdev->lock);
1015
1016		if (!vchan->phy && list_empty(&vchan->node)) {
1017			list_add_tail(&vchan->node, &sdev->pending);
1018			tasklet_schedule(&sdev->task);
1019			dev_dbg(chan2dev(chan), "vchan %p: issued\n",
1020				&vchan->vc);
1021		}
1022
1023		spin_unlock(&sdev->lock);
1024	} else {
1025		dev_dbg(chan2dev(chan), "vchan %p: nothing to issue\n",
1026			&vchan->vc);
1027	}
1028
1029	spin_unlock_irqrestore(&vchan->vc.lock, flags);
1030}
1031
1032static void sun6i_dma_free_chan_resources(struct dma_chan *chan)
1033{
1034	struct sun6i_dma_dev *sdev = to_sun6i_dma_dev(chan->device);
1035	struct sun6i_vchan *vchan = to_sun6i_vchan(chan);
1036	unsigned long flags;
1037
1038	spin_lock_irqsave(&sdev->lock, flags);
1039	list_del_init(&vchan->node);
1040	spin_unlock_irqrestore(&sdev->lock, flags);
1041
1042	vchan_free_chan_resources(&vchan->vc);
1043}
1044
1045static struct dma_chan *sun6i_dma_of_xlate(struct of_phandle_args *dma_spec,
1046					   struct of_dma *ofdma)
1047{
1048	struct sun6i_dma_dev *sdev = ofdma->of_dma_data;
1049	struct sun6i_vchan *vchan;
1050	struct dma_chan *chan;
1051	u8 port = dma_spec->args[0];
1052
1053	if (port > sdev->max_request)
1054		return NULL;
1055
1056	chan = dma_get_any_slave_channel(&sdev->slave);
1057	if (!chan)
1058		return NULL;
1059
1060	vchan = to_sun6i_vchan(chan);
1061	vchan->port = port;
1062
1063	return chan;
1064}
1065
1066static inline void sun6i_kill_tasklet(struct sun6i_dma_dev *sdev)
1067{
1068	/* Disable all interrupts from DMA */
1069	writel(0, sdev->base + DMA_IRQ_EN(0));
1070	writel(0, sdev->base + DMA_IRQ_EN(1));
1071
1072	/* Prevent spurious interrupts from scheduling the tasklet */
1073	atomic_inc(&sdev->tasklet_shutdown);
1074
1075	/* Make sure we won't have any further interrupts */
1076	devm_free_irq(sdev->slave.dev, sdev->irq, sdev);
1077
1078	/* Actually prevent the tasklet from being scheduled */
1079	tasklet_kill(&sdev->task);
1080}
1081
1082static inline void sun6i_dma_free(struct sun6i_dma_dev *sdev)
1083{
1084	int i;
1085
1086	for (i = 0; i < sdev->num_vchans; i++) {
1087		struct sun6i_vchan *vchan = &sdev->vchans[i];
1088
1089		list_del(&vchan->vc.chan.device_node);
1090		tasklet_kill(&vchan->vc.task);
1091	}
1092}
1093
1094/*
1095 * For A31:
1096 *
1097 * There's 16 physical channels that can work in parallel.
1098 *
1099 * However we have 30 different endpoints for our requests.
1100 *
1101 * Since the channels are able to handle only an unidirectional
1102 * transfer, we need to allocate more virtual channels so that
1103 * everyone can grab one channel.
1104 *
1105 * Some devices can't work in both direction (mostly because it
1106 * wouldn't make sense), so we have a bit fewer virtual channels than
1107 * 2 channels per endpoints.
1108 */
1109
1110static struct sun6i_dma_config sun6i_a31_dma_cfg = {
1111	.nr_max_channels = 16,
1112	.nr_max_requests = 30,
1113	.nr_max_vchans   = 53,
1114	.set_burst_length = sun6i_set_burst_length_a31,
1115	.set_drq          = sun6i_set_drq_a31,
1116	.set_mode         = sun6i_set_mode_a31,
1117	.src_burst_lengths = BIT(1) | BIT(8),
1118	.dst_burst_lengths = BIT(1) | BIT(8),
1119	.src_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1120			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1121			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1122	.dst_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1123			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1124			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1125};
1126
1127/*
1128 * The A23 only has 8 physical channels, a maximum DRQ port id of 24,
1129 * and a total of 37 usable source and destination endpoints.
1130 */
1131
1132static struct sun6i_dma_config sun8i_a23_dma_cfg = {
1133	.nr_max_channels = 8,
1134	.nr_max_requests = 24,
1135	.nr_max_vchans   = 37,
1136	.clock_autogate_enable = sun6i_enable_clock_autogate_a23,
1137	.set_burst_length = sun6i_set_burst_length_a31,
1138	.set_drq          = sun6i_set_drq_a31,
1139	.set_mode         = sun6i_set_mode_a31,
1140	.src_burst_lengths = BIT(1) | BIT(8),
1141	.dst_burst_lengths = BIT(1) | BIT(8),
1142	.src_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1143			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1144			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1145	.dst_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1146			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1147			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1148};
1149
1150static struct sun6i_dma_config sun8i_a83t_dma_cfg = {
1151	.nr_max_channels = 8,
1152	.nr_max_requests = 28,
1153	.nr_max_vchans   = 39,
1154	.clock_autogate_enable = sun6i_enable_clock_autogate_a23,
1155	.set_burst_length = sun6i_set_burst_length_a31,
1156	.set_drq          = sun6i_set_drq_a31,
1157	.set_mode         = sun6i_set_mode_a31,
1158	.src_burst_lengths = BIT(1) | BIT(8),
1159	.dst_burst_lengths = BIT(1) | BIT(8),
1160	.src_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1161			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1162			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1163	.dst_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1164			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1165			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1166};
1167
1168/*
1169 * The H3 has 12 physical channels, a maximum DRQ port id of 27,
1170 * and a total of 34 usable source and destination endpoints.
1171 * It also supports additional burst lengths and bus widths,
1172 * and the burst length fields have different offsets.
1173 */
1174
1175static struct sun6i_dma_config sun8i_h3_dma_cfg = {
1176	.nr_max_channels = 12,
1177	.nr_max_requests = 27,
1178	.nr_max_vchans   = 34,
1179	.clock_autogate_enable = sun6i_enable_clock_autogate_h3,
1180	.set_burst_length = sun6i_set_burst_length_h3,
1181	.set_drq          = sun6i_set_drq_a31,
1182	.set_mode         = sun6i_set_mode_a31,
1183	.src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1184	.dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1185	.src_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1186			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1187			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1188			     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1189	.dst_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1190			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1191			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1192			     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1193};
1194
1195/*
1196 * The A64 binding uses the number of dma channels from the
1197 * device tree node.
1198 */
1199static struct sun6i_dma_config sun50i_a64_dma_cfg = {
1200	.clock_autogate_enable = sun6i_enable_clock_autogate_h3,
1201	.set_burst_length = sun6i_set_burst_length_h3,
1202	.set_drq          = sun6i_set_drq_a31,
1203	.set_mode         = sun6i_set_mode_a31,
1204	.src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1205	.dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1206	.src_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1207			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1208			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1209			     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1210	.dst_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1211			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1212			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1213			     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1214};
1215
1216/*
1217 * The A100 binding uses the number of dma channels from the
1218 * device tree node.
1219 */
1220static struct sun6i_dma_config sun50i_a100_dma_cfg = {
1221	.clock_autogate_enable = sun6i_enable_clock_autogate_h3,
1222	.set_burst_length = sun6i_set_burst_length_h3,
1223	.set_drq          = sun6i_set_drq_h6,
1224	.set_mode         = sun6i_set_mode_h6,
1225	.src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1226	.dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1227	.src_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1228			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1229			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1230			     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1231	.dst_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1232			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1233			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1234			     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1235	.has_high_addr = true,
1236	.has_mbus_clk = true,
1237};
1238
1239/*
1240 * The H6 binding uses the number of dma channels from the
1241 * device tree node.
1242 */
1243static struct sun6i_dma_config sun50i_h6_dma_cfg = {
1244	.clock_autogate_enable = sun6i_enable_clock_autogate_h3,
1245	.set_burst_length = sun6i_set_burst_length_h3,
1246	.set_drq          = sun6i_set_drq_h6,
1247	.set_mode         = sun6i_set_mode_h6,
1248	.src_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1249	.dst_burst_lengths = BIT(1) | BIT(4) | BIT(8) | BIT(16),
1250	.src_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1251			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1252			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1253			     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1254	.dst_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1255			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1256			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1257			     BIT(DMA_SLAVE_BUSWIDTH_8_BYTES),
1258	.has_mbus_clk = true,
1259};
1260
1261/*
1262 * The V3s have only 8 physical channels, a maximum DRQ port id of 23,
1263 * and a total of 24 usable source and destination endpoints.
1264 */
1265
1266static struct sun6i_dma_config sun8i_v3s_dma_cfg = {
1267	.nr_max_channels = 8,
1268	.nr_max_requests = 23,
1269	.nr_max_vchans   = 24,
1270	.clock_autogate_enable = sun6i_enable_clock_autogate_a23,
1271	.set_burst_length = sun6i_set_burst_length_a31,
1272	.set_drq          = sun6i_set_drq_a31,
1273	.set_mode         = sun6i_set_mode_a31,
1274	.src_burst_lengths = BIT(1) | BIT(8),
1275	.dst_burst_lengths = BIT(1) | BIT(8),
1276	.src_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1277			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1278			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1279	.dst_addr_widths   = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1280			     BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1281			     BIT(DMA_SLAVE_BUSWIDTH_4_BYTES),
1282};
1283
1284static const struct of_device_id sun6i_dma_match[] = {
1285	{ .compatible = "allwinner,sun6i-a31-dma", .data = &sun6i_a31_dma_cfg },
1286	{ .compatible = "allwinner,sun8i-a23-dma", .data = &sun8i_a23_dma_cfg },
1287	{ .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg },
1288	{ .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg },
1289	{ .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg },
1290	{ .compatible = "allwinner,sun20i-d1-dma", .data = &sun50i_a100_dma_cfg },
1291	{ .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg },
1292	{ .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg },
1293	{ .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg },
1294	{ /* sentinel */ }
1295};
1296MODULE_DEVICE_TABLE(of, sun6i_dma_match);
1297
1298static int sun6i_dma_probe(struct platform_device *pdev)
1299{
1300	struct device_node *np = pdev->dev.of_node;
1301	struct sun6i_dma_dev *sdc;
1302	int ret, i;
1303
1304	sdc = devm_kzalloc(&pdev->dev, sizeof(*sdc), GFP_KERNEL);
1305	if (!sdc)
1306		return -ENOMEM;
1307
1308	sdc->cfg = of_device_get_match_data(&pdev->dev);
1309	if (!sdc->cfg)
1310		return -ENODEV;
1311
1312	sdc->base = devm_platform_ioremap_resource(pdev, 0);
1313	if (IS_ERR(sdc->base))
1314		return PTR_ERR(sdc->base);
1315
1316	sdc->irq = platform_get_irq(pdev, 0);
1317	if (sdc->irq < 0)
1318		return sdc->irq;
1319
1320	sdc->clk = devm_clk_get(&pdev->dev, NULL);
1321	if (IS_ERR(sdc->clk)) {
1322		dev_err(&pdev->dev, "No clock specified\n");
1323		return PTR_ERR(sdc->clk);
1324	}
1325
1326	if (sdc->cfg->has_mbus_clk) {
1327		sdc->clk_mbus = devm_clk_get(&pdev->dev, "mbus");
1328		if (IS_ERR(sdc->clk_mbus)) {
1329			dev_err(&pdev->dev, "No mbus clock specified\n");
1330			return PTR_ERR(sdc->clk_mbus);
1331		}
1332	}
1333
1334	sdc->rstc = devm_reset_control_get(&pdev->dev, NULL);
1335	if (IS_ERR(sdc->rstc)) {
1336		dev_err(&pdev->dev, "No reset controller specified\n");
1337		return PTR_ERR(sdc->rstc);
1338	}
1339
1340	sdc->pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
1341				     sizeof(struct sun6i_dma_lli), 4, 0);
1342	if (!sdc->pool) {
1343		dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
1344		return -ENOMEM;
1345	}
1346
1347	platform_set_drvdata(pdev, sdc);
1348	INIT_LIST_HEAD(&sdc->pending);
1349	spin_lock_init(&sdc->lock);
1350
1351	dma_set_max_seg_size(&pdev->dev, SZ_32M - 1);
1352
1353	dma_cap_set(DMA_PRIVATE, sdc->slave.cap_mask);
1354	dma_cap_set(DMA_MEMCPY, sdc->slave.cap_mask);
1355	dma_cap_set(DMA_SLAVE, sdc->slave.cap_mask);
1356	dma_cap_set(DMA_CYCLIC, sdc->slave.cap_mask);
1357
1358	INIT_LIST_HEAD(&sdc->slave.channels);
1359	sdc->slave.device_free_chan_resources	= sun6i_dma_free_chan_resources;
1360	sdc->slave.device_tx_status		= sun6i_dma_tx_status;
1361	sdc->slave.device_issue_pending		= sun6i_dma_issue_pending;
1362	sdc->slave.device_prep_slave_sg		= sun6i_dma_prep_slave_sg;
1363	sdc->slave.device_prep_dma_memcpy	= sun6i_dma_prep_dma_memcpy;
1364	sdc->slave.device_prep_dma_cyclic	= sun6i_dma_prep_dma_cyclic;
1365	sdc->slave.copy_align			= DMAENGINE_ALIGN_4_BYTES;
1366	sdc->slave.device_config		= sun6i_dma_config;
1367	sdc->slave.device_pause			= sun6i_dma_pause;
1368	sdc->slave.device_resume		= sun6i_dma_resume;
1369	sdc->slave.device_terminate_all		= sun6i_dma_terminate_all;
1370	sdc->slave.src_addr_widths		= sdc->cfg->src_addr_widths;
1371	sdc->slave.dst_addr_widths		= sdc->cfg->dst_addr_widths;
1372	sdc->slave.directions			= BIT(DMA_DEV_TO_MEM) |
1373						  BIT(DMA_MEM_TO_DEV);
1374	sdc->slave.residue_granularity		= DMA_RESIDUE_GRANULARITY_BURST;
1375	sdc->slave.dev = &pdev->dev;
1376
1377	sdc->num_pchans = sdc->cfg->nr_max_channels;
1378	sdc->num_vchans = sdc->cfg->nr_max_vchans;
1379	sdc->max_request = sdc->cfg->nr_max_requests;
1380
1381	ret = of_property_read_u32(np, "dma-channels", &sdc->num_pchans);
1382	if (ret && !sdc->num_pchans) {
1383		dev_err(&pdev->dev, "Can't get dma-channels.\n");
1384		return ret;
1385	}
1386
1387	ret = of_property_read_u32(np, "dma-requests", &sdc->max_request);
1388	if (ret && !sdc->max_request) {
1389		dev_info(&pdev->dev, "Missing dma-requests, using %u.\n",
1390			 DMA_CHAN_MAX_DRQ_A31);
1391		sdc->max_request = DMA_CHAN_MAX_DRQ_A31;
1392	}
1393
1394	/*
1395	 * If the number of vchans is not specified, derive it from the
1396	 * highest port number, at most one channel per port and direction.
1397	 */
1398	if (!sdc->num_vchans)
1399		sdc->num_vchans = 2 * (sdc->max_request + 1);
1400
1401	sdc->pchans = devm_kcalloc(&pdev->dev, sdc->num_pchans,
1402				   sizeof(struct sun6i_pchan), GFP_KERNEL);
1403	if (!sdc->pchans)
1404		return -ENOMEM;
1405
1406	sdc->vchans = devm_kcalloc(&pdev->dev, sdc->num_vchans,
1407				   sizeof(struct sun6i_vchan), GFP_KERNEL);
1408	if (!sdc->vchans)
1409		return -ENOMEM;
1410
1411	tasklet_setup(&sdc->task, sun6i_dma_tasklet);
1412
1413	for (i = 0; i < sdc->num_pchans; i++) {
1414		struct sun6i_pchan *pchan = &sdc->pchans[i];
1415
1416		pchan->idx = i;
1417		pchan->base = sdc->base + 0x100 + i * 0x40;
1418	}
1419
1420	for (i = 0; i < sdc->num_vchans; i++) {
1421		struct sun6i_vchan *vchan = &sdc->vchans[i];
1422
1423		INIT_LIST_HEAD(&vchan->node);
1424		vchan->vc.desc_free = sun6i_dma_free_desc;
1425		vchan_init(&vchan->vc, &sdc->slave);
1426	}
1427
1428	ret = reset_control_deassert(sdc->rstc);
1429	if (ret) {
1430		dev_err(&pdev->dev, "Couldn't deassert the device from reset\n");
1431		goto err_chan_free;
1432	}
1433
1434	ret = clk_prepare_enable(sdc->clk);
1435	if (ret) {
1436		dev_err(&pdev->dev, "Couldn't enable the clock\n");
1437		goto err_reset_assert;
1438	}
1439
1440	if (sdc->cfg->has_mbus_clk) {
1441		ret = clk_prepare_enable(sdc->clk_mbus);
1442		if (ret) {
1443			dev_err(&pdev->dev, "Couldn't enable mbus clock\n");
1444			goto err_clk_disable;
1445		}
1446	}
1447
1448	ret = devm_request_irq(&pdev->dev, sdc->irq, sun6i_dma_interrupt, 0,
1449			       dev_name(&pdev->dev), sdc);
1450	if (ret) {
1451		dev_err(&pdev->dev, "Cannot request IRQ\n");
1452		goto err_mbus_clk_disable;
1453	}
1454
1455	ret = dma_async_device_register(&sdc->slave);
1456	if (ret) {
1457		dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1458		goto err_irq_disable;
1459	}
1460
1461	ret = of_dma_controller_register(pdev->dev.of_node, sun6i_dma_of_xlate,
1462					 sdc);
1463	if (ret) {
1464		dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1465		goto err_dma_unregister;
1466	}
1467
1468	if (sdc->cfg->clock_autogate_enable)
1469		sdc->cfg->clock_autogate_enable(sdc);
1470
1471	return 0;
1472
1473err_dma_unregister:
1474	dma_async_device_unregister(&sdc->slave);
1475err_irq_disable:
1476	sun6i_kill_tasklet(sdc);
1477err_mbus_clk_disable:
1478	clk_disable_unprepare(sdc->clk_mbus);
1479err_clk_disable:
1480	clk_disable_unprepare(sdc->clk);
1481err_reset_assert:
1482	reset_control_assert(sdc->rstc);
1483err_chan_free:
1484	sun6i_dma_free(sdc);
1485	return ret;
1486}
1487
1488static void sun6i_dma_remove(struct platform_device *pdev)
1489{
1490	struct sun6i_dma_dev *sdc = platform_get_drvdata(pdev);
1491
1492	of_dma_controller_free(pdev->dev.of_node);
1493	dma_async_device_unregister(&sdc->slave);
1494
1495	sun6i_kill_tasklet(sdc);
1496
1497	clk_disable_unprepare(sdc->clk_mbus);
1498	clk_disable_unprepare(sdc->clk);
1499	reset_control_assert(sdc->rstc);
1500
1501	sun6i_dma_free(sdc);
1502}
1503
1504static struct platform_driver sun6i_dma_driver = {
1505	.probe		= sun6i_dma_probe,
1506	.remove		= sun6i_dma_remove,
1507	.driver = {
1508		.name		= "sun6i-dma",
1509		.of_match_table	= sun6i_dma_match,
1510	},
1511};
1512module_platform_driver(sun6i_dma_driver);
1513
1514MODULE_DESCRIPTION("Allwinner A31 DMA Controller Driver");
1515MODULE_AUTHOR("Sugar <shuge@allwinnertech.com>");
1516MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
1517MODULE_LICENSE("GPL");