sun4i-dma.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Copyright (C) 2014 Emilio López
   4 * Emilio López <emilio@elopez.com.ar>
   5 */
   6
   7#include <linux/bitmap.h>
   8#include <linux/bitops.h>
   9#include <linux/clk.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/dmaengine.h>
  12#include <linux/dmapool.h>
  13#include <linux/interrupt.h>
  14#include <linux/module.h>
  15#include <linux/of_dma.h>
  16#include <linux/of_device.h>
  17#include <linux/platform_device.h>
  18#include <linux/reset.h>
  19#include <linux/slab.h>
  20#include <linux/spinlock.h>
  21
  22#include "virt-dma.h"
  23
  24/** Common macros to normal and dedicated DMA registers **/
  25
  26#define SUN4I_DMA_CFG_LOADING			BIT(31)
  27#define SUN4I_DMA_CFG_DST_DATA_WIDTH(width)	((width) << 25)
  28#define SUN4I_DMA_CFG_DST_BURST_LENGTH(len)	((len) << 23)
  29#define SUN4I_DMA_CFG_DST_ADDR_MODE(mode)	((mode) << 21)
  30#define SUN4I_DMA_CFG_DST_DRQ_TYPE(type)	((type) << 16)
  31#define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width)	((width) << 9)
  32#define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len)	((len) << 7)
  33#define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode)	((mode) << 5)
  34#define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type)	(type)
  35
  36#define SUNIV_DMA_CFG_DST_DATA_WIDTH(width)	((width) << 24)
  37#define SUNIV_DMA_CFG_SRC_DATA_WIDTH(width)	((width) << 8)
  38
  39#define SUN4I_MAX_BURST	8
  40#define SUNIV_MAX_BURST	4
  41
  42/** Normal DMA register values **/
  43
  44/* Normal DMA source/destination data request type values */
  45#define SUN4I_NDMA_DRQ_TYPE_SDRAM		0x16
  46#define SUN4I_NDMA_DRQ_TYPE_LIMIT		(0x1F + 1)
  47
  48#define SUNIV_NDMA_DRQ_TYPE_SDRAM		0x11
  49#define SUNIV_NDMA_DRQ_TYPE_LIMIT		(0x17 + 1)
  50
  51/** Normal DMA register layout **/
  52
  53/* Dedicated DMA source/destination address mode values */
  54#define SUN4I_NDMA_ADDR_MODE_LINEAR		0
  55#define SUN4I_NDMA_ADDR_MODE_IO			1
  56
  57/* Normal DMA configuration register layout */
  58#define SUN4I_NDMA_CFG_CONT_MODE		BIT(30)
  59#define SUN4I_NDMA_CFG_WAIT_STATE(n)		((n) << 27)
  60#define SUN4I_NDMA_CFG_DST_NON_SECURE		BIT(22)
  61#define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN	BIT(15)
  62#define SUN4I_NDMA_CFG_SRC_NON_SECURE		BIT(6)
  63
  64#define SUNIV_NDMA_CFG_CONT_MODE		BIT(29)
  65#define SUNIV_NDMA_CFG_WAIT_STATE(n)		((n) << 26)
  66
  67/** Dedicated DMA register values **/
  68
  69/* Dedicated DMA source/destination address mode values */
  70#define SUN4I_DDMA_ADDR_MODE_LINEAR		0
  71#define SUN4I_DDMA_ADDR_MODE_IO			1
  72#define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE	2
  73#define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE	3
  74
  75/* Dedicated DMA source/destination data request type values */
  76#define SUN4I_DDMA_DRQ_TYPE_SDRAM		0x1
  77#define SUN4I_DDMA_DRQ_TYPE_LIMIT		(0x1F + 1)
  78
  79#define SUNIV_DDMA_DRQ_TYPE_SDRAM		0x1
  80#define SUNIV_DDMA_DRQ_TYPE_LIMIT		(0x9 + 1)
  81
  82/** Dedicated DMA register layout **/
  83
  84/* Dedicated DMA configuration register layout */
  85#define SUN4I_DDMA_CFG_BUSY			BIT(30)
  86#define SUN4I_DDMA_CFG_CONT_MODE		BIT(29)
  87#define SUN4I_DDMA_CFG_DST_NON_SECURE		BIT(28)
  88#define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN	BIT(15)
  89#define SUN4I_DDMA_CFG_SRC_NON_SECURE		BIT(12)
  90
  91/* Dedicated DMA parameter register layout */
  92#define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n)	(((n) - 1) << 24)
  93#define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n)	(((n) - 1) << 16)
  94#define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n)	(((n) - 1) << 8)
  95#define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n)	(((n) - 1) << 0)
  96
  97/** DMA register offsets **/
  98
  99/* General register offsets */
 100#define SUN4I_DMA_IRQ_ENABLE_REG		0x0
 101#define SUN4I_DMA_IRQ_PENDING_STATUS_REG	0x4
 102
 103/* Normal DMA register offsets */
 104#define SUN4I_NDMA_CHANNEL_REG_BASE(n)		(0x100 + (n) * 0x20)
 105#define SUN4I_NDMA_CFG_REG			0x0
 106#define SUN4I_NDMA_SRC_ADDR_REG			0x4
 107#define SUN4I_NDMA_DST_ADDR_REG		0x8
 108#define SUN4I_NDMA_BYTE_COUNT_REG		0xC
 109
 110/* Dedicated DMA register offsets */
 111#define SUN4I_DDMA_CHANNEL_REG_BASE(n)		(0x300 + (n) * 0x20)
 112#define SUN4I_DDMA_CFG_REG			0x0
 113#define SUN4I_DDMA_SRC_ADDR_REG			0x4
 114#define SUN4I_DDMA_DST_ADDR_REG		0x8
 115#define SUN4I_DDMA_BYTE_COUNT_REG		0xC
 116#define SUN4I_DDMA_PARA_REG			0x18
 117
 118/** DMA Driver **/
 119
 120/*
 121 * Normal DMA has 8 channels, and Dedicated DMA has another 8, so
 122 * that's 16 channels. As for endpoints, there's 29 and 21
 123 * respectively. Given that the Normal DMA endpoints (other than
 124 * SDRAM) can be used as tx/rx, we need 78 vchans in total
 125 */
 126#define SUN4I_NDMA_NR_MAX_CHANNELS	8
 127#define SUN4I_DDMA_NR_MAX_CHANNELS	8
 128#define SUN4I_DMA_NR_MAX_CHANNELS					\
 129	(SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
 130#define SUN4I_NDMA_NR_MAX_VCHANS	(29 * 2 - 1)
 131#define SUN4I_DDMA_NR_MAX_VCHANS	21
 132#define SUN4I_DMA_NR_MAX_VCHANS						\
 133	(SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
 134
 135#define SUNIV_NDMA_NR_MAX_CHANNELS	4
 136#define SUNIV_DDMA_NR_MAX_CHANNELS	4
 137#define SUNIV_NDMA_NR_MAX_VCHANS	(24 * 2 - 1)
 138#define SUNIV_DDMA_NR_MAX_VCHANS	10
 139
 140/* This set of SUN4I_DDMA timing parameters were found experimentally while
 141 * working with the SPI driver and seem to make it behave correctly */
 142#define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
 143	(SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) |			\
 144	 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) |				\
 145	 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) |				\
 146	 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
 147
 148/*
 149 * Normal DMA supports individual transfers (segments) up to 128k.
 150 * Dedicated DMA supports transfers up to 16M. We can only report
 151 * one size limit, so we have to use the smaller value.
 152 */
 153#define SUN4I_NDMA_MAX_SEG_SIZE		SZ_128K
 154#define SUN4I_DDMA_MAX_SEG_SIZE		SZ_16M
 155#define SUN4I_DMA_MAX_SEG_SIZE		SUN4I_NDMA_MAX_SEG_SIZE
 156
 157/*
 158 * Hardware channels / ports representation
 159 *
 160 * The hardware is used in several SoCs, with differing numbers
 161 * of channels and endpoints. This structure ties those numbers
 162 * to a certain compatible string.
 163 */
 164struct sun4i_dma_config {
 165	u32 ndma_nr_max_channels;
 166	u32 ndma_nr_max_vchans;
 167
 168	u32 ddma_nr_max_channels;
 169	u32 ddma_nr_max_vchans;
 170
 171	u32 dma_nr_max_channels;
 172
 173	void (*set_dst_data_width)(u32 *p_cfg, s8 data_width);
 174	void (*set_src_data_width)(u32 *p_cfg, s8 data_width);
 175	int (*convert_burst)(u32 maxburst);
 176
 177	u8 ndma_drq_sdram;
 178	u8 ddma_drq_sdram;
 179
 180	u8 max_burst;
 181	bool has_reset;
 182};
 183
 184struct sun4i_dma_pchan {
 185	/* Register base of channel */
 186	void __iomem			*base;
 187	/* vchan currently being serviced */
 188	struct sun4i_dma_vchan		*vchan;
 189	/* Is this a dedicated pchan? */
 190	int				is_dedicated;
 191};
 192
 193struct sun4i_dma_vchan {
 194	struct virt_dma_chan		vc;
 195	struct dma_slave_config		cfg;
 196	struct sun4i_dma_pchan		*pchan;
 197	struct sun4i_dma_promise	*processing;
 198	struct sun4i_dma_contract	*contract;
 199	u8				endpoint;
 200	int				is_dedicated;
 201};
 202
 203struct sun4i_dma_promise {
 204	u32				cfg;
 205	u32				para;
 206	dma_addr_t			src;
 207	dma_addr_t			dst;
 208	size_t				len;
 209	struct list_head		list;
 210};
 211
 212/* A contract is a set of promises */
 213struct sun4i_dma_contract {
 214	struct virt_dma_desc		vd;
 215	struct list_head		demands;
 216	struct list_head		completed_demands;
 217	bool				is_cyclic : 1;
 218	bool				use_half_int : 1;
 219};
 220
 221struct sun4i_dma_dev {
 222	unsigned long *pchans_used;
 223	struct dma_device		slave;
 224	struct sun4i_dma_pchan		*pchans;
 225	struct sun4i_dma_vchan		*vchans;
 226	void __iomem			*base;
 227	struct clk			*clk;
 228	int				irq;
 229	spinlock_t			lock;
 230	const struct sun4i_dma_config *cfg;
 231	struct reset_control *rst;
 232};
 233
 234static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
 235{
 236	return container_of(dev, struct sun4i_dma_dev, slave);
 237}
 238
 239static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
 240{
 241	return container_of(chan, struct sun4i_dma_vchan, vc.chan);
 242}
 243
 244static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
 245{
 246	return container_of(vd, struct sun4i_dma_contract, vd);
 247}
 248
 249static struct device *chan2dev(struct dma_chan *chan)
 250{
 251	return &chan->dev->device;
 252}
 253
 254static void set_dst_data_width_a10(u32 *p_cfg, s8 data_width)
 255{
 256	*p_cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(data_width);
 257}
 258
 259static void set_src_data_width_a10(u32 *p_cfg, s8 data_width)
 260{
 261	*p_cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(data_width);
 262}
 263
 264static void set_dst_data_width_f1c100s(u32 *p_cfg, s8 data_width)
 265{
 266	*p_cfg |= SUNIV_DMA_CFG_DST_DATA_WIDTH(data_width);
 267}
 268
 269static void set_src_data_width_f1c100s(u32 *p_cfg, s8 data_width)
 270{
 271	*p_cfg |= SUNIV_DMA_CFG_SRC_DATA_WIDTH(data_width);
 272}
 273
 274static int convert_burst_a10(u32 maxburst)
 275{
 276	if (maxburst > 8)
 277		return -EINVAL;
 278
 279	/* 1 -> 0, 4 -> 1, 8 -> 2 */
 280	return (maxburst >> 2);
 281}
 282
 283static int convert_burst_f1c100s(u32 maxburst)
 284{
 285	if (maxburst > 4)
 286		return -EINVAL;
 287
 288	/* 1 -> 0, 4 -> 1 */
 289	return (maxburst >> 2);
 290}
 291
 292static int convert_buswidth(enum dma_slave_buswidth addr_width)
 293{
 294	if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
 295		return -EINVAL;
 296
 297	/* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */
 298	return (addr_width >> 1);
 299}
 300
 301static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
 302{
 303	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 304
 305	vchan_free_chan_resources(&vchan->vc);
 306}
 307
 308static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
 309						  struct sun4i_dma_vchan *vchan)
 310{
 311	struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
 312	unsigned long flags;
 313	int i, max;
 314
 315	/*
 316	 * pchans 0-priv->cfg->ndma_nr_max_channels are normal, and
 317	 * priv->cfg->ndma_nr_max_channels+ are dedicated ones
 318	 */
 319	if (vchan->is_dedicated) {
 320		i = priv->cfg->ndma_nr_max_channels;
 321		max = priv->cfg->dma_nr_max_channels;
 322	} else {
 323		i = 0;
 324		max = priv->cfg->ndma_nr_max_channels;
 325	}
 326
 327	spin_lock_irqsave(&priv->lock, flags);
 328	for_each_clear_bit_from(i, priv->pchans_used, max) {
 329		pchan = &pchans[i];
 330		pchan->vchan = vchan;
 331		set_bit(i, priv->pchans_used);
 332		break;
 333	}
 334	spin_unlock_irqrestore(&priv->lock, flags);
 335
 336	return pchan;
 337}
 338
 339static void release_pchan(struct sun4i_dma_dev *priv,
 340			  struct sun4i_dma_pchan *pchan)
 341{
 342	unsigned long flags;
 343	int nr = pchan - priv->pchans;
 344
 345	spin_lock_irqsave(&priv->lock, flags);
 346
 347	pchan->vchan = NULL;
 348	clear_bit(nr, priv->pchans_used);
 349
 350	spin_unlock_irqrestore(&priv->lock, flags);
 351}
 352
 353static void configure_pchan(struct sun4i_dma_pchan *pchan,
 354			    struct sun4i_dma_promise *d)
 355{
 356	/*
 357	 * Configure addresses and misc parameters depending on type
 358	 * SUN4I_DDMA has an extra field with timing parameters
 359	 */
 360	if (pchan->is_dedicated) {
 361		writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
 362		writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
 363		writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
 364		writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
 365		writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
 366	} else {
 367		writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
 368		writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
 369		writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
 370		writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
 371	}
 372}
 373
 374static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
 375				struct sun4i_dma_pchan *pchan,
 376				int half, int end)
 377{
 378	u32 reg;
 379	int pchan_number = pchan - priv->pchans;
 380	unsigned long flags;
 381
 382	spin_lock_irqsave(&priv->lock, flags);
 383
 384	reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
 385
 386	if (half)
 387		reg |= BIT(pchan_number * 2);
 388	else
 389		reg &= ~BIT(pchan_number * 2);
 390
 391	if (end)
 392		reg |= BIT(pchan_number * 2 + 1);
 393	else
 394		reg &= ~BIT(pchan_number * 2 + 1);
 395
 396	writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
 397
 398	spin_unlock_irqrestore(&priv->lock, flags);
 399}
 400
 401/*
 402 * Execute pending operations on a vchan
 403 *
 404 * When given a vchan, this function will try to acquire a suitable
 405 * pchan and, if successful, will configure it to fulfill a promise
 406 * from the next pending contract.
 407 *
 408 * This function must be called with &vchan->vc.lock held.
 409 */
 410static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
 411				   struct sun4i_dma_vchan *vchan)
 412{
 413	struct sun4i_dma_promise *promise = NULL;
 414	struct sun4i_dma_contract *contract = NULL;
 415	struct sun4i_dma_pchan *pchan;
 416	struct virt_dma_desc *vd;
 417	int ret;
 418
 419	lockdep_assert_held(&vchan->vc.lock);
 420
 421	/* We need a pchan to do anything, so secure one if available */
 422	pchan = find_and_use_pchan(priv, vchan);
 423	if (!pchan)
 424		return -EBUSY;
 425
 426	/*
 427	 * Channel endpoints must not be repeated, so if this vchan
 428	 * has already submitted some work, we can't do anything else
 429	 */
 430	if (vchan->processing) {
 431		dev_dbg(chan2dev(&vchan->vc.chan),
 432			"processing something to this endpoint already\n");
 433		ret = -EBUSY;
 434		goto release_pchan;
 435	}
 436
 437	do {
 438		/* Figure out which contract we're working with today */
 439		vd = vchan_next_desc(&vchan->vc);
 440		if (!vd) {
 441			dev_dbg(chan2dev(&vchan->vc.chan),
 442				"No pending contract found");
 443			ret = 0;
 444			goto release_pchan;
 445		}
 446
 447		contract = to_sun4i_dma_contract(vd);
 448		if (list_empty(&contract->demands)) {
 449			/* The contract has been completed so mark it as such */
 450			list_del(&contract->vd.node);
 451			vchan_cookie_complete(&contract->vd);
 452			dev_dbg(chan2dev(&vchan->vc.chan),
 453				"Empty contract found and marked complete");
 454		}
 455	} while (list_empty(&contract->demands));
 456
 457	/* Now find out what we need to do */
 458	promise = list_first_entry(&contract->demands,
 459				   struct sun4i_dma_promise, list);
 460	vchan->processing = promise;
 461
 462	/* ... and make it reality */
 463	if (promise) {
 464		vchan->contract = contract;
 465		vchan->pchan = pchan;
 466		set_pchan_interrupt(priv, pchan, contract->use_half_int, 1);
 467		configure_pchan(pchan, promise);
 468	}
 469
 470	return 0;
 471
 472release_pchan:
 473	release_pchan(priv, pchan);
 474	return ret;
 475}
 476
 477static int sanitize_config(struct dma_slave_config *sconfig,
 478			   enum dma_transfer_direction direction)
 479{
 480	switch (direction) {
 481	case DMA_MEM_TO_DEV:
 482		if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
 483		    !sconfig->dst_maxburst)
 484			return -EINVAL;
 485
 486		if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
 487			sconfig->src_addr_width = sconfig->dst_addr_width;
 488
 489		if (!sconfig->src_maxburst)
 490			sconfig->src_maxburst = sconfig->dst_maxburst;
 491
 492		break;
 493
 494	case DMA_DEV_TO_MEM:
 495		if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
 496		    !sconfig->src_maxburst)
 497			return -EINVAL;
 498
 499		if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
 500			sconfig->dst_addr_width = sconfig->src_addr_width;
 501
 502		if (!sconfig->dst_maxburst)
 503			sconfig->dst_maxburst = sconfig->src_maxburst;
 504
 505		break;
 506	default:
 507		return 0;
 508	}
 509
 510	return 0;
 511}
 512
 513/*
 514 * Generate a promise, to be used in a normal DMA contract.
 515 *
 516 * A NDMA promise contains all the information required to program the
 517 * normal part of the DMA Engine and get data copied. A non-executed
 518 * promise will live in the demands list on a contract. Once it has been
 519 * completed, it will be moved to the completed demands list for later freeing.
 520 * All linked promises will be freed when the corresponding contract is freed
 521 */
 522static struct sun4i_dma_promise *
 523generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
 524		      size_t len, struct dma_slave_config *sconfig,
 525		      enum dma_transfer_direction direction)
 526{
 527	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
 528	struct sun4i_dma_promise *promise;
 529	int ret;
 530
 531	ret = sanitize_config(sconfig, direction);
 532	if (ret)
 533		return NULL;
 534
 535	promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
 536	if (!promise)
 537		return NULL;
 538
 539	promise->src = src;
 540	promise->dst = dest;
 541	promise->len = len;
 542	promise->cfg = SUN4I_DMA_CFG_LOADING |
 543		SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
 544
 545	dev_dbg(chan2dev(chan),
 546		"src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
 547		sconfig->src_maxburst, sconfig->dst_maxburst,
 548		sconfig->src_addr_width, sconfig->dst_addr_width);
 549
 550	/* Source burst */
 551	ret = priv->cfg->convert_burst(sconfig->src_maxburst);
 552	if (ret < 0)
 553		goto fail;
 554	promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
 555
 556	/* Destination burst */
 557	ret = priv->cfg->convert_burst(sconfig->dst_maxburst);
 558	if (ret < 0)
 559		goto fail;
 560	promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
 561
 562	/* Source bus width */
 563	ret = convert_buswidth(sconfig->src_addr_width);
 564	if (ret < 0)
 565		goto fail;
 566	priv->cfg->set_src_data_width(&promise->cfg, ret);
 567
 568	/* Destination bus width */
 569	ret = convert_buswidth(sconfig->dst_addr_width);
 570	if (ret < 0)
 571		goto fail;
 572	priv->cfg->set_dst_data_width(&promise->cfg, ret);
 573
 574	return promise;
 575
 576fail:
 577	kfree(promise);
 578	return NULL;
 579}
 580
 581/*
 582 * Generate a promise, to be used in a dedicated DMA contract.
 583 *
 584 * A DDMA promise contains all the information required to program the
 585 * Dedicated part of the DMA Engine and get data copied. A non-executed
 586 * promise will live in the demands list on a contract. Once it has been
 587 * completed, it will be moved to the completed demands list for later freeing.
 588 * All linked promises will be freed when the corresponding contract is freed
 589 */
 590static struct sun4i_dma_promise *
 591generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
 592		      size_t len, struct dma_slave_config *sconfig)
 593{
 594	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
 595	struct sun4i_dma_promise *promise;
 596	int ret;
 597
 598	promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
 599	if (!promise)
 600		return NULL;
 601
 602	promise->src = src;
 603	promise->dst = dest;
 604	promise->len = len;
 605	promise->cfg = SUN4I_DMA_CFG_LOADING |
 606		SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
 607
 608	/* Source burst */
 609	ret = priv->cfg->convert_burst(sconfig->src_maxburst);
 610	if (ret < 0)
 611		goto fail;
 612	promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
 613
 614	/* Destination burst */
 615	ret = priv->cfg->convert_burst(sconfig->dst_maxburst);
 616	if (ret < 0)
 617		goto fail;
 618	promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
 619
 620	/* Source bus width */
 621	ret = convert_buswidth(sconfig->src_addr_width);
 622	if (ret < 0)
 623		goto fail;
 624	priv->cfg->set_src_data_width(&promise->cfg, ret);
 625
 626	/* Destination bus width */
 627	ret = convert_buswidth(sconfig->dst_addr_width);
 628	if (ret < 0)
 629		goto fail;
 630	priv->cfg->set_dst_data_width(&promise->cfg, ret);
 631
 632	return promise;
 633
 634fail:
 635	kfree(promise);
 636	return NULL;
 637}
 638
 639/*
 640 * Generate a contract
 641 *
 642 * Contracts function as DMA descriptors. As our hardware does not support
 643 * linked lists, we need to implement SG via software. We use a contract
 644 * to hold all the pieces of the request and process them serially one
 645 * after another. Each piece is represented as a promise.
 646 */
 647static struct sun4i_dma_contract *generate_dma_contract(void)
 648{
 649	struct sun4i_dma_contract *contract;
 650
 651	contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
 652	if (!contract)
 653		return NULL;
 654
 655	INIT_LIST_HEAD(&contract->demands);
 656	INIT_LIST_HEAD(&contract->completed_demands);
 657
 658	return contract;
 659}
 660
 661/*
 662 * Get next promise on a cyclic transfer
 663 *
 664 * Cyclic contracts contain a series of promises which are executed on a
 665 * loop. This function returns the next promise from a cyclic contract,
 666 * so it can be programmed into the hardware.
 667 */
 668static struct sun4i_dma_promise *
 669get_next_cyclic_promise(struct sun4i_dma_contract *contract)
 670{
 671	struct sun4i_dma_promise *promise;
 672
 673	promise = list_first_entry_or_null(&contract->demands,
 674					   struct sun4i_dma_promise, list);
 675	if (!promise) {
 676		list_splice_init(&contract->completed_demands,
 677				 &contract->demands);
 678		promise = list_first_entry(&contract->demands,
 679					   struct sun4i_dma_promise, list);
 680	}
 681
 682	return promise;
 683}
 684
 685/*
 686 * Free a contract and all its associated promises
 687 */
 688static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
 689{
 690	struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
 691	struct sun4i_dma_promise *promise, *tmp;
 692
 693	/* Free all the demands and completed demands */
 694	list_for_each_entry_safe(promise, tmp, &contract->demands, list)
 695		kfree(promise);
 696
 697	list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
 698		kfree(promise);
 699
 700	kfree(contract);
 701}
 702
 703static struct dma_async_tx_descriptor *
 704sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
 705			  dma_addr_t src, size_t len, unsigned long flags)
 706{
 707	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
 708	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 709	struct dma_slave_config *sconfig = &vchan->cfg;
 710	struct sun4i_dma_promise *promise;
 711	struct sun4i_dma_contract *contract;
 712
 713	contract = generate_dma_contract();
 714	if (!contract)
 715		return NULL;
 716
 717	/*
 718	 * We can only do the copy to bus aligned addresses, so
 719	 * choose the best one so we get decent performance. We also
 720	 * maximize the burst size for this same reason.
 721	 */
 722	sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 723	sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 724	sconfig->src_maxburst = priv->cfg->max_burst;
 725	sconfig->dst_maxburst = priv->cfg->max_burst;
 726
 727	if (vchan->is_dedicated)
 728		promise = generate_ddma_promise(chan, src, dest, len, sconfig);
 729	else
 730		promise = generate_ndma_promise(chan, src, dest, len, sconfig,
 731						DMA_MEM_TO_MEM);
 732
 733	if (!promise) {
 734		kfree(contract);
 735		return NULL;
 736	}
 737
 738	/* Configure memcpy mode */
 739	if (vchan->is_dedicated) {
 740		promise->cfg |=
 741			SUN4I_DMA_CFG_SRC_DRQ_TYPE(priv->cfg->ddma_drq_sdram) |
 742			SUN4I_DMA_CFG_DST_DRQ_TYPE(priv->cfg->ddma_drq_sdram);
 743	} else {
 744		promise->cfg |=
 745			SUN4I_DMA_CFG_SRC_DRQ_TYPE(priv->cfg->ndma_drq_sdram) |
 746			SUN4I_DMA_CFG_DST_DRQ_TYPE(priv->cfg->ndma_drq_sdram);
 747	}
 748
 749	/* Fill the contract with our only promise */
 750	list_add_tail(&promise->list, &contract->demands);
 751
 752	/* And add it to the vchan */
 753	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
 754}
 755
 756static struct dma_async_tx_descriptor *
 757sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
 758			  size_t period_len, enum dma_transfer_direction dir,
 759			  unsigned long flags)
 760{
 761	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
 762	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 763	struct dma_slave_config *sconfig = &vchan->cfg;
 764	struct sun4i_dma_promise *promise;
 765	struct sun4i_dma_contract *contract;
 766	dma_addr_t src, dest;
 767	u32 endpoints;
 768	int nr_periods, offset, plength, i;
 769	u8 ram_type, io_mode, linear_mode;
 770
 771	if (!is_slave_direction(dir)) {
 772		dev_err(chan2dev(chan), "Invalid DMA direction\n");
 773		return NULL;
 774	}
 775
 776	contract = generate_dma_contract();
 777	if (!contract)
 778		return NULL;
 779
 780	contract->is_cyclic = 1;
 781
 782	if (vchan->is_dedicated) {
 783		io_mode = SUN4I_DDMA_ADDR_MODE_IO;
 784		linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
 785		ram_type = priv->cfg->ddma_drq_sdram;
 786	} else {
 787		io_mode = SUN4I_NDMA_ADDR_MODE_IO;
 788		linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
 789		ram_type = priv->cfg->ndma_drq_sdram;
 790	}
 791
 792	if (dir == DMA_MEM_TO_DEV) {
 793		src = buf;
 794		dest = sconfig->dst_addr;
 795		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
 796			    SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
 797			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
 798			    SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
 799	} else {
 800		src = sconfig->src_addr;
 801		dest = buf;
 802		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
 803			    SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
 804			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
 805			    SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
 806	}
 807
 808	/*
 809	 * We will be using half done interrupts to make two periods
 810	 * out of a promise, so we need to program the DMA engine less
 811	 * often
 812	 */
 813
 814	/*
 815	 * The engine can interrupt on half-transfer, so we can use
 816	 * this feature to program the engine half as often as if we
 817	 * didn't use it (keep in mind the hardware doesn't support
 818	 * linked lists).
 819	 *
 820	 * Say you have a set of periods (| marks the start/end, I for
 821	 * interrupt, P for programming the engine to do a new
 822	 * transfer), the easy but slow way would be to do
 823	 *
 824	 *  |---|---|---|---| (periods / promises)
 825	 *  P  I,P I,P I,P  I
 826	 *
 827	 * Using half transfer interrupts you can do
 828	 *
 829	 *  |-------|-------| (promises as configured on hw)
 830	 *  |---|---|---|---| (periods)
 831	 *  P   I  I,P  I   I
 832	 *
 833	 * Which requires half the engine programming for the same
 834	 * functionality.
 835	 *
 836	 * This only works if two periods fit in a single promise. That will
 837	 * always be the case for dedicated DMA, where the hardware has a much
 838	 * larger maximum transfer size than advertised to clients.
 839	 */
 840	if (vchan->is_dedicated || period_len <= SUN4I_NDMA_MAX_SEG_SIZE / 2) {
 841		period_len *= 2;
 842		contract->use_half_int = 1;
 843	}
 844
 845	nr_periods = DIV_ROUND_UP(len, period_len);
 846	for (i = 0; i < nr_periods; i++) {
 847		/* Calculate the offset in the buffer and the length needed */
 848		offset = i * period_len;
 849		plength = min((len - offset), period_len);
 850		if (dir == DMA_MEM_TO_DEV)
 851			src = buf + offset;
 852		else
 853			dest = buf + offset;
 854
 855		/* Make the promise */
 856		if (vchan->is_dedicated)
 857			promise = generate_ddma_promise(chan, src, dest,
 858							plength, sconfig);
 859		else
 860			promise = generate_ndma_promise(chan, src, dest,
 861							plength, sconfig, dir);
 862
 863		if (!promise) {
 864			/* TODO: should we free everything? */
 865			return NULL;
 866		}
 867		promise->cfg |= endpoints;
 868
 869		/* Then add it to the contract */
 870		list_add_tail(&promise->list, &contract->demands);
 871	}
 872
 873	/* And add it to the vchan */
 874	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
 875}
 876
 877static struct dma_async_tx_descriptor *
 878sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 879			unsigned int sg_len, enum dma_transfer_direction dir,
 880			unsigned long flags, void *context)
 881{
 882	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
 883	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 884	struct dma_slave_config *sconfig = &vchan->cfg;
 885	struct sun4i_dma_promise *promise;
 886	struct sun4i_dma_contract *contract;
 887	u8 ram_type, io_mode, linear_mode;
 888	struct scatterlist *sg;
 889	dma_addr_t srcaddr, dstaddr;
 890	u32 endpoints, para;
 891	int i;
 892
 893	if (!sgl)
 894		return NULL;
 895
 896	if (!is_slave_direction(dir)) {
 897		dev_err(chan2dev(chan), "Invalid DMA direction\n");
 898		return NULL;
 899	}
 900
 901	contract = generate_dma_contract();
 902	if (!contract)
 903		return NULL;
 904
 905	if (vchan->is_dedicated) {
 906		io_mode = SUN4I_DDMA_ADDR_MODE_IO;
 907		linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
 908		ram_type = priv->cfg->ddma_drq_sdram;
 909	} else {
 910		io_mode = SUN4I_NDMA_ADDR_MODE_IO;
 911		linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
 912		ram_type = priv->cfg->ndma_drq_sdram;
 913	}
 914
 915	if (dir == DMA_MEM_TO_DEV)
 916		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
 917			    SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
 918			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
 919			    SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
 920	else
 921		endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
 922			    SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
 923			    SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
 924			    SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
 925
 926	for_each_sg(sgl, sg, sg_len, i) {
 927		/* Figure out addresses */
 928		if (dir == DMA_MEM_TO_DEV) {
 929			srcaddr = sg_dma_address(sg);
 930			dstaddr = sconfig->dst_addr;
 931		} else {
 932			srcaddr = sconfig->src_addr;
 933			dstaddr = sg_dma_address(sg);
 934		}
 935
 936		/*
 937		 * These are the magic DMA engine timings that keep SPI going.
 938		 * I haven't seen any interface on DMAEngine to configure
 939		 * timings, and so far they seem to work for everything we
 940		 * support, so I've kept them here. I don't know if other
 941		 * devices need different timings because, as usual, we only
 942		 * have the "para" bitfield meanings, but no comment on what
 943		 * the values should be when doing a certain operation :|
 944		 */
 945		para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
 946
 947		/* And make a suitable promise */
 948		if (vchan->is_dedicated)
 949			promise = generate_ddma_promise(chan, srcaddr, dstaddr,
 950							sg_dma_len(sg),
 951							sconfig);
 952		else
 953			promise = generate_ndma_promise(chan, srcaddr, dstaddr,
 954							sg_dma_len(sg),
 955							sconfig, dir);
 956
 957		if (!promise)
 958			return NULL; /* TODO: should we free everything? */
 959
 960		promise->cfg |= endpoints;
 961		promise->para = para;
 962
 963		/* Then add it to the contract */
 964		list_add_tail(&promise->list, &contract->demands);
 965	}
 966
 967	/*
 968	 * Once we've got all the promises ready, add the contract
 969	 * to the pending list on the vchan
 970	 */
 971	return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
 972}
 973
 974static int sun4i_dma_terminate_all(struct dma_chan *chan)
 975{
 976	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
 977	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
 978	struct sun4i_dma_pchan *pchan = vchan->pchan;
 979	LIST_HEAD(head);
 980	unsigned long flags;
 981
 982	spin_lock_irqsave(&vchan->vc.lock, flags);
 983	vchan_get_all_descriptors(&vchan->vc, &head);
 984	spin_unlock_irqrestore(&vchan->vc.lock, flags);
 985
 986	/*
 987	 * Clearing the configuration register will halt the pchan. Interrupts
 988	 * may still trigger, so don't forget to disable them.
 989	 */
 990	if (pchan) {
 991		if (pchan->is_dedicated)
 992			writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
 993		else
 994			writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
 995		set_pchan_interrupt(priv, pchan, 0, 0);
 996		release_pchan(priv, pchan);
 997	}
 998
 999	spin_lock_irqsave(&vchan->vc.lock, flags);
1000	/* Clear these so the vchan is usable again */
1001	vchan->processing = NULL;
1002	vchan->pchan = NULL;
1003	spin_unlock_irqrestore(&vchan->vc.lock, flags);
1004
1005	vchan_dma_desc_free_list(&vchan->vc, &head);
1006
1007	return 0;
1008}
1009
1010static int sun4i_dma_config(struct dma_chan *chan,
1011			    struct dma_slave_config *config)
1012{
1013	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
1014
1015	memcpy(&vchan->cfg, config, sizeof(*config));
1016
1017	return 0;
1018}
1019
1020static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
1021					   struct of_dma *ofdma)
1022{
1023	struct sun4i_dma_dev *priv = ofdma->of_dma_data;
1024	struct sun4i_dma_vchan *vchan;
1025	struct dma_chan *chan;
1026	u8 is_dedicated = dma_spec->args[0];
1027	u8 endpoint = dma_spec->args[1];
1028
1029	/* Check if type is Normal or Dedicated */
1030	if (is_dedicated != 0 && is_dedicated != 1)
1031		return NULL;
1032
1033	/* Make sure the endpoint looks sane */
1034	if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
1035	    (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
1036		return NULL;
1037
1038	chan = dma_get_any_slave_channel(&priv->slave);
1039	if (!chan)
1040		return NULL;
1041
1042	/* Assign the endpoint to the vchan */
1043	vchan = to_sun4i_dma_vchan(chan);
1044	vchan->is_dedicated = is_dedicated;
1045	vchan->endpoint = endpoint;
1046
1047	return chan;
1048}
1049
1050static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
1051					   dma_cookie_t cookie,
1052					   struct dma_tx_state *state)
1053{
1054	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
1055	struct sun4i_dma_pchan *pchan = vchan->pchan;
1056	struct sun4i_dma_contract *contract;
1057	struct sun4i_dma_promise *promise;
1058	struct virt_dma_desc *vd;
1059	unsigned long flags;
1060	enum dma_status ret;
1061	size_t bytes = 0;
1062
1063	ret = dma_cookie_status(chan, cookie, state);
1064	if (!state || (ret == DMA_COMPLETE))
1065		return ret;
1066
1067	spin_lock_irqsave(&vchan->vc.lock, flags);
1068	vd = vchan_find_desc(&vchan->vc, cookie);
1069	if (!vd)
1070		goto exit;
1071	contract = to_sun4i_dma_contract(vd);
1072
1073	list_for_each_entry(promise, &contract->demands, list)
1074		bytes += promise->len;
1075
1076	/*
1077	 * The hardware is configured to return the remaining byte
1078	 * quantity. If possible, replace the first listed element's
1079	 * full size with the actual remaining amount
1080	 */
1081	promise = list_first_entry_or_null(&contract->demands,
1082					   struct sun4i_dma_promise, list);
1083	if (promise && pchan) {
1084		bytes -= promise->len;
1085		if (pchan->is_dedicated)
1086			bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
1087		else
1088			bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
1089	}
1090
1091exit:
1092
1093	dma_set_residue(state, bytes);
1094	spin_unlock_irqrestore(&vchan->vc.lock, flags);
1095
1096	return ret;
1097}
1098
1099static void sun4i_dma_issue_pending(struct dma_chan *chan)
1100{
1101	struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
1102	struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
1103	unsigned long flags;
1104
1105	spin_lock_irqsave(&vchan->vc.lock, flags);
1106
1107	/*
1108	 * If there are pending transactions for this vchan, push one of
1109	 * them into the engine to get the ball rolling.
1110	 */
1111	if (vchan_issue_pending(&vchan->vc))
1112		__execute_vchan_pending(priv, vchan);
1113
1114	spin_unlock_irqrestore(&vchan->vc.lock, flags);
1115}
1116
1117static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1118{
1119	struct sun4i_dma_dev *priv = dev_id;
1120	struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1121	struct sun4i_dma_vchan *vchan;
1122	struct sun4i_dma_contract *contract;
1123	struct sun4i_dma_promise *promise;
1124	unsigned long pendirq, irqs, disableirqs;
1125	int bit, i, free_room, allow_mitigation = 1;
1126
1127	pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1128
1129handle_pending:
1130
1131	disableirqs = 0;
1132	free_room = 0;
1133
1134	for_each_set_bit(bit, &pendirq, 32) {
1135		pchan = &pchans[bit >> 1];
1136		vchan = pchan->vchan;
1137		if (!vchan) /* a terminated channel may still interrupt */
1138			continue;
1139		contract = vchan->contract;
1140
1141		/*
1142		 * Disable the IRQ and free the pchan if it's an end
1143		 * interrupt (odd bit)
1144		 */
1145		if (bit & 1) {
1146			spin_lock(&vchan->vc.lock);
1147
1148			/*
1149			 * Move the promise into the completed list now that
1150			 * we're done with it
1151			 */
1152			list_move_tail(&vchan->processing->list,
1153				       &contract->completed_demands);
1154
1155			/*
1156			 * Cyclic DMA transfers are special:
1157			 * - There's always something we can dispatch
1158			 * - We need to run the callback
1159			 * - Latency is very important, as this is used by audio
1160			 * We therefore just cycle through the list and dispatch
1161			 * whatever we have here, reusing the pchan. There's
1162			 * no need to run the thread after this.
1163			 *
1164			 * For non-cyclic transfers we need to look around,
1165			 * so we can program some more work, or notify the
1166			 * client that their transfers have been completed.
1167			 */
1168			if (contract->is_cyclic) {
1169				promise = get_next_cyclic_promise(contract);
1170				vchan->processing = promise;
1171				configure_pchan(pchan, promise);
1172				vchan_cyclic_callback(&contract->vd);
1173			} else {
1174				vchan->processing = NULL;
1175				vchan->pchan = NULL;
1176
1177				free_room = 1;
1178				disableirqs |= BIT(bit);
1179				release_pchan(priv, pchan);
1180			}
1181
1182			spin_unlock(&vchan->vc.lock);
1183		} else {
1184			/* Half done interrupt */
1185			if (contract->is_cyclic)
1186				vchan_cyclic_callback(&contract->vd);
1187			else
1188				disableirqs |= BIT(bit);
1189		}
1190	}
1191
1192	/* Disable the IRQs for events we handled */
1193	spin_lock(&priv->lock);
1194	irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1195	writel_relaxed(irqs & ~disableirqs,
1196		       priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1197	spin_unlock(&priv->lock);
1198
1199	/* Writing 1 to the pending field will clear the pending interrupt */
1200	writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1201
1202	/*
1203	 * If a pchan was freed, we may be able to schedule something else,
1204	 * so have a look around
1205	 */
1206	if (free_room) {
1207		for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1208			vchan = &priv->vchans[i];
1209			spin_lock(&vchan->vc.lock);
1210			__execute_vchan_pending(priv, vchan);
1211			spin_unlock(&vchan->vc.lock);
1212		}
1213	}
1214
1215	/*
1216	 * Handle newer interrupts if some showed up, but only do it once
1217	 * to avoid a too long a loop
1218	 */
1219	if (allow_mitigation) {
1220		pendirq = readl_relaxed(priv->base +
1221					SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1222		if (pendirq) {
1223			allow_mitigation = 0;
1224			goto handle_pending;
1225		}
1226	}
1227
1228	return IRQ_HANDLED;
1229}
1230
1231static int sun4i_dma_probe(struct platform_device *pdev)
1232{
1233	struct sun4i_dma_dev *priv;
1234	int i, j, ret;
1235
1236	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1237	if (!priv)
1238		return -ENOMEM;
1239
1240	priv->cfg = of_device_get_match_data(&pdev->dev);
1241	if (!priv->cfg)
1242		return -ENODEV;
1243
1244	priv->base = devm_platform_ioremap_resource(pdev, 0);
1245	if (IS_ERR(priv->base))
1246		return PTR_ERR(priv->base);
1247
1248	priv->irq = platform_get_irq(pdev, 0);
1249	if (priv->irq < 0)
1250		return priv->irq;
1251
1252	priv->clk = devm_clk_get_enabled(&pdev->dev, NULL);
1253	if (IS_ERR(priv->clk))
1254		return dev_err_probe(&pdev->dev, PTR_ERR(priv->clk),
1255				     "Couldn't start the clock\n");
1256
1257	if (priv->cfg->has_reset) {
1258		priv->rst = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL);
1259		if (IS_ERR(priv->rst))
1260			return dev_err_probe(&pdev->dev, PTR_ERR(priv->rst),
1261					     "Failed to get reset control\n");
1262	}
1263
1264	platform_set_drvdata(pdev, priv);
1265	spin_lock_init(&priv->lock);
1266
1267	dma_set_max_seg_size(&pdev->dev, SUN4I_DMA_MAX_SEG_SIZE);
1268
1269	dma_cap_zero(priv->slave.cap_mask);
1270	dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1271	dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1272	dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1273	dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1274
1275	INIT_LIST_HEAD(&priv->slave.channels);
1276	priv->slave.device_free_chan_resources	= sun4i_dma_free_chan_resources;
1277	priv->slave.device_tx_status		= sun4i_dma_tx_status;
1278	priv->slave.device_issue_pending	= sun4i_dma_issue_pending;
1279	priv->slave.device_prep_slave_sg	= sun4i_dma_prep_slave_sg;
1280	priv->slave.device_prep_dma_memcpy	= sun4i_dma_prep_dma_memcpy;
1281	priv->slave.device_prep_dma_cyclic	= sun4i_dma_prep_dma_cyclic;
1282	priv->slave.device_config		= sun4i_dma_config;
1283	priv->slave.device_terminate_all	= sun4i_dma_terminate_all;
1284	priv->slave.copy_align			= 2;
1285	priv->slave.src_addr_widths		= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1286						  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1287						  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1288	priv->slave.dst_addr_widths		= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1289						  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1290						  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1291	priv->slave.directions			= BIT(DMA_DEV_TO_MEM) |
1292						  BIT(DMA_MEM_TO_DEV);
1293	priv->slave.residue_granularity		= DMA_RESIDUE_GRANULARITY_BURST;
1294
1295	priv->slave.dev = &pdev->dev;
1296
1297	priv->pchans = devm_kcalloc(&pdev->dev, priv->cfg->dma_nr_max_channels,
1298				    sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1299	priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1300				    sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1301	priv->pchans_used = devm_kcalloc(&pdev->dev,
1302					 BITS_TO_LONGS(priv->cfg->dma_nr_max_channels),
1303					 sizeof(unsigned long), GFP_KERNEL);
1304	if (!priv->vchans || !priv->pchans || !priv->pchans_used)
1305		return -ENOMEM;
1306
1307	/*
1308	 * [0..priv->cfg->ndma_nr_max_channels) are normal pchans, and
1309	 * [priv->cfg->ndma_nr_max_channels..priv->cfg->dma_nr_max_channels) are
1310	 * dedicated ones
1311	 */
1312	for (i = 0; i < priv->cfg->ndma_nr_max_channels; i++)
1313		priv->pchans[i].base = priv->base +
1314			SUN4I_NDMA_CHANNEL_REG_BASE(i);
1315
1316	for (j = 0; i < priv->cfg->dma_nr_max_channels; i++, j++) {
1317		priv->pchans[i].base = priv->base +
1318			SUN4I_DDMA_CHANNEL_REG_BASE(j);
1319		priv->pchans[i].is_dedicated = 1;
1320	}
1321
1322	for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1323		struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1324
1325		spin_lock_init(&vchan->vc.lock);
1326		vchan->vc.desc_free = sun4i_dma_free_contract;
1327		vchan_init(&vchan->vc, &priv->slave);
1328	}
1329
1330	/*
1331	 * Make sure the IRQs are all disabled and accounted for. The bootloader
1332	 * likes to leave these dirty
1333	 */
1334	writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1335	writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1336
1337	ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1338			       0, dev_name(&pdev->dev), priv);
1339	if (ret)
1340		return dev_err_probe(&pdev->dev, ret, "Cannot request IRQ\n");
1341
1342	ret = dmaenginem_async_device_register(&priv->slave);
1343	if (ret)
1344		return dev_err_probe(&pdev->dev, ret,
1345				     "Failed to register DMA engine device\n");
1346
1347	ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1348					 priv);
1349	if (ret)
1350		return dev_err_probe(&pdev->dev, ret,
1351				     "Failed to register translation function\n");
1352
1353	dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1354
1355	return 0;
1356}
1357
1358static void sun4i_dma_remove(struct platform_device *pdev)
1359{
1360	struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1361
1362	/* Disable IRQ so no more work is scheduled */
1363	disable_irq(priv->irq);
1364
1365	of_dma_controller_free(pdev->dev.of_node);
1366}
1367
1368static struct sun4i_dma_config sun4i_a10_dma_cfg = {
1369	.ndma_nr_max_channels	= SUN4I_NDMA_NR_MAX_CHANNELS,
1370	.ndma_nr_max_vchans	= SUN4I_NDMA_NR_MAX_VCHANS,
1371
1372	.ddma_nr_max_channels	= SUN4I_DDMA_NR_MAX_CHANNELS,
1373	.ddma_nr_max_vchans	= SUN4I_DDMA_NR_MAX_VCHANS,
1374
1375	.dma_nr_max_channels	= SUN4I_DMA_NR_MAX_CHANNELS,
1376
1377	.set_dst_data_width	= set_dst_data_width_a10,
1378	.set_src_data_width	= set_src_data_width_a10,
1379	.convert_burst		= convert_burst_a10,
1380
1381	.ndma_drq_sdram		= SUN4I_NDMA_DRQ_TYPE_SDRAM,
1382	.ddma_drq_sdram		= SUN4I_DDMA_DRQ_TYPE_SDRAM,
1383
1384	.max_burst		= SUN4I_MAX_BURST,
1385	.has_reset		= false,
1386};
1387
1388static struct sun4i_dma_config suniv_f1c100s_dma_cfg = {
1389	.ndma_nr_max_channels	= SUNIV_NDMA_NR_MAX_CHANNELS,
1390	.ndma_nr_max_vchans	= SUNIV_NDMA_NR_MAX_VCHANS,
1391
1392	.ddma_nr_max_channels	= SUNIV_DDMA_NR_MAX_CHANNELS,
1393	.ddma_nr_max_vchans	= SUNIV_DDMA_NR_MAX_VCHANS,
1394
1395	.dma_nr_max_channels	= SUNIV_NDMA_NR_MAX_CHANNELS +
1396		SUNIV_DDMA_NR_MAX_CHANNELS,
1397
1398	.set_dst_data_width	= set_dst_data_width_f1c100s,
1399	.set_src_data_width	= set_src_data_width_f1c100s,
1400	.convert_burst		= convert_burst_f1c100s,
1401
1402	.ndma_drq_sdram		= SUNIV_NDMA_DRQ_TYPE_SDRAM,
1403	.ddma_drq_sdram		= SUNIV_DDMA_DRQ_TYPE_SDRAM,
1404
1405	.max_burst		= SUNIV_MAX_BURST,
1406	.has_reset		= true,
1407};
1408
1409static const struct of_device_id sun4i_dma_match[] = {
1410	{ .compatible = "allwinner,sun4i-a10-dma", .data = &sun4i_a10_dma_cfg },
1411	{ .compatible = "allwinner,suniv-f1c100s-dma",
1412		.data = &suniv_f1c100s_dma_cfg },
1413	{ /* sentinel */ },
1414};
1415MODULE_DEVICE_TABLE(of, sun4i_dma_match);
1416
1417static struct platform_driver sun4i_dma_driver = {
1418	.probe	= sun4i_dma_probe,
1419	.remove = sun4i_dma_remove,
1420	.driver	= {
1421		.name		= "sun4i-dma",
1422		.of_match_table	= sun4i_dma_match,
1423	},
1424};
1425
1426module_platform_driver(sun4i_dma_driver);
1427
1428MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1429MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1430MODULE_LICENSE("GPL");