omap-dma.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * OMAP DMAengine support
   4 */
   5#include <linux/cpu_pm.h>
   6#include <linux/delay.h>
   7#include <linux/dmaengine.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/dmapool.h>
  10#include <linux/err.h>
  11#include <linux/init.h>
  12#include <linux/interrupt.h>
  13#include <linux/list.h>
  14#include <linux/module.h>
  15#include <linux/omap-dma.h>
  16#include <linux/platform_device.h>
  17#include <linux/slab.h>
  18#include <linux/spinlock.h>
  19#include <linux/of.h>
  20#include <linux/of_dma.h>
  21
  22#include "../virt-dma.h"
  23
  24#define OMAP_SDMA_REQUESTS	127
  25#define OMAP_SDMA_CHANNELS	32
  26
  27struct omap_dma_config {
  28	int lch_end;
  29	unsigned int rw_priority:1;
  30	unsigned int needs_busy_check:1;
  31	unsigned int may_lose_context:1;
  32	unsigned int needs_lch_clear:1;
  33};
  34
  35struct omap_dma_context {
  36	u32 irqenable_l0;
  37	u32 irqenable_l1;
  38	u32 ocp_sysconfig;
  39	u32 gcr;
  40};
  41
  42struct omap_dmadev {
  43	struct dma_device ddev;
  44	spinlock_t lock;
  45	void __iomem *base;
  46	const struct omap_dma_reg *reg_map;
  47	struct omap_system_dma_plat_info *plat;
  48	const struct omap_dma_config *cfg;
  49	struct notifier_block nb;
  50	struct omap_dma_context context;
  51	int lch_count;
  52	DECLARE_BITMAP(lch_bitmap, OMAP_SDMA_CHANNELS);
  53	struct mutex lch_lock;		/* for assigning logical channels */
  54	bool legacy;
  55	bool ll123_supported;
  56	struct dma_pool *desc_pool;
  57	unsigned dma_requests;
  58	spinlock_t irq_lock;
  59	uint32_t irq_enable_mask;
  60	struct omap_chan **lch_map;
  61};
  62
  63struct omap_chan {
  64	struct virt_dma_chan vc;
  65	void __iomem *channel_base;
  66	const struct omap_dma_reg *reg_map;
  67	uint32_t ccr;
  68
  69	struct dma_slave_config	cfg;
  70	unsigned dma_sig;
  71	bool cyclic;
  72	bool paused;
  73	bool running;
  74
  75	int dma_ch;
  76	struct omap_desc *desc;
  77	unsigned sgidx;
  78};
  79
  80#define DESC_NXT_SV_REFRESH	(0x1 << 24)
  81#define DESC_NXT_SV_REUSE	(0x2 << 24)
  82#define DESC_NXT_DV_REFRESH	(0x1 << 26)
  83#define DESC_NXT_DV_REUSE	(0x2 << 26)
  84#define DESC_NTYPE_TYPE2	(0x2 << 29)
  85
  86/* Type 2 descriptor with Source or Destination address update */
  87struct omap_type2_desc {
  88	uint32_t next_desc;
  89	uint32_t en;
  90	uint32_t addr; /* src or dst */
  91	uint16_t fn;
  92	uint16_t cicr;
  93	int16_t cdei;
  94	int16_t csei;
  95	int32_t cdfi;
  96	int32_t csfi;
  97} __packed;
  98
  99struct omap_sg {
 100	dma_addr_t addr;
 101	uint32_t en;		/* number of elements (24-bit) */
 102	uint32_t fn;		/* number of frames (16-bit) */
 103	int32_t fi;		/* for double indexing */
 104	int16_t ei;		/* for double indexing */
 105
 106	/* Linked list */
 107	struct omap_type2_desc *t2_desc;
 108	dma_addr_t t2_desc_paddr;
 109};
 110
 111struct omap_desc {
 112	struct virt_dma_desc vd;
 113	bool using_ll;
 114	enum dma_transfer_direction dir;
 115	dma_addr_t dev_addr;
 116	bool polled;
 117
 118	int32_t fi;		/* for OMAP_DMA_SYNC_PACKET / double indexing */
 119	int16_t ei;		/* for double indexing */
 120	uint8_t es;		/* CSDP_DATA_TYPE_xxx */
 121	uint32_t ccr;		/* CCR value */
 122	uint16_t clnk_ctrl;	/* CLNK_CTRL value */
 123	uint16_t cicr;		/* CICR value */
 124	uint32_t csdp;		/* CSDP value */
 125
 126	unsigned sglen;
 127	struct omap_sg sg[] __counted_by(sglen);
 128};
 129
 130enum {
 131	CAPS_0_SUPPORT_LL123	= BIT(20),	/* Linked List type1/2/3 */
 132	CAPS_0_SUPPORT_LL4	= BIT(21),	/* Linked List type4 */
 133
 134	CCR_FS			= BIT(5),
 135	CCR_READ_PRIORITY	= BIT(6),
 136	CCR_ENABLE		= BIT(7),
 137	CCR_AUTO_INIT		= BIT(8),	/* OMAP1 only */
 138	CCR_REPEAT		= BIT(9),	/* OMAP1 only */
 139	CCR_OMAP31_DISABLE	= BIT(10),	/* OMAP1 only */
 140	CCR_SUSPEND_SENSITIVE	= BIT(8),	/* OMAP2+ only */
 141	CCR_RD_ACTIVE		= BIT(9),	/* OMAP2+ only */
 142	CCR_WR_ACTIVE		= BIT(10),	/* OMAP2+ only */
 143	CCR_SRC_AMODE_CONSTANT	= 0 << 12,
 144	CCR_SRC_AMODE_POSTINC	= 1 << 12,
 145	CCR_SRC_AMODE_SGLIDX	= 2 << 12,
 146	CCR_SRC_AMODE_DBLIDX	= 3 << 12,
 147	CCR_DST_AMODE_CONSTANT	= 0 << 14,
 148	CCR_DST_AMODE_POSTINC	= 1 << 14,
 149	CCR_DST_AMODE_SGLIDX	= 2 << 14,
 150	CCR_DST_AMODE_DBLIDX	= 3 << 14,
 151	CCR_CONSTANT_FILL	= BIT(16),
 152	CCR_TRANSPARENT_COPY	= BIT(17),
 153	CCR_BS			= BIT(18),
 154	CCR_SUPERVISOR		= BIT(22),
 155	CCR_PREFETCH		= BIT(23),
 156	CCR_TRIGGER_SRC		= BIT(24),
 157	CCR_BUFFERING_DISABLE	= BIT(25),
 158	CCR_WRITE_PRIORITY	= BIT(26),
 159	CCR_SYNC_ELEMENT	= 0,
 160	CCR_SYNC_FRAME		= CCR_FS,
 161	CCR_SYNC_BLOCK		= CCR_BS,
 162	CCR_SYNC_PACKET		= CCR_BS | CCR_FS,
 163
 164	CSDP_DATA_TYPE_8	= 0,
 165	CSDP_DATA_TYPE_16	= 1,
 166	CSDP_DATA_TYPE_32	= 2,
 167	CSDP_SRC_PORT_EMIFF	= 0 << 2, /* OMAP1 only */
 168	CSDP_SRC_PORT_EMIFS	= 1 << 2, /* OMAP1 only */
 169	CSDP_SRC_PORT_OCP_T1	= 2 << 2, /* OMAP1 only */
 170	CSDP_SRC_PORT_TIPB	= 3 << 2, /* OMAP1 only */
 171	CSDP_SRC_PORT_OCP_T2	= 4 << 2, /* OMAP1 only */
 172	CSDP_SRC_PORT_MPUI	= 5 << 2, /* OMAP1 only */
 173	CSDP_SRC_PACKED		= BIT(6),
 174	CSDP_SRC_BURST_1	= 0 << 7,
 175	CSDP_SRC_BURST_16	= 1 << 7,
 176	CSDP_SRC_BURST_32	= 2 << 7,
 177	CSDP_SRC_BURST_64	= 3 << 7,
 178	CSDP_DST_PORT_EMIFF	= 0 << 9, /* OMAP1 only */
 179	CSDP_DST_PORT_EMIFS	= 1 << 9, /* OMAP1 only */
 180	CSDP_DST_PORT_OCP_T1	= 2 << 9, /* OMAP1 only */
 181	CSDP_DST_PORT_TIPB	= 3 << 9, /* OMAP1 only */
 182	CSDP_DST_PORT_OCP_T2	= 4 << 9, /* OMAP1 only */
 183	CSDP_DST_PORT_MPUI	= 5 << 9, /* OMAP1 only */
 184	CSDP_DST_PACKED		= BIT(13),
 185	CSDP_DST_BURST_1	= 0 << 14,
 186	CSDP_DST_BURST_16	= 1 << 14,
 187	CSDP_DST_BURST_32	= 2 << 14,
 188	CSDP_DST_BURST_64	= 3 << 14,
 189	CSDP_WRITE_NON_POSTED	= 0 << 16,
 190	CSDP_WRITE_POSTED	= 1 << 16,
 191	CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
 192
 193	CICR_TOUT_IE		= BIT(0),	/* OMAP1 only */
 194	CICR_DROP_IE		= BIT(1),
 195	CICR_HALF_IE		= BIT(2),
 196	CICR_FRAME_IE		= BIT(3),
 197	CICR_LAST_IE		= BIT(4),
 198	CICR_BLOCK_IE		= BIT(5),
 199	CICR_PKT_IE		= BIT(7),	/* OMAP2+ only */
 200	CICR_TRANS_ERR_IE	= BIT(8),	/* OMAP2+ only */
 201	CICR_SUPERVISOR_ERR_IE	= BIT(10),	/* OMAP2+ only */
 202	CICR_MISALIGNED_ERR_IE	= BIT(11),	/* OMAP2+ only */
 203	CICR_DRAIN_IE		= BIT(12),	/* OMAP2+ only */
 204	CICR_SUPER_BLOCK_IE	= BIT(14),	/* OMAP2+ only */
 205
 206	CLNK_CTRL_ENABLE_LNK	= BIT(15),
 207
 208	CDP_DST_VALID_INC	= 0 << 0,
 209	CDP_DST_VALID_RELOAD	= 1 << 0,
 210	CDP_DST_VALID_REUSE	= 2 << 0,
 211	CDP_SRC_VALID_INC	= 0 << 2,
 212	CDP_SRC_VALID_RELOAD	= 1 << 2,
 213	CDP_SRC_VALID_REUSE	= 2 << 2,
 214	CDP_NTYPE_TYPE1		= 1 << 4,
 215	CDP_NTYPE_TYPE2		= 2 << 4,
 216	CDP_NTYPE_TYPE3		= 3 << 4,
 217	CDP_TMODE_NORMAL	= 0 << 8,
 218	CDP_TMODE_LLIST		= 1 << 8,
 219	CDP_FAST		= BIT(10),
 220};
 221
 222static const unsigned es_bytes[] = {
 223	[CSDP_DATA_TYPE_8] = 1,
 224	[CSDP_DATA_TYPE_16] = 2,
 225	[CSDP_DATA_TYPE_32] = 4,
 226};
 227
 228static bool omap_dma_filter_fn(struct dma_chan *chan, void *param);
 229static struct of_dma_filter_info omap_dma_info = {
 230	.filter_fn = omap_dma_filter_fn,
 231};
 232
 233static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
 234{
 235	return container_of(d, struct omap_dmadev, ddev);
 236}
 237
 238static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
 239{
 240	return container_of(c, struct omap_chan, vc.chan);
 241}
 242
 243static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
 244{
 245	return container_of(t, struct omap_desc, vd.tx);
 246}
 247
 248static void omap_dma_desc_free(struct virt_dma_desc *vd)
 249{
 250	struct omap_desc *d = to_omap_dma_desc(&vd->tx);
 251
 252	if (d->using_ll) {
 253		struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
 254		int i;
 255
 256		for (i = 0; i < d->sglen; i++) {
 257			if (d->sg[i].t2_desc)
 258				dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
 259					      d->sg[i].t2_desc_paddr);
 260		}
 261	}
 262
 263	kfree(d);
 264}
 265
 266static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
 267				     enum dma_transfer_direction dir, bool last)
 268{
 269	struct omap_sg *sg = &d->sg[idx];
 270	struct omap_type2_desc *t2_desc = sg->t2_desc;
 271
 272	if (idx)
 273		d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
 274	if (last)
 275		t2_desc->next_desc = 0xfffffffc;
 276
 277	t2_desc->en = sg->en;
 278	t2_desc->addr = sg->addr;
 279	t2_desc->fn = sg->fn & 0xffff;
 280	t2_desc->cicr = d->cicr;
 281	if (!last)
 282		t2_desc->cicr &= ~CICR_BLOCK_IE;
 283
 284	switch (dir) {
 285	case DMA_DEV_TO_MEM:
 286		t2_desc->cdei = sg->ei;
 287		t2_desc->csei = d->ei;
 288		t2_desc->cdfi = sg->fi;
 289		t2_desc->csfi = d->fi;
 290
 291		t2_desc->en |= DESC_NXT_DV_REFRESH;
 292		t2_desc->en |= DESC_NXT_SV_REUSE;
 293		break;
 294	case DMA_MEM_TO_DEV:
 295		t2_desc->cdei = d->ei;
 296		t2_desc->csei = sg->ei;
 297		t2_desc->cdfi = d->fi;
 298		t2_desc->csfi = sg->fi;
 299
 300		t2_desc->en |= DESC_NXT_SV_REFRESH;
 301		t2_desc->en |= DESC_NXT_DV_REUSE;
 302		break;
 303	default:
 304		return;
 305	}
 306
 307	t2_desc->en |= DESC_NTYPE_TYPE2;
 308}
 309
 310static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
 311{
 312	switch (type) {
 313	case OMAP_DMA_REG_16BIT:
 314		writew_relaxed(val, addr);
 315		break;
 316	case OMAP_DMA_REG_2X16BIT:
 317		writew_relaxed(val, addr);
 318		writew_relaxed(val >> 16, addr + 2);
 319		break;
 320	case OMAP_DMA_REG_32BIT:
 321		writel_relaxed(val, addr);
 322		break;
 323	default:
 324		WARN_ON(1);
 325	}
 326}
 327
 328static unsigned omap_dma_read(unsigned type, void __iomem *addr)
 329{
 330	unsigned val;
 331
 332	switch (type) {
 333	case OMAP_DMA_REG_16BIT:
 334		val = readw_relaxed(addr);
 335		break;
 336	case OMAP_DMA_REG_2X16BIT:
 337		val = readw_relaxed(addr);
 338		val |= readw_relaxed(addr + 2) << 16;
 339		break;
 340	case OMAP_DMA_REG_32BIT:
 341		val = readl_relaxed(addr);
 342		break;
 343	default:
 344		WARN_ON(1);
 345		val = 0;
 346	}
 347
 348	return val;
 349}
 350
 351static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
 352{
 353	const struct omap_dma_reg *r = od->reg_map + reg;
 354
 355	WARN_ON(r->stride);
 356
 357	omap_dma_write(val, r->type, od->base + r->offset);
 358}
 359
 360static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
 361{
 362	const struct omap_dma_reg *r = od->reg_map + reg;
 363
 364	WARN_ON(r->stride);
 365
 366	return omap_dma_read(r->type, od->base + r->offset);
 367}
 368
 369static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
 370{
 371	const struct omap_dma_reg *r = c->reg_map + reg;
 372
 373	omap_dma_write(val, r->type, c->channel_base + r->offset);
 374}
 375
 376static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
 377{
 378	const struct omap_dma_reg *r = c->reg_map + reg;
 379
 380	return omap_dma_read(r->type, c->channel_base + r->offset);
 381}
 382
 383static void omap_dma_clear_csr(struct omap_chan *c)
 384{
 385	if (dma_omap1())
 386		omap_dma_chan_read(c, CSR);
 387	else
 388		omap_dma_chan_write(c, CSR, ~0);
 389}
 390
 391static unsigned omap_dma_get_csr(struct omap_chan *c)
 392{
 393	unsigned val = omap_dma_chan_read(c, CSR);
 394
 395	if (!dma_omap1())
 396		omap_dma_chan_write(c, CSR, val);
 397
 398	return val;
 399}
 400
 401static void omap_dma_clear_lch(struct omap_dmadev *od, int lch)
 402{
 403	struct omap_chan *c;
 404	int i;
 405
 406	c = od->lch_map[lch];
 407	if (!c)
 408		return;
 409
 410	for (i = CSDP; i <= od->cfg->lch_end; i++)
 411		omap_dma_chan_write(c, i, 0);
 412}
 413
 414static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
 415	unsigned lch)
 416{
 417	c->channel_base = od->base + od->plat->channel_stride * lch;
 418
 419	od->lch_map[lch] = c;
 420}
 421
 422static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
 423{
 424	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 425	uint16_t cicr = d->cicr;
 426
 427	if (__dma_omap15xx(od->plat->dma_attr))
 428		omap_dma_chan_write(c, CPC, 0);
 429	else
 430		omap_dma_chan_write(c, CDAC, 0);
 431
 432	omap_dma_clear_csr(c);
 433
 434	if (d->using_ll) {
 435		uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
 436
 437		if (d->dir == DMA_DEV_TO_MEM)
 438			cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
 439		else
 440			cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
 441		omap_dma_chan_write(c, CDP, cdp);
 442
 443		omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
 444		omap_dma_chan_write(c, CCDN, 0);
 445		omap_dma_chan_write(c, CCFN, 0xffff);
 446		omap_dma_chan_write(c, CCEN, 0xffffff);
 447
 448		cicr &= ~CICR_BLOCK_IE;
 449	} else if (od->ll123_supported) {
 450		omap_dma_chan_write(c, CDP, 0);
 451	}
 452
 453	/* Enable interrupts */
 454	omap_dma_chan_write(c, CICR, cicr);
 455
 456	/* Enable channel */
 457	omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
 458
 459	c->running = true;
 460}
 461
 462static void omap_dma_drain_chan(struct omap_chan *c)
 463{
 464	int i;
 465	u32 val;
 466
 467	/* Wait for sDMA FIFO to drain */
 468	for (i = 0; ; i++) {
 469		val = omap_dma_chan_read(c, CCR);
 470		if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
 471			break;
 472
 473		if (i > 100)
 474			break;
 475
 476		udelay(5);
 477	}
 478
 479	if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
 480		dev_err(c->vc.chan.device->dev,
 481			"DMA drain did not complete on lch %d\n",
 482			c->dma_ch);
 483}
 484
 485static int omap_dma_stop(struct omap_chan *c)
 486{
 487	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 488	uint32_t val;
 489
 490	/* disable irq */
 491	omap_dma_chan_write(c, CICR, 0);
 492
 493	omap_dma_clear_csr(c);
 494
 495	val = omap_dma_chan_read(c, CCR);
 496	if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
 497		uint32_t sysconfig;
 498
 499		sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
 500		val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
 501		val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
 502		omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
 503
 504		val = omap_dma_chan_read(c, CCR);
 505		val &= ~CCR_ENABLE;
 506		omap_dma_chan_write(c, CCR, val);
 507
 508		if (!(c->ccr & CCR_BUFFERING_DISABLE))
 509			omap_dma_drain_chan(c);
 510
 511		omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
 512	} else {
 513		if (!(val & CCR_ENABLE))
 514			return -EINVAL;
 515
 516		val &= ~CCR_ENABLE;
 517		omap_dma_chan_write(c, CCR, val);
 518
 519		if (!(c->ccr & CCR_BUFFERING_DISABLE))
 520			omap_dma_drain_chan(c);
 521	}
 522
 523	mb();
 524
 525	if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
 526		val = omap_dma_chan_read(c, CLNK_CTRL);
 527
 528		if (dma_omap1())
 529			val |= 1 << 14; /* set the STOP_LNK bit */
 530		else
 531			val &= ~CLNK_CTRL_ENABLE_LNK;
 532
 533		omap_dma_chan_write(c, CLNK_CTRL, val);
 534	}
 535	c->running = false;
 536	return 0;
 537}
 538
 539static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
 540{
 541	struct omap_sg *sg = d->sg + c->sgidx;
 542	unsigned cxsa, cxei, cxfi;
 543
 544	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 545		cxsa = CDSA;
 546		cxei = CDEI;
 547		cxfi = CDFI;
 548	} else {
 549		cxsa = CSSA;
 550		cxei = CSEI;
 551		cxfi = CSFI;
 552	}
 553
 554	omap_dma_chan_write(c, cxsa, sg->addr);
 555	omap_dma_chan_write(c, cxei, sg->ei);
 556	omap_dma_chan_write(c, cxfi, sg->fi);
 557	omap_dma_chan_write(c, CEN, sg->en);
 558	omap_dma_chan_write(c, CFN, sg->fn);
 559
 560	omap_dma_start(c, d);
 561	c->sgidx++;
 562}
 563
 564static void omap_dma_start_desc(struct omap_chan *c)
 565{
 566	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 567	struct omap_desc *d;
 568	unsigned cxsa, cxei, cxfi;
 569
 570	if (!vd) {
 571		c->desc = NULL;
 572		return;
 573	}
 574
 575	list_del(&vd->node);
 576
 577	c->desc = d = to_omap_dma_desc(&vd->tx);
 578	c->sgidx = 0;
 579
 580	/*
 581	 * This provides the necessary barrier to ensure data held in
 582	 * DMA coherent memory is visible to the DMA engine prior to
 583	 * the transfer starting.
 584	 */
 585	mb();
 586
 587	omap_dma_chan_write(c, CCR, d->ccr);
 588	if (dma_omap1())
 589		omap_dma_chan_write(c, CCR2, d->ccr >> 16);
 590
 591	if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
 592		cxsa = CSSA;
 593		cxei = CSEI;
 594		cxfi = CSFI;
 595	} else {
 596		cxsa = CDSA;
 597		cxei = CDEI;
 598		cxfi = CDFI;
 599	}
 600
 601	omap_dma_chan_write(c, cxsa, d->dev_addr);
 602	omap_dma_chan_write(c, cxei, d->ei);
 603	omap_dma_chan_write(c, cxfi, d->fi);
 604	omap_dma_chan_write(c, CSDP, d->csdp);
 605	omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
 606
 607	omap_dma_start_sg(c, d);
 608}
 609
 610static void omap_dma_callback(int ch, u16 status, void *data)
 611{
 612	struct omap_chan *c = data;
 613	struct omap_desc *d;
 614	unsigned long flags;
 615
 616	spin_lock_irqsave(&c->vc.lock, flags);
 617	d = c->desc;
 618	if (d) {
 619		if (c->cyclic) {
 620			vchan_cyclic_callback(&d->vd);
 621		} else if (d->using_ll || c->sgidx == d->sglen) {
 622			omap_dma_start_desc(c);
 623			vchan_cookie_complete(&d->vd);
 624		} else {
 625			omap_dma_start_sg(c, d);
 626		}
 627	}
 628	spin_unlock_irqrestore(&c->vc.lock, flags);
 629}
 630
 631static irqreturn_t omap_dma_irq(int irq, void *devid)
 632{
 633	struct omap_dmadev *od = devid;
 634	unsigned status, channel;
 635
 636	spin_lock(&od->irq_lock);
 637
 638	status = omap_dma_glbl_read(od, IRQSTATUS_L1);
 639	status &= od->irq_enable_mask;
 640	if (status == 0) {
 641		spin_unlock(&od->irq_lock);
 642		return IRQ_NONE;
 643	}
 644
 645	while ((channel = ffs(status)) != 0) {
 646		unsigned mask, csr;
 647		struct omap_chan *c;
 648
 649		channel -= 1;
 650		mask = BIT(channel);
 651		status &= ~mask;
 652
 653		c = od->lch_map[channel];
 654		if (c == NULL) {
 655			/* This should never happen */
 656			dev_err(od->ddev.dev, "invalid channel %u\n", channel);
 657			continue;
 658		}
 659
 660		csr = omap_dma_get_csr(c);
 661		omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
 662
 663		omap_dma_callback(channel, csr, c);
 664	}
 665
 666	spin_unlock(&od->irq_lock);
 667
 668	return IRQ_HANDLED;
 669}
 670
 671static int omap_dma_get_lch(struct omap_dmadev *od, int *lch)
 672{
 673	int channel;
 674
 675	mutex_lock(&od->lch_lock);
 676	channel = find_first_zero_bit(od->lch_bitmap, od->lch_count);
 677	if (channel >= od->lch_count)
 678		goto out_busy;
 679	set_bit(channel, od->lch_bitmap);
 680	mutex_unlock(&od->lch_lock);
 681
 682	omap_dma_clear_lch(od, channel);
 683	*lch = channel;
 684
 685	return 0;
 686
 687out_busy:
 688	mutex_unlock(&od->lch_lock);
 689	*lch = -EINVAL;
 690
 691	return -EBUSY;
 692}
 693
 694static void omap_dma_put_lch(struct omap_dmadev *od, int lch)
 695{
 696	omap_dma_clear_lch(od, lch);
 697	mutex_lock(&od->lch_lock);
 698	clear_bit(lch, od->lch_bitmap);
 699	mutex_unlock(&od->lch_lock);
 700}
 701
 702static inline bool omap_dma_legacy(struct omap_dmadev *od)
 703{
 704	return IS_ENABLED(CONFIG_ARCH_OMAP1) && od->legacy;
 705}
 706
 707static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
 708{
 709	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 710	struct omap_chan *c = to_omap_dma_chan(chan);
 711	struct device *dev = od->ddev.dev;
 712	int ret;
 713
 714	if (omap_dma_legacy(od)) {
 715		ret = omap_request_dma(c->dma_sig, "DMA engine",
 716				       omap_dma_callback, c, &c->dma_ch);
 717	} else {
 718		ret = omap_dma_get_lch(od, &c->dma_ch);
 719	}
 720
 721	dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
 722
 723	if (ret >= 0) {
 724		omap_dma_assign(od, c, c->dma_ch);
 725
 726		if (!omap_dma_legacy(od)) {
 727			unsigned val;
 728
 729			spin_lock_irq(&od->irq_lock);
 730			val = BIT(c->dma_ch);
 731			omap_dma_glbl_write(od, IRQSTATUS_L1, val);
 732			od->irq_enable_mask |= val;
 733			omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 734
 735			val = omap_dma_glbl_read(od, IRQENABLE_L0);
 736			val &= ~BIT(c->dma_ch);
 737			omap_dma_glbl_write(od, IRQENABLE_L0, val);
 738			spin_unlock_irq(&od->irq_lock);
 739		}
 740	}
 741
 742	if (dma_omap1()) {
 743		if (__dma_omap16xx(od->plat->dma_attr)) {
 744			c->ccr = CCR_OMAP31_DISABLE;
 745			/* Duplicate what plat-omap/dma.c does */
 746			c->ccr |= c->dma_ch + 1;
 747		} else {
 748			c->ccr = c->dma_sig & 0x1f;
 749		}
 750	} else {
 751		c->ccr = c->dma_sig & 0x1f;
 752		c->ccr |= (c->dma_sig & ~0x1f) << 14;
 753	}
 754	if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
 755		c->ccr |= CCR_BUFFERING_DISABLE;
 756
 757	return ret;
 758}
 759
 760static void omap_dma_free_chan_resources(struct dma_chan *chan)
 761{
 762	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 763	struct omap_chan *c = to_omap_dma_chan(chan);
 764
 765	if (!omap_dma_legacy(od)) {
 766		spin_lock_irq(&od->irq_lock);
 767		od->irq_enable_mask &= ~BIT(c->dma_ch);
 768		omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
 769		spin_unlock_irq(&od->irq_lock);
 770	}
 771
 772	c->channel_base = NULL;
 773	od->lch_map[c->dma_ch] = NULL;
 774	vchan_free_chan_resources(&c->vc);
 775
 776	if (omap_dma_legacy(od))
 777		omap_free_dma(c->dma_ch);
 778	else
 779		omap_dma_put_lch(od, c->dma_ch);
 780
 781	dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
 782		c->dma_sig);
 783	c->dma_sig = 0;
 784}
 785
 786static size_t omap_dma_sg_size(struct omap_sg *sg)
 787{
 788	return sg->en * sg->fn;
 789}
 790
 791static size_t omap_dma_desc_size(struct omap_desc *d)
 792{
 793	unsigned i;
 794	size_t size;
 795
 796	for (size = i = 0; i < d->sglen; i++)
 797		size += omap_dma_sg_size(&d->sg[i]);
 798
 799	return size * es_bytes[d->es];
 800}
 801
 802static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
 803{
 804	unsigned i;
 805	size_t size, es_size = es_bytes[d->es];
 806
 807	for (size = i = 0; i < d->sglen; i++) {
 808		size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
 809
 810		if (size)
 811			size += this_size;
 812		else if (addr >= d->sg[i].addr &&
 813			 addr < d->sg[i].addr + this_size)
 814			size += d->sg[i].addr + this_size - addr;
 815	}
 816	return size;
 817}
 818
 819/*
 820 * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
 821 * read before the DMA controller finished disabling the channel.
 822 */
 823static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
 824{
 825	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 826	uint32_t val;
 827
 828	val = omap_dma_chan_read(c, reg);
 829	if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
 830		val = omap_dma_chan_read(c, reg);
 831
 832	return val;
 833}
 834
 835static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
 836{
 837	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 838	dma_addr_t addr, cdac;
 839
 840	if (__dma_omap15xx(od->plat->dma_attr)) {
 841		addr = omap_dma_chan_read(c, CPC);
 842	} else {
 843		addr = omap_dma_chan_read_3_3(c, CSAC);
 844		cdac = omap_dma_chan_read_3_3(c, CDAC);
 845
 846		/*
 847		 * CDAC == 0 indicates that the DMA transfer on the channel has
 848		 * not been started (no data has been transferred so far).
 849		 * Return the programmed source start address in this case.
 850		 */
 851		if (cdac == 0)
 852			addr = omap_dma_chan_read(c, CSSA);
 853	}
 854
 855	if (dma_omap1())
 856		addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
 857
 858	return addr;
 859}
 860
 861static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
 862{
 863	struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
 864	dma_addr_t addr;
 865
 866	if (__dma_omap15xx(od->plat->dma_attr)) {
 867		addr = omap_dma_chan_read(c, CPC);
 868	} else {
 869		addr = omap_dma_chan_read_3_3(c, CDAC);
 870
 871		/*
 872		 * CDAC == 0 indicates that the DMA transfer on the channel
 873		 * has not been started (no data has been transferred so
 874		 * far).  Return the programmed destination start address in
 875		 * this case.
 876		 */
 877		if (addr == 0)
 878			addr = omap_dma_chan_read(c, CDSA);
 879	}
 880
 881	if (dma_omap1())
 882		addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
 883
 884	return addr;
 885}
 886
 887static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
 888	dma_cookie_t cookie, struct dma_tx_state *txstate)
 889{
 890	struct omap_chan *c = to_omap_dma_chan(chan);
 891	enum dma_status ret;
 892	unsigned long flags;
 893	struct omap_desc *d = NULL;
 894
 895	ret = dma_cookie_status(chan, cookie, txstate);
 896	if (ret == DMA_COMPLETE)
 897		return ret;
 898
 899	spin_lock_irqsave(&c->vc.lock, flags);
 900	if (c->desc && c->desc->vd.tx.cookie == cookie)
 901		d = c->desc;
 902
 903	if (!txstate)
 904		goto out;
 905
 906	if (d) {
 907		dma_addr_t pos;
 908
 909		if (d->dir == DMA_MEM_TO_DEV)
 910			pos = omap_dma_get_src_pos(c);
 911		else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
 912			pos = omap_dma_get_dst_pos(c);
 913		else
 914			pos = 0;
 915
 916		txstate->residue = omap_dma_desc_size_pos(d, pos);
 917	} else {
 918		struct virt_dma_desc *vd = vchan_find_desc(&c->vc, cookie);
 919
 920		if (vd)
 921			txstate->residue = omap_dma_desc_size(
 922						to_omap_dma_desc(&vd->tx));
 923		else
 924			txstate->residue = 0;
 925	}
 926
 927out:
 928	if (ret == DMA_IN_PROGRESS && c->paused) {
 929		ret = DMA_PAUSED;
 930	} else if (d && d->polled && c->running) {
 931		uint32_t ccr = omap_dma_chan_read(c, CCR);
 932		/*
 933		 * The channel is no longer active, set the return value
 934		 * accordingly and mark it as completed
 935		 */
 936		if (!(ccr & CCR_ENABLE)) {
 937			ret = DMA_COMPLETE;
 938			omap_dma_start_desc(c);
 939			vchan_cookie_complete(&d->vd);
 940		}
 941	}
 942
 943	spin_unlock_irqrestore(&c->vc.lock, flags);
 944
 945	return ret;
 946}
 947
 948static void omap_dma_issue_pending(struct dma_chan *chan)
 949{
 950	struct omap_chan *c = to_omap_dma_chan(chan);
 951	unsigned long flags;
 952
 953	spin_lock_irqsave(&c->vc.lock, flags);
 954	if (vchan_issue_pending(&c->vc) && !c->desc)
 955		omap_dma_start_desc(c);
 956	spin_unlock_irqrestore(&c->vc.lock, flags);
 957}
 958
 959static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
 960	struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
 961	enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
 962{
 963	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
 964	struct omap_chan *c = to_omap_dma_chan(chan);
 965	enum dma_slave_buswidth dev_width;
 966	struct scatterlist *sgent;
 967	struct omap_desc *d;
 968	dma_addr_t dev_addr;
 969	unsigned i, es, en, frame_bytes;
 970	bool ll_failed = false;
 971	u32 burst;
 972	u32 port_window, port_window_bytes;
 973
 974	if (dir == DMA_DEV_TO_MEM) {
 975		dev_addr = c->cfg.src_addr;
 976		dev_width = c->cfg.src_addr_width;
 977		burst = c->cfg.src_maxburst;
 978		port_window = c->cfg.src_port_window_size;
 979	} else if (dir == DMA_MEM_TO_DEV) {
 980		dev_addr = c->cfg.dst_addr;
 981		dev_width = c->cfg.dst_addr_width;
 982		burst = c->cfg.dst_maxburst;
 983		port_window = c->cfg.dst_port_window_size;
 984	} else {
 985		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
 986		return NULL;
 987	}
 988
 989	/* Bus width translates to the element size (ES) */
 990	switch (dev_width) {
 991	case DMA_SLAVE_BUSWIDTH_1_BYTE:
 992		es = CSDP_DATA_TYPE_8;
 993		break;
 994	case DMA_SLAVE_BUSWIDTH_2_BYTES:
 995		es = CSDP_DATA_TYPE_16;
 996		break;
 997	case DMA_SLAVE_BUSWIDTH_4_BYTES:
 998		es = CSDP_DATA_TYPE_32;
 999		break;
1000	default: /* not reached */
1001		return NULL;
1002	}
1003
1004	/* Now allocate and setup the descriptor. */
1005	d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC);
1006	if (!d)
1007		return NULL;
1008	d->sglen = sglen;
1009
1010	d->dir = dir;
1011	d->dev_addr = dev_addr;
1012	d->es = es;
1013
1014	/* When the port_window is used, one frame must cover the window */
1015	if (port_window) {
1016		burst = port_window;
1017		port_window_bytes = port_window * es_bytes[es];
1018
1019		d->ei = 1;
1020		/*
1021		 * One frame covers the port_window and by  configure
1022		 * the source frame index to be -1 * (port_window - 1)
1023		 * we instruct the sDMA that after a frame is processed
1024		 * it should move back to the start of the window.
1025		 */
1026		d->fi = -(port_window_bytes - 1);
1027	}
1028
1029	d->ccr = c->ccr | CCR_SYNC_FRAME;
1030	if (dir == DMA_DEV_TO_MEM) {
1031		d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
1032
1033		d->ccr |= CCR_DST_AMODE_POSTINC;
1034		if (port_window) {
1035			d->ccr |= CCR_SRC_AMODE_DBLIDX;
1036
1037			if (port_window_bytes >= 64)
1038				d->csdp |= CSDP_SRC_BURST_64;
1039			else if (port_window_bytes >= 32)
1040				d->csdp |= CSDP_SRC_BURST_32;
1041			else if (port_window_bytes >= 16)
1042				d->csdp |= CSDP_SRC_BURST_16;
1043
1044		} else {
1045			d->ccr |= CCR_SRC_AMODE_CONSTANT;
1046		}
1047	} else {
1048		d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
1049
1050		d->ccr |= CCR_SRC_AMODE_POSTINC;
1051		if (port_window) {
1052			d->ccr |= CCR_DST_AMODE_DBLIDX;
1053
1054			if (port_window_bytes >= 64)
1055				d->csdp |= CSDP_DST_BURST_64;
1056			else if (port_window_bytes >= 32)
1057				d->csdp |= CSDP_DST_BURST_32;
1058			else if (port_window_bytes >= 16)
1059				d->csdp |= CSDP_DST_BURST_16;
1060		} else {
1061			d->ccr |= CCR_DST_AMODE_CONSTANT;
1062		}
1063	}
1064
1065	d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
1066	d->csdp |= es;
1067
1068	if (dma_omap1()) {
1069		d->cicr |= CICR_TOUT_IE;
1070
1071		if (dir == DMA_DEV_TO_MEM)
1072			d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
1073		else
1074			d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
1075	} else {
1076		if (dir == DMA_DEV_TO_MEM)
1077			d->ccr |= CCR_TRIGGER_SRC;
1078
1079		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1080
1081		if (port_window)
1082			d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
1083	}
1084	if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
1085		d->clnk_ctrl = c->dma_ch;
1086
1087	/*
1088	 * Build our scatterlist entries: each contains the address,
1089	 * the number of elements (EN) in each frame, and the number of
1090	 * frames (FN).  Number of bytes for this entry = ES * EN * FN.
1091	 *
1092	 * Burst size translates to number of elements with frame sync.
1093	 * Note: DMA engine defines burst to be the number of dev-width
1094	 * transfers.
1095	 */
1096	en = burst;
1097	frame_bytes = es_bytes[es] * en;
1098
1099	if (sglen >= 2)
1100		d->using_ll = od->ll123_supported;
1101
1102	for_each_sg(sgl, sgent, sglen, i) {
1103		struct omap_sg *osg = &d->sg[i];
1104
1105		osg->addr = sg_dma_address(sgent);
1106		osg->en = en;
1107		osg->fn = sg_dma_len(sgent) / frame_bytes;
1108
1109		if (d->using_ll) {
1110			osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
1111						      &osg->t2_desc_paddr);
1112			if (!osg->t2_desc) {
1113				dev_err(chan->device->dev,
1114					"t2_desc[%d] allocation failed\n", i);
1115				ll_failed = true;
1116				d->using_ll = false;
1117				continue;
1118			}
1119
1120			omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
1121		}
1122	}
1123
1124	/* Release the dma_pool entries if one allocation failed */
1125	if (ll_failed) {
1126		for (i = 0; i < d->sglen; i++) {
1127			struct omap_sg *osg = &d->sg[i];
1128
1129			if (osg->t2_desc) {
1130				dma_pool_free(od->desc_pool, osg->t2_desc,
1131					      osg->t2_desc_paddr);
1132				osg->t2_desc = NULL;
1133			}
1134		}
1135	}
1136
1137	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1138}
1139
1140static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
1141	struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1142	size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
1143{
1144	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1145	struct omap_chan *c = to_omap_dma_chan(chan);
1146	enum dma_slave_buswidth dev_width;
1147	struct omap_desc *d;
1148	dma_addr_t dev_addr;
1149	unsigned es;
1150	u32 burst;
1151
1152	if (dir == DMA_DEV_TO_MEM) {
1153		dev_addr = c->cfg.src_addr;
1154		dev_width = c->cfg.src_addr_width;
1155		burst = c->cfg.src_maxburst;
1156	} else if (dir == DMA_MEM_TO_DEV) {
1157		dev_addr = c->cfg.dst_addr;
1158		dev_width = c->cfg.dst_addr_width;
1159		burst = c->cfg.dst_maxburst;
1160	} else {
1161		dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1162		return NULL;
1163	}
1164
1165	/* Bus width translates to the element size (ES) */
1166	switch (dev_width) {
1167	case DMA_SLAVE_BUSWIDTH_1_BYTE:
1168		es = CSDP_DATA_TYPE_8;
1169		break;
1170	case DMA_SLAVE_BUSWIDTH_2_BYTES:
1171		es = CSDP_DATA_TYPE_16;
1172		break;
1173	case DMA_SLAVE_BUSWIDTH_4_BYTES:
1174		es = CSDP_DATA_TYPE_32;
1175		break;
1176	default: /* not reached */
1177		return NULL;
1178	}
1179
1180	/* Now allocate and setup the descriptor. */
1181	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1182	if (!d)
1183		return NULL;
1184
1185	d->dir = dir;
1186	d->dev_addr = dev_addr;
1187	d->fi = burst;
1188	d->es = es;
1189	d->sglen = 1;
1190	d->sg[0].addr = buf_addr;
1191	d->sg[0].en = period_len / es_bytes[es];
1192	d->sg[0].fn = buf_len / period_len;
1193
1194	d->ccr = c->ccr;
1195	if (dir == DMA_DEV_TO_MEM)
1196		d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1197	else
1198		d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1199
1200	d->cicr = CICR_DROP_IE;
1201	if (flags & DMA_PREP_INTERRUPT)
1202		d->cicr |= CICR_FRAME_IE;
1203
1204	d->csdp = es;
1205
1206	if (dma_omap1()) {
1207		d->cicr |= CICR_TOUT_IE;
1208
1209		if (dir == DMA_DEV_TO_MEM)
1210			d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1211		else
1212			d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1213	} else {
1214		if (burst)
1215			d->ccr |= CCR_SYNC_PACKET;
1216		else
1217			d->ccr |= CCR_SYNC_ELEMENT;
1218
1219		if (dir == DMA_DEV_TO_MEM) {
1220			d->ccr |= CCR_TRIGGER_SRC;
1221			d->csdp |= CSDP_DST_PACKED;
1222		} else {
1223			d->csdp |= CSDP_SRC_PACKED;
1224		}
1225
1226		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1227
1228		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1229	}
1230
1231	if (__dma_omap15xx(od->plat->dma_attr))
1232		d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1233	else
1234		d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1235
1236	c->cyclic = true;
1237
1238	return vchan_tx_prep(&c->vc, &d->vd, flags);
1239}
1240
1241static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1242	struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1243	size_t len, unsigned long tx_flags)
1244{
1245	struct omap_chan *c = to_omap_dma_chan(chan);
1246	struct omap_desc *d;
1247	uint8_t data_type;
1248
1249	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1250	if (!d)
1251		return NULL;
1252
1253	data_type = __ffs((src | dest | len));
1254	if (data_type > CSDP_DATA_TYPE_32)
1255		data_type = CSDP_DATA_TYPE_32;
1256
1257	d->dir = DMA_MEM_TO_MEM;
1258	d->dev_addr = src;
1259	d->fi = 0;
1260	d->es = data_type;
1261	d->sglen = 1;
1262	d->sg[0].en = len / BIT(data_type);
1263	d->sg[0].fn = 1;
1264	d->sg[0].addr = dest;
1265	d->ccr = c->ccr;
1266	d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1267
1268	if (tx_flags & DMA_PREP_INTERRUPT)
1269		d->cicr |= CICR_FRAME_IE;
1270	else
1271		d->polled = true;
1272
1273	d->csdp = data_type;
1274
1275	if (dma_omap1()) {
1276		d->cicr |= CICR_TOUT_IE;
1277		d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1278	} else {
1279		d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1280		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1281		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1282	}
1283
1284	return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1285}
1286
1287static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1288	struct dma_chan *chan, struct dma_interleaved_template *xt,
1289	unsigned long flags)
1290{
1291	struct omap_chan *c = to_omap_dma_chan(chan);
1292	struct omap_desc *d;
1293	struct omap_sg *sg;
1294	uint8_t data_type;
1295	size_t src_icg, dst_icg;
1296
1297	/* Slave mode is not supported */
1298	if (is_slave_direction(xt->dir))
1299		return NULL;
1300
1301	if (xt->frame_size != 1 || xt->numf == 0)
1302		return NULL;
1303
1304	d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1305	if (!d)
1306		return NULL;
1307
1308	data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1309	if (data_type > CSDP_DATA_TYPE_32)
1310		data_type = CSDP_DATA_TYPE_32;
1311
1312	d->sglen = 1;
1313	sg = &d->sg[0];
1314	d->dir = DMA_MEM_TO_MEM;
1315	d->dev_addr = xt->src_start;
1316	d->es = data_type;
1317	sg->en = xt->sgl[0].size / BIT(data_type);
1318	sg->fn = xt->numf;
1319	sg->addr = xt->dst_start;
1320	d->ccr = c->ccr;
1321
1322	src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1323	dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1324	if (src_icg) {
1325		d->ccr |= CCR_SRC_AMODE_DBLIDX;
1326		d->ei = 1;
1327		d->fi = src_icg + 1;
1328	} else if (xt->src_inc) {
1329		d->ccr |= CCR_SRC_AMODE_POSTINC;
1330		d->fi = 0;
1331	} else {
1332		dev_err(chan->device->dev,
1333			"%s: SRC constant addressing is not supported\n",
1334			__func__);
1335		kfree(d);
1336		return NULL;
1337	}
1338
1339	if (dst_icg) {
1340		d->ccr |= CCR_DST_AMODE_DBLIDX;
1341		sg->ei = 1;
1342		sg->fi = dst_icg + 1;
1343	} else if (xt->dst_inc) {
1344		d->ccr |= CCR_DST_AMODE_POSTINC;
1345		sg->fi = 0;
1346	} else {
1347		dev_err(chan->device->dev,
1348			"%s: DST constant addressing is not supported\n",
1349			__func__);
1350		kfree(d);
1351		return NULL;
1352	}
1353
1354	d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1355
1356	d->csdp = data_type;
1357
1358	if (dma_omap1()) {
1359		d->cicr |= CICR_TOUT_IE;
1360		d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1361	} else {
1362		d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1363		d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1364		d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1365	}
1366
1367	return vchan_tx_prep(&c->vc, &d->vd, flags);
1368}
1369
1370static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1371{
1372	struct omap_chan *c = to_omap_dma_chan(chan);
1373
1374	if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1375	    cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1376		return -EINVAL;
1377
1378	if (cfg->src_maxburst > chan->device->max_burst ||
1379	    cfg->dst_maxburst > chan->device->max_burst)
1380		return -EINVAL;
1381
1382	memcpy(&c->cfg, cfg, sizeof(c->cfg));
1383
1384	return 0;
1385}
1386
1387static int omap_dma_terminate_all(struct dma_chan *chan)
1388{
1389	struct omap_chan *c = to_omap_dma_chan(chan);
1390	unsigned long flags;
1391	LIST_HEAD(head);
1392
1393	spin_lock_irqsave(&c->vc.lock, flags);
1394
1395	/*
1396	 * Stop DMA activity: we assume the callback will not be called
1397	 * after omap_dma_stop() returns (even if it does, it will see
1398	 * c->desc is NULL and exit.)
1399	 */
1400	if (c->desc) {
1401		vchan_terminate_vdesc(&c->desc->vd);
1402		c->desc = NULL;
1403		/* Avoid stopping the dma twice */
1404		if (!c->paused)
1405			omap_dma_stop(c);
1406	}
1407
1408	c->cyclic = false;
1409	c->paused = false;
1410
1411	vchan_get_all_descriptors(&c->vc, &head);
1412	spin_unlock_irqrestore(&c->vc.lock, flags);
1413	vchan_dma_desc_free_list(&c->vc, &head);
1414
1415	return 0;
1416}
1417
1418static void omap_dma_synchronize(struct dma_chan *chan)
1419{
1420	struct omap_chan *c = to_omap_dma_chan(chan);
1421
1422	vchan_synchronize(&c->vc);
1423}
1424
1425static int omap_dma_pause(struct dma_chan *chan)
1426{
1427	struct omap_chan *c = to_omap_dma_chan(chan);
1428	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1429	unsigned long flags;
1430	int ret = -EINVAL;
1431	bool can_pause = false;
1432
1433	spin_lock_irqsave(&od->irq_lock, flags);
1434
1435	if (!c->desc)
1436		goto out;
1437
1438	if (c->cyclic)
1439		can_pause = true;
1440
1441	/*
1442	 * We do not allow DMA_MEM_TO_DEV transfers to be paused.
1443	 * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
1444	 * "When a channel is disabled during a transfer, the channel undergoes
1445	 * an abort, unless it is hardware-source-synchronized …".
1446	 * A source-synchronised channel is one where the fetching of data is
1447	 * under control of the device. In other words, a device-to-memory
1448	 * transfer. So, a destination-synchronised channel (which would be a
1449	 * memory-to-device transfer) undergoes an abort if the CCR_ENABLE
1450	 * bit is cleared.
1451	 * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
1452	 * aborts immediately after completion of current read/write
1453	 * transactions and then the FIFO is cleaned up." The term "cleaned up"
1454	 * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
1455	 * are both clear _before_ disabling the channel, otherwise data loss
1456	 * will occur.
1457	 * The problem is that if the channel is active, then device activity
1458	 * can result in DMA activity starting between reading those as both
1459	 * clear and the write to DMA_CCR to clear the enable bit hitting the
1460	 * hardware. If the DMA hardware can't drain the data in its FIFO to the
1461	 * destination, then data loss "might" occur (say if we write to an UART
1462	 * and the UART is not accepting any further data).
1463	 */
1464	else if (c->desc->dir == DMA_DEV_TO_MEM)
1465		can_pause = true;
1466
1467	if (can_pause && !c->paused) {
1468		ret = omap_dma_stop(c);
1469		if (!ret)
1470			c->paused = true;
1471	}
1472out:
1473	spin_unlock_irqrestore(&od->irq_lock, flags);
1474
1475	return ret;
1476}
1477
1478static int omap_dma_resume(struct dma_chan *chan)
1479{
1480	struct omap_chan *c = to_omap_dma_chan(chan);
1481	struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1482	unsigned long flags;
1483	int ret = -EINVAL;
1484
1485	spin_lock_irqsave(&od->irq_lock, flags);
1486
1487	if (c->paused && c->desc) {
1488		mb();
1489
1490		/* Restore channel link register */
1491		omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1492
1493		omap_dma_start(c, c->desc);
1494		c->paused = false;
1495		ret = 0;
1496	}
1497	spin_unlock_irqrestore(&od->irq_lock, flags);
1498
1499	return ret;
1500}
1501
1502static int omap_dma_chan_init(struct omap_dmadev *od)
1503{
1504	struct omap_chan *c;
1505
1506	c = kzalloc(sizeof(*c), GFP_KERNEL);
1507	if (!c)
1508		return -ENOMEM;
1509
1510	c->reg_map = od->reg_map;
1511	c->vc.desc_free = omap_dma_desc_free;
1512	vchan_init(&c->vc, &od->ddev);
1513
1514	return 0;
1515}
1516
1517static void omap_dma_free(struct omap_dmadev *od)
1518{
1519	while (!list_empty(&od->ddev.channels)) {
1520		struct omap_chan *c = list_first_entry(&od->ddev.channels,
1521			struct omap_chan, vc.chan.device_node);
1522
1523		list_del(&c->vc.chan.device_node);
1524		tasklet_kill(&c->vc.task);
1525		kfree(c);
1526	}
1527}
1528
1529/* Currently used by omap2 & 3 to block deeper SoC idle states */
1530static bool omap_dma_busy(struct omap_dmadev *od)
1531{
1532	struct omap_chan *c;
1533	int lch = -1;
1534
1535	while (1) {
1536		lch = find_next_bit(od->lch_bitmap, od->lch_count, lch + 1);
1537		if (lch >= od->lch_count)
1538			break;
1539		c = od->lch_map[lch];
1540		if (!c)
1541			continue;
1542		if (omap_dma_chan_read(c, CCR) & CCR_ENABLE)
1543			return true;
1544	}
1545
1546	return false;
1547}
1548
1549/* Currently only used for omap2. For omap1, also a check for lcd_dma is needed */
1550static int omap_dma_busy_notifier(struct notifier_block *nb,
1551				  unsigned long cmd, void *v)
1552{
1553	struct omap_dmadev *od;
1554
1555	od = container_of(nb, struct omap_dmadev, nb);
1556
1557	switch (cmd) {
1558	case CPU_CLUSTER_PM_ENTER:
1559		if (omap_dma_busy(od))
1560			return NOTIFY_BAD;
1561		break;
1562	case CPU_CLUSTER_PM_ENTER_FAILED:
1563	case CPU_CLUSTER_PM_EXIT:
1564		break;
1565	}
1566
1567	return NOTIFY_OK;
1568}
1569
1570/*
1571 * We are using IRQENABLE_L1, and legacy DMA code was using IRQENABLE_L0.
1572 * As the DSP may be using IRQENABLE_L2 and L3, let's not touch those for
1573 * now. Context save seems to be only currently needed on omap3.
1574 */
1575static void omap_dma_context_save(struct omap_dmadev *od)
1576{
1577	od->context.irqenable_l0 = omap_dma_glbl_read(od, IRQENABLE_L0);
1578	od->context.irqenable_l1 = omap_dma_glbl_read(od, IRQENABLE_L1);
1579	od->context.ocp_sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
1580	od->context.gcr = omap_dma_glbl_read(od, GCR);
1581}
1582
1583static void omap_dma_context_restore(struct omap_dmadev *od)
1584{
1585	int i;
1586
1587	omap_dma_glbl_write(od, GCR, od->context.gcr);
1588	omap_dma_glbl_write(od, OCP_SYSCONFIG, od->context.ocp_sysconfig);
1589	omap_dma_glbl_write(od, IRQENABLE_L0, od->context.irqenable_l0);
1590	omap_dma_glbl_write(od, IRQENABLE_L1, od->context.irqenable_l1);
1591
1592	/* Clear IRQSTATUS_L0 as legacy DMA code is no longer doing it */
1593	if (od->plat->errata & DMA_ROMCODE_BUG)
1594		omap_dma_glbl_write(od, IRQSTATUS_L0, 0);
1595
1596	/* Clear dma channels */
1597	for (i = 0; i < od->lch_count; i++)
1598		omap_dma_clear_lch(od, i);
1599}
1600
1601/* Currently only used for omap3 */
1602static int omap_dma_context_notifier(struct notifier_block *nb,
1603				     unsigned long cmd, void *v)
1604{
1605	struct omap_dmadev *od;
1606
1607	od = container_of(nb, struct omap_dmadev, nb);
1608
1609	switch (cmd) {
1610	case CPU_CLUSTER_PM_ENTER:
1611		if (omap_dma_busy(od))
1612			return NOTIFY_BAD;
1613		omap_dma_context_save(od);
1614		break;
1615	case CPU_CLUSTER_PM_ENTER_FAILED:	/* No need to restore context */
1616		break;
1617	case CPU_CLUSTER_PM_EXIT:
1618		omap_dma_context_restore(od);
1619		break;
1620	}
1621
1622	return NOTIFY_OK;
1623}
1624
1625static void omap_dma_init_gcr(struct omap_dmadev *od, int arb_rate,
1626			      int max_fifo_depth, int tparams)
1627{
1628	u32 val;
1629
1630	/* Set only for omap2430 and later */
1631	if (!od->cfg->rw_priority)
1632		return;
1633
1634	if (max_fifo_depth == 0)
1635		max_fifo_depth = 1;
1636	if (arb_rate == 0)
1637		arb_rate = 1;
1638
1639	val = 0xff & max_fifo_depth;
1640	val |= (0x3 & tparams) << 12;
1641	val |= (arb_rate & 0xff) << 16;
1642
1643	omap_dma_glbl_write(od, GCR, val);
1644}
1645
1646#define OMAP_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1647				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1648				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1649
1650/*
1651 * No flags currently set for default configuration as omap1 is still
1652 * using platform data.
1653 */
1654static const struct omap_dma_config default_cfg;
1655
1656static int omap_dma_probe(struct platform_device *pdev)
1657{
1658	const struct omap_dma_config *conf;
1659	struct omap_dmadev *od;
1660	int rc, i, irq;
1661	u32 val;
1662
1663	od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1664	if (!od)
1665		return -ENOMEM;
1666
1667	od->base = devm_platform_ioremap_resource(pdev, 0);
1668	if (IS_ERR(od->base))
1669		return PTR_ERR(od->base);
1670
1671	conf = of_device_get_match_data(&pdev->dev);
1672	if (conf) {
1673		od->cfg = conf;
1674		od->plat = dev_get_platdata(&pdev->dev);
1675		if (!od->plat) {
1676			dev_err(&pdev->dev, "omap_system_dma_plat_info is missing");
1677			return -ENODEV;
1678		}
1679	} else if (IS_ENABLED(CONFIG_ARCH_OMAP1)) {
1680		od->cfg = &default_cfg;
1681
1682		od->plat = omap_get_plat_info();
1683		if (!od->plat)
1684			return -EPROBE_DEFER;
1685	} else {
1686		return -ENODEV;
1687	}
1688
1689	od->reg_map = od->plat->reg_map;
1690
1691	dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1692	dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1693	dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1694	dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1695	od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1696	od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1697	od->ddev.device_tx_status = omap_dma_tx_status;
1698	od->ddev.device_issue_pending = omap_dma_issue_pending;
1699	od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1700	od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1701	od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1702	od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1703	od->ddev.device_config = omap_dma_slave_config;
1704	od->ddev.device_pause = omap_dma_pause;
1705	od->ddev.device_resume = omap_dma_resume;
1706	od->ddev.device_terminate_all = omap_dma_terminate_all;
1707	od->ddev.device_synchronize = omap_dma_synchronize;
1708	od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1709	od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1710	od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1711	if (__dma_omap15xx(od->plat->dma_attr))
1712		od->ddev.residue_granularity =
1713				DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1714	else
1715		od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1716	od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */
1717	od->ddev.dev = &pdev->dev;
1718	INIT_LIST_HEAD(&od->ddev.channels);
1719	mutex_init(&od->lch_lock);
1720	spin_lock_init(&od->lock);
1721	spin_lock_init(&od->irq_lock);
1722
1723	/* Number of DMA requests */
1724	od->dma_requests = OMAP_SDMA_REQUESTS;
1725	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
1726						      "dma-requests",
1727						      &od->dma_requests)) {
1728		dev_info(&pdev->dev,
1729			 "Missing dma-requests property, using %u.\n",
1730			 OMAP_SDMA_REQUESTS);
1731	}
1732
1733	/* Number of available logical channels */
1734	if (!pdev->dev.of_node) {
1735		od->lch_count = od->plat->dma_attr->lch_count;
1736		if (unlikely(!od->lch_count))
1737			od->lch_count = OMAP_SDMA_CHANNELS;
1738	} else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
1739					&od->lch_count)) {
1740		dev_info(&pdev->dev,
1741			 "Missing dma-channels property, using %u.\n",
1742			 OMAP_SDMA_CHANNELS);
1743		od->lch_count = OMAP_SDMA_CHANNELS;
1744	}
1745
1746	/* Mask of allowed logical channels */
1747	if (pdev->dev.of_node && !of_property_read_u32(pdev->dev.of_node,
1748						       "dma-channel-mask",
1749						       &val)) {
1750		/* Tag channels not in mask as reserved */
1751		val = ~val;
1752		bitmap_from_arr32(od->lch_bitmap, &val, od->lch_count);
1753	}
1754	if (od->plat->dma_attr->dev_caps & HS_CHANNELS_RESERVED)
1755		bitmap_set(od->lch_bitmap, 0, 2);
1756
1757	od->lch_map = devm_kcalloc(&pdev->dev, od->lch_count,
1758				   sizeof(*od->lch_map),
1759				   GFP_KERNEL);
1760	if (!od->lch_map)
1761		return -ENOMEM;
1762
1763	for (i = 0; i < od->dma_requests; i++) {
1764		rc = omap_dma_chan_init(od);
1765		if (rc) {
1766			omap_dma_free(od);
1767			return rc;
1768		}
1769	}
1770
1771	irq = platform_get_irq(pdev, 1);
1772	if (irq <= 0) {
1773		dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1774		od->legacy = true;
1775	} else {
1776		/* Disable all interrupts */
1777		od->irq_enable_mask = 0;
1778		omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1779
1780		rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1781				      IRQF_SHARED, "omap-dma-engine", od);
1782		if (rc) {
1783			omap_dma_free(od);
1784			return rc;
1785		}
1786	}
1787
1788	if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1789		od->ll123_supported = true;
1790
1791	od->ddev.filter.map = od->plat->slave_map;
1792	od->ddev.filter.mapcnt = od->plat->slavecnt;
1793	od->ddev.filter.fn = omap_dma_filter_fn;
1794
1795	if (od->ll123_supported) {
1796		od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1797						&pdev->dev,
1798						sizeof(struct omap_type2_desc),
1799						4, 0);
1800		if (!od->desc_pool) {
1801			dev_err(&pdev->dev,
1802				"unable to allocate descriptor pool\n");
1803			od->ll123_supported = false;
1804		}
1805	}
1806
1807	rc = dma_async_device_register(&od->ddev);
1808	if (rc) {
1809		pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1810			rc);
1811		if (od->ll123_supported)
1812			dma_pool_destroy(od->desc_pool);
1813		omap_dma_free(od);
1814		return rc;
1815	}
1816
1817	platform_set_drvdata(pdev, od);
1818
1819	if (pdev->dev.of_node) {
1820		omap_dma_info.dma_cap = od->ddev.cap_mask;
1821
1822		/* Device-tree DMA controller registration */
1823		rc = of_dma_controller_register(pdev->dev.of_node,
1824				of_dma_simple_xlate, &omap_dma_info);
1825		if (rc) {
1826			pr_warn("OMAP-DMA: failed to register DMA controller\n");
1827			dma_async_device_unregister(&od->ddev);
1828			if (od->ll123_supported)
1829				dma_pool_destroy(od->desc_pool);
1830			omap_dma_free(od);
1831		}
1832	}
1833
1834	omap_dma_init_gcr(od, DMA_DEFAULT_ARB_RATE, DMA_DEFAULT_FIFO_DEPTH, 0);
1835
1836	if (od->cfg->needs_busy_check) {
1837		od->nb.notifier_call = omap_dma_busy_notifier;
1838		cpu_pm_register_notifier(&od->nb);
1839	} else if (od->cfg->may_lose_context) {
1840		od->nb.notifier_call = omap_dma_context_notifier;
1841		cpu_pm_register_notifier(&od->nb);
1842	}
1843
1844	dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1845		 od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1846
1847	return rc;
1848}
1849
1850static void omap_dma_remove(struct platform_device *pdev)
1851{
1852	struct omap_dmadev *od = platform_get_drvdata(pdev);
1853	int irq;
1854
1855	if (od->cfg->may_lose_context)
1856		cpu_pm_unregister_notifier(&od->nb);
1857
1858	if (pdev->dev.of_node)
1859		of_dma_controller_free(pdev->dev.of_node);
1860
1861	irq = platform_get_irq(pdev, 1);
1862	devm_free_irq(&pdev->dev, irq, od);
1863
1864	dma_async_device_unregister(&od->ddev);
1865
1866	if (!omap_dma_legacy(od)) {
1867		/* Disable all interrupts */
1868		omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1869	}
1870
1871	if (od->ll123_supported)
1872		dma_pool_destroy(od->desc_pool);
1873
1874	omap_dma_free(od);
1875}
1876
1877static const struct omap_dma_config omap2420_data = {
1878	.lch_end = CCFN,
1879	.rw_priority = true,
1880	.needs_lch_clear = true,
1881	.needs_busy_check = true,
1882};
1883
1884static const struct omap_dma_config omap2430_data = {
1885	.lch_end = CCFN,
1886	.rw_priority = true,
1887	.needs_lch_clear = true,
1888};
1889
1890static const struct omap_dma_config omap3430_data = {
1891	.lch_end = CCFN,
1892	.rw_priority = true,
1893	.needs_lch_clear = true,
1894	.may_lose_context = true,
1895};
1896
1897static const struct omap_dma_config omap3630_data = {
1898	.lch_end = CCDN,
1899	.rw_priority = true,
1900	.needs_lch_clear = true,
1901	.may_lose_context = true,
1902};
1903
1904static const struct omap_dma_config omap4_data = {
1905	.lch_end = CCDN,
1906	.rw_priority = true,
1907	.needs_lch_clear = true,
1908};
1909
1910static const struct of_device_id omap_dma_match[] = {
1911	{ .compatible = "ti,omap2420-sdma", .data = &omap2420_data, },
1912	{ .compatible = "ti,omap2430-sdma", .data = &omap2430_data, },
1913	{ .compatible = "ti,omap3430-sdma", .data = &omap3430_data, },
1914	{ .compatible = "ti,omap3630-sdma", .data = &omap3630_data, },
1915	{ .compatible = "ti,omap4430-sdma", .data = &omap4_data, },
1916	{},
1917};
1918MODULE_DEVICE_TABLE(of, omap_dma_match);
1919
1920static struct platform_driver omap_dma_driver = {
1921	.probe	= omap_dma_probe,
1922	.remove = omap_dma_remove,
1923	.driver = {
1924		.name = "omap-dma-engine",
1925		.of_match_table = omap_dma_match,
1926	},
1927};
1928
1929static bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1930{
1931	if (chan->device->dev->driver == &omap_dma_driver.driver) {
1932		struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1933		struct omap_chan *c = to_omap_dma_chan(chan);
1934		unsigned req = *(unsigned *)param;
1935
1936		if (req <= od->dma_requests) {
1937			c->dma_sig = req;
1938			return true;
1939		}
1940	}
1941	return false;
1942}
1943
1944static int omap_dma_init(void)
1945{
1946	return platform_driver_register(&omap_dma_driver);
1947}
1948subsys_initcall(omap_dma_init);
1949
1950static void __exit omap_dma_exit(void)
1951{
1952	platform_driver_unregister(&omap_dma_driver);
1953}
1954module_exit(omap_dma_exit);
1955
1956MODULE_AUTHOR("Russell King");
1957MODULE_DESCRIPTION("Texas Instruments sDMA DMAengine support");
1958MODULE_LICENSE("GPL");