rz-dmac.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Renesas RZ/G2L DMA Controller Driver
   4 *
   5 * Based on imx-dma.c
   6 *
   7 * Copyright (C) 2021 Renesas Electronics Corp.
   8 * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
   9 * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com>
  10 */
  11
  12#include <linux/bitfield.h>
  13#include <linux/cleanup.h>
  14#include <linux/dma-mapping.h>
  15#include <linux/dmaengine.h>
  16#include <linux/interrupt.h>
  17#include <linux/iopoll.h>
  18#include <linux/irqchip/irq-renesas-rzv2h.h>
  19#include <linux/list.h>
  20#include <linux/module.h>
  21#include <linux/of.h>
  22#include <linux/of_dma.h>
  23#include <linux/of_platform.h>
  24#include <linux/platform_device.h>
  25#include <linux/pm_runtime.h>
  26#include <linux/reset.h>
  27#include <linux/slab.h>
  28#include <linux/spinlock.h>
  29
  30#include "../dmaengine.h"
  31#include "../virt-dma.h"
  32
  33enum  rz_dmac_prep_type {
  34	RZ_DMAC_DESC_MEMCPY,
  35	RZ_DMAC_DESC_SLAVE_SG,
  36};
  37
  38struct rz_lmdesc {
  39	u32 header;
  40	u32 sa;
  41	u32 da;
  42	u32 tb;
  43	u32 chcfg;
  44	u32 chitvl;
  45	u32 chext;
  46	u32 nxla;
  47};
  48
  49struct rz_dmac_desc {
  50	struct virt_dma_desc vd;
  51	dma_addr_t src;
  52	dma_addr_t dest;
  53	size_t len;
  54	struct list_head node;
  55	enum dma_transfer_direction direction;
  56	enum rz_dmac_prep_type type;
  57	/* For slave sg */
  58	struct scatterlist *sg;
  59	unsigned int sgcount;
  60};
  61
  62#define to_rz_dmac_desc(d)	container_of(d, struct rz_dmac_desc, vd)
  63
  64struct rz_dmac_chan {
  65	struct virt_dma_chan vc;
  66	void __iomem *ch_base;
  67	void __iomem *ch_cmn_base;
  68	unsigned int index;
  69	int irq;
  70	struct rz_dmac_desc *desc;
  71	int descs_allocated;
  72
  73	dma_addr_t src_per_address;
  74	dma_addr_t dst_per_address;
  75
  76	u32 chcfg;
  77	u32 chctrl;
  78	int mid_rid;
  79
  80	struct list_head ld_free;
  81	struct list_head ld_queue;
  82	struct list_head ld_active;
  83
  84	struct {
  85		struct rz_lmdesc *base;
  86		struct rz_lmdesc *head;
  87		struct rz_lmdesc *tail;
  88		dma_addr_t base_dma;
  89	} lmdesc;
  90};
  91
  92#define to_rz_dmac_chan(c)	container_of(c, struct rz_dmac_chan, vc.chan)
  93
  94struct rz_dmac_icu {
  95	struct platform_device *pdev;
  96	u8 dmac_index;
  97};
  98
  99struct rz_dmac {
 100	struct dma_device engine;
 101	struct rz_dmac_icu icu;
 102	struct device *dev;
 103	struct reset_control *rstc;
 104	void __iomem *base;
 105	void __iomem *ext_base;
 106
 107	unsigned int n_channels;
 108	struct rz_dmac_chan *channels;
 109
 110	bool has_icu;
 111
 112	DECLARE_BITMAP(modules, 1024);
 113};
 114
 115#define to_rz_dmac(d)	container_of(d, struct rz_dmac, engine)
 116
 117/*
 118 * -----------------------------------------------------------------------------
 119 * Registers
 120 */
 121
 122#define CHSTAT				0x0024
 123#define CHCTRL				0x0028
 124#define CHCFG				0x002c
 125#define NXLA				0x0038
 126
 127#define DCTRL				0x0000
 128
 129#define EACH_CHANNEL_OFFSET		0x0040
 130#define CHANNEL_0_7_OFFSET		0x0000
 131#define CHANNEL_0_7_COMMON_BASE		0x0300
 132#define CHANNEL_8_15_OFFSET		0x0400
 133#define CHANNEL_8_15_COMMON_BASE	0x0700
 134
 135#define CHSTAT_ER			BIT(4)
 136#define CHSTAT_EN			BIT(0)
 137
 138#define CHCTRL_CLRINTMSK		BIT(17)
 139#define CHCTRL_CLRSUS			BIT(9)
 140#define CHCTRL_CLRTC			BIT(6)
 141#define CHCTRL_CLREND			BIT(5)
 142#define CHCTRL_CLRRQ			BIT(4)
 143#define CHCTRL_SWRST			BIT(3)
 144#define CHCTRL_STG			BIT(2)
 145#define CHCTRL_CLREN			BIT(1)
 146#define CHCTRL_SETEN			BIT(0)
 147#define CHCTRL_DEFAULT			(CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \
 148					 CHCTRL_CLRTC |	CHCTRL_CLREND | \
 149					 CHCTRL_CLRRQ | CHCTRL_SWRST | \
 150					 CHCTRL_CLREN)
 151
 152#define CHCFG_DMS			BIT(31)
 153#define CHCFG_DEM			BIT(24)
 154#define CHCFG_DAD			BIT(21)
 155#define CHCFG_SAD			BIT(20)
 156#define CHCFG_REQD			BIT(3)
 157#define CHCFG_SEL(bits)			((bits) & 0x07)
 158#define CHCFG_MEM_COPY			(0x80400008)
 159#define CHCFG_FILL_DDS_MASK		GENMASK(19, 16)
 160#define CHCFG_FILL_SDS_MASK		GENMASK(15, 12)
 161#define CHCFG_FILL_TM(a)		(((a) & BIT(5)) << 22)
 162#define CHCFG_FILL_AM(a)		(((a) & GENMASK(4, 2)) << 6)
 163#define CHCFG_FILL_LVL(a)		(((a) & BIT(1)) << 5)
 164#define CHCFG_FILL_HIEN(a)		(((a) & BIT(0)) << 5)
 165
 166#define MID_RID_MASK			GENMASK(9, 0)
 167#define CHCFG_MASK			GENMASK(15, 10)
 168#define CHCFG_DS_INVALID		0xFF
 169#define DCTRL_LVINT			BIT(1)
 170#define DCTRL_PR			BIT(0)
 171#define DCTRL_DEFAULT			(DCTRL_LVINT | DCTRL_PR)
 172
 173/* LINK MODE DESCRIPTOR */
 174#define HEADER_LV			BIT(0)
 175
 176#define RZ_DMAC_MAX_CHAN_DESCRIPTORS	16
 177#define RZ_DMAC_MAX_CHANNELS		16
 178#define DMAC_NR_LMDESC			64
 179
 180/* RZ/V2H ICU related */
 181#define RZV2H_MAX_DMAC_INDEX		4
 182
 183/*
 184 * -----------------------------------------------------------------------------
 185 * Device access
 186 */
 187
 188static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val,
 189			   unsigned int offset)
 190{
 191	writel(val, dmac->base + offset);
 192}
 193
 194static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val,
 195			       unsigned int offset)
 196{
 197	writel(val, dmac->ext_base + offset);
 198}
 199
 200static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset)
 201{
 202	return readl(dmac->ext_base + offset);
 203}
 204
 205static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val,
 206			      unsigned int offset, int which)
 207{
 208	if (which)
 209		writel(val, channel->ch_base + offset);
 210	else
 211		writel(val, channel->ch_cmn_base + offset);
 212}
 213
 214static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel,
 215			    unsigned int offset, int which)
 216{
 217	if (which)
 218		return readl(channel->ch_base + offset);
 219	else
 220		return readl(channel->ch_cmn_base + offset);
 221}
 222
 223/*
 224 * -----------------------------------------------------------------------------
 225 * Initialization
 226 */
 227
 228static void rz_lmdesc_setup(struct rz_dmac_chan *channel,
 229			    struct rz_lmdesc *lmdesc)
 230{
 231	u32 nxla;
 232
 233	channel->lmdesc.base = lmdesc;
 234	channel->lmdesc.head = lmdesc;
 235	channel->lmdesc.tail = lmdesc;
 236	nxla = channel->lmdesc.base_dma;
 237	while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) {
 238		lmdesc->header = 0;
 239		nxla += sizeof(*lmdesc);
 240		lmdesc->nxla = nxla;
 241		lmdesc++;
 242	}
 243
 244	lmdesc->header = 0;
 245	lmdesc->nxla = channel->lmdesc.base_dma;
 246}
 247
 248/*
 249 * -----------------------------------------------------------------------------
 250 * Descriptors preparation
 251 */
 252
 253static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel)
 254{
 255	struct rz_lmdesc *lmdesc = channel->lmdesc.head;
 256
 257	while (!(lmdesc->header & HEADER_LV)) {
 258		lmdesc->header = 0;
 259		lmdesc++;
 260		if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
 261			lmdesc = channel->lmdesc.base;
 262	}
 263	channel->lmdesc.head = lmdesc;
 264}
 265
 266static void rz_dmac_enable_hw(struct rz_dmac_chan *channel)
 267{
 268	struct dma_chan *chan = &channel->vc.chan;
 269	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 270	unsigned long flags;
 271	u32 nxla;
 272	u32 chctrl;
 273	u32 chstat;
 274
 275	dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
 276
 277	local_irq_save(flags);
 278
 279	rz_dmac_lmdesc_recycle(channel);
 280
 281	nxla = channel->lmdesc.base_dma +
 282		(sizeof(struct rz_lmdesc) * (channel->lmdesc.head -
 283					     channel->lmdesc.base));
 284
 285	chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
 286	if (!(chstat & CHSTAT_EN)) {
 287		chctrl = (channel->chctrl | CHCTRL_SETEN);
 288		rz_dmac_ch_writel(channel, nxla, NXLA, 1);
 289		rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1);
 290		rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1);
 291		rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1);
 292	}
 293
 294	local_irq_restore(flags);
 295}
 296
 297static void rz_dmac_disable_hw(struct rz_dmac_chan *channel)
 298{
 299	struct dma_chan *chan = &channel->vc.chan;
 300	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 301
 302	dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index);
 303
 304	rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 305}
 306
 307static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars)
 308{
 309	u32 dmars_offset = (nr / 2) * 4;
 310	u32 shift = (nr % 2) * 16;
 311	u32 dmars32;
 312
 313	dmars32 = rz_dmac_ext_readl(dmac, dmars_offset);
 314	dmars32 &= ~(0xffff << shift);
 315	dmars32 |= dmars << shift;
 316
 317	rz_dmac_ext_writel(dmac, dmars32, dmars_offset);
 318}
 319
 320static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel)
 321{
 322	struct dma_chan *chan = &channel->vc.chan;
 323	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 324	struct rz_lmdesc *lmdesc = channel->lmdesc.tail;
 325	struct rz_dmac_desc *d = channel->desc;
 326	u32 chcfg = CHCFG_MEM_COPY;
 327
 328	/* prepare descriptor */
 329	lmdesc->sa = d->src;
 330	lmdesc->da = d->dest;
 331	lmdesc->tb = d->len;
 332	lmdesc->chcfg = chcfg;
 333	lmdesc->chitvl = 0;
 334	lmdesc->chext = 0;
 335	lmdesc->header = HEADER_LV;
 336
 337	if (dmac->has_icu) {
 338		rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
 339					   channel->index,
 340					   RZV2H_ICU_DMAC_REQ_NO_DEFAULT);
 341	} else {
 342		rz_dmac_set_dmars_register(dmac, channel->index, 0);
 343	}
 344
 345	channel->chcfg = chcfg;
 346	channel->chctrl = CHCTRL_STG | CHCTRL_SETEN;
 347}
 348
 349static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel)
 350{
 351	struct dma_chan *chan = &channel->vc.chan;
 352	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 353	struct rz_dmac_desc *d = channel->desc;
 354	struct scatterlist *sg, *sgl = d->sg;
 355	struct rz_lmdesc *lmdesc;
 356	unsigned int i, sg_len = d->sgcount;
 357
 358	channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS;
 359
 360	if (d->direction == DMA_DEV_TO_MEM) {
 361		channel->chcfg |= CHCFG_SAD;
 362		channel->chcfg &= ~CHCFG_REQD;
 363	} else {
 364		channel->chcfg |= CHCFG_DAD | CHCFG_REQD;
 365	}
 366
 367	lmdesc = channel->lmdesc.tail;
 368
 369	for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) {
 370		if (d->direction == DMA_DEV_TO_MEM) {
 371			lmdesc->sa = channel->src_per_address;
 372			lmdesc->da = sg_dma_address(sg);
 373		} else {
 374			lmdesc->sa = sg_dma_address(sg);
 375			lmdesc->da = channel->dst_per_address;
 376		}
 377
 378		lmdesc->tb = sg_dma_len(sg);
 379		lmdesc->chitvl = 0;
 380		lmdesc->chext = 0;
 381		if (i == (sg_len - 1)) {
 382			lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM);
 383			lmdesc->header = HEADER_LV;
 384		} else {
 385			lmdesc->chcfg = channel->chcfg;
 386			lmdesc->header = HEADER_LV;
 387		}
 388		if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC))
 389			lmdesc = channel->lmdesc.base;
 390	}
 391
 392	channel->lmdesc.tail = lmdesc;
 393
 394	if (dmac->has_icu) {
 395		rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
 396					   channel->index, channel->mid_rid);
 397	} else {
 398		rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid);
 399	}
 400
 401	channel->chctrl = CHCTRL_SETEN;
 402}
 403
 404static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan)
 405{
 406	struct rz_dmac_desc *d = chan->desc;
 407	struct virt_dma_desc *vd;
 408
 409	vd = vchan_next_desc(&chan->vc);
 410	if (!vd)
 411		return 0;
 412
 413	list_del(&vd->node);
 414
 415	switch (d->type) {
 416	case RZ_DMAC_DESC_MEMCPY:
 417		rz_dmac_prepare_desc_for_memcpy(chan);
 418		break;
 419
 420	case RZ_DMAC_DESC_SLAVE_SG:
 421		rz_dmac_prepare_descs_for_slave_sg(chan);
 422		break;
 423
 424	default:
 425		return -EINVAL;
 426	}
 427
 428	rz_dmac_enable_hw(chan);
 429
 430	return 0;
 431}
 432
 433/*
 434 * -----------------------------------------------------------------------------
 435 * DMA engine operations
 436 */
 437
 438static int rz_dmac_alloc_chan_resources(struct dma_chan *chan)
 439{
 440	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 441
 442	while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) {
 443		struct rz_dmac_desc *desc;
 444
 445		desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 446		if (!desc)
 447			break;
 448
 449		/* No need to lock. This is called only for the 1st client. */
 450		list_add_tail(&desc->node, &channel->ld_free);
 451		channel->descs_allocated++;
 452	}
 453
 454	if (!channel->descs_allocated)
 455		return -ENOMEM;
 456
 457	return channel->descs_allocated;
 458}
 459
 460static void rz_dmac_free_chan_resources(struct dma_chan *chan)
 461{
 462	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 463	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 464	struct rz_lmdesc *lmdesc = channel->lmdesc.base;
 465	struct rz_dmac_desc *desc, *_desc;
 466	unsigned long flags;
 467	unsigned int i;
 468
 469	spin_lock_irqsave(&channel->vc.lock, flags);
 470
 471	for (i = 0; i < DMAC_NR_LMDESC; i++)
 472		lmdesc[i].header = 0;
 473
 474	rz_dmac_disable_hw(channel);
 475	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
 476	list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
 477
 478	if (channel->mid_rid >= 0) {
 479		clear_bit(channel->mid_rid, dmac->modules);
 480		channel->mid_rid = -EINVAL;
 481	}
 482
 483	spin_unlock_irqrestore(&channel->vc.lock, flags);
 484
 485	list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) {
 486		kfree(desc);
 487		channel->descs_allocated--;
 488	}
 489
 490	INIT_LIST_HEAD(&channel->ld_free);
 491	vchan_free_chan_resources(&channel->vc);
 492}
 493
 494static struct dma_async_tx_descriptor *
 495rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 496			size_t len, unsigned long flags)
 497{
 498	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 499	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 500	struct rz_dmac_desc *desc;
 501
 502	dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n",
 503		__func__, channel->index, &src, &dest, len);
 504
 505	scoped_guard(spinlock_irqsave, &channel->vc.lock) {
 506		if (list_empty(&channel->ld_free))
 507			return NULL;
 508
 509		desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
 510
 511		desc->type = RZ_DMAC_DESC_MEMCPY;
 512		desc->src = src;
 513		desc->dest = dest;
 514		desc->len = len;
 515		desc->direction = DMA_MEM_TO_MEM;
 516
 517		list_move_tail(channel->ld_free.next, &channel->ld_queue);
 518	}
 519
 520	return vchan_tx_prep(&channel->vc, &desc->vd, flags);
 521}
 522
 523static struct dma_async_tx_descriptor *
 524rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 525		      unsigned int sg_len,
 526		      enum dma_transfer_direction direction,
 527		      unsigned long flags, void *context)
 528{
 529	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 530	struct rz_dmac_desc *desc;
 531	struct scatterlist *sg;
 532	int dma_length = 0;
 533	int i = 0;
 534
 535	scoped_guard(spinlock_irqsave, &channel->vc.lock) {
 536		if (list_empty(&channel->ld_free))
 537			return NULL;
 538
 539		desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node);
 540
 541		for_each_sg(sgl, sg, sg_len, i)
 542			dma_length += sg_dma_len(sg);
 543
 544		desc->type = RZ_DMAC_DESC_SLAVE_SG;
 545		desc->sg = sgl;
 546		desc->sgcount = sg_len;
 547		desc->len = dma_length;
 548		desc->direction = direction;
 549
 550		if (direction == DMA_DEV_TO_MEM)
 551			desc->src = channel->src_per_address;
 552		else
 553			desc->dest = channel->dst_per_address;
 554
 555		list_move_tail(channel->ld_free.next, &channel->ld_queue);
 556	}
 557
 558	return vchan_tx_prep(&channel->vc, &desc->vd, flags);
 559}
 560
 561static int rz_dmac_terminate_all(struct dma_chan *chan)
 562{
 563	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 564	struct rz_lmdesc *lmdesc = channel->lmdesc.base;
 565	unsigned long flags;
 566	unsigned int i;
 567	LIST_HEAD(head);
 568
 569	spin_lock_irqsave(&channel->vc.lock, flags);
 570	rz_dmac_disable_hw(channel);
 571	for (i = 0; i < DMAC_NR_LMDESC; i++)
 572		lmdesc[i].header = 0;
 573
 574	list_splice_tail_init(&channel->ld_active, &channel->ld_free);
 575	list_splice_tail_init(&channel->ld_queue, &channel->ld_free);
 576	vchan_get_all_descriptors(&channel->vc, &head);
 577	spin_unlock_irqrestore(&channel->vc.lock, flags);
 578	vchan_dma_desc_free_list(&channel->vc, &head);
 579
 580	return 0;
 581}
 582
 583static void rz_dmac_issue_pending(struct dma_chan *chan)
 584{
 585	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 586	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 587	struct rz_dmac_desc *desc;
 588	unsigned long flags;
 589
 590	spin_lock_irqsave(&channel->vc.lock, flags);
 591
 592	if (!list_empty(&channel->ld_queue)) {
 593		desc = list_first_entry(&channel->ld_queue,
 594					struct rz_dmac_desc, node);
 595		channel->desc = desc;
 596		if (vchan_issue_pending(&channel->vc)) {
 597			if (rz_dmac_xfer_desc(channel) < 0)
 598				dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n",
 599					 channel->index);
 600			else
 601				list_move_tail(channel->ld_queue.next,
 602					       &channel->ld_active);
 603		}
 604	}
 605
 606	spin_unlock_irqrestore(&channel->vc.lock, flags);
 607}
 608
 609static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds)
 610{
 611	u8 i;
 612	static const enum dma_slave_buswidth ds_lut[] = {
 613		DMA_SLAVE_BUSWIDTH_1_BYTE,
 614		DMA_SLAVE_BUSWIDTH_2_BYTES,
 615		DMA_SLAVE_BUSWIDTH_4_BYTES,
 616		DMA_SLAVE_BUSWIDTH_8_BYTES,
 617		DMA_SLAVE_BUSWIDTH_16_BYTES,
 618		DMA_SLAVE_BUSWIDTH_32_BYTES,
 619		DMA_SLAVE_BUSWIDTH_64_BYTES,
 620		DMA_SLAVE_BUSWIDTH_128_BYTES,
 621	};
 622
 623	for (i = 0; i < ARRAY_SIZE(ds_lut); i++) {
 624		if (ds_lut[i] == ds)
 625			return i;
 626	}
 627
 628	return CHCFG_DS_INVALID;
 629}
 630
 631static int rz_dmac_config(struct dma_chan *chan,
 632			  struct dma_slave_config *config)
 633{
 634	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 635	u32 val;
 636
 637	channel->dst_per_address = config->dst_addr;
 638	channel->chcfg &= ~CHCFG_FILL_DDS_MASK;
 639	if (channel->dst_per_address) {
 640		val = rz_dmac_ds_to_val_mapping(config->dst_addr_width);
 641		if (val == CHCFG_DS_INVALID)
 642			return -EINVAL;
 643
 644		channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val);
 645	}
 646
 647	channel->src_per_address = config->src_addr;
 648	channel->chcfg &= ~CHCFG_FILL_SDS_MASK;
 649	if (channel->src_per_address) {
 650		val = rz_dmac_ds_to_val_mapping(config->src_addr_width);
 651		if (val == CHCFG_DS_INVALID)
 652			return -EINVAL;
 653
 654		channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val);
 655	}
 656
 657	return 0;
 658}
 659
 660static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd)
 661{
 662	/*
 663	 * Place holder
 664	 * Descriptor allocation is done during alloc_chan_resources and
 665	 * get freed during free_chan_resources.
 666	 * list is used to manage the descriptors and avoid any memory
 667	 * allocation/free during DMA read/write.
 668	 */
 669}
 670
 671static void rz_dmac_device_synchronize(struct dma_chan *chan)
 672{
 673	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 674	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 675	u32 chstat;
 676	int ret;
 677
 678	ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN),
 679				100, 100000, false, channel, CHSTAT, 1);
 680	if (ret < 0)
 681		dev_warn(dmac->dev, "DMA Timeout");
 682
 683	if (dmac->has_icu) {
 684		rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index,
 685					   channel->index,
 686					   RZV2H_ICU_DMAC_REQ_NO_DEFAULT);
 687	} else {
 688		rz_dmac_set_dmars_register(dmac, channel->index, 0);
 689	}
 690}
 691
 692/*
 693 * -----------------------------------------------------------------------------
 694 * IRQ handling
 695 */
 696
 697static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel)
 698{
 699	struct dma_chan *chan = &channel->vc.chan;
 700	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 701	u32 chstat, chctrl;
 702
 703	chstat = rz_dmac_ch_readl(channel, CHSTAT, 1);
 704	if (chstat & CHSTAT_ER) {
 705		dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n",
 706			channel->index, chstat);
 707
 708		scoped_guard(spinlock_irqsave, &channel->vc.lock)
 709			rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 710		goto done;
 711	}
 712
 713	chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1);
 714	rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1);
 715done:
 716	return;
 717}
 718
 719static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id)
 720{
 721	struct rz_dmac_chan *channel = dev_id;
 722
 723	if (channel) {
 724		rz_dmac_irq_handle_channel(channel);
 725		return IRQ_WAKE_THREAD;
 726	}
 727	/* handle DMAERR irq */
 728	return IRQ_HANDLED;
 729}
 730
 731static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id)
 732{
 733	struct rz_dmac_chan *channel = dev_id;
 734	struct rz_dmac_desc *desc = NULL;
 735	unsigned long flags;
 736
 737	spin_lock_irqsave(&channel->vc.lock, flags);
 738
 739	if (list_empty(&channel->ld_active)) {
 740		/* Someone might have called terminate all */
 741		goto out;
 742	}
 743
 744	desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node);
 745	vchan_cookie_complete(&desc->vd);
 746	list_move_tail(channel->ld_active.next, &channel->ld_free);
 747	if (!list_empty(&channel->ld_queue)) {
 748		desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc,
 749					node);
 750		channel->desc = desc;
 751		if (rz_dmac_xfer_desc(channel) == 0)
 752			list_move_tail(channel->ld_queue.next, &channel->ld_active);
 753	}
 754out:
 755	spin_unlock_irqrestore(&channel->vc.lock, flags);
 756
 757	return IRQ_HANDLED;
 758}
 759
 760/*
 761 * -----------------------------------------------------------------------------
 762 * OF xlate and channel filter
 763 */
 764
 765static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg)
 766{
 767	struct rz_dmac_chan *channel = to_rz_dmac_chan(chan);
 768	struct rz_dmac *dmac = to_rz_dmac(chan->device);
 769	struct of_phandle_args *dma_spec = arg;
 770	u32 ch_cfg;
 771
 772	channel->mid_rid = dma_spec->args[0] & MID_RID_MASK;
 773	ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10;
 774	channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) |
 775			 CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg);
 776
 777	return !test_and_set_bit(channel->mid_rid, dmac->modules);
 778}
 779
 780static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec,
 781					 struct of_dma *ofdma)
 782{
 783	dma_cap_mask_t mask;
 784
 785	if (dma_spec->args_count != 1)
 786		return NULL;
 787
 788	/* Only slave DMA channels can be allocated via DT */
 789	dma_cap_zero(mask);
 790	dma_cap_set(DMA_SLAVE, mask);
 791
 792	return __dma_request_channel(&mask, rz_dmac_chan_filter, dma_spec,
 793				     ofdma->of_node);
 794}
 795
 796/*
 797 * -----------------------------------------------------------------------------
 798 * Probe and remove
 799 */
 800
 801static int rz_dmac_chan_probe(struct rz_dmac *dmac,
 802			      struct rz_dmac_chan *channel,
 803			      u8 index)
 804{
 805	struct platform_device *pdev = to_platform_device(dmac->dev);
 806	struct rz_lmdesc *lmdesc;
 807	char pdev_irqname[6];
 808	char *irqname;
 809	int ret;
 810
 811	channel->index = index;
 812	channel->mid_rid = -EINVAL;
 813
 814	/* Request the channel interrupt. */
 815	scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index);
 816	channel->irq = platform_get_irq_byname(pdev, pdev_irqname);
 817	if (channel->irq < 0)
 818		return channel->irq;
 819
 820	irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u",
 821				 dev_name(dmac->dev), index);
 822	if (!irqname)
 823		return -ENOMEM;
 824
 825	ret = devm_request_threaded_irq(dmac->dev, channel->irq,
 826					rz_dmac_irq_handler,
 827					rz_dmac_irq_handler_thread, 0,
 828					irqname, channel);
 829	if (ret) {
 830		dev_err(dmac->dev, "failed to request IRQ %u (%d)\n",
 831			channel->irq, ret);
 832		return ret;
 833	}
 834
 835	/* Set io base address for each channel */
 836	if (index < 8) {
 837		channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET +
 838			EACH_CHANNEL_OFFSET * index;
 839		channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE;
 840	} else {
 841		channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET +
 842			EACH_CHANNEL_OFFSET * (index - 8);
 843		channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE;
 844	}
 845
 846	/* Allocate descriptors */
 847	lmdesc = dma_alloc_coherent(&pdev->dev,
 848				    sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
 849				    &channel->lmdesc.base_dma, GFP_KERNEL);
 850	if (!lmdesc) {
 851		dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n");
 852		return -ENOMEM;
 853	}
 854	rz_lmdesc_setup(channel, lmdesc);
 855
 856	/* Initialize register for each channel */
 857	rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1);
 858
 859	channel->vc.desc_free = rz_dmac_virt_desc_free;
 860	vchan_init(&channel->vc, &dmac->engine);
 861	INIT_LIST_HEAD(&channel->ld_queue);
 862	INIT_LIST_HEAD(&channel->ld_free);
 863	INIT_LIST_HEAD(&channel->ld_active);
 864
 865	return 0;
 866}
 867
 868static void rz_dmac_put_device(void *_dev)
 869{
 870	struct device *dev = _dev;
 871
 872	put_device(dev);
 873}
 874
 875static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac)
 876{
 877	struct device_node *np = dev->of_node;
 878	struct of_phandle_args args;
 879	uint32_t dmac_index;
 880	int ret;
 881
 882	ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args);
 883	if (ret == -ENOENT)
 884		return 0;
 885	if (ret)
 886		return ret;
 887
 888	dmac->has_icu = true;
 889
 890	dmac->icu.pdev = of_find_device_by_node(args.np);
 891	of_node_put(args.np);
 892	if (!dmac->icu.pdev) {
 893		dev_err(dev, "ICU device not found.\n");
 894		return -ENODEV;
 895	}
 896
 897	ret = devm_add_action_or_reset(dev, rz_dmac_put_device, &dmac->icu.pdev->dev);
 898	if (ret)
 899		return ret;
 900
 901	dmac_index = args.args[0];
 902	if (dmac_index > RZV2H_MAX_DMAC_INDEX) {
 903		dev_err(dev, "DMAC index %u invalid.\n", dmac_index);
 904		return -EINVAL;
 905	}
 906	dmac->icu.dmac_index = dmac_index;
 907
 908	return 0;
 909}
 910
 911static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac)
 912{
 913	struct device_node *np = dev->of_node;
 914	int ret;
 915
 916	ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels);
 917	if (ret < 0) {
 918		dev_err(dev, "unable to read dma-channels property\n");
 919		return ret;
 920	}
 921
 922	if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) {
 923		dev_err(dev, "invalid number of channels %u\n", dmac->n_channels);
 924		return -EINVAL;
 925	}
 926
 927	return rz_dmac_parse_of_icu(dev, dmac);
 928}
 929
 930static int rz_dmac_probe(struct platform_device *pdev)
 931{
 932	const char *irqname = "error";
 933	struct dma_device *engine;
 934	struct rz_dmac *dmac;
 935	int channel_num;
 936	int ret;
 937	int irq;
 938	u8 i;
 939
 940	dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
 941	if (!dmac)
 942		return -ENOMEM;
 943
 944	dmac->dev = &pdev->dev;
 945	platform_set_drvdata(pdev, dmac);
 946
 947	ret = rz_dmac_parse_of(&pdev->dev, dmac);
 948	if (ret < 0)
 949		return ret;
 950
 951	dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels,
 952				      sizeof(*dmac->channels), GFP_KERNEL);
 953	if (!dmac->channels)
 954		return -ENOMEM;
 955
 956	/* Request resources */
 957	dmac->base = devm_platform_ioremap_resource(pdev, 0);
 958	if (IS_ERR(dmac->base))
 959		return PTR_ERR(dmac->base);
 960
 961	if (!dmac->has_icu) {
 962		dmac->ext_base = devm_platform_ioremap_resource(pdev, 1);
 963		if (IS_ERR(dmac->ext_base))
 964			return PTR_ERR(dmac->ext_base);
 965	}
 966
 967	/* Register interrupt handler for error */
 968	irq = platform_get_irq_byname(pdev, irqname);
 969	if (irq < 0)
 970		return irq;
 971
 972	ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0,
 973			       irqname, NULL);
 974	if (ret) {
 975		dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n",
 976			irq, ret);
 977		return ret;
 978	}
 979
 980	/* Initialize the channels. */
 981	INIT_LIST_HEAD(&dmac->engine.channels);
 982
 983	dmac->rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev);
 984	if (IS_ERR(dmac->rstc))
 985		return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc),
 986				     "failed to get resets\n");
 987
 988	pm_runtime_enable(&pdev->dev);
 989	ret = pm_runtime_resume_and_get(&pdev->dev);
 990	if (ret < 0) {
 991		dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n");
 992		goto err_pm_disable;
 993	}
 994
 995	ret = reset_control_deassert(dmac->rstc);
 996	if (ret)
 997		goto err_pm_runtime_put;
 998
 999	for (i = 0; i < dmac->n_channels; i++) {
1000		ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i);
1001		if (ret < 0)
1002			goto err;
1003	}
1004
1005	/* Register the DMAC as a DMA provider for DT. */
1006	ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate,
1007					 NULL);
1008	if (ret < 0)
1009		goto err;
1010
1011	/* Register the DMA engine device. */
1012	engine = &dmac->engine;
1013	dma_cap_set(DMA_SLAVE, engine->cap_mask);
1014	dma_cap_set(DMA_MEMCPY, engine->cap_mask);
1015	rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL);
1016	rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL);
1017
1018	engine->dev = &pdev->dev;
1019
1020	engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources;
1021	engine->device_free_chan_resources = rz_dmac_free_chan_resources;
1022	engine->device_tx_status = dma_cookie_status;
1023	engine->device_prep_slave_sg = rz_dmac_prep_slave_sg;
1024	engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy;
1025	engine->device_config = rz_dmac_config;
1026	engine->device_terminate_all = rz_dmac_terminate_all;
1027	engine->device_issue_pending = rz_dmac_issue_pending;
1028	engine->device_synchronize = rz_dmac_device_synchronize;
1029
1030	engine->copy_align = DMAENGINE_ALIGN_1_BYTE;
1031	dma_set_max_seg_size(engine->dev, U32_MAX);
1032
1033	ret = dma_async_device_register(engine);
1034	if (ret < 0) {
1035		dev_err(&pdev->dev, "unable to register\n");
1036		goto dma_register_err;
1037	}
1038	return 0;
1039
1040dma_register_err:
1041	of_dma_controller_free(pdev->dev.of_node);
1042err:
1043	channel_num = i ? i - 1 : 0;
1044	for (i = 0; i < channel_num; i++) {
1045		struct rz_dmac_chan *channel = &dmac->channels[i];
1046
1047		dma_free_coherent(&pdev->dev,
1048				  sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
1049				  channel->lmdesc.base,
1050				  channel->lmdesc.base_dma);
1051	}
1052
1053	reset_control_assert(dmac->rstc);
1054err_pm_runtime_put:
1055	pm_runtime_put(&pdev->dev);
1056err_pm_disable:
1057	pm_runtime_disable(&pdev->dev);
1058
1059	return ret;
1060}
1061
1062static void rz_dmac_remove(struct platform_device *pdev)
1063{
1064	struct rz_dmac *dmac = platform_get_drvdata(pdev);
1065	unsigned int i;
1066
1067	dma_async_device_unregister(&dmac->engine);
1068	of_dma_controller_free(pdev->dev.of_node);
1069	for (i = 0; i < dmac->n_channels; i++) {
1070		struct rz_dmac_chan *channel = &dmac->channels[i];
1071
1072		dma_free_coherent(&pdev->dev,
1073				  sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC,
1074				  channel->lmdesc.base,
1075				  channel->lmdesc.base_dma);
1076	}
1077	reset_control_assert(dmac->rstc);
1078	pm_runtime_put(&pdev->dev);
1079	pm_runtime_disable(&pdev->dev);
1080}
1081
1082static const struct of_device_id of_rz_dmac_match[] = {
1083	{ .compatible = "renesas,r9a09g057-dmac", },
1084	{ .compatible = "renesas,rz-dmac", },
1085	{ /* Sentinel */ }
1086};
1087MODULE_DEVICE_TABLE(of, of_rz_dmac_match);
1088
1089static struct platform_driver rz_dmac_driver = {
1090	.driver		= {
1091		.name	= "rz-dmac",
1092		.of_match_table = of_rz_dmac_match,
1093	},
1094	.probe		= rz_dmac_probe,
1095	.remove		= rz_dmac_remove,
1096};
1097
1098module_platform_driver(rz_dmac_driver);
1099
1100MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver");
1101MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
1102MODULE_LICENSE("GPL v2");