dma-axi-dmac.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Driver for the Analog Devices AXI-DMAC core
   4 *
   5 * Copyright 2013-2019 Analog Devices Inc.
   6 *  Author: Lars-Peter Clausen <lars@metafoo.de>
   7 */
   8
   9#include <linux/adi-axi-common.h>
  10#include <linux/bitfield.h>
  11#include <linux/clk.h>
  12#include <linux/device.h>
  13#include <linux/dma-mapping.h>
  14#include <linux/dmaengine.h>
  15#include <linux/err.h>
  16#include <linux/interrupt.h>
  17#include <linux/io.h>
  18#include <linux/kernel.h>
  19#include <linux/module.h>
  20#include <linux/of.h>
  21#include <linux/of_dma.h>
  22#include <linux/of_address.h>
  23#include <linux/platform_device.h>
  24#include <linux/regmap.h>
  25#include <linux/slab.h>
  26
  27#include <dt-bindings/dma/axi-dmac.h>
  28
  29#include "dmaengine.h"
  30#include "virt-dma.h"
  31
  32/*
  33 * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has
  34 * various instantiation parameters which decided the exact feature set support
  35 * by the core.
  36 *
  37 * Each channel of the core has a source interface and a destination interface.
  38 * The number of channels and the type of the channel interfaces is selected at
  39 * configuration time. A interface can either be a connected to a central memory
  40 * interconnect, which allows access to system memory, or it can be connected to
  41 * a dedicated bus which is directly connected to a data port on a peripheral.
  42 * Given that those are configuration options of the core that are selected when
  43 * it is instantiated this means that they can not be changed by software at
  44 * runtime. By extension this means that each channel is uni-directional. It can
  45 * either be device to memory or memory to device, but not both. Also since the
  46 * device side is a dedicated data bus only connected to a single peripheral
  47 * there is no address than can or needs to be configured for the device side.
  48 */
  49
  50#define AXI_DMAC_REG_INTERFACE_DESC	0x10
  51#define   AXI_DMAC_DMA_SRC_TYPE_MSK	GENMASK(13, 12)
  52#define   AXI_DMAC_DMA_SRC_TYPE_GET(x)	FIELD_GET(AXI_DMAC_DMA_SRC_TYPE_MSK, x)
  53#define   AXI_DMAC_DMA_SRC_WIDTH_MSK	GENMASK(11, 8)
  54#define   AXI_DMAC_DMA_SRC_WIDTH_GET(x)	FIELD_GET(AXI_DMAC_DMA_SRC_WIDTH_MSK, x)
  55#define   AXI_DMAC_DMA_DST_TYPE_MSK	GENMASK(5, 4)
  56#define   AXI_DMAC_DMA_DST_TYPE_GET(x)	FIELD_GET(AXI_DMAC_DMA_DST_TYPE_MSK, x)
  57#define   AXI_DMAC_DMA_DST_WIDTH_MSK	GENMASK(3, 0)
  58#define   AXI_DMAC_DMA_DST_WIDTH_GET(x)	FIELD_GET(AXI_DMAC_DMA_DST_WIDTH_MSK, x)
  59#define AXI_DMAC_REG_COHERENCY_DESC	0x14
  60#define   AXI_DMAC_DST_COHERENT_MSK	BIT(0)
  61#define   AXI_DMAC_DST_COHERENT_GET(x)	FIELD_GET(AXI_DMAC_DST_COHERENT_MSK, x)
  62
  63#define AXI_DMAC_REG_IRQ_MASK		0x80
  64#define AXI_DMAC_REG_IRQ_PENDING	0x84
  65#define AXI_DMAC_REG_IRQ_SOURCE		0x88
  66
  67#define AXI_DMAC_REG_CTRL		0x400
  68#define AXI_DMAC_REG_TRANSFER_ID	0x404
  69#define AXI_DMAC_REG_START_TRANSFER	0x408
  70#define AXI_DMAC_REG_FLAGS		0x40c
  71#define AXI_DMAC_REG_DEST_ADDRESS	0x410
  72#define AXI_DMAC_REG_SRC_ADDRESS	0x414
  73#define AXI_DMAC_REG_X_LENGTH		0x418
  74#define AXI_DMAC_REG_Y_LENGTH		0x41c
  75#define AXI_DMAC_REG_DEST_STRIDE	0x420
  76#define AXI_DMAC_REG_SRC_STRIDE		0x424
  77#define AXI_DMAC_REG_TRANSFER_DONE	0x428
  78#define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c
  79#define AXI_DMAC_REG_STATUS		0x430
  80#define AXI_DMAC_REG_CURRENT_SRC_ADDR	0x434
  81#define AXI_DMAC_REG_CURRENT_DEST_ADDR	0x438
  82#define AXI_DMAC_REG_PARTIAL_XFER_LEN	0x44c
  83#define AXI_DMAC_REG_PARTIAL_XFER_ID	0x450
  84#define AXI_DMAC_REG_CURRENT_SG_ID	0x454
  85#define AXI_DMAC_REG_SG_ADDRESS		0x47c
  86#define AXI_DMAC_REG_SG_ADDRESS_HIGH	0x4bc
  87
  88#define AXI_DMAC_CTRL_ENABLE		BIT(0)
  89#define AXI_DMAC_CTRL_PAUSE		BIT(1)
  90#define AXI_DMAC_CTRL_ENABLE_SG		BIT(2)
  91
  92#define AXI_DMAC_IRQ_SOT		BIT(0)
  93#define AXI_DMAC_IRQ_EOT		BIT(1)
  94
  95#define AXI_DMAC_FLAG_CYCLIC		BIT(0)
  96#define AXI_DMAC_FLAG_LAST		BIT(1)
  97#define AXI_DMAC_FLAG_PARTIAL_REPORT	BIT(2)
  98
  99#define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31)
 100
 101/* The maximum ID allocated by the hardware is 31 */
 102#define AXI_DMAC_SG_UNUSED 32U
 103
 104/* Flags for axi_dmac_hw_desc.flags */
 105#define AXI_DMAC_HW_FLAG_LAST		BIT(0)
 106#define AXI_DMAC_HW_FLAG_IRQ		BIT(1)
 107
 108struct axi_dmac_hw_desc {
 109	u32 flags;
 110	u32 id;
 111	u64 dest_addr;
 112	u64 src_addr;
 113	u64 next_sg_addr;
 114	u32 y_len;
 115	u32 x_len;
 116	u32 src_stride;
 117	u32 dst_stride;
 118	u64 __pad[2];
 119};
 120
 121struct axi_dmac_sg {
 122	unsigned int partial_len;
 123	bool schedule_when_free;
 124
 125	struct axi_dmac_hw_desc *hw;
 126	dma_addr_t hw_phys;
 127};
 128
 129struct axi_dmac_desc {
 130	struct virt_dma_desc vdesc;
 131	struct axi_dmac_chan *chan;
 132
 133	bool cyclic;
 134	bool have_partial_xfer;
 135
 136	unsigned int num_submitted;
 137	unsigned int num_completed;
 138	unsigned int num_sgs;
 139	struct axi_dmac_sg sg[] __counted_by(num_sgs);
 140};
 141
 142struct axi_dmac_chan {
 143	struct virt_dma_chan vchan;
 144
 145	struct axi_dmac_desc *next_desc;
 146	struct list_head active_descs;
 147	enum dma_transfer_direction direction;
 148
 149	unsigned int src_width;
 150	unsigned int dest_width;
 151	unsigned int src_type;
 152	unsigned int dest_type;
 153
 154	unsigned int max_length;
 155	unsigned int address_align_mask;
 156	unsigned int length_align_mask;
 157
 158	bool hw_partial_xfer;
 159	bool hw_cyclic;
 160	bool hw_2d;
 161	bool hw_sg;
 162};
 163
 164struct axi_dmac {
 165	void __iomem *base;
 166	int irq;
 167
 168	struct clk *clk;
 169
 170	struct dma_device dma_dev;
 171	struct axi_dmac_chan chan;
 172};
 173
 174static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan)
 175{
 176	return container_of(chan->vchan.chan.device, struct axi_dmac,
 177		dma_dev);
 178}
 179
 180static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c)
 181{
 182	return container_of(c, struct axi_dmac_chan, vchan.chan);
 183}
 184
 185static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc)
 186{
 187	return container_of(vdesc, struct axi_dmac_desc, vdesc);
 188}
 189
 190static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg,
 191	unsigned int val)
 192{
 193	writel(val, axi_dmac->base + reg);
 194}
 195
 196static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg)
 197{
 198	return readl(axi_dmac->base + reg);
 199}
 200
 201static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan)
 202{
 203	return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM;
 204}
 205
 206static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan)
 207{
 208	return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM;
 209}
 210
 211static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len)
 212{
 213	if (len == 0)
 214		return false;
 215	if ((len & chan->length_align_mask) != 0) /* Not aligned */
 216		return false;
 217	return true;
 218}
 219
 220static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr)
 221{
 222	if ((addr & chan->address_align_mask) != 0) /* Not aligned */
 223		return false;
 224	return true;
 225}
 226
 227static void axi_dmac_start_transfer(struct axi_dmac_chan *chan)
 228{
 229	struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 230	struct virt_dma_desc *vdesc;
 231	struct axi_dmac_desc *desc;
 232	struct axi_dmac_sg *sg;
 233	unsigned int flags = 0;
 234	unsigned int val;
 235
 236	val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER);
 237	if (val) /* Queue is full, wait for the next SOT IRQ */
 238		return;
 239
 240	desc = chan->next_desc;
 241
 242	if (!desc) {
 243		vdesc = vchan_next_desc(&chan->vchan);
 244		if (!vdesc)
 245			return;
 246		list_move_tail(&vdesc->node, &chan->active_descs);
 247		desc = to_axi_dmac_desc(vdesc);
 248		chan->next_desc = desc;
 249	}
 250	sg = &desc->sg[desc->num_submitted];
 251
 252	/* Already queued in cyclic mode. Wait for it to finish */
 253	if (sg->hw->id != AXI_DMAC_SG_UNUSED) {
 254		sg->schedule_when_free = true;
 255		return;
 256	}
 257
 258	if (chan->hw_sg) {
 259		chan->next_desc = NULL;
 260	} else if (++desc->num_submitted == desc->num_sgs ||
 261		   desc->have_partial_xfer) {
 262		if (desc->cyclic)
 263			desc->num_submitted = 0; /* Start again */
 264		else
 265			chan->next_desc = NULL;
 266		flags |= AXI_DMAC_FLAG_LAST;
 267	}
 268
 269	sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID);
 270
 271	if (!chan->hw_sg) {
 272		if (axi_dmac_dest_is_mem(chan)) {
 273			axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr);
 274			axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride);
 275		}
 276
 277		if (axi_dmac_src_is_mem(chan)) {
 278			axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr);
 279			axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride);
 280		}
 281	}
 282
 283	/*
 284	 * If the hardware supports cyclic transfers and there is no callback to
 285	 * call, enable hw cyclic mode to avoid unnecessary interrupts.
 286	 */
 287	if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) {
 288		if (chan->hw_sg)
 289			desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ;
 290		else if (desc->num_sgs == 1)
 291			flags |= AXI_DMAC_FLAG_CYCLIC;
 292	}
 293
 294	if (chan->hw_partial_xfer)
 295		flags |= AXI_DMAC_FLAG_PARTIAL_REPORT;
 296
 297	if (chan->hw_sg) {
 298		axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys);
 299		axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH,
 300			       (u64)sg->hw_phys >> 32);
 301	} else {
 302		axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len);
 303		axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len);
 304	}
 305	axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags);
 306	axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1);
 307}
 308
 309static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan)
 310{
 311	return list_first_entry_or_null(&chan->active_descs,
 312		struct axi_dmac_desc, vdesc.node);
 313}
 314
 315static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan,
 316	struct axi_dmac_sg *sg)
 317{
 318	if (chan->hw_2d)
 319		return (sg->hw->x_len + 1) * (sg->hw->y_len + 1);
 320	else
 321		return (sg->hw->x_len + 1);
 322}
 323
 324static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan)
 325{
 326	struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 327	struct axi_dmac_desc *desc;
 328	struct axi_dmac_sg *sg;
 329	u32 xfer_done, len, id, i;
 330	bool found_sg;
 331
 332	do {
 333		len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN);
 334		id  = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID);
 335
 336		found_sg = false;
 337		list_for_each_entry(desc, &chan->active_descs, vdesc.node) {
 338			for (i = 0; i < desc->num_sgs; i++) {
 339				sg = &desc->sg[i];
 340				if (sg->hw->id == AXI_DMAC_SG_UNUSED)
 341					continue;
 342				if (sg->hw->id == id) {
 343					desc->have_partial_xfer = true;
 344					sg->partial_len = len;
 345					found_sg = true;
 346					break;
 347				}
 348			}
 349			if (found_sg)
 350				break;
 351		}
 352
 353		if (found_sg) {
 354			dev_dbg(dmac->dma_dev.dev,
 355				"Found partial segment id=%u, len=%u\n",
 356				id, len);
 357		} else {
 358			dev_warn(dmac->dma_dev.dev,
 359				 "Not found partial segment id=%u, len=%u\n",
 360				 id, len);
 361		}
 362
 363		/* Check if we have any more partial transfers */
 364		xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
 365		xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE);
 366
 367	} while (!xfer_done);
 368}
 369
 370static void axi_dmac_compute_residue(struct axi_dmac_chan *chan,
 371	struct axi_dmac_desc *active)
 372{
 373	struct dmaengine_result *rslt = &active->vdesc.tx_result;
 374	unsigned int start = active->num_completed - 1;
 375	struct axi_dmac_sg *sg;
 376	unsigned int i, total;
 377
 378	rslt->result = DMA_TRANS_NOERROR;
 379	rslt->residue = 0;
 380
 381	if (chan->hw_sg)
 382		return;
 383
 384	/*
 385	 * We get here if the last completed segment is partial, which
 386	 * means we can compute the residue from that segment onwards
 387	 */
 388	for (i = start; i < active->num_sgs; i++) {
 389		sg = &active->sg[i];
 390		total = axi_dmac_total_sg_bytes(chan, sg);
 391		rslt->residue += (total - sg->partial_len);
 392	}
 393}
 394
 395static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan,
 396	unsigned int completed_transfers)
 397{
 398	struct axi_dmac_desc *active;
 399	struct axi_dmac_sg *sg;
 400	bool start_next = false;
 401
 402	active = axi_dmac_active_desc(chan);
 403	if (!active)
 404		return false;
 405
 406	if (chan->hw_partial_xfer &&
 407	    (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE))
 408		axi_dmac_dequeue_partial_xfers(chan);
 409
 410	if (chan->hw_sg) {
 411		if (active->cyclic) {
 412			vchan_cyclic_callback(&active->vdesc);
 413		} else {
 414			list_del(&active->vdesc.node);
 415			vchan_cookie_complete(&active->vdesc);
 416			active = axi_dmac_active_desc(chan);
 417			start_next = !!active;
 418		}
 419	} else {
 420		do {
 421			sg = &active->sg[active->num_completed];
 422			if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */
 423				break;
 424			if (!(BIT(sg->hw->id) & completed_transfers))
 425				break;
 426			active->num_completed++;
 427			sg->hw->id = AXI_DMAC_SG_UNUSED;
 428			if (sg->schedule_when_free) {
 429				sg->schedule_when_free = false;
 430				start_next = true;
 431			}
 432
 433			if (sg->partial_len)
 434				axi_dmac_compute_residue(chan, active);
 435
 436			if (active->cyclic)
 437				vchan_cyclic_callback(&active->vdesc);
 438
 439			if (active->num_completed == active->num_sgs ||
 440			    sg->partial_len) {
 441				if (active->cyclic) {
 442					active->num_completed = 0; /* wrap around */
 443				} else {
 444					list_del(&active->vdesc.node);
 445					vchan_cookie_complete(&active->vdesc);
 446					active = axi_dmac_active_desc(chan);
 447				}
 448			}
 449		} while (active);
 450	}
 451
 452	return start_next;
 453}
 454
 455static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid)
 456{
 457	struct axi_dmac *dmac = devid;
 458	unsigned int pending;
 459	bool start_next = false;
 460
 461	pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING);
 462	if (!pending)
 463		return IRQ_NONE;
 464
 465	axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending);
 466
 467	spin_lock(&dmac->chan.vchan.lock);
 468	/* One or more transfers have finished */
 469	if (pending & AXI_DMAC_IRQ_EOT) {
 470		unsigned int completed;
 471
 472		completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE);
 473		start_next = axi_dmac_transfer_done(&dmac->chan, completed);
 474	}
 475	/* Space has become available in the descriptor queue */
 476	if ((pending & AXI_DMAC_IRQ_SOT) || start_next)
 477		axi_dmac_start_transfer(&dmac->chan);
 478	spin_unlock(&dmac->chan.vchan.lock);
 479
 480	return IRQ_HANDLED;
 481}
 482
 483static int axi_dmac_terminate_all(struct dma_chan *c)
 484{
 485	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 486	struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 487	unsigned long flags;
 488	LIST_HEAD(head);
 489
 490	spin_lock_irqsave(&chan->vchan.lock, flags);
 491	axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0);
 492	chan->next_desc = NULL;
 493	vchan_get_all_descriptors(&chan->vchan, &head);
 494	list_splice_tail_init(&chan->active_descs, &head);
 495	spin_unlock_irqrestore(&chan->vchan.lock, flags);
 496
 497	vchan_dma_desc_free_list(&chan->vchan, &head);
 498
 499	return 0;
 500}
 501
 502static void axi_dmac_synchronize(struct dma_chan *c)
 503{
 504	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 505
 506	vchan_synchronize(&chan->vchan);
 507}
 508
 509static void axi_dmac_issue_pending(struct dma_chan *c)
 510{
 511	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 512	struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 513	unsigned long flags;
 514	u32 ctrl = AXI_DMAC_CTRL_ENABLE;
 515
 516	if (chan->hw_sg)
 517		ctrl |= AXI_DMAC_CTRL_ENABLE_SG;
 518
 519	axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl);
 520
 521	spin_lock_irqsave(&chan->vchan.lock, flags);
 522	if (vchan_issue_pending(&chan->vchan))
 523		axi_dmac_start_transfer(chan);
 524	spin_unlock_irqrestore(&chan->vchan.lock, flags);
 525}
 526
 527static struct axi_dmac_desc *
 528axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs)
 529{
 530	struct axi_dmac *dmac = chan_to_axi_dmac(chan);
 531	struct device *dev = dmac->dma_dev.dev;
 532	struct axi_dmac_hw_desc *hws;
 533	struct axi_dmac_desc *desc;
 534	dma_addr_t hw_phys;
 535	unsigned int i;
 536
 537	desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT);
 538	if (!desc)
 539		return NULL;
 540	desc->num_sgs = num_sgs;
 541	desc->chan = chan;
 542
 543	hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)),
 544				&hw_phys, GFP_ATOMIC);
 545	if (!hws) {
 546		kfree(desc);
 547		return NULL;
 548	}
 549
 550	for (i = 0; i < num_sgs; i++) {
 551		desc->sg[i].hw = &hws[i];
 552		desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws);
 553
 554		hws[i].id = AXI_DMAC_SG_UNUSED;
 555		hws[i].flags = 0;
 556
 557		/* Link hardware descriptors */
 558		hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws);
 559	}
 560
 561	/* The last hardware descriptor will trigger an interrupt */
 562	desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ;
 563
 564	return desc;
 565}
 566
 567static void axi_dmac_free_desc(struct axi_dmac_desc *desc)
 568{
 569	struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan);
 570	struct device *dev = dmac->dma_dev.dev;
 571	struct axi_dmac_hw_desc *hw = desc->sg[0].hw;
 572	dma_addr_t hw_phys = desc->sg[0].hw_phys;
 573
 574	dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)),
 575			  hw, hw_phys);
 576	kfree(desc);
 577}
 578
 579static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan,
 580	enum dma_transfer_direction direction, dma_addr_t addr,
 581	unsigned int num_periods, unsigned int period_len,
 582	struct axi_dmac_sg *sg)
 583{
 584	unsigned int num_segments, i;
 585	unsigned int segment_size;
 586	unsigned int len;
 587
 588	/* Split into multiple equally sized segments if necessary */
 589	num_segments = DIV_ROUND_UP(period_len, chan->max_length);
 590	segment_size = DIV_ROUND_UP(period_len, num_segments);
 591	/* Take care of alignment */
 592	segment_size = ((segment_size - 1) | chan->length_align_mask) + 1;
 593
 594	for (i = 0; i < num_periods; i++) {
 595		for (len = period_len; len > segment_size; sg++) {
 596			if (direction == DMA_DEV_TO_MEM)
 597				sg->hw->dest_addr = addr;
 598			else
 599				sg->hw->src_addr = addr;
 600			sg->hw->x_len = segment_size - 1;
 601			sg->hw->y_len = 0;
 602			sg->hw->flags = 0;
 603			addr += segment_size;
 604			len -= segment_size;
 605		}
 606
 607		if (direction == DMA_DEV_TO_MEM)
 608			sg->hw->dest_addr = addr;
 609		else
 610			sg->hw->src_addr = addr;
 611		sg->hw->x_len = len - 1;
 612		sg->hw->y_len = 0;
 613		sg++;
 614		addr += len;
 615	}
 616
 617	return sg;
 618}
 619
 620static struct dma_async_tx_descriptor *
 621axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs,
 622				 size_t nb, enum dma_transfer_direction direction,
 623				 unsigned long flags)
 624{
 625	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 626	struct axi_dmac_desc *desc;
 627	unsigned int num_sgs = 0;
 628	struct axi_dmac_sg *dsg;
 629	size_t i;
 630
 631	if (direction != chan->direction)
 632		return NULL;
 633
 634	for (i = 0; i < nb; i++)
 635		num_sgs += DIV_ROUND_UP(vecs[i].len, chan->max_length);
 636
 637	desc = axi_dmac_alloc_desc(chan, num_sgs);
 638	if (!desc)
 639		return NULL;
 640
 641	dsg = desc->sg;
 642
 643	for (i = 0; i < nb; i++) {
 644		if (!axi_dmac_check_addr(chan, vecs[i].addr) ||
 645		    !axi_dmac_check_len(chan, vecs[i].len)) {
 646			kfree(desc);
 647			return NULL;
 648		}
 649
 650		dsg = axi_dmac_fill_linear_sg(chan, direction, vecs[i].addr, 1,
 651					      vecs[i].len, dsg);
 652	}
 653
 654	desc->cyclic = false;
 655
 656	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 657}
 658
 659static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg(
 660	struct dma_chan *c, struct scatterlist *sgl,
 661	unsigned int sg_len, enum dma_transfer_direction direction,
 662	unsigned long flags, void *context)
 663{
 664	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 665	struct axi_dmac_desc *desc;
 666	struct axi_dmac_sg *dsg;
 667	struct scatterlist *sg;
 668	unsigned int num_sgs;
 669	unsigned int i;
 670
 671	if (direction != chan->direction)
 672		return NULL;
 673
 674	num_sgs = 0;
 675	for_each_sg(sgl, sg, sg_len, i)
 676		num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length);
 677
 678	desc = axi_dmac_alloc_desc(chan, num_sgs);
 679	if (!desc)
 680		return NULL;
 681
 682	dsg = desc->sg;
 683
 684	for_each_sg(sgl, sg, sg_len, i) {
 685		if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) ||
 686		    !axi_dmac_check_len(chan, sg_dma_len(sg))) {
 687			axi_dmac_free_desc(desc);
 688			return NULL;
 689		}
 690
 691		dsg = axi_dmac_fill_linear_sg(chan, direction, sg_dma_address(sg), 1,
 692			sg_dma_len(sg), dsg);
 693	}
 694
 695	desc->cyclic = false;
 696
 697	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 698}
 699
 700static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic(
 701	struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len,
 702	size_t period_len, enum dma_transfer_direction direction,
 703	unsigned long flags)
 704{
 705	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 706	struct axi_dmac_desc *desc;
 707	unsigned int num_periods, num_segments, num_sgs;
 708
 709	if (direction != chan->direction)
 710		return NULL;
 711
 712	if (!axi_dmac_check_len(chan, buf_len) ||
 713	    !axi_dmac_check_addr(chan, buf_addr))
 714		return NULL;
 715
 716	if (period_len == 0 || buf_len % period_len)
 717		return NULL;
 718
 719	num_periods = buf_len / period_len;
 720	num_segments = DIV_ROUND_UP(period_len, chan->max_length);
 721	num_sgs = num_periods * num_segments;
 722
 723	desc = axi_dmac_alloc_desc(chan, num_sgs);
 724	if (!desc)
 725		return NULL;
 726
 727	/* Chain the last descriptor to the first, and remove its "last" flag */
 728	desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys;
 729	desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST;
 730
 731	axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods,
 732		period_len, desc->sg);
 733
 734	desc->cyclic = true;
 735
 736	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 737}
 738
 739static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved(
 740	struct dma_chan *c, struct dma_interleaved_template *xt,
 741	unsigned long flags)
 742{
 743	struct axi_dmac_chan *chan = to_axi_dmac_chan(c);
 744	struct axi_dmac_desc *desc;
 745	size_t dst_icg, src_icg;
 746
 747	if (xt->frame_size != 1)
 748		return NULL;
 749
 750	if (xt->dir != chan->direction)
 751		return NULL;
 752
 753	if (axi_dmac_src_is_mem(chan)) {
 754		if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start))
 755			return NULL;
 756	}
 757
 758	if (axi_dmac_dest_is_mem(chan)) {
 759		if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start))
 760			return NULL;
 761	}
 762
 763	dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
 764	src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
 765
 766	if (chan->hw_2d) {
 767		if (!axi_dmac_check_len(chan, xt->sgl[0].size) ||
 768		    xt->numf == 0)
 769			return NULL;
 770		if (xt->sgl[0].size + dst_icg > chan->max_length ||
 771		    xt->sgl[0].size + src_icg > chan->max_length)
 772			return NULL;
 773	} else {
 774		if (dst_icg != 0 || src_icg != 0)
 775			return NULL;
 776		if (chan->max_length / xt->sgl[0].size < xt->numf)
 777			return NULL;
 778		if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf))
 779			return NULL;
 780	}
 781
 782	desc = axi_dmac_alloc_desc(chan, 1);
 783	if (!desc)
 784		return NULL;
 785
 786	if (axi_dmac_src_is_mem(chan)) {
 787		desc->sg[0].hw->src_addr = xt->src_start;
 788		desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg;
 789	}
 790
 791	if (axi_dmac_dest_is_mem(chan)) {
 792		desc->sg[0].hw->dest_addr = xt->dst_start;
 793		desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg;
 794	}
 795
 796	if (chan->hw_2d) {
 797		desc->sg[0].hw->x_len = xt->sgl[0].size - 1;
 798		desc->sg[0].hw->y_len = xt->numf - 1;
 799	} else {
 800		desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1;
 801		desc->sg[0].hw->y_len = 0;
 802	}
 803
 804	if (flags & DMA_CYCLIC)
 805		desc->cyclic = true;
 806
 807	return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
 808}
 809
 810static void axi_dmac_free_chan_resources(struct dma_chan *c)
 811{
 812	vchan_free_chan_resources(to_virt_chan(c));
 813}
 814
 815static void axi_dmac_desc_free(struct virt_dma_desc *vdesc)
 816{
 817	axi_dmac_free_desc(to_axi_dmac_desc(vdesc));
 818}
 819
 820static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg)
 821{
 822	switch (reg) {
 823	case AXI_DMAC_REG_IRQ_MASK:
 824	case AXI_DMAC_REG_IRQ_SOURCE:
 825	case AXI_DMAC_REG_IRQ_PENDING:
 826	case AXI_DMAC_REG_CTRL:
 827	case AXI_DMAC_REG_TRANSFER_ID:
 828	case AXI_DMAC_REG_START_TRANSFER:
 829	case AXI_DMAC_REG_FLAGS:
 830	case AXI_DMAC_REG_DEST_ADDRESS:
 831	case AXI_DMAC_REG_SRC_ADDRESS:
 832	case AXI_DMAC_REG_X_LENGTH:
 833	case AXI_DMAC_REG_Y_LENGTH:
 834	case AXI_DMAC_REG_DEST_STRIDE:
 835	case AXI_DMAC_REG_SRC_STRIDE:
 836	case AXI_DMAC_REG_TRANSFER_DONE:
 837	case AXI_DMAC_REG_ACTIVE_TRANSFER_ID:
 838	case AXI_DMAC_REG_STATUS:
 839	case AXI_DMAC_REG_CURRENT_SRC_ADDR:
 840	case AXI_DMAC_REG_CURRENT_DEST_ADDR:
 841	case AXI_DMAC_REG_PARTIAL_XFER_LEN:
 842	case AXI_DMAC_REG_PARTIAL_XFER_ID:
 843	case AXI_DMAC_REG_CURRENT_SG_ID:
 844	case AXI_DMAC_REG_SG_ADDRESS:
 845	case AXI_DMAC_REG_SG_ADDRESS_HIGH:
 846		return true;
 847	default:
 848		return false;
 849	}
 850}
 851
 852static const struct regmap_config axi_dmac_regmap_config = {
 853	.reg_bits = 32,
 854	.val_bits = 32,
 855	.reg_stride = 4,
 856	.max_register = AXI_DMAC_REG_PARTIAL_XFER_ID,
 857	.readable_reg = axi_dmac_regmap_rdwr,
 858	.writeable_reg = axi_dmac_regmap_rdwr,
 859};
 860
 861static void axi_dmac_adjust_chan_params(struct axi_dmac_chan *chan)
 862{
 863	chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1;
 864
 865	if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
 866		chan->direction = DMA_MEM_TO_MEM;
 867	else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan))
 868		chan->direction = DMA_MEM_TO_DEV;
 869	else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan))
 870		chan->direction = DMA_DEV_TO_MEM;
 871	else
 872		chan->direction = DMA_DEV_TO_DEV;
 873}
 874
 875/*
 876 * The configuration stored in the devicetree matches the configuration
 877 * parameters of the peripheral instance and allows the driver to know which
 878 * features are implemented and how it should behave.
 879 */
 880static int axi_dmac_parse_chan_dt(struct device_node *of_chan,
 881	struct axi_dmac_chan *chan)
 882{
 883	u32 val;
 884	int ret;
 885
 886	ret = of_property_read_u32(of_chan, "reg", &val);
 887	if (ret)
 888		return ret;
 889
 890	/* We only support 1 channel for now */
 891	if (val != 0)
 892		return -EINVAL;
 893
 894	ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val);
 895	if (ret)
 896		return ret;
 897	if (val > AXI_DMAC_BUS_TYPE_FIFO)
 898		return -EINVAL;
 899	chan->src_type = val;
 900
 901	ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val);
 902	if (ret)
 903		return ret;
 904	if (val > AXI_DMAC_BUS_TYPE_FIFO)
 905		return -EINVAL;
 906	chan->dest_type = val;
 907
 908	ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val);
 909	if (ret)
 910		return ret;
 911	chan->src_width = val / 8;
 912
 913	ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val);
 914	if (ret)
 915		return ret;
 916	chan->dest_width = val / 8;
 917
 918	axi_dmac_adjust_chan_params(chan);
 919
 920	return 0;
 921}
 922
 923static int axi_dmac_parse_dt(struct device *dev, struct axi_dmac *dmac)
 924{
 925	struct device_node *of_channels, *of_chan;
 926	int ret;
 927
 928	of_channels = of_get_child_by_name(dev->of_node, "adi,channels");
 929	if (of_channels == NULL)
 930		return -ENODEV;
 931
 932	for_each_child_of_node(of_channels, of_chan) {
 933		ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan);
 934		if (ret) {
 935			of_node_put(of_chan);
 936			of_node_put(of_channels);
 937			return -EINVAL;
 938		}
 939	}
 940	of_node_put(of_channels);
 941
 942	return 0;
 943}
 944
 945static int axi_dmac_read_chan_config(struct device *dev, struct axi_dmac *dmac)
 946{
 947	struct axi_dmac_chan *chan = &dmac->chan;
 948	unsigned int val, desc;
 949
 950	desc = axi_dmac_read(dmac, AXI_DMAC_REG_INTERFACE_DESC);
 951	if (desc == 0) {
 952		dev_err(dev, "DMA interface register reads zero\n");
 953		return -EFAULT;
 954	}
 955
 956	val = AXI_DMAC_DMA_SRC_TYPE_GET(desc);
 957	if (val > AXI_DMAC_BUS_TYPE_FIFO) {
 958		dev_err(dev, "Invalid source bus type read: %d\n", val);
 959		return -EINVAL;
 960	}
 961	chan->src_type = val;
 962
 963	val = AXI_DMAC_DMA_DST_TYPE_GET(desc);
 964	if (val > AXI_DMAC_BUS_TYPE_FIFO) {
 965		dev_err(dev, "Invalid destination bus type read: %d\n", val);
 966		return -EINVAL;
 967	}
 968	chan->dest_type = val;
 969
 970	val = AXI_DMAC_DMA_SRC_WIDTH_GET(desc);
 971	if (val == 0) {
 972		dev_err(dev, "Source bus width is zero\n");
 973		return -EINVAL;
 974	}
 975	/* widths are stored in log2 */
 976	chan->src_width = 1 << val;
 977
 978	val = AXI_DMAC_DMA_DST_WIDTH_GET(desc);
 979	if (val == 0) {
 980		dev_err(dev, "Destination bus width is zero\n");
 981		return -EINVAL;
 982	}
 983	chan->dest_width = 1 << val;
 984
 985	axi_dmac_adjust_chan_params(chan);
 986
 987	return 0;
 988}
 989
 990static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version)
 991{
 992	struct axi_dmac_chan *chan = &dmac->chan;
 993
 994	axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC);
 995	if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC)
 996		chan->hw_cyclic = true;
 997
 998	axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff);
 999	if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS))
1000		chan->hw_sg = true;
1001
1002	axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1);
1003	if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1)
1004		chan->hw_2d = true;
1005
1006	axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0xffffffff);
1007	chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
1008	if (chan->max_length != UINT_MAX)
1009		chan->max_length++;
1010
1011	axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff);
1012	if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 &&
1013	    chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
1014		dev_err(dmac->dma_dev.dev,
1015			"Destination memory-mapped interface not supported.");
1016		return -ENODEV;
1017	}
1018
1019	axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff);
1020	if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 &&
1021	    chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) {
1022		dev_err(dmac->dma_dev.dev,
1023			"Source memory-mapped interface not supported.");
1024		return -ENODEV;
1025	}
1026
1027	if (version >= ADI_AXI_PCORE_VER(4, 2, 'a'))
1028		chan->hw_partial_xfer = true;
1029
1030	if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) {
1031		axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00);
1032		chan->length_align_mask =
1033			axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH);
1034	} else {
1035		chan->length_align_mask = chan->address_align_mask;
1036	}
1037
1038	return 0;
1039}
1040
1041static void axi_dmac_tasklet_kill(void *task)
1042{
1043	tasklet_kill(task);
1044}
1045
1046static void axi_dmac_free_dma_controller(void *of_node)
1047{
1048	of_dma_controller_free(of_node);
1049}
1050
1051static int axi_dmac_probe(struct platform_device *pdev)
1052{
1053	struct dma_device *dma_dev;
1054	struct axi_dmac *dmac;
1055	struct regmap *regmap;
1056	unsigned int version;
1057	u32 irq_mask = 0;
1058	int ret;
1059
1060	dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL);
1061	if (!dmac)
1062		return -ENOMEM;
1063
1064	dmac->irq = platform_get_irq(pdev, 0);
1065	if (dmac->irq < 0)
1066		return dmac->irq;
1067	if (dmac->irq == 0)
1068		return -EINVAL;
1069
1070	dmac->base = devm_platform_ioremap_resource(pdev, 0);
1071	if (IS_ERR(dmac->base))
1072		return PTR_ERR(dmac->base);
1073
1074	dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL);
1075	if (IS_ERR(dmac->clk))
1076		return PTR_ERR(dmac->clk);
1077
1078	version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION);
1079
1080	if (version >= ADI_AXI_PCORE_VER(4, 3, 'a'))
1081		ret = axi_dmac_read_chan_config(&pdev->dev, dmac);
1082	else
1083		ret = axi_dmac_parse_dt(&pdev->dev, dmac);
1084
1085	if (ret < 0)
1086		return ret;
1087
1088	INIT_LIST_HEAD(&dmac->chan.active_descs);
1089
1090	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
1091
1092	dma_dev = &dmac->dma_dev;
1093	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1094	dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1095	dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask);
1096	dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources;
1097	dma_dev->device_tx_status = dma_cookie_status;
1098	dma_dev->device_issue_pending = axi_dmac_issue_pending;
1099	dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg;
1100	dma_dev->device_prep_peripheral_dma_vec = axi_dmac_prep_peripheral_dma_vec;
1101	dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic;
1102	dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved;
1103	dma_dev->device_terminate_all = axi_dmac_terminate_all;
1104	dma_dev->device_synchronize = axi_dmac_synchronize;
1105	dma_dev->dev = &pdev->dev;
1106	dma_dev->src_addr_widths = BIT(dmac->chan.src_width);
1107	dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width);
1108	dma_dev->directions = BIT(dmac->chan.direction);
1109	dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1110	dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */
1111	INIT_LIST_HEAD(&dma_dev->channels);
1112
1113	dmac->chan.vchan.desc_free = axi_dmac_desc_free;
1114	vchan_init(&dmac->chan.vchan, dma_dev);
1115
1116	ret = axi_dmac_detect_caps(dmac, version);
1117	if (ret)
1118		return ret;
1119
1120	dma_dev->copy_align = (dmac->chan.address_align_mask + 1);
1121
1122	if (dmac->chan.hw_sg)
1123		irq_mask |= AXI_DMAC_IRQ_SOT;
1124
1125	axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask);
1126
1127	if (of_dma_is_coherent(pdev->dev.of_node)) {
1128		ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC);
1129
1130		if (version < ADI_AXI_PCORE_VER(4, 4, 'a') ||
1131		    !AXI_DMAC_DST_COHERENT_GET(ret)) {
1132			dev_err(dmac->dma_dev.dev,
1133				"Coherent DMA not supported in hardware");
1134			return -EINVAL;
1135		}
1136	}
1137
1138	ret = dmaenginem_async_device_register(dma_dev);
1139	if (ret)
1140		return ret;
1141
1142	/*
1143	 * Put the action in here so it get's done before unregistering the DMA
1144	 * device.
1145	 */
1146	ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill,
1147				       &dmac->chan.vchan.task);
1148	if (ret)
1149		return ret;
1150
1151	ret = of_dma_controller_register(pdev->dev.of_node,
1152		of_dma_xlate_by_chan_id, dma_dev);
1153	if (ret)
1154		return ret;
1155
1156	ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_free_dma_controller,
1157				       pdev->dev.of_node);
1158	if (ret)
1159		return ret;
1160
1161	ret = devm_request_irq(&pdev->dev, dmac->irq, axi_dmac_interrupt_handler,
1162			       IRQF_SHARED, dev_name(&pdev->dev), dmac);
1163	if (ret)
1164		return ret;
1165
1166	regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base,
1167		 &axi_dmac_regmap_config);
1168
1169	return PTR_ERR_OR_ZERO(regmap);
1170}
1171
1172static const struct of_device_id axi_dmac_of_match_table[] = {
1173	{ .compatible = "adi,axi-dmac-1.00.a" },
1174	{ },
1175};
1176MODULE_DEVICE_TABLE(of, axi_dmac_of_match_table);
1177
1178static struct platform_driver axi_dmac_driver = {
1179	.driver = {
1180		.name = "dma-axi-dmac",
1181		.of_match_table = axi_dmac_of_match_table,
1182	},
1183	.probe = axi_dmac_probe,
1184};
1185module_platform_driver(axi_dmac_driver);
1186
1187MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>");
1188MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller");
1189MODULE_LICENSE("GPL v2");