nbpfaxi.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2013-2014 Renesas Electronics Europe Ltd.
   4 * Author: Guennadi Liakhovetski <g.liakhovetski@gmx.de>
   5 */
   6
   7#include <linux/bitmap.h>
   8#include <linux/bitops.h>
   9#include <linux/clk.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/dmaengine.h>
  12#include <linux/err.h>
  13#include <linux/interrupt.h>
  14#include <linux/io.h>
  15#include <linux/log2.h>
  16#include <linux/module.h>
  17#include <linux/of.h>
  18#include <linux/of_dma.h>
  19#include <linux/platform_device.h>
  20#include <linux/slab.h>
  21
  22#include <dt-bindings/dma/nbpfaxi.h>
  23
  24#include "dmaengine.h"
  25
  26#define NBPF_REG_CHAN_OFFSET	0
  27#define NBPF_REG_CHAN_SIZE	0x40
  28
  29/* Channel Current Transaction Byte register */
  30#define NBPF_CHAN_CUR_TR_BYTE	0x20
  31
  32/* Channel Status register */
  33#define NBPF_CHAN_STAT	0x24
  34#define NBPF_CHAN_STAT_EN	1
  35#define NBPF_CHAN_STAT_TACT	4
  36#define NBPF_CHAN_STAT_ERR	0x10
  37#define NBPF_CHAN_STAT_END	0x20
  38#define NBPF_CHAN_STAT_TC	0x40
  39#define NBPF_CHAN_STAT_DER	0x400
  40
  41/* Channel Control register */
  42#define NBPF_CHAN_CTRL	0x28
  43#define NBPF_CHAN_CTRL_SETEN	1
  44#define NBPF_CHAN_CTRL_CLREN	2
  45#define NBPF_CHAN_CTRL_STG	4
  46#define NBPF_CHAN_CTRL_SWRST	8
  47#define NBPF_CHAN_CTRL_CLRRQ	0x10
  48#define NBPF_CHAN_CTRL_CLREND	0x20
  49#define NBPF_CHAN_CTRL_CLRTC	0x40
  50#define NBPF_CHAN_CTRL_SETSUS	0x100
  51#define NBPF_CHAN_CTRL_CLRSUS	0x200
  52
  53/* Channel Configuration register */
  54#define NBPF_CHAN_CFG	0x2c
  55#define NBPF_CHAN_CFG_SEL	7		/* terminal SELect: 0..7 */
  56#define NBPF_CHAN_CFG_REQD	8		/* REQuest Direction: DMAREQ is 0: input, 1: output */
  57#define NBPF_CHAN_CFG_LOEN	0x10		/* LOw ENable: low DMA request line is: 0: inactive, 1: active */
  58#define NBPF_CHAN_CFG_HIEN	0x20		/* HIgh ENable: high DMA request line is: 0: inactive, 1: active */
  59#define NBPF_CHAN_CFG_LVL	0x40		/* LeVeL: DMA request line is sensed as 0: edge, 1: level */
  60#define NBPF_CHAN_CFG_AM	0x700		/* ACK Mode: 0: Pulse mode, 1: Level mode, b'1x: Bus Cycle */
  61#define NBPF_CHAN_CFG_SDS	0xf000		/* Source Data Size: 0: 8 bits,... , 7: 1024 bits */
  62#define NBPF_CHAN_CFG_DDS	0xf0000		/* Destination Data Size: as above */
  63#define NBPF_CHAN_CFG_SAD	0x100000	/* Source ADdress counting: 0: increment, 1: fixed */
  64#define NBPF_CHAN_CFG_DAD	0x200000	/* Destination ADdress counting: 0: increment, 1: fixed */
  65#define NBPF_CHAN_CFG_TM	0x400000	/* Transfer Mode: 0: single, 1: block TM */
  66#define NBPF_CHAN_CFG_DEM	0x1000000	/* DMAEND interrupt Mask */
  67#define NBPF_CHAN_CFG_TCM	0x2000000	/* DMATCO interrupt Mask */
  68#define NBPF_CHAN_CFG_SBE	0x8000000	/* Sweep Buffer Enable */
  69#define NBPF_CHAN_CFG_RSEL	0x10000000	/* RM: Register Set sELect */
  70#define NBPF_CHAN_CFG_RSW	0x20000000	/* RM: Register Select sWitch */
  71#define NBPF_CHAN_CFG_REN	0x40000000	/* RM: Register Set Enable */
  72#define NBPF_CHAN_CFG_DMS	0x80000000	/* 0: register mode (RM), 1: link mode (LM) */
  73
  74#define NBPF_CHAN_NXLA	0x38
  75#define NBPF_CHAN_CRLA	0x3c
  76
  77/* Link Header field */
  78#define NBPF_HEADER_LV	1
  79#define NBPF_HEADER_LE	2
  80#define NBPF_HEADER_WBD	4
  81#define NBPF_HEADER_DIM	8
  82
  83#define NBPF_CTRL	0x300
  84#define NBPF_CTRL_PR	1		/* 0: fixed priority, 1: round robin */
  85#define NBPF_CTRL_LVINT	2		/* DMAEND and DMAERR signalling: 0: pulse, 1: level */
  86
  87#define NBPF_DSTAT_ER	0x314
  88#define NBPF_DSTAT_END	0x318
  89
  90#define NBPF_DMA_BUSWIDTHS \
  91	(BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
  92	 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  93	 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
  94	 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
  95	 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
  96
  97struct nbpf_config {
  98	int num_channels;
  99	int buffer_size;
 100};
 101
 102/*
 103 * We've got 3 types of objects, used to describe DMA transfers:
 104 * 1. high-level descriptor, containing a struct dma_async_tx_descriptor object
 105 *	in it, used to communicate with the user
 106 * 2. hardware DMA link descriptors, that we pass to DMAC for DMA transfer
 107 *	queuing, these must be DMAable, using either the streaming DMA API or
 108 *	allocated from coherent memory - one per SG segment
 109 * 3. one per SG segment descriptors, used to manage HW link descriptors from
 110 *	(2). They do not have to be DMAable. They can either be (a) allocated
 111 *	together with link descriptors as mixed (DMA / CPU) objects, or (b)
 112 *	separately. Even if allocated separately it would be best to link them
 113 *	to link descriptors once during channel resource allocation and always
 114 *	use them as a single object.
 115 * Therefore for both cases (a) and (b) at run-time objects (2) and (3) shall be
 116 * treated as a single SG segment descriptor.
 117 */
 118
 119struct nbpf_link_reg {
 120	u32	header;
 121	u32	src_addr;
 122	u32	dst_addr;
 123	u32	transaction_size;
 124	u32	config;
 125	u32	interval;
 126	u32	extension;
 127	u32	next;
 128} __packed;
 129
 130struct nbpf_device;
 131struct nbpf_channel;
 132struct nbpf_desc;
 133
 134struct nbpf_link_desc {
 135	struct nbpf_link_reg *hwdesc;
 136	dma_addr_t hwdesc_dma_addr;
 137	struct nbpf_desc *desc;
 138	struct list_head node;
 139};
 140
 141/**
 142 * struct nbpf_desc - DMA transfer descriptor
 143 * @async_tx:	dmaengine object
 144 * @user_wait:	waiting for a user ack
 145 * @length:	total transfer length
 146 * @chan:	associated DMAC channel
 147 * @sg:		list of hardware descriptors, represented by struct nbpf_link_desc
 148 * @node:	member in channel descriptor lists
 149 */
 150struct nbpf_desc {
 151	struct dma_async_tx_descriptor async_tx;
 152	bool user_wait;
 153	size_t length;
 154	struct nbpf_channel *chan;
 155	struct list_head sg;
 156	struct list_head node;
 157};
 158
 159/* Take a wild guess: allocate 4 segments per descriptor */
 160#define NBPF_SEGMENTS_PER_DESC 4
 161#define NBPF_DESCS_PER_PAGE ((PAGE_SIZE - sizeof(struct list_head)) /	\
 162	(sizeof(struct nbpf_desc) +					\
 163	 NBPF_SEGMENTS_PER_DESC *					\
 164	 (sizeof(struct nbpf_link_desc) + sizeof(struct nbpf_link_reg))))
 165#define NBPF_SEGMENTS_PER_PAGE (NBPF_SEGMENTS_PER_DESC * NBPF_DESCS_PER_PAGE)
 166
 167struct nbpf_desc_page {
 168	struct list_head node;
 169	struct nbpf_desc desc[NBPF_DESCS_PER_PAGE];
 170	struct nbpf_link_desc ldesc[NBPF_SEGMENTS_PER_PAGE];
 171	struct nbpf_link_reg hwdesc[NBPF_SEGMENTS_PER_PAGE];
 172};
 173
 174/**
 175 * struct nbpf_channel - one DMAC channel
 176 * @dma_chan:	standard dmaengine channel object
 177 * @tasklet:	channel specific tasklet used for callbacks
 178 * @base:	register address base
 179 * @nbpf:	DMAC
 180 * @name:	IRQ name
 181 * @irq:	IRQ number
 182 * @slave_src_addr:	source address for slave DMA
 183 * @slave_src_width:	source slave data size in bytes
 184 * @slave_src_burst:	maximum source slave burst size in bytes
 185 * @slave_dst_addr:	destination address for slave DMA
 186 * @slave_dst_width:	destination slave data size in bytes
 187 * @slave_dst_burst:	maximum destination slave burst size in bytes
 188 * @terminal:	DMA terminal, assigned to this channel
 189 * @dmarq_cfg:	DMA request line configuration - high / low, edge / level for NBPF_CHAN_CFG
 190 * @flags:	configuration flags from DT
 191 * @lock:	protect descriptor lists
 192 * @free_links:	list of free link descriptors
 193 * @free:	list of free descriptors
 194 * @queued:	list of queued descriptors
 195 * @active:	list of descriptors, scheduled for processing
 196 * @done:	list of completed descriptors, waiting post-processing
 197 * @desc_page:	list of additionally allocated descriptor pages - if any
 198 * @running:	linked descriptor of running transaction
 199 * @paused:	are translations on this channel paused?
 200 */
 201struct nbpf_channel {
 202	struct dma_chan dma_chan;
 203	struct tasklet_struct tasklet;
 204	void __iomem *base;
 205	struct nbpf_device *nbpf;
 206	char name[16];
 207	int irq;
 208	dma_addr_t slave_src_addr;
 209	size_t slave_src_width;
 210	size_t slave_src_burst;
 211	dma_addr_t slave_dst_addr;
 212	size_t slave_dst_width;
 213	size_t slave_dst_burst;
 214	unsigned int terminal;
 215	u32 dmarq_cfg;
 216	unsigned long flags;
 217	spinlock_t lock;
 218	struct list_head free_links;
 219	struct list_head free;
 220	struct list_head queued;
 221	struct list_head active;
 222	struct list_head done;
 223	struct list_head desc_page;
 224	struct nbpf_desc *running;
 225	bool paused;
 226};
 227
 228struct nbpf_device {
 229	struct dma_device dma_dev;
 230	void __iomem *base;
 231	u32 max_burst_mem_read;
 232	u32 max_burst_mem_write;
 233	struct clk *clk;
 234	const struct nbpf_config *config;
 235	unsigned int eirq;
 236	struct nbpf_channel chan[];
 237};
 238
 239enum nbpf_model {
 240	NBPF1B4,
 241	NBPF1B8,
 242	NBPF1B16,
 243	NBPF4B4,
 244	NBPF4B8,
 245	NBPF4B16,
 246	NBPF8B4,
 247	NBPF8B8,
 248	NBPF8B16,
 249};
 250
 251static struct nbpf_config nbpf_cfg[] = {
 252	[NBPF1B4] = {
 253		.num_channels = 1,
 254		.buffer_size = 4,
 255	},
 256	[NBPF1B8] = {
 257		.num_channels = 1,
 258		.buffer_size = 8,
 259	},
 260	[NBPF1B16] = {
 261		.num_channels = 1,
 262		.buffer_size = 16,
 263	},
 264	[NBPF4B4] = {
 265		.num_channels = 4,
 266		.buffer_size = 4,
 267	},
 268	[NBPF4B8] = {
 269		.num_channels = 4,
 270		.buffer_size = 8,
 271	},
 272	[NBPF4B16] = {
 273		.num_channels = 4,
 274		.buffer_size = 16,
 275	},
 276	[NBPF8B4] = {
 277		.num_channels = 8,
 278		.buffer_size = 4,
 279	},
 280	[NBPF8B8] = {
 281		.num_channels = 8,
 282		.buffer_size = 8,
 283	},
 284	[NBPF8B16] = {
 285		.num_channels = 8,
 286		.buffer_size = 16,
 287	},
 288};
 289
 290#define nbpf_to_chan(d) container_of(d, struct nbpf_channel, dma_chan)
 291
 292/*
 293 * dmaengine drivers seem to have a lot in common and instead of sharing more
 294 * code, they reimplement those common algorithms independently. In this driver
 295 * we try to separate the hardware-specific part from the (largely) generic
 296 * part. This improves code readability and makes it possible in the future to
 297 * reuse the generic code in form of a helper library. That generic code should
 298 * be suitable for various DMA controllers, using transfer descriptors in RAM
 299 * and pushing one SG list at a time to the DMA controller.
 300 */
 301
 302/*		Hardware-specific part		*/
 303
 304static inline u32 nbpf_chan_read(struct nbpf_channel *chan,
 305				 unsigned int offset)
 306{
 307	u32 data = ioread32(chan->base + offset);
 308	dev_dbg(chan->dma_chan.device->dev, "%s(0x%p + 0x%x) = 0x%x\n",
 309		__func__, chan->base, offset, data);
 310	return data;
 311}
 312
 313static inline void nbpf_chan_write(struct nbpf_channel *chan,
 314				   unsigned int offset, u32 data)
 315{
 316	iowrite32(data, chan->base + offset);
 317	dev_dbg(chan->dma_chan.device->dev, "%s(0x%p + 0x%x) = 0x%x\n",
 318		__func__, chan->base, offset, data);
 319}
 320
 321static inline u32 nbpf_read(struct nbpf_device *nbpf,
 322			    unsigned int offset)
 323{
 324	u32 data = ioread32(nbpf->base + offset);
 325	dev_dbg(nbpf->dma_dev.dev, "%s(0x%p + 0x%x) = 0x%x\n",
 326		__func__, nbpf->base, offset, data);
 327	return data;
 328}
 329
 330static inline void nbpf_write(struct nbpf_device *nbpf,
 331			      unsigned int offset, u32 data)
 332{
 333	iowrite32(data, nbpf->base + offset);
 334	dev_dbg(nbpf->dma_dev.dev, "%s(0x%p + 0x%x) = 0x%x\n",
 335		__func__, nbpf->base, offset, data);
 336}
 337
 338static void nbpf_chan_halt(struct nbpf_channel *chan)
 339{
 340	nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREN);
 341}
 342
 343static bool nbpf_status_get(struct nbpf_channel *chan)
 344{
 345	u32 status = nbpf_read(chan->nbpf, NBPF_DSTAT_END);
 346
 347	return status & BIT(chan - chan->nbpf->chan);
 348}
 349
 350static void nbpf_status_ack(struct nbpf_channel *chan)
 351{
 352	nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREND);
 353}
 354
 355static u32 nbpf_error_get(struct nbpf_device *nbpf)
 356{
 357	return nbpf_read(nbpf, NBPF_DSTAT_ER);
 358}
 359
 360static struct nbpf_channel *nbpf_error_get_channel(struct nbpf_device *nbpf, u32 error)
 361{
 362	return nbpf->chan + __ffs(error);
 363}
 364
 365static void nbpf_error_clear(struct nbpf_channel *chan)
 366{
 367	u32 status;
 368	int i;
 369
 370	/* Stop the channel, make sure DMA has been aborted */
 371	nbpf_chan_halt(chan);
 372
 373	for (i = 1000; i; i--) {
 374		status = nbpf_chan_read(chan, NBPF_CHAN_STAT);
 375		if (!(status & NBPF_CHAN_STAT_TACT))
 376			break;
 377		cpu_relax();
 378	}
 379
 380	if (!i)
 381		dev_err(chan->dma_chan.device->dev,
 382			"%s(): abort timeout, channel status 0x%x\n", __func__, status);
 383
 384	nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SWRST);
 385}
 386
 387static int nbpf_start(struct nbpf_desc *desc)
 388{
 389	struct nbpf_channel *chan = desc->chan;
 390	struct nbpf_link_desc *ldesc = list_first_entry(&desc->sg, struct nbpf_link_desc, node);
 391
 392	nbpf_chan_write(chan, NBPF_CHAN_NXLA, (u32)ldesc->hwdesc_dma_addr);
 393	nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SETEN | NBPF_CHAN_CTRL_CLRSUS);
 394	chan->paused = false;
 395
 396	/* Software trigger MEMCPY - only MEMCPY uses the block mode */
 397	if (ldesc->hwdesc->config & NBPF_CHAN_CFG_TM)
 398		nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_STG);
 399
 400	dev_dbg(chan->nbpf->dma_dev.dev, "%s(): next 0x%x, cur 0x%x\n", __func__,
 401		nbpf_chan_read(chan, NBPF_CHAN_NXLA), nbpf_chan_read(chan, NBPF_CHAN_CRLA));
 402
 403	return 0;
 404}
 405
 406static void nbpf_chan_prepare(struct nbpf_channel *chan)
 407{
 408	chan->dmarq_cfg = (chan->flags & NBPF_SLAVE_RQ_HIGH ? NBPF_CHAN_CFG_HIEN : 0) |
 409		(chan->flags & NBPF_SLAVE_RQ_LOW ? NBPF_CHAN_CFG_LOEN : 0) |
 410		(chan->flags & NBPF_SLAVE_RQ_LEVEL ?
 411		 NBPF_CHAN_CFG_LVL | (NBPF_CHAN_CFG_AM & 0x200) : 0) |
 412		chan->terminal;
 413}
 414
 415static void nbpf_chan_prepare_default(struct nbpf_channel *chan)
 416{
 417	/* Don't output DMAACK */
 418	chan->dmarq_cfg = NBPF_CHAN_CFG_AM & 0x400;
 419	chan->terminal = 0;
 420	chan->flags = 0;
 421}
 422
 423static void nbpf_chan_configure(struct nbpf_channel *chan)
 424{
 425	/*
 426	 * We assume, that only the link mode and DMA request line configuration
 427	 * have to be set in the configuration register manually. Dynamic
 428	 * per-transfer configuration will be loaded from transfer descriptors.
 429	 */
 430	nbpf_chan_write(chan, NBPF_CHAN_CFG, NBPF_CHAN_CFG_DMS | chan->dmarq_cfg);
 431}
 432
 433static u32 nbpf_xfer_ds(struct nbpf_device *nbpf, size_t size,
 434			enum dma_transfer_direction direction)
 435{
 436	int max_burst = nbpf->config->buffer_size * 8;
 437
 438	if (nbpf->max_burst_mem_read || nbpf->max_burst_mem_write) {
 439		switch (direction) {
 440		case DMA_MEM_TO_MEM:
 441			max_burst = min_not_zero(nbpf->max_burst_mem_read,
 442						 nbpf->max_burst_mem_write);
 443			break;
 444		case DMA_MEM_TO_DEV:
 445			if (nbpf->max_burst_mem_read)
 446				max_burst = nbpf->max_burst_mem_read;
 447			break;
 448		case DMA_DEV_TO_MEM:
 449			if (nbpf->max_burst_mem_write)
 450				max_burst = nbpf->max_burst_mem_write;
 451			break;
 452		case DMA_DEV_TO_DEV:
 453		default:
 454			break;
 455		}
 456	}
 457
 458	/* Maximum supported bursts depend on the buffer size */
 459	return min_t(int, __ffs(size), ilog2(max_burst));
 460}
 461
 462static size_t nbpf_xfer_size(struct nbpf_device *nbpf,
 463			     enum dma_slave_buswidth width, u32 burst)
 464{
 465	size_t size;
 466
 467	if (!burst)
 468		burst = 1;
 469
 470	switch (width) {
 471	case DMA_SLAVE_BUSWIDTH_8_BYTES:
 472		size = 8 * burst;
 473		break;
 474
 475	case DMA_SLAVE_BUSWIDTH_4_BYTES:
 476		size = 4 * burst;
 477		break;
 478
 479	case DMA_SLAVE_BUSWIDTH_2_BYTES:
 480		size = 2 * burst;
 481		break;
 482
 483	default:
 484		pr_warn("%s(): invalid bus width %u\n", __func__, width);
 485		fallthrough;
 486	case DMA_SLAVE_BUSWIDTH_1_BYTE:
 487		size = burst;
 488	}
 489
 490	return nbpf_xfer_ds(nbpf, size, DMA_TRANS_NONE);
 491}
 492
 493/*
 494 * We need a way to recognise slaves, whose data is sent "raw" over the bus,
 495 * i.e. it isn't known in advance how many bytes will be received. Therefore
 496 * the slave driver has to provide a "large enough" buffer and either read the
 497 * buffer, when it is full, or detect, that some data has arrived, then wait for
 498 * a timeout, if no more data arrives - receive what's already there. We want to
 499 * handle such slaves in a special way to allow an optimised mode for other
 500 * users, for whom the amount of data is known in advance. So far there's no way
 501 * to recognise such slaves. We use a data-width check to distinguish between
 502 * the SD host and the PL011 UART.
 503 */
 504
 505static int nbpf_prep_one(struct nbpf_link_desc *ldesc,
 506			 enum dma_transfer_direction direction,
 507			 dma_addr_t src, dma_addr_t dst, size_t size, bool last)
 508{
 509	struct nbpf_link_reg *hwdesc = ldesc->hwdesc;
 510	struct nbpf_desc *desc = ldesc->desc;
 511	struct nbpf_channel *chan = desc->chan;
 512	struct device *dev = chan->dma_chan.device->dev;
 513	size_t mem_xfer, slave_xfer;
 514	bool can_burst;
 515
 516	hwdesc->header = NBPF_HEADER_WBD | NBPF_HEADER_LV |
 517		(last ? NBPF_HEADER_LE : 0);
 518
 519	hwdesc->src_addr = src;
 520	hwdesc->dst_addr = dst;
 521	hwdesc->transaction_size = size;
 522
 523	/*
 524	 * set config: SAD, DAD, DDS, SDS, etc.
 525	 * Note on transfer sizes: the DMAC can perform unaligned DMA transfers,
 526	 * but it is important to have transaction size a multiple of both
 527	 * receiver and transmitter transfer sizes. It is also possible to use
 528	 * different RAM and device transfer sizes, and it does work well with
 529	 * some devices, e.g. with V08R07S01E SD host controllers, which can use
 530	 * 128 byte transfers. But this doesn't work with other devices,
 531	 * especially when the transaction size is unknown. This is the case,
 532	 * e.g. with serial drivers like amba-pl011.c. For reception it sets up
 533	 * the transaction size of 4K and if fewer bytes are received, it
 534	 * pauses DMA and reads out data received via DMA as well as those left
 535	 * in the Rx FIFO. For this to work with the RAM side using burst
 536	 * transfers we enable the SBE bit and terminate the transfer in our
 537	 * .device_pause handler.
 538	 */
 539	mem_xfer = nbpf_xfer_ds(chan->nbpf, size, direction);
 540
 541	switch (direction) {
 542	case DMA_DEV_TO_MEM:
 543		can_burst = chan->slave_src_width >= 3;
 544		slave_xfer = min(mem_xfer, can_burst ?
 545				 chan->slave_src_burst : chan->slave_src_width);
 546		/*
 547		 * Is the slave narrower than 64 bits, i.e. isn't using the full
 548		 * bus width and cannot use bursts?
 549		 */
 550		if (mem_xfer > chan->slave_src_burst && !can_burst)
 551			mem_xfer = chan->slave_src_burst;
 552		/* Device-to-RAM DMA is unreliable without REQD set */
 553		hwdesc->config = NBPF_CHAN_CFG_SAD | (NBPF_CHAN_CFG_DDS & (mem_xfer << 16)) |
 554			(NBPF_CHAN_CFG_SDS & (slave_xfer << 12)) | NBPF_CHAN_CFG_REQD |
 555			NBPF_CHAN_CFG_SBE;
 556		break;
 557
 558	case DMA_MEM_TO_DEV:
 559		slave_xfer = min(mem_xfer, chan->slave_dst_width >= 3 ?
 560				 chan->slave_dst_burst : chan->slave_dst_width);
 561		hwdesc->config = NBPF_CHAN_CFG_DAD | (NBPF_CHAN_CFG_SDS & (mem_xfer << 12)) |
 562			(NBPF_CHAN_CFG_DDS & (slave_xfer << 16)) | NBPF_CHAN_CFG_REQD;
 563		break;
 564
 565	case DMA_MEM_TO_MEM:
 566		hwdesc->config = NBPF_CHAN_CFG_TCM | NBPF_CHAN_CFG_TM |
 567			(NBPF_CHAN_CFG_SDS & (mem_xfer << 12)) |
 568			(NBPF_CHAN_CFG_DDS & (mem_xfer << 16));
 569		break;
 570
 571	default:
 572		return -EINVAL;
 573	}
 574
 575	hwdesc->config |= chan->dmarq_cfg | (last ? 0 : NBPF_CHAN_CFG_DEM) |
 576		NBPF_CHAN_CFG_DMS;
 577
 578	dev_dbg(dev, "%s(): desc @ %pad: hdr 0x%x, cfg 0x%x, %zu @ %pad -> %pad\n",
 579		__func__, &ldesc->hwdesc_dma_addr, hwdesc->header,
 580		hwdesc->config, size, &src, &dst);
 581
 582	dma_sync_single_for_device(dev, ldesc->hwdesc_dma_addr, sizeof(*hwdesc),
 583				   DMA_TO_DEVICE);
 584
 585	return 0;
 586}
 587
 588static size_t nbpf_bytes_left(struct nbpf_channel *chan)
 589{
 590	return nbpf_chan_read(chan, NBPF_CHAN_CUR_TR_BYTE);
 591}
 592
 593static void nbpf_configure(struct nbpf_device *nbpf)
 594{
 595	nbpf_write(nbpf, NBPF_CTRL, NBPF_CTRL_LVINT);
 596}
 597
 598/*		Generic part			*/
 599
 600/* DMA ENGINE functions */
 601static void nbpf_issue_pending(struct dma_chan *dchan)
 602{
 603	struct nbpf_channel *chan = nbpf_to_chan(dchan);
 604	unsigned long flags;
 605
 606	dev_dbg(dchan->device->dev, "Entry %s()\n", __func__);
 607
 608	spin_lock_irqsave(&chan->lock, flags);
 609	if (list_empty(&chan->queued))
 610		goto unlock;
 611
 612	list_splice_tail_init(&chan->queued, &chan->active);
 613
 614	if (!chan->running) {
 615		struct nbpf_desc *desc = list_first_entry(&chan->active,
 616						struct nbpf_desc, node);
 617		if (!nbpf_start(desc))
 618			chan->running = desc;
 619	}
 620
 621unlock:
 622	spin_unlock_irqrestore(&chan->lock, flags);
 623}
 624
 625static enum dma_status nbpf_tx_status(struct dma_chan *dchan,
 626		dma_cookie_t cookie, struct dma_tx_state *state)
 627{
 628	struct nbpf_channel *chan = nbpf_to_chan(dchan);
 629	enum dma_status status = dma_cookie_status(dchan, cookie, state);
 630
 631	if (state) {
 632		dma_cookie_t running;
 633		unsigned long flags;
 634
 635		spin_lock_irqsave(&chan->lock, flags);
 636		running = chan->running ? chan->running->async_tx.cookie : -EINVAL;
 637
 638		if (cookie == running) {
 639			state->residue = nbpf_bytes_left(chan);
 640			dev_dbg(dchan->device->dev, "%s(): residue %u\n", __func__,
 641				state->residue);
 642		} else if (status == DMA_IN_PROGRESS) {
 643			struct nbpf_desc *desc;
 644			bool found = false;
 645
 646			list_for_each_entry(desc, &chan->active, node)
 647				if (desc->async_tx.cookie == cookie) {
 648					found = true;
 649					break;
 650				}
 651
 652			if (!found)
 653				list_for_each_entry(desc, &chan->queued, node)
 654					if (desc->async_tx.cookie == cookie) {
 655						found = true;
 656						break;
 657
 658					}
 659
 660			state->residue = found ? desc->length : 0;
 661		}
 662
 663		spin_unlock_irqrestore(&chan->lock, flags);
 664	}
 665
 666	if (chan->paused)
 667		status = DMA_PAUSED;
 668
 669	return status;
 670}
 671
 672static dma_cookie_t nbpf_tx_submit(struct dma_async_tx_descriptor *tx)
 673{
 674	struct nbpf_desc *desc = container_of(tx, struct nbpf_desc, async_tx);
 675	struct nbpf_channel *chan = desc->chan;
 676	unsigned long flags;
 677	dma_cookie_t cookie;
 678
 679	spin_lock_irqsave(&chan->lock, flags);
 680	cookie = dma_cookie_assign(tx);
 681	list_add_tail(&desc->node, &chan->queued);
 682	spin_unlock_irqrestore(&chan->lock, flags);
 683
 684	dev_dbg(chan->dma_chan.device->dev, "Entry %s(%d)\n", __func__, cookie);
 685
 686	return cookie;
 687}
 688
 689static int nbpf_desc_page_alloc(struct nbpf_channel *chan)
 690{
 691	struct dma_chan *dchan = &chan->dma_chan;
 692	struct nbpf_desc_page *dpage = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 693	struct nbpf_link_desc *ldesc;
 694	struct nbpf_link_reg *hwdesc;
 695	struct nbpf_desc *desc;
 696	LIST_HEAD(head);
 697	LIST_HEAD(lhead);
 698	int i;
 699	struct device *dev = dchan->device->dev;
 700
 701	if (!dpage)
 702		return -ENOMEM;
 703
 704	dev_dbg(dev, "%s(): alloc %lu descriptors, %lu segments, total alloc %zu\n",
 705		__func__, NBPF_DESCS_PER_PAGE, NBPF_SEGMENTS_PER_PAGE, sizeof(*dpage));
 706
 707	for (i = 0, ldesc = dpage->ldesc, hwdesc = dpage->hwdesc;
 708	     i < ARRAY_SIZE(dpage->ldesc);
 709	     i++, ldesc++, hwdesc++) {
 710		ldesc->hwdesc = hwdesc;
 711		list_add_tail(&ldesc->node, &lhead);
 712		ldesc->hwdesc_dma_addr = dma_map_single(dchan->device->dev,
 713					hwdesc, sizeof(*hwdesc), DMA_TO_DEVICE);
 714		if (dma_mapping_error(dchan->device->dev,
 715				      ldesc->hwdesc_dma_addr))
 716			goto unmap_error;
 717
 718		dev_dbg(dev, "%s(): mapped 0x%p to %pad\n", __func__,
 719			hwdesc, &ldesc->hwdesc_dma_addr);
 720	}
 721
 722	for (i = 0, desc = dpage->desc;
 723	     i < ARRAY_SIZE(dpage->desc);
 724	     i++, desc++) {
 725		dma_async_tx_descriptor_init(&desc->async_tx, dchan);
 726		desc->async_tx.tx_submit = nbpf_tx_submit;
 727		desc->chan = chan;
 728		INIT_LIST_HEAD(&desc->sg);
 729		list_add_tail(&desc->node, &head);
 730	}
 731
 732	/*
 733	 * This function cannot be called from interrupt context, so, no need to
 734	 * save flags
 735	 */
 736	spin_lock_irq(&chan->lock);
 737	list_splice_tail(&lhead, &chan->free_links);
 738	list_splice_tail(&head, &chan->free);
 739	list_add(&dpage->node, &chan->desc_page);
 740	spin_unlock_irq(&chan->lock);
 741
 742	return ARRAY_SIZE(dpage->desc);
 743
 744unmap_error:
 745	while (i--) {
 746		ldesc--; hwdesc--;
 747
 748		dma_unmap_single(dchan->device->dev, ldesc->hwdesc_dma_addr,
 749				 sizeof(hwdesc), DMA_TO_DEVICE);
 750	}
 751
 752	return -ENOMEM;
 753}
 754
 755static void nbpf_desc_put(struct nbpf_desc *desc)
 756{
 757	struct nbpf_channel *chan = desc->chan;
 758	struct nbpf_link_desc *ldesc, *tmp;
 759	unsigned long flags;
 760
 761	spin_lock_irqsave(&chan->lock, flags);
 762	list_for_each_entry_safe(ldesc, tmp, &desc->sg, node)
 763		list_move(&ldesc->node, &chan->free_links);
 764
 765	list_add(&desc->node, &chan->free);
 766	spin_unlock_irqrestore(&chan->lock, flags);
 767}
 768
 769static void nbpf_scan_acked(struct nbpf_channel *chan)
 770{
 771	struct nbpf_desc *desc, *tmp;
 772	unsigned long flags;
 773	LIST_HEAD(head);
 774
 775	spin_lock_irqsave(&chan->lock, flags);
 776	list_for_each_entry_safe(desc, tmp, &chan->done, node)
 777		if (async_tx_test_ack(&desc->async_tx) && desc->user_wait) {
 778			list_move(&desc->node, &head);
 779			desc->user_wait = false;
 780		}
 781	spin_unlock_irqrestore(&chan->lock, flags);
 782
 783	list_for_each_entry_safe(desc, tmp, &head, node) {
 784		list_del(&desc->node);
 785		nbpf_desc_put(desc);
 786	}
 787}
 788
 789/*
 790 * We have to allocate descriptors with the channel lock dropped. This means,
 791 * before we re-acquire the lock buffers can be taken already, so we have to
 792 * re-check after re-acquiring the lock and possibly retry, if buffers are gone
 793 * again.
 794 */
 795static struct nbpf_desc *nbpf_desc_get(struct nbpf_channel *chan, size_t len)
 796{
 797	struct nbpf_desc *desc = NULL;
 798	struct nbpf_link_desc *ldesc, *prev = NULL;
 799
 800	nbpf_scan_acked(chan);
 801
 802	spin_lock_irq(&chan->lock);
 803
 804	do {
 805		int i = 0, ret;
 806
 807		if (list_empty(&chan->free)) {
 808			/* No more free descriptors */
 809			spin_unlock_irq(&chan->lock);
 810			ret = nbpf_desc_page_alloc(chan);
 811			if (ret < 0)
 812				return NULL;
 813			spin_lock_irq(&chan->lock);
 814			continue;
 815		}
 816		desc = list_first_entry(&chan->free, struct nbpf_desc, node);
 817		list_del(&desc->node);
 818
 819		do {
 820			if (list_empty(&chan->free_links)) {
 821				/* No more free link descriptors */
 822				spin_unlock_irq(&chan->lock);
 823				ret = nbpf_desc_page_alloc(chan);
 824				if (ret < 0) {
 825					nbpf_desc_put(desc);
 826					return NULL;
 827				}
 828				spin_lock_irq(&chan->lock);
 829				continue;
 830			}
 831
 832			ldesc = list_first_entry(&chan->free_links,
 833						 struct nbpf_link_desc, node);
 834			ldesc->desc = desc;
 835			if (prev)
 836				prev->hwdesc->next = (u32)ldesc->hwdesc_dma_addr;
 837
 838			prev = ldesc;
 839			list_move_tail(&ldesc->node, &desc->sg);
 840
 841			i++;
 842		} while (i < len);
 843	} while (!desc);
 844
 845	prev->hwdesc->next = 0;
 846
 847	spin_unlock_irq(&chan->lock);
 848
 849	return desc;
 850}
 851
 852static void nbpf_chan_idle(struct nbpf_channel *chan)
 853{
 854	struct nbpf_desc *desc, *tmp;
 855	unsigned long flags;
 856	LIST_HEAD(head);
 857
 858	spin_lock_irqsave(&chan->lock, flags);
 859
 860	list_splice_init(&chan->done, &head);
 861	list_splice_init(&chan->active, &head);
 862	list_splice_init(&chan->queued, &head);
 863
 864	chan->running = NULL;
 865
 866	spin_unlock_irqrestore(&chan->lock, flags);
 867
 868	list_for_each_entry_safe(desc, tmp, &head, node) {
 869		dev_dbg(chan->nbpf->dma_dev.dev, "%s(): force-free desc %p cookie %d\n",
 870			__func__, desc, desc->async_tx.cookie);
 871		list_del(&desc->node);
 872		nbpf_desc_put(desc);
 873	}
 874}
 875
 876static int nbpf_pause(struct dma_chan *dchan)
 877{
 878	struct nbpf_channel *chan = nbpf_to_chan(dchan);
 879
 880	dev_dbg(dchan->device->dev, "Entry %s\n", __func__);
 881
 882	chan->paused = true;
 883	nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SETSUS);
 884	/* See comment in nbpf_prep_one() */
 885	nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREN);
 886
 887	return 0;
 888}
 889
 890static int nbpf_terminate_all(struct dma_chan *dchan)
 891{
 892	struct nbpf_channel *chan = nbpf_to_chan(dchan);
 893
 894	dev_dbg(dchan->device->dev, "Entry %s\n", __func__);
 895	dev_dbg(dchan->device->dev, "Terminating\n");
 896
 897	nbpf_chan_halt(chan);
 898	nbpf_chan_idle(chan);
 899
 900	return 0;
 901}
 902
 903static int nbpf_config(struct dma_chan *dchan,
 904		       struct dma_slave_config *config)
 905{
 906	struct nbpf_channel *chan = nbpf_to_chan(dchan);
 907
 908	dev_dbg(dchan->device->dev, "Entry %s\n", __func__);
 909
 910	/*
 911	 * We could check config->slave_id to match chan->terminal here,
 912	 * but with DT they would be coming from the same source, so
 913	 * such a check would be superfluous
 914	 */
 915
 916	chan->slave_dst_addr = config->dst_addr;
 917	chan->slave_dst_width = nbpf_xfer_size(chan->nbpf,
 918					       config->dst_addr_width, 1);
 919	chan->slave_dst_burst = nbpf_xfer_size(chan->nbpf,
 920					       config->dst_addr_width,
 921					       config->dst_maxburst);
 922	chan->slave_src_addr = config->src_addr;
 923	chan->slave_src_width = nbpf_xfer_size(chan->nbpf,
 924					       config->src_addr_width, 1);
 925	chan->slave_src_burst = nbpf_xfer_size(chan->nbpf,
 926					       config->src_addr_width,
 927					       config->src_maxburst);
 928
 929	return 0;
 930}
 931
 932static struct dma_async_tx_descriptor *nbpf_prep_sg(struct nbpf_channel *chan,
 933		struct scatterlist *src_sg, struct scatterlist *dst_sg,
 934		size_t len, enum dma_transfer_direction direction,
 935		unsigned long flags)
 936{
 937	struct nbpf_link_desc *ldesc;
 938	struct scatterlist *mem_sg;
 939	struct nbpf_desc *desc;
 940	bool inc_src, inc_dst;
 941	size_t data_len = 0;
 942	int i = 0;
 943
 944	switch (direction) {
 945	case DMA_DEV_TO_MEM:
 946		mem_sg = dst_sg;
 947		inc_src = false;
 948		inc_dst = true;
 949		break;
 950
 951	case DMA_MEM_TO_DEV:
 952		mem_sg = src_sg;
 953		inc_src = true;
 954		inc_dst = false;
 955		break;
 956
 957	default:
 958	case DMA_MEM_TO_MEM:
 959		mem_sg = src_sg;
 960		inc_src = true;
 961		inc_dst = true;
 962	}
 963
 964	desc = nbpf_desc_get(chan, len);
 965	if (!desc)
 966		return NULL;
 967
 968	desc->async_tx.flags = flags;
 969	desc->async_tx.cookie = -EBUSY;
 970	desc->user_wait = false;
 971
 972	/*
 973	 * This is a private descriptor list, and we own the descriptor. No need
 974	 * to lock.
 975	 */
 976	list_for_each_entry(ldesc, &desc->sg, node) {
 977		int ret = nbpf_prep_one(ldesc, direction,
 978					sg_dma_address(src_sg),
 979					sg_dma_address(dst_sg),
 980					sg_dma_len(mem_sg),
 981					i == len - 1);
 982		if (ret < 0) {
 983			nbpf_desc_put(desc);
 984			return NULL;
 985		}
 986		data_len += sg_dma_len(mem_sg);
 987		if (inc_src)
 988			src_sg = sg_next(src_sg);
 989		if (inc_dst)
 990			dst_sg = sg_next(dst_sg);
 991		mem_sg = direction == DMA_DEV_TO_MEM ? dst_sg : src_sg;
 992		i++;
 993	}
 994
 995	desc->length = data_len;
 996
 997	/* The user has to return the descriptor to us ASAP via .tx_submit() */
 998	return &desc->async_tx;
 999}
1000
1001static struct dma_async_tx_descriptor *nbpf_prep_memcpy(
1002	struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
1003	size_t len, unsigned long flags)
1004{
1005	struct nbpf_channel *chan = nbpf_to_chan(dchan);
1006	struct scatterlist dst_sg;
1007	struct scatterlist src_sg;
1008
1009	sg_init_table(&dst_sg, 1);
1010	sg_init_table(&src_sg, 1);
1011
1012	sg_dma_address(&dst_sg) = dst;
1013	sg_dma_address(&src_sg) = src;
1014
1015	sg_dma_len(&dst_sg) = len;
1016	sg_dma_len(&src_sg) = len;
1017
1018	dev_dbg(dchan->device->dev, "%s(): %zu @ %pad -> %pad\n",
1019		__func__, len, &src, &dst);
1020
1021	return nbpf_prep_sg(chan, &src_sg, &dst_sg, 1,
1022			    DMA_MEM_TO_MEM, flags);
1023}
1024
1025static struct dma_async_tx_descriptor *nbpf_prep_slave_sg(
1026	struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len,
1027	enum dma_transfer_direction direction, unsigned long flags, void *context)
1028{
1029	struct nbpf_channel *chan = nbpf_to_chan(dchan);
1030	struct scatterlist slave_sg;
1031
1032	dev_dbg(dchan->device->dev, "Entry %s()\n", __func__);
1033
1034	sg_init_table(&slave_sg, 1);
1035
1036	switch (direction) {
1037	case DMA_MEM_TO_DEV:
1038		sg_dma_address(&slave_sg) = chan->slave_dst_addr;
1039		return nbpf_prep_sg(chan, sgl, &slave_sg, sg_len,
1040				    direction, flags);
1041
1042	case DMA_DEV_TO_MEM:
1043		sg_dma_address(&slave_sg) = chan->slave_src_addr;
1044		return nbpf_prep_sg(chan, &slave_sg, sgl, sg_len,
1045				    direction, flags);
1046
1047	default:
1048		return NULL;
1049	}
1050}
1051
1052static int nbpf_alloc_chan_resources(struct dma_chan *dchan)
1053{
1054	struct nbpf_channel *chan = nbpf_to_chan(dchan);
1055	int ret;
1056
1057	INIT_LIST_HEAD(&chan->free);
1058	INIT_LIST_HEAD(&chan->free_links);
1059	INIT_LIST_HEAD(&chan->queued);
1060	INIT_LIST_HEAD(&chan->active);
1061	INIT_LIST_HEAD(&chan->done);
1062
1063	ret = nbpf_desc_page_alloc(chan);
1064	if (ret < 0)
1065		return ret;
1066
1067	dev_dbg(dchan->device->dev, "Entry %s(): terminal %u\n", __func__,
1068		chan->terminal);
1069
1070	nbpf_chan_configure(chan);
1071
1072	return ret;
1073}
1074
1075static void nbpf_free_chan_resources(struct dma_chan *dchan)
1076{
1077	struct nbpf_channel *chan = nbpf_to_chan(dchan);
1078	struct nbpf_desc_page *dpage, *tmp;
1079
1080	dev_dbg(dchan->device->dev, "Entry %s()\n", __func__);
1081
1082	nbpf_chan_halt(chan);
1083	nbpf_chan_idle(chan);
1084	/* Clean up for if a channel is re-used for MEMCPY after slave DMA */
1085	nbpf_chan_prepare_default(chan);
1086
1087	list_for_each_entry_safe(dpage, tmp, &chan->desc_page, node) {
1088		struct nbpf_link_desc *ldesc;
1089		int i;
1090		list_del(&dpage->node);
1091		for (i = 0, ldesc = dpage->ldesc;
1092		     i < ARRAY_SIZE(dpage->ldesc);
1093		     i++, ldesc++)
1094			dma_unmap_single(dchan->device->dev, ldesc->hwdesc_dma_addr,
1095					 sizeof(*ldesc->hwdesc), DMA_TO_DEVICE);
1096		free_page((unsigned long)dpage);
1097	}
1098}
1099
1100static struct dma_chan *nbpf_of_xlate(struct of_phandle_args *dma_spec,
1101				      struct of_dma *ofdma)
1102{
1103	struct nbpf_device *nbpf = ofdma->of_dma_data;
1104	struct dma_chan *dchan;
1105	struct nbpf_channel *chan;
1106
1107	if (dma_spec->args_count != 2)
1108		return NULL;
1109
1110	dchan = dma_get_any_slave_channel(&nbpf->dma_dev);
1111	if (!dchan)
1112		return NULL;
1113
1114	dev_dbg(dchan->device->dev, "Entry %s(%pOFn)\n", __func__,
1115		dma_spec->np);
1116
1117	chan = nbpf_to_chan(dchan);
1118
1119	chan->terminal = dma_spec->args[0];
1120	chan->flags = dma_spec->args[1];
1121
1122	nbpf_chan_prepare(chan);
1123	nbpf_chan_configure(chan);
1124
1125	return dchan;
1126}
1127
1128static void nbpf_chan_tasklet(struct tasklet_struct *t)
1129{
1130	struct nbpf_channel *chan = from_tasklet(chan, t, tasklet);
1131	struct nbpf_desc *desc, *tmp;
1132	struct dmaengine_desc_callback cb;
1133
1134	while (!list_empty(&chan->done)) {
1135		bool found = false, must_put, recycling = false;
1136
1137		spin_lock_irq(&chan->lock);
1138
1139		list_for_each_entry_safe(desc, tmp, &chan->done, node) {
1140			if (!desc->user_wait) {
1141				/* Newly completed descriptor, have to process */
1142				found = true;
1143				break;
1144			} else if (async_tx_test_ack(&desc->async_tx)) {
1145				/*
1146				 * This descriptor was waiting for a user ACK,
1147				 * it can be recycled now.
1148				 */
1149				list_del(&desc->node);
1150				spin_unlock_irq(&chan->lock);
1151				nbpf_desc_put(desc);
1152				recycling = true;
1153				break;
1154			}
1155		}
1156
1157		if (recycling)
1158			continue;
1159
1160		if (!found) {
1161			/* This can happen if TERMINATE_ALL has been called */
1162			spin_unlock_irq(&chan->lock);
1163			break;
1164		}
1165
1166		dma_cookie_complete(&desc->async_tx);
1167
1168		/*
1169		 * With released lock we cannot dereference desc, maybe it's
1170		 * still on the "done" list
1171		 */
1172		if (async_tx_test_ack(&desc->async_tx)) {
1173			list_del(&desc->node);
1174			must_put = true;
1175		} else {
1176			desc->user_wait = true;
1177			must_put = false;
1178		}
1179
1180		dmaengine_desc_get_callback(&desc->async_tx, &cb);
1181
1182		/* ack and callback completed descriptor */
1183		spin_unlock_irq(&chan->lock);
1184
1185		dmaengine_desc_callback_invoke(&cb, NULL);
1186
1187		if (must_put)
1188			nbpf_desc_put(desc);
1189	}
1190}
1191
1192static irqreturn_t nbpf_chan_irq(int irq, void *dev)
1193{
1194	struct nbpf_channel *chan = dev;
1195	bool done = nbpf_status_get(chan);
1196	struct nbpf_desc *desc;
1197	irqreturn_t ret;
1198	bool bh = false;
1199
1200	if (!done)
1201		return IRQ_NONE;
1202
1203	nbpf_status_ack(chan);
1204
1205	dev_dbg(&chan->dma_chan.dev->device, "%s()\n", __func__);
1206
1207	spin_lock(&chan->lock);
1208	desc = chan->running;
1209	if (WARN_ON(!desc)) {
1210		ret = IRQ_NONE;
1211		goto unlock;
1212	} else {
1213		ret = IRQ_HANDLED;
1214		bh = true;
1215	}
1216
1217	list_move_tail(&desc->node, &chan->done);
1218	chan->running = NULL;
1219
1220	if (!list_empty(&chan->active)) {
1221		desc = list_first_entry(&chan->active,
1222					struct nbpf_desc, node);
1223		if (!nbpf_start(desc))
1224			chan->running = desc;
1225	}
1226
1227unlock:
1228	spin_unlock(&chan->lock);
1229
1230	if (bh)
1231		tasklet_schedule(&chan->tasklet);
1232
1233	return ret;
1234}
1235
1236static irqreturn_t nbpf_err_irq(int irq, void *dev)
1237{
1238	struct nbpf_device *nbpf = dev;
1239	u32 error = nbpf_error_get(nbpf);
1240
1241	dev_warn(nbpf->dma_dev.dev, "DMA error IRQ %u\n", irq);
1242
1243	if (!error)
1244		return IRQ_NONE;
1245
1246	do {
1247		struct nbpf_channel *chan = nbpf_error_get_channel(nbpf, error);
1248		/* On error: abort all queued transfers, no callback */
1249		nbpf_error_clear(chan);
1250		nbpf_chan_idle(chan);
1251		error = nbpf_error_get(nbpf);
1252	} while (error);
1253
1254	return IRQ_HANDLED;
1255}
1256
1257static int nbpf_chan_probe(struct nbpf_device *nbpf, int n)
1258{
1259	struct dma_device *dma_dev = &nbpf->dma_dev;
1260	struct nbpf_channel *chan = nbpf->chan + n;
1261	int ret;
1262
1263	chan->nbpf = nbpf;
1264	chan->base = nbpf->base + NBPF_REG_CHAN_OFFSET + NBPF_REG_CHAN_SIZE * n;
1265	INIT_LIST_HEAD(&chan->desc_page);
1266	spin_lock_init(&chan->lock);
1267	chan->dma_chan.device = dma_dev;
1268	dma_cookie_init(&chan->dma_chan);
1269	nbpf_chan_prepare_default(chan);
1270
1271	dev_dbg(dma_dev->dev, "%s(): channel %d: -> %p\n", __func__, n, chan->base);
1272
1273	snprintf(chan->name, sizeof(chan->name), "nbpf %d", n);
1274
1275	tasklet_setup(&chan->tasklet, nbpf_chan_tasklet);
1276	ret = devm_request_irq(dma_dev->dev, chan->irq,
1277			nbpf_chan_irq, IRQF_SHARED,
1278			chan->name, chan);
1279	if (ret < 0)
1280		return ret;
1281
1282	/* Add the channel to DMA device channel list */
1283	list_add_tail(&chan->dma_chan.device_node,
1284		      &dma_dev->channels);
1285
1286	return 0;
1287}
1288
1289static const struct of_device_id nbpf_match[] = {
1290	{.compatible = "renesas,nbpfaxi64dmac1b4",	.data = &nbpf_cfg[NBPF1B4]},
1291	{.compatible = "renesas,nbpfaxi64dmac1b8",	.data = &nbpf_cfg[NBPF1B8]},
1292	{.compatible = "renesas,nbpfaxi64dmac1b16",	.data = &nbpf_cfg[NBPF1B16]},
1293	{.compatible = "renesas,nbpfaxi64dmac4b4",	.data = &nbpf_cfg[NBPF4B4]},
1294	{.compatible = "renesas,nbpfaxi64dmac4b8",	.data = &nbpf_cfg[NBPF4B8]},
1295	{.compatible = "renesas,nbpfaxi64dmac4b16",	.data = &nbpf_cfg[NBPF4B16]},
1296	{.compatible = "renesas,nbpfaxi64dmac8b4",	.data = &nbpf_cfg[NBPF8B4]},
1297	{.compatible = "renesas,nbpfaxi64dmac8b8",	.data = &nbpf_cfg[NBPF8B8]},
1298	{.compatible = "renesas,nbpfaxi64dmac8b16",	.data = &nbpf_cfg[NBPF8B16]},
1299	{}
1300};
1301MODULE_DEVICE_TABLE(of, nbpf_match);
1302
1303static int nbpf_probe(struct platform_device *pdev)
1304{
1305	struct device *dev = &pdev->dev;
1306	struct device_node *np = dev->of_node;
1307	struct nbpf_device *nbpf;
1308	struct dma_device *dma_dev;
1309	const struct nbpf_config *cfg;
1310	int num_channels;
1311	int ret, irq, eirq, i;
1312	int irqbuf[9] /* maximum 8 channels + error IRQ */;
1313	unsigned int irqs = 0;
1314
1315	BUILD_BUG_ON(sizeof(struct nbpf_desc_page) > PAGE_SIZE);
1316
1317	/* DT only */
1318	if (!np)
1319		return -ENODEV;
1320
1321	cfg = of_device_get_match_data(dev);
1322	num_channels = cfg->num_channels;
1323
1324	nbpf = devm_kzalloc(dev, struct_size(nbpf, chan, num_channels),
1325			    GFP_KERNEL);
1326	if (!nbpf)
1327		return -ENOMEM;
1328
1329	dma_dev = &nbpf->dma_dev;
1330	dma_dev->dev = dev;
1331
1332	nbpf->base = devm_platform_ioremap_resource(pdev, 0);
1333	if (IS_ERR(nbpf->base))
1334		return PTR_ERR(nbpf->base);
1335
1336	nbpf->clk = devm_clk_get(dev, NULL);
1337	if (IS_ERR(nbpf->clk))
1338		return PTR_ERR(nbpf->clk);
1339
1340	of_property_read_u32(np, "max-burst-mem-read",
1341			     &nbpf->max_burst_mem_read);
1342	of_property_read_u32(np, "max-burst-mem-write",
1343			     &nbpf->max_burst_mem_write);
1344
1345	nbpf->config = cfg;
1346
1347	for (i = 0; irqs < ARRAY_SIZE(irqbuf); i++) {
1348		irq = platform_get_irq_optional(pdev, i);
1349		if (irq < 0 && irq != -ENXIO)
1350			return irq;
1351		if (irq > 0)
1352			irqbuf[irqs++] = irq;
1353	}
1354
1355	/*
1356	 * 3 IRQ resource schemes are supported:
1357	 * 1. 1 shared IRQ for error and all channels
1358	 * 2. 2 IRQs: one for error and one shared for all channels
1359	 * 3. 1 IRQ for error and an own IRQ for each channel
1360	 */
1361	if (irqs != 1 && irqs != 2 && irqs != num_channels + 1)
1362		return -ENXIO;
1363
1364	if (irqs == 1) {
1365		eirq = irqbuf[0];
1366
1367		for (i = 0; i < num_channels; i++)
1368			nbpf->chan[i].irq = irqbuf[0];
1369	} else {
1370		eirq = platform_get_irq_byname(pdev, "error");
1371		if (eirq < 0)
1372			return eirq;
1373
1374		if (irqs == num_channels + 1) {
1375			struct nbpf_channel *chan;
1376
1377			for (i = 0, chan = nbpf->chan; i < num_channels;
1378			     i++, chan++) {
1379				/* Skip the error IRQ */
1380				if (irqbuf[i] == eirq)
1381					i++;
1382				if (i >= ARRAY_SIZE(irqbuf))
1383					return -EINVAL;
1384				chan->irq = irqbuf[i];
1385			}
1386		} else {
1387			/* 2 IRQs and more than one channel */
1388			if (irqbuf[0] == eirq)
1389				irq = irqbuf[1];
1390			else
1391				irq = irqbuf[0];
1392
1393			for (i = 0; i < num_channels; i++)
1394				nbpf->chan[i].irq = irq;
1395		}
1396	}
1397
1398	ret = devm_request_irq(dev, eirq, nbpf_err_irq,
1399			       IRQF_SHARED, "dma error", nbpf);
1400	if (ret < 0)
1401		return ret;
1402	nbpf->eirq = eirq;
1403
1404	INIT_LIST_HEAD(&dma_dev->channels);
1405
1406	/* Create DMA Channel */
1407	for (i = 0; i < num_channels; i++) {
1408		ret = nbpf_chan_probe(nbpf, i);
1409		if (ret < 0)
1410			return ret;
1411	}
1412
1413	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1414	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1415	dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
1416
1417	/* Common and MEMCPY operations */
1418	dma_dev->device_alloc_chan_resources
1419		= nbpf_alloc_chan_resources;
1420	dma_dev->device_free_chan_resources = nbpf_free_chan_resources;
1421	dma_dev->device_prep_dma_memcpy = nbpf_prep_memcpy;
1422	dma_dev->device_tx_status = nbpf_tx_status;
1423	dma_dev->device_issue_pending = nbpf_issue_pending;
1424
1425	/*
1426	 * If we drop support for unaligned MEMCPY buffer addresses and / or
1427	 * lengths by setting
1428	 * dma_dev->copy_align = 4;
1429	 * then we can set transfer length to 4 bytes in nbpf_prep_one() for
1430	 * DMA_MEM_TO_MEM
1431	 */
1432
1433	/* Compulsory for DMA_SLAVE fields */
1434	dma_dev->device_prep_slave_sg = nbpf_prep_slave_sg;
1435	dma_dev->device_config = nbpf_config;
1436	dma_dev->device_pause = nbpf_pause;
1437	dma_dev->device_terminate_all = nbpf_terminate_all;
1438
1439	dma_dev->src_addr_widths = NBPF_DMA_BUSWIDTHS;
1440	dma_dev->dst_addr_widths = NBPF_DMA_BUSWIDTHS;
1441	dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1442
1443	platform_set_drvdata(pdev, nbpf);
1444
1445	ret = clk_prepare_enable(nbpf->clk);
1446	if (ret < 0)
1447		return ret;
1448
1449	nbpf_configure(nbpf);
1450
1451	ret = dma_async_device_register(dma_dev);
1452	if (ret < 0)
1453		goto e_clk_off;
1454
1455	ret = of_dma_controller_register(np, nbpf_of_xlate, nbpf);
1456	if (ret < 0)
1457		goto e_dma_dev_unreg;
1458
1459	return 0;
1460
1461e_dma_dev_unreg:
1462	dma_async_device_unregister(dma_dev);
1463e_clk_off:
1464	clk_disable_unprepare(nbpf->clk);
1465
1466	return ret;
1467}
1468
1469static void nbpf_remove(struct platform_device *pdev)
1470{
1471	struct nbpf_device *nbpf = platform_get_drvdata(pdev);
1472	int i;
1473
1474	devm_free_irq(&pdev->dev, nbpf->eirq, nbpf);
1475
1476	for (i = 0; i < nbpf->config->num_channels; i++) {
1477		struct nbpf_channel *chan = nbpf->chan + i;
1478
1479		devm_free_irq(&pdev->dev, chan->irq, chan);
1480
1481		tasklet_kill(&chan->tasklet);
1482	}
1483
1484	of_dma_controller_free(pdev->dev.of_node);
1485	dma_async_device_unregister(&nbpf->dma_dev);
1486	clk_disable_unprepare(nbpf->clk);
1487}
1488
1489static const struct platform_device_id nbpf_ids[] = {
1490	{"nbpfaxi64dmac1b4",	(kernel_ulong_t)&nbpf_cfg[NBPF1B4]},
1491	{"nbpfaxi64dmac1b8",	(kernel_ulong_t)&nbpf_cfg[NBPF1B8]},
1492	{"nbpfaxi64dmac1b16",	(kernel_ulong_t)&nbpf_cfg[NBPF1B16]},
1493	{"nbpfaxi64dmac4b4",	(kernel_ulong_t)&nbpf_cfg[NBPF4B4]},
1494	{"nbpfaxi64dmac4b8",	(kernel_ulong_t)&nbpf_cfg[NBPF4B8]},
1495	{"nbpfaxi64dmac4b16",	(kernel_ulong_t)&nbpf_cfg[NBPF4B16]},
1496	{"nbpfaxi64dmac8b4",	(kernel_ulong_t)&nbpf_cfg[NBPF8B4]},
1497	{"nbpfaxi64dmac8b8",	(kernel_ulong_t)&nbpf_cfg[NBPF8B8]},
1498	{"nbpfaxi64dmac8b16",	(kernel_ulong_t)&nbpf_cfg[NBPF8B16]},
1499	{},
1500};
1501MODULE_DEVICE_TABLE(platform, nbpf_ids);
1502
1503static int nbpf_runtime_suspend(struct device *dev)
1504{
1505	struct nbpf_device *nbpf = dev_get_drvdata(dev);
1506	clk_disable_unprepare(nbpf->clk);
1507	return 0;
1508}
1509
1510static int nbpf_runtime_resume(struct device *dev)
1511{
1512	struct nbpf_device *nbpf = dev_get_drvdata(dev);
1513	return clk_prepare_enable(nbpf->clk);
1514}
1515
1516static const struct dev_pm_ops nbpf_pm_ops = {
1517	RUNTIME_PM_OPS(nbpf_runtime_suspend, nbpf_runtime_resume, NULL)
1518};
1519
1520static struct platform_driver nbpf_driver = {
1521	.driver = {
1522		.name = "dma-nbpf",
1523		.of_match_table = nbpf_match,
1524		.pm = pm_ptr(&nbpf_pm_ops),
1525	},
1526	.id_table = nbpf_ids,
1527	.probe = nbpf_probe,
1528	.remove = nbpf_remove,
1529};
1530
1531module_platform_driver(nbpf_driver);
1532
1533MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>");
1534MODULE_DESCRIPTION("dmaengine driver for NBPFAXI64* DMACs");
1535MODULE_LICENSE("GPL v2");