xilinx_dma.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * DMA driver for Xilinx Video DMA Engine
   4 *
   5 * Copyright (C) 2010-2014 Xilinx, Inc. All rights reserved.
   6 *
   7 * Based on the Freescale DMA driver.
   8 *
   9 * Description:
  10 * The AXI Video Direct Memory Access (AXI VDMA) core is a soft Xilinx IP
  11 * core that provides high-bandwidth direct memory access between memory
  12 * and AXI4-Stream type video target peripherals. The core provides efficient
  13 * two dimensional DMA operations with independent asynchronous read (S2MM)
  14 * and write (MM2S) channel operation. It can be configured to have either
  15 * one channel or two channels. If configured as two channels, one is to
  16 * transmit to the video device (MM2S) and another is to receive from the
  17 * video device (S2MM). Initialization, status, interrupt and management
  18 * registers are accessed through an AXI4-Lite slave interface.
  19 *
  20 * The AXI Direct Memory Access (AXI DMA) core is a soft Xilinx IP core that
  21 * provides high-bandwidth one dimensional direct memory access between memory
  22 * and AXI4-Stream target peripherals. It supports one receive and one
  23 * transmit channel, both of them optional at synthesis time.
  24 *
  25 * The AXI CDMA, is a soft IP, which provides high-bandwidth Direct Memory
  26 * Access (DMA) between a memory-mapped source address and a memory-mapped
  27 * destination address.
  28 *
  29 * The AXI Multichannel Direct Memory Access (AXI MCDMA) core is a soft
  30 * Xilinx IP that provides high-bandwidth direct memory access between
  31 * memory and AXI4-Stream target peripherals. It provides scatter gather
  32 * (SG) interface with multiple channels independent configuration support.
  33 *
  34 */
  35
  36#include <linux/bitops.h>
  37#include <linux/dmapool.h>
  38#include <linux/dma/xilinx_dma.h>
  39#include <linux/init.h>
  40#include <linux/interrupt.h>
  41#include <linux/io.h>
  42#include <linux/iopoll.h>
  43#include <linux/module.h>
  44#include <linux/of.h>
  45#include <linux/of_dma.h>
  46#include <linux/of_irq.h>
  47#include <linux/platform_device.h>
  48#include <linux/slab.h>
  49#include <linux/string_choices.h>
  50#include <linux/clk.h>
  51#include <linux/io-64-nonatomic-lo-hi.h>
  52
  53#include "../dmaengine.h"
  54
  55/* Register/Descriptor Offsets */
  56#define XILINX_DMA_MM2S_CTRL_OFFSET		0x0000
  57#define XILINX_DMA_S2MM_CTRL_OFFSET		0x0030
  58#define XILINX_VDMA_MM2S_DESC_OFFSET		0x0050
  59#define XILINX_VDMA_S2MM_DESC_OFFSET		0x00a0
  60
  61/* Control Registers */
  62#define XILINX_DMA_REG_DMACR			0x0000
  63#define XILINX_DMA_DMACR_DELAY_MAX		0xff
  64#define XILINX_DMA_DMACR_DELAY_SHIFT		24
  65#define XILINX_DMA_DMACR_FRAME_COUNT_MAX	0xff
  66#define XILINX_DMA_DMACR_FRAME_COUNT_SHIFT	16
  67#define XILINX_DMA_DMACR_ERR_IRQ		BIT(14)
  68#define XILINX_DMA_DMACR_DLY_CNT_IRQ		BIT(13)
  69#define XILINX_DMA_DMACR_FRM_CNT_IRQ		BIT(12)
  70#define XILINX_DMA_DMACR_MASTER_SHIFT		8
  71#define XILINX_DMA_DMACR_FSYNCSRC_SHIFT	5
  72#define XILINX_DMA_DMACR_FRAMECNT_EN		BIT(4)
  73#define XILINX_DMA_DMACR_GENLOCK_EN		BIT(3)
  74#define XILINX_DMA_DMACR_RESET			BIT(2)
  75#define XILINX_DMA_DMACR_CIRC_EN		BIT(1)
  76#define XILINX_DMA_DMACR_RUNSTOP		BIT(0)
  77#define XILINX_DMA_DMACR_FSYNCSRC_MASK		GENMASK(6, 5)
  78#define XILINX_DMA_DMACR_DELAY_MASK		GENMASK(31, 24)
  79#define XILINX_DMA_DMACR_FRAME_COUNT_MASK	GENMASK(23, 16)
  80#define XILINX_DMA_DMACR_MASTER_MASK		GENMASK(11, 8)
  81
  82#define XILINX_DMA_REG_DMASR			0x0004
  83#define XILINX_DMA_DMASR_EOL_LATE_ERR		BIT(15)
  84#define XILINX_DMA_DMASR_ERR_IRQ		BIT(14)
  85#define XILINX_DMA_DMASR_DLY_CNT_IRQ		BIT(13)
  86#define XILINX_DMA_DMASR_FRM_CNT_IRQ		BIT(12)
  87#define XILINX_DMA_DMASR_SOF_LATE_ERR		BIT(11)
  88#define XILINX_DMA_DMASR_SG_DEC_ERR		BIT(10)
  89#define XILINX_DMA_DMASR_SG_SLV_ERR		BIT(9)
  90#define XILINX_DMA_DMASR_EOF_EARLY_ERR		BIT(8)
  91#define XILINX_DMA_DMASR_SOF_EARLY_ERR		BIT(7)
  92#define XILINX_DMA_DMASR_DMA_DEC_ERR		BIT(6)
  93#define XILINX_DMA_DMASR_DMA_SLAVE_ERR		BIT(5)
  94#define XILINX_DMA_DMASR_DMA_INT_ERR		BIT(4)
  95#define XILINX_DMA_DMASR_SG_MASK		BIT(3)
  96#define XILINX_DMA_DMASR_IDLE			BIT(1)
  97#define XILINX_DMA_DMASR_HALTED		BIT(0)
  98#define XILINX_DMA_DMASR_DELAY_MASK		GENMASK(31, 24)
  99#define XILINX_DMA_DMASR_FRAME_COUNT_MASK	GENMASK(23, 16)
 100
 101#define XILINX_DMA_REG_CURDESC			0x0008
 102#define XILINX_DMA_REG_TAILDESC		0x0010
 103#define XILINX_DMA_REG_REG_INDEX		0x0014
 104#define XILINX_DMA_REG_FRMSTORE		0x0018
 105#define XILINX_DMA_REG_THRESHOLD		0x001c
 106#define XILINX_DMA_REG_FRMPTR_STS		0x0024
 107#define XILINX_DMA_REG_PARK_PTR		0x0028
 108#define XILINX_DMA_PARK_PTR_WR_REF_SHIFT	8
 109#define XILINX_DMA_PARK_PTR_WR_REF_MASK		GENMASK(12, 8)
 110#define XILINX_DMA_PARK_PTR_RD_REF_SHIFT	0
 111#define XILINX_DMA_PARK_PTR_RD_REF_MASK		GENMASK(4, 0)
 112#define XILINX_DMA_REG_VDMA_VERSION		0x002c
 113
 114/* Register Direct Mode Registers */
 115#define XILINX_DMA_REG_VSIZE			0x0000
 116#define XILINX_DMA_VSIZE_MASK			GENMASK(12, 0)
 117#define XILINX_DMA_REG_HSIZE			0x0004
 118#define XILINX_DMA_HSIZE_MASK			GENMASK(15, 0)
 119
 120#define XILINX_DMA_REG_FRMDLY_STRIDE		0x0008
 121#define XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT	24
 122#define XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT	0
 123
 124#define XILINX_VDMA_REG_START_ADDRESS(n)	(0x000c + 4 * (n))
 125#define XILINX_VDMA_REG_START_ADDRESS_64(n)	(0x000c + 8 * (n))
 126
 127#define XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP	0x00ec
 128#define XILINX_VDMA_ENABLE_VERTICAL_FLIP	BIT(0)
 129
 130/* HW specific definitions */
 131#define XILINX_MCDMA_MAX_CHANS_PER_DEVICE	0x20
 132#define XILINX_DMA_MAX_CHANS_PER_DEVICE		0x2
 133#define XILINX_CDMA_MAX_CHANS_PER_DEVICE	0x1
 134#define XILINX_DMA_DFAULT_ADDRWIDTH		0x20
 135
 136#define XILINX_DMA_DMAXR_ALL_IRQ_MASK	\
 137		(XILINX_DMA_DMASR_FRM_CNT_IRQ | \
 138		 XILINX_DMA_DMASR_DLY_CNT_IRQ | \
 139		 XILINX_DMA_DMASR_ERR_IRQ)
 140
 141#define XILINX_DMA_DMASR_ALL_ERR_MASK	\
 142		(XILINX_DMA_DMASR_EOL_LATE_ERR | \
 143		 XILINX_DMA_DMASR_SOF_LATE_ERR | \
 144		 XILINX_DMA_DMASR_SG_DEC_ERR | \
 145		 XILINX_DMA_DMASR_SG_SLV_ERR | \
 146		 XILINX_DMA_DMASR_EOF_EARLY_ERR | \
 147		 XILINX_DMA_DMASR_SOF_EARLY_ERR | \
 148		 XILINX_DMA_DMASR_DMA_DEC_ERR | \
 149		 XILINX_DMA_DMASR_DMA_SLAVE_ERR | \
 150		 XILINX_DMA_DMASR_DMA_INT_ERR)
 151
 152/*
 153 * Recoverable errors are DMA Internal error, SOF Early, EOF Early
 154 * and SOF Late. They are only recoverable when C_FLUSH_ON_FSYNC
 155 * is enabled in the h/w system.
 156 */
 157#define XILINX_DMA_DMASR_ERR_RECOVER_MASK	\
 158		(XILINX_DMA_DMASR_SOF_LATE_ERR | \
 159		 XILINX_DMA_DMASR_EOF_EARLY_ERR | \
 160		 XILINX_DMA_DMASR_SOF_EARLY_ERR | \
 161		 XILINX_DMA_DMASR_DMA_INT_ERR)
 162
 163/* Axi VDMA Flush on Fsync bits */
 164#define XILINX_DMA_FLUSH_S2MM		3
 165#define XILINX_DMA_FLUSH_MM2S		2
 166#define XILINX_DMA_FLUSH_BOTH		1
 167
 168/* Delay loop counter to prevent hardware failure */
 169#define XILINX_DMA_LOOP_COUNT		1000000
 170
 171/* AXI DMA Specific Registers/Offsets */
 172#define XILINX_DMA_REG_SRCDSTADDR	0x18
 173#define XILINX_DMA_REG_BTT		0x28
 174
 175/* AXI DMA Specific Masks/Bit fields */
 176#define XILINX_DMA_MAX_TRANS_LEN_MIN	8
 177#define XILINX_DMA_MAX_TRANS_LEN_MAX	23
 178#define XILINX_DMA_V2_MAX_TRANS_LEN_MAX	26
 179#define XILINX_DMA_CR_COALESCE_MAX	GENMASK(23, 16)
 180#define XILINX_DMA_CR_DELAY_MAX		GENMASK(31, 24)
 181#define XILINX_DMA_CR_CYCLIC_BD_EN_MASK	BIT(4)
 182#define XILINX_DMA_CR_COALESCE_SHIFT	16
 183#define XILINX_DMA_CR_DELAY_SHIFT	24
 184#define XILINX_DMA_BD_SOP		BIT(27)
 185#define XILINX_DMA_BD_EOP		BIT(26)
 186#define XILINX_DMA_BD_COMP_MASK		BIT(31)
 187#define XILINX_DMA_COALESCE_MAX		255
 188#define XILINX_DMA_NUM_DESCS		512
 189#define XILINX_DMA_NUM_APP_WORDS	5
 190
 191/* AXI CDMA Specific Registers/Offsets */
 192#define XILINX_CDMA_REG_SRCADDR		0x18
 193#define XILINX_CDMA_REG_DSTADDR		0x20
 194
 195/* AXI CDMA Specific Masks */
 196#define XILINX_CDMA_CR_SGMODE          BIT(3)
 197
 198#define xilinx_prep_dma_addr_t(addr)	\
 199	((dma_addr_t)((u64)addr##_##msb << 32 | (addr)))
 200
 201/* AXI MCDMA Specific Registers/Offsets */
 202#define XILINX_MCDMA_MM2S_CTRL_OFFSET		0x0000
 203#define XILINX_MCDMA_S2MM_CTRL_OFFSET		0x0500
 204#define XILINX_MCDMA_CHEN_OFFSET		0x0008
 205#define XILINX_MCDMA_CH_ERR_OFFSET		0x0010
 206#define XILINX_MCDMA_RXINT_SER_OFFSET		0x0020
 207#define XILINX_MCDMA_TXINT_SER_OFFSET		0x0028
 208#define XILINX_MCDMA_CHAN_CR_OFFSET(x)		(0x40 + (x) * 0x40)
 209#define XILINX_MCDMA_CHAN_SR_OFFSET(x)		(0x44 + (x) * 0x40)
 210#define XILINX_MCDMA_CHAN_CDESC_OFFSET(x)	(0x48 + (x) * 0x40)
 211#define XILINX_MCDMA_CHAN_TDESC_OFFSET(x)	(0x50 + (x) * 0x40)
 212
 213/* AXI MCDMA Specific Masks/Shifts */
 214#define XILINX_MCDMA_COALESCE_SHIFT		16
 215#define XILINX_MCDMA_COALESCE_MAX		24
 216#define XILINX_MCDMA_IRQ_ALL_MASK		GENMASK(7, 5)
 217#define XILINX_MCDMA_COALESCE_MASK		GENMASK(23, 16)
 218#define XILINX_MCDMA_CR_RUNSTOP_MASK		BIT(0)
 219#define XILINX_MCDMA_IRQ_IOC_MASK		BIT(5)
 220#define XILINX_MCDMA_IRQ_DELAY_MASK		BIT(6)
 221#define XILINX_MCDMA_IRQ_ERR_MASK		BIT(7)
 222#define XILINX_MCDMA_BD_EOP			BIT(30)
 223#define XILINX_MCDMA_BD_SOP			BIT(31)
 224
 225/**
 226 * struct xilinx_vdma_desc_hw - Hardware Descriptor
 227 * @next_desc: Next Descriptor Pointer @0x00
 228 * @pad1: Reserved @0x04
 229 * @buf_addr: Buffer address @0x08
 230 * @buf_addr_msb: MSB of Buffer address @0x0C
 231 * @vsize: Vertical Size @0x10
 232 * @hsize: Horizontal Size @0x14
 233 * @stride: Number of bytes between the first
 234 *	    pixels of each horizontal line @0x18
 235 */
 236struct xilinx_vdma_desc_hw {
 237	u32 next_desc;
 238	u32 pad1;
 239	u32 buf_addr;
 240	u32 buf_addr_msb;
 241	u32 vsize;
 242	u32 hsize;
 243	u32 stride;
 244} __aligned(64);
 245
 246/**
 247 * struct xilinx_axidma_desc_hw - Hardware Descriptor for AXI DMA
 248 * @next_desc: Next Descriptor Pointer @0x00
 249 * @next_desc_msb: MSB of Next Descriptor Pointer @0x04
 250 * @buf_addr: Buffer address @0x08
 251 * @buf_addr_msb: MSB of Buffer address @0x0C
 252 * @reserved1: Reserved @0x10
 253 * @reserved2: Reserved @0x14
 254 * @control: Control field @0x18
 255 * @status: Status field @0x1C
 256 * @app: APP Fields @0x20 - 0x30
 257 */
 258struct xilinx_axidma_desc_hw {
 259	u32 next_desc;
 260	u32 next_desc_msb;
 261	u32 buf_addr;
 262	u32 buf_addr_msb;
 263	u32 reserved1;
 264	u32 reserved2;
 265	u32 control;
 266	u32 status;
 267	u32 app[XILINX_DMA_NUM_APP_WORDS];
 268} __aligned(64);
 269
 270/**
 271 * struct xilinx_aximcdma_desc_hw - Hardware Descriptor for AXI MCDMA
 272 * @next_desc: Next Descriptor Pointer @0x00
 273 * @next_desc_msb: MSB of Next Descriptor Pointer @0x04
 274 * @buf_addr: Buffer address @0x08
 275 * @buf_addr_msb: MSB of Buffer address @0x0C
 276 * @rsvd: Reserved field @0x10
 277 * @control: Control Information field @0x14
 278 * @status: Status field @0x18
 279 * @sideband_status: Status of sideband signals @0x1C
 280 * @app: APP Fields @0x20 - 0x30
 281 */
 282struct xilinx_aximcdma_desc_hw {
 283	u32 next_desc;
 284	u32 next_desc_msb;
 285	u32 buf_addr;
 286	u32 buf_addr_msb;
 287	u32 rsvd;
 288	u32 control;
 289	u32 status;
 290	u32 sideband_status;
 291	u32 app[XILINX_DMA_NUM_APP_WORDS];
 292} __aligned(64);
 293
 294/**
 295 * struct xilinx_cdma_desc_hw - Hardware Descriptor
 296 * @next_desc: Next Descriptor Pointer @0x00
 297 * @next_desc_msb: Next Descriptor Pointer MSB @0x04
 298 * @src_addr: Source address @0x08
 299 * @src_addr_msb: Source address MSB @0x0C
 300 * @dest_addr: Destination address @0x10
 301 * @dest_addr_msb: Destination address MSB @0x14
 302 * @control: Control field @0x18
 303 * @status: Status field @0x1C
 304 */
 305struct xilinx_cdma_desc_hw {
 306	u32 next_desc;
 307	u32 next_desc_msb;
 308	u32 src_addr;
 309	u32 src_addr_msb;
 310	u32 dest_addr;
 311	u32 dest_addr_msb;
 312	u32 control;
 313	u32 status;
 314} __aligned(64);
 315
 316/**
 317 * struct xilinx_vdma_tx_segment - Descriptor segment
 318 * @hw: Hardware descriptor
 319 * @node: Node in the descriptor segments list
 320 * @phys: Physical address of segment
 321 */
 322struct xilinx_vdma_tx_segment {
 323	struct xilinx_vdma_desc_hw hw;
 324	struct list_head node;
 325	dma_addr_t phys;
 326} __aligned(64);
 327
 328/**
 329 * struct xilinx_axidma_tx_segment - Descriptor segment
 330 * @hw: Hardware descriptor
 331 * @node: Node in the descriptor segments list
 332 * @phys: Physical address of segment
 333 */
 334struct xilinx_axidma_tx_segment {
 335	struct xilinx_axidma_desc_hw hw;
 336	struct list_head node;
 337	dma_addr_t phys;
 338} __aligned(64);
 339
 340/**
 341 * struct xilinx_aximcdma_tx_segment - Descriptor segment
 342 * @hw: Hardware descriptor
 343 * @node: Node in the descriptor segments list
 344 * @phys: Physical address of segment
 345 */
 346struct xilinx_aximcdma_tx_segment {
 347	struct xilinx_aximcdma_desc_hw hw;
 348	struct list_head node;
 349	dma_addr_t phys;
 350} __aligned(64);
 351
 352/**
 353 * struct xilinx_cdma_tx_segment - Descriptor segment
 354 * @hw: Hardware descriptor
 355 * @node: Node in the descriptor segments list
 356 * @phys: Physical address of segment
 357 */
 358struct xilinx_cdma_tx_segment {
 359	struct xilinx_cdma_desc_hw hw;
 360	struct list_head node;
 361	dma_addr_t phys;
 362} __aligned(64);
 363
 364/**
 365 * struct xilinx_dma_tx_descriptor - Per Transaction structure
 366 * @async_tx: Async transaction descriptor
 367 * @segments: TX segments list
 368 * @node: Node in the channel descriptors list
 369 * @cyclic: Check for cyclic transfers.
 370 * @err: Whether the descriptor has an error.
 371 * @residue: Residue of the completed descriptor
 372 */
 373struct xilinx_dma_tx_descriptor {
 374	struct dma_async_tx_descriptor async_tx;
 375	struct list_head segments;
 376	struct list_head node;
 377	bool cyclic;
 378	bool err;
 379	u32 residue;
 380};
 381
 382/**
 383 * struct xilinx_dma_chan - Driver specific DMA channel structure
 384 * @xdev: Driver specific device structure
 385 * @ctrl_offset: Control registers offset
 386 * @desc_offset: TX descriptor registers offset
 387 * @lock: Descriptor operation lock
 388 * @pending_list: Descriptors waiting
 389 * @active_list: Descriptors ready to submit
 390 * @done_list: Complete descriptors
 391 * @free_seg_list: Free descriptors
 392 * @common: DMA common channel
 393 * @desc_pool: Descriptors pool
 394 * @dev: The dma device
 395 * @irq: Channel IRQ
 396 * @id: Channel ID
 397 * @direction: Transfer direction
 398 * @num_frms: Number of frames
 399 * @has_sg: Support scatter transfers
 400 * @cyclic: Check for cyclic transfers.
 401 * @genlock: Support genlock mode
 402 * @err: Channel has errors
 403 * @idle: Check for channel idle
 404 * @terminating: Check for channel being synchronized by user
 405 * @tasklet: Cleanup work after irq
 406 * @config: Device configuration info
 407 * @flush_on_fsync: Flush on Frame sync
 408 * @desc_pendingcount: Descriptor pending count
 409 * @ext_addr: Indicates 64 bit addressing is supported by dma channel
 410 * @desc_submitcount: Descriptor h/w submitted count
 411 * @seg_v: Statically allocated segments base
 412 * @seg_mv: Statically allocated segments base for MCDMA
 413 * @seg_p: Physical allocated segments base
 414 * @cyclic_seg_v: Statically allocated segment base for cyclic transfers
 415 * @cyclic_seg_p: Physical allocated segments base for cyclic dma
 416 * @start_transfer: Differentiate b/w DMA IP's transfer
 417 * @stop_transfer: Differentiate b/w DMA IP's quiesce
 418 * @tdest: TDEST value for mcdma
 419 * @has_vflip: S2MM vertical flip
 420 * @irq_delay: Interrupt delay timeout
 421 */
 422struct xilinx_dma_chan {
 423	struct xilinx_dma_device *xdev;
 424	u32 ctrl_offset;
 425	u32 desc_offset;
 426	spinlock_t lock;
 427	struct list_head pending_list;
 428	struct list_head active_list;
 429	struct list_head done_list;
 430	struct list_head free_seg_list;
 431	struct dma_chan common;
 432	struct dma_pool *desc_pool;
 433	struct device *dev;
 434	int irq;
 435	int id;
 436	enum dma_transfer_direction direction;
 437	int num_frms;
 438	bool has_sg;
 439	bool cyclic;
 440	bool genlock;
 441	bool err;
 442	bool idle;
 443	bool terminating;
 444	struct tasklet_struct tasklet;
 445	struct xilinx_vdma_config config;
 446	bool flush_on_fsync;
 447	u32 desc_pendingcount;
 448	bool ext_addr;
 449	u32 desc_submitcount;
 450	struct xilinx_axidma_tx_segment *seg_v;
 451	struct xilinx_aximcdma_tx_segment *seg_mv;
 452	dma_addr_t seg_p;
 453	struct xilinx_axidma_tx_segment *cyclic_seg_v;
 454	dma_addr_t cyclic_seg_p;
 455	void (*start_transfer)(struct xilinx_dma_chan *chan);
 456	int (*stop_transfer)(struct xilinx_dma_chan *chan);
 457	u16 tdest;
 458	bool has_vflip;
 459	u8 irq_delay;
 460};
 461
 462/**
 463 * enum xdma_ip_type - DMA IP type.
 464 *
 465 * @XDMA_TYPE_AXIDMA: Axi dma ip.
 466 * @XDMA_TYPE_CDMA: Axi cdma ip.
 467 * @XDMA_TYPE_VDMA: Axi vdma ip.
 468 * @XDMA_TYPE_AXIMCDMA: Axi MCDMA ip.
 469 *
 470 */
 471enum xdma_ip_type {
 472	XDMA_TYPE_AXIDMA = 0,
 473	XDMA_TYPE_CDMA,
 474	XDMA_TYPE_VDMA,
 475	XDMA_TYPE_AXIMCDMA
 476};
 477
 478struct xilinx_dma_config {
 479	enum xdma_ip_type dmatype;
 480	int (*clk_init)(struct platform_device *pdev, struct clk **axi_clk,
 481			struct clk **tx_clk, struct clk **txs_clk,
 482			struct clk **rx_clk, struct clk **rxs_clk);
 483	irqreturn_t (*irq_handler)(int irq, void *data);
 484	const int max_channels;
 485};
 486
 487/**
 488 * struct xilinx_dma_device - DMA device structure
 489 * @regs: I/O mapped base address
 490 * @dev: Device Structure
 491 * @common: DMA device structure
 492 * @chan: Driver specific DMA channel
 493 * @flush_on_fsync: Flush on frame sync
 494 * @ext_addr: Indicates 64 bit addressing is supported by dma device
 495 * @pdev: Platform device structure pointer
 496 * @dma_config: DMA config structure
 497 * @axi_clk: DMA Axi4-lite interace clock
 498 * @tx_clk: DMA mm2s clock
 499 * @txs_clk: DMA mm2s stream clock
 500 * @rx_clk: DMA s2mm clock
 501 * @rxs_clk: DMA s2mm stream clock
 502 * @s2mm_chan_id: DMA s2mm channel identifier
 503 * @mm2s_chan_id: DMA mm2s channel identifier
 504 * @max_buffer_len: Max buffer length
 505 * @has_axistream_connected: AXI DMA connected to AXI Stream IP
 506 */
 507struct xilinx_dma_device {
 508	void __iomem *regs;
 509	struct device *dev;
 510	struct dma_device common;
 511	struct xilinx_dma_chan *chan[XILINX_MCDMA_MAX_CHANS_PER_DEVICE];
 512	u32 flush_on_fsync;
 513	bool ext_addr;
 514	struct platform_device  *pdev;
 515	const struct xilinx_dma_config *dma_config;
 516	struct clk *axi_clk;
 517	struct clk *tx_clk;
 518	struct clk *txs_clk;
 519	struct clk *rx_clk;
 520	struct clk *rxs_clk;
 521	u32 s2mm_chan_id;
 522	u32 mm2s_chan_id;
 523	u32 max_buffer_len;
 524	bool has_axistream_connected;
 525};
 526
 527/* Macros */
 528#define to_xilinx_chan(chan) \
 529	container_of(chan, struct xilinx_dma_chan, common)
 530#define to_dma_tx_descriptor(tx) \
 531	container_of(tx, struct xilinx_dma_tx_descriptor, async_tx)
 532#define xilinx_dma_poll_timeout(chan, reg, val, cond, delay_us, timeout_us) \
 533	readl_poll_timeout_atomic(chan->xdev->regs + chan->ctrl_offset + reg, \
 534				  val, cond, delay_us, timeout_us)
 535
 536/* IO accessors */
 537static inline u32 dma_read(struct xilinx_dma_chan *chan, u32 reg)
 538{
 539	return ioread32(chan->xdev->regs + reg);
 540}
 541
 542static inline void dma_write(struct xilinx_dma_chan *chan, u32 reg, u32 value)
 543{
 544	iowrite32(value, chan->xdev->regs + reg);
 545}
 546
 547static inline void vdma_desc_write(struct xilinx_dma_chan *chan, u32 reg,
 548				   u32 value)
 549{
 550	dma_write(chan, chan->desc_offset + reg, value);
 551}
 552
 553static inline u32 dma_ctrl_read(struct xilinx_dma_chan *chan, u32 reg)
 554{
 555	return dma_read(chan, chan->ctrl_offset + reg);
 556}
 557
 558static inline void dma_ctrl_write(struct xilinx_dma_chan *chan, u32 reg,
 559				   u32 value)
 560{
 561	dma_write(chan, chan->ctrl_offset + reg, value);
 562}
 563
 564static inline void dma_ctrl_clr(struct xilinx_dma_chan *chan, u32 reg,
 565				 u32 clr)
 566{
 567	dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) & ~clr);
 568}
 569
 570static inline void dma_ctrl_set(struct xilinx_dma_chan *chan, u32 reg,
 571				 u32 set)
 572{
 573	dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) | set);
 574}
 575
 576/**
 577 * vdma_desc_write_64 - 64-bit descriptor write
 578 * @chan: Driver specific VDMA channel
 579 * @reg: Register to write
 580 * @value_lsb: lower address of the descriptor.
 581 * @value_msb: upper address of the descriptor.
 582 *
 583 * Since vdma driver is trying to write to a register offset which is not a
 584 * multiple of 64 bits(ex : 0x5c), we are writing as two separate 32 bits
 585 * instead of a single 64 bit register write.
 586 */
 587static inline void vdma_desc_write_64(struct xilinx_dma_chan *chan, u32 reg,
 588				      u32 value_lsb, u32 value_msb)
 589{
 590	/* Write the lsb 32 bits*/
 591	writel(value_lsb, chan->xdev->regs + chan->desc_offset + reg);
 592
 593	/* Write the msb 32 bits */
 594	writel(value_msb, chan->xdev->regs + chan->desc_offset + reg + 4);
 595}
 596
 597static inline void dma_writeq(struct xilinx_dma_chan *chan, u32 reg, u64 value)
 598{
 599	lo_hi_writeq(value, chan->xdev->regs + chan->ctrl_offset + reg);
 600}
 601
 602static inline void xilinx_write(struct xilinx_dma_chan *chan, u32 reg,
 603				dma_addr_t addr)
 604{
 605	if (chan->ext_addr)
 606		dma_writeq(chan, reg, addr);
 607	else
 608		dma_ctrl_write(chan, reg, addr);
 609}
 610
 611static inline void xilinx_axidma_buf(struct xilinx_dma_chan *chan,
 612				     struct xilinx_axidma_desc_hw *hw,
 613				     dma_addr_t buf_addr, size_t sg_used,
 614				     size_t period_len)
 615{
 616	if (chan->ext_addr) {
 617		hw->buf_addr = lower_32_bits(buf_addr + sg_used + period_len);
 618		hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used +
 619						 period_len);
 620	} else {
 621		hw->buf_addr = buf_addr + sg_used + period_len;
 622	}
 623}
 624
 625static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan,
 626				       struct xilinx_aximcdma_desc_hw *hw,
 627				       dma_addr_t buf_addr, size_t sg_used)
 628{
 629	if (chan->ext_addr) {
 630		hw->buf_addr = lower_32_bits(buf_addr + sg_used);
 631		hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used);
 632	} else {
 633		hw->buf_addr = buf_addr + sg_used;
 634	}
 635}
 636
 637/**
 638 * xilinx_dma_get_metadata_ptr- Populate metadata pointer and payload length
 639 * @tx: async transaction descriptor
 640 * @payload_len: metadata payload length
 641 * @max_len: metadata max length
 642 * Return: The app field pointer.
 643 */
 644static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx,
 645					 size_t *payload_len, size_t *max_len)
 646{
 647	struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx);
 648	struct xilinx_axidma_tx_segment *seg;
 649
 650	*max_len = *payload_len = sizeof(u32) * XILINX_DMA_NUM_APP_WORDS;
 651	seg = list_first_entry(&desc->segments,
 652			       struct xilinx_axidma_tx_segment, node);
 653	return seg->hw.app;
 654}
 655
 656static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = {
 657	.get_ptr = xilinx_dma_get_metadata_ptr,
 658};
 659
 660/* -----------------------------------------------------------------------------
 661 * Descriptors and segments alloc and free
 662 */
 663
 664/**
 665 * xilinx_vdma_alloc_tx_segment - Allocate transaction segment
 666 * @chan: Driver specific DMA channel
 667 *
 668 * Return: The allocated segment on success and NULL on failure.
 669 */
 670static struct xilinx_vdma_tx_segment *
 671xilinx_vdma_alloc_tx_segment(struct xilinx_dma_chan *chan)
 672{
 673	struct xilinx_vdma_tx_segment *segment;
 674	dma_addr_t phys;
 675
 676	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);
 677	if (!segment)
 678		return NULL;
 679
 680	segment->phys = phys;
 681
 682	return segment;
 683}
 684
 685/**
 686 * xilinx_cdma_alloc_tx_segment - Allocate transaction segment
 687 * @chan: Driver specific DMA channel
 688 *
 689 * Return: The allocated segment on success and NULL on failure.
 690 */
 691static struct xilinx_cdma_tx_segment *
 692xilinx_cdma_alloc_tx_segment(struct xilinx_dma_chan *chan)
 693{
 694	struct xilinx_cdma_tx_segment *segment;
 695	dma_addr_t phys;
 696
 697	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);
 698	if (!segment)
 699		return NULL;
 700
 701	segment->phys = phys;
 702
 703	return segment;
 704}
 705
 706/**
 707 * xilinx_axidma_alloc_tx_segment - Allocate transaction segment
 708 * @chan: Driver specific DMA channel
 709 *
 710 * Return: The allocated segment on success and NULL on failure.
 711 */
 712static struct xilinx_axidma_tx_segment *
 713xilinx_axidma_alloc_tx_segment(struct xilinx_dma_chan *chan)
 714{
 715	struct xilinx_axidma_tx_segment *segment = NULL;
 716	unsigned long flags;
 717
 718	spin_lock_irqsave(&chan->lock, flags);
 719	if (!list_empty(&chan->free_seg_list)) {
 720		segment = list_first_entry(&chan->free_seg_list,
 721					   struct xilinx_axidma_tx_segment,
 722					   node);
 723		list_del(&segment->node);
 724	}
 725	spin_unlock_irqrestore(&chan->lock, flags);
 726
 727	if (!segment)
 728		dev_dbg(chan->dev, "Could not find free tx segment\n");
 729
 730	return segment;
 731}
 732
 733/**
 734 * xilinx_aximcdma_alloc_tx_segment - Allocate transaction segment
 735 * @chan: Driver specific DMA channel
 736 *
 737 * Return: The allocated segment on success and NULL on failure.
 738 */
 739static struct xilinx_aximcdma_tx_segment *
 740xilinx_aximcdma_alloc_tx_segment(struct xilinx_dma_chan *chan)
 741{
 742	struct xilinx_aximcdma_tx_segment *segment = NULL;
 743	unsigned long flags;
 744
 745	spin_lock_irqsave(&chan->lock, flags);
 746	if (!list_empty(&chan->free_seg_list)) {
 747		segment = list_first_entry(&chan->free_seg_list,
 748					   struct xilinx_aximcdma_tx_segment,
 749					   node);
 750		list_del(&segment->node);
 751	}
 752	spin_unlock_irqrestore(&chan->lock, flags);
 753
 754	return segment;
 755}
 756
 757static void xilinx_dma_clean_hw_desc(struct xilinx_axidma_desc_hw *hw)
 758{
 759	u32 next_desc = hw->next_desc;
 760	u32 next_desc_msb = hw->next_desc_msb;
 761
 762	memset(hw, 0, sizeof(struct xilinx_axidma_desc_hw));
 763
 764	hw->next_desc = next_desc;
 765	hw->next_desc_msb = next_desc_msb;
 766}
 767
 768static void xilinx_mcdma_clean_hw_desc(struct xilinx_aximcdma_desc_hw *hw)
 769{
 770	u32 next_desc = hw->next_desc;
 771	u32 next_desc_msb = hw->next_desc_msb;
 772
 773	memset(hw, 0, sizeof(struct xilinx_aximcdma_desc_hw));
 774
 775	hw->next_desc = next_desc;
 776	hw->next_desc_msb = next_desc_msb;
 777}
 778
 779/**
 780 * xilinx_dma_free_tx_segment - Free transaction segment
 781 * @chan: Driver specific DMA channel
 782 * @segment: DMA transaction segment
 783 */
 784static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan,
 785				struct xilinx_axidma_tx_segment *segment)
 786{
 787	xilinx_dma_clean_hw_desc(&segment->hw);
 788
 789	list_add_tail(&segment->node, &chan->free_seg_list);
 790}
 791
 792/**
 793 * xilinx_mcdma_free_tx_segment - Free transaction segment
 794 * @chan: Driver specific DMA channel
 795 * @segment: DMA transaction segment
 796 */
 797static void xilinx_mcdma_free_tx_segment(struct xilinx_dma_chan *chan,
 798					 struct xilinx_aximcdma_tx_segment *
 799					 segment)
 800{
 801	xilinx_mcdma_clean_hw_desc(&segment->hw);
 802
 803	list_add_tail(&segment->node, &chan->free_seg_list);
 804}
 805
 806/**
 807 * xilinx_cdma_free_tx_segment - Free transaction segment
 808 * @chan: Driver specific DMA channel
 809 * @segment: DMA transaction segment
 810 */
 811static void xilinx_cdma_free_tx_segment(struct xilinx_dma_chan *chan,
 812				struct xilinx_cdma_tx_segment *segment)
 813{
 814	dma_pool_free(chan->desc_pool, segment, segment->phys);
 815}
 816
 817/**
 818 * xilinx_vdma_free_tx_segment - Free transaction segment
 819 * @chan: Driver specific DMA channel
 820 * @segment: DMA transaction segment
 821 */
 822static void xilinx_vdma_free_tx_segment(struct xilinx_dma_chan *chan,
 823					struct xilinx_vdma_tx_segment *segment)
 824{
 825	dma_pool_free(chan->desc_pool, segment, segment->phys);
 826}
 827
 828/**
 829 * xilinx_dma_alloc_tx_descriptor - Allocate transaction descriptor
 830 * @chan: Driver specific DMA channel
 831 *
 832 * Return: The allocated descriptor on success and NULL on failure.
 833 */
 834static struct xilinx_dma_tx_descriptor *
 835xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan)
 836{
 837	struct xilinx_dma_tx_descriptor *desc;
 838
 839	desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
 840	if (!desc)
 841		return NULL;
 842
 843	INIT_LIST_HEAD(&desc->segments);
 844
 845	return desc;
 846}
 847
 848/**
 849 * xilinx_dma_free_tx_descriptor - Free transaction descriptor
 850 * @chan: Driver specific DMA channel
 851 * @desc: DMA transaction descriptor
 852 */
 853static void
 854xilinx_dma_free_tx_descriptor(struct xilinx_dma_chan *chan,
 855			       struct xilinx_dma_tx_descriptor *desc)
 856{
 857	struct xilinx_vdma_tx_segment *segment, *next;
 858	struct xilinx_cdma_tx_segment *cdma_segment, *cdma_next;
 859	struct xilinx_axidma_tx_segment *axidma_segment, *axidma_next;
 860	struct xilinx_aximcdma_tx_segment *aximcdma_segment, *aximcdma_next;
 861
 862	if (!desc)
 863		return;
 864
 865	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
 866		list_for_each_entry_safe(segment, next, &desc->segments, node) {
 867			list_del(&segment->node);
 868			xilinx_vdma_free_tx_segment(chan, segment);
 869		}
 870	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
 871		list_for_each_entry_safe(cdma_segment, cdma_next,
 872					 &desc->segments, node) {
 873			list_del(&cdma_segment->node);
 874			xilinx_cdma_free_tx_segment(chan, cdma_segment);
 875		}
 876	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
 877		list_for_each_entry_safe(axidma_segment, axidma_next,
 878					 &desc->segments, node) {
 879			list_del(&axidma_segment->node);
 880			xilinx_dma_free_tx_segment(chan, axidma_segment);
 881		}
 882	} else {
 883		list_for_each_entry_safe(aximcdma_segment, aximcdma_next,
 884					 &desc->segments, node) {
 885			list_del(&aximcdma_segment->node);
 886			xilinx_mcdma_free_tx_segment(chan, aximcdma_segment);
 887		}
 888	}
 889
 890	kfree(desc);
 891}
 892
 893/* Required functions */
 894
 895/**
 896 * xilinx_dma_free_desc_list - Free descriptors list
 897 * @chan: Driver specific DMA channel
 898 * @list: List to parse and delete the descriptor
 899 */
 900static void xilinx_dma_free_desc_list(struct xilinx_dma_chan *chan,
 901					struct list_head *list)
 902{
 903	struct xilinx_dma_tx_descriptor *desc, *next;
 904
 905	list_for_each_entry_safe(desc, next, list, node) {
 906		list_del(&desc->node);
 907		xilinx_dma_free_tx_descriptor(chan, desc);
 908	}
 909}
 910
 911/**
 912 * xilinx_dma_free_descriptors - Free channel descriptors
 913 * @chan: Driver specific DMA channel
 914 */
 915static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan)
 916{
 917	unsigned long flags;
 918
 919	spin_lock_irqsave(&chan->lock, flags);
 920
 921	xilinx_dma_free_desc_list(chan, &chan->pending_list);
 922	xilinx_dma_free_desc_list(chan, &chan->done_list);
 923	xilinx_dma_free_desc_list(chan, &chan->active_list);
 924
 925	spin_unlock_irqrestore(&chan->lock, flags);
 926}
 927
 928/**
 929 * xilinx_dma_free_chan_resources - Free channel resources
 930 * @dchan: DMA channel
 931 */
 932static void xilinx_dma_free_chan_resources(struct dma_chan *dchan)
 933{
 934	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
 935	unsigned long flags;
 936
 937	dev_dbg(chan->dev, "Free all channel resources.\n");
 938
 939	xilinx_dma_free_descriptors(chan);
 940
 941	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
 942		spin_lock_irqsave(&chan->lock, flags);
 943		INIT_LIST_HEAD(&chan->free_seg_list);
 944		spin_unlock_irqrestore(&chan->lock, flags);
 945
 946		/* Free memory that is allocated for BD */
 947		dma_free_coherent(chan->dev, sizeof(*chan->seg_v) *
 948				  XILINX_DMA_NUM_DESCS, chan->seg_v,
 949				  chan->seg_p);
 950
 951		/* Free Memory that is allocated for cyclic DMA Mode */
 952		dma_free_coherent(chan->dev, sizeof(*chan->cyclic_seg_v),
 953				  chan->cyclic_seg_v, chan->cyclic_seg_p);
 954	}
 955
 956	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
 957		spin_lock_irqsave(&chan->lock, flags);
 958		INIT_LIST_HEAD(&chan->free_seg_list);
 959		spin_unlock_irqrestore(&chan->lock, flags);
 960
 961		/* Free memory that is allocated for BD */
 962		dma_free_coherent(chan->dev, sizeof(*chan->seg_mv) *
 963				  XILINX_DMA_NUM_DESCS, chan->seg_mv,
 964				  chan->seg_p);
 965	}
 966
 967	if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA &&
 968	    chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA) {
 969		dma_pool_destroy(chan->desc_pool);
 970		chan->desc_pool = NULL;
 971	}
 972
 973}
 974
 975/**
 976 * xilinx_dma_get_residue - Compute residue for a given descriptor
 977 * @chan: Driver specific dma channel
 978 * @desc: dma transaction descriptor
 979 *
 980 * Return: The number of residue bytes for the descriptor.
 981 */
 982static u32 xilinx_dma_get_residue(struct xilinx_dma_chan *chan,
 983				  struct xilinx_dma_tx_descriptor *desc)
 984{
 985	struct xilinx_cdma_tx_segment *cdma_seg;
 986	struct xilinx_axidma_tx_segment *axidma_seg;
 987	struct xilinx_aximcdma_tx_segment *aximcdma_seg;
 988	struct xilinx_cdma_desc_hw *cdma_hw;
 989	struct xilinx_axidma_desc_hw *axidma_hw;
 990	struct xilinx_aximcdma_desc_hw *aximcdma_hw;
 991	struct list_head *entry;
 992	u32 residue = 0;
 993
 994	list_for_each(entry, &desc->segments) {
 995		if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
 996			cdma_seg = list_entry(entry,
 997					      struct xilinx_cdma_tx_segment,
 998					      node);
 999			cdma_hw = &cdma_seg->hw;
1000			residue += (cdma_hw->control & chan->xdev->max_buffer_len) -
1001			           (cdma_hw->status & chan->xdev->max_buffer_len);
1002		} else if (chan->xdev->dma_config->dmatype ==
1003			   XDMA_TYPE_AXIDMA) {
1004			axidma_seg = list_entry(entry,
1005						struct xilinx_axidma_tx_segment,
1006						node);
1007			axidma_hw = &axidma_seg->hw;
1008			residue += (axidma_hw->control & chan->xdev->max_buffer_len) -
1009			           (axidma_hw->status & chan->xdev->max_buffer_len);
1010		} else {
1011			aximcdma_seg =
1012				list_entry(entry,
1013					   struct xilinx_aximcdma_tx_segment,
1014					   node);
1015			aximcdma_hw = &aximcdma_seg->hw;
1016			residue +=
1017				(aximcdma_hw->control & chan->xdev->max_buffer_len) -
1018				(aximcdma_hw->status & chan->xdev->max_buffer_len);
1019		}
1020	}
1021
1022	return residue;
1023}
1024
1025/**
1026 * xilinx_dma_chan_handle_cyclic - Cyclic dma callback
1027 * @chan: Driver specific dma channel
1028 * @desc: dma transaction descriptor
1029 * @flags: flags for spin lock
1030 */
1031static void xilinx_dma_chan_handle_cyclic(struct xilinx_dma_chan *chan,
1032					  struct xilinx_dma_tx_descriptor *desc,
1033					  unsigned long *flags)
1034{
1035	struct dmaengine_desc_callback cb;
1036
1037	dmaengine_desc_get_callback(&desc->async_tx, &cb);
1038	if (dmaengine_desc_callback_valid(&cb)) {
1039		spin_unlock_irqrestore(&chan->lock, *flags);
1040		dmaengine_desc_callback_invoke(&cb, NULL);
1041		spin_lock_irqsave(&chan->lock, *flags);
1042	}
1043}
1044
1045/**
1046 * xilinx_dma_chan_desc_cleanup - Clean channel descriptors
1047 * @chan: Driver specific DMA channel
1048 */
1049static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan)
1050{
1051	struct xilinx_dma_tx_descriptor *desc, *next;
1052	unsigned long flags;
1053
1054	spin_lock_irqsave(&chan->lock, flags);
1055
1056	list_for_each_entry_safe(desc, next, &chan->done_list, node) {
1057		struct dmaengine_result result;
1058
1059		if (desc->cyclic) {
1060			xilinx_dma_chan_handle_cyclic(chan, desc, &flags);
1061			break;
1062		}
1063
1064		/* Remove from the list of running transactions */
1065		list_del(&desc->node);
1066
1067		if (unlikely(desc->err)) {
1068			if (chan->direction == DMA_DEV_TO_MEM)
1069				result.result = DMA_TRANS_READ_FAILED;
1070			else
1071				result.result = DMA_TRANS_WRITE_FAILED;
1072		} else {
1073			result.result = DMA_TRANS_NOERROR;
1074		}
1075
1076		result.residue = desc->residue;
1077
1078		/* Run the link descriptor callback function */
1079		spin_unlock_irqrestore(&chan->lock, flags);
1080		dmaengine_desc_get_callback_invoke(&desc->async_tx, &result);
1081		spin_lock_irqsave(&chan->lock, flags);
1082
1083		/* Run any dependencies, then free the descriptor */
1084		dma_run_dependencies(&desc->async_tx);
1085		xilinx_dma_free_tx_descriptor(chan, desc);
1086
1087		/*
1088		 * While we ran a callback the user called a terminate function,
1089		 * which takes care of cleaning up any remaining descriptors
1090		 */
1091		if (chan->terminating)
1092			break;
1093	}
1094
1095	spin_unlock_irqrestore(&chan->lock, flags);
1096}
1097
1098/**
1099 * xilinx_dma_do_tasklet - Schedule completion tasklet
1100 * @t: Pointer to the Xilinx DMA channel structure
1101 */
1102static void xilinx_dma_do_tasklet(struct tasklet_struct *t)
1103{
1104	struct xilinx_dma_chan *chan = from_tasklet(chan, t, tasklet);
1105
1106	xilinx_dma_chan_desc_cleanup(chan);
1107}
1108
1109/**
1110 * xilinx_dma_alloc_chan_resources - Allocate channel resources
1111 * @dchan: DMA channel
1112 *
1113 * Return: '0' on success and failure value on error
1114 */
1115static int xilinx_dma_alloc_chan_resources(struct dma_chan *dchan)
1116{
1117	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
1118	int i;
1119
1120	/* Has this channel already been allocated? */
1121	if (chan->desc_pool)
1122		return 0;
1123
1124	/*
1125	 * We need the descriptor to be aligned to 64bytes
1126	 * for meeting Xilinx VDMA specification requirement.
1127	 */
1128	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
1129		/* Allocate the buffer descriptors. */
1130		chan->seg_v = dma_alloc_coherent(chan->dev,
1131						 sizeof(*chan->seg_v) * XILINX_DMA_NUM_DESCS,
1132						 &chan->seg_p, GFP_KERNEL);
1133		if (!chan->seg_v) {
1134			dev_err(chan->dev,
1135				"unable to allocate channel %d descriptors\n",
1136				chan->id);
1137			return -ENOMEM;
1138		}
1139		/*
1140		 * For cyclic DMA mode we need to program the tail Descriptor
1141		 * register with a value which is not a part of the BD chain
1142		 * so allocating a desc segment during channel allocation for
1143		 * programming tail descriptor.
1144		 */
1145		chan->cyclic_seg_v = dma_alloc_coherent(chan->dev,
1146							sizeof(*chan->cyclic_seg_v),
1147							&chan->cyclic_seg_p,
1148							GFP_KERNEL);
1149		if (!chan->cyclic_seg_v) {
1150			dev_err(chan->dev,
1151				"unable to allocate desc segment for cyclic DMA\n");
1152			dma_free_coherent(chan->dev, sizeof(*chan->seg_v) *
1153				XILINX_DMA_NUM_DESCS, chan->seg_v,
1154				chan->seg_p);
1155			return -ENOMEM;
1156		}
1157		chan->cyclic_seg_v->phys = chan->cyclic_seg_p;
1158
1159		for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) {
1160			chan->seg_v[i].hw.next_desc =
1161			lower_32_bits(chan->seg_p + sizeof(*chan->seg_v) *
1162				((i + 1) % XILINX_DMA_NUM_DESCS));
1163			chan->seg_v[i].hw.next_desc_msb =
1164			upper_32_bits(chan->seg_p + sizeof(*chan->seg_v) *
1165				((i + 1) % XILINX_DMA_NUM_DESCS));
1166			chan->seg_v[i].phys = chan->seg_p +
1167				sizeof(*chan->seg_v) * i;
1168			list_add_tail(&chan->seg_v[i].node,
1169				      &chan->free_seg_list);
1170		}
1171	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
1172		/* Allocate the buffer descriptors. */
1173		chan->seg_mv = dma_alloc_coherent(chan->dev,
1174						  sizeof(*chan->seg_mv) *
1175						  XILINX_DMA_NUM_DESCS,
1176						  &chan->seg_p, GFP_KERNEL);
1177		if (!chan->seg_mv) {
1178			dev_err(chan->dev,
1179				"unable to allocate channel %d descriptors\n",
1180				chan->id);
1181			return -ENOMEM;
1182		}
1183		for (i = 0; i < XILINX_DMA_NUM_DESCS; i++) {
1184			chan->seg_mv[i].hw.next_desc =
1185			lower_32_bits(chan->seg_p + sizeof(*chan->seg_mv) *
1186				((i + 1) % XILINX_DMA_NUM_DESCS));
1187			chan->seg_mv[i].hw.next_desc_msb =
1188			upper_32_bits(chan->seg_p + sizeof(*chan->seg_mv) *
1189				((i + 1) % XILINX_DMA_NUM_DESCS));
1190			chan->seg_mv[i].phys = chan->seg_p +
1191				sizeof(*chan->seg_mv) * i;
1192			list_add_tail(&chan->seg_mv[i].node,
1193				      &chan->free_seg_list);
1194		}
1195	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
1196		chan->desc_pool = dma_pool_create("xilinx_cdma_desc_pool",
1197				   chan->dev,
1198				   sizeof(struct xilinx_cdma_tx_segment),
1199				   __alignof__(struct xilinx_cdma_tx_segment),
1200				   0);
1201	} else {
1202		chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool",
1203				     chan->dev,
1204				     sizeof(struct xilinx_vdma_tx_segment),
1205				     __alignof__(struct xilinx_vdma_tx_segment),
1206				     0);
1207	}
1208
1209	if (!chan->desc_pool &&
1210	    ((chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) &&
1211		chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA)) {
1212		dev_err(chan->dev,
1213			"unable to allocate channel %d descriptor pool\n",
1214			chan->id);
1215		return -ENOMEM;
1216	}
1217
1218	dma_cookie_init(dchan);
1219
1220	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)
1221		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,
1222			     XILINX_CDMA_CR_SGMODE);
1223
1224	return 0;
1225}
1226
1227/**
1228 * xilinx_dma_calc_copysize - Calculate the amount of data to copy
1229 * @chan: Driver specific DMA channel
1230 * @size: Total data that needs to be copied
1231 * @done: Amount of data that has been already copied
1232 *
1233 * Return: Amount of data that has to be copied
1234 */
1235static int xilinx_dma_calc_copysize(struct xilinx_dma_chan *chan,
1236				    int size, int done)
1237{
1238	size_t copy;
1239
1240	copy = min_t(size_t, size - done,
1241		     chan->xdev->max_buffer_len);
1242
1243	if ((copy + done < size) &&
1244	    chan->xdev->common.copy_align) {
1245		/*
1246		 * If this is not the last descriptor, make sure
1247		 * the next one will be properly aligned
1248		 */
1249		copy = rounddown(copy,
1250				 (1 << chan->xdev->common.copy_align));
1251	}
1252	return copy;
1253}
1254
1255/**
1256 * xilinx_dma_tx_status - Get DMA transaction status
1257 * @dchan: DMA channel
1258 * @cookie: Transaction identifier
1259 * @txstate: Transaction state
1260 *
1261 * Return: DMA transaction status
1262 */
1263static enum dma_status xilinx_dma_tx_status(struct dma_chan *dchan,
1264					dma_cookie_t cookie,
1265					struct dma_tx_state *txstate)
1266{
1267	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
1268	struct xilinx_dma_tx_descriptor *desc;
1269	enum dma_status ret;
1270	unsigned long flags;
1271	u32 residue = 0;
1272
1273	ret = dma_cookie_status(dchan, cookie, txstate);
1274	if (ret == DMA_COMPLETE || !txstate)
1275		return ret;
1276
1277	spin_lock_irqsave(&chan->lock, flags);
1278	if (!list_empty(&chan->active_list)) {
1279		desc = list_last_entry(&chan->active_list,
1280				       struct xilinx_dma_tx_descriptor, node);
1281		/*
1282		 * VDMA and simple mode do not support residue reporting, so the
1283		 * residue field will always be 0.
1284		 */
1285		if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA)
1286			residue = xilinx_dma_get_residue(chan, desc);
1287	}
1288	spin_unlock_irqrestore(&chan->lock, flags);
1289
1290	dma_set_residue(txstate, residue);
1291
1292	return ret;
1293}
1294
1295/**
1296 * xilinx_dma_stop_transfer - Halt DMA channel
1297 * @chan: Driver specific DMA channel
1298 *
1299 * Return: '0' on success and failure value on error
1300 */
1301static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan)
1302{
1303	u32 val;
1304
1305	dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP);
1306
1307	/* Wait for the hardware to halt */
1308	return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
1309				       val & XILINX_DMA_DMASR_HALTED, 0,
1310				       XILINX_DMA_LOOP_COUNT);
1311}
1312
1313/**
1314 * xilinx_cdma_stop_transfer - Wait for the current transfer to complete
1315 * @chan: Driver specific DMA channel
1316 *
1317 * Return: '0' on success and failure value on error
1318 */
1319static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan)
1320{
1321	u32 val;
1322
1323	return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
1324				       val & XILINX_DMA_DMASR_IDLE, 0,
1325				       XILINX_DMA_LOOP_COUNT);
1326}
1327
1328/**
1329 * xilinx_dma_start - Start DMA channel
1330 * @chan: Driver specific DMA channel
1331 */
1332static void xilinx_dma_start(struct xilinx_dma_chan *chan)
1333{
1334	int err;
1335	u32 val;
1336
1337	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP);
1338
1339	/* Wait for the hardware to start */
1340	err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
1341				      !(val & XILINX_DMA_DMASR_HALTED), 0,
1342				      XILINX_DMA_LOOP_COUNT);
1343
1344	if (err) {
1345		dev_err(chan->dev, "Cannot start channel %p: %x\n",
1346			chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR));
1347
1348		chan->err = true;
1349	}
1350}
1351
1352/**
1353 * xilinx_vdma_start_transfer - Starts VDMA transfer
1354 * @chan: Driver specific channel struct pointer
1355 */
1356static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)
1357{
1358	struct xilinx_vdma_config *config = &chan->config;
1359	struct xilinx_dma_tx_descriptor *desc;
1360	u32 reg, j;
1361	struct xilinx_vdma_tx_segment *segment, *last = NULL;
1362	int i = 0;
1363
1364	/* This function was invoked with lock held */
1365	if (chan->err)
1366		return;
1367
1368	if (!chan->idle)
1369		return;
1370
1371	if (list_empty(&chan->pending_list))
1372		return;
1373
1374	desc = list_first_entry(&chan->pending_list,
1375				struct xilinx_dma_tx_descriptor, node);
1376
1377	/* Configure the hardware using info in the config structure */
1378	if (chan->has_vflip) {
1379		reg = dma_read(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP);
1380		reg &= ~XILINX_VDMA_ENABLE_VERTICAL_FLIP;
1381		reg |= config->vflip_en;
1382		dma_write(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP,
1383			  reg);
1384	}
1385
1386	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);
1387
1388	if (config->frm_cnt_en)
1389		reg |= XILINX_DMA_DMACR_FRAMECNT_EN;
1390	else
1391		reg &= ~XILINX_DMA_DMACR_FRAMECNT_EN;
1392
1393	/* If not parking, enable circular mode */
1394	if (config->park)
1395		reg &= ~XILINX_DMA_DMACR_CIRC_EN;
1396	else
1397		reg |= XILINX_DMA_DMACR_CIRC_EN;
1398
1399	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
1400
1401	if (config->park) {
1402		j = chan->desc_submitcount;
1403		reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR);
1404		if (chan->direction == DMA_MEM_TO_DEV) {
1405			reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK;
1406			reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT;
1407		} else {
1408			reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK;
1409			reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT;
1410		}
1411		dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg);
1412	}
1413
1414	/* Start the hardware */
1415	xilinx_dma_start(chan);
1416
1417	if (chan->err)
1418		return;
1419
1420	/* Start the transfer */
1421	if (chan->desc_submitcount < chan->num_frms)
1422		i = chan->desc_submitcount;
1423
1424	list_for_each_entry(segment, &desc->segments, node) {
1425		if (chan->ext_addr)
1426			vdma_desc_write_64(chan,
1427				   XILINX_VDMA_REG_START_ADDRESS_64(i++),
1428				   segment->hw.buf_addr,
1429				   segment->hw.buf_addr_msb);
1430		else
1431			vdma_desc_write(chan,
1432					XILINX_VDMA_REG_START_ADDRESS(i++),
1433					segment->hw.buf_addr);
1434
1435		last = segment;
1436	}
1437
1438	if (!last)
1439		return;
1440
1441	/* HW expects these parameters to be same for one transaction */
1442	vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize);
1443	vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE,
1444			last->hw.stride);
1445	vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize);
1446
1447	chan->desc_submitcount++;
1448	chan->desc_pendingcount--;
1449	list_move_tail(&desc->node, &chan->active_list);
1450	if (chan->desc_submitcount == chan->num_frms)
1451		chan->desc_submitcount = 0;
1452
1453	chan->idle = false;
1454}
1455
1456/**
1457 * xilinx_cdma_start_transfer - Starts cdma transfer
1458 * @chan: Driver specific channel struct pointer
1459 */
1460static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan)
1461{
1462	struct xilinx_dma_tx_descriptor *head_desc, *tail_desc;
1463	struct xilinx_cdma_tx_segment *tail_segment;
1464	u32 ctrl_reg = dma_read(chan, XILINX_DMA_REG_DMACR);
1465
1466	if (chan->err)
1467		return;
1468
1469	if (!chan->idle)
1470		return;
1471
1472	if (list_empty(&chan->pending_list))
1473		return;
1474
1475	head_desc = list_first_entry(&chan->pending_list,
1476				     struct xilinx_dma_tx_descriptor, node);
1477	tail_desc = list_last_entry(&chan->pending_list,
1478				    struct xilinx_dma_tx_descriptor, node);
1479	tail_segment = list_last_entry(&tail_desc->segments,
1480				       struct xilinx_cdma_tx_segment, node);
1481
1482	if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) {
1483		ctrl_reg &= ~XILINX_DMA_CR_COALESCE_MAX;
1484		ctrl_reg |= chan->desc_pendingcount <<
1485				XILINX_DMA_CR_COALESCE_SHIFT;
1486		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, ctrl_reg);
1487	}
1488
1489	if (chan->has_sg) {
1490		dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,
1491			     XILINX_CDMA_CR_SGMODE);
1492
1493		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,
1494			     XILINX_CDMA_CR_SGMODE);
1495
1496		xilinx_write(chan, XILINX_DMA_REG_CURDESC,
1497			     head_desc->async_tx.phys);
1498
1499		/* Update tail ptr register which will start the transfer */
1500		xilinx_write(chan, XILINX_DMA_REG_TAILDESC,
1501			     tail_segment->phys);
1502	} else {
1503		/* In simple mode */
1504		struct xilinx_cdma_tx_segment *segment;
1505		struct xilinx_cdma_desc_hw *hw;
1506
1507		segment = list_first_entry(&head_desc->segments,
1508					   struct xilinx_cdma_tx_segment,
1509					   node);
1510
1511		hw = &segment->hw;
1512
1513		xilinx_write(chan, XILINX_CDMA_REG_SRCADDR,
1514			     xilinx_prep_dma_addr_t(hw->src_addr));
1515		xilinx_write(chan, XILINX_CDMA_REG_DSTADDR,
1516			     xilinx_prep_dma_addr_t(hw->dest_addr));
1517
1518		/* Start the transfer */
1519		dma_ctrl_write(chan, XILINX_DMA_REG_BTT,
1520				hw->control & chan->xdev->max_buffer_len);
1521	}
1522
1523	list_splice_tail_init(&chan->pending_list, &chan->active_list);
1524	chan->desc_pendingcount = 0;
1525	chan->idle = false;
1526}
1527
1528/**
1529 * xilinx_dma_start_transfer - Starts DMA transfer
1530 * @chan: Driver specific channel struct pointer
1531 */
1532static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
1533{
1534	struct xilinx_dma_tx_descriptor *head_desc, *tail_desc;
1535	struct xilinx_axidma_tx_segment *tail_segment;
1536	u32 reg;
1537
1538	if (chan->err)
1539		return;
1540
1541	if (list_empty(&chan->pending_list)) {
1542		if (chan->cyclic) {
1543			struct xilinx_dma_tx_descriptor *desc;
1544			struct list_head *entry;
1545
1546			desc = list_last_entry(&chan->done_list,
1547					       struct xilinx_dma_tx_descriptor, node);
1548			list_for_each(entry, &desc->segments) {
1549				struct xilinx_axidma_tx_segment *axidma_seg;
1550				struct xilinx_axidma_desc_hw *axidma_hw;
1551				axidma_seg = list_entry(entry,
1552							struct xilinx_axidma_tx_segment,
1553							node);
1554				axidma_hw = &axidma_seg->hw;
1555				axidma_hw->status = 0;
1556			}
1557
1558			list_splice_tail_init(&chan->done_list, &chan->active_list);
1559			chan->desc_pendingcount = 0;
1560			chan->idle = false;
1561		}
1562		return;
1563	}
1564
1565	if (!chan->idle)
1566		return;
1567
1568	head_desc = list_first_entry(&chan->pending_list,
1569				     struct xilinx_dma_tx_descriptor, node);
1570	tail_desc = list_last_entry(&chan->pending_list,
1571				    struct xilinx_dma_tx_descriptor, node);
1572	tail_segment = list_last_entry(&tail_desc->segments,
1573				       struct xilinx_axidma_tx_segment, node);
1574
1575	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);
1576
1577	if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) {
1578		reg &= ~XILINX_DMA_CR_COALESCE_MAX;
1579		reg |= chan->desc_pendingcount <<
1580				  XILINX_DMA_CR_COALESCE_SHIFT;
1581		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
1582	}
1583
1584	if (chan->has_sg)
1585		xilinx_write(chan, XILINX_DMA_REG_CURDESC,
1586			     head_desc->async_tx.phys);
1587	reg  &= ~XILINX_DMA_CR_DELAY_MAX;
1588	reg  |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT;
1589	reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK;
1590	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
1591
1592	xilinx_dma_start(chan);
1593
1594	if (chan->err)
1595		return;
1596
1597	/* Start the transfer */
1598	if (chan->has_sg) {
1599		if (chan->cyclic)
1600			xilinx_write(chan, XILINX_DMA_REG_TAILDESC,
1601				     chan->cyclic_seg_v->phys);
1602		else
1603			xilinx_write(chan, XILINX_DMA_REG_TAILDESC,
1604				     tail_segment->phys);
1605	} else {
1606		struct xilinx_axidma_tx_segment *segment;
1607		struct xilinx_axidma_desc_hw *hw;
1608
1609		segment = list_first_entry(&head_desc->segments,
1610					   struct xilinx_axidma_tx_segment,
1611					   node);
1612		hw = &segment->hw;
1613
1614		xilinx_write(chan, XILINX_DMA_REG_SRCDSTADDR,
1615			     xilinx_prep_dma_addr_t(hw->buf_addr));
1616
1617		/* Start the transfer */
1618		dma_ctrl_write(chan, XILINX_DMA_REG_BTT,
1619			       hw->control & chan->xdev->max_buffer_len);
1620	}
1621
1622	list_splice_tail_init(&chan->pending_list, &chan->active_list);
1623	chan->desc_pendingcount = 0;
1624	chan->idle = false;
1625}
1626
1627/**
1628 * xilinx_mcdma_start_transfer - Starts MCDMA transfer
1629 * @chan: Driver specific channel struct pointer
1630 */
1631static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)
1632{
1633	struct xilinx_dma_tx_descriptor *head_desc, *tail_desc;
1634	struct xilinx_aximcdma_tx_segment *tail_segment;
1635	u32 reg;
1636
1637	/*
1638	 * lock has been held by calling functions, so we don't need it
1639	 * to take it here again.
1640	 */
1641
1642	if (chan->err)
1643		return;
1644
1645	if (!chan->idle)
1646		return;
1647
1648	if (list_empty(&chan->pending_list))
1649		return;
1650
1651	head_desc = list_first_entry(&chan->pending_list,
1652				     struct xilinx_dma_tx_descriptor, node);
1653	tail_desc = list_last_entry(&chan->pending_list,
1654				    struct xilinx_dma_tx_descriptor, node);
1655	tail_segment = list_last_entry(&tail_desc->segments,
1656				       struct xilinx_aximcdma_tx_segment, node);
1657
1658	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest));
1659
1660	if (chan->desc_pendingcount <= XILINX_MCDMA_COALESCE_MAX) {
1661		reg &= ~XILINX_MCDMA_COALESCE_MASK;
1662		reg |= chan->desc_pendingcount <<
1663			XILINX_MCDMA_COALESCE_SHIFT;
1664	}
1665
1666	reg |= XILINX_MCDMA_IRQ_ALL_MASK;
1667	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);
1668
1669	/* Program current descriptor */
1670	xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest),
1671		     head_desc->async_tx.phys);
1672
1673	/* Program channel enable register */
1674	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHEN_OFFSET);
1675	reg |= BIT(chan->tdest);
1676	dma_ctrl_write(chan, XILINX_MCDMA_CHEN_OFFSET, reg);
1677
1678	/* Start the fetch of BDs for the channel */
1679	reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest));
1680	reg |= XILINX_MCDMA_CR_RUNSTOP_MASK;
1681	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);
1682
1683	xilinx_dma_start(chan);
1684
1685	if (chan->err)
1686		return;
1687
1688	/* Start the transfer */
1689	xilinx_write(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET(chan->tdest),
1690		     tail_segment->phys);
1691
1692	list_splice_tail_init(&chan->pending_list, &chan->active_list);
1693	chan->desc_pendingcount = 0;
1694	chan->idle = false;
1695}
1696
1697/**
1698 * xilinx_dma_issue_pending - Issue pending transactions
1699 * @dchan: DMA channel
1700 */
1701static void xilinx_dma_issue_pending(struct dma_chan *dchan)
1702{
1703	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
1704	unsigned long flags;
1705
1706	spin_lock_irqsave(&chan->lock, flags);
1707	chan->start_transfer(chan);
1708	spin_unlock_irqrestore(&chan->lock, flags);
1709}
1710
1711/**
1712 * xilinx_dma_device_config - Configure the DMA channel
1713 * @dchan: DMA channel
1714 * @config: channel configuration
1715 *
1716 * Return: 0 always.
1717 */
1718static int xilinx_dma_device_config(struct dma_chan *dchan,
1719				    struct dma_slave_config *config)
1720{
1721	return 0;
1722}
1723
1724/**
1725 * xilinx_dma_complete_descriptor - Mark the active descriptor as complete
1726 * @chan : xilinx DMA channel
1727 *
1728 * CONTEXT: hardirq
1729 */
1730static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan)
1731{
1732	struct xilinx_dma_tx_descriptor *desc, *next;
1733
1734	/* This function was invoked with lock held */
1735	if (list_empty(&chan->active_list))
1736		return;
1737
1738	list_for_each_entry_safe(desc, next, &chan->active_list, node) {
1739		if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
1740			struct xilinx_axidma_tx_segment *seg;
1741
1742			seg = list_last_entry(&desc->segments,
1743					      struct xilinx_axidma_tx_segment, node);
1744			if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg)
1745				break;
1746		}
1747		if (chan->has_sg && chan->xdev->dma_config->dmatype !=
1748		    XDMA_TYPE_VDMA)
1749			desc->residue = xilinx_dma_get_residue(chan, desc);
1750		else
1751			desc->residue = 0;
1752		desc->err = chan->err;
1753
1754		list_del(&desc->node);
1755		if (!desc->cyclic)
1756			dma_cookie_complete(&desc->async_tx);
1757		list_add_tail(&desc->node, &chan->done_list);
1758	}
1759}
1760
1761/**
1762 * xilinx_dma_reset - Reset DMA channel
1763 * @chan: Driver specific DMA channel
1764 *
1765 * Return: '0' on success and failure value on error
1766 */
1767static int xilinx_dma_reset(struct xilinx_dma_chan *chan)
1768{
1769	int err;
1770	u32 tmp;
1771
1772	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RESET);
1773
1774	/* Wait for the hardware to finish reset */
1775	err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp,
1776				      !(tmp & XILINX_DMA_DMACR_RESET), 0,
1777				      XILINX_DMA_LOOP_COUNT);
1778
1779	if (err) {
1780		dev_err(chan->dev, "reset timeout, cr %x, sr %x\n",
1781			dma_ctrl_read(chan, XILINX_DMA_REG_DMACR),
1782			dma_ctrl_read(chan, XILINX_DMA_REG_DMASR));
1783		return -ETIMEDOUT;
1784	}
1785
1786	chan->err = false;
1787	chan->idle = true;
1788	chan->desc_pendingcount = 0;
1789	chan->desc_submitcount = 0;
1790
1791	return err;
1792}
1793
1794/**
1795 * xilinx_dma_chan_reset - Reset DMA channel and enable interrupts
1796 * @chan: Driver specific DMA channel
1797 *
1798 * Return: '0' on success and failure value on error
1799 */
1800static int xilinx_dma_chan_reset(struct xilinx_dma_chan *chan)
1801{
1802	int err;
1803
1804	/* Reset VDMA */
1805	err = xilinx_dma_reset(chan);
1806	if (err)
1807		return err;
1808
1809	/* Enable interrupts */
1810	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,
1811		      XILINX_DMA_DMAXR_ALL_IRQ_MASK);
1812
1813	return 0;
1814}
1815
1816/**
1817 * xilinx_mcdma_irq_handler - MCDMA Interrupt handler
1818 * @irq: IRQ number
1819 * @data: Pointer to the Xilinx MCDMA channel structure
1820 *
1821 * Return: IRQ_HANDLED/IRQ_NONE
1822 */
1823static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data)
1824{
1825	struct xilinx_dma_chan *chan = data;
1826	u32 status, ser_offset, chan_sermask, chan_offset = 0, chan_id;
1827
1828	if (chan->direction == DMA_DEV_TO_MEM)
1829		ser_offset = XILINX_MCDMA_RXINT_SER_OFFSET;
1830	else
1831		ser_offset = XILINX_MCDMA_TXINT_SER_OFFSET;
1832
1833	/* Read the channel id raising the interrupt*/
1834	chan_sermask = dma_ctrl_read(chan, ser_offset);
1835	chan_id = ffs(chan_sermask);
1836
1837	if (!chan_id)
1838		return IRQ_NONE;
1839
1840	if (chan->direction == DMA_DEV_TO_MEM)
1841		chan_offset = chan->xdev->dma_config->max_channels / 2;
1842
1843	chan_offset = chan_offset + (chan_id - 1);
1844	chan = chan->xdev->chan[chan_offset];
1845	/* Read the status and ack the interrupts. */
1846	status = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest));
1847	if (!(status & XILINX_MCDMA_IRQ_ALL_MASK))
1848		return IRQ_NONE;
1849
1850	dma_ctrl_write(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest),
1851		       status & XILINX_MCDMA_IRQ_ALL_MASK);
1852
1853	if (status & XILINX_MCDMA_IRQ_ERR_MASK) {
1854		dev_err(chan->dev, "Channel %p has errors %x cdr %x tdr %x\n",
1855			chan,
1856			dma_ctrl_read(chan, XILINX_MCDMA_CH_ERR_OFFSET),
1857			dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET
1858				      (chan->tdest)),
1859			dma_ctrl_read(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET
1860				      (chan->tdest)));
1861		chan->err = true;
1862	}
1863
1864	if (status & XILINX_MCDMA_IRQ_DELAY_MASK) {
1865		/*
1866		 * Device takes too long to do the transfer when user requires
1867		 * responsiveness.
1868		 */
1869		dev_dbg(chan->dev, "Inter-packet latency too long\n");
1870	}
1871
1872	if (status & XILINX_MCDMA_IRQ_IOC_MASK) {
1873		spin_lock(&chan->lock);
1874		xilinx_dma_complete_descriptor(chan);
1875		chan->idle = true;
1876		chan->start_transfer(chan);
1877		spin_unlock(&chan->lock);
1878	}
1879
1880	tasklet_hi_schedule(&chan->tasklet);
1881	return IRQ_HANDLED;
1882}
1883
1884/**
1885 * xilinx_dma_irq_handler - DMA Interrupt handler
1886 * @irq: IRQ number
1887 * @data: Pointer to the Xilinx DMA channel structure
1888 *
1889 * Return: IRQ_HANDLED/IRQ_NONE
1890 */
1891static irqreturn_t xilinx_dma_irq_handler(int irq, void *data)
1892{
1893	struct xilinx_dma_chan *chan = data;
1894	u32 status;
1895
1896	/* Read the status and ack the interrupts. */
1897	status = dma_ctrl_read(chan, XILINX_DMA_REG_DMASR);
1898	if (!(status & XILINX_DMA_DMAXR_ALL_IRQ_MASK))
1899		return IRQ_NONE;
1900
1901	dma_ctrl_write(chan, XILINX_DMA_REG_DMASR,
1902			status & XILINX_DMA_DMAXR_ALL_IRQ_MASK);
1903
1904	if (status & XILINX_DMA_DMASR_ERR_IRQ) {
1905		/*
1906		 * An error occurred. If C_FLUSH_ON_FSYNC is enabled and the
1907		 * error is recoverable, ignore it. Otherwise flag the error.
1908		 *
1909		 * Only recoverable errors can be cleared in the DMASR register,
1910		 * make sure not to write to other error bits to 1.
1911		 */
1912		u32 errors = status & XILINX_DMA_DMASR_ALL_ERR_MASK;
1913
1914		dma_ctrl_write(chan, XILINX_DMA_REG_DMASR,
1915				errors & XILINX_DMA_DMASR_ERR_RECOVER_MASK);
1916
1917		if (!chan->flush_on_fsync ||
1918		    (errors & ~XILINX_DMA_DMASR_ERR_RECOVER_MASK)) {
1919			dev_err(chan->dev,
1920				"Channel %p has errors %x, cdr %x tdr %x\n",
1921				chan, errors,
1922				dma_ctrl_read(chan, XILINX_DMA_REG_CURDESC),
1923				dma_ctrl_read(chan, XILINX_DMA_REG_TAILDESC));
1924			chan->err = true;
1925		}
1926	}
1927
1928	if (status & (XILINX_DMA_DMASR_FRM_CNT_IRQ |
1929		      XILINX_DMA_DMASR_DLY_CNT_IRQ)) {
1930		spin_lock(&chan->lock);
1931		xilinx_dma_complete_descriptor(chan);
1932		chan->idle = true;
1933		chan->start_transfer(chan);
1934		spin_unlock(&chan->lock);
1935	}
1936
1937	tasklet_schedule(&chan->tasklet);
1938	return IRQ_HANDLED;
1939}
1940
1941/**
1942 * append_desc_queue - Queuing descriptor
1943 * @chan: Driver specific dma channel
1944 * @desc: dma transaction descriptor
1945 */
1946static void append_desc_queue(struct xilinx_dma_chan *chan,
1947			      struct xilinx_dma_tx_descriptor *desc)
1948{
1949	struct xilinx_vdma_tx_segment *tail_segment;
1950	struct xilinx_dma_tx_descriptor *tail_desc;
1951	struct xilinx_axidma_tx_segment *axidma_tail_segment;
1952	struct xilinx_aximcdma_tx_segment *aximcdma_tail_segment;
1953	struct xilinx_cdma_tx_segment *cdma_tail_segment;
1954
1955	if (list_empty(&chan->pending_list))
1956		goto append;
1957
1958	/*
1959	 * Add the hardware descriptor to the chain of hardware descriptors
1960	 * that already exists in memory.
1961	 */
1962	tail_desc = list_last_entry(&chan->pending_list,
1963				    struct xilinx_dma_tx_descriptor, node);
1964	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
1965		tail_segment = list_last_entry(&tail_desc->segments,
1966					       struct xilinx_vdma_tx_segment,
1967					       node);
1968		tail_segment->hw.next_desc = (u32)desc->async_tx.phys;
1969	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
1970		cdma_tail_segment = list_last_entry(&tail_desc->segments,
1971						struct xilinx_cdma_tx_segment,
1972						node);
1973		cdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;
1974	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
1975		axidma_tail_segment = list_last_entry(&tail_desc->segments,
1976					       struct xilinx_axidma_tx_segment,
1977					       node);
1978		axidma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;
1979	} else {
1980		aximcdma_tail_segment =
1981			list_last_entry(&tail_desc->segments,
1982					struct xilinx_aximcdma_tx_segment,
1983					node);
1984		aximcdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys;
1985	}
1986
1987	/*
1988	 * Add the software descriptor and all children to the list
1989	 * of pending transactions
1990	 */
1991append:
1992	list_add_tail(&desc->node, &chan->pending_list);
1993	chan->desc_pendingcount++;
1994
1995	if (chan->has_sg && (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA)
1996	    && unlikely(chan->desc_pendingcount > chan->num_frms)) {
1997		dev_dbg(chan->dev, "desc pendingcount is too high\n");
1998		chan->desc_pendingcount = chan->num_frms;
1999	}
2000}
2001
2002/**
2003 * xilinx_dma_tx_submit - Submit DMA transaction
2004 * @tx: Async transaction descriptor
2005 *
2006 * Return: cookie value on success and failure value on error
2007 */
2008static dma_cookie_t xilinx_dma_tx_submit(struct dma_async_tx_descriptor *tx)
2009{
2010	struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx);
2011	struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan);
2012	dma_cookie_t cookie;
2013	unsigned long flags;
2014	int err;
2015
2016	if (chan->cyclic) {
2017		xilinx_dma_free_tx_descriptor(chan, desc);
2018		return -EBUSY;
2019	}
2020
2021	if (chan->err) {
2022		/*
2023		 * If reset fails, need to hard reset the system.
2024		 * Channel is no longer functional
2025		 */
2026		err = xilinx_dma_chan_reset(chan);
2027		if (err < 0)
2028			return err;
2029	}
2030
2031	spin_lock_irqsave(&chan->lock, flags);
2032
2033	cookie = dma_cookie_assign(tx);
2034
2035	/* Put this transaction onto the tail of the pending queue */
2036	append_desc_queue(chan, desc);
2037
2038	if (desc->cyclic)
2039		chan->cyclic = true;
2040
2041	chan->terminating = false;
2042
2043	spin_unlock_irqrestore(&chan->lock, flags);
2044
2045	return cookie;
2046}
2047
2048/**
2049 * xilinx_vdma_dma_prep_interleaved - prepare a descriptor for a
2050 *	DMA_SLAVE transaction
2051 * @dchan: DMA channel
2052 * @xt: Interleaved template pointer
2053 * @flags: transfer ack flags
2054 *
2055 * Return: Async transaction descriptor on success and NULL on failure
2056 */
2057static struct dma_async_tx_descriptor *
2058xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan,
2059				 struct dma_interleaved_template *xt,
2060				 unsigned long flags)
2061{
2062	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2063	struct xilinx_dma_tx_descriptor *desc;
2064	struct xilinx_vdma_tx_segment *segment;
2065	struct xilinx_vdma_desc_hw *hw;
2066
2067	if (!is_slave_direction(xt->dir))
2068		return NULL;
2069
2070	if (!xt->numf || !xt->sgl[0].size)
2071		return NULL;
2072
2073	if (xt->numf & ~XILINX_DMA_VSIZE_MASK ||
2074	    xt->sgl[0].size & ~XILINX_DMA_HSIZE_MASK)
2075		return NULL;
2076
2077	if (xt->frame_size != 1)
2078		return NULL;
2079
2080	/* Allocate a transaction descriptor. */
2081	desc = xilinx_dma_alloc_tx_descriptor(chan);
2082	if (!desc)
2083		return NULL;
2084
2085	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
2086	desc->async_tx.tx_submit = xilinx_dma_tx_submit;
2087	async_tx_ack(&desc->async_tx);
2088
2089	/* Allocate the link descriptor from DMA pool */
2090	segment = xilinx_vdma_alloc_tx_segment(chan);
2091	if (!segment)
2092		goto error;
2093
2094	/* Fill in the hardware descriptor */
2095	hw = &segment->hw;
2096	hw->vsize = xt->numf;
2097	hw->hsize = xt->sgl[0].size;
2098	hw->stride = (xt->sgl[0].icg + xt->sgl[0].size) <<
2099			XILINX_DMA_FRMDLY_STRIDE_STRIDE_SHIFT;
2100	hw->stride |= chan->config.frm_dly <<
2101			XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT;
2102
2103	if (xt->dir != DMA_MEM_TO_DEV) {
2104		if (chan->ext_addr) {
2105			hw->buf_addr = lower_32_bits(xt->dst_start);
2106			hw->buf_addr_msb = upper_32_bits(xt->dst_start);
2107		} else {
2108			hw->buf_addr = xt->dst_start;
2109		}
2110	} else {
2111		if (chan->ext_addr) {
2112			hw->buf_addr = lower_32_bits(xt->src_start);
2113			hw->buf_addr_msb = upper_32_bits(xt->src_start);
2114		} else {
2115			hw->buf_addr = xt->src_start;
2116		}
2117	}
2118
2119	/* Insert the segment into the descriptor segments list. */
2120	list_add_tail(&segment->node, &desc->segments);
2121
2122	/* Link the last hardware descriptor with the first. */
2123	segment = list_first_entry(&desc->segments,
2124				   struct xilinx_vdma_tx_segment, node);
2125	desc->async_tx.phys = segment->phys;
2126
2127	return &desc->async_tx;
2128
2129error:
2130	xilinx_dma_free_tx_descriptor(chan, desc);
2131	return NULL;
2132}
2133
2134/**
2135 * xilinx_cdma_prep_memcpy - prepare descriptors for a memcpy transaction
2136 * @dchan: DMA channel
2137 * @dma_dst: destination address
2138 * @dma_src: source address
2139 * @len: transfer length
2140 * @flags: transfer ack flags
2141 *
2142 * Return: Async transaction descriptor on success and NULL on failure
2143 */
2144static struct dma_async_tx_descriptor *
2145xilinx_cdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst,
2146			dma_addr_t dma_src, size_t len, unsigned long flags)
2147{
2148	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2149	struct xilinx_dma_tx_descriptor *desc;
2150	struct xilinx_cdma_tx_segment *segment;
2151	struct xilinx_cdma_desc_hw *hw;
2152
2153	if (!len || len > chan->xdev->max_buffer_len)
2154		return NULL;
2155
2156	desc = xilinx_dma_alloc_tx_descriptor(chan);
2157	if (!desc)
2158		return NULL;
2159
2160	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
2161	desc->async_tx.tx_submit = xilinx_dma_tx_submit;
2162
2163	/* Allocate the link descriptor from DMA pool */
2164	segment = xilinx_cdma_alloc_tx_segment(chan);
2165	if (!segment)
2166		goto error;
2167
2168	hw = &segment->hw;
2169	hw->control = len;
2170	hw->src_addr = dma_src;
2171	hw->dest_addr = dma_dst;
2172	if (chan->ext_addr) {
2173		hw->src_addr_msb = upper_32_bits(dma_src);
2174		hw->dest_addr_msb = upper_32_bits(dma_dst);
2175	}
2176
2177	/* Insert the segment into the descriptor segments list. */
2178	list_add_tail(&segment->node, &desc->segments);
2179
2180	desc->async_tx.phys = segment->phys;
2181	hw->next_desc = segment->phys;
2182
2183	return &desc->async_tx;
2184
2185error:
2186	xilinx_dma_free_tx_descriptor(chan, desc);
2187	return NULL;
2188}
2189
2190/**
2191 * xilinx_dma_prep_peripheral_dma_vec - prepare descriptors for a DMA_SLAVE
2192 *	transaction from DMA vectors
2193 * @dchan: DMA channel
2194 * @vecs: Array of DMA vectors that should be transferred
2195 * @nb: number of entries in @vecs
2196 * @direction: DMA direction
2197 * @flags: transfer ack flags
2198 *
2199 * Return: Async transaction descriptor on success and NULL on failure
2200 */
2201static struct dma_async_tx_descriptor *xilinx_dma_prep_peripheral_dma_vec(
2202	struct dma_chan *dchan, const struct dma_vec *vecs, size_t nb,
2203	enum dma_transfer_direction direction, unsigned long flags)
2204{
2205	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2206	struct xilinx_dma_tx_descriptor *desc;
2207	struct xilinx_axidma_tx_segment *segment, *head, *prev = NULL;
2208	size_t copy;
2209	size_t sg_used;
2210	unsigned int i;
2211
2212	if (!is_slave_direction(direction) || direction != chan->direction)
2213		return NULL;
2214
2215	desc = xilinx_dma_alloc_tx_descriptor(chan);
2216	if (!desc)
2217		return NULL;
2218
2219	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
2220	desc->async_tx.tx_submit = xilinx_dma_tx_submit;
2221
2222	/* Build transactions using information from DMA vectors */
2223	for (i = 0; i < nb; i++) {
2224		sg_used = 0;
2225
2226		/* Loop until the entire dma_vec entry is used */
2227		while (sg_used < vecs[i].len) {
2228			struct xilinx_axidma_desc_hw *hw;
2229
2230			/* Get a free segment */
2231			segment = xilinx_axidma_alloc_tx_segment(chan);
2232			if (!segment)
2233				goto error;
2234
2235			/*
2236			 * Calculate the maximum number of bytes to transfer,
2237			 * making sure it is less than the hw limit
2238			 */
2239			copy = xilinx_dma_calc_copysize(chan, vecs[i].len,
2240					sg_used);
2241			hw = &segment->hw;
2242
2243			/* Fill in the descriptor */
2244			xilinx_axidma_buf(chan, hw, vecs[i].addr, sg_used, 0);
2245			hw->control = copy;
2246
2247			if (prev)
2248				prev->hw.next_desc = segment->phys;
2249
2250			prev = segment;
2251			sg_used += copy;
2252
2253			/*
2254			 * Insert the segment into the descriptor segments
2255			 * list.
2256			 */
2257			list_add_tail(&segment->node, &desc->segments);
2258		}
2259	}
2260
2261	head = list_first_entry(&desc->segments, struct xilinx_axidma_tx_segment, node);
2262	desc->async_tx.phys = head->phys;
2263
2264	/* For the last DMA_MEM_TO_DEV transfer, set EOP */
2265	if (chan->direction == DMA_MEM_TO_DEV) {
2266		segment->hw.control |= XILINX_DMA_BD_SOP;
2267		segment = list_last_entry(&desc->segments,
2268					  struct xilinx_axidma_tx_segment,
2269					  node);
2270		segment->hw.control |= XILINX_DMA_BD_EOP;
2271	}
2272
2273	if (chan->xdev->has_axistream_connected)
2274		desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops;
2275
2276	return &desc->async_tx;
2277
2278error:
2279	xilinx_dma_free_tx_descriptor(chan, desc);
2280	return NULL;
2281}
2282
2283/**
2284 * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
2285 * @dchan: DMA channel
2286 * @sgl: scatterlist to transfer to/from
2287 * @sg_len: number of entries in @scatterlist
2288 * @direction: DMA direction
2289 * @flags: transfer ack flags
2290 * @context: APP words of the descriptor
2291 *
2292 * Return: Async transaction descriptor on success and NULL on failure
2293 */
2294static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg(
2295	struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len,
2296	enum dma_transfer_direction direction, unsigned long flags,
2297	void *context)
2298{
2299	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2300	struct xilinx_dma_tx_descriptor *desc;
2301	struct xilinx_axidma_tx_segment *segment = NULL;
2302	u32 *app_w = (u32 *)context;
2303	struct scatterlist *sg;
2304	size_t copy;
2305	size_t sg_used;
2306	unsigned int i;
2307
2308	if (!is_slave_direction(direction))
2309		return NULL;
2310
2311	/* Allocate a transaction descriptor. */
2312	desc = xilinx_dma_alloc_tx_descriptor(chan);
2313	if (!desc)
2314		return NULL;
2315
2316	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
2317	desc->async_tx.tx_submit = xilinx_dma_tx_submit;
2318
2319	/* Build transactions using information in the scatter gather list */
2320	for_each_sg(sgl, sg, sg_len, i) {
2321		sg_used = 0;
2322
2323		/* Loop until the entire scatterlist entry is used */
2324		while (sg_used < sg_dma_len(sg)) {
2325			struct xilinx_axidma_desc_hw *hw;
2326
2327			/* Get a free segment */
2328			segment = xilinx_axidma_alloc_tx_segment(chan);
2329			if (!segment)
2330				goto error;
2331
2332			/*
2333			 * Calculate the maximum number of bytes to transfer,
2334			 * making sure it is less than the hw limit
2335			 */
2336			copy = xilinx_dma_calc_copysize(chan, sg_dma_len(sg),
2337							sg_used);
2338			hw = &segment->hw;
2339
2340			/* Fill in the descriptor */
2341			xilinx_axidma_buf(chan, hw, sg_dma_address(sg),
2342					  sg_used, 0);
2343
2344			hw->control = copy;
2345
2346			if (chan->direction == DMA_MEM_TO_DEV) {
2347				if (app_w)
2348					memcpy(hw->app, app_w, sizeof(u32) *
2349					       XILINX_DMA_NUM_APP_WORDS);
2350			}
2351
2352			sg_used += copy;
2353
2354			/*
2355			 * Insert the segment into the descriptor segments
2356			 * list.
2357			 */
2358			list_add_tail(&segment->node, &desc->segments);
2359		}
2360	}
2361
2362	segment = list_first_entry(&desc->segments,
2363				   struct xilinx_axidma_tx_segment, node);
2364	desc->async_tx.phys = segment->phys;
2365
2366	/* For the last DMA_MEM_TO_DEV transfer, set EOP */
2367	if (chan->direction == DMA_MEM_TO_DEV) {
2368		segment->hw.control |= XILINX_DMA_BD_SOP;
2369		segment = list_last_entry(&desc->segments,
2370					  struct xilinx_axidma_tx_segment,
2371					  node);
2372		segment->hw.control |= XILINX_DMA_BD_EOP;
2373	}
2374
2375	if (chan->xdev->has_axistream_connected)
2376		desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops;
2377
2378	return &desc->async_tx;
2379
2380error:
2381	xilinx_dma_free_tx_descriptor(chan, desc);
2382	return NULL;
2383}
2384
2385/**
2386 * xilinx_dma_prep_dma_cyclic - prepare descriptors for a DMA_SLAVE transaction
2387 * @dchan: DMA channel
2388 * @buf_addr: Physical address of the buffer
2389 * @buf_len: Total length of the cyclic buffers
2390 * @period_len: length of individual cyclic buffer
2391 * @direction: DMA direction
2392 * @flags: transfer ack flags
2393 *
2394 * Return: Async transaction descriptor on success and NULL on failure
2395 */
2396static struct dma_async_tx_descriptor *xilinx_dma_prep_dma_cyclic(
2397	struct dma_chan *dchan, dma_addr_t buf_addr, size_t buf_len,
2398	size_t period_len, enum dma_transfer_direction direction,
2399	unsigned long flags)
2400{
2401	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2402	struct xilinx_dma_tx_descriptor *desc;
2403	struct xilinx_axidma_tx_segment *segment, *head_segment, *prev = NULL;
2404	size_t copy, sg_used;
2405	unsigned int num_periods;
2406	int i;
2407	u32 reg;
2408
2409	if (!period_len)
2410		return NULL;
2411
2412	num_periods = buf_len / period_len;
2413
2414	if (!num_periods)
2415		return NULL;
2416
2417	if (!is_slave_direction(direction))
2418		return NULL;
2419
2420	/* Allocate a transaction descriptor. */
2421	desc = xilinx_dma_alloc_tx_descriptor(chan);
2422	if (!desc)
2423		return NULL;
2424
2425	chan->direction = direction;
2426	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
2427	desc->async_tx.tx_submit = xilinx_dma_tx_submit;
2428
2429	for (i = 0; i < num_periods; ++i) {
2430		sg_used = 0;
2431
2432		while (sg_used < period_len) {
2433			struct xilinx_axidma_desc_hw *hw;
2434
2435			/* Get a free segment */
2436			segment = xilinx_axidma_alloc_tx_segment(chan);
2437			if (!segment)
2438				goto error;
2439
2440			/*
2441			 * Calculate the maximum number of bytes to transfer,
2442			 * making sure it is less than the hw limit
2443			 */
2444			copy = xilinx_dma_calc_copysize(chan, period_len,
2445							sg_used);
2446			hw = &segment->hw;
2447			xilinx_axidma_buf(chan, hw, buf_addr, sg_used,
2448					  period_len * i);
2449			hw->control = copy;
2450
2451			if (prev)
2452				prev->hw.next_desc = segment->phys;
2453
2454			prev = segment;
2455			sg_used += copy;
2456
2457			/*
2458			 * Insert the segment into the descriptor segments
2459			 * list.
2460			 */
2461			list_add_tail(&segment->node, &desc->segments);
2462		}
2463	}
2464
2465	head_segment = list_first_entry(&desc->segments,
2466				   struct xilinx_axidma_tx_segment, node);
2467	desc->async_tx.phys = head_segment->phys;
2468
2469	desc->cyclic = true;
2470	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);
2471	reg |= XILINX_DMA_CR_CYCLIC_BD_EN_MASK;
2472	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
2473
2474	segment = list_last_entry(&desc->segments,
2475				  struct xilinx_axidma_tx_segment,
2476				  node);
2477	segment->hw.next_desc = (u32) head_segment->phys;
2478
2479	/* For the last DMA_MEM_TO_DEV transfer, set EOP */
2480	if (direction == DMA_MEM_TO_DEV) {
2481		head_segment->hw.control |= XILINX_DMA_BD_SOP;
2482		segment->hw.control |= XILINX_DMA_BD_EOP;
2483	}
2484
2485	return &desc->async_tx;
2486
2487error:
2488	xilinx_dma_free_tx_descriptor(chan, desc);
2489	return NULL;
2490}
2491
2492/**
2493 * xilinx_mcdma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
2494 * @dchan: DMA channel
2495 * @sgl: scatterlist to transfer to/from
2496 * @sg_len: number of entries in @scatterlist
2497 * @direction: DMA direction
2498 * @flags: transfer ack flags
2499 * @context: APP words of the descriptor
2500 *
2501 * Return: Async transaction descriptor on success and NULL on failure
2502 */
2503static struct dma_async_tx_descriptor *
2504xilinx_mcdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
2505			   unsigned int sg_len,
2506			   enum dma_transfer_direction direction,
2507			   unsigned long flags, void *context)
2508{
2509	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2510	struct xilinx_dma_tx_descriptor *desc;
2511	struct xilinx_aximcdma_tx_segment *segment = NULL;
2512	u32 *app_w = (u32 *)context;
2513	struct scatterlist *sg;
2514	size_t copy;
2515	size_t sg_used;
2516	unsigned int i;
2517
2518	if (!is_slave_direction(direction))
2519		return NULL;
2520
2521	/* Allocate a transaction descriptor. */
2522	desc = xilinx_dma_alloc_tx_descriptor(chan);
2523	if (!desc)
2524		return NULL;
2525
2526	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
2527	desc->async_tx.tx_submit = xilinx_dma_tx_submit;
2528
2529	/* Build transactions using information in the scatter gather list */
2530	for_each_sg(sgl, sg, sg_len, i) {
2531		sg_used = 0;
2532
2533		/* Loop until the entire scatterlist entry is used */
2534		while (sg_used < sg_dma_len(sg)) {
2535			struct xilinx_aximcdma_desc_hw *hw;
2536
2537			/* Get a free segment */
2538			segment = xilinx_aximcdma_alloc_tx_segment(chan);
2539			if (!segment)
2540				goto error;
2541
2542			/*
2543			 * Calculate the maximum number of bytes to transfer,
2544			 * making sure it is less than the hw limit
2545			 */
2546			copy = min_t(size_t, sg_dma_len(sg) - sg_used,
2547				     chan->xdev->max_buffer_len);
2548			hw = &segment->hw;
2549
2550			/* Fill in the descriptor */
2551			xilinx_aximcdma_buf(chan, hw, sg_dma_address(sg),
2552					    sg_used);
2553			hw->control = copy;
2554
2555			if (chan->direction == DMA_MEM_TO_DEV && app_w) {
2556				memcpy(hw->app, app_w, sizeof(u32) *
2557				       XILINX_DMA_NUM_APP_WORDS);
2558			}
2559
2560			sg_used += copy;
2561			/*
2562			 * Insert the segment into the descriptor segments
2563			 * list.
2564			 */
2565			list_add_tail(&segment->node, &desc->segments);
2566		}
2567	}
2568
2569	segment = list_first_entry(&desc->segments,
2570				   struct xilinx_aximcdma_tx_segment, node);
2571	desc->async_tx.phys = segment->phys;
2572
2573	/* For the last DMA_MEM_TO_DEV transfer, set EOP */
2574	if (chan->direction == DMA_MEM_TO_DEV) {
2575		segment->hw.control |= XILINX_MCDMA_BD_SOP;
2576		segment = list_last_entry(&desc->segments,
2577					  struct xilinx_aximcdma_tx_segment,
2578					  node);
2579		segment->hw.control |= XILINX_MCDMA_BD_EOP;
2580	}
2581
2582	return &desc->async_tx;
2583
2584error:
2585	xilinx_dma_free_tx_descriptor(chan, desc);
2586
2587	return NULL;
2588}
2589
2590/**
2591 * xilinx_dma_terminate_all - Halt the channel and free descriptors
2592 * @dchan: Driver specific DMA Channel pointer
2593 *
2594 * Return: '0' always.
2595 */
2596static int xilinx_dma_terminate_all(struct dma_chan *dchan)
2597{
2598	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2599	u32 reg;
2600	int err;
2601
2602	if (!chan->cyclic) {
2603		err = chan->stop_transfer(chan);
2604		if (err) {
2605			dev_err(chan->dev, "Cannot stop channel %p: %x\n",
2606				chan, dma_ctrl_read(chan,
2607				XILINX_DMA_REG_DMASR));
2608			chan->err = true;
2609		}
2610	}
2611
2612	xilinx_dma_chan_reset(chan);
2613	/* Remove and free all of the descriptors in the lists */
2614	chan->terminating = true;
2615	xilinx_dma_free_descriptors(chan);
2616	chan->idle = true;
2617
2618	if (chan->cyclic) {
2619		reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);
2620		reg &= ~XILINX_DMA_CR_CYCLIC_BD_EN_MASK;
2621		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
2622		chan->cyclic = false;
2623	}
2624
2625	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)
2626		dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,
2627			     XILINX_CDMA_CR_SGMODE);
2628
2629	return 0;
2630}
2631
2632static void xilinx_dma_synchronize(struct dma_chan *dchan)
2633{
2634	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2635
2636	tasklet_kill(&chan->tasklet);
2637}
2638
2639/**
2640 * xilinx_vdma_channel_set_config - Configure VDMA channel
2641 * Run-time configuration for Axi VDMA, supports:
2642 * . halt the channel
2643 * . configure interrupt coalescing and inter-packet delay threshold
2644 * . start/stop parking
2645 * . enable genlock
2646 *
2647 * @dchan: DMA channel
2648 * @cfg: VDMA device configuration pointer
2649 *
2650 * Return: '0' on success and failure value on error
2651 */
2652int xilinx_vdma_channel_set_config(struct dma_chan *dchan,
2653					struct xilinx_vdma_config *cfg)
2654{
2655	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);
2656	u32 dmacr;
2657
2658	if (cfg->reset)
2659		return xilinx_dma_chan_reset(chan);
2660
2661	dmacr = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);
2662
2663	chan->config.frm_dly = cfg->frm_dly;
2664	chan->config.park = cfg->park;
2665
2666	/* genlock settings */
2667	chan->config.gen_lock = cfg->gen_lock;
2668	chan->config.master = cfg->master;
2669
2670	dmacr &= ~XILINX_DMA_DMACR_GENLOCK_EN;
2671	if (cfg->gen_lock && chan->genlock) {
2672		dmacr |= XILINX_DMA_DMACR_GENLOCK_EN;
2673		dmacr &= ~XILINX_DMA_DMACR_MASTER_MASK;
2674		dmacr |= cfg->master << XILINX_DMA_DMACR_MASTER_SHIFT;
2675	}
2676
2677	chan->config.frm_cnt_en = cfg->frm_cnt_en;
2678	chan->config.vflip_en = cfg->vflip_en;
2679
2680	if (cfg->park)
2681		chan->config.park_frm = cfg->park_frm;
2682	else
2683		chan->config.park_frm = -1;
2684
2685	chan->config.coalesc = cfg->coalesc;
2686	chan->config.delay = cfg->delay;
2687
2688	if (cfg->coalesc <= XILINX_DMA_DMACR_FRAME_COUNT_MAX) {
2689		dmacr &= ~XILINX_DMA_DMACR_FRAME_COUNT_MASK;
2690		dmacr |= cfg->coalesc << XILINX_DMA_DMACR_FRAME_COUNT_SHIFT;
2691		chan->config.coalesc = cfg->coalesc;
2692	}
2693
2694	if (cfg->delay <= XILINX_DMA_DMACR_DELAY_MAX) {
2695		dmacr &= ~XILINX_DMA_DMACR_DELAY_MASK;
2696		dmacr |= cfg->delay << XILINX_DMA_DMACR_DELAY_SHIFT;
2697		chan->config.delay = cfg->delay;
2698	}
2699
2700	/* FSync Source selection */
2701	dmacr &= ~XILINX_DMA_DMACR_FSYNCSRC_MASK;
2702	dmacr |= cfg->ext_fsync << XILINX_DMA_DMACR_FSYNCSRC_SHIFT;
2703
2704	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, dmacr);
2705
2706	return 0;
2707}
2708EXPORT_SYMBOL(xilinx_vdma_channel_set_config);
2709
2710/* -----------------------------------------------------------------------------
2711 * Probe and remove
2712 */
2713
2714/**
2715 * xilinx_dma_chan_remove - Per Channel remove function
2716 * @chan: Driver specific DMA channel
2717 */
2718static void xilinx_dma_chan_remove(struct xilinx_dma_chan *chan)
2719{
2720	/* Disable all interrupts */
2721	dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,
2722		      XILINX_DMA_DMAXR_ALL_IRQ_MASK);
2723
2724	if (chan->irq > 0)
2725		free_irq(chan->irq, chan);
2726
2727	tasklet_kill(&chan->tasklet);
2728
2729	list_del(&chan->common.device_node);
2730}
2731
2732static int axidma_clk_init(struct platform_device *pdev, struct clk **axi_clk,
2733			    struct clk **tx_clk, struct clk **rx_clk,
2734			    struct clk **sg_clk, struct clk **tmp_clk)
2735{
2736	int err;
2737
2738	*tmp_clk = NULL;
2739
2740	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
2741	if (IS_ERR(*axi_clk))
2742		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");
2743
2744	*tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk");
2745	if (IS_ERR(*tx_clk))
2746		*tx_clk = NULL;
2747
2748	*rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk");
2749	if (IS_ERR(*rx_clk))
2750		*rx_clk = NULL;
2751
2752	*sg_clk = devm_clk_get(&pdev->dev, "m_axi_sg_aclk");
2753	if (IS_ERR(*sg_clk))
2754		*sg_clk = NULL;
2755
2756	err = clk_prepare_enable(*axi_clk);
2757	if (err) {
2758		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err);
2759		return err;
2760	}
2761
2762	err = clk_prepare_enable(*tx_clk);
2763	if (err) {
2764		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);
2765		goto err_disable_axiclk;
2766	}
2767
2768	err = clk_prepare_enable(*rx_clk);
2769	if (err) {
2770		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);
2771		goto err_disable_txclk;
2772	}
2773
2774	err = clk_prepare_enable(*sg_clk);
2775	if (err) {
2776		dev_err(&pdev->dev, "failed to enable sg_clk (%d)\n", err);
2777		goto err_disable_rxclk;
2778	}
2779
2780	return 0;
2781
2782err_disable_rxclk:
2783	clk_disable_unprepare(*rx_clk);
2784err_disable_txclk:
2785	clk_disable_unprepare(*tx_clk);
2786err_disable_axiclk:
2787	clk_disable_unprepare(*axi_clk);
2788
2789	return err;
2790}
2791
2792static int axicdma_clk_init(struct platform_device *pdev, struct clk **axi_clk,
2793			    struct clk **dev_clk, struct clk **tmp_clk,
2794			    struct clk **tmp1_clk, struct clk **tmp2_clk)
2795{
2796	int err;
2797
2798	*tmp_clk = NULL;
2799	*tmp1_clk = NULL;
2800	*tmp2_clk = NULL;
2801
2802	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
2803	if (IS_ERR(*axi_clk))
2804		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");
2805
2806	*dev_clk = devm_clk_get(&pdev->dev, "m_axi_aclk");
2807	if (IS_ERR(*dev_clk))
2808		return dev_err_probe(&pdev->dev, PTR_ERR(*dev_clk), "failed to get dev_clk\n");
2809
2810	err = clk_prepare_enable(*axi_clk);
2811	if (err) {
2812		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err);
2813		return err;
2814	}
2815
2816	err = clk_prepare_enable(*dev_clk);
2817	if (err) {
2818		dev_err(&pdev->dev, "failed to enable dev_clk (%d)\n", err);
2819		goto err_disable_axiclk;
2820	}
2821
2822	return 0;
2823
2824err_disable_axiclk:
2825	clk_disable_unprepare(*axi_clk);
2826
2827	return err;
2828}
2829
2830static int axivdma_clk_init(struct platform_device *pdev, struct clk **axi_clk,
2831			    struct clk **tx_clk, struct clk **txs_clk,
2832			    struct clk **rx_clk, struct clk **rxs_clk)
2833{
2834	int err;
2835
2836	*axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk");
2837	if (IS_ERR(*axi_clk))
2838		return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n");
2839
2840	*tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk");
2841	if (IS_ERR(*tx_clk))
2842		*tx_clk = NULL;
2843
2844	*txs_clk = devm_clk_get(&pdev->dev, "m_axis_mm2s_aclk");
2845	if (IS_ERR(*txs_clk))
2846		*txs_clk = NULL;
2847
2848	*rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk");
2849	if (IS_ERR(*rx_clk))
2850		*rx_clk = NULL;
2851
2852	*rxs_clk = devm_clk_get(&pdev->dev, "s_axis_s2mm_aclk");
2853	if (IS_ERR(*rxs_clk))
2854		*rxs_clk = NULL;
2855
2856	err = clk_prepare_enable(*axi_clk);
2857	if (err) {
2858		dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n",
2859			err);
2860		return err;
2861	}
2862
2863	err = clk_prepare_enable(*tx_clk);
2864	if (err) {
2865		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);
2866		goto err_disable_axiclk;
2867	}
2868
2869	err = clk_prepare_enable(*txs_clk);
2870	if (err) {
2871		dev_err(&pdev->dev, "failed to enable txs_clk (%d)\n", err);
2872		goto err_disable_txclk;
2873	}
2874
2875	err = clk_prepare_enable(*rx_clk);
2876	if (err) {
2877		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);
2878		goto err_disable_txsclk;
2879	}
2880
2881	err = clk_prepare_enable(*rxs_clk);
2882	if (err) {
2883		dev_err(&pdev->dev, "failed to enable rxs_clk (%d)\n", err);
2884		goto err_disable_rxclk;
2885	}
2886
2887	return 0;
2888
2889err_disable_rxclk:
2890	clk_disable_unprepare(*rx_clk);
2891err_disable_txsclk:
2892	clk_disable_unprepare(*txs_clk);
2893err_disable_txclk:
2894	clk_disable_unprepare(*tx_clk);
2895err_disable_axiclk:
2896	clk_disable_unprepare(*axi_clk);
2897
2898	return err;
2899}
2900
2901static void xdma_disable_allclks(struct xilinx_dma_device *xdev)
2902{
2903	clk_disable_unprepare(xdev->rxs_clk);
2904	clk_disable_unprepare(xdev->rx_clk);
2905	clk_disable_unprepare(xdev->txs_clk);
2906	clk_disable_unprepare(xdev->tx_clk);
2907	clk_disable_unprepare(xdev->axi_clk);
2908}
2909
2910/**
2911 * xilinx_dma_chan_probe - Per Channel Probing
2912 * It get channel features from the device tree entry and
2913 * initialize special channel handling routines
2914 *
2915 * @xdev: Driver specific device structure
2916 * @node: Device node
2917 *
2918 * Return: '0' on success and failure value on error
2919 */
2920static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,
2921				  struct device_node *node)
2922{
2923	struct xilinx_dma_chan *chan;
2924	bool has_dre = false;
2925	u32 value, width;
2926	int err;
2927
2928	/* Allocate and initialize the channel structure */
2929	chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL);
2930	if (!chan)
2931		return -ENOMEM;
2932
2933	chan->dev = xdev->dev;
2934	chan->xdev = xdev;
2935	chan->desc_pendingcount = 0x0;
2936	chan->ext_addr = xdev->ext_addr;
2937	/* This variable ensures that descriptors are not
2938	 * Submitted when dma engine is in progress. This variable is
2939	 * Added to avoid polling for a bit in the status register to
2940	 * Know dma state in the driver hot path.
2941	 */
2942	chan->idle = true;
2943
2944	spin_lock_init(&chan->lock);
2945	INIT_LIST_HEAD(&chan->pending_list);
2946	INIT_LIST_HEAD(&chan->done_list);
2947	INIT_LIST_HEAD(&chan->active_list);
2948	INIT_LIST_HEAD(&chan->free_seg_list);
2949
2950	/* Retrieve the channel properties from the device tree */
2951	has_dre = of_property_read_bool(node, "xlnx,include-dre");
2952
2953	of_property_read_u8(node, "xlnx,irq-delay", &chan->irq_delay);
2954
2955	chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode");
2956
2957	err = of_property_read_u32(node, "xlnx,datawidth", &value);
2958	if (err) {
2959		dev_err(xdev->dev, "missing xlnx,datawidth property\n");
2960		return err;
2961	}
2962	width = value >> 3; /* Convert bits to bytes */
2963
2964	/* If data width is greater than 8 bytes, DRE is not in hw */
2965	if (width > 8)
2966		has_dre = false;
2967
2968	if (!has_dre)
2969		xdev->common.copy_align = (enum dmaengine_alignment)fls(width - 1);
2970
2971	if (of_device_is_compatible(node, "xlnx,axi-vdma-mm2s-channel") ||
2972	    of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") ||
2973	    of_device_is_compatible(node, "xlnx,axi-cdma-channel")) {
2974		chan->direction = DMA_MEM_TO_DEV;
2975		chan->id = xdev->mm2s_chan_id++;
2976		chan->tdest = chan->id;
2977
2978		chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET;
2979		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
2980			chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET;
2981			chan->config.park = 1;
2982
2983			if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH ||
2984			    xdev->flush_on_fsync == XILINX_DMA_FLUSH_MM2S)
2985				chan->flush_on_fsync = true;
2986		}
2987	} else if (of_device_is_compatible(node,
2988					   "xlnx,axi-vdma-s2mm-channel") ||
2989		   of_device_is_compatible(node,
2990					   "xlnx,axi-dma-s2mm-channel")) {
2991		chan->direction = DMA_DEV_TO_MEM;
2992		chan->id = xdev->s2mm_chan_id++;
2993		chan->tdest = chan->id - xdev->dma_config->max_channels / 2;
2994		chan->has_vflip = of_property_read_bool(node,
2995					"xlnx,enable-vert-flip");
2996		if (chan->has_vflip) {
2997			chan->config.vflip_en = dma_read(chan,
2998				XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP) &
2999				XILINX_VDMA_ENABLE_VERTICAL_FLIP;
3000		}
3001
3002		if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA)
3003			chan->ctrl_offset = XILINX_MCDMA_S2MM_CTRL_OFFSET;
3004		else
3005			chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET;
3006
3007		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
3008			chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET;
3009			chan->config.park = 1;
3010
3011			if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH ||
3012			    xdev->flush_on_fsync == XILINX_DMA_FLUSH_S2MM)
3013				chan->flush_on_fsync = true;
3014		}
3015	} else {
3016		dev_err(xdev->dev, "Invalid channel compatible node\n");
3017		return -EINVAL;
3018	}
3019
3020	xdev->common.directions |= BIT(chan->direction);
3021
3022	/* Request the interrupt */
3023	chan->irq = of_irq_get(node, chan->tdest);
3024	if (chan->irq < 0)
3025		return dev_err_probe(xdev->dev, chan->irq, "failed to get irq\n");
3026	err = request_irq(chan->irq, xdev->dma_config->irq_handler,
3027			  IRQF_SHARED, "xilinx-dma-controller", chan);
3028	if (err) {
3029		dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq);
3030		return err;
3031	}
3032
3033	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
3034		chan->start_transfer = xilinx_dma_start_transfer;
3035		chan->stop_transfer = xilinx_dma_stop_transfer;
3036	} else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
3037		chan->start_transfer = xilinx_mcdma_start_transfer;
3038		chan->stop_transfer = xilinx_dma_stop_transfer;
3039	} else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
3040		chan->start_transfer = xilinx_cdma_start_transfer;
3041		chan->stop_transfer = xilinx_cdma_stop_transfer;
3042	} else {
3043		chan->start_transfer = xilinx_vdma_start_transfer;
3044		chan->stop_transfer = xilinx_dma_stop_transfer;
3045	}
3046
3047	/* check if SG is enabled (only for AXIDMA, AXIMCDMA, and CDMA) */
3048	if (xdev->dma_config->dmatype != XDMA_TYPE_VDMA) {
3049		if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA ||
3050		    dma_ctrl_read(chan, XILINX_DMA_REG_DMASR) &
3051			    XILINX_DMA_DMASR_SG_MASK)
3052			chan->has_sg = true;
3053		dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id,
3054			str_enabled_disabled(chan->has_sg));
3055	}
3056
3057	/* Initialize the tasklet */
3058	tasklet_setup(&chan->tasklet, xilinx_dma_do_tasklet);
3059
3060	/*
3061	 * Initialize the DMA channel and add it to the DMA engine channels
3062	 * list.
3063	 */
3064	chan->common.device = &xdev->common;
3065
3066	list_add_tail(&chan->common.device_node, &xdev->common.channels);
3067	xdev->chan[chan->id] = chan;
3068
3069	/* Reset the channel */
3070	err = xilinx_dma_chan_reset(chan);
3071	if (err < 0) {
3072		dev_err(xdev->dev, "Reset channel failed\n");
3073		return err;
3074	}
3075
3076	return 0;
3077}
3078
3079/**
3080 * xilinx_dma_child_probe - Per child node probe
3081 * It get number of dma-channels per child node from
3082 * device-tree and initializes all the channels.
3083 *
3084 * @xdev: Driver specific device structure
3085 * @node: Device node
3086 *
3087 * Return: '0' on success and failure value on error.
3088 */
3089static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev,
3090				    struct device_node *node)
3091{
3092	int ret, i;
3093	u32 nr_channels = 1;
3094
3095	ret = of_property_read_u32(node, "dma-channels", &nr_channels);
3096	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA && ret < 0)
3097		dev_warn(xdev->dev, "missing dma-channels property\n");
3098
3099	for (i = 0; i < nr_channels; i++) {
3100		ret = xilinx_dma_chan_probe(xdev, node);
3101		if (ret)
3102			return ret;
3103	}
3104
3105	return 0;
3106}
3107
3108/**
3109 * of_dma_xilinx_xlate - Translation function
3110 * @dma_spec: Pointer to DMA specifier as found in the device tree
3111 * @ofdma: Pointer to DMA controller data
3112 *
3113 * Return: DMA channel pointer on success and NULL on error
3114 */
3115static struct dma_chan *of_dma_xilinx_xlate(struct of_phandle_args *dma_spec,
3116						struct of_dma *ofdma)
3117{
3118	struct xilinx_dma_device *xdev = ofdma->of_dma_data;
3119	int chan_id = dma_spec->args[0];
3120
3121	if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id])
3122		return NULL;
3123
3124	return dma_get_slave_channel(&xdev->chan[chan_id]->common);
3125}
3126
3127static const struct xilinx_dma_config axidma_config = {
3128	.dmatype = XDMA_TYPE_AXIDMA,
3129	.clk_init = axidma_clk_init,
3130	.irq_handler = xilinx_dma_irq_handler,
3131	.max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE,
3132};
3133
3134static const struct xilinx_dma_config aximcdma_config = {
3135	.dmatype = XDMA_TYPE_AXIMCDMA,
3136	.clk_init = axidma_clk_init,
3137	.irq_handler = xilinx_mcdma_irq_handler,
3138	.max_channels = XILINX_MCDMA_MAX_CHANS_PER_DEVICE,
3139};
3140static const struct xilinx_dma_config axicdma_config = {
3141	.dmatype = XDMA_TYPE_CDMA,
3142	.clk_init = axicdma_clk_init,
3143	.irq_handler = xilinx_dma_irq_handler,
3144	.max_channels = XILINX_CDMA_MAX_CHANS_PER_DEVICE,
3145};
3146
3147static const struct xilinx_dma_config axivdma_config = {
3148	.dmatype = XDMA_TYPE_VDMA,
3149	.clk_init = axivdma_clk_init,
3150	.irq_handler = xilinx_dma_irq_handler,
3151	.max_channels = XILINX_DMA_MAX_CHANS_PER_DEVICE,
3152};
3153
3154static const struct of_device_id xilinx_dma_of_ids[] = {
3155	{ .compatible = "xlnx,axi-dma-1.00.a", .data = &axidma_config },
3156	{ .compatible = "xlnx,axi-cdma-1.00.a", .data = &axicdma_config },
3157	{ .compatible = "xlnx,axi-vdma-1.00.a", .data = &axivdma_config },
3158	{ .compatible = "xlnx,axi-mcdma-1.00.a", .data = &aximcdma_config },
3159	{}
3160};
3161MODULE_DEVICE_TABLE(of, xilinx_dma_of_ids);
3162
3163/**
3164 * xilinx_dma_probe - Driver probe function
3165 * @pdev: Pointer to the platform_device structure
3166 *
3167 * Return: '0' on success and failure value on error
3168 */
3169static int xilinx_dma_probe(struct platform_device *pdev)
3170{
3171	int (*clk_init)(struct platform_device *, struct clk **, struct clk **,
3172			struct clk **, struct clk **, struct clk **)
3173					= axivdma_clk_init;
3174	struct device_node *node = pdev->dev.of_node;
3175	struct xilinx_dma_device *xdev;
3176	struct device_node *child, *np = pdev->dev.of_node;
3177	u32 num_frames, addr_width = XILINX_DMA_DFAULT_ADDRWIDTH, len_width;
3178	int i, err;
3179
3180	/* Allocate and initialize the DMA engine structure */
3181	xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);
3182	if (!xdev)
3183		return -ENOMEM;
3184
3185	xdev->dev = &pdev->dev;
3186	if (np) {
3187		const struct of_device_id *match;
3188
3189		match = of_match_node(xilinx_dma_of_ids, np);
3190		if (match && match->data) {
3191			xdev->dma_config = match->data;
3192			clk_init = xdev->dma_config->clk_init;
3193		}
3194	}
3195
3196	err = clk_init(pdev, &xdev->axi_clk, &xdev->tx_clk, &xdev->txs_clk,
3197		       &xdev->rx_clk, &xdev->rxs_clk);
3198	if (err)
3199		return err;
3200
3201	/* Request and map I/O memory */
3202	xdev->regs = devm_platform_ioremap_resource(pdev, 0);
3203	if (IS_ERR(xdev->regs)) {
3204		err = PTR_ERR(xdev->regs);
3205		goto disable_clks;
3206	}
3207	/* Retrieve the DMA engine properties from the device tree */
3208	xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0);
3209	xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2;
3210
3211	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA ||
3212	    xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
3213		if (!of_property_read_u32(node, "xlnx,sg-length-width",
3214					  &len_width)) {
3215			if (len_width < XILINX_DMA_MAX_TRANS_LEN_MIN ||
3216			    len_width > XILINX_DMA_V2_MAX_TRANS_LEN_MAX) {
3217				dev_warn(xdev->dev,
3218					 "invalid xlnx,sg-length-width property value. Using default width\n");
3219			} else {
3220				if (len_width > XILINX_DMA_MAX_TRANS_LEN_MAX)
3221					dev_warn(xdev->dev, "Please ensure that IP supports buffer length > 23 bits\n");
3222				xdev->max_buffer_len =
3223					GENMASK(len_width - 1, 0);
3224			}
3225		}
3226	}
3227
3228	dma_set_max_seg_size(xdev->dev, xdev->max_buffer_len);
3229
3230	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
3231		xdev->has_axistream_connected =
3232			of_property_read_bool(node, "xlnx,axistream-connected");
3233	}
3234
3235	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
3236		err = of_property_read_u32(node, "xlnx,num-fstores",
3237					   &num_frames);
3238		if (err < 0) {
3239			dev_err(xdev->dev,
3240				"missing xlnx,num-fstores property\n");
3241			goto disable_clks;
3242		}
3243
3244		err = of_property_read_u32(node, "xlnx,flush-fsync",
3245					   &xdev->flush_on_fsync);
3246		if (err < 0)
3247			dev_warn(xdev->dev,
3248				 "missing xlnx,flush-fsync property\n");
3249	}
3250
3251	err = of_property_read_u32(node, "xlnx,addrwidth", &addr_width);
3252	if (err < 0)
3253		dev_warn(xdev->dev,
3254			 "missing xlnx,addrwidth property, using default value %d\n",
3255			 XILINX_DMA_DFAULT_ADDRWIDTH);
3256
3257	if (addr_width > 32)
3258		xdev->ext_addr = true;
3259	else
3260		xdev->ext_addr = false;
3261
3262	/* Set metadata mode */
3263	if (xdev->has_axistream_connected)
3264		xdev->common.desc_metadata_modes = DESC_METADATA_ENGINE;
3265
3266	/* Set the dma mask bits */
3267	err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width));
3268	if (err < 0) {
3269		dev_err(xdev->dev, "DMA mask error %d\n", err);
3270		goto disable_clks;
3271	}
3272
3273	/* Initialize the DMA engine */
3274	xdev->common.dev = &pdev->dev;
3275
3276	INIT_LIST_HEAD(&xdev->common.channels);
3277	if (!(xdev->dma_config->dmatype == XDMA_TYPE_CDMA)) {
3278		dma_cap_set(DMA_SLAVE, xdev->common.cap_mask);
3279		dma_cap_set(DMA_PRIVATE, xdev->common.cap_mask);
3280	}
3281
3282	xdev->common.device_alloc_chan_resources =
3283				xilinx_dma_alloc_chan_resources;
3284	xdev->common.device_free_chan_resources =
3285				xilinx_dma_free_chan_resources;
3286	xdev->common.device_terminate_all = xilinx_dma_terminate_all;
3287	xdev->common.device_synchronize = xilinx_dma_synchronize;
3288	xdev->common.device_tx_status = xilinx_dma_tx_status;
3289	xdev->common.device_issue_pending = xilinx_dma_issue_pending;
3290	xdev->common.device_config = xilinx_dma_device_config;
3291	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {
3292		dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask);
3293		xdev->common.device_prep_peripheral_dma_vec = xilinx_dma_prep_peripheral_dma_vec;
3294		xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg;
3295		xdev->common.device_prep_dma_cyclic =
3296					  xilinx_dma_prep_dma_cyclic;
3297		/* Residue calculation is supported by only AXI DMA and CDMA */
3298		xdev->common.residue_granularity =
3299					  DMA_RESIDUE_GRANULARITY_SEGMENT;
3300	} else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {
3301		dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask);
3302		xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy;
3303		/* Residue calculation is supported by only AXI DMA and CDMA */
3304		xdev->common.residue_granularity =
3305					  DMA_RESIDUE_GRANULARITY_SEGMENT;
3306	} else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {
3307		xdev->common.device_prep_slave_sg = xilinx_mcdma_prep_slave_sg;
3308	} else {
3309		xdev->common.device_prep_interleaved_dma =
3310				xilinx_vdma_dma_prep_interleaved;
3311	}
3312
3313	platform_set_drvdata(pdev, xdev);
3314
3315	/* Initialize the channels */
3316	for_each_child_of_node(node, child) {
3317		err = xilinx_dma_child_probe(xdev, child);
3318		if (err < 0) {
3319			of_node_put(child);
3320			goto error;
3321		}
3322	}
3323
3324	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {
3325		for (i = 0; i < xdev->dma_config->max_channels; i++)
3326			if (xdev->chan[i])
3327				xdev->chan[i]->num_frms = num_frames;
3328	}
3329
3330	/* Register the DMA engine with the core */
3331	err = dma_async_device_register(&xdev->common);
3332	if (err) {
3333		dev_err(xdev->dev, "failed to register the dma device\n");
3334		goto error;
3335	}
3336
3337	err = of_dma_controller_register(node, of_dma_xilinx_xlate,
3338					 xdev);
3339	if (err < 0) {
3340		dev_err(&pdev->dev, "Unable to register DMA to DT\n");
3341		dma_async_device_unregister(&xdev->common);
3342		goto error;
3343	}
3344
3345	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA)
3346		dev_info(&pdev->dev, "Xilinx AXI DMA Engine Driver Probed!!\n");
3347	else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA)
3348		dev_info(&pdev->dev, "Xilinx AXI CDMA Engine Driver Probed!!\n");
3349	else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA)
3350		dev_info(&pdev->dev, "Xilinx AXI MCDMA Engine Driver Probed!!\n");
3351	else
3352		dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n");
3353
3354	return 0;
3355
3356error:
3357	for (i = 0; i < xdev->dma_config->max_channels; i++)
3358		if (xdev->chan[i])
3359			xilinx_dma_chan_remove(xdev->chan[i]);
3360disable_clks:
3361	xdma_disable_allclks(xdev);
3362
3363	return err;
3364}
3365
3366/**
3367 * xilinx_dma_remove - Driver remove function
3368 * @pdev: Pointer to the platform_device structure
3369 */
3370static void xilinx_dma_remove(struct platform_device *pdev)
3371{
3372	struct xilinx_dma_device *xdev = platform_get_drvdata(pdev);
3373	int i;
3374
3375	of_dma_controller_free(pdev->dev.of_node);
3376
3377	dma_async_device_unregister(&xdev->common);
3378
3379	for (i = 0; i < xdev->dma_config->max_channels; i++)
3380		if (xdev->chan[i])
3381			xilinx_dma_chan_remove(xdev->chan[i]);
3382
3383	xdma_disable_allclks(xdev);
3384}
3385
3386static struct platform_driver xilinx_vdma_driver = {
3387	.driver = {
3388		.name = "xilinx-vdma",
3389		.of_match_table = xilinx_dma_of_ids,
3390	},
3391	.probe = xilinx_dma_probe,
3392	.remove = xilinx_dma_remove,
3393};
3394
3395module_platform_driver(xilinx_vdma_driver);
3396
3397MODULE_AUTHOR("Xilinx, Inc.");
3398MODULE_DESCRIPTION("Xilinx VDMA driver");
3399MODULE_LICENSE("GPL v2");