tsi721_dma.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * DMA Engine support for Tsi721 PCIExpress-to-SRIO bridge
   4 *
   5 * Copyright (c) 2011-2014 Integrated Device Technology, Inc.
   6 * Alexandre Bounine <alexandre.bounine@idt.com>
   7 */
   8
   9#include <linux/io.h>
  10#include <linux/errno.h>
  11#include <linux/init.h>
  12#include <linux/ioport.h>
  13#include <linux/kernel.h>
  14#include <linux/module.h>
  15#include <linux/pci.h>
  16#include <linux/rio.h>
  17#include <linux/rio_drv.h>
  18#include <linux/dma-mapping.h>
  19#include <linux/interrupt.h>
  20#include <linux/kfifo.h>
  21#include <linux/sched.h>
  22#include <linux/delay.h>
  23#include "../../dma/dmaengine.h"
  24
  25#include "tsi721.h"
  26
  27#ifdef CONFIG_PCI_MSI
  28static irqreturn_t tsi721_bdma_msix(int irq, void *ptr);
  29#endif
  30static int tsi721_submit_sg(struct tsi721_tx_desc *desc);
  31
  32static unsigned int dma_desc_per_channel = 128;
  33module_param(dma_desc_per_channel, uint, S_IRUGO);
  34MODULE_PARM_DESC(dma_desc_per_channel,
  35		 "Number of DMA descriptors per channel (default: 128)");
  36
  37static unsigned int dma_txqueue_sz = 16;
  38module_param(dma_txqueue_sz, uint, S_IRUGO);
  39MODULE_PARM_DESC(dma_txqueue_sz,
  40		 "DMA Transactions Queue Size (default: 16)");
  41
  42static u8 dma_sel = 0x7f;
  43module_param(dma_sel, byte, S_IRUGO);
  44MODULE_PARM_DESC(dma_sel,
  45		 "DMA Channel Selection Mask (default: 0x7f = all)");
  46
  47static inline struct tsi721_bdma_chan *to_tsi721_chan(struct dma_chan *chan)
  48{
  49	return container_of(chan, struct tsi721_bdma_chan, dchan);
  50}
  51
  52static inline struct tsi721_device *to_tsi721(struct dma_device *ddev)
  53{
  54	return container_of(ddev, struct rio_mport, dma)->priv;
  55}
  56
  57static inline
  58struct tsi721_tx_desc *to_tsi721_desc(struct dma_async_tx_descriptor *txd)
  59{
  60	return container_of(txd, struct tsi721_tx_desc, txd);
  61}
  62
  63static int tsi721_bdma_ch_init(struct tsi721_bdma_chan *bdma_chan, int bd_num)
  64{
  65	struct tsi721_dma_desc *bd_ptr;
  66	struct device *dev = bdma_chan->dchan.device->dev;
  67	u64		*sts_ptr;
  68	dma_addr_t	bd_phys;
  69	dma_addr_t	sts_phys;
  70	int		sts_size;
  71#ifdef CONFIG_PCI_MSI
  72	struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
  73#endif
  74
  75	tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id);
  76
  77	/*
  78	 * Allocate space for DMA descriptors
  79	 * (add an extra element for link descriptor)
  80	 */
  81	bd_ptr = dma_alloc_coherent(dev,
  82				    (bd_num + 1) * sizeof(struct tsi721_dma_desc),
  83				    &bd_phys, GFP_ATOMIC);
  84	if (!bd_ptr)
  85		return -ENOMEM;
  86
  87	bdma_chan->bd_num = bd_num;
  88	bdma_chan->bd_phys = bd_phys;
  89	bdma_chan->bd_base = bd_ptr;
  90
  91	tsi_debug(DMA, &bdma_chan->dchan.dev->device,
  92		  "DMAC%d descriptors @ %p (phys = %pad)",
  93		  bdma_chan->id, bd_ptr, &bd_phys);
  94
  95	/* Allocate space for descriptor status FIFO */
  96	sts_size = ((bd_num + 1) >= TSI721_DMA_MINSTSSZ) ?
  97					(bd_num + 1) : TSI721_DMA_MINSTSSZ;
  98	sts_size = roundup_pow_of_two(sts_size);
  99	sts_ptr = dma_alloc_coherent(dev,
 100				     sts_size * sizeof(struct tsi721_dma_sts),
 101				     &sts_phys, GFP_ATOMIC);
 102	if (!sts_ptr) {
 103		/* Free space allocated for DMA descriptors */
 104		dma_free_coherent(dev,
 105				  (bd_num + 1) * sizeof(struct tsi721_dma_desc),
 106				  bd_ptr, bd_phys);
 107		bdma_chan->bd_base = NULL;
 108		return -ENOMEM;
 109	}
 110
 111	bdma_chan->sts_phys = sts_phys;
 112	bdma_chan->sts_base = sts_ptr;
 113	bdma_chan->sts_size = sts_size;
 114
 115	tsi_debug(DMA, &bdma_chan->dchan.dev->device,
 116		"DMAC%d desc status FIFO @ %p (phys = %pad) size=0x%x",
 117		bdma_chan->id, sts_ptr, &sts_phys, sts_size);
 118
 119	/* Initialize DMA descriptors ring using added link descriptor */
 120	bd_ptr[bd_num].type_id = cpu_to_le32(DTYPE3 << 29);
 121	bd_ptr[bd_num].next_lo = cpu_to_le32((u64)bd_phys &
 122						 TSI721_DMAC_DPTRL_MASK);
 123	bd_ptr[bd_num].next_hi = cpu_to_le32((u64)bd_phys >> 32);
 124
 125	/* Setup DMA descriptor pointers */
 126	iowrite32(((u64)bd_phys >> 32),
 127		bdma_chan->regs + TSI721_DMAC_DPTRH);
 128	iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK),
 129		bdma_chan->regs + TSI721_DMAC_DPTRL);
 130
 131	/* Setup descriptor status FIFO */
 132	iowrite32(((u64)sts_phys >> 32),
 133		bdma_chan->regs + TSI721_DMAC_DSBH);
 134	iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK),
 135		bdma_chan->regs + TSI721_DMAC_DSBL);
 136	iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size),
 137		bdma_chan->regs + TSI721_DMAC_DSSZ);
 138
 139	/* Clear interrupt bits */
 140	iowrite32(TSI721_DMAC_INT_ALL,
 141		bdma_chan->regs + TSI721_DMAC_INT);
 142
 143	ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 144
 145#ifdef CONFIG_PCI_MSI
 146	/* Request interrupt service if we are in MSI-X mode */
 147	if (priv->flags & TSI721_USING_MSIX) {
 148		int rc, idx;
 149
 150		idx = TSI721_VECT_DMA0_DONE + bdma_chan->id;
 151
 152		rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0,
 153				 priv->msix[idx].irq_name, (void *)bdma_chan);
 154
 155		if (rc) {
 156			tsi_debug(DMA, &bdma_chan->dchan.dev->device,
 157				  "Unable to get MSI-X for DMAC%d-DONE",
 158				  bdma_chan->id);
 159			goto err_out;
 160		}
 161
 162		idx = TSI721_VECT_DMA0_INT + bdma_chan->id;
 163
 164		rc = request_irq(priv->msix[idx].vector, tsi721_bdma_msix, 0,
 165				priv->msix[idx].irq_name, (void *)bdma_chan);
 166
 167		if (rc)	{
 168			tsi_debug(DMA, &bdma_chan->dchan.dev->device,
 169				  "Unable to get MSI-X for DMAC%d-INT",
 170				  bdma_chan->id);
 171			free_irq(
 172				priv->msix[TSI721_VECT_DMA0_DONE +
 173					    bdma_chan->id].vector,
 174				(void *)bdma_chan);
 175		}
 176
 177err_out:
 178		if (rc) {
 179			/* Free space allocated for DMA descriptors */
 180			dma_free_coherent(dev,
 181				(bd_num + 1) * sizeof(struct tsi721_dma_desc),
 182				bd_ptr, bd_phys);
 183			bdma_chan->bd_base = NULL;
 184
 185			/* Free space allocated for status descriptors */
 186			dma_free_coherent(dev,
 187				sts_size * sizeof(struct tsi721_dma_sts),
 188				sts_ptr, sts_phys);
 189			bdma_chan->sts_base = NULL;
 190
 191			return -EIO;
 192		}
 193	}
 194#endif /* CONFIG_PCI_MSI */
 195
 196	/* Toggle DMA channel initialization */
 197	iowrite32(TSI721_DMAC_CTL_INIT,	bdma_chan->regs + TSI721_DMAC_CTL);
 198	ioread32(bdma_chan->regs + TSI721_DMAC_CTL);
 199	bdma_chan->wr_count = bdma_chan->wr_count_next = 0;
 200	bdma_chan->sts_rdptr = 0;
 201	udelay(10);
 202
 203	return 0;
 204}
 205
 206static int tsi721_bdma_ch_free(struct tsi721_bdma_chan *bdma_chan)
 207{
 208	u32 ch_stat;
 209#ifdef CONFIG_PCI_MSI
 210	struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
 211#endif
 212
 213	if (!bdma_chan->bd_base)
 214		return 0;
 215
 216	/* Check if DMA channel still running */
 217	ch_stat = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
 218	if (ch_stat & TSI721_DMAC_STS_RUN)
 219		return -EFAULT;
 220
 221	/* Put DMA channel into init state */
 222	iowrite32(TSI721_DMAC_CTL_INIT,	bdma_chan->regs + TSI721_DMAC_CTL);
 223
 224#ifdef CONFIG_PCI_MSI
 225	if (priv->flags & TSI721_USING_MSIX) {
 226		free_irq(priv->msix[TSI721_VECT_DMA0_DONE +
 227				    bdma_chan->id].vector, (void *)bdma_chan);
 228		free_irq(priv->msix[TSI721_VECT_DMA0_INT +
 229				    bdma_chan->id].vector, (void *)bdma_chan);
 230	}
 231#endif /* CONFIG_PCI_MSI */
 232
 233	/* Free space allocated for DMA descriptors */
 234	dma_free_coherent(bdma_chan->dchan.device->dev,
 235		(bdma_chan->bd_num + 1) * sizeof(struct tsi721_dma_desc),
 236		bdma_chan->bd_base, bdma_chan->bd_phys);
 237	bdma_chan->bd_base = NULL;
 238
 239	/* Free space allocated for status FIFO */
 240	dma_free_coherent(bdma_chan->dchan.device->dev,
 241		bdma_chan->sts_size * sizeof(struct tsi721_dma_sts),
 242		bdma_chan->sts_base, bdma_chan->sts_phys);
 243	bdma_chan->sts_base = NULL;
 244	return 0;
 245}
 246
 247static void
 248tsi721_bdma_interrupt_enable(struct tsi721_bdma_chan *bdma_chan, int enable)
 249{
 250	if (enable) {
 251		/* Clear pending BDMA channel interrupts */
 252		iowrite32(TSI721_DMAC_INT_ALL,
 253			bdma_chan->regs + TSI721_DMAC_INT);
 254		ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 255		/* Enable BDMA channel interrupts */
 256		iowrite32(TSI721_DMAC_INT_ALL,
 257			bdma_chan->regs + TSI721_DMAC_INTE);
 258	} else {
 259		/* Disable BDMA channel interrupts */
 260		iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE);
 261		/* Clear pending BDMA channel interrupts */
 262		iowrite32(TSI721_DMAC_INT_ALL,
 263			bdma_chan->regs + TSI721_DMAC_INT);
 264	}
 265
 266}
 267
 268static bool tsi721_dma_is_idle(struct tsi721_bdma_chan *bdma_chan)
 269{
 270	u32 sts;
 271
 272	sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
 273	return ((sts & TSI721_DMAC_STS_RUN) == 0);
 274}
 275
 276void tsi721_bdma_handler(struct tsi721_bdma_chan *bdma_chan)
 277{
 278	/* Disable BDMA channel interrupts */
 279	iowrite32(0, bdma_chan->regs + TSI721_DMAC_INTE);
 280	if (bdma_chan->active)
 281		tasklet_hi_schedule(&bdma_chan->tasklet);
 282}
 283
 284#ifdef CONFIG_PCI_MSI
 285/**
 286 * tsi721_bdma_msix - MSI-X interrupt handler for BDMA channels
 287 * @irq: Linux interrupt number
 288 * @ptr: Pointer to interrupt-specific data (BDMA channel structure)
 289 *
 290 * Handles BDMA channel interrupts signaled using MSI-X.
 291 *
 292 * Returns: %IRQ_HANDLED
 293 */
 294static irqreturn_t tsi721_bdma_msix(int irq, void *ptr)
 295{
 296	struct tsi721_bdma_chan *bdma_chan = ptr;
 297
 298	if (bdma_chan->active)
 299		tasklet_hi_schedule(&bdma_chan->tasklet);
 300	return IRQ_HANDLED;
 301}
 302#endif /* CONFIG_PCI_MSI */
 303
 304/* Must be called with the spinlock held */
 305static void tsi721_start_dma(struct tsi721_bdma_chan *bdma_chan)
 306{
 307	if (!tsi721_dma_is_idle(bdma_chan)) {
 308		tsi_err(&bdma_chan->dchan.dev->device,
 309			"DMAC%d Attempt to start non-idle channel",
 310			bdma_chan->id);
 311		return;
 312	}
 313
 314	if (bdma_chan->wr_count == bdma_chan->wr_count_next) {
 315		tsi_err(&bdma_chan->dchan.dev->device,
 316			"DMAC%d Attempt to start DMA with no BDs ready %d",
 317			bdma_chan->id, task_pid_nr(current));
 318		return;
 319	}
 320
 321	tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d (wrc=%d) %d",
 322		  bdma_chan->id, bdma_chan->wr_count_next,
 323		  task_pid_nr(current));
 324
 325	iowrite32(bdma_chan->wr_count_next,
 326		bdma_chan->regs + TSI721_DMAC_DWRCNT);
 327	ioread32(bdma_chan->regs + TSI721_DMAC_DWRCNT);
 328
 329	bdma_chan->wr_count = bdma_chan->wr_count_next;
 330}
 331
 332static int
 333tsi721_desc_fill_init(struct tsi721_tx_desc *desc,
 334		      struct tsi721_dma_desc *bd_ptr,
 335		      struct scatterlist *sg, u32 sys_size)
 336{
 337	u64 rio_addr;
 338
 339	if (!bd_ptr)
 340		return -EINVAL;
 341
 342	/* Initialize DMA descriptor */
 343	bd_ptr->type_id = cpu_to_le32((DTYPE1 << 29) |
 344				      (desc->rtype << 19) | desc->destid);
 345	bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) |
 346				     (sys_size << 26));
 347	rio_addr = (desc->rio_addr >> 2) |
 348				((u64)(desc->rio_addr_u & 0x3) << 62);
 349	bd_ptr->raddr_lo = cpu_to_le32(rio_addr & 0xffffffff);
 350	bd_ptr->raddr_hi = cpu_to_le32(rio_addr >> 32);
 351	bd_ptr->t1.bufptr_lo = cpu_to_le32(
 352					(u64)sg_dma_address(sg) & 0xffffffff);
 353	bd_ptr->t1.bufptr_hi = cpu_to_le32((u64)sg_dma_address(sg) >> 32);
 354	bd_ptr->t1.s_dist = 0;
 355	bd_ptr->t1.s_size = 0;
 356
 357	return 0;
 358}
 359
 360static int
 361tsi721_desc_fill_end(struct tsi721_dma_desc *bd_ptr, u32 bcount, bool interrupt)
 362{
 363	if (!bd_ptr)
 364		return -EINVAL;
 365
 366	/* Update DMA descriptor */
 367	if (interrupt)
 368		bd_ptr->type_id |= cpu_to_le32(TSI721_DMAD_IOF);
 369	bd_ptr->bcount |= cpu_to_le32(bcount & TSI721_DMAD_BCOUNT1);
 370
 371	return 0;
 372}
 373
 374static void tsi721_dma_tx_err(struct tsi721_bdma_chan *bdma_chan,
 375			      struct tsi721_tx_desc *desc)
 376{
 377	struct dma_async_tx_descriptor *txd = &desc->txd;
 378	dma_async_tx_callback callback = txd->callback;
 379	void *param = txd->callback_param;
 380
 381	list_move(&desc->desc_node, &bdma_chan->free_list);
 382
 383	if (callback)
 384		callback(param);
 385}
 386
 387static void tsi721_clr_stat(struct tsi721_bdma_chan *bdma_chan)
 388{
 389	u32 srd_ptr;
 390	u64 *sts_ptr;
 391	int i, j;
 392
 393	/* Check and clear descriptor status FIFO entries */
 394	srd_ptr = bdma_chan->sts_rdptr;
 395	sts_ptr = bdma_chan->sts_base;
 396	j = srd_ptr * 8;
 397	while (sts_ptr[j]) {
 398		for (i = 0; i < 8 && sts_ptr[j]; i++, j++)
 399			sts_ptr[j] = 0;
 400
 401		++srd_ptr;
 402		srd_ptr %= bdma_chan->sts_size;
 403		j = srd_ptr * 8;
 404	}
 405
 406	iowrite32(srd_ptr, bdma_chan->regs + TSI721_DMAC_DSRP);
 407	bdma_chan->sts_rdptr = srd_ptr;
 408}
 409
 410/* Must be called with the channel spinlock held */
 411static int tsi721_submit_sg(struct tsi721_tx_desc *desc)
 412{
 413	struct dma_chan *dchan = desc->txd.chan;
 414	struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 415	u32 sys_size;
 416	u64 rio_addr;
 417	dma_addr_t next_addr;
 418	u32 bcount;
 419	struct scatterlist *sg;
 420	unsigned int i;
 421	int err = 0;
 422	struct tsi721_dma_desc *bd_ptr = NULL;
 423	u32 idx, rd_idx;
 424	u32 add_count = 0;
 425	struct device *ch_dev = &dchan->dev->device;
 426
 427	if (!tsi721_dma_is_idle(bdma_chan)) {
 428		tsi_err(ch_dev, "DMAC%d ERR: Attempt to use non-idle channel",
 429			bdma_chan->id);
 430		return -EIO;
 431	}
 432
 433	/*
 434	 * Fill DMA channel's hardware buffer descriptors.
 435	 * (NOTE: RapidIO destination address is limited to 64 bits for now)
 436	 */
 437	rio_addr = desc->rio_addr;
 438	next_addr = -1;
 439	bcount = 0;
 440	sys_size = dma_to_mport(dchan->device)->sys_size;
 441
 442	rd_idx = ioread32(bdma_chan->regs + TSI721_DMAC_DRDCNT);
 443	rd_idx %= (bdma_chan->bd_num + 1);
 444
 445	idx = bdma_chan->wr_count_next % (bdma_chan->bd_num + 1);
 446	if (idx == bdma_chan->bd_num) {
 447		/* wrap around link descriptor */
 448		idx = 0;
 449		add_count++;
 450	}
 451
 452	tsi_debug(DMA, ch_dev, "DMAC%d BD ring status: rdi=%d wri=%d",
 453		  bdma_chan->id, rd_idx, idx);
 454
 455	for_each_sg(desc->sg, sg, desc->sg_len, i) {
 456
 457		tsi_debug(DMAV, ch_dev, "DMAC%d sg%d/%d addr: 0x%llx len: %d",
 458			bdma_chan->id, i, desc->sg_len,
 459			(unsigned long long)sg_dma_address(sg), sg_dma_len(sg));
 460
 461		if (sg_dma_len(sg) > TSI721_BDMA_MAX_BCOUNT) {
 462			tsi_err(ch_dev, "DMAC%d SG entry %d is too large",
 463				bdma_chan->id, i);
 464			err = -EINVAL;
 465			break;
 466		}
 467
 468		/*
 469		 * If this sg entry forms contiguous block with previous one,
 470		 * try to merge it into existing DMA descriptor
 471		 */
 472		if (next_addr == sg_dma_address(sg) &&
 473		    bcount + sg_dma_len(sg) <= TSI721_BDMA_MAX_BCOUNT) {
 474			/* Adjust byte count of the descriptor */
 475			bcount += sg_dma_len(sg);
 476			goto entry_done;
 477		} else if (next_addr != -1) {
 478			/* Finalize descriptor using total byte count value */
 479			tsi721_desc_fill_end(bd_ptr, bcount, 0);
 480			tsi_debug(DMAV, ch_dev,	"DMAC%d prev desc final len: %d",
 481				  bdma_chan->id, bcount);
 482		}
 483
 484		desc->rio_addr = rio_addr;
 485
 486		if (i && idx == rd_idx) {
 487			tsi_debug(DMAV, ch_dev,
 488				  "DMAC%d HW descriptor ring is full @ %d",
 489				  bdma_chan->id, i);
 490			desc->sg = sg;
 491			desc->sg_len -= i;
 492			break;
 493		}
 494
 495		bd_ptr = &((struct tsi721_dma_desc *)bdma_chan->bd_base)[idx];
 496		err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size);
 497		if (err) {
 498			tsi_err(ch_dev, "Failed to build desc: err=%d", err);
 499			break;
 500		}
 501
 502		tsi_debug(DMAV, ch_dev, "DMAC%d bd_ptr = %p did=%d raddr=0x%llx",
 503			  bdma_chan->id, bd_ptr, desc->destid, desc->rio_addr);
 504
 505		next_addr = sg_dma_address(sg);
 506		bcount = sg_dma_len(sg);
 507
 508		add_count++;
 509		if (++idx == bdma_chan->bd_num) {
 510			/* wrap around link descriptor */
 511			idx = 0;
 512			add_count++;
 513		}
 514
 515entry_done:
 516		if (sg_is_last(sg)) {
 517			tsi721_desc_fill_end(bd_ptr, bcount, 0);
 518			tsi_debug(DMAV, ch_dev,
 519				  "DMAC%d last desc final len: %d",
 520				  bdma_chan->id, bcount);
 521			desc->sg_len = 0;
 522		} else {
 523			rio_addr += sg_dma_len(sg);
 524			next_addr += sg_dma_len(sg);
 525		}
 526	}
 527
 528	if (!err)
 529		bdma_chan->wr_count_next += add_count;
 530
 531	return err;
 532}
 533
 534static void tsi721_advance_work(struct tsi721_bdma_chan *bdma_chan,
 535				struct tsi721_tx_desc *desc)
 536{
 537	int err;
 538
 539	tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d", bdma_chan->id);
 540
 541	if (!tsi721_dma_is_idle(bdma_chan))
 542		return;
 543
 544	/*
 545	 * If there is no data transfer in progress, fetch new descriptor from
 546	 * the pending queue.
 547	*/
 548	if (!desc && !bdma_chan->active_tx && !list_empty(&bdma_chan->queue)) {
 549		desc = list_first_entry(&bdma_chan->queue,
 550					struct tsi721_tx_desc, desc_node);
 551		list_del_init((&desc->desc_node));
 552		bdma_chan->active_tx = desc;
 553	}
 554
 555	if (desc) {
 556		err = tsi721_submit_sg(desc);
 557		if (!err)
 558			tsi721_start_dma(bdma_chan);
 559		else {
 560			tsi721_dma_tx_err(bdma_chan, desc);
 561			tsi_debug(DMA, &bdma_chan->dchan.dev->device,
 562				"DMAC%d ERR: tsi721_submit_sg failed with err=%d",
 563				bdma_chan->id, err);
 564		}
 565	}
 566
 567	tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d Exit",
 568		  bdma_chan->id);
 569}
 570
 571static void tsi721_dma_tasklet(unsigned long data)
 572{
 573	struct tsi721_bdma_chan *bdma_chan = (struct tsi721_bdma_chan *)data;
 574	u32 dmac_int, dmac_sts;
 575
 576	dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 577	tsi_debug(DMA, &bdma_chan->dchan.dev->device, "DMAC%d_INT = 0x%x",
 578		  bdma_chan->id, dmac_int);
 579	/* Clear channel interrupts */
 580	iowrite32(dmac_int, bdma_chan->regs + TSI721_DMAC_INT);
 581
 582	if (dmac_int & TSI721_DMAC_INT_ERR) {
 583		int i = 10000;
 584		struct tsi721_tx_desc *desc;
 585
 586		desc = bdma_chan->active_tx;
 587		dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
 588		tsi_err(&bdma_chan->dchan.dev->device,
 589			"DMAC%d_STS = 0x%x did=%d raddr=0x%llx",
 590			bdma_chan->id, dmac_sts, desc->destid, desc->rio_addr);
 591
 592		/* Re-initialize DMA channel if possible */
 593
 594		if ((dmac_sts & TSI721_DMAC_STS_ABORT) == 0)
 595			goto err_out;
 596
 597		tsi721_clr_stat(bdma_chan);
 598
 599		spin_lock(&bdma_chan->lock);
 600
 601		/* Put DMA channel into init state */
 602		iowrite32(TSI721_DMAC_CTL_INIT,
 603			  bdma_chan->regs + TSI721_DMAC_CTL);
 604		do {
 605			udelay(1);
 606			dmac_sts = ioread32(bdma_chan->regs + TSI721_DMAC_STS);
 607			i--;
 608		} while ((dmac_sts & TSI721_DMAC_STS_ABORT) && i);
 609
 610		if (dmac_sts & TSI721_DMAC_STS_ABORT) {
 611			tsi_err(&bdma_chan->dchan.dev->device,
 612				"Failed to re-initiate DMAC%d",	bdma_chan->id);
 613			spin_unlock(&bdma_chan->lock);
 614			goto err_out;
 615		}
 616
 617		/* Setup DMA descriptor pointers */
 618		iowrite32(((u64)bdma_chan->bd_phys >> 32),
 619			bdma_chan->regs + TSI721_DMAC_DPTRH);
 620		iowrite32(((u64)bdma_chan->bd_phys & TSI721_DMAC_DPTRL_MASK),
 621			bdma_chan->regs + TSI721_DMAC_DPTRL);
 622
 623		/* Setup descriptor status FIFO */
 624		iowrite32(((u64)bdma_chan->sts_phys >> 32),
 625			bdma_chan->regs + TSI721_DMAC_DSBH);
 626		iowrite32(((u64)bdma_chan->sts_phys & TSI721_DMAC_DSBL_MASK),
 627			bdma_chan->regs + TSI721_DMAC_DSBL);
 628		iowrite32(TSI721_DMAC_DSSZ_SIZE(bdma_chan->sts_size),
 629			bdma_chan->regs + TSI721_DMAC_DSSZ);
 630
 631		/* Clear interrupt bits */
 632		iowrite32(TSI721_DMAC_INT_ALL,
 633			bdma_chan->regs + TSI721_DMAC_INT);
 634
 635		ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 636
 637		bdma_chan->wr_count = bdma_chan->wr_count_next = 0;
 638		bdma_chan->sts_rdptr = 0;
 639		udelay(10);
 640
 641		desc = bdma_chan->active_tx;
 642		desc->status = DMA_ERROR;
 643		dma_cookie_complete(&desc->txd);
 644		list_add(&desc->desc_node, &bdma_chan->free_list);
 645		bdma_chan->active_tx = NULL;
 646		if (bdma_chan->active)
 647			tsi721_advance_work(bdma_chan, NULL);
 648		spin_unlock(&bdma_chan->lock);
 649	}
 650
 651	if (dmac_int & TSI721_DMAC_INT_STFULL) {
 652		tsi_err(&bdma_chan->dchan.dev->device,
 653			"DMAC%d descriptor status FIFO is full",
 654			bdma_chan->id);
 655	}
 656
 657	if (dmac_int & (TSI721_DMAC_INT_DONE | TSI721_DMAC_INT_IOFDONE)) {
 658		struct tsi721_tx_desc *desc;
 659
 660		tsi721_clr_stat(bdma_chan);
 661		spin_lock(&bdma_chan->lock);
 662		desc = bdma_chan->active_tx;
 663
 664		if (desc->sg_len == 0) {
 665			dma_async_tx_callback callback = NULL;
 666			void *param = NULL;
 667
 668			desc->status = DMA_COMPLETE;
 669			dma_cookie_complete(&desc->txd);
 670			if (desc->txd.flags & DMA_PREP_INTERRUPT) {
 671				callback = desc->txd.callback;
 672				param = desc->txd.callback_param;
 673			}
 674			list_add(&desc->desc_node, &bdma_chan->free_list);
 675			bdma_chan->active_tx = NULL;
 676			if (bdma_chan->active)
 677				tsi721_advance_work(bdma_chan, NULL);
 678			spin_unlock(&bdma_chan->lock);
 679			if (callback)
 680				callback(param);
 681		} else {
 682			if (bdma_chan->active)
 683				tsi721_advance_work(bdma_chan,
 684						    bdma_chan->active_tx);
 685			spin_unlock(&bdma_chan->lock);
 686		}
 687	}
 688err_out:
 689	/* Re-Enable BDMA channel interrupts */
 690	iowrite32(TSI721_DMAC_INT_ALL, bdma_chan->regs + TSI721_DMAC_INTE);
 691}
 692
 693static dma_cookie_t tsi721_tx_submit(struct dma_async_tx_descriptor *txd)
 694{
 695	struct tsi721_tx_desc *desc = to_tsi721_desc(txd);
 696	struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(txd->chan);
 697	dma_cookie_t cookie;
 698
 699	/* Check if the descriptor is detached from any lists */
 700	if (!list_empty(&desc->desc_node)) {
 701		tsi_err(&bdma_chan->dchan.dev->device,
 702			"DMAC%d wrong state of descriptor %p",
 703			bdma_chan->id, txd);
 704		return -EIO;
 705	}
 706
 707	spin_lock_bh(&bdma_chan->lock);
 708
 709	if (!bdma_chan->active) {
 710		spin_unlock_bh(&bdma_chan->lock);
 711		return -ENODEV;
 712	}
 713
 714	cookie = dma_cookie_assign(txd);
 715	desc->status = DMA_IN_PROGRESS;
 716	list_add_tail(&desc->desc_node, &bdma_chan->queue);
 717	tsi721_advance_work(bdma_chan, NULL);
 718
 719	spin_unlock_bh(&bdma_chan->lock);
 720	return cookie;
 721}
 722
 723static int tsi721_alloc_chan_resources(struct dma_chan *dchan)
 724{
 725	struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 726	struct tsi721_tx_desc *desc;
 727	int i;
 728
 729	tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id);
 730
 731	if (bdma_chan->bd_base)
 732		return dma_txqueue_sz;
 733
 734	/* Initialize BDMA channel */
 735	if (tsi721_bdma_ch_init(bdma_chan, dma_desc_per_channel)) {
 736		tsi_err(&dchan->dev->device, "Unable to initialize DMAC%d",
 737			bdma_chan->id);
 738		return -ENODEV;
 739	}
 740
 741	/* Allocate queue of transaction descriptors */
 742	desc = kcalloc(dma_txqueue_sz, sizeof(struct tsi721_tx_desc),
 743			GFP_ATOMIC);
 744	if (!desc) {
 745		tsi721_bdma_ch_free(bdma_chan);
 746		return -ENOMEM;
 747	}
 748
 749	bdma_chan->tx_desc = desc;
 750
 751	for (i = 0; i < dma_txqueue_sz; i++) {
 752		dma_async_tx_descriptor_init(&desc[i].txd, dchan);
 753		desc[i].txd.tx_submit = tsi721_tx_submit;
 754		desc[i].txd.flags = DMA_CTRL_ACK;
 755		list_add(&desc[i].desc_node, &bdma_chan->free_list);
 756	}
 757
 758	dma_cookie_init(dchan);
 759
 760	bdma_chan->active = true;
 761	tsi721_bdma_interrupt_enable(bdma_chan, 1);
 762
 763	return dma_txqueue_sz;
 764}
 765
 766static void tsi721_sync_dma_irq(struct tsi721_bdma_chan *bdma_chan)
 767{
 768	struct tsi721_device *priv = to_tsi721(bdma_chan->dchan.device);
 769
 770#ifdef CONFIG_PCI_MSI
 771	if (priv->flags & TSI721_USING_MSIX) {
 772		synchronize_irq(priv->msix[TSI721_VECT_DMA0_DONE +
 773					   bdma_chan->id].vector);
 774		synchronize_irq(priv->msix[TSI721_VECT_DMA0_INT +
 775					   bdma_chan->id].vector);
 776	} else
 777#endif
 778	synchronize_irq(priv->pdev->irq);
 779}
 780
 781static void tsi721_free_chan_resources(struct dma_chan *dchan)
 782{
 783	struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 784
 785	tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id);
 786
 787	if (!bdma_chan->bd_base)
 788		return;
 789
 790	tsi721_bdma_interrupt_enable(bdma_chan, 0);
 791	bdma_chan->active = false;
 792	tsi721_sync_dma_irq(bdma_chan);
 793	tasklet_kill(&bdma_chan->tasklet);
 794	INIT_LIST_HEAD(&bdma_chan->free_list);
 795	kfree(bdma_chan->tx_desc);
 796	tsi721_bdma_ch_free(bdma_chan);
 797}
 798
 799static
 800enum dma_status tsi721_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
 801				 struct dma_tx_state *txstate)
 802{
 803	struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 804	enum dma_status	status;
 805
 806	spin_lock_bh(&bdma_chan->lock);
 807	status = dma_cookie_status(dchan, cookie, txstate);
 808	spin_unlock_bh(&bdma_chan->lock);
 809	return status;
 810}
 811
 812static void tsi721_issue_pending(struct dma_chan *dchan)
 813{
 814	struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 815
 816	tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id);
 817
 818	spin_lock_bh(&bdma_chan->lock);
 819	if (tsi721_dma_is_idle(bdma_chan) && bdma_chan->active) {
 820		tsi721_advance_work(bdma_chan, NULL);
 821	}
 822	spin_unlock_bh(&bdma_chan->lock);
 823}
 824
 825static
 826struct dma_async_tx_descriptor *tsi721_prep_rio_sg(struct dma_chan *dchan,
 827			struct scatterlist *sgl, unsigned int sg_len,
 828			enum dma_transfer_direction dir, unsigned long flags,
 829			void *tinfo)
 830{
 831	struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 832	struct tsi721_tx_desc *desc;
 833	struct rio_dma_ext *rext = tinfo;
 834	enum dma_rtype rtype;
 835	struct dma_async_tx_descriptor *txd = NULL;
 836
 837	if (!sgl || !sg_len) {
 838		tsi_err(&dchan->dev->device, "DMAC%d No SG list",
 839			bdma_chan->id);
 840		return ERR_PTR(-EINVAL);
 841	}
 842
 843	tsi_debug(DMA, &dchan->dev->device, "DMAC%d %s", bdma_chan->id,
 844		  (dir == DMA_DEV_TO_MEM)?"READ":"WRITE");
 845
 846	if (dir == DMA_DEV_TO_MEM)
 847		rtype = NREAD;
 848	else if (dir == DMA_MEM_TO_DEV) {
 849		switch (rext->wr_type) {
 850		case RDW_ALL_NWRITE:
 851			rtype = ALL_NWRITE;
 852			break;
 853		case RDW_ALL_NWRITE_R:
 854			rtype = ALL_NWRITE_R;
 855			break;
 856		case RDW_LAST_NWRITE_R:
 857		default:
 858			rtype = LAST_NWRITE_R;
 859			break;
 860		}
 861	} else {
 862		tsi_err(&dchan->dev->device,
 863			"DMAC%d Unsupported DMA direction option",
 864			bdma_chan->id);
 865		return ERR_PTR(-EINVAL);
 866	}
 867
 868	spin_lock_bh(&bdma_chan->lock);
 869
 870	if (!list_empty(&bdma_chan->free_list)) {
 871		desc = list_first_entry(&bdma_chan->free_list,
 872				struct tsi721_tx_desc, desc_node);
 873		list_del_init(&desc->desc_node);
 874		desc->destid = rext->destid;
 875		desc->rio_addr = rext->rio_addr;
 876		desc->rio_addr_u = 0;
 877		desc->rtype = rtype;
 878		desc->sg_len	= sg_len;
 879		desc->sg	= sgl;
 880		txd		= &desc->txd;
 881		txd->flags	= flags;
 882	}
 883
 884	spin_unlock_bh(&bdma_chan->lock);
 885
 886	if (!txd) {
 887		tsi_debug(DMA, &dchan->dev->device,
 888			  "DMAC%d free TXD is not available", bdma_chan->id);
 889		return ERR_PTR(-EBUSY);
 890	}
 891
 892	return txd;
 893}
 894
 895static int tsi721_terminate_all(struct dma_chan *dchan)
 896{
 897	struct tsi721_bdma_chan *bdma_chan = to_tsi721_chan(dchan);
 898	struct tsi721_tx_desc *desc, *_d;
 899	LIST_HEAD(list);
 900
 901	tsi_debug(DMA, &dchan->dev->device, "DMAC%d", bdma_chan->id);
 902
 903	spin_lock_bh(&bdma_chan->lock);
 904
 905	bdma_chan->active = false;
 906
 907	while (!tsi721_dma_is_idle(bdma_chan)) {
 908
 909		udelay(5);
 910#if (0)
 911		/* make sure to stop the transfer */
 912		iowrite32(TSI721_DMAC_CTL_SUSP,
 913			  bdma_chan->regs + TSI721_DMAC_CTL);
 914
 915		/* Wait until DMA channel stops */
 916		do {
 917			dmac_int = ioread32(bdma_chan->regs + TSI721_DMAC_INT);
 918		} while ((dmac_int & TSI721_DMAC_INT_SUSP) == 0);
 919#endif
 920	}
 921
 922	if (bdma_chan->active_tx)
 923		list_add(&bdma_chan->active_tx->desc_node, &list);
 924	list_splice_init(&bdma_chan->queue, &list);
 925
 926	list_for_each_entry_safe(desc, _d, &list, desc_node)
 927		tsi721_dma_tx_err(bdma_chan, desc);
 928
 929	spin_unlock_bh(&bdma_chan->lock);
 930
 931	return 0;
 932}
 933
 934static void tsi721_dma_stop(struct tsi721_bdma_chan *bdma_chan)
 935{
 936	if (!bdma_chan->active)
 937		return;
 938	spin_lock_bh(&bdma_chan->lock);
 939	if (!tsi721_dma_is_idle(bdma_chan)) {
 940		int timeout = 100000;
 941
 942		/* stop the transfer in progress */
 943		iowrite32(TSI721_DMAC_CTL_SUSP,
 944			  bdma_chan->regs + TSI721_DMAC_CTL);
 945
 946		/* Wait until DMA channel stops */
 947		while (!tsi721_dma_is_idle(bdma_chan) && --timeout)
 948			udelay(1);
 949	}
 950
 951	spin_unlock_bh(&bdma_chan->lock);
 952}
 953
 954void tsi721_dma_stop_all(struct tsi721_device *priv)
 955{
 956	int i;
 957
 958	for (i = 0; i < TSI721_DMA_MAXCH; i++) {
 959		if ((i != TSI721_DMACH_MAINT) && (dma_sel & (1 << i)))
 960			tsi721_dma_stop(&priv->bdma[i]);
 961	}
 962}
 963
 964int tsi721_register_dma(struct tsi721_device *priv)
 965{
 966	int i;
 967	int nr_channels = 0;
 968	int err;
 969	struct rio_mport *mport = &priv->mport;
 970
 971	INIT_LIST_HEAD(&mport->dma.channels);
 972
 973	for (i = 0; i < TSI721_DMA_MAXCH; i++) {
 974		struct tsi721_bdma_chan *bdma_chan = &priv->bdma[i];
 975
 976		if ((i == TSI721_DMACH_MAINT) || (dma_sel & (1 << i)) == 0)
 977			continue;
 978
 979		bdma_chan->regs = priv->regs + TSI721_DMAC_BASE(i);
 980
 981		bdma_chan->dchan.device = &mport->dma;
 982		bdma_chan->dchan.cookie = 1;
 983		bdma_chan->dchan.chan_id = i;
 984		bdma_chan->id = i;
 985		bdma_chan->active = false;
 986
 987		spin_lock_init(&bdma_chan->lock);
 988
 989		bdma_chan->active_tx = NULL;
 990		INIT_LIST_HEAD(&bdma_chan->queue);
 991		INIT_LIST_HEAD(&bdma_chan->free_list);
 992
 993		tasklet_init(&bdma_chan->tasklet, tsi721_dma_tasklet,
 994			     (unsigned long)bdma_chan);
 995		list_add_tail(&bdma_chan->dchan.device_node,
 996			      &mport->dma.channels);
 997		nr_channels++;
 998	}
 999
1000	mport->dma.chancnt = nr_channels;
1001	dma_cap_zero(mport->dma.cap_mask);
1002	dma_cap_set(DMA_PRIVATE, mport->dma.cap_mask);
1003	dma_cap_set(DMA_SLAVE, mport->dma.cap_mask);
1004
1005	mport->dma.dev = &priv->pdev->dev;
1006	mport->dma.device_alloc_chan_resources = tsi721_alloc_chan_resources;
1007	mport->dma.device_free_chan_resources = tsi721_free_chan_resources;
1008	mport->dma.device_tx_status = tsi721_tx_status;
1009	mport->dma.device_issue_pending = tsi721_issue_pending;
1010	mport->dma.device_prep_slave_sg = tsi721_prep_rio_sg;
1011	mport->dma.device_terminate_all = tsi721_terminate_all;
1012
1013	err = dma_async_device_register(&mport->dma);
1014	if (err)
1015		tsi_err(&priv->pdev->dev, "Failed to register DMA device");
1016
1017	return err;
1018}
1019
1020void tsi721_unregister_dma(struct tsi721_device *priv)
1021{
1022	struct rio_mport *mport = &priv->mport;
1023	struct dma_chan *chan, *_c;
1024	struct tsi721_bdma_chan *bdma_chan;
1025
1026	tsi721_dma_stop_all(priv);
1027	dma_async_device_unregister(&mport->dma);
1028
1029	list_for_each_entry_safe(chan, _c, &mport->dma.channels,
1030					device_node) {
1031		bdma_chan = to_tsi721_chan(chan);
1032		if (bdma_chan->active) {
1033			tsi721_bdma_interrupt_enable(bdma_chan, 0);
1034			bdma_chan->active = false;
1035			tsi721_sync_dma_irq(bdma_chan);
1036			tasklet_kill(&bdma_chan->tasklet);
1037			INIT_LIST_HEAD(&bdma_chan->free_list);
1038			kfree(bdma_chan->tx_desc);
1039			tsi721_bdma_ch_free(bdma_chan);
1040		}
1041
1042		list_del(&chan->device_node);
1043	}
1044}