dma.c

   1/*
   2 * Copyright (c) 2010 Broadcom Corporation
   3 *
   4 * Permission to use, copy, modify, and/or distribute this software for any
   5 * purpose with or without fee is hereby granted, provided that the above
   6 * copyright notice and this permission notice appear in all copies.
   7 *
   8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  11 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  13 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  14 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  15 */
  16
  17#include <linux/slab.h>
  18#include <linux/delay.h>
  19#include <linux/pci.h>
  20#include <net/cfg80211.h>
  21#include <net/mac80211.h>
  22
  23#include <brcmu_utils.h>
  24#include <aiutils.h>
  25#include "types.h"
  26#include "main.h"
  27#include "dma.h"
  28#include "soc.h"
  29#include "scb.h"
  30#include "ampdu.h"
  31#include "debug.h"
  32#include "brcms_trace_events.h"
  33
  34/*
  35 * dma register field offset calculation
  36 */
  37#define DMA64REGOFFS(field)		offsetof(struct dma64regs, field)
  38#define DMA64TXREGOFFS(di, field)	(di->d64txregbase + DMA64REGOFFS(field))
  39#define DMA64RXREGOFFS(di, field)	(di->d64rxregbase + DMA64REGOFFS(field))
  40
  41/*
  42 * DMA hardware requires each descriptor ring to be 8kB aligned, and fit within
  43 * a contiguous 8kB physical address.
  44 */
  45#define D64RINGALIGN_BITS	13
  46#define	D64MAXRINGSZ		(1 << D64RINGALIGN_BITS)
  47#define	D64RINGALIGN		(1 << D64RINGALIGN_BITS)
  48
  49#define	D64MAXDD	(D64MAXRINGSZ / sizeof(struct dma64desc))
  50
  51/* transmit channel control */
  52#define	D64_XC_XE		0x00000001	/* transmit enable */
  53#define	D64_XC_SE		0x00000002	/* transmit suspend request */
  54#define	D64_XC_LE		0x00000004	/* loopback enable */
  55#define	D64_XC_FL		0x00000010	/* flush request */
  56#define	D64_XC_PD		0x00000800	/* parity check disable */
  57#define	D64_XC_AE		0x00030000	/* address extension bits */
  58#define	D64_XC_AE_SHIFT		16
  59
  60/* transmit descriptor table pointer */
  61#define	D64_XP_LD_MASK		0x00000fff	/* last valid descriptor */
  62
  63/* transmit channel status */
  64#define	D64_XS0_CD_MASK		0x00001fff	/* current descriptor pointer */
  65#define	D64_XS0_XS_MASK		0xf0000000	/* transmit state */
  66#define	D64_XS0_XS_SHIFT		28
  67#define	D64_XS0_XS_DISABLED	0x00000000	/* disabled */
  68#define	D64_XS0_XS_ACTIVE	0x10000000	/* active */
  69#define	D64_XS0_XS_IDLE		0x20000000	/* idle wait */
  70#define	D64_XS0_XS_STOPPED	0x30000000	/* stopped */
  71#define	D64_XS0_XS_SUSP		0x40000000	/* suspend pending */
  72
  73#define	D64_XS1_AD_MASK		0x00001fff	/* active descriptor */
  74#define	D64_XS1_XE_MASK		0xf0000000	/* transmit errors */
  75#define	D64_XS1_XE_SHIFT		28
  76#define	D64_XS1_XE_NOERR	0x00000000	/* no error */
  77#define	D64_XS1_XE_DPE		0x10000000	/* descriptor protocol error */
  78#define	D64_XS1_XE_DFU		0x20000000	/* data fifo underrun */
  79#define	D64_XS1_XE_DTE		0x30000000	/* data transfer error */
  80#define	D64_XS1_XE_DESRE	0x40000000	/* descriptor read error */
  81#define	D64_XS1_XE_COREE	0x50000000	/* core error */
  82
  83/* receive channel control */
  84/* receive enable */
  85#define	D64_RC_RE		0x00000001
  86/* receive frame offset */
  87#define	D64_RC_RO_MASK		0x000000fe
  88#define	D64_RC_RO_SHIFT		1
  89/* direct fifo receive (pio) mode */
  90#define	D64_RC_FM		0x00000100
  91/* separate rx header descriptor enable */
  92#define	D64_RC_SH		0x00000200
  93/* overflow continue */
  94#define	D64_RC_OC		0x00000400
  95/* parity check disable */
  96#define	D64_RC_PD		0x00000800
  97/* address extension bits */
  98#define	D64_RC_AE		0x00030000
  99#define	D64_RC_AE_SHIFT		16
 100
 101/* flags for dma controller */
 102/* partity enable */
 103#define DMA_CTRL_PEN		(1 << 0)
 104/* rx overflow continue */
 105#define DMA_CTRL_ROC		(1 << 1)
 106/* allow rx scatter to multiple descriptors */
 107#define DMA_CTRL_RXMULTI	(1 << 2)
 108/* Unframed Rx/Tx data */
 109#define DMA_CTRL_UNFRAMED	(1 << 3)
 110
 111/* receive descriptor table pointer */
 112#define	D64_RP_LD_MASK		0x00000fff	/* last valid descriptor */
 113
 114/* receive channel status */
 115#define	D64_RS0_CD_MASK		0x00001fff	/* current descriptor pointer */
 116#define	D64_RS0_RS_MASK		0xf0000000	/* receive state */
 117#define	D64_RS0_RS_SHIFT		28
 118#define	D64_RS0_RS_DISABLED	0x00000000	/* disabled */
 119#define	D64_RS0_RS_ACTIVE	0x10000000	/* active */
 120#define	D64_RS0_RS_IDLE		0x20000000	/* idle wait */
 121#define	D64_RS0_RS_STOPPED	0x30000000	/* stopped */
 122#define	D64_RS0_RS_SUSP		0x40000000	/* suspend pending */
 123
 124#define	D64_RS1_AD_MASK		0x0001ffff	/* active descriptor */
 125#define	D64_RS1_RE_MASK		0xf0000000	/* receive errors */
 126#define	D64_RS1_RE_SHIFT		28
 127#define	D64_RS1_RE_NOERR	0x00000000	/* no error */
 128#define	D64_RS1_RE_DPO		0x10000000	/* descriptor protocol error */
 129#define	D64_RS1_RE_DFU		0x20000000	/* data fifo overflow */
 130#define	D64_RS1_RE_DTE		0x30000000	/* data transfer error */
 131#define	D64_RS1_RE_DESRE	0x40000000	/* descriptor read error */
 132#define	D64_RS1_RE_COREE	0x50000000	/* core error */
 133
 134/* fifoaddr */
 135#define	D64_FA_OFF_MASK		0xffff	/* offset */
 136#define	D64_FA_SEL_MASK		0xf0000	/* select */
 137#define	D64_FA_SEL_SHIFT	16
 138#define	D64_FA_SEL_XDD		0x00000	/* transmit dma data */
 139#define	D64_FA_SEL_XDP		0x10000	/* transmit dma pointers */
 140#define	D64_FA_SEL_RDD		0x40000	/* receive dma data */
 141#define	D64_FA_SEL_RDP		0x50000	/* receive dma pointers */
 142#define	D64_FA_SEL_XFD		0x80000	/* transmit fifo data */
 143#define	D64_FA_SEL_XFP		0x90000	/* transmit fifo pointers */
 144#define	D64_FA_SEL_RFD		0xc0000	/* receive fifo data */
 145#define	D64_FA_SEL_RFP		0xd0000	/* receive fifo pointers */
 146#define	D64_FA_SEL_RSD		0xe0000	/* receive frame status data */
 147#define	D64_FA_SEL_RSP		0xf0000	/* receive frame status pointers */
 148
 149/* descriptor control flags 1 */
 150#define D64_CTRL_COREFLAGS	0x0ff00000	/* core specific flags */
 151#define	D64_CTRL1_EOT		((u32)1 << 28)	/* end of descriptor table */
 152#define	D64_CTRL1_IOC		((u32)1 << 29)	/* interrupt on completion */
 153#define	D64_CTRL1_EOF		((u32)1 << 30)	/* end of frame */
 154#define	D64_CTRL1_SOF		((u32)1 << 31)	/* start of frame */
 155
 156/* descriptor control flags 2 */
 157/* buffer byte count. real data len must <= 16KB */
 158#define	D64_CTRL2_BC_MASK	0x00007fff
 159/* address extension bits */
 160#define	D64_CTRL2_AE		0x00030000
 161#define	D64_CTRL2_AE_SHIFT	16
 162/* parity bit */
 163#define D64_CTRL2_PARITY	0x00040000
 164
 165/* control flags in the range [27:20] are core-specific and not defined here */
 166#define	D64_CTRL_CORE_MASK	0x0ff00000
 167
 168#define D64_RX_FRM_STS_LEN	0x0000ffff	/* frame length mask */
 169#define D64_RX_FRM_STS_OVFL	0x00800000	/* RxOverFlow */
 170#define D64_RX_FRM_STS_DSCRCNT	0x0f000000  /* no. of descriptors used - 1 */
 171#define D64_RX_FRM_STS_DATATYPE	0xf0000000	/* core-dependent data type */
 172
 173/*
 174 * packet headroom necessary to accommodate the largest header
 175 * in the system, (i.e TXOFF). By doing, we avoid the need to
 176 * allocate an extra buffer for the header when bridging to WL.
 177 * There is a compile time check in wlc.c which ensure that this
 178 * value is at least as big as TXOFF. This value is used in
 179 * dma_rxfill().
 180 */
 181
 182#define BCMEXTRAHDROOM 172
 183
 184#define	MAXNAMEL	8	/* 8 char names */
 185
 186/* macros to convert between byte offsets and indexes */
 187#define	B2I(bytes, type)	((bytes) / sizeof(type))
 188#define	I2B(index, type)	((index) * sizeof(type))
 189
 190#define	PCI32ADDR_HIGH		0xc0000000	/* address[31:30] */
 191#define	PCI32ADDR_HIGH_SHIFT	30	/* address[31:30] */
 192
 193#define	PCI64ADDR_HIGH		0x80000000	/* address[63] */
 194#define	PCI64ADDR_HIGH_SHIFT	31	/* address[63] */
 195
 196/*
 197 * DMA Descriptor
 198 * Descriptors are only read by the hardware, never written back.
 199 */
 200struct dma64desc {
 201	__le32 ctrl1;	/* misc control bits & bufcount */
 202	__le32 ctrl2;	/* buffer count and address extension */
 203	__le32 addrlow;	/* memory address of the date buffer, bits 31:0 */
 204	__le32 addrhigh; /* memory address of the date buffer, bits 63:32 */
 205};
 206
 207/* dma engine software state */
 208struct dma_info {
 209	struct dma_pub dma; /* exported structure */
 210	char name[MAXNAMEL];	/* callers name for diag msgs */
 211
 212	struct bcma_device *core;
 213	struct device *dmadev;
 214
 215	/* session information for AMPDU */
 216	struct brcms_ampdu_session ampdu_session;
 217
 218	bool dma64;	/* this dma engine is operating in 64-bit mode */
 219	bool addrext;	/* this dma engine supports DmaExtendedAddrChanges */
 220
 221	/* 64-bit dma tx engine registers */
 222	uint d64txregbase;
 223	/* 64-bit dma rx engine registers */
 224	uint d64rxregbase;
 225	/* pointer to dma64 tx descriptor ring */
 226	struct dma64desc *txd64;
 227	/* pointer to dma64 rx descriptor ring */
 228	struct dma64desc *rxd64;
 229
 230	u16 dmadesc_align;	/* alignment requirement for dma descriptors */
 231
 232	u16 ntxd;		/* # tx descriptors tunable */
 233	u16 txin;		/* index of next descriptor to reclaim */
 234	u16 txout;		/* index of next descriptor to post */
 235	/* pointer to parallel array of pointers to packets */
 236	struct sk_buff **txp;
 237	/* Aligned physical address of descriptor ring */
 238	dma_addr_t txdpa;
 239	/* Original physical address of descriptor ring */
 240	dma_addr_t txdpaorig;
 241	u16 txdalign;	/* #bytes added to alloc'd mem to align txd */
 242	u32 txdalloc;	/* #bytes allocated for the ring */
 243	u32 xmtptrbase;	/* When using unaligned descriptors, the ptr register
 244			 * is not just an index, it needs all 13 bits to be
 245			 * an offset from the addr register.
 246			 */
 247
 248	u16 nrxd;	/* # rx descriptors tunable */
 249	u16 rxin;	/* index of next descriptor to reclaim */
 250	u16 rxout;	/* index of next descriptor to post */
 251	/* pointer to parallel array of pointers to packets */
 252	struct sk_buff **rxp;
 253	/* Aligned physical address of descriptor ring */
 254	dma_addr_t rxdpa;
 255	/* Original physical address of descriptor ring */
 256	dma_addr_t rxdpaorig;
 257	u16 rxdalign;	/* #bytes added to alloc'd mem to align rxd */
 258	u32 rxdalloc;	/* #bytes allocated for the ring */
 259	u32 rcvptrbase;	/* Base for ptr reg when using unaligned descriptors */
 260
 261	/* tunables */
 262	unsigned int rxbufsize;	/* rx buffer size in bytes, not including
 263				 * the extra headroom
 264				 */
 265	uint rxextrahdrroom;	/* extra rx headroom, reverseved to assist upper
 266				 * stack, e.g. some rx pkt buffers will be
 267				 * bridged to tx side without byte copying.
 268				 * The extra headroom needs to be large enough
 269				 * to fit txheader needs. Some dongle driver may
 270				 * not need it.
 271				 */
 272	uint nrxpost;		/* # rx buffers to keep posted */
 273	unsigned int rxoffset;	/* rxcontrol offset */
 274	/* add to get dma address of descriptor ring, low 32 bits */
 275	uint ddoffsetlow;
 276	/*   high 32 bits */
 277	uint ddoffsethigh;
 278	/* add to get dma address of data buffer, low 32 bits */
 279	uint dataoffsetlow;
 280	/*   high 32 bits */
 281	uint dataoffsethigh;
 282	/* descriptor base need to be aligned or not */
 283	bool aligndesc_4k;
 284};
 285
 286/* Check for odd number of 1's */
 287static u32 parity32(__le32 data)
 288{
 289	/* no swap needed for counting 1's */
 290	u32 par_data = *(u32 *)&data;
 291
 292	par_data ^= par_data >> 16;
 293	par_data ^= par_data >> 8;
 294	par_data ^= par_data >> 4;
 295	par_data ^= par_data >> 2;
 296	par_data ^= par_data >> 1;
 297
 298	return par_data & 1;
 299}
 300
 301static bool dma64_dd_parity(struct dma64desc *dd)
 302{
 303	return parity32(dd->addrlow ^ dd->addrhigh ^ dd->ctrl1 ^ dd->ctrl2);
 304}
 305
 306/* descriptor bumping functions */
 307
 308static uint xxd(uint x, uint n)
 309{
 310	return x & (n - 1); /* faster than %, but n must be power of 2 */
 311}
 312
 313static uint txd(struct dma_info *di, uint x)
 314{
 315	return xxd(x, di->ntxd);
 316}
 317
 318static uint rxd(struct dma_info *di, uint x)
 319{
 320	return xxd(x, di->nrxd);
 321}
 322
 323static uint nexttxd(struct dma_info *di, uint i)
 324{
 325	return txd(di, i + 1);
 326}
 327
 328static uint prevtxd(struct dma_info *di, uint i)
 329{
 330	return txd(di, i - 1);
 331}
 332
 333static uint nextrxd(struct dma_info *di, uint i)
 334{
 335	return rxd(di, i + 1);
 336}
 337
 338static uint ntxdactive(struct dma_info *di, uint h, uint t)
 339{
 340	return txd(di, t-h);
 341}
 342
 343static uint nrxdactive(struct dma_info *di, uint h, uint t)
 344{
 345	return rxd(di, t-h);
 346}
 347
 348static uint _dma_ctrlflags(struct dma_info *di, uint mask, uint flags)
 349{
 350	uint dmactrlflags;
 351
 352	if (di == NULL)
 353		return 0;
 354
 355	dmactrlflags = di->dma.dmactrlflags;
 356	dmactrlflags &= ~mask;
 357	dmactrlflags |= flags;
 358
 359	/* If trying to enable parity, check if parity is actually supported */
 360	if (dmactrlflags & DMA_CTRL_PEN) {
 361		u32 control;
 362
 363		control = bcma_read32(di->core, DMA64TXREGOFFS(di, control));
 364		bcma_write32(di->core, DMA64TXREGOFFS(di, control),
 365		      control | D64_XC_PD);
 366		if (bcma_read32(di->core, DMA64TXREGOFFS(di, control)) &
 367		    D64_XC_PD)
 368			/* We *can* disable it so it is supported,
 369			 * restore control register
 370			 */
 371			bcma_write32(di->core, DMA64TXREGOFFS(di, control),
 372				     control);
 373		else
 374			/* Not supported, don't allow it to be enabled */
 375			dmactrlflags &= ~DMA_CTRL_PEN;
 376	}
 377
 378	di->dma.dmactrlflags = dmactrlflags;
 379
 380	return dmactrlflags;
 381}
 382
 383static bool _dma64_addrext(struct dma_info *di, uint ctrl_offset)
 384{
 385	u32 w;
 386	bcma_set32(di->core, ctrl_offset, D64_XC_AE);
 387	w = bcma_read32(di->core, ctrl_offset);
 388	bcma_mask32(di->core, ctrl_offset, ~D64_XC_AE);
 389	return (w & D64_XC_AE) == D64_XC_AE;
 390}
 391
 392/*
 393 * return true if this dma engine supports DmaExtendedAddrChanges,
 394 * otherwise false
 395 */
 396static bool _dma_isaddrext(struct dma_info *di)
 397{
 398	/* DMA64 supports full 32- or 64-bit operation. AE is always valid */
 399
 400	/* not all tx or rx channel are available */
 401	if (di->d64txregbase != 0) {
 402		if (!_dma64_addrext(di, DMA64TXREGOFFS(di, control)))
 403			brcms_dbg_dma(di->core,
 404				      "%s: DMA64 tx doesn't have AE set\n",
 405				      di->name);
 406		return true;
 407	} else if (di->d64rxregbase != 0) {
 408		if (!_dma64_addrext(di, DMA64RXREGOFFS(di, control)))
 409			brcms_dbg_dma(di->core,
 410				      "%s: DMA64 rx doesn't have AE set\n",
 411				      di->name);
 412		return true;
 413	}
 414
 415	return false;
 416}
 417
 418static bool _dma_descriptor_align(struct dma_info *di)
 419{
 420	u32 addrl;
 421
 422	/* Check to see if the descriptors need to be aligned on 4K/8K or not */
 423	if (di->d64txregbase != 0) {
 424		bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 0xff0);
 425		addrl = bcma_read32(di->core, DMA64TXREGOFFS(di, addrlow));
 426		if (addrl != 0)
 427			return false;
 428	} else if (di->d64rxregbase != 0) {
 429		bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 0xff0);
 430		addrl = bcma_read32(di->core, DMA64RXREGOFFS(di, addrlow));
 431		if (addrl != 0)
 432			return false;
 433	}
 434	return true;
 435}
 436
 437/*
 438 * Descriptor table must start at the DMA hardware dictated alignment, so
 439 * allocated memory must be large enough to support this requirement.
 440 */
 441static void *dma_alloc_consistent(struct dma_info *di, uint size,
 442				  u16 align_bits, uint *alloced,
 443				  dma_addr_t *pap)
 444{
 445	if (align_bits) {
 446		u16 align = (1 << align_bits);
 447		if (!IS_ALIGNED(PAGE_SIZE, align))
 448			size += align;
 449		*alloced = size;
 450	}
 451	return dma_alloc_coherent(di->dmadev, size, pap, GFP_ATOMIC);
 452}
 453
 454static
 455u8 dma_align_sizetobits(uint size)
 456{
 457	u8 bitpos = 0;
 458	while (size >>= 1)
 459		bitpos++;
 460	return bitpos;
 461}
 462
 463/* This function ensures that the DMA descriptor ring will not get allocated
 464 * across Page boundary. If the allocation is done across the page boundary
 465 * at the first time, then it is freed and the allocation is done at
 466 * descriptor ring size aligned location. This will ensure that the ring will
 467 * not cross page boundary
 468 */
 469static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size,
 470			   u16 *alignbits, uint *alloced,
 471			   dma_addr_t *descpa)
 472{
 473	void *va;
 474	u32 desc_strtaddr;
 475	u32 alignbytes = 1 << *alignbits;
 476
 477	va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa);
 478
 479	if (NULL == va)
 480		return NULL;
 481
 482	desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes);
 483	if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
 484							& boundary)) {
 485		*alignbits = dma_align_sizetobits(size);
 486		dma_free_coherent(di->dmadev, *alloced, va, *descpa);
 487		va = dma_alloc_consistent(di, size, *alignbits,
 488			alloced, descpa);
 489	}
 490	return va;
 491}
 492
 493static bool dma64_alloc(struct dma_info *di, uint direction)
 494{
 495	u16 size;
 496	uint ddlen;
 497	void *va;
 498	uint alloced = 0;
 499	u16 align;
 500	u16 align_bits;
 501
 502	ddlen = sizeof(struct dma64desc);
 503
 504	size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
 505	align_bits = di->dmadesc_align;
 506	align = (1 << align_bits);
 507
 508	if (direction == DMA_TX) {
 509		va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
 510			&alloced, &di->txdpaorig);
 511		if (va == NULL) {
 512			brcms_dbg_dma(di->core,
 513				      "%s: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
 514				      di->name);
 515			return false;
 516		}
 517		align = (1 << align_bits);
 518		di->txd64 = (struct dma64desc *)
 519					roundup((unsigned long)va, align);
 520		di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va);
 521		di->txdpa = di->txdpaorig + di->txdalign;
 522		di->txdalloc = alloced;
 523	} else {
 524		va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
 525			&alloced, &di->rxdpaorig);
 526		if (va == NULL) {
 527			brcms_dbg_dma(di->core,
 528				      "%s: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
 529				      di->name);
 530			return false;
 531		}
 532		align = (1 << align_bits);
 533		di->rxd64 = (struct dma64desc *)
 534					roundup((unsigned long)va, align);
 535		di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va);
 536		di->rxdpa = di->rxdpaorig + di->rxdalign;
 537		di->rxdalloc = alloced;
 538	}
 539
 540	return true;
 541}
 542
 543static bool _dma_alloc(struct dma_info *di, uint direction)
 544{
 545	return dma64_alloc(di, direction);
 546}
 547
 548struct dma_pub *dma_attach(char *name, struct brcms_c_info *wlc,
 549			   uint txregbase, uint rxregbase, uint ntxd, uint nrxd,
 550			   uint rxbufsize, int rxextheadroom,
 551			   uint nrxpost, uint rxoffset)
 552{
 553	struct si_pub *sih = wlc->hw->sih;
 554	struct bcma_device *core = wlc->hw->d11core;
 555	struct dma_info *di;
 556	u8 rev = core->id.rev;
 557	uint size;
 558	struct si_info *sii = container_of(sih, struct si_info, pub);
 559
 560	/* allocate private info structure */
 561	di = kzalloc(sizeof(*di), GFP_ATOMIC);
 562	if (di == NULL)
 563		return NULL;
 564
 565	di->dma64 =
 566		((bcma_aread32(core, BCMA_IOST) & SISF_DMA64) == SISF_DMA64);
 567
 568	/* init dma reg info */
 569	di->core = core;
 570	di->d64txregbase = txregbase;
 571	di->d64rxregbase = rxregbase;
 572
 573	/*
 574	 * Default flags (which can be changed by the driver calling
 575	 * dma_ctrlflags before enable): For backwards compatibility
 576	 * both Rx Overflow Continue and Parity are DISABLED.
 577	 */
 578	_dma_ctrlflags(di, DMA_CTRL_ROC | DMA_CTRL_PEN, 0);
 579
 580	brcms_dbg_dma(di->core, "%s: %s flags 0x%x ntxd %d nrxd %d "
 581		      "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d "
 582		      "txregbase %u rxregbase %u\n", name, "DMA64",
 583		      di->dma.dmactrlflags, ntxd, nrxd, rxbufsize,
 584		      rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase);
 585
 586	/* make a private copy of our callers name */
 587	strscpy(di->name, name, sizeof(di->name));
 588
 589	di->dmadev = core->dma_dev;
 590
 591	/* save tunables */
 592	di->ntxd = (u16) ntxd;
 593	di->nrxd = (u16) nrxd;
 594
 595	/* the actual dma size doesn't include the extra headroom */
 596	di->rxextrahdrroom =
 597	    (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
 598	if (rxbufsize > BCMEXTRAHDROOM)
 599		di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom);
 600	else
 601		di->rxbufsize = (u16) rxbufsize;
 602
 603	di->nrxpost = (u16) nrxpost;
 604	di->rxoffset = (u8) rxoffset;
 605
 606	/*
 607	 * figure out the DMA physical address offset for dd and data
 608	 *     PCI/PCIE: they map silicon backplace address to zero
 609	 *     based memory, need offset
 610	 *     Other bus: use zero SI_BUS BIGENDIAN kludge: use sdram
 611	 *     swapped region for data buffer, not descriptor
 612	 */
 613	di->ddoffsetlow = 0;
 614	di->dataoffsetlow = 0;
 615	/* for pci bus, add offset */
 616	if (sii->icbus->hosttype == BCMA_HOSTTYPE_PCI) {
 617		/* add offset for pcie with DMA64 bus */
 618		di->ddoffsetlow = 0;
 619		di->ddoffsethigh = SI_PCIE_DMA_H32;
 620	}
 621	di->dataoffsetlow = di->ddoffsetlow;
 622	di->dataoffsethigh = di->ddoffsethigh;
 623
 624	/* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
 625	if ((core->id.id == BCMA_CORE_SDIO_DEV)
 626	    && ((rev > 0) && (rev <= 2)))
 627		di->addrext = false;
 628	else if ((core->id.id == BCMA_CORE_I2S) &&
 629		 ((rev == 0) || (rev == 1)))
 630		di->addrext = false;
 631	else
 632		di->addrext = _dma_isaddrext(di);
 633
 634	/* does the descriptor need to be aligned and if yes, on 4K/8K or not */
 635	di->aligndesc_4k = _dma_descriptor_align(di);
 636	if (di->aligndesc_4k) {
 637		di->dmadesc_align = D64RINGALIGN_BITS;
 638		if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2))
 639			/* for smaller dd table, HW relax alignment reqmnt */
 640			di->dmadesc_align = D64RINGALIGN_BITS - 1;
 641	} else {
 642		di->dmadesc_align = 4;	/* 16 byte alignment */
 643	}
 644
 645	brcms_dbg_dma(di->core, "DMA descriptor align_needed %d, align %d\n",
 646		      di->aligndesc_4k, di->dmadesc_align);
 647
 648	/* allocate tx packet pointer vector */
 649	if (ntxd) {
 650		size = ntxd * sizeof(void *);
 651		di->txp = kzalloc(size, GFP_ATOMIC);
 652		if (di->txp == NULL)
 653			goto fail;
 654	}
 655
 656	/* allocate rx packet pointer vector */
 657	if (nrxd) {
 658		size = nrxd * sizeof(void *);
 659		di->rxp = kzalloc(size, GFP_ATOMIC);
 660		if (di->rxp == NULL)
 661			goto fail;
 662	}
 663
 664	/*
 665	 * allocate transmit descriptor ring, only need ntxd descriptors
 666	 * but it must be aligned
 667	 */
 668	if (ntxd) {
 669		if (!_dma_alloc(di, DMA_TX))
 670			goto fail;
 671	}
 672
 673	/*
 674	 * allocate receive descriptor ring, only need nrxd descriptors
 675	 * but it must be aligned
 676	 */
 677	if (nrxd) {
 678		if (!_dma_alloc(di, DMA_RX))
 679			goto fail;
 680	}
 681
 682	if ((di->ddoffsetlow != 0) && !di->addrext) {
 683		if (di->txdpa > SI_PCI_DMA_SZ) {
 684			brcms_dbg_dma(di->core,
 685				      "%s: txdpa 0x%x: addrext not supported\n",
 686				      di->name, (u32)di->txdpa);
 687			goto fail;
 688		}
 689		if (di->rxdpa > SI_PCI_DMA_SZ) {
 690			brcms_dbg_dma(di->core,
 691				      "%s: rxdpa 0x%x: addrext not supported\n",
 692				      di->name, (u32)di->rxdpa);
 693			goto fail;
 694		}
 695	}
 696
 697	/* Initialize AMPDU session */
 698	brcms_c_ampdu_reset_session(&di->ampdu_session, wlc);
 699
 700	brcms_dbg_dma(di->core,
 701		      "ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh 0x%x addrext %d\n",
 702		      di->ddoffsetlow, di->ddoffsethigh,
 703		      di->dataoffsetlow, di->dataoffsethigh,
 704		      di->addrext);
 705
 706	return (struct dma_pub *) di;
 707
 708 fail:
 709	dma_detach((struct dma_pub *)di);
 710	return NULL;
 711}
 712
 713static inline void
 714dma64_dd_upd(struct dma_info *di, struct dma64desc *ddring,
 715	     dma_addr_t pa, uint outidx, u32 *flags, u32 bufcount)
 716{
 717	u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
 718
 719	/* PCI bus with big(>1G) physical address, use address extension */
 720	if ((di->dataoffsetlow == 0) || !(pa & PCI32ADDR_HIGH)) {
 721		ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
 722		ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
 723		ddring[outidx].ctrl1 = cpu_to_le32(*flags);
 724		ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
 725	} else {
 726		/* address extension for 32-bit PCI */
 727		u32 ae;
 728
 729		ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
 730		pa &= ~PCI32ADDR_HIGH;
 731
 732		ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
 733		ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
 734		ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
 735		ddring[outidx].ctrl1 = cpu_to_le32(*flags);
 736		ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
 737	}
 738	if (di->dma.dmactrlflags & DMA_CTRL_PEN) {
 739		if (dma64_dd_parity(&ddring[outidx]))
 740			ddring[outidx].ctrl2 =
 741			     cpu_to_le32(ctrl2 | D64_CTRL2_PARITY);
 742	}
 743}
 744
 745/* !! may be called with core in reset */
 746void dma_detach(struct dma_pub *pub)
 747{
 748	struct dma_info *di = container_of(pub, struct dma_info, dma);
 749
 750	brcms_dbg_dma(di->core, "%s:\n", di->name);
 751
 752	/* free dma descriptor rings */
 753	if (di->txd64)
 754		dma_free_coherent(di->dmadev, di->txdalloc,
 755				  ((s8 *)di->txd64 - di->txdalign),
 756				  (di->txdpaorig));
 757	if (di->rxd64)
 758		dma_free_coherent(di->dmadev, di->rxdalloc,
 759				  ((s8 *)di->rxd64 - di->rxdalign),
 760				  (di->rxdpaorig));
 761
 762	/* free packet pointer vectors */
 763	kfree(di->txp);
 764	kfree(di->rxp);
 765
 766	/* free our private info structure */
 767	kfree(di);
 768
 769}
 770
 771/* initialize descriptor table base address */
 772static void
 773_dma_ddtable_init(struct dma_info *di, uint direction, dma_addr_t pa)
 774{
 775	if (!di->aligndesc_4k) {
 776		if (direction == DMA_TX)
 777			di->xmtptrbase = pa;
 778		else
 779			di->rcvptrbase = pa;
 780	}
 781
 782	if ((di->ddoffsetlow == 0)
 783	    || !(pa & PCI32ADDR_HIGH)) {
 784		if (direction == DMA_TX) {
 785			bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
 786				     pa + di->ddoffsetlow);
 787			bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
 788				     di->ddoffsethigh);
 789		} else {
 790			bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
 791				     pa + di->ddoffsetlow);
 792			bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
 793				     di->ddoffsethigh);
 794		}
 795	} else {
 796		/* DMA64 32bits address extension */
 797		u32 ae;
 798
 799		/* shift the high bit(s) from pa to ae */
 800		ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
 801		pa &= ~PCI32ADDR_HIGH;
 802
 803		if (direction == DMA_TX) {
 804			bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
 805				     pa + di->ddoffsetlow);
 806			bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
 807				     di->ddoffsethigh);
 808			bcma_maskset32(di->core, DMA64TXREGOFFS(di, control),
 809				       D64_XC_AE, (ae << D64_XC_AE_SHIFT));
 810		} else {
 811			bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
 812				     pa + di->ddoffsetlow);
 813			bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
 814				     di->ddoffsethigh);
 815			bcma_maskset32(di->core, DMA64RXREGOFFS(di, control),
 816				       D64_RC_AE, (ae << D64_RC_AE_SHIFT));
 817		}
 818	}
 819}
 820
 821static void _dma_rxenable(struct dma_info *di)
 822{
 823	uint dmactrlflags = di->dma.dmactrlflags;
 824	u32 control;
 825
 826	brcms_dbg_dma(di->core, "%s:\n", di->name);
 827
 828	control = D64_RC_RE | (bcma_read32(di->core,
 829					   DMA64RXREGOFFS(di, control)) &
 830			       D64_RC_AE);
 831
 832	if ((dmactrlflags & DMA_CTRL_PEN) == 0)
 833		control |= D64_RC_PD;
 834
 835	if (dmactrlflags & DMA_CTRL_ROC)
 836		control |= D64_RC_OC;
 837
 838	bcma_write32(di->core, DMA64RXREGOFFS(di, control),
 839		((di->rxoffset << D64_RC_RO_SHIFT) | control));
 840}
 841
 842void dma_rxinit(struct dma_pub *pub)
 843{
 844	struct dma_info *di = container_of(pub, struct dma_info, dma);
 845
 846	brcms_dbg_dma(di->core, "%s:\n", di->name);
 847
 848	if (di->nrxd == 0)
 849		return;
 850
 851	di->rxin = di->rxout = 0;
 852
 853	/* clear rx descriptor ring */
 854	memset(di->rxd64, '\0', di->nrxd * sizeof(struct dma64desc));
 855
 856	/* DMA engine with out alignment requirement requires table to be inited
 857	 * before enabling the engine
 858	 */
 859	if (!di->aligndesc_4k)
 860		_dma_ddtable_init(di, DMA_RX, di->rxdpa);
 861
 862	_dma_rxenable(di);
 863
 864	if (di->aligndesc_4k)
 865		_dma_ddtable_init(di, DMA_RX, di->rxdpa);
 866}
 867
 868static struct sk_buff *dma64_getnextrxp(struct dma_info *di, bool forceall)
 869{
 870	uint i, curr;
 871	struct sk_buff *rxp;
 872	dma_addr_t pa;
 873
 874	i = di->rxin;
 875
 876	/* return if no packets posted */
 877	if (i == di->rxout)
 878		return NULL;
 879
 880	curr =
 881	    B2I(((bcma_read32(di->core,
 882			      DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) -
 883		 di->rcvptrbase) & D64_RS0_CD_MASK, struct dma64desc);
 884
 885	/* ignore curr if forceall */
 886	if (!forceall && (i == curr))
 887		return NULL;
 888
 889	/* get the packet pointer that corresponds to the rx descriptor */
 890	rxp = di->rxp[i];
 891	di->rxp[i] = NULL;
 892
 893	pa = le32_to_cpu(di->rxd64[i].addrlow) - di->dataoffsetlow;
 894
 895	/* clear this packet from the descriptor ring */
 896	dma_unmap_single(di->dmadev, pa, di->rxbufsize, DMA_FROM_DEVICE);
 897
 898	di->rxd64[i].addrlow = cpu_to_le32(0xdeadbeef);
 899	di->rxd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
 900
 901	di->rxin = nextrxd(di, i);
 902
 903	return rxp;
 904}
 905
 906static struct sk_buff *_dma_getnextrxp(struct dma_info *di, bool forceall)
 907{
 908	if (di->nrxd == 0)
 909		return NULL;
 910
 911	return dma64_getnextrxp(di, forceall);
 912}
 913
 914/*
 915 * !! rx entry routine
 916 * returns the number packages in the next frame, or 0 if there are no more
 917 *   if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is
 918 *   supported with pkts chain
 919 *   otherwise, it's treated as giant pkt and will be tossed.
 920 *   The DMA scattering starts with normal DMA header, followed by first
 921 *   buffer data. After it reaches the max size of buffer, the data continues
 922 *   in next DMA descriptor buffer WITHOUT DMA header
 923 */
 924int dma_rx(struct dma_pub *pub, struct sk_buff_head *skb_list)
 925{
 926	struct dma_info *di = container_of(pub, struct dma_info, dma);
 927	struct sk_buff_head dma_frames;
 928	struct sk_buff *p, *next;
 929	uint len;
 930	uint pkt_len;
 931	int resid = 0;
 932	int pktcnt = 1;
 933
 934	skb_queue_head_init(&dma_frames);
 935 next_frame:
 936	p = _dma_getnextrxp(di, false);
 937	if (p == NULL)
 938		return 0;
 939
 940	len = le16_to_cpu(*(__le16 *) (p->data));
 941	brcms_dbg_dma(di->core, "%s: dma_rx len %d\n", di->name, len);
 942	dma_spin_for_len(len, p);
 943
 944	/* set actual length */
 945	pkt_len = min((di->rxoffset + len), di->rxbufsize);
 946	__skb_trim(p, pkt_len);
 947	skb_queue_tail(&dma_frames, p);
 948	resid = len - (di->rxbufsize - di->rxoffset);
 949
 950	/* check for single or multi-buffer rx */
 951	if (resid > 0) {
 952		while ((resid > 0) && (p = _dma_getnextrxp(di, false))) {
 953			pkt_len = min_t(uint, resid, di->rxbufsize);
 954			__skb_trim(p, pkt_len);
 955			skb_queue_tail(&dma_frames, p);
 956			resid -= di->rxbufsize;
 957			pktcnt++;
 958		}
 959
 960#ifdef DEBUG
 961		if (resid > 0) {
 962			uint cur;
 963			cur =
 964			    B2I(((bcma_read32(di->core,
 965					      DMA64RXREGOFFS(di, status0)) &
 966				  D64_RS0_CD_MASK) - di->rcvptrbase) &
 967				D64_RS0_CD_MASK, struct dma64desc);
 968			brcms_dbg_dma(di->core,
 969				      "rxin %d rxout %d, hw_curr %d\n",
 970				      di->rxin, di->rxout, cur);
 971		}
 972#endif				/* DEBUG */
 973
 974		if ((di->dma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
 975			brcms_dbg_dma(di->core, "%s: bad frame length (%d)\n",
 976				      di->name, len);
 977			skb_queue_walk_safe(&dma_frames, p, next) {
 978				skb_unlink(p, &dma_frames);
 979				brcmu_pkt_buf_free_skb(p);
 980			}
 981			di->dma.rxgiants++;
 982			pktcnt = 1;
 983			goto next_frame;
 984		}
 985	}
 986
 987	skb_queue_splice_tail(&dma_frames, skb_list);
 988	return pktcnt;
 989}
 990
 991static bool dma64_rxidle(struct dma_info *di)
 992{
 993	brcms_dbg_dma(di->core, "%s:\n", di->name);
 994
 995	if (di->nrxd == 0)
 996		return true;
 997
 998	return ((bcma_read32(di->core,
 999			     DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) ==
1000		(bcma_read32(di->core, DMA64RXREGOFFS(di, ptr)) &
1001		 D64_RS0_CD_MASK));
1002}
1003
1004static bool dma64_txidle(struct dma_info *di)
1005{
1006	if (di->ntxd == 0)
1007		return true;
1008
1009	return ((bcma_read32(di->core,
1010			     DMA64TXREGOFFS(di, status0)) & D64_XS0_CD_MASK) ==
1011		(bcma_read32(di->core, DMA64TXREGOFFS(di, ptr)) &
1012		 D64_XS0_CD_MASK));
1013}
1014
1015/*
1016 * post receive buffers
1017 *  Return false if refill failed completely or dma mapping failed. The ring
1018 *  is empty, which will stall the rx dma and user might want to call rxfill
1019 *  again asap. This is unlikely to happen on a memory-rich NIC, but often on
1020 *  memory-constrained dongle.
1021 */
1022bool dma_rxfill(struct dma_pub *pub)
1023{
1024	struct dma_info *di = container_of(pub, struct dma_info, dma);
1025	struct sk_buff *p;
1026	u16 rxin, rxout;
1027	u32 flags = 0;
1028	uint n;
1029	uint i;
1030	dma_addr_t pa;
1031	uint extra_offset = 0;
1032	bool ring_empty;
1033
1034	ring_empty = false;
1035
1036	/*
1037	 * Determine how many receive buffers we're lacking
1038	 * from the full complement, allocate, initialize,
1039	 * and post them, then update the chip rx lastdscr.
1040	 */
1041
1042	rxin = di->rxin;
1043	rxout = di->rxout;
1044
1045	n = di->nrxpost - nrxdactive(di, rxin, rxout);
1046
1047	brcms_dbg_dma(di->core, "%s: post %d\n", di->name, n);
1048
1049	if (di->rxbufsize > BCMEXTRAHDROOM)
1050		extra_offset = di->rxextrahdrroom;
1051
1052	for (i = 0; i < n; i++) {
1053		/*
1054		 * the di->rxbufsize doesn't include the extra headroom,
1055		 * we need to add it to the size to be allocated
1056		 */
1057		p = brcmu_pkt_buf_get_skb(di->rxbufsize + extra_offset);
1058
1059		if (p == NULL) {
1060			brcms_dbg_dma(di->core, "%s: out of rxbufs\n",
1061				      di->name);
1062			if (i == 0 && dma64_rxidle(di)) {
1063				brcms_dbg_dma(di->core, "%s: ring is empty !\n",
1064					      di->name);
1065				ring_empty = true;
1066			}
1067			di->dma.rxnobuf++;
1068			break;
1069		}
1070		/* reserve an extra headroom, if applicable */
1071		if (extra_offset)
1072			skb_pull(p, extra_offset);
1073
1074		/* Do a cached write instead of uncached write since DMA_MAP
1075		 * will flush the cache.
1076		 */
1077		*(u32 *) (p->data) = 0;
1078
1079		pa = dma_map_single(di->dmadev, p->data, di->rxbufsize,
1080				    DMA_FROM_DEVICE);
1081		if (dma_mapping_error(di->dmadev, pa)) {
1082			brcmu_pkt_buf_free_skb(p);
1083			return false;
1084		}
1085
1086		/* save the free packet pointer */
1087		di->rxp[rxout] = p;
1088
1089		/* reset flags for each descriptor */
1090		flags = 0;
1091		if (rxout == (di->nrxd - 1))
1092			flags = D64_CTRL1_EOT;
1093
1094		dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,
1095			     di->rxbufsize);
1096		rxout = nextrxd(di, rxout);
1097	}
1098
1099	di->rxout = rxout;
1100
1101	/* update the chip lastdscr pointer */
1102	bcma_write32(di->core, DMA64RXREGOFFS(di, ptr),
1103	      di->rcvptrbase + I2B(rxout, struct dma64desc));
1104
1105	return ring_empty;
1106}
1107
1108void dma_rxreclaim(struct dma_pub *pub)
1109{
1110	struct dma_info *di = container_of(pub, struct dma_info, dma);
1111	struct sk_buff *p;
1112
1113	brcms_dbg_dma(di->core, "%s:\n", di->name);
1114
1115	while ((p = _dma_getnextrxp(di, true)))
1116		brcmu_pkt_buf_free_skb(p);
1117}
1118
1119void dma_counterreset(struct dma_pub *pub)
1120{
1121	/* reset all software counters */
1122	pub->rxgiants = 0;
1123	pub->rxnobuf = 0;
1124	pub->txnobuf = 0;
1125}
1126
1127/* get the address of the var in order to change later */
1128unsigned long dma_getvar(struct dma_pub *pub, const char *name)
1129{
1130	struct dma_info *di = container_of(pub, struct dma_info, dma);
1131
1132	if (!strcmp(name, "&txavail"))
1133		return (unsigned long)&(di->dma.txavail);
1134	return 0;
1135}
1136
1137/* 64-bit DMA functions */
1138
1139void dma_txinit(struct dma_pub *pub)
1140{
1141	struct dma_info *di = container_of(pub, struct dma_info, dma);
1142	u32 control = D64_XC_XE;
1143
1144	brcms_dbg_dma(di->core, "%s:\n", di->name);
1145
1146	if (di->ntxd == 0)
1147		return;
1148
1149	di->txin = di->txout = 0;
1150	di->dma.txavail = di->ntxd - 1;
1151
1152	/* clear tx descriptor ring */
1153	memset(di->txd64, '\0', (di->ntxd * sizeof(struct dma64desc)));
1154
1155	/* DMA engine with out alignment requirement requires table to be inited
1156	 * before enabling the engine
1157	 */
1158	if (!di->aligndesc_4k)
1159		_dma_ddtable_init(di, DMA_TX, di->txdpa);
1160
1161	if ((di->dma.dmactrlflags & DMA_CTRL_PEN) == 0)
1162		control |= D64_XC_PD;
1163	bcma_set32(di->core, DMA64TXREGOFFS(di, control), control);
1164
1165	/* DMA engine with alignment requirement requires table to be inited
1166	 * before enabling the engine
1167	 */
1168	if (di->aligndesc_4k)
1169		_dma_ddtable_init(di, DMA_TX, di->txdpa);
1170}
1171
1172void dma_txsuspend(struct dma_pub *pub)
1173{
1174	struct dma_info *di = container_of(pub, struct dma_info, dma);
1175
1176	brcms_dbg_dma(di->core, "%s:\n", di->name);
1177
1178	if (di->ntxd == 0)
1179		return;
1180
1181	bcma_set32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
1182}
1183
1184void dma_txresume(struct dma_pub *pub)
1185{
1186	struct dma_info *di = container_of(pub, struct dma_info, dma);
1187
1188	brcms_dbg_dma(di->core, "%s:\n", di->name);
1189
1190	if (di->ntxd == 0)
1191		return;
1192
1193	bcma_mask32(di->core, DMA64TXREGOFFS(di, control), ~D64_XC_SE);
1194}
1195
1196bool dma_txsuspended(struct dma_pub *pub)
1197{
1198	struct dma_info *di = container_of(pub, struct dma_info, dma);
1199
1200	return (di->ntxd == 0) ||
1201	       ((bcma_read32(di->core,
1202			     DMA64TXREGOFFS(di, control)) & D64_XC_SE) ==
1203		D64_XC_SE);
1204}
1205
1206void dma_txreclaim(struct dma_pub *pub, enum txd_range range)
1207{
1208	struct dma_info *di = container_of(pub, struct dma_info, dma);
1209	struct sk_buff *p;
1210
1211	brcms_dbg_dma(di->core, "%s: %s\n",
1212		      di->name,
1213		      range == DMA_RANGE_ALL ? "all" :
1214		      range == DMA_RANGE_TRANSMITTED ? "transmitted" :
1215		      "transferred");
1216
1217	if (di->txin == di->txout)
1218		return;
1219
1220	while ((p = dma_getnexttxp(pub, range))) {
1221		/* For unframed data, we don't have any packets to free */
1222		if (!(di->dma.dmactrlflags & DMA_CTRL_UNFRAMED))
1223			brcmu_pkt_buf_free_skb(p);
1224	}
1225}
1226
1227bool dma_txreset(struct dma_pub *pub)
1228{
1229	struct dma_info *di = container_of(pub, struct dma_info, dma);
1230	u32 status;
1231
1232	if (di->ntxd == 0)
1233		return true;
1234
1235	/* suspend tx DMA first */
1236	bcma_write32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
1237	SPINWAIT(((status =
1238		   (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
1239		    D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED) &&
1240		  (status != D64_XS0_XS_IDLE) && (status != D64_XS0_XS_STOPPED),
1241		 10000);
1242
1243	bcma_write32(di->core, DMA64TXREGOFFS(di, control), 0);
1244	SPINWAIT(((status =
1245		   (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
1246		    D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED), 10000);
1247
1248	/* wait for the last transaction to complete */
1249	udelay(300);
1250
1251	return status == D64_XS0_XS_DISABLED;
1252}
1253
1254bool dma_rxreset(struct dma_pub *pub)
1255{
1256	struct dma_info *di = container_of(pub, struct dma_info, dma);
1257	u32 status;
1258
1259	if (di->nrxd == 0)
1260		return true;
1261
1262	bcma_write32(di->core, DMA64RXREGOFFS(di, control), 0);
1263	SPINWAIT(((status =
1264		   (bcma_read32(di->core, DMA64RXREGOFFS(di, status0)) &
1265		    D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED), 10000);
1266
1267	return status == D64_RS0_RS_DISABLED;
1268}
1269
1270static void dma_txenq(struct dma_info *di, struct sk_buff *p)
1271{
1272	unsigned char *data;
1273	uint len;
1274	u16 txout;
1275	u32 flags = 0;
1276	dma_addr_t pa;
1277
1278	txout = di->txout;
1279
1280	if (WARN_ON(nexttxd(di, txout) == di->txin))
1281		return;
1282
1283	/*
1284	 * obtain and initialize transmit descriptor entry.
1285	 */
1286	data = p->data;
1287	len = p->len;
1288
1289	/* get physical address of buffer start */
1290	pa = dma_map_single(di->dmadev, data, len, DMA_TO_DEVICE);
1291	/* if mapping failed, free skb */
1292	if (dma_mapping_error(di->dmadev, pa)) {
1293		brcmu_pkt_buf_free_skb(p);
1294		return;
1295	}
1296	/* With a DMA segment list, Descriptor table is filled
1297	 * using the segment list instead of looping over
1298	 * buffers in multi-chain DMA. Therefore, EOF for SGLIST
1299	 * is when end of segment list is reached.
1300	 */
1301	flags = D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF;
1302	if (txout == (di->ntxd - 1))
1303		flags |= D64_CTRL1_EOT;
1304
1305	dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
1306
1307	txout = nexttxd(di, txout);
1308
1309	/* save the packet */
1310	di->txp[prevtxd(di, txout)] = p;
1311
1312	/* bump the tx descriptor index */
1313	di->txout = txout;
1314}
1315
1316static void ampdu_finalize(struct dma_info *di)
1317{
1318	struct brcms_ampdu_session *session = &di->ampdu_session;
1319	struct sk_buff *p;
1320
1321	trace_brcms_ampdu_session(&session->wlc->hw->d11core->dev,
1322				  session->max_ampdu_len,
1323				  session->max_ampdu_frames,
1324				  session->ampdu_len,
1325				  skb_queue_len(&session->skb_list),
1326				  session->dma_len);
1327
1328	if (WARN_ON(skb_queue_empty(&session->skb_list)))
1329		return;
1330
1331	brcms_c_ampdu_finalize(session);
1332
1333	while (!skb_queue_empty(&session->skb_list)) {
1334		p = skb_dequeue(&session->skb_list);
1335		dma_txenq(di, p);
1336	}
1337
1338	bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
1339		     di->xmtptrbase + I2B(di->txout, struct dma64desc));
1340	brcms_c_ampdu_reset_session(session, session->wlc);
1341}
1342
1343static void prep_ampdu_frame(struct dma_info *di, struct sk_buff *p)
1344{
1345	struct brcms_ampdu_session *session = &di->ampdu_session;
1346	int ret;
1347
1348	ret = brcms_c_ampdu_add_frame(session, p);
1349	if (ret == -ENOSPC) {
1350		/*
1351		 * AMPDU cannot accommodate this frame. Close out the in-
1352		 * progress AMPDU session and start a new one.
1353		 */
1354		ampdu_finalize(di);
1355		ret = brcms_c_ampdu_add_frame(session, p);
1356	}
1357
1358	WARN_ON(ret);
1359}
1360
1361/* Update count of available tx descriptors based on current DMA state */
1362static void dma_update_txavail(struct dma_info *di)
1363{
1364	/*
1365	 * Available space is number of descriptors less the number of
1366	 * active descriptors and the number of queued AMPDU frames.
1367	 */
1368	di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) -
1369			  skb_queue_len(&di->ampdu_session.skb_list) - 1;
1370}
1371
1372/*
1373 * !! tx entry routine
1374 * WARNING: call must check the return value for error.
1375 *   the error(toss frames) could be fatal and cause many subsequent hard
1376 *   to debug problems
1377 */
1378int dma_txfast(struct brcms_c_info *wlc, struct dma_pub *pub,
1379	       struct sk_buff *p)
1380{
1381	struct dma_info *di = container_of(pub, struct dma_info, dma);
1382	struct brcms_ampdu_session *session = &di->ampdu_session;
1383	struct ieee80211_tx_info *tx_info;
1384	bool is_ampdu;
1385
1386	/* no use to transmit a zero length packet */
1387	if (p->len == 0)
1388		return 0;
1389
1390	/* return nonzero if out of tx descriptors */
1391	if (di->dma.txavail == 0 || nexttxd(di, di->txout) == di->txin)
1392		goto outoftxd;
1393
1394	tx_info = IEEE80211_SKB_CB(p);
1395	is_ampdu = tx_info->flags & IEEE80211_TX_CTL_AMPDU;
1396	if (is_ampdu)
1397		prep_ampdu_frame(di, p);
1398	else
1399		dma_txenq(di, p);
1400
1401	/* tx flow control */
1402	dma_update_txavail(di);
1403
1404	/* kick the chip */
1405	if (is_ampdu) {
1406		/*
1407		 * Start sending data if we've got a full AMPDU, there's
1408		 * no more space in the DMA ring, or the ring isn't
1409		 * currently transmitting.
1410		 */
1411		if (skb_queue_len(&session->skb_list) == session->max_ampdu_frames ||
1412		    di->dma.txavail == 0 || dma64_txidle(di))
1413			ampdu_finalize(di);
1414	} else {
1415		bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
1416			     di->xmtptrbase + I2B(di->txout, struct dma64desc));
1417	}
1418
1419	return 0;
1420
1421 outoftxd:
1422	brcms_dbg_dma(di->core, "%s: out of txds !!!\n", di->name);
1423	brcmu_pkt_buf_free_skb(p);
1424	di->dma.txavail = 0;
1425	di->dma.txnobuf++;
1426	return -ENOSPC;
1427}
1428
1429int dma_txpending(struct dma_pub *pub)
1430{
1431	struct dma_info *di = container_of(pub, struct dma_info, dma);
1432	return ntxdactive(di, di->txin, di->txout);
1433}
1434
1435/*
1436 * If we have an active AMPDU session and are not transmitting,
1437 * this function will force tx to start.
1438 */
1439void dma_kick_tx(struct dma_pub *pub)
1440{
1441	struct dma_info *di = container_of(pub, struct dma_info, dma);
1442	struct brcms_ampdu_session *session = &di->ampdu_session;
1443
1444	if (!skb_queue_empty(&session->skb_list) && dma64_txidle(di))
1445		ampdu_finalize(di);
1446}
1447
1448/*
1449 * Reclaim next completed txd (txds if using chained buffers) in the range
1450 * specified and return associated packet.
1451 * If range is DMA_RANGE_TRANSMITTED, reclaim descriptors that have be
1452 * transmitted as noted by the hardware "CurrDescr" pointer.
1453 * If range is DMA_RANGE_TRANSFERED, reclaim descriptors that have be
1454 * transferred by the DMA as noted by the hardware "ActiveDescr" pointer.
1455 * If range is DMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
1456 * return associated packet regardless of the value of hardware pointers.
1457 */
1458struct sk_buff *dma_getnexttxp(struct dma_pub *pub, enum txd_range range)
1459{
1460	struct dma_info *di = container_of(pub, struct dma_info, dma);
1461	u16 start, end, i;
1462	u16 active_desc;
1463	struct sk_buff *txp;
1464
1465	brcms_dbg_dma(di->core, "%s: %s\n",
1466		      di->name,
1467		      range == DMA_RANGE_ALL ? "all" :
1468		      range == DMA_RANGE_TRANSMITTED ? "transmitted" :
1469		      "transferred");
1470
1471	if (di->ntxd == 0)
1472		return NULL;
1473
1474	txp = NULL;
1475
1476	start = di->txin;
1477	if (range == DMA_RANGE_ALL)
1478		end = di->txout;
1479	else {
1480		end = (u16) (B2I(((bcma_read32(di->core,
1481					       DMA64TXREGOFFS(di, status0)) &
1482				   D64_XS0_CD_MASK) - di->xmtptrbase) &
1483				 D64_XS0_CD_MASK, struct dma64desc));
1484
1485		if (range == DMA_RANGE_TRANSFERED) {
1486			active_desc =
1487				(u16)(bcma_read32(di->core,
1488						  DMA64TXREGOFFS(di, status1)) &
1489				      D64_XS1_AD_MASK);
1490			active_desc =
1491			    (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
1492			active_desc = B2I(active_desc, struct dma64desc);
1493			if (end != active_desc)
1494				end = prevtxd(di, active_desc);
1495		}
1496	}
1497
1498	if ((start == 0) && (end > di->txout))
1499		goto bogus;
1500
1501	for (i = start; i != end && !txp; i = nexttxd(di, i)) {
1502		dma_addr_t pa;
1503		uint size;
1504
1505		pa = le32_to_cpu(di->txd64[i].addrlow) - di->dataoffsetlow;
1506
1507		size =
1508		    (le32_to_cpu(di->txd64[i].ctrl2) &
1509		     D64_CTRL2_BC_MASK);
1510
1511		di->txd64[i].addrlow = cpu_to_le32(0xdeadbeef);
1512		di->txd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
1513
1514		txp = di->txp[i];
1515		di->txp[i] = NULL;
1516
1517		dma_unmap_single(di->dmadev, pa, size, DMA_TO_DEVICE);
1518	}
1519
1520	di->txin = i;
1521
1522	/* tx flow control */
1523	dma_update_txavail(di);
1524
1525	return txp;
1526
1527 bogus:
1528	brcms_dbg_dma(di->core, "bogus curr: start %d end %d txout %d\n",
1529		      start, end, di->txout);
1530	return NULL;
1531}
1532
1533/*
1534 * Mac80211 initiated actions sometimes require packets in the DMA queue to be
1535 * modified. The modified portion of the packet is not under control of the DMA
1536 * engine. This function calls a caller-supplied function for each packet in
1537 * the caller specified dma chain.
1538 */
1539void dma_walk_packets(struct dma_pub *dmah, void (*callback_fnc)
1540		      (void *pkt, void *arg_a), void *arg_a)
1541{
1542	struct dma_info *di = container_of(dmah, struct dma_info, dma);
1543	uint i =   di->txin;
1544	uint end = di->txout;
1545	struct sk_buff *skb;
1546	struct ieee80211_tx_info *tx_info;
1547
1548	while (i != end) {
1549		skb = di->txp[i];
1550		if (skb != NULL) {
1551			tx_info = (struct ieee80211_tx_info *)skb->cb;
1552			(callback_fnc)(tx_info, arg_a);
1553		}
1554		i = nexttxd(di, i);
1555	}
1556}