csio_wr.c

   1/*
   2 * This file is part of the Chelsio FCoE driver for Linux.
   3 *
   4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
   5 *
   6 * This software is available to you under a choice of one of two
   7 * licenses.  You may choose to be licensed under the terms of the GNU
   8 * General Public License (GPL) Version 2, available from the file
   9 * COPYING in the main directory of this source tree, or the
  10 * OpenIB.org BSD license below:
  11 *
  12 *     Redistribution and use in source and binary forms, with or
  13 *     without modification, are permitted provided that the following
  14 *     conditions are met:
  15 *
  16 *      - Redistributions of source code must retain the above
  17 *        copyright notice, this list of conditions and the following
  18 *        disclaimer.
  19 *
  20 *      - Redistributions in binary form must reproduce the above
  21 *        copyright notice, this list of conditions and the following
  22 *        disclaimer in the documentation and/or other materials
  23 *        provided with the distribution.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32 * SOFTWARE.
  33 */
  34
  35#include <linux/kernel.h>
  36#include <linux/string.h>
  37#include <linux/compiler.h>
  38#include <linux/slab.h>
  39#include <asm/page.h>
  40#include <linux/cache.h>
  41
  42#include "t4_values.h"
  43#include "csio_hw.h"
  44#include "csio_wr.h"
  45#include "csio_mb.h"
  46#include "csio_defs.h"
  47
  48int csio_intr_coalesce_cnt;		/* value:SGE_INGRESS_RX_THRESHOLD[0] */
  49static int csio_sge_thresh_reg;		/* SGE_INGRESS_RX_THRESHOLD[0] */
  50
  51int csio_intr_coalesce_time = 10;	/* value:SGE_TIMER_VALUE_1 */
  52static int csio_sge_timer_reg = 1;
  53
  54#define CSIO_SET_FLBUF_SIZE(_hw, _reg, _val)				\
  55	csio_wr_reg32((_hw), (_val), SGE_FL_BUFFER_SIZE##_reg##_A)
  56
  57static void
  58csio_get_flbuf_size(struct csio_hw *hw, struct csio_sge *sge, uint32_t reg)
  59{
  60	sge->sge_fl_buf_size[reg] = csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE0_A +
  61							reg * sizeof(uint32_t));
  62}
  63
  64/* Free list buffer size */
  65static inline uint32_t
  66csio_wr_fl_bufsz(struct csio_sge *sge, struct csio_dma_buf *buf)
  67{
  68	return sge->sge_fl_buf_size[buf->paddr & 0xF];
  69}
  70
  71/* Size of the egress queue status page */
  72static inline uint32_t
  73csio_wr_qstat_pgsz(struct csio_hw *hw)
  74{
  75	return (hw->wrm.sge.sge_control & EGRSTATUSPAGESIZE_F) ?  128 : 64;
  76}
  77
  78/* Ring freelist doorbell */
  79static inline void
  80csio_wr_ring_fldb(struct csio_hw *hw, struct csio_q *flq)
  81{
  82	/*
  83	 * Ring the doorbell only when we have atleast CSIO_QCREDIT_SZ
  84	 * number of bytes in the freelist queue. This translates to atleast
  85	 * 8 freelist buffer pointers (since each pointer is 8 bytes).
  86	 */
  87	if (flq->inc_idx >= 8) {
  88		csio_wr_reg32(hw, DBPRIO_F | QID_V(flq->un.fl.flid) |
  89				  PIDX_T5_V(flq->inc_idx / 8) | DBTYPE_F,
  90				  MYPF_REG(SGE_PF_KDOORBELL_A));
  91		flq->inc_idx &= 7;
  92	}
  93}
  94
  95/* Write a 0 cidx increment value to enable SGE interrupts for this queue */
  96static void
  97csio_wr_sge_intr_enable(struct csio_hw *hw, uint16_t iqid)
  98{
  99	csio_wr_reg32(hw, CIDXINC_V(0)		|
 100			  INGRESSQID_V(iqid)	|
 101			  TIMERREG_V(X_TIMERREG_RESTART_COUNTER),
 102			  MYPF_REG(SGE_PF_GTS_A));
 103}
 104
 105/*
 106 * csio_wr_fill_fl - Populate the FL buffers of a FL queue.
 107 * @hw: HW module.
 108 * @flq: Freelist queue.
 109 *
 110 * Fill up freelist buffer entries with buffers of size specified
 111 * in the size register.
 112 *
 113 */
 114static int
 115csio_wr_fill_fl(struct csio_hw *hw, struct csio_q *flq)
 116{
 117	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 118	struct csio_sge *sge = &wrm->sge;
 119	__be64 *d = (__be64 *)(flq->vstart);
 120	struct csio_dma_buf *buf = &flq->un.fl.bufs[0];
 121	uint64_t paddr;
 122	int sreg = flq->un.fl.sreg;
 123	int n = flq->credits;
 124
 125	while (n--) {
 126		buf->len = sge->sge_fl_buf_size[sreg];
 127		buf->vaddr = dma_alloc_coherent(&hw->pdev->dev, buf->len,
 128						&buf->paddr, GFP_KERNEL);
 129		if (!buf->vaddr) {
 130			csio_err(hw, "Could only fill %d buffers!\n", n + 1);
 131			return -ENOMEM;
 132		}
 133
 134		paddr = buf->paddr | (sreg & 0xF);
 135
 136		*d++ = cpu_to_be64(paddr);
 137		buf++;
 138	}
 139
 140	return 0;
 141}
 142
 143/*
 144 * csio_wr_update_fl -
 145 * @hw: HW module.
 146 * @flq: Freelist queue.
 147 *
 148 *
 149 */
 150static inline void
 151csio_wr_update_fl(struct csio_hw *hw, struct csio_q *flq, uint16_t n)
 152{
 153
 154	flq->inc_idx += n;
 155	flq->pidx += n;
 156	if (unlikely(flq->pidx >= flq->credits))
 157		flq->pidx -= (uint16_t)flq->credits;
 158
 159	CSIO_INC_STATS(flq, n_flq_refill);
 160}
 161
 162/*
 163 * csio_wr_alloc_q - Allocate a WR queue and initialize it.
 164 * @hw: HW module
 165 * @qsize: Size of the queue in bytes
 166 * @wrsize: Since of WR in this queue, if fixed.
 167 * @type: Type of queue (Ingress/Egress/Freelist)
 168 * @owner: Module that owns this queue.
 169 * @nflb: Number of freelist buffers for FL.
 170 * @sreg: What is the FL buffer size register?
 171 * @iq_int_handler: Ingress queue handler in INTx mode.
 172 *
 173 * This function allocates and sets up a queue for the caller
 174 * of size qsize, aligned at the required boundary. This is subject to
 175 * be free entries being available in the queue array. If one is found,
 176 * it is initialized with the allocated queue, marked as being used (owner),
 177 * and a handle returned to the caller in form of the queue's index
 178 * into the q_arr array.
 179 * If user has indicated a freelist (by specifying nflb > 0), create
 180 * another queue (with its own index into q_arr) for the freelist. Allocate
 181 * memory for DMA buffer metadata (vaddr, len etc). Save off the freelist
 182 * idx in the ingress queue's flq.idx. This is how a Freelist is associated
 183 * with its owning ingress queue.
 184 */
 185int
 186csio_wr_alloc_q(struct csio_hw *hw, uint32_t qsize, uint32_t wrsize,
 187		uint16_t type, void *owner, uint32_t nflb, int sreg,
 188		iq_handler_t iq_intx_handler)
 189{
 190	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 191	struct csio_q	*q, *flq;
 192	int		free_idx = wrm->free_qidx;
 193	int		ret_idx = free_idx;
 194	uint32_t	qsz;
 195	int flq_idx;
 196
 197	if (free_idx >= wrm->num_q) {
 198		csio_err(hw, "No more free queues.\n");
 199		return -1;
 200	}
 201
 202	switch (type) {
 203	case CSIO_EGRESS:
 204		qsz = ALIGN(qsize, CSIO_QCREDIT_SZ) + csio_wr_qstat_pgsz(hw);
 205		break;
 206	case CSIO_INGRESS:
 207		switch (wrsize) {
 208		case 16:
 209		case 32:
 210		case 64:
 211		case 128:
 212			break;
 213		default:
 214			csio_err(hw, "Invalid Ingress queue WR size:%d\n",
 215				    wrsize);
 216			return -1;
 217		}
 218
 219		/*
 220		 * Number of elements must be a multiple of 16
 221		 * So this includes status page size
 222		 */
 223		qsz = ALIGN(qsize/wrsize, 16) * wrsize;
 224
 225		break;
 226	case CSIO_FREELIST:
 227		qsz = ALIGN(qsize/wrsize, 8) * wrsize + csio_wr_qstat_pgsz(hw);
 228		break;
 229	default:
 230		csio_err(hw, "Invalid queue type: 0x%x\n", type);
 231		return -1;
 232	}
 233
 234	q = wrm->q_arr[free_idx];
 235
 236	q->vstart = dma_alloc_coherent(&hw->pdev->dev, qsz, &q->pstart,
 237				       GFP_KERNEL);
 238	if (!q->vstart) {
 239		csio_err(hw,
 240			 "Failed to allocate DMA memory for "
 241			 "queue at id: %d size: %d\n", free_idx, qsize);
 242		return -1;
 243	}
 244
 245	q->type		= type;
 246	q->owner	= owner;
 247	q->pidx		= q->cidx = q->inc_idx = 0;
 248	q->size		= qsz;
 249	q->wr_sz	= wrsize;	/* If using fixed size WRs */
 250
 251	wrm->free_qidx++;
 252
 253	if (type == CSIO_INGRESS) {
 254		/* Since queue area is set to zero */
 255		q->un.iq.genbit	= 1;
 256
 257		/*
 258		 * Ingress queue status page size is always the size of
 259		 * the ingress queue entry.
 260		 */
 261		q->credits	= (qsz - q->wr_sz) / q->wr_sz;
 262		q->vwrap	= (void *)((uintptr_t)(q->vstart) + qsz
 263							- q->wr_sz);
 264
 265		/* Allocate memory for FL if requested */
 266		if (nflb > 0) {
 267			flq_idx = csio_wr_alloc_q(hw, nflb * sizeof(__be64),
 268						  sizeof(__be64), CSIO_FREELIST,
 269						  owner, 0, sreg, NULL);
 270			if (flq_idx == -1) {
 271				csio_err(hw,
 272					 "Failed to allocate FL queue"
 273					 " for IQ idx:%d\n", free_idx);
 274				return -1;
 275			}
 276
 277			/* Associate the new FL with the Ingress quue */
 278			q->un.iq.flq_idx = flq_idx;
 279
 280			flq = wrm->q_arr[q->un.iq.flq_idx];
 281			flq->un.fl.bufs = kcalloc(flq->credits,
 282						  sizeof(struct csio_dma_buf),
 283						  GFP_KERNEL);
 284			if (!flq->un.fl.bufs) {
 285				csio_err(hw,
 286					 "Failed to allocate FL queue bufs"
 287					 " for IQ idx:%d\n", free_idx);
 288				return -1;
 289			}
 290
 291			flq->un.fl.packen = 0;
 292			flq->un.fl.offset = 0;
 293			flq->un.fl.sreg = sreg;
 294
 295			/* Fill up the free list buffers */
 296			if (csio_wr_fill_fl(hw, flq))
 297				return -1;
 298
 299			/*
 300			 * Make sure in a FLQ, atleast 1 credit (8 FL buffers)
 301			 * remains unpopulated,otherwise HW thinks
 302			 * FLQ is empty.
 303			 */
 304			flq->pidx = flq->inc_idx = flq->credits - 8;
 305		} else {
 306			q->un.iq.flq_idx = -1;
 307		}
 308
 309		/* Associate the IQ INTx handler. */
 310		q->un.iq.iq_intx_handler = iq_intx_handler;
 311
 312		csio_q_iqid(hw, ret_idx) = CSIO_MAX_QID;
 313
 314	} else if (type == CSIO_EGRESS) {
 315		q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / CSIO_QCREDIT_SZ;
 316		q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
 317						- csio_wr_qstat_pgsz(hw));
 318		csio_q_eqid(hw, ret_idx) = CSIO_MAX_QID;
 319	} else { /* Freelist */
 320		q->credits = (qsz - csio_wr_qstat_pgsz(hw)) / sizeof(__be64);
 321		q->vwrap   = (void *)((uintptr_t)(q->vstart) + qsz
 322						- csio_wr_qstat_pgsz(hw));
 323		csio_q_flid(hw, ret_idx) = CSIO_MAX_QID;
 324	}
 325
 326	return ret_idx;
 327}
 328
 329/*
 330 * csio_wr_iq_create_rsp - Response handler for IQ creation.
 331 * @hw: The HW module.
 332 * @mbp: Mailbox.
 333 * @iq_idx: Ingress queue that got created.
 334 *
 335 * Handle FW_IQ_CMD mailbox completion. Save off the assigned IQ/FL ids.
 336 */
 337static int
 338csio_wr_iq_create_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
 339{
 340	struct csio_iq_params iqp;
 341	enum fw_retval retval;
 342	uint32_t iq_id;
 343	int flq_idx;
 344
 345	memset(&iqp, 0, sizeof(struct csio_iq_params));
 346
 347	csio_mb_iq_alloc_write_rsp(hw, mbp, &retval, &iqp);
 348
 349	if (retval != FW_SUCCESS) {
 350		csio_err(hw, "IQ cmd returned 0x%x!\n", retval);
 351		mempool_free(mbp, hw->mb_mempool);
 352		return -EINVAL;
 353	}
 354
 355	csio_q_iqid(hw, iq_idx)		= iqp.iqid;
 356	csio_q_physiqid(hw, iq_idx)	= iqp.physiqid;
 357	csio_q_pidx(hw, iq_idx)		= csio_q_cidx(hw, iq_idx) = 0;
 358	csio_q_inc_idx(hw, iq_idx)	= 0;
 359
 360	/* Actual iq-id. */
 361	iq_id = iqp.iqid - hw->wrm.fw_iq_start;
 362
 363	/* Set the iq-id to iq map table. */
 364	if (iq_id >= CSIO_MAX_IQ) {
 365		csio_err(hw,
 366			 "Exceeding MAX_IQ(%d) supported!"
 367			 " iqid:%d rel_iqid:%d FW iq_start:%d\n",
 368			 CSIO_MAX_IQ, iq_id, iqp.iqid, hw->wrm.fw_iq_start);
 369		mempool_free(mbp, hw->mb_mempool);
 370		return -EINVAL;
 371	}
 372	csio_q_set_intr_map(hw, iq_idx, iq_id);
 373
 374	/*
 375	 * During FW_IQ_CMD, FW sets interrupt_sent bit to 1 in the SGE
 376	 * ingress context of this queue. This will block interrupts to
 377	 * this queue until the next GTS write. Therefore, we do a
 378	 * 0-cidx increment GTS write for this queue just to clear the
 379	 * interrupt_sent bit. This will re-enable interrupts to this
 380	 * queue.
 381	 */
 382	csio_wr_sge_intr_enable(hw, iqp.physiqid);
 383
 384	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
 385	if (flq_idx != -1) {
 386		struct csio_q *flq = hw->wrm.q_arr[flq_idx];
 387
 388		csio_q_flid(hw, flq_idx) = iqp.fl0id;
 389		csio_q_cidx(hw, flq_idx) = 0;
 390		csio_q_pidx(hw, flq_idx)    = csio_q_credits(hw, flq_idx) - 8;
 391		csio_q_inc_idx(hw, flq_idx) = csio_q_credits(hw, flq_idx) - 8;
 392
 393		/* Now update SGE about the buffers allocated during init */
 394		csio_wr_ring_fldb(hw, flq);
 395	}
 396
 397	mempool_free(mbp, hw->mb_mempool);
 398
 399	return 0;
 400}
 401
 402/*
 403 * csio_wr_iq_create - Configure an Ingress queue with FW.
 404 * @hw: The HW module.
 405 * @priv: Private data object.
 406 * @iq_idx: Ingress queue index in the WR module.
 407 * @vec: MSIX vector.
 408 * @portid: PCIE Channel to be associated with this queue.
 409 * @async: Is this a FW asynchronous message handling queue?
 410 * @cbfn: Completion callback.
 411 *
 412 * This API configures an ingress queue with FW by issuing a FW_IQ_CMD mailbox
 413 * with alloc/write bits set.
 414 */
 415int
 416csio_wr_iq_create(struct csio_hw *hw, void *priv, int iq_idx,
 417		  uint32_t vec, uint8_t portid, bool async,
 418		  void (*cbfn) (struct csio_hw *, struct csio_mb *))
 419{
 420	struct csio_mb  *mbp;
 421	struct csio_iq_params iqp;
 422	int flq_idx;
 423
 424	memset(&iqp, 0, sizeof(struct csio_iq_params));
 425	csio_q_portid(hw, iq_idx) = portid;
 426
 427	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
 428	if (!mbp) {
 429		csio_err(hw, "IQ command out of memory!\n");
 430		return -ENOMEM;
 431	}
 432
 433	switch (hw->intr_mode) {
 434	case CSIO_IM_INTX:
 435	case CSIO_IM_MSI:
 436		/* For interrupt forwarding queue only */
 437		if (hw->intr_iq_idx == iq_idx)
 438			iqp.iqandst	= X_INTERRUPTDESTINATION_PCIE;
 439		else
 440			iqp.iqandst	= X_INTERRUPTDESTINATION_IQ;
 441		iqp.iqandstindex	=
 442			csio_q_physiqid(hw, hw->intr_iq_idx);
 443		break;
 444	case CSIO_IM_MSIX:
 445		iqp.iqandst		= X_INTERRUPTDESTINATION_PCIE;
 446		iqp.iqandstindex	= (uint16_t)vec;
 447		break;
 448	case CSIO_IM_NONE:
 449		mempool_free(mbp, hw->mb_mempool);
 450		return -EINVAL;
 451	}
 452
 453	/* Pass in the ingress queue cmd parameters */
 454	iqp.pfn			= hw->pfn;
 455	iqp.vfn			= 0;
 456	iqp.iq_start		= 1;
 457	iqp.viid		= 0;
 458	iqp.type		= FW_IQ_TYPE_FL_INT_CAP;
 459	iqp.iqasynch		= async;
 460	if (csio_intr_coalesce_cnt)
 461		iqp.iqanus	= X_UPDATESCHEDULING_COUNTER_OPTTIMER;
 462	else
 463		iqp.iqanus	= X_UPDATESCHEDULING_TIMER;
 464	iqp.iqanud		= X_UPDATEDELIVERY_INTERRUPT;
 465	iqp.iqpciech		= portid;
 466	iqp.iqintcntthresh	= (uint8_t)csio_sge_thresh_reg;
 467
 468	switch (csio_q_wr_sz(hw, iq_idx)) {
 469	case 16:
 470		iqp.iqesize = 0; break;
 471	case 32:
 472		iqp.iqesize = 1; break;
 473	case 64:
 474		iqp.iqesize = 2; break;
 475	case 128:
 476		iqp.iqesize = 3; break;
 477	}
 478
 479	iqp.iqsize		= csio_q_size(hw, iq_idx) /
 480						csio_q_wr_sz(hw, iq_idx);
 481	iqp.iqaddr		= csio_q_pstart(hw, iq_idx);
 482
 483	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
 484	if (flq_idx != -1) {
 485		enum chip_type chip = CHELSIO_CHIP_VERSION(hw->chip_id);
 486		struct csio_q *flq = hw->wrm.q_arr[flq_idx];
 487
 488		iqp.fl0paden	= 1;
 489		iqp.fl0packen	= flq->un.fl.packen ? 1 : 0;
 490		iqp.fl0fbmin	= X_FETCHBURSTMIN_64B;
 491		iqp.fl0fbmax	= ((chip == CHELSIO_T5) ?
 492				  X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B);
 493		iqp.fl0size	= csio_q_size(hw, flq_idx) / CSIO_QCREDIT_SZ;
 494		iqp.fl0addr	= csio_q_pstart(hw, flq_idx);
 495	}
 496
 497	csio_mb_iq_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
 498
 499	if (csio_mb_issue(hw, mbp)) {
 500		csio_err(hw, "Issue of IQ cmd failed!\n");
 501		mempool_free(mbp, hw->mb_mempool);
 502		return -EINVAL;
 503	}
 504
 505	if (cbfn != NULL)
 506		return 0;
 507
 508	return csio_wr_iq_create_rsp(hw, mbp, iq_idx);
 509}
 510
 511/*
 512 * csio_wr_eq_create_rsp - Response handler for EQ creation.
 513 * @hw: The HW module.
 514 * @mbp: Mailbox.
 515 * @eq_idx: Egress queue that got created.
 516 *
 517 * Handle FW_EQ_OFLD_CMD mailbox completion. Save off the assigned EQ ids.
 518 */
 519static int
 520csio_wr_eq_cfg_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
 521{
 522	struct csio_eq_params eqp;
 523	enum fw_retval retval;
 524
 525	memset(&eqp, 0, sizeof(struct csio_eq_params));
 526
 527	csio_mb_eq_ofld_alloc_write_rsp(hw, mbp, &retval, &eqp);
 528
 529	if (retval != FW_SUCCESS) {
 530		csio_err(hw, "EQ OFLD cmd returned 0x%x!\n", retval);
 531		mempool_free(mbp, hw->mb_mempool);
 532		return -EINVAL;
 533	}
 534
 535	csio_q_eqid(hw, eq_idx)	= (uint16_t)eqp.eqid;
 536	csio_q_physeqid(hw, eq_idx) = (uint16_t)eqp.physeqid;
 537	csio_q_pidx(hw, eq_idx)	= csio_q_cidx(hw, eq_idx) = 0;
 538	csio_q_inc_idx(hw, eq_idx) = 0;
 539
 540	mempool_free(mbp, hw->mb_mempool);
 541
 542	return 0;
 543}
 544
 545/*
 546 * csio_wr_eq_create - Configure an Egress queue with FW.
 547 * @hw: HW module.
 548 * @priv: Private data.
 549 * @eq_idx: Egress queue index in the WR module.
 550 * @iq_idx: Associated ingress queue index.
 551 * @cbfn: Completion callback.
 552 *
 553 * This API configures a offload egress queue with FW by issuing a
 554 * FW_EQ_OFLD_CMD  (with alloc + write ) mailbox.
 555 */
 556int
 557csio_wr_eq_create(struct csio_hw *hw, void *priv, int eq_idx,
 558		  int iq_idx, uint8_t portid,
 559		  void (*cbfn) (struct csio_hw *, struct csio_mb *))
 560{
 561	struct csio_mb  *mbp;
 562	struct csio_eq_params eqp;
 563
 564	memset(&eqp, 0, sizeof(struct csio_eq_params));
 565
 566	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
 567	if (!mbp) {
 568		csio_err(hw, "EQ command out of memory!\n");
 569		return -ENOMEM;
 570	}
 571
 572	eqp.pfn			= hw->pfn;
 573	eqp.vfn			= 0;
 574	eqp.eqstart		= 1;
 575	eqp.hostfcmode		= X_HOSTFCMODE_STATUS_PAGE;
 576	eqp.iqid		= csio_q_iqid(hw, iq_idx);
 577	eqp.fbmin		= X_FETCHBURSTMIN_64B;
 578	eqp.fbmax		= X_FETCHBURSTMAX_512B;
 579	eqp.cidxfthresh		= 0;
 580	eqp.pciechn		= portid;
 581	eqp.eqsize		= csio_q_size(hw, eq_idx) / CSIO_QCREDIT_SZ;
 582	eqp.eqaddr		= csio_q_pstart(hw, eq_idx);
 583
 584	csio_mb_eq_ofld_alloc_write(hw, mbp, priv, CSIO_MB_DEFAULT_TMO,
 585				    &eqp, cbfn);
 586
 587	if (csio_mb_issue(hw, mbp)) {
 588		csio_err(hw, "Issue of EQ OFLD cmd failed!\n");
 589		mempool_free(mbp, hw->mb_mempool);
 590		return -EINVAL;
 591	}
 592
 593	if (cbfn != NULL)
 594		return 0;
 595
 596	return csio_wr_eq_cfg_rsp(hw, mbp, eq_idx);
 597}
 598
 599/*
 600 * csio_wr_iq_destroy_rsp - Response handler for IQ removal.
 601 * @hw: The HW module.
 602 * @mbp: Mailbox.
 603 * @iq_idx: Ingress queue that was freed.
 604 *
 605 * Handle FW_IQ_CMD (free) mailbox completion.
 606 */
 607static int
 608csio_wr_iq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int iq_idx)
 609{
 610	enum fw_retval retval = csio_mb_fw_retval(mbp);
 611	int rv = 0;
 612
 613	if (retval != FW_SUCCESS)
 614		rv = -EINVAL;
 615
 616	mempool_free(mbp, hw->mb_mempool);
 617
 618	return rv;
 619}
 620
 621/*
 622 * csio_wr_iq_destroy - Free an ingress queue.
 623 * @hw: The HW module.
 624 * @priv: Private data object.
 625 * @iq_idx: Ingress queue index to destroy
 626 * @cbfn: Completion callback.
 627 *
 628 * This API frees an ingress queue by issuing the FW_IQ_CMD
 629 * with the free bit set.
 630 */
 631static int
 632csio_wr_iq_destroy(struct csio_hw *hw, void *priv, int iq_idx,
 633		   void (*cbfn)(struct csio_hw *, struct csio_mb *))
 634{
 635	int rv = 0;
 636	struct csio_mb  *mbp;
 637	struct csio_iq_params iqp;
 638	int flq_idx;
 639
 640	memset(&iqp, 0, sizeof(struct csio_iq_params));
 641
 642	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
 643	if (!mbp)
 644		return -ENOMEM;
 645
 646	iqp.pfn		= hw->pfn;
 647	iqp.vfn		= 0;
 648	iqp.iqid	= csio_q_iqid(hw, iq_idx);
 649	iqp.type	= FW_IQ_TYPE_FL_INT_CAP;
 650
 651	flq_idx = csio_q_iq_flq_idx(hw, iq_idx);
 652	if (flq_idx != -1)
 653		iqp.fl0id = csio_q_flid(hw, flq_idx);
 654	else
 655		iqp.fl0id = 0xFFFF;
 656
 657	iqp.fl1id = 0xFFFF;
 658
 659	csio_mb_iq_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &iqp, cbfn);
 660
 661	rv = csio_mb_issue(hw, mbp);
 662	if (rv != 0) {
 663		mempool_free(mbp, hw->mb_mempool);
 664		return rv;
 665	}
 666
 667	if (cbfn != NULL)
 668		return 0;
 669
 670	return csio_wr_iq_destroy_rsp(hw, mbp, iq_idx);
 671}
 672
 673/*
 674 * csio_wr_eq_destroy_rsp - Response handler for OFLD EQ creation.
 675 * @hw: The HW module.
 676 * @mbp: Mailbox.
 677 * @eq_idx: Egress queue that was freed.
 678 *
 679 * Handle FW_OFLD_EQ_CMD (free) mailbox completion.
 680 */
 681static int
 682csio_wr_eq_destroy_rsp(struct csio_hw *hw, struct csio_mb *mbp, int eq_idx)
 683{
 684	enum fw_retval retval = csio_mb_fw_retval(mbp);
 685	int rv = 0;
 686
 687	if (retval != FW_SUCCESS)
 688		rv = -EINVAL;
 689
 690	mempool_free(mbp, hw->mb_mempool);
 691
 692	return rv;
 693}
 694
 695/*
 696 * csio_wr_eq_destroy - Free an Egress queue.
 697 * @hw: The HW module.
 698 * @priv: Private data object.
 699 * @eq_idx: Egress queue index to destroy
 700 * @cbfn: Completion callback.
 701 *
 702 * This API frees an Egress queue by issuing the FW_EQ_OFLD_CMD
 703 * with the free bit set.
 704 */
 705static int
 706csio_wr_eq_destroy(struct csio_hw *hw, void *priv, int eq_idx,
 707		   void (*cbfn) (struct csio_hw *, struct csio_mb *))
 708{
 709	int rv = 0;
 710	struct csio_mb  *mbp;
 711	struct csio_eq_params eqp;
 712
 713	memset(&eqp, 0, sizeof(struct csio_eq_params));
 714
 715	mbp = mempool_alloc(hw->mb_mempool, GFP_ATOMIC);
 716	if (!mbp)
 717		return -ENOMEM;
 718
 719	eqp.pfn		= hw->pfn;
 720	eqp.vfn		= 0;
 721	eqp.eqid	= csio_q_eqid(hw, eq_idx);
 722
 723	csio_mb_eq_ofld_free(hw, mbp, priv, CSIO_MB_DEFAULT_TMO, &eqp, cbfn);
 724
 725	rv = csio_mb_issue(hw, mbp);
 726	if (rv != 0) {
 727		mempool_free(mbp, hw->mb_mempool);
 728		return rv;
 729	}
 730
 731	if (cbfn != NULL)
 732		return 0;
 733
 734	return csio_wr_eq_destroy_rsp(hw, mbp, eq_idx);
 735}
 736
 737/*
 738 * csio_wr_cleanup_eq_stpg - Cleanup Egress queue status page
 739 * @hw: HW module
 740 * @qidx: Egress queue index
 741 *
 742 * Cleanup the Egress queue status page.
 743 */
 744static void
 745csio_wr_cleanup_eq_stpg(struct csio_hw *hw, int qidx)
 746{
 747	struct csio_q	*q = csio_hw_to_wrm(hw)->q_arr[qidx];
 748	struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
 749
 750	memset(stp, 0, sizeof(*stp));
 751}
 752
 753/*
 754 * csio_wr_cleanup_iq_ftr - Cleanup Footer entries in IQ
 755 * @hw: HW module
 756 * @qidx: Ingress queue index
 757 *
 758 * Cleanup the footer entries in the given ingress queue,
 759 * set to 1 the internal copy of genbit.
 760 */
 761static void
 762csio_wr_cleanup_iq_ftr(struct csio_hw *hw, int qidx)
 763{
 764	struct csio_wrm *wrm	= csio_hw_to_wrm(hw);
 765	struct csio_q	*q	= wrm->q_arr[qidx];
 766	void *wr;
 767	struct csio_iqwr_footer *ftr;
 768	uint32_t i = 0;
 769
 770	/* set to 1 since we are just about zero out genbit */
 771	q->un.iq.genbit = 1;
 772
 773	for (i = 0; i < q->credits; i++) {
 774		/* Get the WR */
 775		wr = (void *)((uintptr_t)q->vstart +
 776					   (i * q->wr_sz));
 777		/* Get the footer */
 778		ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
 779					  (q->wr_sz - sizeof(*ftr)));
 780		/* Zero out footer */
 781		memset(ftr, 0, sizeof(*ftr));
 782	}
 783}
 784
 785int
 786csio_wr_destroy_queues(struct csio_hw *hw, bool cmd)
 787{
 788	int i, flq_idx;
 789	struct csio_q *q;
 790	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 791	int rv;
 792
 793	for (i = 0; i < wrm->free_qidx; i++) {
 794		q = wrm->q_arr[i];
 795
 796		switch (q->type) {
 797		case CSIO_EGRESS:
 798			if (csio_q_eqid(hw, i) != CSIO_MAX_QID) {
 799				csio_wr_cleanup_eq_stpg(hw, i);
 800				if (!cmd) {
 801					csio_q_eqid(hw, i) = CSIO_MAX_QID;
 802					continue;
 803				}
 804
 805				rv = csio_wr_eq_destroy(hw, NULL, i, NULL);
 806				if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
 807					cmd = false;
 808
 809				csio_q_eqid(hw, i) = CSIO_MAX_QID;
 810			}
 811			fallthrough;
 812		case CSIO_INGRESS:
 813			if (csio_q_iqid(hw, i) != CSIO_MAX_QID) {
 814				csio_wr_cleanup_iq_ftr(hw, i);
 815				if (!cmd) {
 816					csio_q_iqid(hw, i) = CSIO_MAX_QID;
 817					flq_idx = csio_q_iq_flq_idx(hw, i);
 818					if (flq_idx != -1)
 819						csio_q_flid(hw, flq_idx) =
 820								CSIO_MAX_QID;
 821					continue;
 822				}
 823
 824				rv = csio_wr_iq_destroy(hw, NULL, i, NULL);
 825				if ((rv == -EBUSY) || (rv == -ETIMEDOUT))
 826					cmd = false;
 827
 828				csio_q_iqid(hw, i) = CSIO_MAX_QID;
 829				flq_idx = csio_q_iq_flq_idx(hw, i);
 830				if (flq_idx != -1)
 831					csio_q_flid(hw, flq_idx) = CSIO_MAX_QID;
 832			}
 833			break;
 834		default:
 835			break;
 836		}
 837	}
 838
 839	hw->flags &= ~CSIO_HWF_Q_FW_ALLOCED;
 840
 841	return 0;
 842}
 843
 844/*
 845 * csio_wr_get - Get requested size of WR entry/entries from queue.
 846 * @hw: HW module.
 847 * @qidx: Index of queue.
 848 * @size: Cumulative size of Work request(s).
 849 * @wrp: Work request pair.
 850 *
 851 * If requested credits are available, return the start address of the
 852 * work request in the work request pair. Set pidx accordingly and
 853 * return.
 854 *
 855 * NOTE about WR pair:
 856 * ==================
 857 * A WR can start towards the end of a queue, and then continue at the
 858 * beginning, since the queue is considered to be circular. This will
 859 * require a pair of address/size to be passed back to the caller -
 860 * hence Work request pair format.
 861 */
 862int
 863csio_wr_get(struct csio_hw *hw, int qidx, uint32_t size,
 864	    struct csio_wr_pair *wrp)
 865{
 866	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 867	struct csio_q *q = wrm->q_arr[qidx];
 868	void *cwr = (void *)((uintptr_t)(q->vstart) +
 869						(q->pidx * CSIO_QCREDIT_SZ));
 870	struct csio_qstatus_page *stp = (struct csio_qstatus_page *)q->vwrap;
 871	uint16_t cidx = q->cidx = ntohs(stp->cidx);
 872	uint16_t pidx = q->pidx;
 873	uint32_t req_sz	= ALIGN(size, CSIO_QCREDIT_SZ);
 874	int req_credits	= req_sz / CSIO_QCREDIT_SZ;
 875	int credits;
 876
 877	CSIO_DB_ASSERT(q->owner != NULL);
 878	CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
 879	CSIO_DB_ASSERT(cidx <= q->credits);
 880
 881	/* Calculate credits */
 882	if (pidx > cidx) {
 883		credits = q->credits - (pidx - cidx) - 1;
 884	} else if (cidx > pidx) {
 885		credits = cidx - pidx - 1;
 886	} else {
 887		/* cidx == pidx, empty queue */
 888		credits = q->credits;
 889		CSIO_INC_STATS(q, n_qempty);
 890	}
 891
 892	/*
 893	 * Check if we have enough credits.
 894	 * credits = 1 implies queue is full.
 895	 */
 896	if (!credits || (req_credits > credits)) {
 897		CSIO_INC_STATS(q, n_qfull);
 898		return -EBUSY;
 899	}
 900
 901	/*
 902	 * If we are here, we have enough credits to satisfy the
 903	 * request. Check if we are near the end of q, and if WR spills over.
 904	 * If it does, use the first addr/size to cover the queue until
 905	 * the end. Fit the remainder portion of the request at the top
 906	 * of queue and return it in the second addr/len. Set pidx
 907	 * accordingly.
 908	 */
 909	if (unlikely(((uintptr_t)cwr + req_sz) > (uintptr_t)(q->vwrap))) {
 910		wrp->addr1 = cwr;
 911		wrp->size1 = (uint32_t)((uintptr_t)q->vwrap - (uintptr_t)cwr);
 912		wrp->addr2 = q->vstart;
 913		wrp->size2 = req_sz - wrp->size1;
 914		q->pidx	= (uint16_t)(ALIGN(wrp->size2, CSIO_QCREDIT_SZ) /
 915							CSIO_QCREDIT_SZ);
 916		CSIO_INC_STATS(q, n_qwrap);
 917		CSIO_INC_STATS(q, n_eq_wr_split);
 918	} else {
 919		wrp->addr1 = cwr;
 920		wrp->size1 = req_sz;
 921		wrp->addr2 = NULL;
 922		wrp->size2 = 0;
 923		q->pidx	+= (uint16_t)req_credits;
 924
 925		/* We are the end of queue, roll back pidx to top of queue */
 926		if (unlikely(q->pidx == q->credits)) {
 927			q->pidx = 0;
 928			CSIO_INC_STATS(q, n_qwrap);
 929		}
 930	}
 931
 932	q->inc_idx = (uint16_t)req_credits;
 933
 934	CSIO_INC_STATS(q, n_tot_reqs);
 935
 936	return 0;
 937}
 938
 939/*
 940 * csio_wr_copy_to_wrp - Copies given data into WR.
 941 * @data_buf - Data buffer
 942 * @wrp - Work request pair.
 943 * @wr_off - Work request offset.
 944 * @data_len - Data length.
 945 *
 946 * Copies the given data in Work Request. Work request pair(wrp) specifies
 947 * address information of Work request.
 948 * Returns: none
 949 */
 950void
 951csio_wr_copy_to_wrp(void *data_buf, struct csio_wr_pair *wrp,
 952		   uint32_t wr_off, uint32_t data_len)
 953{
 954	uint32_t nbytes;
 955
 956	/* Number of space available in buffer addr1 of WRP */
 957	nbytes = ((wrp->size1 - wr_off) >= data_len) ?
 958					data_len : (wrp->size1 - wr_off);
 959
 960	memcpy((uint8_t *) wrp->addr1 + wr_off, data_buf, nbytes);
 961	data_len -= nbytes;
 962
 963	/* Write the remaining data from the beginning of circular buffer */
 964	if (data_len) {
 965		CSIO_DB_ASSERT(data_len <= wrp->size2);
 966		CSIO_DB_ASSERT(wrp->addr2 != NULL);
 967		memcpy(wrp->addr2, (uint8_t *) data_buf + nbytes, data_len);
 968	}
 969}
 970
 971/*
 972 * csio_wr_issue - Notify chip of Work request.
 973 * @hw: HW module.
 974 * @qidx: Index of queue.
 975 * @prio: 0: Low priority, 1: High priority
 976 *
 977 * Rings the SGE Doorbell by writing the current producer index of the passed
 978 * in queue into the register.
 979 *
 980 */
 981int
 982csio_wr_issue(struct csio_hw *hw, int qidx, bool prio)
 983{
 984	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
 985	struct csio_q *q = wrm->q_arr[qidx];
 986
 987	CSIO_DB_ASSERT((qidx >= 0) && (qidx < wrm->free_qidx));
 988
 989	wmb();
 990	/* Ring SGE Doorbell writing q->pidx into it */
 991	csio_wr_reg32(hw, DBPRIO_V(prio) | QID_V(q->un.eq.physeqid) |
 992			  PIDX_T5_V(q->inc_idx) | DBTYPE_F,
 993			  MYPF_REG(SGE_PF_KDOORBELL_A));
 994	q->inc_idx = 0;
 995
 996	return 0;
 997}
 998
 999static inline uint32_t
1000csio_wr_avail_qcredits(struct csio_q *q)
1001{
1002	if (q->pidx > q->cidx)
1003		return q->pidx - q->cidx;
1004	else if (q->cidx > q->pidx)
1005		return q->credits - (q->cidx - q->pidx);
1006	else
1007		return 0;	/* cidx == pidx, empty queue */
1008}
1009
1010/*
1011 * csio_wr_inval_flq_buf - Invalidate a free list buffer entry.
1012 * @hw: HW module.
1013 * @flq: The freelist queue.
1014 *
1015 * Invalidate the driver's version of a freelist buffer entry,
1016 * without freeing the associated the DMA memory. The entry
1017 * to be invalidated is picked up from the current Free list
1018 * queue cidx.
1019 *
1020 */
1021static inline void
1022csio_wr_inval_flq_buf(struct csio_hw *hw, struct csio_q *flq)
1023{
1024	flq->cidx++;
1025	if (flq->cidx == flq->credits) {
1026		flq->cidx = 0;
1027		CSIO_INC_STATS(flq, n_qwrap);
1028	}
1029}
1030
1031/*
1032 * csio_wr_process_fl - Process a freelist completion.
1033 * @hw: HW module.
1034 * @q: The ingress queue attached to the Freelist.
1035 * @wr: The freelist completion WR in the ingress queue.
1036 * @len_to_qid: The lower 32-bits of the first flit of the RSP footer
1037 * @iq_handler: Caller's handler for this completion.
1038 * @priv: Private pointer of caller
1039 *
1040 */
1041static inline void
1042csio_wr_process_fl(struct csio_hw *hw, struct csio_q *q,
1043		   void *wr, uint32_t len_to_qid,
1044		   void (*iq_handler)(struct csio_hw *, void *,
1045				      uint32_t, struct csio_fl_dma_buf *,
1046				      void *),
1047		   void *priv)
1048{
1049	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1050	struct csio_sge *sge = &wrm->sge;
1051	struct csio_fl_dma_buf flb;
1052	struct csio_dma_buf *buf, *fbuf;
1053	uint32_t bufsz, len, lastlen = 0;
1054	struct csio_q *flq = hw->wrm.q_arr[q->un.iq.flq_idx];
1055
1056	CSIO_DB_ASSERT(flq != NULL);
1057
1058	len = len_to_qid;
1059
1060	if (len & IQWRF_NEWBUF) {
1061		if (flq->un.fl.offset > 0) {
1062			csio_wr_inval_flq_buf(hw, flq);
1063			flq->un.fl.offset = 0;
1064		}
1065		len = IQWRF_LEN_GET(len);
1066	}
1067
1068	CSIO_DB_ASSERT(len != 0);
1069
1070	flb.totlen = len;
1071
1072	/* Consume all freelist buffers used for len bytes */
1073	for (fbuf = flb.flbufs; ; fbuf++) {
1074		buf = &flq->un.fl.bufs[flq->cidx];
1075		bufsz = csio_wr_fl_bufsz(sge, buf);
1076
1077		fbuf->paddr	= buf->paddr;
1078		fbuf->vaddr	= buf->vaddr;
1079
1080		flb.offset	= flq->un.fl.offset;
1081		lastlen		= min(bufsz, len);
1082		fbuf->len	= lastlen;
1083
1084		len -= lastlen;
1085		if (!len)
1086			break;
1087		csio_wr_inval_flq_buf(hw, flq);
1088	}
1089
1090	flb.defer_free = flq->un.fl.packen ? 0 : 1;
1091
1092	iq_handler(hw, wr, q->wr_sz - sizeof(struct csio_iqwr_footer),
1093		   &flb, priv);
1094
1095	if (flq->un.fl.packen)
1096		flq->un.fl.offset += ALIGN(lastlen, sge->csio_fl_align);
1097	else
1098		csio_wr_inval_flq_buf(hw, flq);
1099
1100}
1101
1102/*
1103 * csio_is_new_iqwr - Is this a new Ingress queue entry ?
1104 * @q: Ingress quueue.
1105 * @ftr: Ingress queue WR SGE footer.
1106 *
1107 * The entry is new if our generation bit matches the corresponding
1108 * bit in the footer of the current WR.
1109 */
1110static inline bool
1111csio_is_new_iqwr(struct csio_q *q, struct csio_iqwr_footer *ftr)
1112{
1113	return (q->un.iq.genbit == (ftr->u.type_gen >> IQWRF_GEN_SHIFT));
1114}
1115
1116/*
1117 * csio_wr_process_iq - Process elements in Ingress queue.
1118 * @hw:  HW pointer
1119 * @qidx: Index of queue
1120 * @iq_handler: Handler for this queue
1121 * @priv: Caller's private pointer
1122 *
1123 * This routine walks through every entry of the ingress queue, calling
1124 * the provided iq_handler with the entry, until the generation bit
1125 * flips.
1126 */
1127int
1128csio_wr_process_iq(struct csio_hw *hw, struct csio_q *q,
1129		   void (*iq_handler)(struct csio_hw *, void *,
1130				      uint32_t, struct csio_fl_dma_buf *,
1131				      void *),
1132		   void *priv)
1133{
1134	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1135	void *wr = (void *)((uintptr_t)q->vstart + (q->cidx * q->wr_sz));
1136	struct csio_iqwr_footer *ftr;
1137	uint32_t wr_type, fw_qid, qid;
1138	struct csio_q *q_completed;
1139	struct csio_q *flq = csio_iq_has_fl(q) ?
1140					wrm->q_arr[q->un.iq.flq_idx] : NULL;
1141	int rv = 0;
1142
1143	/* Get the footer */
1144	ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1145					  (q->wr_sz - sizeof(*ftr)));
1146
1147	/*
1148	 * When q wrapped around last time, driver should have inverted
1149	 * ic.genbit as well.
1150	 */
1151	while (csio_is_new_iqwr(q, ftr)) {
1152
1153		CSIO_DB_ASSERT(((uintptr_t)wr + q->wr_sz) <=
1154						(uintptr_t)q->vwrap);
1155		rmb();
1156		wr_type = IQWRF_TYPE_GET(ftr->u.type_gen);
1157
1158		switch (wr_type) {
1159		case X_RSPD_TYPE_CPL:
1160			/* Subtract footer from WR len */
1161			iq_handler(hw, wr, q->wr_sz - sizeof(*ftr), NULL, priv);
1162			break;
1163		case X_RSPD_TYPE_FLBUF:
1164			csio_wr_process_fl(hw, q, wr,
1165					   ntohl(ftr->pldbuflen_qid),
1166					   iq_handler, priv);
1167			break;
1168		case X_RSPD_TYPE_INTR:
1169			fw_qid = ntohl(ftr->pldbuflen_qid);
1170			qid = fw_qid - wrm->fw_iq_start;
1171			q_completed = hw->wrm.intr_map[qid];
1172
1173			if (unlikely(qid ==
1174					csio_q_physiqid(hw, hw->intr_iq_idx))) {
1175				/*
1176				 * We are already in the Forward Interrupt
1177				 * Interrupt Queue Service! Do-not service
1178				 * again!
1179				 *
1180				 */
1181			} else {
1182				CSIO_DB_ASSERT(q_completed);
1183				CSIO_DB_ASSERT(
1184					q_completed->un.iq.iq_intx_handler);
1185
1186				/* Call the queue handler. */
1187				q_completed->un.iq.iq_intx_handler(hw, NULL,
1188						0, NULL, (void *)q_completed);
1189			}
1190			break;
1191		default:
1192			csio_warn(hw, "Unknown resp type 0x%x received\n",
1193				 wr_type);
1194			CSIO_INC_STATS(q, n_rsp_unknown);
1195			break;
1196		}
1197
1198		/*
1199		 * Ingress *always* has fixed size WR entries. Therefore,
1200		 * there should always be complete WRs towards the end of
1201		 * queue.
1202		 */
1203		if (((uintptr_t)wr + q->wr_sz) == (uintptr_t)q->vwrap) {
1204
1205			/* Roll over to start of queue */
1206			q->cidx = 0;
1207			wr	= q->vstart;
1208
1209			/* Toggle genbit */
1210			q->un.iq.genbit ^= 0x1;
1211
1212			CSIO_INC_STATS(q, n_qwrap);
1213		} else {
1214			q->cidx++;
1215			wr	= (void *)((uintptr_t)(q->vstart) +
1216					   (q->cidx * q->wr_sz));
1217		}
1218
1219		ftr = (struct csio_iqwr_footer *)((uintptr_t)wr +
1220						  (q->wr_sz - sizeof(*ftr)));
1221		q->inc_idx++;
1222
1223	} /* while (q->un.iq.genbit == hdr->genbit) */
1224
1225	/*
1226	 * We need to re-arm SGE interrupts in case we got a stray interrupt,
1227	 * especially in msix mode. With INTx, this may be a common occurrence.
1228	 */
1229	if (unlikely(!q->inc_idx)) {
1230		CSIO_INC_STATS(q, n_stray_comp);
1231		rv = -EINVAL;
1232		goto restart;
1233	}
1234
1235	/* Replenish free list buffers if pending falls below low water mark */
1236	if (flq) {
1237		uint32_t avail  = csio_wr_avail_qcredits(flq);
1238		if (avail <= 16) {
1239			/* Make sure in FLQ, atleast 1 credit (8 FL buffers)
1240			 * remains unpopulated otherwise HW thinks
1241			 * FLQ is empty.
1242			 */
1243			csio_wr_update_fl(hw, flq, (flq->credits - 8) - avail);
1244			csio_wr_ring_fldb(hw, flq);
1245		}
1246	}
1247
1248restart:
1249	/* Now inform SGE about our incremental index value */
1250	csio_wr_reg32(hw, CIDXINC_V(q->inc_idx)		|
1251			  INGRESSQID_V(q->un.iq.physiqid)	|
1252			  TIMERREG_V(csio_sge_timer_reg),
1253			  MYPF_REG(SGE_PF_GTS_A));
1254	q->stats.n_tot_rsps += q->inc_idx;
1255
1256	q->inc_idx = 0;
1257
1258	return rv;
1259}
1260
1261int
1262csio_wr_process_iq_idx(struct csio_hw *hw, int qidx,
1263		   void (*iq_handler)(struct csio_hw *, void *,
1264				      uint32_t, struct csio_fl_dma_buf *,
1265				      void *),
1266		   void *priv)
1267{
1268	struct csio_wrm *wrm	= csio_hw_to_wrm(hw);
1269	struct csio_q	*iq	= wrm->q_arr[qidx];
1270
1271	return csio_wr_process_iq(hw, iq, iq_handler, priv);
1272}
1273
1274static int
1275csio_closest_timer(struct csio_sge *s, int time)
1276{
1277	int i, delta, match = 0, min_delta = INT_MAX;
1278
1279	for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1280		delta = time - s->timer_val[i];
1281		if (delta < 0)
1282			delta = -delta;
1283		if (delta < min_delta) {
1284			min_delta = delta;
1285			match = i;
1286		}
1287	}
1288	return match;
1289}
1290
1291static int
1292csio_closest_thresh(struct csio_sge *s, int cnt)
1293{
1294	int i, delta, match = 0, min_delta = INT_MAX;
1295
1296	for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1297		delta = cnt - s->counter_val[i];
1298		if (delta < 0)
1299			delta = -delta;
1300		if (delta < min_delta) {
1301			min_delta = delta;
1302			match = i;
1303		}
1304	}
1305	return match;
1306}
1307
1308static void
1309csio_wr_fixup_host_params(struct csio_hw *hw)
1310{
1311	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1312	struct csio_sge *sge = &wrm->sge;
1313	uint32_t clsz = L1_CACHE_BYTES;
1314	uint32_t s_hps = PAGE_SHIFT - 10;
1315	uint32_t stat_len = clsz > 64 ? 128 : 64;
1316	u32 fl_align = clsz < 32 ? 32 : clsz;
1317	u32 pack_align;
1318	u32 ingpad, ingpack;
1319
1320	csio_wr_reg32(hw, HOSTPAGESIZEPF0_V(s_hps) | HOSTPAGESIZEPF1_V(s_hps) |
1321		      HOSTPAGESIZEPF2_V(s_hps) | HOSTPAGESIZEPF3_V(s_hps) |
1322		      HOSTPAGESIZEPF4_V(s_hps) | HOSTPAGESIZEPF5_V(s_hps) |
1323		      HOSTPAGESIZEPF6_V(s_hps) | HOSTPAGESIZEPF7_V(s_hps),
1324		      SGE_HOST_PAGE_SIZE_A);
1325
1326	/* T5 introduced the separation of the Free List Padding and
1327	 * Packing Boundaries.  Thus, we can select a smaller Padding
1328	 * Boundary to avoid uselessly chewing up PCIe Link and Memory
1329	 * Bandwidth, and use a Packing Boundary which is large enough
1330	 * to avoid false sharing between CPUs, etc.
1331	 *
1332	 * For the PCI Link, the smaller the Padding Boundary the
1333	 * better.  For the Memory Controller, a smaller Padding
1334	 * Boundary is better until we cross under the Memory Line
1335	 * Size (the minimum unit of transfer to/from Memory).  If we
1336	 * have a Padding Boundary which is smaller than the Memory
1337	 * Line Size, that'll involve a Read-Modify-Write cycle on the
1338	 * Memory Controller which is never good.
1339	 */
1340
1341	/* We want the Packing Boundary to be based on the Cache Line
1342	 * Size in order to help avoid False Sharing performance
1343	 * issues between CPUs, etc.  We also want the Packing
1344	 * Boundary to incorporate the PCI-E Maximum Payload Size.  We
1345	 * get best performance when the Packing Boundary is a
1346	 * multiple of the Maximum Payload Size.
1347	 */
1348	pack_align = fl_align;
1349	if (pci_is_pcie(hw->pdev)) {
1350		u32 mps, mps_log;
1351		u16 devctl;
1352
1353		/* The PCIe Device Control Maximum Payload Size field
1354		 * [bits 7:5] encodes sizes as powers of 2 starting at
1355		 * 128 bytes.
1356		 */
1357		pcie_capability_read_word(hw->pdev, PCI_EXP_DEVCTL, &devctl);
1358		mps_log = ((devctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5) + 7;
1359		mps = 1 << mps_log;
1360		if (mps > pack_align)
1361			pack_align = mps;
1362	}
1363
1364	/* T5/T6 have a special interpretation of the "0"
1365	 * value for the Packing Boundary.  This corresponds to 16
1366	 * bytes instead of the expected 32 bytes.
1367	 */
1368	if (pack_align <= 16) {
1369		ingpack = INGPACKBOUNDARY_16B_X;
1370		fl_align = 16;
1371	} else if (pack_align == 32) {
1372		ingpack = INGPACKBOUNDARY_64B_X;
1373		fl_align = 64;
1374	} else {
1375		u32 pack_align_log = fls(pack_align) - 1;
1376
1377		ingpack = pack_align_log - INGPACKBOUNDARY_SHIFT_X;
1378		fl_align = pack_align;
1379	}
1380
1381	/* Use the smallest Ingress Padding which isn't smaller than
1382	 * the Memory Controller Read/Write Size.  We'll take that as
1383	 * being 8 bytes since we don't know of any system with a
1384	 * wider Memory Controller Bus Width.
1385	 */
1386	if (csio_is_t5(hw->pdev->device & CSIO_HW_CHIP_MASK))
1387		ingpad = INGPADBOUNDARY_32B_X;
1388	else
1389		ingpad = T6_INGPADBOUNDARY_8B_X;
1390
1391	csio_set_reg_field(hw, SGE_CONTROL_A,
1392			   INGPADBOUNDARY_V(INGPADBOUNDARY_M) |
1393			   EGRSTATUSPAGESIZE_F,
1394			   INGPADBOUNDARY_V(ingpad) |
1395			   EGRSTATUSPAGESIZE_V(stat_len != 64));
1396	csio_set_reg_field(hw, SGE_CONTROL2_A,
1397			   INGPACKBOUNDARY_V(INGPACKBOUNDARY_M),
1398			   INGPACKBOUNDARY_V(ingpack));
1399
1400	/* FL BUFFER SIZE#0 is Page size i,e already aligned to cache line */
1401	csio_wr_reg32(hw, PAGE_SIZE, SGE_FL_BUFFER_SIZE0_A);
1402
1403	/*
1404	 * If using hard params, the following will get set correctly
1405	 * in csio_wr_set_sge().
1406	 */
1407	if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS) {
1408		csio_wr_reg32(hw,
1409			(csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE2_A) +
1410			fl_align - 1) & ~(fl_align - 1),
1411			SGE_FL_BUFFER_SIZE2_A);
1412		csio_wr_reg32(hw,
1413			(csio_rd_reg32(hw, SGE_FL_BUFFER_SIZE3_A) +
1414			fl_align - 1) & ~(fl_align - 1),
1415			SGE_FL_BUFFER_SIZE3_A);
1416	}
1417
1418	sge->csio_fl_align = fl_align;
1419
1420	csio_wr_reg32(hw, HPZ0_V(PAGE_SHIFT - 12), ULP_RX_TDDP_PSZ_A);
1421
1422	/* default value of rx_dma_offset of the NIC driver */
1423	csio_set_reg_field(hw, SGE_CONTROL_A,
1424			   PKTSHIFT_V(PKTSHIFT_M),
1425			   PKTSHIFT_V(CSIO_SGE_RX_DMA_OFFSET));
1426
1427	csio_hw_tp_wr_bits_indirect(hw, TP_INGRESS_CONFIG_A,
1428				    CSUM_HAS_PSEUDO_HDR_F, 0);
1429}
1430
1431static void
1432csio_init_intr_coalesce_parms(struct csio_hw *hw)
1433{
1434	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1435	struct csio_sge *sge = &wrm->sge;
1436
1437	csio_sge_thresh_reg = csio_closest_thresh(sge, csio_intr_coalesce_cnt);
1438	if (csio_intr_coalesce_cnt) {
1439		csio_sge_thresh_reg = 0;
1440		csio_sge_timer_reg = X_TIMERREG_RESTART_COUNTER;
1441		return;
1442	}
1443
1444	csio_sge_timer_reg = csio_closest_timer(sge, csio_intr_coalesce_time);
1445}
1446
1447/*
1448 * csio_wr_get_sge - Get SGE register values.
1449 * @hw: HW module.
1450 *
1451 * Used by non-master functions and by master-functions relying on config file.
1452 */
1453static void
1454csio_wr_get_sge(struct csio_hw *hw)
1455{
1456	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1457	struct csio_sge *sge = &wrm->sge;
1458	uint32_t ingpad;
1459	int i;
1460	u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
1461	u32 ingress_rx_threshold;
1462
1463	sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);
1464
1465	ingpad = INGPADBOUNDARY_G(sge->sge_control);
1466
1467	switch (ingpad) {
1468	case X_INGPCIEBOUNDARY_32B:
1469		sge->csio_fl_align = 32; break;
1470	case X_INGPCIEBOUNDARY_64B:
1471		sge->csio_fl_align = 64; break;
1472	case X_INGPCIEBOUNDARY_128B:
1473		sge->csio_fl_align = 128; break;
1474	case X_INGPCIEBOUNDARY_256B:
1475		sge->csio_fl_align = 256; break;
1476	case X_INGPCIEBOUNDARY_512B:
1477		sge->csio_fl_align = 512; break;
1478	case X_INGPCIEBOUNDARY_1024B:
1479		sge->csio_fl_align = 1024; break;
1480	case X_INGPCIEBOUNDARY_2048B:
1481		sge->csio_fl_align = 2048; break;
1482	case X_INGPCIEBOUNDARY_4096B:
1483		sge->csio_fl_align = 4096; break;
1484	}
1485
1486	for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1487		csio_get_flbuf_size(hw, sge, i);
1488
1489	timer_value_0_and_1 = csio_rd_reg32(hw, SGE_TIMER_VALUE_0_AND_1_A);
1490	timer_value_2_and_3 = csio_rd_reg32(hw, SGE_TIMER_VALUE_2_AND_3_A);
1491	timer_value_4_and_5 = csio_rd_reg32(hw, SGE_TIMER_VALUE_4_AND_5_A);
1492
1493	sge->timer_val[0] = (uint16_t)csio_core_ticks_to_us(hw,
1494					TIMERVALUE0_G(timer_value_0_and_1));
1495	sge->timer_val[1] = (uint16_t)csio_core_ticks_to_us(hw,
1496					TIMERVALUE1_G(timer_value_0_and_1));
1497	sge->timer_val[2] = (uint16_t)csio_core_ticks_to_us(hw,
1498					TIMERVALUE2_G(timer_value_2_and_3));
1499	sge->timer_val[3] = (uint16_t)csio_core_ticks_to_us(hw,
1500					TIMERVALUE3_G(timer_value_2_and_3));
1501	sge->timer_val[4] = (uint16_t)csio_core_ticks_to_us(hw,
1502					TIMERVALUE4_G(timer_value_4_and_5));
1503	sge->timer_val[5] = (uint16_t)csio_core_ticks_to_us(hw,
1504					TIMERVALUE5_G(timer_value_4_and_5));
1505
1506	ingress_rx_threshold = csio_rd_reg32(hw, SGE_INGRESS_RX_THRESHOLD_A);
1507	sge->counter_val[0] = THRESHOLD_0_G(ingress_rx_threshold);
1508	sge->counter_val[1] = THRESHOLD_1_G(ingress_rx_threshold);
1509	sge->counter_val[2] = THRESHOLD_2_G(ingress_rx_threshold);
1510	sge->counter_val[3] = THRESHOLD_3_G(ingress_rx_threshold);
1511
1512	csio_init_intr_coalesce_parms(hw);
1513}
1514
1515/*
1516 * csio_wr_set_sge - Initialize SGE registers
1517 * @hw: HW module.
1518 *
1519 * Used by Master function to initialize SGE registers in the absence
1520 * of a config file.
1521 */
1522static void
1523csio_wr_set_sge(struct csio_hw *hw)
1524{
1525	struct csio_wrm *wrm = csio_hw_to_wrm(hw);
1526	struct csio_sge *sge = &wrm->sge;
1527	int i;
1528
1529	/*
1530	 * Set up our basic SGE mode to deliver CPL messages to our Ingress
1531	 * Queue and Packet Date to the Free List.
1532	 */
1533	csio_set_reg_field(hw, SGE_CONTROL_A, RXPKTCPLMODE_F, RXPKTCPLMODE_F);
1534
1535	sge->sge_control = csio_rd_reg32(hw, SGE_CONTROL_A);
1536
1537	/* sge->csio_fl_align is set up by csio_wr_fixup_host_params(). */
1538
1539	/*
1540	 * Set up to drop DOORBELL writes when the DOORBELL FIFO overflows
1541	 * and generate an interrupt when this occurs so we can recover.
1542	 */
1543	csio_set_reg_field(hw, SGE_DBFIFO_STATUS_A,
1544			   LP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
1545			   LP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
1546	csio_set_reg_field(hw, SGE_DBFIFO_STATUS2_A,
1547			   HP_INT_THRESH_T5_V(LP_INT_THRESH_T5_M),
1548			   HP_INT_THRESH_T5_V(CSIO_SGE_DBFIFO_INT_THRESH));
1549
1550	csio_set_reg_field(hw, SGE_DOORBELL_CONTROL_A, ENABLE_DROP_F,
1551			   ENABLE_DROP_F);
1552
1553	/* SGE_FL_BUFFER_SIZE0 is set up by csio_wr_fixup_host_params(). */
1554
1555	CSIO_SET_FLBUF_SIZE(hw, 1, CSIO_SGE_FLBUF_SIZE1);
1556	csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE2 + sge->csio_fl_align - 1)
1557		      & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE2_A);
1558	csio_wr_reg32(hw, (CSIO_SGE_FLBUF_SIZE3 + sge->csio_fl_align - 1)
1559		      & ~(sge->csio_fl_align - 1), SGE_FL_BUFFER_SIZE3_A);
1560	CSIO_SET_FLBUF_SIZE(hw, 4, CSIO_SGE_FLBUF_SIZE4);
1561	CSIO_SET_FLBUF_SIZE(hw, 5, CSIO_SGE_FLBUF_SIZE5);
1562	CSIO_SET_FLBUF_SIZE(hw, 6, CSIO_SGE_FLBUF_SIZE6);
1563	CSIO_SET_FLBUF_SIZE(hw, 7, CSIO_SGE_FLBUF_SIZE7);
1564	CSIO_SET_FLBUF_SIZE(hw, 8, CSIO_SGE_FLBUF_SIZE8);
1565
1566	for (i = 0; i < CSIO_SGE_FL_SIZE_REGS; i++)
1567		csio_get_flbuf_size(hw, sge, i);
1568
1569	/* Initialize interrupt coalescing attributes */
1570	sge->timer_val[0] = CSIO_SGE_TIMER_VAL_0;
1571	sge->timer_val[1] = CSIO_SGE_TIMER_VAL_1;
1572	sge->timer_val[2] = CSIO_SGE_TIMER_VAL_2;
1573	sge->timer_val[3] = CSIO_SGE_TIMER_VAL_3;
1574	sge->timer_val[4] = CSIO_SGE_TIMER_VAL_4;
1575	sge->timer_val[5] = CSIO_SGE_TIMER_VAL_5;
1576
1577	sge->counter_val[0] = CSIO_SGE_INT_CNT_VAL_0;
1578	sge->counter_val[1] = CSIO_SGE_INT_CNT_VAL_1;
1579	sge->counter_val[2] = CSIO_SGE_INT_CNT_VAL_2;
1580	sge->counter_val[3] = CSIO_SGE_INT_CNT_VAL_3;
1581
1582	csio_wr_reg32(hw, THRESHOLD_0_V(sge->counter_val[0]) |
1583		      THRESHOLD_1_V(sge->counter_val[1]) |
1584		      THRESHOLD_2_V(sge->counter_val[2]) |
1585		      THRESHOLD_3_V(sge->counter_val[3]),
1586		      SGE_INGRESS_RX_THRESHOLD_A);
1587
1588	csio_wr_reg32(hw,
1589		   TIMERVALUE0_V(csio_us_to_core_ticks(hw, sge->timer_val[0])) |
1590		   TIMERVALUE1_V(csio_us_to_core_ticks(hw, sge->timer_val[1])),
1591		   SGE_TIMER_VALUE_0_AND_1_A);
1592
1593	csio_wr_reg32(hw,
1594		   TIMERVALUE2_V(csio_us_to_core_ticks(hw, sge->timer_val[2])) |
1595		   TIMERVALUE3_V(csio_us_to_core_ticks(hw, sge->timer_val[3])),
1596		   SGE_TIMER_VALUE_2_AND_3_A);
1597
1598	csio_wr_reg32(hw,
1599		   TIMERVALUE4_V(csio_us_to_core_ticks(hw, sge->timer_val[4])) |
1600		   TIMERVALUE5_V(csio_us_to_core_ticks(hw, sge->timer_val[5])),
1601		   SGE_TIMER_VALUE_4_AND_5_A);
1602
1603	csio_init_intr_coalesce_parms(hw);
1604}
1605
1606void
1607csio_wr_sge_init(struct csio_hw *hw)
1608{
1609	/*
1610	 * If we are master and chip is not initialized:
1611	 *    - If we plan to use the config file, we need to fixup some
1612	 *      host specific registers, and read the rest of the SGE
1613	 *      configuration.
1614	 *    - If we dont plan to use the config file, we need to initialize
1615	 *      SGE entirely, including fixing the host specific registers.
1616	 * If we are master and chip is initialized, just read and work off of
1617	 *	the already initialized SGE values.
1618	 * If we arent the master, we are only allowed to read and work off of
1619	 *      the already initialized SGE values.
1620	 *
1621	 * Therefore, before calling this function, we assume that the master-
1622	 * ship of the card, state and whether to use config file or not, have
1623	 * already been decided.
1624	 */
1625	if (csio_is_hw_master(hw)) {
1626		if (hw->fw_state != CSIO_DEV_STATE_INIT)
1627			csio_wr_fixup_host_params(hw);
1628
1629		if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS)
1630			csio_wr_get_sge(hw);
1631		else
1632			csio_wr_set_sge(hw);
1633	} else
1634		csio_wr_get_sge(hw);
1635}
1636
1637/*
1638 * csio_wrm_init - Initialize Work request module.
1639 * @wrm: WR module
1640 * @hw: HW pointer
1641 *
1642 * Allocates memory for an array of queue pointers starting at q_arr.
1643 */
1644int
1645csio_wrm_init(struct csio_wrm *wrm, struct csio_hw *hw)
1646{
1647	int i;
1648
1649	if (!wrm->num_q) {
1650		csio_err(hw, "Num queues is not set\n");
1651		return -EINVAL;
1652	}
1653
1654	wrm->q_arr = kcalloc(wrm->num_q, sizeof(struct csio_q *), GFP_KERNEL);
1655	if (!wrm->q_arr)
1656		goto err;
1657
1658	for (i = 0; i < wrm->num_q; i++) {
1659		wrm->q_arr[i] = kzalloc(sizeof(struct csio_q), GFP_KERNEL);
1660		if (!wrm->q_arr[i]) {
1661			while (--i >= 0)
1662				kfree(wrm->q_arr[i]);
1663			goto err_free_arr;
1664		}
1665	}
1666	wrm->free_qidx	= 0;
1667
1668	return 0;
1669
1670err_free_arr:
1671	kfree(wrm->q_arr);
1672err:
1673	return -ENOMEM;
1674}
1675
1676/*
1677 * csio_wrm_exit - Initialize Work request module.
1678 * @wrm: WR module
1679 * @hw: HW module
1680 *
1681 * Uninitialize WR module. Free q_arr and pointers in it.
1682 * We have the additional job of freeing the DMA memory associated
1683 * with the queues.
1684 */
1685void
1686csio_wrm_exit(struct csio_wrm *wrm, struct csio_hw *hw)
1687{
1688	int i;
1689	uint32_t j;
1690	struct csio_q *q;
1691	struct csio_dma_buf *buf;
1692
1693	for (i = 0; i < wrm->num_q; i++) {
1694		q = wrm->q_arr[i];
1695
1696		if (wrm->free_qidx && (i < wrm->free_qidx)) {
1697			if (q->type == CSIO_FREELIST) {
1698				if (!q->un.fl.bufs)
1699					continue;
1700				for (j = 0; j < q->credits; j++) {
1701					buf = &q->un.fl.bufs[j];
1702					if (!buf->vaddr)
1703						continue;
1704					dma_free_coherent(&hw->pdev->dev,
1705							buf->len, buf->vaddr,
1706							buf->paddr);
1707				}
1708				kfree(q->un.fl.bufs);
1709			}
1710			dma_free_coherent(&hw->pdev->dev, q->size,
1711					q->vstart, q->pstart);
1712		}
1713		kfree(q);
1714	}
1715
1716	hw->flags &= ~CSIO_HWF_Q_MEM_ALLOCED;
1717
1718	kfree(wrm->q_arr);
1719}