puda.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2015 - 2021 Intel Corporation */
   3#include "osdep.h"
   4#include "hmc.h"
   5#include "defs.h"
   6#include "type.h"
   7#include "protos.h"
   8#include "puda.h"
   9#include "ws.h"
  10
  11static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
  12			      struct irdma_puda_buf *buf);
  13static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid);
  14static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
  15				     struct irdma_puda_buf *buf, u32 wqe_idx);
  16/**
  17 * irdma_puda_get_listbuf - get buffer from puda list
  18 * @list: list to use for buffers (ILQ or IEQ)
  19 */
  20static struct irdma_puda_buf *irdma_puda_get_listbuf(struct list_head *list)
  21{
  22	struct irdma_puda_buf *buf = NULL;
  23
  24	if (!list_empty(list)) {
  25		buf = (struct irdma_puda_buf *)list->next;
  26		list_del((struct list_head *)&buf->list);
  27	}
  28
  29	return buf;
  30}
  31
  32/**
  33 * irdma_puda_get_bufpool - return buffer from resource
  34 * @rsrc: resource to use for buffer
  35 */
  36struct irdma_puda_buf *irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc)
  37{
  38	struct irdma_puda_buf *buf = NULL;
  39	struct list_head *list = &rsrc->bufpool;
  40	unsigned long flags;
  41
  42	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
  43	buf = irdma_puda_get_listbuf(list);
  44	if (buf) {
  45		rsrc->avail_buf_count--;
  46		buf->vsi = rsrc->vsi;
  47	} else {
  48		rsrc->stats_buf_alloc_fail++;
  49	}
  50	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
  51
  52	return buf;
  53}
  54
  55/**
  56 * irdma_puda_ret_bufpool - return buffer to rsrc list
  57 * @rsrc: resource to use for buffer
  58 * @buf: buffer to return to resource
  59 */
  60void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,
  61			    struct irdma_puda_buf *buf)
  62{
  63	unsigned long flags;
  64
  65	buf->do_lpb = false;
  66	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
  67	list_add(&buf->list, &rsrc->bufpool);
  68	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
  69	rsrc->avail_buf_count++;
  70}
  71
  72/**
  73 * irdma_puda_post_recvbuf - set wqe for rcv buffer
  74 * @rsrc: resource ptr
  75 * @wqe_idx: wqe index to use
  76 * @buf: puda buffer for rcv q
  77 * @initial: flag if during init time
  78 */
  79static void irdma_puda_post_recvbuf(struct irdma_puda_rsrc *rsrc, u32 wqe_idx,
  80				    struct irdma_puda_buf *buf, bool initial)
  81{
  82	__le64 *wqe;
  83	struct irdma_sc_qp *qp = &rsrc->qp;
  84	u64 offset24 = 0;
  85
  86	/* Synch buffer for use by device */
  87	dma_sync_single_for_device(rsrc->dev->hw->device, buf->mem.pa,
  88				   buf->mem.size, DMA_BIDIRECTIONAL);
  89	qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;
  90	wqe = qp->qp_uk.rq_base[wqe_idx].elem;
  91	if (!initial)
  92		get_64bit_val(wqe, 24, &offset24);
  93
  94	offset24 = (offset24) ? 0 : FIELD_PREP(IRDMAQPSQ_VALID, 1);
  95
  96	set_64bit_val(wqe, 16, 0);
  97	set_64bit_val(wqe, 0, buf->mem.pa);
  98	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
  99		set_64bit_val(wqe, 8,
 100			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, buf->mem.size));
 101	} else {
 102		set_64bit_val(wqe, 8,
 103			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, buf->mem.size) |
 104			      offset24);
 105	}
 106	dma_wmb(); /* make sure WQE is written before valid bit is set */
 107
 108	set_64bit_val(wqe, 24, offset24);
 109}
 110
 111/**
 112 * irdma_puda_replenish_rq - post rcv buffers
 113 * @rsrc: resource to use for buffer
 114 * @initial: flag if during init time
 115 */
 116static int irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)
 117{
 118	u32 i;
 119	u32 invalid_cnt = rsrc->rxq_invalid_cnt;
 120	struct irdma_puda_buf *buf = NULL;
 121
 122	for (i = 0; i < invalid_cnt; i++) {
 123		buf = irdma_puda_get_bufpool(rsrc);
 124		if (!buf)
 125			return -ENOBUFS;
 126		irdma_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf, initial);
 127		rsrc->rx_wqe_idx = ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);
 128		rsrc->rxq_invalid_cnt--;
 129	}
 130
 131	return 0;
 132}
 133
 134/**
 135 * irdma_puda_alloc_buf - allocate mem for buffer
 136 * @dev: iwarp device
 137 * @len: length of buffer
 138 */
 139static struct irdma_puda_buf *irdma_puda_alloc_buf(struct irdma_sc_dev *dev,
 140						   u32 len)
 141{
 142	struct irdma_puda_buf *buf;
 143	struct irdma_virt_mem buf_mem;
 144
 145	buf_mem.size = sizeof(struct irdma_puda_buf);
 146	buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
 147	if (!buf_mem.va)
 148		return NULL;
 149
 150	buf = buf_mem.va;
 151	buf->mem.size = len;
 152	buf->mem.va = kzalloc(buf->mem.size, GFP_KERNEL);
 153	if (!buf->mem.va)
 154		goto free_virt;
 155	buf->mem.pa = dma_map_single(dev->hw->device, buf->mem.va,
 156				     buf->mem.size, DMA_BIDIRECTIONAL);
 157	if (dma_mapping_error(dev->hw->device, buf->mem.pa)) {
 158		kfree(buf->mem.va);
 159		goto free_virt;
 160	}
 161
 162	buf->buf_mem.va = buf_mem.va;
 163	buf->buf_mem.size = buf_mem.size;
 164
 165	return buf;
 166
 167free_virt:
 168	kfree(buf_mem.va);
 169	return NULL;
 170}
 171
 172/**
 173 * irdma_puda_dele_buf - delete buffer back to system
 174 * @dev: iwarp device
 175 * @buf: buffer to free
 176 */
 177static void irdma_puda_dele_buf(struct irdma_sc_dev *dev,
 178				struct irdma_puda_buf *buf)
 179{
 180	dma_unmap_single(dev->hw->device, buf->mem.pa, buf->mem.size,
 181			 DMA_BIDIRECTIONAL);
 182	kfree(buf->mem.va);
 183	kfree(buf->buf_mem.va);
 184}
 185
 186/**
 187 * irdma_puda_get_next_send_wqe - return next wqe for processing
 188 * @qp: puda qp for wqe
 189 * @wqe_idx: wqe index for caller
 190 */
 191static __le64 *irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp,
 192					    u32 *wqe_idx)
 193{
 194	int ret_code = 0;
 195
 196	*wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
 197	if (!*wqe_idx)
 198		qp->swqe_polarity = !qp->swqe_polarity;
 199	IRDMA_RING_MOVE_HEAD(qp->sq_ring, ret_code);
 200	if (ret_code)
 201		return NULL;
 202
 203	return qp->sq_base[*wqe_idx].elem;
 204}
 205
 206/**
 207 * irdma_puda_poll_info - poll cq for completion
 208 * @cq: cq for poll
 209 * @info: info return for successful completion
 210 */
 211static int irdma_puda_poll_info(struct irdma_sc_cq *cq,
 212				struct irdma_puda_cmpl_info *info)
 213{
 214	struct irdma_cq_uk *cq_uk = &cq->cq_uk;
 215	u64 qword0, qword2, qword3, qword6;
 216	__le64 *cqe;
 217	__le64 *ext_cqe = NULL;
 218	u64 qword7 = 0;
 219	u64 comp_ctx;
 220	bool valid_bit;
 221	bool ext_valid = 0;
 222	u32 major_err, minor_err;
 223	u32 peek_head;
 224	bool error;
 225	u8 polarity;
 226
 227	cqe = IRDMA_GET_CURRENT_CQ_ELEM(&cq->cq_uk);
 228	get_64bit_val(cqe, 24, &qword3);
 229	valid_bit = (bool)FIELD_GET(IRDMA_CQ_VALID, qword3);
 230	if (valid_bit != cq_uk->polarity)
 231		return -ENOENT;
 232
 233	/* Ensure CQE contents are read after valid bit is checked */
 234	dma_rmb();
 235
 236	if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
 237		ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
 238
 239	if (ext_valid) {
 240		peek_head = (cq_uk->cq_ring.head + 1) % cq_uk->cq_ring.size;
 241		ext_cqe = cq_uk->cq_base[peek_head].buf;
 242		get_64bit_val(ext_cqe, 24, &qword7);
 243		polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
 244		if (!peek_head)
 245			polarity ^= 1;
 246		if (polarity != cq_uk->polarity)
 247			return -ENOENT;
 248
 249		/* Ensure ext CQE contents are read after ext valid bit is checked */
 250		dma_rmb();
 251
 252		IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
 253		if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
 254			cq_uk->polarity = !cq_uk->polarity;
 255		/* update cq tail in cq shadow memory also */
 256		IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
 257	}
 258
 259	print_hex_dump_debug("PUDA: PUDA CQE", DUMP_PREFIX_OFFSET, 16, 8, cqe,
 260			     32, false);
 261	if (ext_valid)
 262		print_hex_dump_debug("PUDA: PUDA EXT-CQE", DUMP_PREFIX_OFFSET,
 263				     16, 8, ext_cqe, 32, false);
 264
 265	error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
 266	if (error) {
 267		ibdev_dbg(to_ibdev(cq->dev), "PUDA: receive error\n");
 268		major_err = (u32)(FIELD_GET(IRDMA_CQ_MAJERR, qword3));
 269		minor_err = (u32)(FIELD_GET(IRDMA_CQ_MINERR, qword3));
 270		info->compl_error = major_err << 16 | minor_err;
 271		return -EIO;
 272	}
 273
 274	get_64bit_val(cqe, 0, &qword0);
 275	get_64bit_val(cqe, 16, &qword2);
 276
 277	info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
 278	info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
 279	if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
 280		info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
 281
 282	get_64bit_val(cqe, 8, &comp_ctx);
 283	info->qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
 284	info->wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
 285
 286	if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
 287		if (ext_valid) {
 288			info->vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
 289			if (info->vlan_valid) {
 290				get_64bit_val(ext_cqe, 16, &qword6);
 291				info->vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
 292			}
 293			info->smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
 294			if (info->smac_valid) {
 295				get_64bit_val(ext_cqe, 16, &qword6);
 296				info->smac[0] = (u8)((qword6 >> 40) & 0xFF);
 297				info->smac[1] = (u8)((qword6 >> 32) & 0xFF);
 298				info->smac[2] = (u8)((qword6 >> 24) & 0xFF);
 299				info->smac[3] = (u8)((qword6 >> 16) & 0xFF);
 300				info->smac[4] = (u8)((qword6 >> 8) & 0xFF);
 301				info->smac[5] = (u8)(qword6 & 0xFF);
 302			}
 303		}
 304
 305		if (cq->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
 306			info->vlan_valid = (bool)FIELD_GET(IRDMA_VLAN_TAG_VALID, qword3);
 307			info->l4proto = (u8)FIELD_GET(IRDMA_UDA_L4PROTO, qword2);
 308			info->l3proto = (u8)FIELD_GET(IRDMA_UDA_L3PROTO, qword2);
 309		}
 310
 311		info->payload_len = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
 312	}
 313
 314	return 0;
 315}
 316
 317/**
 318 * irdma_puda_poll_cmpl - processes completion for cq
 319 * @dev: iwarp device
 320 * @cq: cq getting interrupt
 321 * @compl_err: return any completion err
 322 */
 323int irdma_puda_poll_cmpl(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq,
 324			 u32 *compl_err)
 325{
 326	struct irdma_qp_uk *qp;
 327	struct irdma_cq_uk *cq_uk = &cq->cq_uk;
 328	struct irdma_puda_cmpl_info info = {};
 329	int ret = 0;
 330	struct irdma_puda_buf *buf;
 331	struct irdma_puda_rsrc *rsrc;
 332	u8 cq_type = cq->cq_type;
 333	unsigned long flags;
 334
 335	if (cq_type == IRDMA_CQ_TYPE_ILQ || cq_type == IRDMA_CQ_TYPE_IEQ) {
 336		rsrc = (cq_type == IRDMA_CQ_TYPE_ILQ) ? cq->vsi->ilq :
 337							cq->vsi->ieq;
 338	} else {
 339		ibdev_dbg(to_ibdev(dev), "PUDA: qp_type error\n");
 340		return -EINVAL;
 341	}
 342
 343	ret = irdma_puda_poll_info(cq, &info);
 344	*compl_err = info.compl_error;
 345	if (ret == -ENOENT)
 346		return ret;
 347	if (ret)
 348		goto done;
 349
 350	qp = info.qp;
 351	if (!qp || !rsrc) {
 352		ret = -EFAULT;
 353		goto done;
 354	}
 355
 356	if (qp->qp_id != rsrc->qp_id) {
 357		ret = -EFAULT;
 358		goto done;
 359	}
 360
 361	if (info.q_type == IRDMA_CQE_QTYPE_RQ) {
 362		buf = (struct irdma_puda_buf *)(uintptr_t)
 363			      qp->rq_wrid_array[info.wqe_idx];
 364
 365		/* reusing so synch the buffer for CPU use */
 366		dma_sync_single_for_cpu(dev->hw->device, buf->mem.pa,
 367					buf->mem.size, DMA_BIDIRECTIONAL);
 368		/* Get all the tcpip information in the buf header */
 369		ret = irdma_puda_get_tcpip_info(&info, buf);
 370		if (ret) {
 371			rsrc->stats_rcvd_pkt_err++;
 372			if (cq_type == IRDMA_CQ_TYPE_ILQ) {
 373				irdma_ilq_putback_rcvbuf(&rsrc->qp, buf,
 374							 info.wqe_idx);
 375			} else {
 376				irdma_puda_ret_bufpool(rsrc, buf);
 377				irdma_puda_replenish_rq(rsrc, false);
 378			}
 379			goto done;
 380		}
 381
 382		rsrc->stats_pkt_rcvd++;
 383		rsrc->compl_rxwqe_idx = info.wqe_idx;
 384		ibdev_dbg(to_ibdev(dev), "PUDA: RQ completion\n");
 385		rsrc->receive(rsrc->vsi, buf);
 386		if (cq_type == IRDMA_CQ_TYPE_ILQ)
 387			irdma_ilq_putback_rcvbuf(&rsrc->qp, buf, info.wqe_idx);
 388		else
 389			irdma_puda_replenish_rq(rsrc, false);
 390
 391	} else {
 392		ibdev_dbg(to_ibdev(dev), "PUDA: SQ completion\n");
 393		buf = (struct irdma_puda_buf *)(uintptr_t)
 394					qp->sq_wrtrk_array[info.wqe_idx].wrid;
 395
 396		/* reusing so synch the buffer for CPU use */
 397		dma_sync_single_for_cpu(dev->hw->device, buf->mem.pa,
 398					buf->mem.size, DMA_BIDIRECTIONAL);
 399		IRDMA_RING_SET_TAIL(qp->sq_ring, info.wqe_idx);
 400		rsrc->xmit_complete(rsrc->vsi, buf);
 401		spin_lock_irqsave(&rsrc->bufpool_lock, flags);
 402		rsrc->tx_wqe_avail_cnt++;
 403		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
 404		if (!list_empty(&rsrc->txpend))
 405			irdma_puda_send_buf(rsrc, NULL);
 406	}
 407
 408done:
 409	IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
 410	if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
 411		cq_uk->polarity = !cq_uk->polarity;
 412	/* update cq tail in cq shadow memory also */
 413	IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
 414	set_64bit_val(cq_uk->shadow_area, 0,
 415		      IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring));
 416
 417	return ret;
 418}
 419
 420/**
 421 * irdma_puda_send - complete send wqe for transmit
 422 * @qp: puda qp for send
 423 * @info: buffer information for transmit
 424 */
 425int irdma_puda_send(struct irdma_sc_qp *qp, struct irdma_puda_send_info *info)
 426{
 427	__le64 *wqe;
 428	u32 iplen, l4len;
 429	u64 hdr[2];
 430	u32 wqe_idx;
 431	u8 iipt;
 432
 433	/* number of 32 bits DWORDS in header */
 434	l4len = info->tcplen >> 2;
 435	if (info->ipv4) {
 436		iipt = 3;
 437		iplen = 5;
 438	} else {
 439		iipt = 1;
 440		iplen = 10;
 441	}
 442
 443	wqe = irdma_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx);
 444	if (!wqe)
 445		return -ENOMEM;
 446
 447	qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch;
 448	/* Third line of WQE descriptor */
 449	/* maclen is in words */
 450
 451	if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
 452		hdr[0] = 0; /* Dest_QPN and Dest_QKey only for UD */
 453		hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
 454			 FIELD_PREP(IRDMA_UDA_QPSQ_L4LEN, l4len) |
 455			 FIELD_PREP(IRDMAQPSQ_AHID, info->ah_id) |
 456			 FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
 457			 FIELD_PREP(IRDMA_UDA_QPSQ_VALID,
 458				    qp->qp_uk.swqe_polarity);
 459
 460		/* Forth line of WQE descriptor */
 461
 462		set_64bit_val(wqe, 0, info->paddr);
 463		set_64bit_val(wqe, 8,
 464			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, info->len) |
 465			      FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity));
 466	} else {
 467		hdr[0] = FIELD_PREP(IRDMA_UDA_QPSQ_MACLEN, info->maclen >> 1) |
 468			 FIELD_PREP(IRDMA_UDA_QPSQ_IPLEN, iplen) |
 469			 FIELD_PREP(IRDMA_UDA_QPSQ_L4T, 1) |
 470			 FIELD_PREP(IRDMA_UDA_QPSQ_IIPT, iipt) |
 471			 FIELD_PREP(IRDMA_GEN1_UDA_QPSQ_L4LEN, l4len);
 472
 473		hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
 474			 FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
 475			 FIELD_PREP(IRDMA_UDA_QPSQ_DOLOOPBACK, info->do_lpb) |
 476			 FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity);
 477
 478		/* Forth line of WQE descriptor */
 479
 480		set_64bit_val(wqe, 0, info->paddr);
 481		set_64bit_val(wqe, 8,
 482			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, info->len));
 483	}
 484
 485	set_64bit_val(wqe, 16, hdr[0]);
 486	dma_wmb(); /* make sure WQE is written before valid bit is set */
 487
 488	set_64bit_val(wqe, 24, hdr[1]);
 489
 490	print_hex_dump_debug("PUDA: PUDA SEND WQE", DUMP_PREFIX_OFFSET, 16, 8,
 491			     wqe, 32, false);
 492	irdma_uk_qp_post_wr(&qp->qp_uk);
 493	return 0;
 494}
 495
 496/**
 497 * irdma_puda_send_buf - transmit puda buffer
 498 * @rsrc: resource to use for buffer
 499 * @buf: puda buffer to transmit
 500 */
 501void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,
 502			 struct irdma_puda_buf *buf)
 503{
 504	struct irdma_puda_send_info info;
 505	int ret = 0;
 506	unsigned long flags;
 507
 508	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
 509	/* if no wqe available or not from a completion and we have
 510	 * pending buffers, we must queue new buffer
 511	 */
 512	if (!rsrc->tx_wqe_avail_cnt || (buf && !list_empty(&rsrc->txpend))) {
 513		list_add_tail(&buf->list, &rsrc->txpend);
 514		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
 515		rsrc->stats_sent_pkt_q++;
 516		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
 517			ibdev_dbg(to_ibdev(rsrc->dev),
 518				  "PUDA: adding to txpend\n");
 519		return;
 520	}
 521	rsrc->tx_wqe_avail_cnt--;
 522	/* if we are coming from a completion and have pending buffers
 523	 * then Get one from pending list
 524	 */
 525	if (!buf) {
 526		buf = irdma_puda_get_listbuf(&rsrc->txpend);
 527		if (!buf)
 528			goto done;
 529	}
 530
 531	info.scratch = buf;
 532	info.paddr = buf->mem.pa;
 533	info.len = buf->totallen;
 534	info.tcplen = buf->tcphlen;
 535	info.ipv4 = buf->ipv4;
 536
 537	if (rsrc->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
 538		info.ah_id = buf->ah_id;
 539	} else {
 540		info.maclen = buf->maclen;
 541		info.do_lpb = buf->do_lpb;
 542	}
 543
 544	/* Synch buffer for use by device */
 545	dma_sync_single_for_cpu(rsrc->dev->hw->device, buf->mem.pa,
 546				buf->mem.size, DMA_BIDIRECTIONAL);
 547	ret = irdma_puda_send(&rsrc->qp, &info);
 548	if (ret) {
 549		rsrc->tx_wqe_avail_cnt++;
 550		rsrc->stats_sent_pkt_q++;
 551		list_add(&buf->list, &rsrc->txpend);
 552		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
 553			ibdev_dbg(to_ibdev(rsrc->dev),
 554				  "PUDA: adding to puda_send\n");
 555	} else {
 556		rsrc->stats_pkt_sent++;
 557	}
 558done:
 559	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
 560}
 561
 562/**
 563 * irdma_puda_qp_setctx - during init, set qp's context
 564 * @rsrc: qp's resource
 565 */
 566static void irdma_puda_qp_setctx(struct irdma_puda_rsrc *rsrc)
 567{
 568	struct irdma_sc_qp *qp = &rsrc->qp;
 569	__le64 *qp_ctx = qp->hw_host_ctx;
 570
 571	set_64bit_val(qp_ctx, 8, qp->sq_pa);
 572	set_64bit_val(qp_ctx, 16, qp->rq_pa);
 573	set_64bit_val(qp_ctx, 24,
 574		      FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
 575		      FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size));
 576	set_64bit_val(qp_ctx, 48,
 577		      FIELD_PREP(IRDMAQPC_SNDMSS, rsrc->buf_size));
 578	set_64bit_val(qp_ctx, 56, 0);
 579	if (qp->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
 580		set_64bit_val(qp_ctx, 64, 1);
 581	set_64bit_val(qp_ctx, 136,
 582		      FIELD_PREP(IRDMAQPC_TXCQNUM, rsrc->cq_id) |
 583		      FIELD_PREP(IRDMAQPC_RXCQNUM, rsrc->cq_id));
 584	set_64bit_val(qp_ctx, 144,
 585		      FIELD_PREP(IRDMAQPC_STAT_INDEX, rsrc->stats_idx));
 586	set_64bit_val(qp_ctx, 160,
 587		      FIELD_PREP(IRDMAQPC_PRIVEN, 1) |
 588		      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, rsrc->stats_idx_valid));
 589	set_64bit_val(qp_ctx, 168,
 590		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, (uintptr_t)qp));
 591	set_64bit_val(qp_ctx, 176,
 592		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
 593		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
 594		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
 595
 596	print_hex_dump_debug("PUDA: PUDA QP CONTEXT", DUMP_PREFIX_OFFSET, 16,
 597			     8, qp_ctx, IRDMA_QP_CTX_SIZE, false);
 598}
 599
 600/**
 601 * irdma_puda_qp_wqe - setup wqe for qp create
 602 * @dev: Device
 603 * @qp: Resource qp
 604 */
 605static int irdma_puda_qp_wqe(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
 606{
 607	struct irdma_sc_cqp *cqp;
 608	__le64 *wqe;
 609	u64 hdr;
 610	struct irdma_ccq_cqe_info compl_info;
 611	int status = 0;
 612
 613	cqp = dev->cqp;
 614	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
 615	if (!wqe)
 616		return -ENOMEM;
 617
 618	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
 619	set_64bit_val(wqe, 40, qp->shadow_area_pa);
 620
 621	hdr = qp->qp_uk.qp_id |
 622	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
 623	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, IRDMA_QP_TYPE_UDA) |
 624	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, 1) |
 625	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, 2) |
 626	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 627	dma_wmb(); /* make sure WQE is written before valid bit is set */
 628
 629	set_64bit_val(wqe, 24, hdr);
 630
 631	print_hex_dump_debug("PUDA: PUDA QP CREATE", DUMP_PREFIX_OFFSET, 16,
 632			     8, wqe, 40, false);
 633	irdma_sc_cqp_post_sq(cqp);
 634	status = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_CREATE_QP,
 635					       &compl_info);
 636
 637	return status;
 638}
 639
 640/**
 641 * irdma_puda_qp_create - create qp for resource
 642 * @rsrc: resource to use for buffer
 643 */
 644static int irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
 645{
 646	struct irdma_sc_qp *qp = &rsrc->qp;
 647	struct irdma_qp_uk *ukqp = &qp->qp_uk;
 648	int ret = 0;
 649	u32 sq_size, rq_size;
 650	struct irdma_dma_mem *mem;
 651
 652	sq_size = rsrc->sq_size * IRDMA_QP_WQE_MIN_SIZE;
 653	rq_size = rsrc->rq_size * IRDMA_QP_WQE_MIN_SIZE;
 654	rsrc->qpmem.size = ALIGN((sq_size + rq_size + (IRDMA_SHADOW_AREA_SIZE << 3) + IRDMA_QP_CTX_SIZE),
 655				 IRDMA_HW_PAGE_SIZE);
 656	rsrc->qpmem.va = dma_alloc_coherent(rsrc->dev->hw->device,
 657					    rsrc->qpmem.size, &rsrc->qpmem.pa,
 658					    GFP_KERNEL);
 659	if (!rsrc->qpmem.va)
 660		return -ENOMEM;
 661
 662	mem = &rsrc->qpmem;
 663	memset(mem->va, 0, rsrc->qpmem.size);
 664	qp->hw_sq_size = irdma_get_encoded_wqe_size(rsrc->sq_size, IRDMA_QUEUE_TYPE_SQ_RQ);
 665	qp->hw_rq_size = irdma_get_encoded_wqe_size(rsrc->rq_size, IRDMA_QUEUE_TYPE_SQ_RQ);
 666	qp->pd = &rsrc->sc_pd;
 667	qp->qp_uk.qp_type = IRDMA_QP_TYPE_UDA;
 668	qp->dev = rsrc->dev;
 669	qp->qp_uk.back_qp = rsrc;
 670	qp->sq_pa = mem->pa;
 671	qp->rq_pa = qp->sq_pa + sq_size;
 672	qp->vsi = rsrc->vsi;
 673	ukqp->sq_base = mem->va;
 674	ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];
 675	ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;
 676	ukqp->uk_attrs = &qp->dev->hw_attrs.uk_attrs;
 677	qp->shadow_area_pa = qp->rq_pa + rq_size;
 678	qp->hw_host_ctx = ukqp->shadow_area + IRDMA_SHADOW_AREA_SIZE;
 679	qp->hw_host_ctx_pa = qp->shadow_area_pa + (IRDMA_SHADOW_AREA_SIZE << 3);
 680	qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
 681	ukqp->qp_id = rsrc->qp_id;
 682	ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array;
 683	ukqp->rq_wrid_array = rsrc->rq_wrid_array;
 684	ukqp->sq_size = rsrc->sq_size;
 685	ukqp->rq_size = rsrc->rq_size;
 686
 687	IRDMA_RING_INIT(ukqp->sq_ring, ukqp->sq_size);
 688	IRDMA_RING_INIT(ukqp->rq_ring, ukqp->rq_size);
 689	ukqp->wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
 690
 691	ret = rsrc->dev->ws_add(qp->vsi, qp->user_pri);
 692	if (ret) {
 693		dma_free_coherent(rsrc->dev->hw->device, rsrc->qpmem.size,
 694				  rsrc->qpmem.va, rsrc->qpmem.pa);
 695		rsrc->qpmem.va = NULL;
 696		return ret;
 697	}
 698
 699	irdma_qp_add_qos(qp);
 700	irdma_puda_qp_setctx(rsrc);
 701
 702	if (rsrc->dev->ceq_valid)
 703		ret = irdma_cqp_qp_create_cmd(rsrc->dev, qp);
 704	else
 705		ret = irdma_puda_qp_wqe(rsrc->dev, qp);
 706	if (ret) {
 707		irdma_qp_rem_qos(qp);
 708		rsrc->dev->ws_remove(qp->vsi, qp->user_pri);
 709		dma_free_coherent(rsrc->dev->hw->device, rsrc->qpmem.size,
 710				  rsrc->qpmem.va, rsrc->qpmem.pa);
 711		rsrc->qpmem.va = NULL;
 712	}
 713
 714	return ret;
 715}
 716
 717/**
 718 * irdma_puda_cq_wqe - setup wqe for CQ create
 719 * @dev: Device
 720 * @cq: resource for cq
 721 */
 722static int irdma_puda_cq_wqe(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
 723{
 724	__le64 *wqe;
 725	struct irdma_sc_cqp *cqp;
 726	u64 hdr;
 727	struct irdma_ccq_cqe_info compl_info;
 728
 729	cqp = dev->cqp;
 730	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
 731	if (!wqe)
 732		return -ENOMEM;
 733
 734	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
 735	set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
 736	set_64bit_val(wqe, 16,
 737		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
 738	set_64bit_val(wqe, 32, cq->cq_pa);
 739	set_64bit_val(wqe, 40, cq->shadow_area_pa);
 740	set_64bit_val(wqe, 56,
 741		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
 742		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
 743
 744	hdr = cq->cq_uk.cq_id |
 745	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
 746	      FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, 1) |
 747	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, 1) |
 748	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, 1) |
 749	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 750	dma_wmb(); /* make sure WQE is written before valid bit is set */
 751
 752	set_64bit_val(wqe, 24, hdr);
 753
 754	print_hex_dump_debug("PUDA: PUDA CREATE CQ", DUMP_PREFIX_OFFSET, 16,
 755			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
 756	irdma_sc_cqp_post_sq(dev->cqp);
 757	return irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_CREATE_CQ,
 758					     &compl_info);
 759}
 760
 761/**
 762 * irdma_puda_cq_create - create cq for resource
 763 * @rsrc: resource for which cq to create
 764 */
 765static int irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)
 766{
 767	struct irdma_sc_dev *dev = rsrc->dev;
 768	struct irdma_sc_cq *cq = &rsrc->cq;
 769	int ret = 0;
 770	u32 cqsize;
 771	struct irdma_dma_mem *mem;
 772	struct irdma_cq_init_info info = {};
 773	struct irdma_cq_uk_init_info *init_info = &info.cq_uk_init_info;
 774
 775	cq->vsi = rsrc->vsi;
 776	cqsize = rsrc->cq_size * (sizeof(struct irdma_cqe));
 777	rsrc->cqmem.size = ALIGN(cqsize + sizeof(struct irdma_cq_shadow_area),
 778				 IRDMA_CQ0_ALIGNMENT);
 779	rsrc->cqmem.va = dma_alloc_coherent(dev->hw->device, rsrc->cqmem.size,
 780					    &rsrc->cqmem.pa, GFP_KERNEL);
 781	if (!rsrc->cqmem.va)
 782		return -ENOMEM;
 783
 784	mem = &rsrc->cqmem;
 785	info.dev = dev;
 786	info.type = (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ) ?
 787		    IRDMA_CQ_TYPE_ILQ : IRDMA_CQ_TYPE_IEQ;
 788	info.shadow_read_threshold = rsrc->cq_size >> 2;
 789	info.cq_base_pa = mem->pa;
 790	info.shadow_area_pa = mem->pa + cqsize;
 791	init_info->cq_base = mem->va;
 792	init_info->shadow_area = (__le64 *)((u8 *)mem->va + cqsize);
 793	init_info->cq_size = rsrc->cq_size;
 794	init_info->cq_id = rsrc->cq_id;
 795	info.ceqe_mask = true;
 796	info.ceq_id_valid = true;
 797	info.vsi = rsrc->vsi;
 798
 799	ret = irdma_sc_cq_init(cq, &info);
 800	if (ret)
 801		goto error;
 802
 803	if (rsrc->dev->ceq_valid)
 804		ret = irdma_cqp_cq_create_cmd(dev, cq);
 805	else
 806		ret = irdma_puda_cq_wqe(dev, cq);
 807error:
 808	if (ret) {
 809		dma_free_coherent(dev->hw->device, rsrc->cqmem.size,
 810				  rsrc->cqmem.va, rsrc->cqmem.pa);
 811		rsrc->cqmem.va = NULL;
 812	}
 813
 814	return ret;
 815}
 816
 817/**
 818 * irdma_puda_free_qp - free qp for resource
 819 * @rsrc: resource for which qp to free
 820 */
 821static void irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)
 822{
 823	int ret;
 824	struct irdma_ccq_cqe_info compl_info;
 825	struct irdma_sc_dev *dev = rsrc->dev;
 826
 827	if (rsrc->dev->ceq_valid) {
 828		irdma_cqp_qp_destroy_cmd(dev, &rsrc->qp);
 829		rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
 830		return;
 831	}
 832
 833	ret = irdma_sc_qp_destroy(&rsrc->qp, 0, false, true, true);
 834	if (ret)
 835		ibdev_dbg(to_ibdev(dev),
 836			  "PUDA: error puda qp destroy wqe, status = %d\n",
 837			  ret);
 838	if (!ret) {
 839		ret = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_DESTROY_QP,
 840						    &compl_info);
 841		if (ret)
 842			ibdev_dbg(to_ibdev(dev),
 843				  "PUDA: error puda qp destroy failed, status = %d\n",
 844				  ret);
 845	}
 846	rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
 847}
 848
 849/**
 850 * irdma_puda_free_cq - free cq for resource
 851 * @rsrc: resource for which cq to free
 852 */
 853static void irdma_puda_free_cq(struct irdma_puda_rsrc *rsrc)
 854{
 855	int ret;
 856	struct irdma_ccq_cqe_info compl_info;
 857	struct irdma_sc_dev *dev = rsrc->dev;
 858
 859	if (rsrc->dev->ceq_valid) {
 860		irdma_cqp_cq_destroy_cmd(dev, &rsrc->cq);
 861		return;
 862	}
 863
 864	ret = irdma_sc_cq_destroy(&rsrc->cq, 0, true);
 865	if (ret)
 866		ibdev_dbg(to_ibdev(dev), "PUDA: error ieq cq destroy\n");
 867	if (!ret) {
 868		ret = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_DESTROY_CQ,
 869						    &compl_info);
 870		if (ret)
 871			ibdev_dbg(to_ibdev(dev),
 872				  "PUDA: error ieq qp destroy done\n");
 873	}
 874}
 875
 876/**
 877 * irdma_puda_dele_rsrc - delete all resources during close
 878 * @vsi: VSI structure of device
 879 * @type: type of resource to dele
 880 * @reset: true if reset chip
 881 */
 882void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
 883			  bool reset)
 884{
 885	struct irdma_sc_dev *dev = vsi->dev;
 886	struct irdma_puda_rsrc *rsrc;
 887	struct irdma_puda_buf *buf = NULL;
 888	struct irdma_puda_buf *nextbuf = NULL;
 889	struct irdma_virt_mem *vmem;
 890
 891	switch (type) {
 892	case IRDMA_PUDA_RSRC_TYPE_ILQ:
 893		rsrc = vsi->ilq;
 894		vmem = &vsi->ilq_mem;
 895		vsi->ilq = NULL;
 896		break;
 897	case IRDMA_PUDA_RSRC_TYPE_IEQ:
 898		rsrc = vsi->ieq;
 899		vmem = &vsi->ieq_mem;
 900		vsi->ieq = NULL;
 901		break;
 902	default:
 903		ibdev_dbg(to_ibdev(dev), "PUDA: error resource type = 0x%x\n",
 904			  type);
 905		return;
 906	}
 907
 908	switch (rsrc->cmpl) {
 909	case PUDA_HASH_CRC_COMPLETE:
 910	case PUDA_QP_CREATED:
 911		irdma_qp_rem_qos(&rsrc->qp);
 912
 913		if (!reset)
 914			irdma_puda_free_qp(rsrc);
 915
 916		dma_free_coherent(dev->hw->device, rsrc->qpmem.size,
 917				  rsrc->qpmem.va, rsrc->qpmem.pa);
 918		rsrc->qpmem.va = NULL;
 919		fallthrough;
 920	case PUDA_CQ_CREATED:
 921		if (!reset)
 922			irdma_puda_free_cq(rsrc);
 923
 924		dma_free_coherent(dev->hw->device, rsrc->cqmem.size,
 925				  rsrc->cqmem.va, rsrc->cqmem.pa);
 926		rsrc->cqmem.va = NULL;
 927		break;
 928	default:
 929		ibdev_dbg(to_ibdev(rsrc->dev), "PUDA: error no resources\n");
 930		break;
 931	}
 932	/* Free all allocated puda buffers for both tx and rx */
 933	buf = rsrc->alloclist;
 934	while (buf) {
 935		nextbuf = buf->next;
 936		irdma_puda_dele_buf(dev, buf);
 937		buf = nextbuf;
 938		rsrc->alloc_buf_count--;
 939	}
 940
 941	kfree(vmem->va);
 942}
 943
 944/**
 945 * irdma_puda_allocbufs - allocate buffers for resource
 946 * @rsrc: resource for buffer allocation
 947 * @count: number of buffers to create
 948 */
 949static int irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count)
 950{
 951	u32 i;
 952	struct irdma_puda_buf *buf;
 953	struct irdma_puda_buf *nextbuf;
 954
 955	for (i = 0; i < count; i++) {
 956		buf = irdma_puda_alloc_buf(rsrc->dev, rsrc->buf_size);
 957		if (!buf) {
 958			rsrc->stats_buf_alloc_fail++;
 959			return -ENOMEM;
 960		}
 961		irdma_puda_ret_bufpool(rsrc, buf);
 962		rsrc->alloc_buf_count++;
 963		if (!rsrc->alloclist) {
 964			rsrc->alloclist = buf;
 965		} else {
 966			nextbuf = rsrc->alloclist;
 967			rsrc->alloclist = buf;
 968			buf->next = nextbuf;
 969		}
 970	}
 971
 972	rsrc->avail_buf_count = rsrc->alloc_buf_count;
 973
 974	return 0;
 975}
 976
 977/**
 978 * irdma_puda_create_rsrc - create resource (ilq or ieq)
 979 * @vsi: sc VSI struct
 980 * @info: resource information
 981 */
 982int irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
 983			   struct irdma_puda_rsrc_info *info)
 984{
 985	struct irdma_sc_dev *dev = vsi->dev;
 986	int ret = 0;
 987	struct irdma_puda_rsrc *rsrc;
 988	u32 pudasize;
 989	u32 sqwridsize, rqwridsize;
 990	struct irdma_virt_mem *vmem;
 991
 992	info->count = 1;
 993	pudasize = sizeof(struct irdma_puda_rsrc);
 994	sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);
 995	rqwridsize = info->rq_size * 8;
 996	switch (info->type) {
 997	case IRDMA_PUDA_RSRC_TYPE_ILQ:
 998		vmem = &vsi->ilq_mem;
 999		break;
1000	case IRDMA_PUDA_RSRC_TYPE_IEQ:
1001		vmem = &vsi->ieq_mem;
1002		break;
1003	default:
1004		return -EOPNOTSUPP;
1005	}
1006	vmem->size = pudasize + sqwridsize + rqwridsize;
1007	vmem->va = kzalloc(vmem->size, GFP_KERNEL);
1008	if (!vmem->va)
1009		return -ENOMEM;
1010
1011	rsrc = vmem->va;
1012	spin_lock_init(&rsrc->bufpool_lock);
1013	switch (info->type) {
1014	case IRDMA_PUDA_RSRC_TYPE_ILQ:
1015		vsi->ilq = vmem->va;
1016		vsi->ilq_count = info->count;
1017		rsrc->receive = info->receive;
1018		rsrc->xmit_complete = info->xmit_complete;
1019		break;
1020	case IRDMA_PUDA_RSRC_TYPE_IEQ:
1021		vsi->ieq_count = info->count;
1022		vsi->ieq = vmem->va;
1023		rsrc->receive = irdma_ieq_receive;
1024		rsrc->xmit_complete = irdma_ieq_tx_compl;
1025		break;
1026	default:
1027		return -EOPNOTSUPP;
1028	}
1029
1030	rsrc->type = info->type;
1031	rsrc->sq_wrtrk_array = (struct irdma_sq_uk_wr_trk_info *)
1032			       ((u8 *)vmem->va + pudasize);
1033	rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize);
1034	/* Initialize all ieq lists */
1035	INIT_LIST_HEAD(&rsrc->bufpool);
1036	INIT_LIST_HEAD(&rsrc->txpend);
1037
1038	rsrc->tx_wqe_avail_cnt = info->sq_size - 1;
1039	irdma_sc_pd_init(dev, &rsrc->sc_pd, info->pd_id, info->abi_ver);
1040	rsrc->qp_id = info->qp_id;
1041	rsrc->cq_id = info->cq_id;
1042	rsrc->sq_size = info->sq_size;
1043	rsrc->rq_size = info->rq_size;
1044	rsrc->cq_size = info->rq_size + info->sq_size;
1045	if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1046		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
1047			rsrc->cq_size += info->rq_size;
1048	}
1049	rsrc->buf_size = info->buf_size;
1050	rsrc->dev = dev;
1051	rsrc->vsi = vsi;
1052	rsrc->stats_idx = info->stats_idx;
1053	rsrc->stats_idx_valid = info->stats_idx_valid;
1054
1055	ret = irdma_puda_cq_create(rsrc);
1056	if (!ret) {
1057		rsrc->cmpl = PUDA_CQ_CREATED;
1058		ret = irdma_puda_qp_create(rsrc);
1059	}
1060	if (ret) {
1061		ibdev_dbg(to_ibdev(dev),
1062			  "PUDA: error qp_create type=%d, status=%d\n",
1063			  rsrc->type, ret);
1064		goto error;
1065	}
1066	rsrc->cmpl = PUDA_QP_CREATED;
1067
1068	ret = irdma_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);
1069	if (ret) {
1070		ibdev_dbg(to_ibdev(dev), "PUDA: error alloc_buf\n");
1071		goto error;
1072	}
1073
1074	rsrc->rxq_invalid_cnt = info->rq_size;
1075	ret = irdma_puda_replenish_rq(rsrc, true);
1076	if (ret)
1077		goto error;
1078
1079	if (info->type == IRDMA_PUDA_RSRC_TYPE_IEQ) {
1080		rsrc->check_crc = true;
1081		rsrc->cmpl = PUDA_HASH_CRC_COMPLETE;
1082	}
1083
1084	irdma_sc_ccq_arm(&rsrc->cq);
1085	return 0;
1086
1087error:
1088	irdma_puda_dele_rsrc(vsi, info->type, false);
1089
1090	return ret;
1091}
1092
1093/**
1094 * irdma_ilq_putback_rcvbuf - ilq buffer to put back on rq
1095 * @qp: ilq's qp resource
1096 * @buf: puda buffer for rcv q
1097 * @wqe_idx:  wqe index of completed rcvbuf
1098 */
1099static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
1100				     struct irdma_puda_buf *buf, u32 wqe_idx)
1101{
1102	__le64 *wqe;
1103	u64 offset8, offset24;
1104
1105	/* Synch buffer for use by device */
1106	dma_sync_single_for_device(qp->dev->hw->device, buf->mem.pa,
1107				   buf->mem.size, DMA_BIDIRECTIONAL);
1108	wqe = qp->qp_uk.rq_base[wqe_idx].elem;
1109	get_64bit_val(wqe, 24, &offset24);
1110	if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1111		get_64bit_val(wqe, 8, &offset8);
1112		if (offset24)
1113			offset8 &= ~FIELD_PREP(IRDMAQPSQ_VALID, 1);
1114		else
1115			offset8 |= FIELD_PREP(IRDMAQPSQ_VALID, 1);
1116		set_64bit_val(wqe, 8, offset8);
1117		dma_wmb(); /* make sure WQE is written before valid bit is set */
1118	}
1119	if (offset24)
1120		offset24 = 0;
1121	else
1122		offset24 = FIELD_PREP(IRDMAQPSQ_VALID, 1);
1123
1124	set_64bit_val(wqe, 24, offset24);
1125}
1126
1127/**
1128 * irdma_ieq_get_fpdu_len - get length of fpdu with or without marker
1129 * @pfpdu: pointer to fpdu
1130 * @datap: pointer to data in the buffer
1131 * @rcv_seq: seqnum of the data buffer
1132 */
1133static u16 irdma_ieq_get_fpdu_len(struct irdma_pfpdu *pfpdu, u8 *datap,
1134				  u32 rcv_seq)
1135{
1136	u32 marker_seq, end_seq, blk_start;
1137	u8 marker_len = pfpdu->marker_len;
1138	u16 total_len = 0;
1139	u16 fpdu_len;
1140
1141	blk_start = (pfpdu->rcv_start_seq - rcv_seq) & (IRDMA_MRK_BLK_SZ - 1);
1142	if (!blk_start) {
1143		total_len = marker_len;
1144		marker_seq = rcv_seq + IRDMA_MRK_BLK_SZ;
1145		if (marker_len && *(u32 *)datap)
1146			return 0;
1147	} else {
1148		marker_seq = rcv_seq + blk_start;
1149	}
1150
1151	datap += total_len;
1152	fpdu_len = ntohs(*(__be16 *)datap);
1153	fpdu_len += IRDMA_IEQ_MPA_FRAMING;
1154	fpdu_len = (fpdu_len + 3) & 0xfffc;
1155
1156	if (fpdu_len > pfpdu->max_fpdu_data)
1157		return 0;
1158
1159	total_len += fpdu_len;
1160	end_seq = rcv_seq + total_len;
1161	while ((int)(marker_seq - end_seq) < 0) {
1162		total_len += marker_len;
1163		end_seq += marker_len;
1164		marker_seq += IRDMA_MRK_BLK_SZ;
1165	}
1166
1167	return total_len;
1168}
1169
1170/**
1171 * irdma_ieq_copy_to_txbuf - copydata from rcv buf to tx buf
1172 * @buf: rcv buffer with partial
1173 * @txbuf: tx buffer for sending back
1174 * @buf_offset: rcv buffer offset to copy from
1175 * @txbuf_offset: at offset in tx buf to copy
1176 * @len: length of data to copy
1177 */
1178static void irdma_ieq_copy_to_txbuf(struct irdma_puda_buf *buf,
1179				    struct irdma_puda_buf *txbuf,
1180				    u16 buf_offset, u32 txbuf_offset, u32 len)
1181{
1182	void *mem1 = (u8 *)buf->mem.va + buf_offset;
1183	void *mem2 = (u8 *)txbuf->mem.va + txbuf_offset;
1184
1185	memcpy(mem2, mem1, len);
1186}
1187
1188/**
1189 * irdma_ieq_setup_tx_buf - setup tx buffer for partial handling
1190 * @buf: reeive buffer with partial
1191 * @txbuf: buffer to prepare
1192 */
1193static void irdma_ieq_setup_tx_buf(struct irdma_puda_buf *buf,
1194				   struct irdma_puda_buf *txbuf)
1195{
1196	txbuf->tcphlen = buf->tcphlen;
1197	txbuf->ipv4 = buf->ipv4;
1198
1199	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1200		txbuf->hdrlen = txbuf->tcphlen;
1201		irdma_ieq_copy_to_txbuf(buf, txbuf, IRDMA_TCP_OFFSET, 0,
1202					txbuf->hdrlen);
1203	} else {
1204		txbuf->maclen = buf->maclen;
1205		txbuf->hdrlen = buf->hdrlen;
1206		irdma_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf->hdrlen);
1207	}
1208}
1209
1210/**
1211 * irdma_ieq_check_first_buf - check if rcv buffer's seq is in range
1212 * @buf: receive exception buffer
1213 * @fps: first partial sequence number
1214 */
1215static void irdma_ieq_check_first_buf(struct irdma_puda_buf *buf, u32 fps)
1216{
1217	u32 offset;
1218
1219	if (buf->seqnum < fps) {
1220		offset = fps - buf->seqnum;
1221		if (offset > buf->datalen)
1222			return;
1223		buf->data += offset;
1224		buf->datalen -= (u16)offset;
1225		buf->seqnum = fps;
1226	}
1227}
1228
1229/**
1230 * irdma_ieq_compl_pfpdu - write txbuf with full fpdu
1231 * @ieq: ieq resource
1232 * @rxlist: ieq's received buffer list
1233 * @pbufl: temporary list for buffers for fpddu
1234 * @txbuf: tx buffer for fpdu
1235 * @fpdu_len: total length of fpdu
1236 */
1237static void irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc *ieq,
1238				  struct list_head *rxlist,
1239				  struct list_head *pbufl,
1240				  struct irdma_puda_buf *txbuf, u16 fpdu_len)
1241{
1242	struct irdma_puda_buf *buf;
1243	u32 nextseqnum;
1244	u16 txoffset, bufoffset;
1245
1246	buf = irdma_puda_get_listbuf(pbufl);
1247	if (!buf)
1248		return;
1249
1250	nextseqnum = buf->seqnum + fpdu_len;
1251	irdma_ieq_setup_tx_buf(buf, txbuf);
1252	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1253		txoffset = txbuf->hdrlen;
1254		txbuf->totallen = txbuf->hdrlen + fpdu_len;
1255		txbuf->data = (u8 *)txbuf->mem.va + txoffset;
1256	} else {
1257		txoffset = buf->hdrlen;
1258		txbuf->totallen = buf->hdrlen + fpdu_len;
1259		txbuf->data = (u8 *)txbuf->mem.va + buf->hdrlen;
1260	}
1261	bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
1262
1263	do {
1264		if (buf->datalen >= fpdu_len) {
1265			/* copied full fpdu */
1266			irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
1267						fpdu_len);
1268			buf->datalen -= fpdu_len;
1269			buf->data += fpdu_len;
1270			buf->seqnum = nextseqnum;
1271			break;
1272		}
1273		/* copy partial fpdu */
1274		irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
1275					buf->datalen);
1276		txoffset += buf->datalen;
1277		fpdu_len -= buf->datalen;
1278		irdma_puda_ret_bufpool(ieq, buf);
1279		buf = irdma_puda_get_listbuf(pbufl);
1280		if (!buf)
1281			return;
1282
1283		bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
1284	} while (1);
1285
1286	/* last buffer on the list*/
1287	if (buf->datalen)
1288		list_add(&buf->list, rxlist);
1289	else
1290		irdma_puda_ret_bufpool(ieq, buf);
1291}
1292
1293/**
1294 * irdma_ieq_create_pbufl - create buffer list for single fpdu
1295 * @pfpdu: pointer to fpdu
1296 * @rxlist: resource list for receive ieq buffes
1297 * @pbufl: temp. list for buffers for fpddu
1298 * @buf: first receive buffer
1299 * @fpdu_len: total length of fpdu
1300 */
1301static int irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu,
1302				  struct list_head *rxlist,
1303				  struct list_head *pbufl,
1304				  struct irdma_puda_buf *buf, u16 fpdu_len)
1305{
1306	int status = 0;
1307	struct irdma_puda_buf *nextbuf;
1308	u32 nextseqnum;
1309	u16 plen = fpdu_len - buf->datalen;
1310	bool done = false;
1311
1312	nextseqnum = buf->seqnum + buf->datalen;
1313	do {
1314		nextbuf = irdma_puda_get_listbuf(rxlist);
1315		if (!nextbuf) {
1316			status = -ENOBUFS;
1317			break;
1318		}
1319		list_add_tail(&nextbuf->list, pbufl);
1320		if (nextbuf->seqnum != nextseqnum) {
1321			pfpdu->bad_seq_num++;
1322			status = -ERANGE;
1323			break;
1324		}
1325		if (nextbuf->datalen >= plen) {
1326			done = true;
1327		} else {
1328			plen -= nextbuf->datalen;
1329			nextseqnum = nextbuf->seqnum + nextbuf->datalen;
1330		}
1331
1332	} while (!done);
1333
1334	return status;
1335}
1336
1337/**
1338 * irdma_ieq_handle_partial - process partial fpdu buffer
1339 * @ieq: ieq resource
1340 * @pfpdu: partial management per user qp
1341 * @buf: receive buffer
1342 * @fpdu_len: fpdu len in the buffer
1343 */
1344static int irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq,
1345				    struct irdma_pfpdu *pfpdu,
1346				    struct irdma_puda_buf *buf, u16 fpdu_len)
1347{
1348	int status = 0;
1349	u8 *crcptr;
1350	u32 mpacrc;
1351	u32 seqnum = buf->seqnum;
1352	struct list_head pbufl; /* partial buffer list */
1353	struct irdma_puda_buf *txbuf = NULL;
1354	struct list_head *rxlist = &pfpdu->rxlist;
1355
1356	ieq->partials_handled++;
1357
1358	INIT_LIST_HEAD(&pbufl);
1359	list_add(&buf->list, &pbufl);
1360
1361	status = irdma_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len);
1362	if (status)
1363		goto error;
1364
1365	txbuf = irdma_puda_get_bufpool(ieq);
1366	if (!txbuf) {
1367		pfpdu->no_tx_bufs++;
1368		status = -ENOBUFS;
1369		goto error;
1370	}
1371
1372	irdma_ieq_compl_pfpdu(ieq, rxlist, &pbufl, txbuf, fpdu_len);
1373	irdma_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);
1374
1375	crcptr = txbuf->data + fpdu_len - 4;
1376	mpacrc = *(u32 *)crcptr;
1377	if (ieq->check_crc) {
1378		status = irdma_ieq_check_mpacrc(txbuf->data, fpdu_len - 4,
1379						mpacrc);
1380		if (status) {
1381			ibdev_dbg(to_ibdev(ieq->dev), "IEQ: error bad crc\n");
1382			goto error;
1383		}
1384	}
1385
1386	print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,
1387			     txbuf->mem.va, txbuf->totallen, false);
1388	if (ieq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1389		txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
1390	txbuf->do_lpb = true;
1391	irdma_puda_send_buf(ieq, txbuf);
1392	pfpdu->rcv_nxt = seqnum + fpdu_len;
1393	return status;
1394
1395error:
1396	while (!list_empty(&pbufl)) {
1397		buf = list_last_entry(&pbufl, struct irdma_puda_buf, list);
1398		list_move(&buf->list, rxlist);
1399	}
1400	if (txbuf)
1401		irdma_puda_ret_bufpool(ieq, txbuf);
1402
1403	return status;
1404}
1405
1406/**
1407 * irdma_ieq_process_buf - process buffer rcvd for ieq
1408 * @ieq: ieq resource
1409 * @pfpdu: partial management per user qp
1410 * @buf: receive buffer
1411 */
1412static int irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
1413				 struct irdma_pfpdu *pfpdu,
1414				 struct irdma_puda_buf *buf)
1415{
1416	u16 fpdu_len = 0;
1417	u16 datalen = buf->datalen;
1418	u8 *datap = buf->data;
1419	u8 *crcptr;
1420	u16 ioffset = 0;
1421	u32 mpacrc;
1422	u32 seqnum = buf->seqnum;
1423	u16 len = 0;
1424	u16 full = 0;
1425	bool partial = false;
1426	struct irdma_puda_buf *txbuf;
1427	struct list_head *rxlist = &pfpdu->rxlist;
1428	int ret = 0;
1429
1430	ioffset = (u16)(buf->data - (u8 *)buf->mem.va);
1431	while (datalen) {
1432		fpdu_len = irdma_ieq_get_fpdu_len(pfpdu, datap, buf->seqnum);
1433		if (!fpdu_len) {
1434			ibdev_dbg(to_ibdev(ieq->dev),
1435				  "IEQ: error bad fpdu len\n");
1436			list_add(&buf->list, rxlist);
1437			return -EINVAL;
1438		}
1439
1440		if (datalen < fpdu_len) {
1441			partial = true;
1442			break;
1443		}
1444		crcptr = datap + fpdu_len - 4;
1445		mpacrc = *(u32 *)crcptr;
1446		if (ieq->check_crc)
1447			ret = irdma_ieq_check_mpacrc(datap, fpdu_len - 4,
1448						     mpacrc);
1449		if (ret) {
1450			list_add(&buf->list, rxlist);
1451			ibdev_dbg(to_ibdev(ieq->dev),
1452				  "ERR: IRDMA_ERR_MPA_CRC\n");
1453			return -EINVAL;
1454		}
1455		full++;
1456		pfpdu->fpdu_processed++;
1457		ieq->fpdu_processed++;
1458		datap += fpdu_len;
1459		len += fpdu_len;
1460		datalen -= fpdu_len;
1461	}
1462	if (full) {
1463		/* copy full pdu's in the txbuf and send them out */
1464		txbuf = irdma_puda_get_bufpool(ieq);
1465		if (!txbuf) {
1466			pfpdu->no_tx_bufs++;
1467			list_add(&buf->list, rxlist);
1468			return -ENOBUFS;
1469		}
1470		/* modify txbuf's buffer header */
1471		irdma_ieq_setup_tx_buf(buf, txbuf);
1472		/* copy full fpdu's to new buffer */
1473		if (ieq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1474			irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
1475						txbuf->hdrlen, len);
1476			txbuf->totallen = txbuf->hdrlen + len;
1477			txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
1478		} else {
1479			irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
1480						buf->hdrlen, len);
1481			txbuf->totallen = buf->hdrlen + len;
1482		}
1483		irdma_ieq_update_tcpip_info(txbuf, len, buf->seqnum);
1484		print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET,
1485				     16, 8, txbuf->mem.va, txbuf->totallen,
1486				     false);
1487		txbuf->do_lpb = true;
1488		irdma_puda_send_buf(ieq, txbuf);
1489
1490		if (!datalen) {
1491			pfpdu->rcv_nxt = buf->seqnum + len;
1492			irdma_puda_ret_bufpool(ieq, buf);
1493			return 0;
1494		}
1495		buf->data = datap;
1496		buf->seqnum = seqnum + len;
1497		buf->datalen = datalen;
1498		pfpdu->rcv_nxt = buf->seqnum;
1499	}
1500	if (partial)
1501		return irdma_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);
1502
1503	return 0;
1504}
1505
1506/**
1507 * irdma_ieq_process_fpdus - process fpdu's buffers on its list
1508 * @qp: qp for which partial fpdus
1509 * @ieq: ieq resource
1510 */
1511void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
1512			     struct irdma_puda_rsrc *ieq)
1513{
1514	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1515	struct list_head *rxlist = &pfpdu->rxlist;
1516	struct irdma_puda_buf *buf;
1517	int status;
1518
1519	do {
1520		if (list_empty(rxlist))
1521			break;
1522		buf = irdma_puda_get_listbuf(rxlist);
1523		if (!buf) {
1524			ibdev_dbg(to_ibdev(ieq->dev), "IEQ: error no buf\n");
1525			break;
1526		}
1527		if (buf->seqnum != pfpdu->rcv_nxt) {
1528			/* This could be out of order or missing packet */
1529			pfpdu->out_of_order++;
1530			list_add(&buf->list, rxlist);
1531			break;
1532		}
1533		/* keep processing buffers from the head of the list */
1534		status = irdma_ieq_process_buf(ieq, pfpdu, buf);
1535		if (status == -EINVAL) {
1536			pfpdu->mpa_crc_err = true;
1537			while (!list_empty(rxlist)) {
1538				buf = irdma_puda_get_listbuf(rxlist);
1539				irdma_puda_ret_bufpool(ieq, buf);
1540				pfpdu->crc_err++;
1541				ieq->crc_err++;
1542			}
1543			/* create CQP for AE */
1544			irdma_ieq_mpa_crc_ae(ieq->dev, qp);
1545		}
1546	} while (!status);
1547}
1548
1549/**
1550 * irdma_ieq_create_ah - create an address handle for IEQ
1551 * @qp: qp pointer
1552 * @buf: buf received on IEQ used to create AH
1553 */
1554static int irdma_ieq_create_ah(struct irdma_sc_qp *qp, struct irdma_puda_buf *buf)
1555{
1556	struct irdma_ah_info ah_info = {};
1557
1558	qp->pfpdu.ah_buf = buf;
1559	irdma_puda_ieq_get_ah_info(qp, &ah_info);
1560	return irdma_puda_create_ah(qp->vsi->dev, &ah_info, false,
1561				    IRDMA_PUDA_RSRC_TYPE_IEQ, qp,
1562				    &qp->pfpdu.ah);
1563}
1564
1565/**
1566 * irdma_ieq_handle_exception - handle qp's exception
1567 * @ieq: ieq resource
1568 * @qp: qp receiving excpetion
1569 * @buf: receive buffer
1570 */
1571static void irdma_ieq_handle_exception(struct irdma_puda_rsrc *ieq,
1572				       struct irdma_sc_qp *qp,
1573				       struct irdma_puda_buf *buf)
1574{
1575	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1576	u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
1577	u32 rcv_wnd = hw_host_ctx[23];
1578	/* first partial seq # in q2 */
1579	u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
1580	struct list_head *rxlist = &pfpdu->rxlist;
1581	unsigned long flags = 0;
1582	u8 hw_rev = qp->dev->hw_attrs.uk_attrs.hw_rev;
1583
1584	print_hex_dump_debug("IEQ: IEQ RX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,
1585			     buf->mem.va, buf->totallen, false);
1586
1587	spin_lock_irqsave(&pfpdu->lock, flags);
1588	pfpdu->total_ieq_bufs++;
1589	if (pfpdu->mpa_crc_err) {
1590		pfpdu->crc_err++;
1591		goto error;
1592	}
1593	if (pfpdu->mode && fps != pfpdu->fps) {
1594		/* clean up qp as it is new partial sequence */
1595		irdma_ieq_cleanup_qp(ieq, qp);
1596		ibdev_dbg(to_ibdev(ieq->dev), "IEQ: restarting new partial\n");
1597		pfpdu->mode = false;
1598	}
1599
1600	if (!pfpdu->mode) {
1601		print_hex_dump_debug("IEQ: Q2 BUFFER", DUMP_PREFIX_OFFSET, 16,
1602				     8, (u64 *)qp->q2_buf, 128, false);
1603		/* First_Partial_Sequence_Number check */
1604		pfpdu->rcv_nxt = fps;
1605		pfpdu->fps = fps;
1606		pfpdu->mode = true;
1607		pfpdu->max_fpdu_data = (buf->ipv4) ?
1608				       (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV4) :
1609				       (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV6);
1610		pfpdu->pmode_count++;
1611		ieq->pmode_count++;
1612		INIT_LIST_HEAD(rxlist);
1613		irdma_ieq_check_first_buf(buf, fps);
1614	}
1615
1616	if (!(rcv_wnd >= (buf->seqnum - pfpdu->rcv_nxt))) {
1617		pfpdu->bad_seq_num++;
1618		ieq->bad_seq_num++;
1619		goto error;
1620	}
1621
1622	if (!list_empty(rxlist)) {
1623		if (buf->seqnum != pfpdu->nextseqnum) {
1624			irdma_send_ieq_ack(qp);
1625			/* throw away out-of-order, duplicates*/
1626			goto error;
1627		}
1628	}
1629	/* Insert buf before head */
1630	list_add_tail(&buf->list, rxlist);
1631	pfpdu->nextseqnum = buf->seqnum + buf->datalen;
1632	pfpdu->lastrcv_buf = buf;
1633	if (hw_rev >= IRDMA_GEN_2 && !pfpdu->ah) {
1634		irdma_ieq_create_ah(qp, buf);
1635		if (!pfpdu->ah)
1636			goto error;
1637		goto exit;
1638	}
1639	if (hw_rev == IRDMA_GEN_1)
1640		irdma_ieq_process_fpdus(qp, ieq);
1641	else if (pfpdu->ah && pfpdu->ah->ah_info.ah_valid)
1642		irdma_ieq_process_fpdus(qp, ieq);
1643exit:
1644	spin_unlock_irqrestore(&pfpdu->lock, flags);
1645
1646	return;
1647
1648error:
1649	irdma_puda_ret_bufpool(ieq, buf);
1650	spin_unlock_irqrestore(&pfpdu->lock, flags);
1651}
1652
1653/**
1654 * irdma_ieq_receive - received exception buffer
1655 * @vsi: VSI of device
1656 * @buf: exception buffer received
1657 */
1658static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
1659			      struct irdma_puda_buf *buf)
1660{
1661	struct irdma_puda_rsrc *ieq = vsi->ieq;
1662	struct irdma_sc_qp *qp = NULL;
1663	u32 wqe_idx = ieq->compl_rxwqe_idx;
1664
1665	qp = irdma_ieq_get_qp(vsi->dev, buf);
1666	if (!qp) {
1667		ieq->stats_bad_qp_id++;
1668		irdma_puda_ret_bufpool(ieq, buf);
1669	} else {
1670		irdma_ieq_handle_exception(ieq, qp, buf);
1671	}
1672	/*
1673	 * ieq->rx_wqe_idx is used by irdma_puda_replenish_rq()
1674	 * on which wqe_idx to start replenish rq
1675	 */
1676	if (!ieq->rxq_invalid_cnt)
1677		ieq->rx_wqe_idx = wqe_idx;
1678	ieq->rxq_invalid_cnt++;
1679}
1680
1681/**
1682 * irdma_ieq_tx_compl - put back after sending completed exception buffer
1683 * @vsi: sc VSI struct
1684 * @sqwrid: pointer to puda buffer
1685 */
1686static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid)
1687{
1688	struct irdma_puda_rsrc *ieq = vsi->ieq;
1689	struct irdma_puda_buf *buf = sqwrid;
1690
1691	irdma_puda_ret_bufpool(ieq, buf);
1692}
1693
1694/**
1695 * irdma_ieq_cleanup_qp - qp is being destroyed
1696 * @ieq: ieq resource
1697 * @qp: all pending fpdu buffers
1698 */
1699void irdma_ieq_cleanup_qp(struct irdma_puda_rsrc *ieq, struct irdma_sc_qp *qp)
1700{
1701	struct irdma_puda_buf *buf;
1702	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1703	struct list_head *rxlist = &pfpdu->rxlist;
1704
1705	if (qp->pfpdu.ah) {
1706		irdma_puda_free_ah(ieq->dev, qp->pfpdu.ah);
1707		qp->pfpdu.ah = NULL;
1708		qp->pfpdu.ah_buf = NULL;
1709	}
1710
1711	if (!pfpdu->mode)
1712		return;
1713
1714	while (!list_empty(rxlist)) {
1715		buf = irdma_puda_get_listbuf(rxlist);
1716		irdma_puda_ret_bufpool(ieq, buf);
1717	}
1718}