verbs.c

   1/* QLogic qedr NIC Driver
   2 * Copyright (c) 2015-2016  QLogic Corporation
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and /or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32#include <linux/dma-mapping.h>
  33#include <linux/crc32.h>
  34#include <net/ip.h>
  35#include <net/ipv6.h>
  36#include <net/udp.h>
  37#include <linux/iommu.h>
  38
  39#include <rdma/ib_verbs.h>
  40#include <rdma/ib_user_verbs.h>
  41#include <rdma/iw_cm.h>
  42#include <rdma/ib_umem.h>
  43#include <rdma/ib_addr.h>
  44#include <rdma/ib_cache.h>
  45#include <rdma/uverbs_ioctl.h>
  46
  47#include <linux/qed/common_hsi.h>
  48#include "qedr_hsi_rdma.h"
  49#include <linux/qed/qed_if.h>
  50#include "qedr.h"
  51#include "verbs.h"
  52#include <rdma/qedr-abi.h>
  53#include "qedr_roce_cm.h"
  54#include "qedr_iw_cm.h"
  55
  56#define QEDR_SRQ_WQE_ELEM_SIZE	sizeof(union rdma_srq_elm)
  57#define	RDMA_MAX_SGE_PER_SRQ	(4)
  58#define RDMA_MAX_SRQ_WQE_SIZE	(RDMA_MAX_SGE_PER_SRQ + 1)
  59
  60#define DB_ADDR_SHIFT(addr)		((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  61
  62enum {
  63	QEDR_USER_MMAP_IO_WC = 0,
  64	QEDR_USER_MMAP_PHYS_PAGE,
  65};
  66
  67static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  68					size_t len)
  69{
  70	size_t min_len = min_t(size_t, len, udata->outlen);
  71
  72	return ib_copy_to_udata(udata, src, min_len);
  73}
  74
  75int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
  76{
  77	if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  78		return -EINVAL;
  79
  80	*pkey = QEDR_ROCE_PKEY_DEFAULT;
  81	return 0;
  82}
  83
  84int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
  85		      int index, union ib_gid *sgid)
  86{
  87	struct qedr_dev *dev = get_qedr_dev(ibdev);
  88
  89	memset(sgid->raw, 0, sizeof(sgid->raw));
  90	ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  91
  92	DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  93		 sgid->global.interface_id, sgid->global.subnet_prefix);
  94
  95	return 0;
  96}
  97
  98int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  99{
 100	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 101	struct qedr_device_attr *qattr = &dev->attr;
 102	struct qedr_srq *srq = get_qedr_srq(ibsrq);
 103
 104	srq_attr->srq_limit = srq->srq_limit;
 105	srq_attr->max_wr = qattr->max_srq_wr;
 106	srq_attr->max_sge = qattr->max_sge;
 107
 108	return 0;
 109}
 110
 111int qedr_query_device(struct ib_device *ibdev,
 112		      struct ib_device_attr *attr, struct ib_udata *udata)
 113{
 114	struct qedr_dev *dev = get_qedr_dev(ibdev);
 115	struct qedr_device_attr *qattr = &dev->attr;
 116
 117	if (!dev->rdma_ctx) {
 118		DP_ERR(dev,
 119		       "qedr_query_device called with invalid params rdma_ctx=%p\n",
 120		       dev->rdma_ctx);
 121		return -EINVAL;
 122	}
 123
 124	memset(attr, 0, sizeof(*attr));
 125
 126	attr->fw_ver = qattr->fw_ver;
 127	attr->sys_image_guid = qattr->sys_image_guid;
 128	attr->max_mr_size = qattr->max_mr_size;
 129	attr->page_size_cap = qattr->page_size_caps;
 130	attr->vendor_id = qattr->vendor_id;
 131	attr->vendor_part_id = qattr->vendor_part_id;
 132	attr->hw_ver = qattr->hw_ver;
 133	attr->max_qp = qattr->max_qp;
 134	attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 135	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 136	    IB_DEVICE_RC_RNR_NAK_GEN |
 137	    IB_DEVICE_MEM_MGT_EXTENSIONS;
 138	attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
 139
 140	if (!rdma_protocol_iwarp(&dev->ibdev, 1))
 141		attr->device_cap_flags |= IB_DEVICE_XRC;
 142	attr->max_send_sge = qattr->max_sge;
 143	attr->max_recv_sge = qattr->max_sge;
 144	attr->max_sge_rd = qattr->max_sge;
 145	attr->max_cq = qattr->max_cq;
 146	attr->max_cqe = qattr->max_cqe;
 147	attr->max_mr = qattr->max_mr;
 148	attr->max_mw = qattr->max_mw;
 149	attr->max_pd = qattr->max_pd;
 150	attr->atomic_cap = dev->atomic_cap;
 151	attr->max_qp_init_rd_atom =
 152	    1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 153	attr->max_qp_rd_atom =
 154	    min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 155		attr->max_qp_init_rd_atom);
 156
 157	attr->max_srq = qattr->max_srq;
 158	attr->max_srq_sge = qattr->max_srq_sge;
 159	attr->max_srq_wr = qattr->max_srq_wr;
 160
 161	attr->local_ca_ack_delay = qattr->dev_ack_delay;
 162	attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 163	attr->max_pkeys = qattr->max_pkey;
 164	attr->max_ah = qattr->max_ah;
 165
 166	return 0;
 167}
 168
 169static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
 170					    u8 *ib_width)
 171{
 172	switch (speed) {
 173	case 1000:
 174		*ib_speed = IB_SPEED_SDR;
 175		*ib_width = IB_WIDTH_1X;
 176		break;
 177	case 10000:
 178		*ib_speed = IB_SPEED_QDR;
 179		*ib_width = IB_WIDTH_1X;
 180		break;
 181
 182	case 20000:
 183		*ib_speed = IB_SPEED_DDR;
 184		*ib_width = IB_WIDTH_4X;
 185		break;
 186
 187	case 25000:
 188		*ib_speed = IB_SPEED_EDR;
 189		*ib_width = IB_WIDTH_1X;
 190		break;
 191
 192	case 40000:
 193		*ib_speed = IB_SPEED_QDR;
 194		*ib_width = IB_WIDTH_4X;
 195		break;
 196
 197	case 50000:
 198		*ib_speed = IB_SPEED_HDR;
 199		*ib_width = IB_WIDTH_1X;
 200		break;
 201
 202	case 100000:
 203		*ib_speed = IB_SPEED_EDR;
 204		*ib_width = IB_WIDTH_4X;
 205		break;
 206
 207	default:
 208		/* Unsupported */
 209		*ib_speed = IB_SPEED_SDR;
 210		*ib_width = IB_WIDTH_1X;
 211	}
 212}
 213
 214int qedr_query_port(struct ib_device *ibdev, u32 port,
 215		    struct ib_port_attr *attr)
 216{
 217	struct qedr_dev *dev;
 218	struct qed_rdma_port *rdma_port;
 219
 220	dev = get_qedr_dev(ibdev);
 221
 222	if (!dev->rdma_ctx) {
 223		DP_ERR(dev, "rdma_ctx is NULL\n");
 224		return -EINVAL;
 225	}
 226
 227	rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 228
 229	/* *attr being zeroed by the caller, avoid zeroing it here */
 230	if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 231		attr->state = IB_PORT_ACTIVE;
 232		attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 233	} else {
 234		attr->state = IB_PORT_DOWN;
 235		attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 236	}
 237	attr->max_mtu = IB_MTU_4096;
 238	attr->lid = 0;
 239	attr->lmc = 0;
 240	attr->sm_lid = 0;
 241	attr->sm_sl = 0;
 242	attr->ip_gids = true;
 243	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 244		attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
 245		attr->gid_tbl_len = 1;
 246	} else {
 247		attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 248		attr->gid_tbl_len = QEDR_MAX_SGID;
 249		attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 250	}
 251	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 252	attr->qkey_viol_cntr = 0;
 253	get_link_speed_and_width(rdma_port->link_speed,
 254				 &attr->active_speed, &attr->active_width);
 255	attr->max_msg_sz = rdma_port->max_msg_size;
 256	attr->max_vl_num = 4;
 257
 258	return 0;
 259}
 260
 261int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 262{
 263	struct ib_device *ibdev = uctx->device;
 264	int rc;
 265	struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 266	struct qedr_alloc_ucontext_resp uresp = {};
 267	struct qedr_alloc_ucontext_req ureq = {};
 268	struct qedr_dev *dev = get_qedr_dev(ibdev);
 269	struct qed_rdma_add_user_out_params oparams;
 270	struct qedr_user_mmap_entry *entry;
 271
 272	if (!udata)
 273		return -EFAULT;
 274
 275	if (udata->inlen) {
 276		rc = ib_copy_from_udata(&ureq, udata,
 277					min(sizeof(ureq), udata->inlen));
 278		if (rc) {
 279			DP_ERR(dev, "Problem copying data from user space\n");
 280			return -EFAULT;
 281		}
 282		ctx->edpm_mode = !!(ureq.context_flags &
 283				    QEDR_ALLOC_UCTX_EDPM_MODE);
 284		ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
 285	}
 286
 287	rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 288	if (rc) {
 289		DP_ERR(dev,
 290		       "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 291		       rc);
 292		return rc;
 293	}
 294
 295	ctx->dpi = oparams.dpi;
 296	ctx->dpi_addr = oparams.dpi_addr;
 297	ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 298	ctx->dpi_size = oparams.dpi_size;
 299	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 300	if (!entry) {
 301		rc = -ENOMEM;
 302		goto err;
 303	}
 304
 305	entry->io_address = ctx->dpi_phys_addr;
 306	entry->length = ctx->dpi_size;
 307	entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
 308	entry->dpi = ctx->dpi;
 309	entry->dev = dev;
 310	rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
 311					 ctx->dpi_size);
 312	if (rc) {
 313		kfree(entry);
 314		goto err;
 315	}
 316	ctx->db_mmap_entry = &entry->rdma_entry;
 317
 318	if (!dev->user_dpm_enabled)
 319		uresp.dpm_flags = 0;
 320	else if (rdma_protocol_iwarp(&dev->ibdev, 1))
 321		uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
 322	else
 323		uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
 324				  QEDR_DPM_TYPE_ROCE_LEGACY |
 325				  QEDR_DPM_TYPE_ROCE_EDPM_MODE;
 326
 327	if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
 328		uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
 329		uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
 330		uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
 331		uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
 332	}
 333
 334	uresp.wids_enabled = 1;
 335	uresp.wid_count = oparams.wid_count;
 336	uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
 337	uresp.db_size = ctx->dpi_size;
 338	uresp.max_send_wr = dev->attr.max_sqe;
 339	uresp.max_recv_wr = dev->attr.max_rqe;
 340	uresp.max_srq_wr = dev->attr.max_srq_wr;
 341	uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 342	uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 343	uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 344	uresp.max_cqes = QEDR_MAX_CQES;
 345
 346	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 347	if (rc)
 348		goto err;
 349
 350	ctx->dev = dev;
 351
 352	DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 353		 &ctx->ibucontext);
 354	return 0;
 355
 356err:
 357	if (!ctx->db_mmap_entry)
 358		dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
 359	else
 360		rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
 361
 362	return rc;
 363}
 364
 365void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 366{
 367	struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 368
 369	DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 370		 uctx);
 371
 372	rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
 373}
 374
 375void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
 376{
 377	struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
 378	struct qedr_dev *dev = entry->dev;
 379
 380	if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
 381		free_page((unsigned long)entry->address);
 382	else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
 383		dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
 384
 385	kfree(entry);
 386}
 387
 388int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
 389{
 390	struct ib_device *dev = ucontext->device;
 391	size_t length = vma->vm_end - vma->vm_start;
 392	struct rdma_user_mmap_entry *rdma_entry;
 393	struct qedr_user_mmap_entry *entry;
 394	int rc = 0;
 395	u64 pfn;
 396
 397	ibdev_dbg(dev,
 398		  "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
 399		  vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
 400
 401	rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
 402	if (!rdma_entry) {
 403		ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
 404			  vma->vm_pgoff);
 405		return -EINVAL;
 406	}
 407	entry = get_qedr_mmap_entry(rdma_entry);
 408	ibdev_dbg(dev,
 409		  "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
 410		  entry->io_address, length, entry->mmap_flag);
 411
 412	switch (entry->mmap_flag) {
 413	case QEDR_USER_MMAP_IO_WC:
 414		pfn = entry->io_address >> PAGE_SHIFT;
 415		rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
 416				       pgprot_writecombine(vma->vm_page_prot),
 417				       rdma_entry);
 418		break;
 419	case QEDR_USER_MMAP_PHYS_PAGE:
 420		rc = vm_insert_page(vma, vma->vm_start,
 421				    virt_to_page(entry->address));
 422		break;
 423	default:
 424		rc = -EINVAL;
 425	}
 426
 427	if (rc)
 428		ibdev_dbg(dev,
 429			  "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 430			  entry->io_address, length, entry->mmap_flag, rc);
 431
 432	rdma_user_mmap_entry_put(rdma_entry);
 433	return rc;
 434}
 435
 436int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 437{
 438	struct ib_device *ibdev = ibpd->device;
 439	struct qedr_dev *dev = get_qedr_dev(ibdev);
 440	struct qedr_pd *pd = get_qedr_pd(ibpd);
 441	u16 pd_id;
 442	int rc;
 443
 444	DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 445		 udata ? "User Lib" : "Kernel");
 446
 447	if (!dev->rdma_ctx) {
 448		DP_ERR(dev, "invalid RDMA context\n");
 449		return -EINVAL;
 450	}
 451
 452	rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 453	if (rc)
 454		return rc;
 455
 456	pd->pd_id = pd_id;
 457
 458	if (udata) {
 459		struct qedr_alloc_pd_uresp uresp = {
 460			.pd_id = pd_id,
 461		};
 462		struct qedr_ucontext *context = rdma_udata_to_drv_context(
 463			udata, struct qedr_ucontext, ibucontext);
 464
 465		rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 466		if (rc) {
 467			DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 468			dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 469			return rc;
 470		}
 471
 472		pd->uctx = context;
 473		pd->uctx->pd = pd;
 474	}
 475
 476	return 0;
 477}
 478
 479int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 480{
 481	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 482	struct qedr_pd *pd = get_qedr_pd(ibpd);
 483
 484	DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 485	dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 486	return 0;
 487}
 488
 489
 490int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 491{
 492	struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 493	struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
 494
 495	return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
 496}
 497
 498int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 499{
 500	struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 501	u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
 502
 503	dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
 504	return 0;
 505}
 506static void qedr_free_pbl(struct qedr_dev *dev,
 507			  struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 508{
 509	struct pci_dev *pdev = dev->pdev;
 510	int i;
 511
 512	for (i = 0; i < pbl_info->num_pbls; i++) {
 513		if (!pbl[i].va)
 514			continue;
 515		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 516				  pbl[i].va, pbl[i].pa);
 517	}
 518
 519	kfree(pbl);
 520}
 521
 522#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 523#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 524
 525#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 526#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 527#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 528
 529static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 530					   struct qedr_pbl_info *pbl_info,
 531					   gfp_t flags)
 532{
 533	struct pci_dev *pdev = dev->pdev;
 534	struct qedr_pbl *pbl_table;
 535	dma_addr_t *pbl_main_tbl;
 536	dma_addr_t pa;
 537	void *va;
 538	int i;
 539
 540	pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 541	if (!pbl_table)
 542		return ERR_PTR(-ENOMEM);
 543
 544	for (i = 0; i < pbl_info->num_pbls; i++) {
 545		va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 546					flags);
 547		if (!va)
 548			goto err;
 549
 550		pbl_table[i].va = va;
 551		pbl_table[i].pa = pa;
 552	}
 553
 554	/* Two-Layer PBLs, if we have more than one pbl we need to initialize
 555	 * the first one with physical pointers to all of the rest
 556	 */
 557	pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 558	for (i = 0; i < pbl_info->num_pbls - 1; i++)
 559		pbl_main_tbl[i] = pbl_table[i + 1].pa;
 560
 561	return pbl_table;
 562
 563err:
 564	for (i--; i >= 0; i--)
 565		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 566				  pbl_table[i].va, pbl_table[i].pa);
 567
 568	qedr_free_pbl(dev, pbl_info, pbl_table);
 569
 570	return ERR_PTR(-ENOMEM);
 571}
 572
 573static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 574				struct qedr_pbl_info *pbl_info,
 575				u32 num_pbes, int two_layer_capable)
 576{
 577	u32 pbl_capacity;
 578	u32 pbl_size;
 579	u32 num_pbls;
 580
 581	if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 582		if (num_pbes > MAX_PBES_TWO_LAYER) {
 583			DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 584			       num_pbes);
 585			return -EINVAL;
 586		}
 587
 588		/* calculate required pbl page size */
 589		pbl_size = MIN_FW_PBL_PAGE_SIZE;
 590		pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 591			       NUM_PBES_ON_PAGE(pbl_size);
 592
 593		while (pbl_capacity < num_pbes) {
 594			pbl_size *= 2;
 595			pbl_capacity = pbl_size / sizeof(u64);
 596			pbl_capacity = pbl_capacity * pbl_capacity;
 597		}
 598
 599		num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 600		num_pbls++;	/* One for the layer0 ( points to the pbls) */
 601		pbl_info->two_layered = true;
 602	} else {
 603		/* One layered PBL */
 604		num_pbls = 1;
 605		pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 606				 roundup_pow_of_two((num_pbes * sizeof(u64))));
 607		pbl_info->two_layered = false;
 608	}
 609
 610	pbl_info->num_pbls = num_pbls;
 611	pbl_info->pbl_size = pbl_size;
 612	pbl_info->num_pbes = num_pbes;
 613
 614	DP_DEBUG(dev, QEDR_MSG_MR,
 615		 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 616		 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 617
 618	return 0;
 619}
 620
 621static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 622			       struct qedr_pbl *pbl,
 623			       struct qedr_pbl_info *pbl_info, u32 pg_shift)
 624{
 625	int pbe_cnt, total_num_pbes = 0;
 626	struct qedr_pbl *pbl_tbl;
 627	struct ib_block_iter biter;
 628	struct regpair *pbe;
 629
 630	if (!pbl_info->num_pbes)
 631		return;
 632
 633	/* If we have a two layered pbl, the first pbl points to the rest
 634	 * of the pbls and the first entry lays on the second pbl in the table
 635	 */
 636	if (pbl_info->two_layered)
 637		pbl_tbl = &pbl[1];
 638	else
 639		pbl_tbl = pbl;
 640
 641	pbe = (struct regpair *)pbl_tbl->va;
 642	if (!pbe) {
 643		DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 644		return;
 645	}
 646
 647	pbe_cnt = 0;
 648
 649	rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
 650		u64 pg_addr = rdma_block_iter_dma_address(&biter);
 651
 652		pbe->lo = cpu_to_le32(pg_addr);
 653		pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 654
 655		pbe_cnt++;
 656		total_num_pbes++;
 657		pbe++;
 658
 659		if (total_num_pbes == pbl_info->num_pbes)
 660			return;
 661
 662		/* If the given pbl is full storing the pbes, move to next pbl.
 663		 */
 664		if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 665			pbl_tbl++;
 666			pbe = (struct regpair *)pbl_tbl->va;
 667			pbe_cnt = 0;
 668		}
 669	}
 670}
 671
 672static int qedr_db_recovery_add(struct qedr_dev *dev,
 673				void __iomem *db_addr,
 674				void *db_data,
 675				enum qed_db_rec_width db_width,
 676				enum qed_db_rec_space db_space)
 677{
 678	if (!db_data) {
 679		DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 680		return 0;
 681	}
 682
 683	return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
 684						 db_width, db_space);
 685}
 686
 687static void qedr_db_recovery_del(struct qedr_dev *dev,
 688				 void __iomem *db_addr,
 689				 void *db_data)
 690{
 691	if (!db_data) {
 692		DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 693		return;
 694	}
 695
 696	/* Ignore return code as there is not much we can do about it. Error
 697	 * log will be printed inside.
 698	 */
 699	dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 700}
 701
 702static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 703			      struct qedr_cq *cq, struct ib_udata *udata,
 704			      u32 db_offset)
 705{
 706	struct qedr_create_cq_uresp uresp;
 707	int rc;
 708
 709	memset(&uresp, 0, sizeof(uresp));
 710
 711	uresp.db_offset = db_offset;
 712	uresp.icid = cq->icid;
 713	if (cq->q.db_mmap_entry)
 714		uresp.db_rec_addr =
 715			rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 716
 717	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 718	if (rc)
 719		DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 720
 721	return rc;
 722}
 723
 724static void consume_cqe(struct qedr_cq *cq)
 725{
 726	if (cq->latest_cqe == cq->toggle_cqe)
 727		cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 728
 729	cq->latest_cqe = qed_chain_consume(&cq->pbl);
 730}
 731
 732static inline int qedr_align_cq_entries(int entries)
 733{
 734	u64 size, aligned_size;
 735
 736	/* We allocate an extra entry that we don't report to the FW. */
 737	size = (entries + 1) * QEDR_CQE_SIZE;
 738	aligned_size = ALIGN(size, PAGE_SIZE);
 739
 740	return aligned_size / QEDR_CQE_SIZE;
 741}
 742
 743static int qedr_init_user_db_rec(struct ib_udata *udata,
 744				 struct qedr_dev *dev, struct qedr_userq *q,
 745				 bool requires_db_rec)
 746{
 747	struct qedr_ucontext *uctx =
 748		rdma_udata_to_drv_context(udata, struct qedr_ucontext,
 749					  ibucontext);
 750	struct qedr_user_mmap_entry *entry;
 751	int rc;
 752
 753	/* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
 754	if (requires_db_rec == 0 || !uctx->db_rec)
 755		return 0;
 756
 757	/* Allocate a page for doorbell recovery, add to mmap */
 758	q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
 759	if (!q->db_rec_data) {
 760		DP_ERR(dev, "get_zeroed_page failed\n");
 761		return -ENOMEM;
 762	}
 763
 764	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 765	if (!entry)
 766		goto err_free_db_data;
 767
 768	entry->address = q->db_rec_data;
 769	entry->length = PAGE_SIZE;
 770	entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
 771	rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
 772					 &entry->rdma_entry,
 773					 PAGE_SIZE);
 774	if (rc)
 775		goto err_free_entry;
 776
 777	q->db_mmap_entry = &entry->rdma_entry;
 778
 779	return 0;
 780
 781err_free_entry:
 782	kfree(entry);
 783
 784err_free_db_data:
 785	free_page((unsigned long)q->db_rec_data);
 786	q->db_rec_data = NULL;
 787	return -ENOMEM;
 788}
 789
 790static inline int qedr_init_user_queue(struct ib_udata *udata,
 791				       struct qedr_dev *dev,
 792				       struct qedr_userq *q, u64 buf_addr,
 793				       size_t buf_len, bool requires_db_rec,
 794				       int access,
 795				       int alloc_and_init)
 796{
 797	u32 fw_pages;
 798	int rc;
 799
 800	q->buf_addr = buf_addr;
 801	q->buf_len = buf_len;
 802	q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
 803	if (IS_ERR(q->umem)) {
 804		DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 805		       PTR_ERR(q->umem));
 806		return PTR_ERR(q->umem);
 807	}
 808
 809	fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
 810	rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 811	if (rc)
 812		goto err0;
 813
 814	if (alloc_and_init) {
 815		q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 816		if (IS_ERR(q->pbl_tbl)) {
 817			rc = PTR_ERR(q->pbl_tbl);
 818			goto err0;
 819		}
 820		qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 821				   FW_PAGE_SHIFT);
 822	} else {
 823		q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 824		if (!q->pbl_tbl) {
 825			rc = -ENOMEM;
 826			goto err0;
 827		}
 828	}
 829
 830	/* mmap the user address used to store doorbell data for recovery */
 831	return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
 832
 833err0:
 834	ib_umem_release(q->umem);
 835	q->umem = NULL;
 836
 837	return rc;
 838}
 839
 840static inline void qedr_init_cq_params(struct qedr_cq *cq,
 841				       struct qedr_ucontext *ctx,
 842				       struct qedr_dev *dev, int vector,
 843				       int chain_entries, int page_cnt,
 844				       u64 pbl_ptr,
 845				       struct qed_rdma_create_cq_in_params
 846				       *params)
 847{
 848	memset(params, 0, sizeof(*params));
 849	params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 850	params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 851	params->cnq_id = vector;
 852	params->cq_size = chain_entries - 1;
 853	params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 854	params->pbl_num_pages = page_cnt;
 855	params->pbl_ptr = pbl_ptr;
 856	params->pbl_two_level = 0;
 857}
 858
 859static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 860{
 861	cq->db.data.agg_flags = flags;
 862	cq->db.data.value = cpu_to_le32(cons);
 863	writeq(cq->db.raw, cq->db_addr);
 864}
 865
 866int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 867{
 868	struct qedr_cq *cq = get_qedr_cq(ibcq);
 869	unsigned long sflags;
 870	struct qedr_dev *dev;
 871
 872	dev = get_qedr_dev(ibcq->device);
 873
 874	if (cq->destroyed) {
 875		DP_ERR(dev,
 876		       "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 877		       cq, cq->icid);
 878		return -EINVAL;
 879	}
 880
 881
 882	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 883		return 0;
 884
 885	spin_lock_irqsave(&cq->cq_lock, sflags);
 886
 887	cq->arm_flags = 0;
 888
 889	if (flags & IB_CQ_SOLICITED)
 890		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 891
 892	if (flags & IB_CQ_NEXT_COMP)
 893		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 894
 895	doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 896
 897	spin_unlock_irqrestore(&cq->cq_lock, sflags);
 898
 899	return 0;
 900}
 901
 902int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 903		   struct uverbs_attr_bundle *attrs)
 904{
 905	struct ib_udata *udata = &attrs->driver_udata;
 906	struct ib_device *ibdev = ibcq->device;
 907	struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 908		udata, struct qedr_ucontext, ibucontext);
 909	struct qed_rdma_destroy_cq_out_params destroy_oparams;
 910	struct qed_rdma_destroy_cq_in_params destroy_iparams;
 911	struct qed_chain_init_params chain_params = {
 912		.mode		= QED_CHAIN_MODE_PBL,
 913		.intended_use	= QED_CHAIN_USE_TO_CONSUME,
 914		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
 915		.elem_size	= sizeof(union rdma_cqe),
 916	};
 917	struct qedr_dev *dev = get_qedr_dev(ibdev);
 918	struct qed_rdma_create_cq_in_params params;
 919	struct qedr_create_cq_ureq ureq = {};
 920	int vector = attr->comp_vector;
 921	int entries = attr->cqe;
 922	struct qedr_cq *cq = get_qedr_cq(ibcq);
 923	int chain_entries;
 924	u32 db_offset;
 925	int page_cnt;
 926	u64 pbl_ptr;
 927	u16 icid;
 928	int rc;
 929
 930	DP_DEBUG(dev, QEDR_MSG_INIT,
 931		 "create_cq: called from %s. entries=%d, vector=%d\n",
 932		 udata ? "User Lib" : "Kernel", entries, vector);
 933
 934	if (attr->flags)
 935		return -EOPNOTSUPP;
 936
 937	if (entries > QEDR_MAX_CQES) {
 938		DP_ERR(dev,
 939		       "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 940		       entries, QEDR_MAX_CQES);
 941		return -EINVAL;
 942	}
 943
 944	chain_entries = qedr_align_cq_entries(entries);
 945	chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 946	chain_params.num_elems = chain_entries;
 947
 948	/* calc db offset. user will add DPI base, kernel will add db addr */
 949	db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 950
 951	if (udata) {
 952		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
 953							 udata->inlen))) {
 954			DP_ERR(dev,
 955			       "create cq: problem copying data from user space\n");
 956			goto err0;
 957		}
 958
 959		if (!ureq.len) {
 960			DP_ERR(dev,
 961			       "create cq: cannot create a cq with 0 entries\n");
 962			goto err0;
 963		}
 964
 965		cq->cq_type = QEDR_CQ_TYPE_USER;
 966
 967		rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 968					  ureq.len, true, IB_ACCESS_LOCAL_WRITE,
 969					  1);
 970		if (rc)
 971			goto err0;
 972
 973		pbl_ptr = cq->q.pbl_tbl->pa;
 974		page_cnt = cq->q.pbl_info.num_pbes;
 975
 976		cq->ibcq.cqe = chain_entries;
 977		cq->q.db_addr = ctx->dpi_addr + db_offset;
 978	} else {
 979		cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 980
 981		rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
 982						   &chain_params);
 983		if (rc)
 984			goto err0;
 985
 986		page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 987		pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 988		cq->ibcq.cqe = cq->pbl.capacity;
 989	}
 990
 991	qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 992			    pbl_ptr, &params);
 993
 994	rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 995	if (rc)
 996		goto err1;
 997
 998	cq->icid = icid;
 999	cq->sig = QEDR_CQ_MAGIC_NUMBER;
1000	spin_lock_init(&cq->cq_lock);
1001
1002	if (udata) {
1003		rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1004		if (rc)
1005			goto err2;
1006
1007		rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1008					  &cq->q.db_rec_data->db_data,
1009					  DB_REC_WIDTH_64B,
1010					  DB_REC_USER);
1011		if (rc)
1012			goto err2;
1013
1014	} else {
1015		/* Generate doorbell address. */
1016		cq->db.data.icid = cq->icid;
1017		cq->db_addr = dev->db_addr + db_offset;
1018		cq->db.data.params = DB_AGG_CMD_MAX <<
1019		    RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1020
1021		/* point to the very last element, passing it we will toggle */
1022		cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1023		cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1024		cq->latest_cqe = NULL;
1025		consume_cqe(cq);
1026		cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1027
1028		rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1029					  DB_REC_WIDTH_64B, DB_REC_KERNEL);
1030		if (rc)
1031			goto err2;
1032	}
1033
1034	DP_DEBUG(dev, QEDR_MSG_CQ,
1035		 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1036		 cq->icid, cq, params.cq_size);
1037
1038	return 0;
1039
1040err2:
1041	destroy_iparams.icid = cq->icid;
1042	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1043				  &destroy_oparams);
1044err1:
1045	if (udata) {
1046		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1047		ib_umem_release(cq->q.umem);
1048		if (cq->q.db_mmap_entry)
1049			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1050	} else {
1051		dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1052	}
1053err0:
1054	return -EINVAL;
1055}
1056
1057#define QEDR_DESTROY_CQ_MAX_ITERATIONS		(10)
1058#define QEDR_DESTROY_CQ_ITER_DURATION		(10)
1059
1060int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1061{
1062	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1063	struct qed_rdma_destroy_cq_out_params oparams;
1064	struct qed_rdma_destroy_cq_in_params iparams;
1065	struct qedr_cq *cq = get_qedr_cq(ibcq);
1066	int iter;
1067
1068	DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1069
1070	cq->destroyed = 1;
1071
1072	/* GSIs CQs are handled by driver, so they don't exist in the FW */
1073	if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1074		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1075		return 0;
1076	}
1077
1078	iparams.icid = cq->icid;
1079	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1080	dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1081
1082	if (udata) {
1083		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1084		ib_umem_release(cq->q.umem);
1085
1086		if (cq->q.db_rec_data) {
1087			qedr_db_recovery_del(dev, cq->q.db_addr,
1088					     &cq->q.db_rec_data->db_data);
1089			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1090		}
1091	} else {
1092		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1093	}
1094
1095	/* We don't want the IRQ handler to handle a non-existing CQ so we
1096	 * wait until all CNQ interrupts, if any, are received. This will always
1097	 * happen and will always happen very fast. If not, then a serious error
1098	 * has occured. That is why we can use a long delay.
1099	 * We spin for a short time so we don’t lose time on context switching
1100	 * in case all the completions are handled in that span. Otherwise
1101	 * we sleep for a while and check again. Since the CNQ may be
1102	 * associated with (only) the current CPU we use msleep to allow the
1103	 * current CPU to be freed.
1104	 * The CNQ notification is increased in qedr_irq_handler().
1105	 */
1106	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1107	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1108		udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1109		iter--;
1110	}
1111
1112	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1113	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1114		msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1115		iter--;
1116	}
1117
1118	/* Note that we don't need to have explicit code to wait for the
1119	 * completion of the event handler because it is invoked from the EQ.
1120	 * Since the destroy CQ ramrod has also been received on the EQ we can
1121	 * be certain that there's no event handler in process.
1122	 */
1123	return 0;
1124}
1125
1126static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1127					  struct ib_qp_attr *attr,
1128					  int attr_mask,
1129					  struct qed_rdma_modify_qp_in_params
1130					  *qp_params)
1131{
1132	const struct ib_gid_attr *gid_attr;
1133	enum rdma_network_type nw_type;
1134	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1135	u32 ipv4_addr;
1136	int ret;
1137	int i;
1138
1139	gid_attr = grh->sgid_attr;
1140	ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1141	if (ret)
1142		return ret;
1143
1144	nw_type = rdma_gid_attr_network_type(gid_attr);
1145	switch (nw_type) {
1146	case RDMA_NETWORK_IPV6:
1147		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1148		       sizeof(qp_params->sgid));
1149		memcpy(&qp_params->dgid.bytes[0],
1150		       &grh->dgid,
1151		       sizeof(qp_params->dgid));
1152		qp_params->roce_mode = ROCE_V2_IPV6;
1153		SET_FIELD(qp_params->modify_flags,
1154			  QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1155		break;
1156	case RDMA_NETWORK_ROCE_V1:
1157		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1158		       sizeof(qp_params->sgid));
1159		memcpy(&qp_params->dgid.bytes[0],
1160		       &grh->dgid,
1161		       sizeof(qp_params->dgid));
1162		qp_params->roce_mode = ROCE_V1;
1163		break;
1164	case RDMA_NETWORK_IPV4:
1165		memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1166		memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1167		ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1168		qp_params->sgid.ipv4_addr = ipv4_addr;
1169		ipv4_addr =
1170		    qedr_get_ipv4_from_gid(grh->dgid.raw);
1171		qp_params->dgid.ipv4_addr = ipv4_addr;
1172		SET_FIELD(qp_params->modify_flags,
1173			  QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1174		qp_params->roce_mode = ROCE_V2_IPV4;
1175		break;
1176	default:
1177		return -EINVAL;
1178	}
1179
1180	for (i = 0; i < 4; i++) {
1181		qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1182		qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1183	}
1184
1185	if (qp_params->vlan_id >= VLAN_CFI_MASK)
1186		qp_params->vlan_id = 0;
1187
1188	return 0;
1189}
1190
1191static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1192			       struct ib_qp_init_attr *attrs,
1193			       struct ib_udata *udata)
1194{
1195	struct qedr_device_attr *qattr = &dev->attr;
1196
1197	/* QP0... attrs->qp_type == IB_QPT_GSI */
1198	if (attrs->qp_type != IB_QPT_RC &&
1199	    attrs->qp_type != IB_QPT_GSI &&
1200	    attrs->qp_type != IB_QPT_XRC_INI &&
1201	    attrs->qp_type != IB_QPT_XRC_TGT) {
1202		DP_DEBUG(dev, QEDR_MSG_QP,
1203			 "create qp: unsupported qp type=0x%x requested\n",
1204			 attrs->qp_type);
1205		return -EOPNOTSUPP;
1206	}
1207
1208	if (attrs->cap.max_send_wr > qattr->max_sqe) {
1209		DP_ERR(dev,
1210		       "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1211		       attrs->cap.max_send_wr, qattr->max_sqe);
1212		return -EINVAL;
1213	}
1214
1215	if (attrs->cap.max_inline_data > qattr->max_inline) {
1216		DP_ERR(dev,
1217		       "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1218		       attrs->cap.max_inline_data, qattr->max_inline);
1219		return -EINVAL;
1220	}
1221
1222	if (attrs->cap.max_send_sge > qattr->max_sge) {
1223		DP_ERR(dev,
1224		       "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1225		       attrs->cap.max_send_sge, qattr->max_sge);
1226		return -EINVAL;
1227	}
1228
1229	if (attrs->cap.max_recv_sge > qattr->max_sge) {
1230		DP_ERR(dev,
1231		       "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1232		       attrs->cap.max_recv_sge, qattr->max_sge);
1233		return -EINVAL;
1234	}
1235
1236	/* verify consumer QPs are not trying to use GSI QP's CQ.
1237	 * TGT QP isn't associated with RQ/SQ
1238	 */
1239	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1240	    (attrs->qp_type != IB_QPT_XRC_TGT) &&
1241	    (attrs->qp_type != IB_QPT_XRC_INI)) {
1242		struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1243		struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1244
1245		if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1246		    (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1247			DP_ERR(dev,
1248			       "create qp: consumer QP cannot use GSI CQs.\n");
1249			return -EINVAL;
1250		}
1251	}
1252
1253	return 0;
1254}
1255
1256static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1257			       struct qedr_srq *srq, struct ib_udata *udata)
1258{
1259	struct qedr_create_srq_uresp uresp = {};
1260	int rc;
1261
1262	uresp.srq_id = srq->srq_id;
1263
1264	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1265	if (rc)
1266		DP_ERR(dev, "create srq: problem copying data to user space\n");
1267
1268	return rc;
1269}
1270
1271static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1272			       struct qedr_create_qp_uresp *uresp,
1273			       struct qedr_qp *qp)
1274{
1275	/* iWARP requires two doorbells per RQ. */
1276	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1277		uresp->rq_db_offset =
1278		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1279		uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1280	} else {
1281		uresp->rq_db_offset =
1282		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1283	}
1284
1285	uresp->rq_icid = qp->icid;
1286	if (qp->urq.db_mmap_entry)
1287		uresp->rq_db_rec_addr =
1288			rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1289}
1290
1291static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1292			       struct qedr_create_qp_uresp *uresp,
1293			       struct qedr_qp *qp)
1294{
1295	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1296
1297	/* iWARP uses the same cid for rq and sq */
1298	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1299		uresp->sq_icid = qp->icid;
1300	else
1301		uresp->sq_icid = qp->icid + 1;
1302
1303	if (qp->usq.db_mmap_entry)
1304		uresp->sq_db_rec_addr =
1305			rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1306}
1307
1308static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1309			      struct qedr_qp *qp, struct ib_udata *udata,
1310			      struct qedr_create_qp_uresp *uresp)
1311{
1312	int rc;
1313
1314	memset(uresp, 0, sizeof(*uresp));
1315
1316	if (qedr_qp_has_sq(qp))
1317		qedr_copy_sq_uresp(dev, uresp, qp);
1318
1319	if (qedr_qp_has_rq(qp))
1320		qedr_copy_rq_uresp(dev, uresp, qp);
1321
1322	uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1323	uresp->qp_id = qp->qp_id;
1324
1325	rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1326	if (rc)
1327		DP_ERR(dev,
1328		       "create qp: failed a copy to user space with qp icid=0x%x.\n",
1329		       qp->icid);
1330
1331	return rc;
1332}
1333
1334static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1335{
1336	qed_chain_reset(&qph->pbl);
1337	qph->prod = 0;
1338	qph->cons = 0;
1339	qph->wqe_cons = 0;
1340	qph->db_data.data.value = cpu_to_le16(0);
1341}
1342
1343static void qedr_set_common_qp_params(struct qedr_dev *dev,
1344				      struct qedr_qp *qp,
1345				      struct qedr_pd *pd,
1346				      struct ib_qp_init_attr *attrs)
1347{
1348	spin_lock_init(&qp->q_lock);
1349	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1350		kref_init(&qp->refcnt);
1351		init_completion(&qp->iwarp_cm_comp);
1352		init_completion(&qp->qp_rel_comp);
1353	}
1354
1355	qp->pd = pd;
1356	qp->qp_type = attrs->qp_type;
1357	qp->max_inline_data = attrs->cap.max_inline_data;
1358	qp->state = QED_ROCE_QP_STATE_RESET;
1359
1360	qp->prev_wqe_size = 0;
1361
1362	qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1363	qp->dev = dev;
1364	if (qedr_qp_has_sq(qp)) {
1365		qedr_reset_qp_hwq_info(&qp->sq);
1366		qp->sq.max_sges = attrs->cap.max_send_sge;
1367		qp->sq_cq = get_qedr_cq(attrs->send_cq);
1368		DP_DEBUG(dev, QEDR_MSG_QP,
1369			 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1370			 qp->sq.max_sges, qp->sq_cq->icid);
1371	}
1372
1373	if (attrs->srq)
1374		qp->srq = get_qedr_srq(attrs->srq);
1375
1376	if (qedr_qp_has_rq(qp)) {
1377		qedr_reset_qp_hwq_info(&qp->rq);
1378		qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1379		qp->rq.max_sges = attrs->cap.max_recv_sge;
1380		DP_DEBUG(dev, QEDR_MSG_QP,
1381			 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1382			 qp->rq.max_sges, qp->rq_cq->icid);
1383	}
1384
1385	DP_DEBUG(dev, QEDR_MSG_QP,
1386		 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1387		 pd->pd_id, qp->qp_type, qp->max_inline_data,
1388		 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1389	DP_DEBUG(dev, QEDR_MSG_QP,
1390		 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1391		 qp->sq.max_sges, qp->sq_cq->icid);
1392}
1393
1394static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1395{
1396	int rc = 0;
1397
1398	if (qedr_qp_has_sq(qp)) {
1399		qp->sq.db = dev->db_addr +
1400			    DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1401		qp->sq.db_data.data.icid = qp->icid + 1;
1402		rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1403					  DB_REC_WIDTH_32B, DB_REC_KERNEL);
1404		if (rc)
1405			return rc;
1406	}
1407
1408	if (qedr_qp_has_rq(qp)) {
1409		qp->rq.db = dev->db_addr +
1410			    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1411		qp->rq.db_data.data.icid = qp->icid;
1412		rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1413					  DB_REC_WIDTH_32B, DB_REC_KERNEL);
1414		if (rc && qedr_qp_has_sq(qp))
1415			qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1416	}
1417
1418	return rc;
1419}
1420
1421static int qedr_check_srq_params(struct qedr_dev *dev,
1422				 struct ib_srq_init_attr *attrs,
1423				 struct ib_udata *udata)
1424{
1425	struct qedr_device_attr *qattr = &dev->attr;
1426
1427	if (attrs->attr.max_wr > qattr->max_srq_wr) {
1428		DP_ERR(dev,
1429		       "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1430		       attrs->attr.max_wr, qattr->max_srq_wr);
1431		return -EINVAL;
1432	}
1433
1434	if (attrs->attr.max_sge > qattr->max_sge) {
1435		DP_ERR(dev,
1436		       "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1437		       attrs->attr.max_sge, qattr->max_sge);
1438	}
1439
1440	if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1441		DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1442		return -EINVAL;
1443	}
1444
1445	return 0;
1446}
1447
1448static void qedr_free_srq_user_params(struct qedr_srq *srq)
1449{
1450	qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1451	ib_umem_release(srq->usrq.umem);
1452	ib_umem_release(srq->prod_umem);
1453}
1454
1455static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1456{
1457	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1458	struct qedr_dev *dev = srq->dev;
1459
1460	dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1461
1462	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1463			  hw_srq->virt_prod_pair_addr,
1464			  hw_srq->phy_prod_pair_addr);
1465}
1466
1467static int qedr_init_srq_user_params(struct ib_udata *udata,
1468				     struct qedr_srq *srq,
1469				     struct qedr_create_srq_ureq *ureq,
1470				     int access)
1471{
1472	struct scatterlist *sg;
1473	int rc;
1474
1475	rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1476				  ureq->srq_len, false, access, 1);
1477	if (rc)
1478		return rc;
1479
1480	srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1481				     sizeof(struct rdma_srq_producers), access);
1482	if (IS_ERR(srq->prod_umem)) {
1483		qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1484		ib_umem_release(srq->usrq.umem);
1485		DP_ERR(srq->dev,
1486		       "create srq: failed ib_umem_get for producer, got %ld\n",
1487		       PTR_ERR(srq->prod_umem));
1488		return PTR_ERR(srq->prod_umem);
1489	}
1490
1491	sg = srq->prod_umem->sgt_append.sgt.sgl;
1492	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1493
1494	return 0;
1495}
1496
1497static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1498					struct qedr_dev *dev,
1499					struct ib_srq_init_attr *init_attr)
1500{
1501	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1502	struct qed_chain_init_params params = {
1503		.mode		= QED_CHAIN_MODE_PBL,
1504		.intended_use	= QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1505		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
1506		.elem_size	= QEDR_SRQ_WQE_ELEM_SIZE,
1507	};
1508	dma_addr_t phy_prod_pair_addr;
1509	u32 num_elems;
1510	void *va;
1511	int rc;
1512
1513	va = dma_alloc_coherent(&dev->pdev->dev,
1514				sizeof(struct rdma_srq_producers),
1515				&phy_prod_pair_addr, GFP_KERNEL);
1516	if (!va) {
1517		DP_ERR(dev,
1518		       "create srq: failed to allocate dma memory for producer\n");
1519		return -ENOMEM;
1520	}
1521
1522	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1523	hw_srq->virt_prod_pair_addr = va;
1524
1525	num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1526	params.num_elems = num_elems;
1527
1528	rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1529	if (rc)
1530		goto err0;
1531
1532	hw_srq->num_elems = num_elems;
1533
1534	return 0;
1535
1536err0:
1537	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1538			  va, phy_prod_pair_addr);
1539	return rc;
1540}
1541
1542int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1543		    struct ib_udata *udata)
1544{
1545	struct qed_rdma_destroy_srq_in_params destroy_in_params;
1546	struct qed_rdma_create_srq_in_params in_params = {};
1547	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1548	struct qed_rdma_create_srq_out_params out_params;
1549	struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1550	struct qedr_create_srq_ureq ureq = {};
1551	u64 pbl_base_addr, phy_prod_pair_addr;
1552	struct qedr_srq_hwq_info *hw_srq;
1553	u32 page_cnt, page_size;
1554	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1555	int rc = 0;
1556
1557	DP_DEBUG(dev, QEDR_MSG_QP,
1558		 "create SRQ called from %s (pd %p)\n",
1559		 (udata) ? "User lib" : "kernel", pd);
1560
1561	if (init_attr->srq_type != IB_SRQT_BASIC &&
1562	    init_attr->srq_type != IB_SRQT_XRC)
1563		return -EOPNOTSUPP;
1564
1565	rc = qedr_check_srq_params(dev, init_attr, udata);
1566	if (rc)
1567		return -EINVAL;
1568
1569	srq->dev = dev;
1570	srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1571	hw_srq = &srq->hw_srq;
1572	spin_lock_init(&srq->lock);
1573
1574	hw_srq->max_wr = init_attr->attr.max_wr;
1575	hw_srq->max_sges = init_attr->attr.max_sge;
1576
1577	if (udata) {
1578		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1579							 udata->inlen))) {
1580			DP_ERR(dev,
1581			       "create srq: problem copying data from user space\n");
1582			goto err0;
1583		}
1584
1585		rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1586		if (rc)
1587			goto err0;
1588
1589		page_cnt = srq->usrq.pbl_info.num_pbes;
1590		pbl_base_addr = srq->usrq.pbl_tbl->pa;
1591		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1592		page_size = PAGE_SIZE;
1593	} else {
1594		struct qed_chain *pbl;
1595
1596		rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1597		if (rc)
1598			goto err0;
1599
1600		pbl = &hw_srq->pbl;
1601		page_cnt = qed_chain_get_page_cnt(pbl);
1602		pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1603		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1604		page_size = QED_CHAIN_PAGE_SIZE;
1605	}
1606
1607	in_params.pd_id = pd->pd_id;
1608	in_params.pbl_base_addr = pbl_base_addr;
1609	in_params.prod_pair_addr = phy_prod_pair_addr;
1610	in_params.num_pages = page_cnt;
1611	in_params.page_size = page_size;
1612	if (srq->is_xrc) {
1613		struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1614		struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1615
1616		in_params.is_xrc = 1;
1617		in_params.xrcd_id = xrcd->xrcd_id;
1618		in_params.cq_cid = cq->icid;
1619	}
1620
1621	rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1622	if (rc)
1623		goto err1;
1624
1625	srq->srq_id = out_params.srq_id;
1626
1627	if (udata) {
1628		rc = qedr_copy_srq_uresp(dev, srq, udata);
1629		if (rc)
1630			goto err2;
1631	}
1632
1633	rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1634	if (rc)
1635		goto err2;
1636
1637	DP_DEBUG(dev, QEDR_MSG_SRQ,
1638		 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1639	return 0;
1640
1641err2:
1642	destroy_in_params.srq_id = srq->srq_id;
1643
1644	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1645err1:
1646	if (udata)
1647		qedr_free_srq_user_params(srq);
1648	else
1649		qedr_free_srq_kernel_params(srq);
1650err0:
1651	return -EFAULT;
1652}
1653
1654int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1655{
1656	struct qed_rdma_destroy_srq_in_params in_params = {};
1657	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1658	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1659
1660	xa_erase_irq(&dev->srqs, srq->srq_id);
1661	in_params.srq_id = srq->srq_id;
1662	in_params.is_xrc = srq->is_xrc;
1663	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1664
1665	if (ibsrq->uobject)
1666		qedr_free_srq_user_params(srq);
1667	else
1668		qedr_free_srq_kernel_params(srq);
1669
1670	DP_DEBUG(dev, QEDR_MSG_SRQ,
1671		 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1672		 srq->srq_id);
1673	return 0;
1674}
1675
1676int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1677		    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1678{
1679	struct qed_rdma_modify_srq_in_params in_params = {};
1680	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1681	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1682	int rc;
1683
1684	if (attr_mask & IB_SRQ_MAX_WR) {
1685		DP_ERR(dev,
1686		       "modify srq: invalid attribute mask=0x%x specified for %p\n",
1687		       attr_mask, srq);
1688		return -EINVAL;
1689	}
1690
1691	if (attr_mask & IB_SRQ_LIMIT) {
1692		if (attr->srq_limit >= srq->hw_srq.max_wr) {
1693			DP_ERR(dev,
1694			       "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1695			       attr->srq_limit, srq->hw_srq.max_wr);
1696			return -EINVAL;
1697		}
1698
1699		in_params.srq_id = srq->srq_id;
1700		in_params.wqe_limit = attr->srq_limit;
1701		rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1702		if (rc)
1703			return rc;
1704	}
1705
1706	srq->srq_limit = attr->srq_limit;
1707
1708	DP_DEBUG(dev, QEDR_MSG_SRQ,
1709		 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1710
1711	return 0;
1712}
1713
1714static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1715{
1716	switch (ib_qp_type) {
1717	case IB_QPT_RC:
1718		return QED_RDMA_QP_TYPE_RC;
1719	case IB_QPT_XRC_INI:
1720		return QED_RDMA_QP_TYPE_XRC_INI;
1721	case IB_QPT_XRC_TGT:
1722		return QED_RDMA_QP_TYPE_XRC_TGT;
1723	default:
1724		return QED_RDMA_QP_TYPE_INVAL;
1725	}
1726}
1727
1728static inline void
1729qedr_init_common_qp_in_params(struct qedr_dev *dev,
1730			      struct qedr_pd *pd,
1731			      struct qedr_qp *qp,
1732			      struct ib_qp_init_attr *attrs,
1733			      bool fmr_and_reserved_lkey,
1734			      struct qed_rdma_create_qp_in_params *params)
1735{
1736	/* QP handle to be written in an async event */
1737	params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1738	params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1739
1740	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1741	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1742	params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1743	params->stats_queue = 0;
1744
1745	if (pd) {
1746		params->pd = pd->pd_id;
1747		params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1748	}
1749
1750	if (qedr_qp_has_sq(qp))
1751		params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1752
1753	if (qedr_qp_has_rq(qp))
1754		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1755
1756	if (qedr_qp_has_srq(qp)) {
1757		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1758		params->srq_id = qp->srq->srq_id;
1759		params->use_srq = true;
1760	} else {
1761		params->srq_id = 0;
1762		params->use_srq = false;
1763	}
1764}
1765
1766static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1767{
1768	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1769		 "qp=%p. "
1770		 "sq_addr=0x%llx, "
1771		 "sq_len=%zd, "
1772		 "rq_addr=0x%llx, "
1773		 "rq_len=%zd"
1774		 "\n",
1775		 qp,
1776		 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1777		 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1778		 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1779		 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1780}
1781
1782static inline void
1783qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1784			    struct qedr_qp *qp,
1785			    struct qed_rdma_create_qp_out_params *out_params)
1786{
1787	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1788	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1789
1790	qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1791			   &qp->usq.pbl_info, FW_PAGE_SHIFT);
1792	if (!qp->srq) {
1793		qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1794		qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1795	}
1796
1797	qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1798			   &qp->urq.pbl_info, FW_PAGE_SHIFT);
1799}
1800
1801static void qedr_cleanup_user(struct qedr_dev *dev,
1802			      struct qedr_ucontext *ctx,
1803			      struct qedr_qp *qp)
1804{
1805	if (qedr_qp_has_sq(qp)) {
1806		ib_umem_release(qp->usq.umem);
1807		qp->usq.umem = NULL;
1808	}
1809
1810	if (qedr_qp_has_rq(qp)) {
1811		ib_umem_release(qp->urq.umem);
1812		qp->urq.umem = NULL;
1813	}
1814
1815	if (rdma_protocol_roce(&dev->ibdev, 1)) {
1816		qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1817		qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1818	} else {
1819		kfree(qp->usq.pbl_tbl);
1820		kfree(qp->urq.pbl_tbl);
1821	}
1822
1823	if (qp->usq.db_rec_data) {
1824		qedr_db_recovery_del(dev, qp->usq.db_addr,
1825				     &qp->usq.db_rec_data->db_data);
1826		rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1827	}
1828
1829	if (qp->urq.db_rec_data) {
1830		qedr_db_recovery_del(dev, qp->urq.db_addr,
1831				     &qp->urq.db_rec_data->db_data);
1832		rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1833	}
1834
1835	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1836		qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1837				     &qp->urq.db_rec_db2_data);
1838}
1839
1840static int qedr_create_user_qp(struct qedr_dev *dev,
1841			       struct qedr_qp *qp,
1842			       struct ib_pd *ibpd,
1843			       struct ib_udata *udata,
1844			       struct ib_qp_init_attr *attrs)
1845{
1846	struct qed_rdma_create_qp_in_params in_params;
1847	struct qed_rdma_create_qp_out_params out_params;
1848	struct qedr_create_qp_uresp uresp = {};
1849	struct qedr_create_qp_ureq ureq = {};
1850	int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1851	struct qedr_ucontext *ctx = NULL;
1852	struct qedr_pd *pd = NULL;
1853	int rc = 0;
1854
1855	qp->create_type = QEDR_QP_CREATE_USER;
1856
1857	if (ibpd) {
1858		pd = get_qedr_pd(ibpd);
1859		ctx = pd->uctx;
1860	}
1861
1862	if (udata) {
1863		rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1864					udata->inlen));
1865		if (rc) {
1866			DP_ERR(dev, "Problem copying data from user space\n");
1867			return rc;
1868		}
1869	}
1870
1871	if (qedr_qp_has_sq(qp)) {
1872		/* SQ - read access only (0) */
1873		rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1874					  ureq.sq_len, true, 0, alloc_and_init);
1875		if (rc)
1876			return rc;
1877	}
1878
1879	if (qedr_qp_has_rq(qp)) {
1880		/* RQ - read access only (0) */
1881		rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1882					  ureq.rq_len, true, 0, alloc_and_init);
1883		if (rc) {
1884			ib_umem_release(qp->usq.umem);
1885			qp->usq.umem = NULL;
1886			if (rdma_protocol_roce(&dev->ibdev, 1)) {
1887				qedr_free_pbl(dev, &qp->usq.pbl_info,
1888					      qp->usq.pbl_tbl);
1889			} else {
1890				kfree(qp->usq.pbl_tbl);
1891			}
1892			return rc;
1893		}
1894	}
1895
1896	memset(&in_params, 0, sizeof(in_params));
1897	qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1898	in_params.qp_handle_lo = ureq.qp_handle_lo;
1899	in_params.qp_handle_hi = ureq.qp_handle_hi;
1900
1901	if (qp->qp_type == IB_QPT_XRC_TGT) {
1902		struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1903
1904		in_params.xrcd_id = xrcd->xrcd_id;
1905		in_params.qp_handle_lo = qp->qp_id;
1906		in_params.use_srq = 1;
1907	}
1908
1909	if (qedr_qp_has_sq(qp)) {
1910		in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1911		in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1912	}
1913
1914	if (qedr_qp_has_rq(qp)) {
1915		in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1916		in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1917	}
1918
1919	if (ctx)
1920		SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1921
1922	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1923					      &in_params, &out_params);
1924
1925	if (!qp->qed_qp) {
1926		rc = -ENOMEM;
1927		goto err1;
1928	}
1929
1930	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1931		qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1932
1933	qp->qp_id = out_params.qp_id;
1934	qp->icid = out_params.icid;
1935
1936	if (udata) {
1937		rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1938		if (rc)
1939			goto err;
1940	}
1941
1942	/* db offset was calculated in copy_qp_uresp, now set in the user q */
1943	if (qedr_qp_has_sq(qp)) {
1944		qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1945		qp->sq.max_wr = attrs->cap.max_send_wr;
1946		rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1947					  &qp->usq.db_rec_data->db_data,
1948					  DB_REC_WIDTH_32B,
1949					  DB_REC_USER);
1950		if (rc)
1951			goto err;
1952	}
1953
1954	if (qedr_qp_has_rq(qp)) {
1955		qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1956		qp->rq.max_wr = attrs->cap.max_recv_wr;
1957		rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1958					  &qp->urq.db_rec_data->db_data,
1959					  DB_REC_WIDTH_32B,
1960					  DB_REC_USER);
1961		if (rc)
1962			goto err;
1963	}
1964
1965	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1966		qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1967
1968		/* calculate the db_rec_db2 data since it is constant so no
1969		 * need to reflect from user
1970		 */
1971		qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1972		qp->urq.db_rec_db2_data.data.value =
1973			cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1974
1975		rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1976					  &qp->urq.db_rec_db2_data,
1977					  DB_REC_WIDTH_32B,
1978					  DB_REC_USER);
1979		if (rc)
1980			goto err;
1981	}
1982	qedr_qp_user_print(dev, qp);
1983	return rc;
1984err:
1985	rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1986	if (rc)
1987		DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1988
1989err1:
1990	qedr_cleanup_user(dev, ctx, qp);
1991	return rc;
1992}
1993
1994static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1995{
1996	int rc;
1997
1998	qp->sq.db = dev->db_addr +
1999	    DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2000	qp->sq.db_data.data.icid = qp->icid;
2001
2002	rc = qedr_db_recovery_add(dev, qp->sq.db,
2003				  &qp->sq.db_data,
2004				  DB_REC_WIDTH_32B,
2005				  DB_REC_KERNEL);
2006	if (rc)
2007		return rc;
2008
2009	qp->rq.db = dev->db_addr +
2010		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2011	qp->rq.db_data.data.icid = qp->icid;
2012	qp->rq.iwarp_db2 = dev->db_addr +
2013			   DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2014	qp->rq.iwarp_db2_data.data.icid = qp->icid;
2015	qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2016
2017	rc = qedr_db_recovery_add(dev, qp->rq.db,
2018				  &qp->rq.db_data,
2019				  DB_REC_WIDTH_32B,
2020				  DB_REC_KERNEL);
2021	if (rc)
2022		return rc;
2023
2024	rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2025				  &qp->rq.iwarp_db2_data,
2026				  DB_REC_WIDTH_32B,
2027				  DB_REC_KERNEL);
2028	return rc;
2029}
2030
2031static int
2032qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2033			   struct qedr_qp *qp,
2034			   struct qed_rdma_create_qp_in_params *in_params,
2035			   u32 n_sq_elems, u32 n_rq_elems)
2036{
2037	struct qed_rdma_create_qp_out_params out_params;
2038	struct qed_chain_init_params params = {
2039		.mode		= QED_CHAIN_MODE_PBL,
2040		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
2041	};
2042	int rc;
2043
2044	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2045	params.num_elems = n_sq_elems;
2046	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2047
2048	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2049	if (rc)
2050		return rc;
2051
2052	in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2053	in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2054
2055	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2056	params.num_elems = n_rq_elems;
2057	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2058
2059	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2060	if (rc)
2061		return rc;
2062
2063	in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2064	in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2065
2066	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2067					      in_params, &out_params);
2068
2069	if (!qp->qed_qp)
2070		return -EINVAL;
2071
2072	qp->qp_id = out_params.qp_id;
2073	qp->icid = out_params.icid;
2074
2075	return qedr_set_roce_db_info(dev, qp);
2076}
2077
2078static int
2079qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2080			    struct qedr_qp *qp,
2081			    struct qed_rdma_create_qp_in_params *in_params,
2082			    u32 n_sq_elems, u32 n_rq_elems)
2083{
2084	struct qed_rdma_create_qp_out_params out_params;
2085	struct qed_chain_init_params params = {
2086		.mode		= QED_CHAIN_MODE_PBL,
2087		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
2088	};
2089	int rc;
2090
2091	in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2092						     QEDR_SQE_ELEMENT_SIZE,
2093						     QED_CHAIN_PAGE_SIZE,
2094						     QED_CHAIN_MODE_PBL);
2095	in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2096						     QEDR_RQE_ELEMENT_SIZE,
2097						     QED_CHAIN_PAGE_SIZE,
2098						     QED_CHAIN_MODE_PBL);
2099
2100	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2101					      in_params, &out_params);
2102
2103	if (!qp->qed_qp)
2104		return -EINVAL;
2105
2106	/* Now we allocate the chain */
2107
2108	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2109	params.num_elems = n_sq_elems;
2110	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2111	params.ext_pbl_virt = out_params.sq_pbl_virt;
2112	params.ext_pbl_phys = out_params.sq_pbl_phys;
2113
2114	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2115	if (rc)
2116		goto err;
2117
2118	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2119	params.num_elems = n_rq_elems;
2120	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2121	params.ext_pbl_virt = out_params.rq_pbl_virt;
2122	params.ext_pbl_phys = out_params.rq_pbl_phys;
2123
2124	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2125	if (rc)
2126		goto err;
2127
2128	qp->qp_id = out_params.qp_id;
2129	qp->icid = out_params.icid;
2130
2131	return qedr_set_iwarp_db_info(dev, qp);
2132
2133err:
2134	dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2135
2136	return rc;
2137}
2138
2139static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2140{
2141	dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2142	kfree(qp->wqe_wr_id);
2143
2144	dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2145	kfree(qp->rqe_wr_id);
2146
2147	/* GSI qp is not registered to db mechanism so no need to delete */
2148	if (qp->qp_type == IB_QPT_GSI)
2149		return;
2150
2151	qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2152
2153	if (!qp->srq) {
2154		qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2155
2156		if (rdma_protocol_iwarp(&dev->ibdev, 1))
2157			qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2158					     &qp->rq.iwarp_db2_data);
2159	}
2160}
2161
2162static int qedr_create_kernel_qp(struct qedr_dev *dev,
2163				 struct qedr_qp *qp,
2164				 struct ib_pd *ibpd,
2165				 struct ib_qp_init_attr *attrs)
2166{
2167	struct qed_rdma_create_qp_in_params in_params;
2168	struct qedr_pd *pd = get_qedr_pd(ibpd);
2169	int rc = -EINVAL;
2170	u32 n_rq_elems;
2171	u32 n_sq_elems;
2172	u32 n_sq_entries;
2173
2174	memset(&in_params, 0, sizeof(in_params));
2175	qp->create_type = QEDR_QP_CREATE_KERNEL;
2176
2177	/* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2178	 * the ring. The ring should allow at least a single WR, even if the
2179	 * user requested none, due to allocation issues.
2180	 * We should add an extra WR since the prod and cons indices of
2181	 * wqe_wr_id are managed in such a way that the WQ is considered full
2182	 * when (prod+1)%max_wr==cons. We currently don't do that because we
2183	 * double the number of entries due an iSER issue that pushes far more
2184	 * WRs than indicated. If we decline its ib_post_send() then we get
2185	 * error prints in the dmesg we'd like to avoid.
2186	 */
2187	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2188			      dev->attr.max_sqe);
2189
2190	qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2191				GFP_KERNEL);
2192	if (!qp->wqe_wr_id) {
2193		DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2194		return -ENOMEM;
2195	}
2196
2197	/* QP handle to be written in CQE */
2198	in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2199	in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2200
2201	/* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2202	 * the ring. There ring should allow at least a single WR, even if the
2203	 * user requested none, due to allocation issues.
2204	 */
2205	qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2206
2207	/* Allocate driver internal RQ array */
2208	qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2209				GFP_KERNEL);
2210	if (!qp->rqe_wr_id) {
2211		DP_ERR(dev,
2212		       "create qp: failed RQ shadow memory allocation\n");
2213		kfree(qp->wqe_wr_id);
2214		return -ENOMEM;
2215	}
2216
2217	qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2218
2219	n_sq_entries = attrs->cap.max_send_wr;
2220	n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2221	n_sq_entries = max_t(u32, n_sq_entries, 1);
2222	n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2223
2224	n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2225
2226	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2227		rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2228						 n_sq_elems, n_rq_elems);
2229	else
2230		rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2231						n_sq_elems, n_rq_elems);
2232	if (rc)
2233		qedr_cleanup_kernel(dev, qp);
2234
2235	return rc;
2236}
2237
2238static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2239				  struct ib_udata *udata)
2240{
2241	struct qedr_ucontext *ctx =
2242		rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2243					  ibucontext);
2244	int rc;
2245
2246	if (qp->qp_type != IB_QPT_GSI) {
2247		rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2248		if (rc)
2249			return rc;
2250	}
2251
2252	if (qp->create_type == QEDR_QP_CREATE_USER)
2253		qedr_cleanup_user(dev, ctx, qp);
2254	else
2255		qedr_cleanup_kernel(dev, qp);
2256
2257	return 0;
2258}
2259
2260int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2261		   struct ib_udata *udata)
2262{
2263	struct qedr_xrcd *xrcd = NULL;
2264	struct ib_pd *ibpd = ibqp->pd;
2265	struct qedr_pd *pd = get_qedr_pd(ibpd);
2266	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2267	struct qedr_qp *qp = get_qedr_qp(ibqp);
2268	int rc = 0;
2269
2270	if (attrs->create_flags)
2271		return -EOPNOTSUPP;
2272
2273	if (attrs->qp_type == IB_QPT_XRC_TGT)
2274		xrcd = get_qedr_xrcd(attrs->xrcd);
2275	else
2276		pd = get_qedr_pd(ibpd);
2277
2278	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2279		 udata ? "user library" : "kernel", pd);
2280
2281	rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2282	if (rc)
2283		return rc;
2284
2285	DP_DEBUG(dev, QEDR_MSG_QP,
2286		 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2287		 udata ? "user library" : "kernel", attrs->event_handler, pd,
2288		 get_qedr_cq(attrs->send_cq),
2289		 get_qedr_cq(attrs->send_cq)->icid,
2290		 get_qedr_cq(attrs->recv_cq),
2291		 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2292
2293	qedr_set_common_qp_params(dev, qp, pd, attrs);
2294
2295	if (attrs->qp_type == IB_QPT_GSI)
2296		return qedr_create_gsi_qp(dev, attrs, qp);
2297
2298	if (udata || xrcd)
2299		rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2300	else
2301		rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2302
2303	if (rc)
2304		return rc;
2305
2306	qp->ibqp.qp_num = qp->qp_id;
2307
2308	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2309		rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2310		if (rc)
2311			goto out_free_qp_resources;
2312	}
2313
2314	return 0;
2315
2316out_free_qp_resources:
2317	qedr_free_qp_resources(dev, qp, udata);
2318	return -EFAULT;
2319}
2320
2321static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2322{
2323	switch (qp_state) {
2324	case QED_ROCE_QP_STATE_RESET:
2325		return IB_QPS_RESET;
2326	case QED_ROCE_QP_STATE_INIT:
2327		return IB_QPS_INIT;
2328	case QED_ROCE_QP_STATE_RTR:
2329		return IB_QPS_RTR;
2330	case QED_ROCE_QP_STATE_RTS:
2331		return IB_QPS_RTS;
2332	case QED_ROCE_QP_STATE_SQD:
2333		return IB_QPS_SQD;
2334	case QED_ROCE_QP_STATE_ERR:
2335		return IB_QPS_ERR;
2336	case QED_ROCE_QP_STATE_SQE:
2337		return IB_QPS_SQE;
2338	}
2339	return IB_QPS_ERR;
2340}
2341
2342static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2343					enum ib_qp_state qp_state)
2344{
2345	switch (qp_state) {
2346	case IB_QPS_RESET:
2347		return QED_ROCE_QP_STATE_RESET;
2348	case IB_QPS_INIT:
2349		return QED_ROCE_QP_STATE_INIT;
2350	case IB_QPS_RTR:
2351		return QED_ROCE_QP_STATE_RTR;
2352	case IB_QPS_RTS:
2353		return QED_ROCE_QP_STATE_RTS;
2354	case IB_QPS_SQD:
2355		return QED_ROCE_QP_STATE_SQD;
2356	case IB_QPS_ERR:
2357		return QED_ROCE_QP_STATE_ERR;
2358	default:
2359		return QED_ROCE_QP_STATE_ERR;
2360	}
2361}
2362
2363static int qedr_update_qp_state(struct qedr_dev *dev,
2364				struct qedr_qp *qp,
2365				enum qed_roce_qp_state cur_state,
2366				enum qed_roce_qp_state new_state)
2367{
2368	int status = 0;
2369
2370	if (new_state == cur_state)
2371		return 0;
2372
2373	switch (cur_state) {
2374	case QED_ROCE_QP_STATE_RESET:
2375		switch (new_state) {
2376		case QED_ROCE_QP_STATE_INIT:
2377			break;
2378		default:
2379			status = -EINVAL;
2380			break;
2381		}
2382		break;
2383	case QED_ROCE_QP_STATE_INIT:
2384		switch (new_state) {
2385		case QED_ROCE_QP_STATE_RTR:
2386			/* Update doorbell (in case post_recv was
2387			 * done before move to RTR)
2388			 */
2389
2390			if (rdma_protocol_roce(&dev->ibdev, 1)) {
2391				writel(qp->rq.db_data.raw, qp->rq.db);
2392			}
2393			break;
2394		case QED_ROCE_QP_STATE_ERR:
2395			break;
2396		default:
2397			/* Invalid state change. */
2398			status = -EINVAL;
2399			break;
2400		}
2401		break;
2402	case QED_ROCE_QP_STATE_RTR:
2403		/* RTR->XXX */
2404		switch (new_state) {
2405		case QED_ROCE_QP_STATE_RTS:
2406			break;
2407		case QED_ROCE_QP_STATE_ERR:
2408			break;
2409		default:
2410			/* Invalid state change. */
2411			status = -EINVAL;
2412			break;
2413		}
2414		break;
2415	case QED_ROCE_QP_STATE_RTS:
2416		/* RTS->XXX */
2417		switch (new_state) {
2418		case QED_ROCE_QP_STATE_SQD:
2419			break;
2420		case QED_ROCE_QP_STATE_ERR:
2421			break;
2422		default:
2423			/* Invalid state change. */
2424			status = -EINVAL;
2425			break;
2426		}
2427		break;
2428	case QED_ROCE_QP_STATE_SQD:
2429		/* SQD->XXX */
2430		switch (new_state) {
2431		case QED_ROCE_QP_STATE_RTS:
2432		case QED_ROCE_QP_STATE_ERR:
2433			break;
2434		default:
2435			/* Invalid state change. */
2436			status = -EINVAL;
2437			break;
2438		}
2439		break;
2440	case QED_ROCE_QP_STATE_ERR:
2441		/* ERR->XXX */
2442		switch (new_state) {
2443		case QED_ROCE_QP_STATE_RESET:
2444			if ((qp->rq.prod != qp->rq.cons) ||
2445			    (qp->sq.prod != qp->sq.cons)) {
2446				DP_NOTICE(dev,
2447					  "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2448					  qp->rq.prod, qp->rq.cons, qp->sq.prod,
2449					  qp->sq.cons);
2450				status = -EINVAL;
2451			}
2452			break;
2453		default:
2454			status = -EINVAL;
2455			break;
2456		}
2457		break;
2458	default:
2459		status = -EINVAL;
2460		break;
2461	}
2462
2463	return status;
2464}
2465
2466int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2467		   int attr_mask, struct ib_udata *udata)
2468{
2469	struct qedr_qp *qp = get_qedr_qp(ibqp);
2470	struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2471	struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2472	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2473	enum ib_qp_state old_qp_state, new_qp_state;
2474	enum qed_roce_qp_state cur_state;
2475	int rc = 0;
2476
2477	DP_DEBUG(dev, QEDR_MSG_QP,
2478		 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2479		 attr->qp_state);
2480
2481	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2482		return -EOPNOTSUPP;
2483
2484	old_qp_state = qedr_get_ibqp_state(qp->state);
2485	if (attr_mask & IB_QP_STATE)
2486		new_qp_state = attr->qp_state;
2487	else
2488		new_qp_state = old_qp_state;
2489
2490	if (rdma_protocol_roce(&dev->ibdev, 1)) {
2491		if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2492					ibqp->qp_type, attr_mask)) {
2493			DP_ERR(dev,
2494			       "modify qp: invalid attribute mask=0x%x specified for\n"
2495			       "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2496			       attr_mask, qp->qp_id, ibqp->qp_type,
2497			       old_qp_state, new_qp_state);
2498			rc = -EINVAL;
2499			goto err;
2500		}
2501	}
2502
2503	/* Translate the masks... */
2504	if (attr_mask & IB_QP_STATE) {
2505		SET_FIELD(qp_params.modify_flags,
2506			  QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2507		qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2508	}
2509
2510	if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2511		qp_params.sqd_async = true;
2512
2513	if (attr_mask & IB_QP_PKEY_INDEX) {
2514		SET_FIELD(qp_params.modify_flags,
2515			  QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2516		if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2517			rc = -EINVAL;
2518			goto err;
2519		}
2520
2521		qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2522	}
2523
2524	if (attr_mask & IB_QP_QKEY)
2525		qp->qkey = attr->qkey;
2526
2527	if (attr_mask & IB_QP_ACCESS_FLAGS) {
2528		SET_FIELD(qp_params.modify_flags,
2529			  QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2530		qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2531						  IB_ACCESS_REMOTE_READ;
2532		qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2533						   IB_ACCESS_REMOTE_WRITE;
2534		qp_params.incoming_atomic_en = attr->qp_access_flags &
2535					       IB_ACCESS_REMOTE_ATOMIC;
2536	}
2537
2538	if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2539		if (rdma_protocol_iwarp(&dev->ibdev, 1))
2540			return -EINVAL;
2541
2542		if (attr_mask & IB_QP_PATH_MTU) {
2543			if (attr->path_mtu < IB_MTU_256 ||
2544			    attr->path_mtu > IB_MTU_4096) {
2545				pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2546				rc = -EINVAL;
2547				goto err;
2548			}
2549			qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2550				      ib_mtu_enum_to_int(iboe_get_mtu
2551							 (dev->ndev->mtu)));
2552		}
2553
2554		if (!qp->mtu) {
2555			qp->mtu =
2556			ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2557			pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2558		}
2559
2560		SET_FIELD(qp_params.modify_flags,
2561			  QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2562
2563		qp_params.traffic_class_tos = grh->traffic_class;
2564		qp_params.flow_label = grh->flow_label;
2565		qp_params.hop_limit_ttl = grh->hop_limit;
2566
2567		qp->sgid_idx = grh->sgid_index;
2568
2569		rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2570		if (rc) {
2571			DP_ERR(dev,
2572			       "modify qp: problems with GID index %d (rc=%d)\n",
2573			       grh->sgid_index, rc);
2574			return rc;
2575		}
2576
2577		rc = qedr_get_dmac(dev, &attr->ah_attr,
2578				   qp_params.remote_mac_addr);
2579		if (rc)
2580			return rc;
2581
2582		qp_params.use_local_mac = true;
2583		ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2584
2585		DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2586			 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2587			 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2588		DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2589			 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2590			 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2591		DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2592			 qp_params.remote_mac_addr);
2593
2594		qp_params.mtu = qp->mtu;
2595		qp_params.lb_indication = false;
2596	}
2597
2598	if (!qp_params.mtu) {
2599		/* Stay with current MTU */
2600		if (qp->mtu)
2601			qp_params.mtu = qp->mtu;
2602		else
2603			qp_params.mtu =
2604			    ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2605	}
2606
2607	if (attr_mask & IB_QP_TIMEOUT) {
2608		SET_FIELD(qp_params.modify_flags,
2609			  QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2610
2611		/* The received timeout value is an exponent used like this:
2612		 *    "12.7.34 LOCAL ACK TIMEOUT
2613		 *    Value representing the transport (ACK) timeout for use by
2614		 *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2615		 * The FW expects timeout in msec so we need to divide the usec
2616		 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2617		 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2618		 * The value of zero means infinite so we use a 'max_t' to make
2619		 * sure that sub 1 msec values will be configured as 1 msec.
2620		 */
2621		if (attr->timeout)
2622			qp_params.ack_timeout =
2623					1 << max_t(int, attr->timeout - 8, 0);
2624		else
2625			qp_params.ack_timeout = 0;
2626
2627		qp->timeout = attr->timeout;
2628	}
2629
2630	if (attr_mask & IB_QP_RETRY_CNT) {
2631		SET_FIELD(qp_params.modify_flags,
2632			  QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2633		qp_params.retry_cnt = attr->retry_cnt;
2634	}
2635
2636	if (attr_mask & IB_QP_RNR_RETRY) {
2637		SET_FIELD(qp_params.modify_flags,
2638			  QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2639		qp_params.rnr_retry_cnt = attr->rnr_retry;
2640	}
2641
2642	if (attr_mask & IB_QP_RQ_PSN) {
2643		SET_FIELD(qp_params.modify_flags,
2644			  QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2645		qp_params.rq_psn = attr->rq_psn;
2646		qp->rq_psn = attr->rq_psn;
2647	}
2648
2649	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2650		if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2651			rc = -EINVAL;
2652			DP_ERR(dev,
2653			       "unsupported max_rd_atomic=%d, supported=%d\n",
2654			       attr->max_rd_atomic,
2655			       dev->attr.max_qp_req_rd_atomic_resc);
2656			goto err;
2657		}
2658
2659		SET_FIELD(qp_params.modify_flags,
2660			  QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2661		qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2662	}
2663
2664	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2665		SET_FIELD(qp_params.modify_flags,
2666			  QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2667		qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2668	}
2669
2670	if (attr_mask & IB_QP_SQ_PSN) {
2671		SET_FIELD(qp_params.modify_flags,
2672			  QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2673		qp_params.sq_psn = attr->sq_psn;
2674		qp->sq_psn = attr->sq_psn;
2675	}
2676
2677	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2678		if (attr->max_dest_rd_atomic >
2679		    dev->attr.max_qp_resp_rd_atomic_resc) {
2680			DP_ERR(dev,
2681			       "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2682			       attr->max_dest_rd_atomic,
2683			       dev->attr.max_qp_resp_rd_atomic_resc);
2684
2685			rc = -EINVAL;
2686			goto err;
2687		}
2688
2689		SET_FIELD(qp_params.modify_flags,
2690			  QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2691		qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2692	}
2693
2694	if (attr_mask & IB_QP_DEST_QPN) {
2695		SET_FIELD(qp_params.modify_flags,
2696			  QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2697
2698		qp_params.dest_qp = attr->dest_qp_num;
2699		qp->dest_qp_num = attr->dest_qp_num;
2700	}
2701
2702	cur_state = qp->state;
2703
2704	/* Update the QP state before the actual ramrod to prevent a race with
2705	 * fast path. Modifying the QP state to error will cause the device to
2706	 * flush the CQEs and while polling the flushed CQEs will considered as
2707	 * a potential issue if the QP isn't in error state.
2708	 */
2709	if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2710	    !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2711		qp->state = QED_ROCE_QP_STATE_ERR;
2712
2713	if (qp->qp_type != IB_QPT_GSI)
2714		rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2715					      qp->qed_qp, &qp_params);
2716
2717	if (attr_mask & IB_QP_STATE) {
2718		if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2719			rc = qedr_update_qp_state(dev, qp, cur_state,
2720						  qp_params.new_state);
2721		qp->state = qp_params.new_state;
2722	}
2723
2724err:
2725	return rc;
2726}
2727
2728static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2729{
2730	int ib_qp_acc_flags = 0;
2731
2732	if (params->incoming_rdma_write_en)
2733		ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2734	if (params->incoming_rdma_read_en)
2735		ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2736	if (params->incoming_atomic_en)
2737		ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2738	ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2739	return ib_qp_acc_flags;
2740}
2741
2742int qedr_query_qp(struct ib_qp *ibqp,
2743		  struct ib_qp_attr *qp_attr,
2744		  int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2745{
2746	struct qed_rdma_query_qp_out_params params;
2747	struct qedr_qp *qp = get_qedr_qp(ibqp);
2748	struct qedr_dev *dev = qp->dev;
2749	int rc = 0;
2750
2751	memset(&params, 0, sizeof(params));
2752	memset(qp_attr, 0, sizeof(*qp_attr));
2753	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2754
2755	if (qp->qp_type != IB_QPT_GSI) {
2756		rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2757		if (rc)
2758			goto err;
2759		qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2760	} else {
2761		qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2762	}
2763
2764	qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2765	qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2766	qp_attr->path_mig_state = IB_MIG_MIGRATED;
2767	qp_attr->rq_psn = params.rq_psn;
2768	qp_attr->sq_psn = params.sq_psn;
2769	qp_attr->dest_qp_num = params.dest_qp;
2770
2771	qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2772
2773	qp_attr->cap.max_send_wr = qp->sq.max_wr;
2774	qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2775	qp_attr->cap.max_send_sge = qp->sq.max_sges;
2776	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2777	qp_attr->cap.max_inline_data = dev->attr.max_inline;
2778	qp_init_attr->cap = qp_attr->cap;
2779
2780	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2781	rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2782			params.flow_label, qp->sgid_idx,
2783			params.hop_limit_ttl, params.traffic_class_tos);
2784	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2785	rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2786	rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2787	qp_attr->timeout = qp->timeout;
2788	qp_attr->rnr_retry = params.rnr_retry;
2789	qp_attr->retry_cnt = params.retry_cnt;
2790	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2791	qp_attr->pkey_index = params.pkey_index;
2792	qp_attr->port_num = 1;
2793	rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2794	rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2795	qp_attr->alt_pkey_index = 0;
2796	qp_attr->alt_port_num = 0;
2797	qp_attr->alt_timeout = 0;
2798	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2799
2800	qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2801	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2802	qp_attr->max_rd_atomic = params.max_rd_atomic;
2803	qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2804
2805	DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2806		 qp_attr->cap.max_inline_data);
2807
2808err:
2809	return rc;
2810}
2811
2812int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2813{
2814	struct qedr_qp *qp = get_qedr_qp(ibqp);
2815	struct qedr_dev *dev = qp->dev;
2816	struct ib_qp_attr attr;
2817	int attr_mask = 0;
2818
2819	DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2820		 qp, qp->qp_type);
2821
2822	if (rdma_protocol_roce(&dev->ibdev, 1)) {
2823		if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2824		    (qp->state != QED_ROCE_QP_STATE_ERR) &&
2825		    (qp->state != QED_ROCE_QP_STATE_INIT)) {
2826
2827			attr.qp_state = IB_QPS_ERR;
2828			attr_mask |= IB_QP_STATE;
2829
2830			/* Change the QP state to ERROR */
2831			qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2832		}
2833	} else {
2834		/* If connection establishment started the WAIT_FOR_CONNECT
2835		 * bit will be on and we need to Wait for the establishment
2836		 * to complete before destroying the qp.
2837		 */
2838		if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2839				     &qp->iwarp_cm_flags))
2840			wait_for_completion(&qp->iwarp_cm_comp);
2841
2842		/* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2843		 * bit will be on, and we need to wait for the disconnect to
2844		 * complete before continuing. We can use the same completion,
2845		 * iwarp_cm_comp, since this is the only place that waits for
2846		 * this completion and it is sequential. In addition,
2847		 * disconnect can't occur before the connection is fully
2848		 * established, therefore if WAIT_FOR_DISCONNECT is on it
2849		 * means WAIT_FOR_CONNECT is also on and the completion for
2850		 * CONNECT already occurred.
2851		 */
2852		if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2853				     &qp->iwarp_cm_flags))
2854			wait_for_completion(&qp->iwarp_cm_comp);
2855	}
2856
2857	if (qp->qp_type == IB_QPT_GSI)
2858		qedr_destroy_gsi_qp(dev);
2859
2860	/* We need to remove the entry from the xarray before we release the
2861	 * qp_id to avoid a race of the qp_id being reallocated and failing
2862	 * on xa_insert
2863	 */
2864	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2865		xa_erase(&dev->qps, qp->qp_id);
2866
2867	qedr_free_qp_resources(dev, qp, udata);
2868
2869	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2870		qedr_iw_qp_rem_ref(&qp->ibqp);
2871		wait_for_completion(&qp->qp_rel_comp);
2872	}
2873
2874	return 0;
2875}
2876
2877int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2878		   struct ib_udata *udata)
2879{
2880	struct qedr_ah *ah = get_qedr_ah(ibah);
2881
2882	rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2883
2884	return 0;
2885}
2886
2887int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2888{
2889	struct qedr_ah *ah = get_qedr_ah(ibah);
2890
2891	rdma_destroy_ah_attr(&ah->attr);
2892	return 0;
2893}
2894
2895static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2896{
2897	struct qedr_pbl *pbl, *tmp;
2898
2899	if (info->pbl_table)
2900		list_add_tail(&info->pbl_table->list_entry,
2901			      &info->free_pbl_list);
2902
2903	if (!list_empty(&info->inuse_pbl_list))
2904		list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2905
2906	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2907		list_del(&pbl->list_entry);
2908		qedr_free_pbl(dev, &info->pbl_info, pbl);
2909	}
2910}
2911
2912static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2913			size_t page_list_len, bool two_layered)
2914{
2915	struct qedr_pbl *tmp;
2916	int rc;
2917
2918	INIT_LIST_HEAD(&info->free_pbl_list);
2919	INIT_LIST_HEAD(&info->inuse_pbl_list);
2920
2921	rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2922				  page_list_len, two_layered);
2923	if (rc)
2924		goto done;
2925
2926	info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2927	if (IS_ERR(info->pbl_table)) {
2928		rc = PTR_ERR(info->pbl_table);
2929		goto done;
2930	}
2931
2932	DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2933		 &info->pbl_table->pa);
2934
2935	/* in usual case we use 2 PBLs, so we add one to free
2936	 * list and allocating another one
2937	 */
2938	tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2939	if (IS_ERR(tmp)) {
2940		DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2941		goto done;
2942	}
2943
2944	list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2945
2946	DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2947
2948done:
2949	if (rc)
2950		free_mr_info(dev, info);
2951
2952	return rc;
2953}
2954
2955struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2956			       u64 usr_addr, int acc, struct ib_dmah *dmah,
2957			       struct ib_udata *udata)
2958{
2959	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2960	struct qedr_mr *mr;
2961	struct qedr_pd *pd;
2962	int rc = -ENOMEM;
2963
2964	if (dmah)
2965		return ERR_PTR(-EOPNOTSUPP);
2966
2967	pd = get_qedr_pd(ibpd);
2968	DP_DEBUG(dev, QEDR_MSG_MR,
2969		 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2970		 pd->pd_id, start, len, usr_addr, acc);
2971
2972	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2973		return ERR_PTR(-EINVAL);
2974
2975	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2976	if (!mr)
2977		return ERR_PTR(rc);
2978
2979	mr->type = QEDR_MR_USER;
2980
2981	mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2982	if (IS_ERR(mr->umem)) {
2983		rc = -EFAULT;
2984		goto err0;
2985	}
2986
2987	rc = init_mr_info(dev, &mr->info,
2988			  ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2989	if (rc)
2990		goto err1;
2991
2992	qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2993			   &mr->info.pbl_info, PAGE_SHIFT);
2994
2995	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2996	if (rc) {
2997		if (rc == -EINVAL)
2998			DP_ERR(dev, "Out of MR resources\n");
2999		else
3000			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3001
3002		goto err1;
3003	}
3004
3005	/* Index only, 18 bit long, lkey = itid << 8 | key */
3006	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3007	mr->hw_mr.key = 0;
3008	mr->hw_mr.pd = pd->pd_id;
3009	mr->hw_mr.local_read = 1;
3010	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3011	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3012	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3013	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3014	mr->hw_mr.mw_bind = false;
3015	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3016	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3017	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3018	mr->hw_mr.page_size_log = PAGE_SHIFT;
3019	mr->hw_mr.length = len;
3020	mr->hw_mr.vaddr = usr_addr;
3021	mr->hw_mr.phy_mr = false;
3022	mr->hw_mr.dma_mr = false;
3023
3024	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3025	if (rc) {
3026		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3027		goto err2;
3028	}
3029
3030	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3031	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3032	    mr->hw_mr.remote_atomic)
3033		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3034
3035	DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3036		 mr->ibmr.lkey);
3037	return &mr->ibmr;
3038
3039err2:
3040	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3041err1:
3042	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3043err0:
3044	kfree(mr);
3045	return ERR_PTR(rc);
3046}
3047
3048int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3049{
3050	struct qedr_mr *mr = get_qedr_mr(ib_mr);
3051	struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3052	int rc = 0;
3053
3054	rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3055	if (rc)
3056		return rc;
3057
3058	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3059
3060	if (mr->type != QEDR_MR_DMA)
3061		free_mr_info(dev, &mr->info);
3062
3063	/* it could be user registered memory. */
3064	ib_umem_release(mr->umem);
3065
3066	kfree(mr);
3067
3068	return rc;
3069}
3070
3071static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3072				       int max_page_list_len)
3073{
3074	struct qedr_pd *pd = get_qedr_pd(ibpd);
3075	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3076	struct qedr_mr *mr;
3077	int rc = -ENOMEM;
3078
3079	DP_DEBUG(dev, QEDR_MSG_MR,
3080		 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3081		 max_page_list_len);
3082
3083	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3084	if (!mr)
3085		return ERR_PTR(rc);
3086
3087	mr->dev = dev;
3088	mr->type = QEDR_MR_FRMR;
3089
3090	rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3091	if (rc)
3092		goto err0;
3093
3094	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3095	if (rc) {
3096		if (rc == -EINVAL)
3097			DP_ERR(dev, "Out of MR resources\n");
3098		else
3099			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3100
3101		goto err1;
3102	}
3103
3104	/* Index only, 18 bit long, lkey = itid << 8 | key */
3105	mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3106	mr->hw_mr.key = 0;
3107	mr->hw_mr.pd = pd->pd_id;
3108	mr->hw_mr.local_read = 1;
3109	mr->hw_mr.local_write = 0;
3110	mr->hw_mr.remote_read = 0;
3111	mr->hw_mr.remote_write = 0;
3112	mr->hw_mr.remote_atomic = 0;
3113	mr->hw_mr.mw_bind = false;
3114	mr->hw_mr.pbl_ptr = 0;
3115	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3116	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3117	mr->hw_mr.length = 0;
3118	mr->hw_mr.vaddr = 0;
3119	mr->hw_mr.phy_mr = true;
3120	mr->hw_mr.dma_mr = false;
3121
3122	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3123	if (rc) {
3124		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3125		goto err2;
3126	}
3127
3128	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3129	mr->ibmr.rkey = mr->ibmr.lkey;
3130
3131	DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3132	return mr;
3133
3134err2:
3135	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3136err1:
3137	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3138err0:
3139	kfree(mr);
3140	return ERR_PTR(rc);
3141}
3142
3143struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3144			    u32 max_num_sg)
3145{
3146	struct qedr_mr *mr;
3147
3148	if (mr_type != IB_MR_TYPE_MEM_REG)
3149		return ERR_PTR(-EINVAL);
3150
3151	mr = __qedr_alloc_mr(ibpd, max_num_sg);
3152
3153	if (IS_ERR(mr))
3154		return ERR_PTR(-EINVAL);
3155
3156	return &mr->ibmr;
3157}
3158
3159static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3160{
3161	struct qedr_mr *mr = get_qedr_mr(ibmr);
3162	struct qedr_pbl *pbl_table;
3163	struct regpair *pbe;
3164	u32 pbes_in_page;
3165
3166	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3167		DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3168		return -ENOMEM;
3169	}
3170
3171	DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3172		 mr->npages, addr);
3173
3174	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3175	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3176	pbe = (struct regpair *)pbl_table->va;
3177	pbe +=  mr->npages % pbes_in_page;
3178	pbe->lo = cpu_to_le32((u32)addr);
3179	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3180
3181	mr->npages++;
3182
3183	return 0;
3184}
3185
3186static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3187{
3188	int work = info->completed - info->completed_handled - 1;
3189
3190	DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3191	while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3192		struct qedr_pbl *pbl;
3193
3194		/* Free all the page list that are possible to be freed
3195		 * (all the ones that were invalidated), under the assumption
3196		 * that if an FMR was completed successfully that means that
3197		 * if there was an invalidate operation before it also ended
3198		 */
3199		pbl = list_first_entry(&info->inuse_pbl_list,
3200				       struct qedr_pbl, list_entry);
3201		list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3202		info->completed_handled++;
3203	}
3204}
3205
3206int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3207		   int sg_nents, unsigned int *sg_offset)
3208{
3209	struct qedr_mr *mr = get_qedr_mr(ibmr);
3210
3211	mr->npages = 0;
3212
3213	handle_completed_mrs(mr->dev, &mr->info);
3214	return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3215}
3216
3217struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3218{
3219	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3220	struct qedr_pd *pd = get_qedr_pd(ibpd);
3221	struct qedr_mr *mr;
3222	int rc;
3223
3224	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3225	if (!mr)
3226		return ERR_PTR(-ENOMEM);
3227
3228	mr->type = QEDR_MR_DMA;
3229
3230	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3231	if (rc) {
3232		if (rc == -EINVAL)
3233			DP_ERR(dev, "Out of MR resources\n");
3234		else
3235			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3236
3237		goto err1;
3238	}
3239
3240	/* index only, 18 bit long, lkey = itid << 8 | key */
3241	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3242	mr->hw_mr.pd = pd->pd_id;
3243	mr->hw_mr.local_read = 1;
3244	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3245	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3246	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3247	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3248	mr->hw_mr.dma_mr = true;
3249
3250	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3251	if (rc) {
3252		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3253		goto err2;
3254	}
3255
3256	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3257	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3258	    mr->hw_mr.remote_atomic)
3259		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3260
3261	DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3262	return &mr->ibmr;
3263
3264err2:
3265	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3266err1:
3267	kfree(mr);
3268	return ERR_PTR(rc);
3269}
3270
3271static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3272{
3273	return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3274}
3275
3276static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3277{
3278	int i, len = 0;
3279
3280	for (i = 0; i < num_sge; i++)
3281		len += sg_list[i].length;
3282
3283	return len;
3284}
3285
3286static void swap_wqe_data64(u64 *p)
3287{
3288	int i;
3289
3290	for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3291		*p = cpu_to_be64(cpu_to_le64(*p));
3292}
3293
3294static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3295				       struct qedr_qp *qp, u8 *wqe_size,
3296				       const struct ib_send_wr *wr,
3297				       const struct ib_send_wr **bad_wr,
3298				       u8 *bits, u8 bit)
3299{
3300	u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3301	char *seg_prt, *wqe;
3302	int i, seg_siz;
3303
3304	if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3305		DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3306		*bad_wr = wr;
3307		return 0;
3308	}
3309
3310	if (!data_size)
3311		return data_size;
3312
3313	*bits |= bit;
3314
3315	seg_prt = NULL;
3316	wqe = NULL;
3317	seg_siz = 0;
3318
3319	/* Copy data inline */
3320	for (i = 0; i < wr->num_sge; i++) {
3321		u32 len = wr->sg_list[i].length;
3322		void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3323
3324		while (len > 0) {
3325			u32 cur;
3326
3327			/* New segment required */
3328			if (!seg_siz) {
3329				wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3330				seg_prt = wqe;
3331				seg_siz = sizeof(struct rdma_sq_common_wqe);
3332				(*wqe_size)++;
3333			}
3334
3335			/* Calculate currently allowed length */
3336			cur = min_t(u32, len, seg_siz);
3337			memcpy(seg_prt, src, cur);
3338
3339			/* Update segment variables */
3340			seg_prt += cur;
3341			seg_siz -= cur;
3342
3343			/* Update sge variables */
3344			src += cur;
3345			len -= cur;
3346
3347			/* Swap fully-completed segments */
3348			if (!seg_siz)
3349				swap_wqe_data64((u64 *)wqe);
3350		}
3351	}
3352
3353	/* swap last not completed segment */
3354	if (seg_siz)
3355		swap_wqe_data64((u64 *)wqe);
3356
3357	return data_size;
3358}
3359
3360#define RQ_SGE_SET(sge, vaddr, vlength, vflags)			\
3361	do {							\
3362		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3363		(sge)->length = cpu_to_le32(vlength);		\
3364		(sge)->flags = cpu_to_le32(vflags);		\
3365	} while (0)
3366
3367#define SRQ_HDR_SET(hdr, vwr_id, num_sge)			\
3368	do {							\
3369		DMA_REGPAIR_LE(hdr->wr_id, vwr_id);		\
3370		(hdr)->num_sges = num_sge;			\
3371	} while (0)
3372
3373#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)			\
3374	do {							\
3375		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3376		(sge)->length = cpu_to_le32(vlength);		\
3377		(sge)->l_key = cpu_to_le32(vlkey);		\
3378	} while (0)
3379
3380static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3381				const struct ib_send_wr *wr)
3382{
3383	u32 data_size = 0;
3384	int i;
3385
3386	for (i = 0; i < wr->num_sge; i++) {
3387		struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3388
3389		DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3390		sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3391		sge->length = cpu_to_le32(wr->sg_list[i].length);
3392		data_size += wr->sg_list[i].length;
3393	}
3394
3395	if (wqe_size)
3396		*wqe_size += wr->num_sge;
3397
3398	return data_size;
3399}
3400
3401static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3402				     struct qedr_qp *qp,
3403				     struct rdma_sq_rdma_wqe_1st *rwqe,
3404				     struct rdma_sq_rdma_wqe_2nd *rwqe2,
3405				     const struct ib_send_wr *wr,
3406				     const struct ib_send_wr **bad_wr)
3407{
3408	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3409	DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3410
3411	if (wr->send_flags & IB_SEND_INLINE &&
3412	    (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3413	     wr->opcode == IB_WR_RDMA_WRITE)) {
3414		u8 flags = 0;
3415
3416		SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3417		return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3418						   bad_wr, &rwqe->flags, flags);
3419	}
3420
3421	return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3422}
3423
3424static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3425				     struct qedr_qp *qp,
3426				     struct rdma_sq_send_wqe_1st *swqe,
3427				     struct rdma_sq_send_wqe_2st *swqe2,
3428				     const struct ib_send_wr *wr,
3429				     const struct ib_send_wr **bad_wr)
3430{
3431	memset(swqe2, 0, sizeof(*swqe2));
3432	if (wr->send_flags & IB_SEND_INLINE) {
3433		u8 flags = 0;
3434
3435		SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3436		return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3437						   bad_wr, &swqe->flags, flags);
3438	}
3439
3440	return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3441}
3442
3443static int qedr_prepare_reg(struct qedr_qp *qp,
3444			    struct rdma_sq_fmr_wqe_1st *fwqe1,
3445			    const struct ib_reg_wr *wr)
3446{
3447	struct qedr_mr *mr = get_qedr_mr(wr->mr);
3448	struct rdma_sq_fmr_wqe_2nd *fwqe2;
3449
3450	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3451	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3452	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3453	fwqe1->l_key = wr->key;
3454
3455	fwqe2->access_ctrl = 0;
3456
3457	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3458		   !!(wr->access & IB_ACCESS_REMOTE_READ));
3459	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3460		   !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3461	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3462		   !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3463	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3464	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3465		   !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3466	fwqe2->fmr_ctrl = 0;
3467
3468	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3469		   ilog2(mr->ibmr.page_size) - 12);
3470
3471	fwqe2->length_hi = 0;
3472	fwqe2->length_lo = mr->ibmr.length;
3473	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3474	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3475
3476	qp->wqe_wr_id[qp->sq.prod].mr = mr;
3477
3478	return 0;
3479}
3480
3481static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3482{
3483	switch (opcode) {
3484	case IB_WR_RDMA_WRITE:
3485	case IB_WR_RDMA_WRITE_WITH_IMM:
3486		return IB_WC_RDMA_WRITE;
3487	case IB_WR_SEND_WITH_IMM:
3488	case IB_WR_SEND:
3489	case IB_WR_SEND_WITH_INV:
3490		return IB_WC_SEND;
3491	case IB_WR_RDMA_READ:
3492	case IB_WR_RDMA_READ_WITH_INV:
3493		return IB_WC_RDMA_READ;
3494	case IB_WR_ATOMIC_CMP_AND_SWP:
3495		return IB_WC_COMP_SWAP;
3496	case IB_WR_ATOMIC_FETCH_AND_ADD:
3497		return IB_WC_FETCH_ADD;
3498	case IB_WR_REG_MR:
3499		return IB_WC_REG_MR;
3500	case IB_WR_LOCAL_INV:
3501		return IB_WC_LOCAL_INV;
3502	default:
3503		return IB_WC_SEND;
3504	}
3505}
3506
3507static inline bool qedr_can_post_send(struct qedr_qp *qp,
3508				      const struct ib_send_wr *wr)
3509{
3510	int wq_is_full, err_wr, pbl_is_full;
3511	struct qedr_dev *dev = qp->dev;
3512
3513	/* prevent SQ overflow and/or processing of a bad WR */
3514	err_wr = wr->num_sge > qp->sq.max_sges;
3515	wq_is_full = qedr_wq_is_full(&qp->sq);
3516	pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3517		      QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3518	if (wq_is_full || err_wr || pbl_is_full) {
3519		if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3520			DP_ERR(dev,
3521			       "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3522			       qp);
3523			qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3524		}
3525
3526		if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3527			DP_ERR(dev,
3528			       "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3529			       qp);
3530			qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3531		}
3532
3533		if (pbl_is_full &&
3534		    !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3535			DP_ERR(dev,
3536			       "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3537			       qp);
3538			qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3539		}
3540		return false;
3541	}
3542	return true;
3543}
3544
3545static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3546			    const struct ib_send_wr **bad_wr)
3547{
3548	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3549	struct qedr_qp *qp = get_qedr_qp(ibqp);
3550	struct rdma_sq_atomic_wqe_1st *awqe1;
3551	struct rdma_sq_atomic_wqe_2nd *awqe2;
3552	struct rdma_sq_atomic_wqe_3rd *awqe3;
3553	struct rdma_sq_send_wqe_2st *swqe2;
3554	struct rdma_sq_local_inv_wqe *iwqe;
3555	struct rdma_sq_rdma_wqe_2nd *rwqe2;
3556	struct rdma_sq_send_wqe_1st *swqe;
3557	struct rdma_sq_rdma_wqe_1st *rwqe;
3558	struct rdma_sq_fmr_wqe_1st *fwqe1;
3559	struct rdma_sq_common_wqe *wqe;
3560	u32 length;
3561	int rc = 0;
3562	bool comp;
3563
3564	if (!qedr_can_post_send(qp, wr)) {
3565		*bad_wr = wr;
3566		return -ENOMEM;
3567	}
3568
3569	wqe = qed_chain_produce(&qp->sq.pbl);
3570	qp->wqe_wr_id[qp->sq.prod].signaled =
3571		!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3572
3573	wqe->flags = 0;
3574	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3575		   !!(wr->send_flags & IB_SEND_SOLICITED));
3576	comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3577	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3578	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3579		   !!(wr->send_flags & IB_SEND_FENCE));
3580	wqe->prev_wqe_size = qp->prev_wqe_size;
3581
3582	qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3583
3584	switch (wr->opcode) {
3585	case IB_WR_SEND_WITH_IMM:
3586		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3587			rc = -EINVAL;
3588			*bad_wr = wr;
3589			break;
3590		}
3591		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3592		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3593		swqe->wqe_size = 2;
3594		swqe2 = qed_chain_produce(&qp->sq.pbl);
3595
3596		swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3597		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3598						   wr, bad_wr);
3599		swqe->length = cpu_to_le32(length);
3600		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3601		qp->prev_wqe_size = swqe->wqe_size;
3602		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3603		break;
3604	case IB_WR_SEND:
3605		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3606		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3607
3608		swqe->wqe_size = 2;
3609		swqe2 = qed_chain_produce(&qp->sq.pbl);
3610		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3611						   wr, bad_wr);
3612		swqe->length = cpu_to_le32(length);
3613		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3614		qp->prev_wqe_size = swqe->wqe_size;
3615		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3616		break;
3617	case IB_WR_SEND_WITH_INV:
3618		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3619		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3620		swqe2 = qed_chain_produce(&qp->sq.pbl);
3621		swqe->wqe_size = 2;
3622		swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3623		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3624						   wr, bad_wr);
3625		swqe->length = cpu_to_le32(length);
3626		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3627		qp->prev_wqe_size = swqe->wqe_size;
3628		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3629		break;
3630
3631	case IB_WR_RDMA_WRITE_WITH_IMM:
3632		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3633			rc = -EINVAL;
3634			*bad_wr = wr;
3635			break;
3636		}
3637		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3638		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3639
3640		rwqe->wqe_size = 2;
3641		rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3642		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3643		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3644						   wr, bad_wr);
3645		rwqe->length = cpu_to_le32(length);
3646		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3647		qp->prev_wqe_size = rwqe->wqe_size;
3648		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3649		break;
3650	case IB_WR_RDMA_WRITE:
3651		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3652		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3653
3654		rwqe->wqe_size = 2;
3655		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3656		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3657						   wr, bad_wr);
3658		rwqe->length = cpu_to_le32(length);
3659		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3660		qp->prev_wqe_size = rwqe->wqe_size;
3661		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3662		break;
3663	case IB_WR_RDMA_READ_WITH_INV:
3664		SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3665		fallthrough;	/* same is identical to RDMA READ */
3666
3667	case IB_WR_RDMA_READ:
3668		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3669		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3670
3671		rwqe->wqe_size = 2;
3672		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3673		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3674						   wr, bad_wr);
3675		rwqe->length = cpu_to_le32(length);
3676		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3677		qp->prev_wqe_size = rwqe->wqe_size;
3678		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3679		break;
3680
3681	case IB_WR_ATOMIC_CMP_AND_SWP:
3682	case IB_WR_ATOMIC_FETCH_AND_ADD:
3683		awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3684		awqe1->wqe_size = 4;
3685
3686		awqe2 = qed_chain_produce(&qp->sq.pbl);
3687		DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3688		awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3689
3690		awqe3 = qed_chain_produce(&qp->sq.pbl);
3691
3692		if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3693			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3694			DMA_REGPAIR_LE(awqe3->swap_data,
3695				       atomic_wr(wr)->compare_add);
3696		} else {
3697			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3698			DMA_REGPAIR_LE(awqe3->swap_data,
3699				       atomic_wr(wr)->swap);
3700			DMA_REGPAIR_LE(awqe3->cmp_data,
3701				       atomic_wr(wr)->compare_add);
3702		}
3703
3704		qedr_prepare_sq_sges(qp, NULL, wr);
3705
3706		qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3707		qp->prev_wqe_size = awqe1->wqe_size;
3708		break;
3709
3710	case IB_WR_LOCAL_INV:
3711		iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3712		iwqe->wqe_size = 1;
3713
3714		iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3715		iwqe->inv_l_key = wr->ex.invalidate_rkey;
3716		qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3717		qp->prev_wqe_size = iwqe->wqe_size;
3718		break;
3719	case IB_WR_REG_MR:
3720		DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3721		wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3722		fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3723		fwqe1->wqe_size = 2;
3724
3725		rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3726		if (rc) {
3727			DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3728			*bad_wr = wr;
3729			break;
3730		}
3731
3732		qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3733		qp->prev_wqe_size = fwqe1->wqe_size;
3734		break;
3735	default:
3736		DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3737		rc = -EINVAL;
3738		*bad_wr = wr;
3739		break;
3740	}
3741
3742	if (*bad_wr) {
3743		u16 value;
3744
3745		/* Restore prod to its position before
3746		 * this WR was processed
3747		 */
3748		value = le16_to_cpu(qp->sq.db_data.data.value);
3749		qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3750
3751		/* Restore prev_wqe_size */
3752		qp->prev_wqe_size = wqe->prev_wqe_size;
3753		rc = -EINVAL;
3754		DP_ERR(dev, "POST SEND FAILED\n");
3755	}
3756
3757	return rc;
3758}
3759
3760int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3761		   const struct ib_send_wr **bad_wr)
3762{
3763	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3764	struct qedr_qp *qp = get_qedr_qp(ibqp);
3765	unsigned long flags;
3766	int rc = 0;
3767
3768	*bad_wr = NULL;
3769
3770	if (qp->qp_type == IB_QPT_GSI)
3771		return qedr_gsi_post_send(ibqp, wr, bad_wr);
3772
3773	spin_lock_irqsave(&qp->q_lock, flags);
3774
3775	if (rdma_protocol_roce(&dev->ibdev, 1)) {
3776		if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3777		    (qp->state != QED_ROCE_QP_STATE_ERR) &&
3778		    (qp->state != QED_ROCE_QP_STATE_SQD)) {
3779			spin_unlock_irqrestore(&qp->q_lock, flags);
3780			*bad_wr = wr;
3781			DP_DEBUG(dev, QEDR_MSG_CQ,
3782				 "QP in wrong state! QP icid=0x%x state %d\n",
3783				 qp->icid, qp->state);
3784			return -EINVAL;
3785		}
3786	}
3787
3788	while (wr) {
3789		rc = __qedr_post_send(ibqp, wr, bad_wr);
3790		if (rc)
3791			break;
3792
3793		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3794
3795		qedr_inc_sw_prod(&qp->sq);
3796
3797		qp->sq.db_data.data.value++;
3798
3799		wr = wr->next;
3800	}
3801
3802	/* Trigger doorbell
3803	 * If there was a failure in the first WR then it will be triggered in
3804	 * vane. However this is not harmful (as long as the producer value is
3805	 * unchanged). For performance reasons we avoid checking for this
3806	 * redundant doorbell.
3807	 *
3808	 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3809	 * soon as we give the doorbell, we could get a completion
3810	 * for this wr, therefore we need to make sure that the
3811	 * memory is updated before giving the doorbell.
3812	 * During qedr_poll_cq, rmb is called before accessing the
3813	 * cqe. This covers for the smp_rmb as well.
3814	 */
3815	smp_wmb();
3816	writel(qp->sq.db_data.raw, qp->sq.db);
3817
3818	spin_unlock_irqrestore(&qp->q_lock, flags);
3819
3820	return rc;
3821}
3822
3823static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3824{
3825	u32 used;
3826
3827	/* Calculate number of elements used based on producer
3828	 * count and consumer count and subtract it from max
3829	 * work request supported so that we get elements left.
3830	 */
3831	used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3832
3833	return hw_srq->max_wr - used;
3834}
3835
3836int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3837		       const struct ib_recv_wr **bad_wr)
3838{
3839	struct qedr_srq *srq = get_qedr_srq(ibsrq);
3840	struct qedr_srq_hwq_info *hw_srq;
3841	struct qedr_dev *dev = srq->dev;
3842	struct qed_chain *pbl;
3843	unsigned long flags;
3844	int status = 0;
3845	u32 num_sge;
3846
3847	spin_lock_irqsave(&srq->lock, flags);
3848
3849	hw_srq = &srq->hw_srq;
3850	pbl = &srq->hw_srq.pbl;
3851	while (wr) {
3852		struct rdma_srq_wqe_header *hdr;
3853		int i;
3854
3855		if (!qedr_srq_elem_left(hw_srq) ||
3856		    wr->num_sge > srq->hw_srq.max_sges) {
3857			DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3858			       hw_srq->wr_prod_cnt,
3859			       atomic_read(&hw_srq->wr_cons_cnt),
3860			       wr->num_sge, srq->hw_srq.max_sges);
3861			status = -ENOMEM;
3862			*bad_wr = wr;
3863			break;
3864		}
3865
3866		hdr = qed_chain_produce(pbl);
3867		num_sge = wr->num_sge;
3868		/* Set number of sge and work request id in header */
3869		SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3870
3871		srq->hw_srq.wr_prod_cnt++;
3872		hw_srq->wqe_prod++;
3873		hw_srq->sge_prod++;
3874
3875		DP_DEBUG(dev, QEDR_MSG_SRQ,
3876			 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3877			 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3878
3879		for (i = 0; i < wr->num_sge; i++) {
3880			struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3881
3882			/* Set SGE length, lkey and address */
3883			SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3884				    wr->sg_list[i].length, wr->sg_list[i].lkey);
3885
3886			DP_DEBUG(dev, QEDR_MSG_SRQ,
3887				 "[%d]: len %d key %x addr %x:%x\n",
3888				 i, srq_sge->length, srq_sge->l_key,
3889				 srq_sge->addr.hi, srq_sge->addr.lo);
3890			hw_srq->sge_prod++;
3891		}
3892
3893		/* Update WQE and SGE information before
3894		 * updating producer.
3895		 */
3896		dma_wmb();
3897
3898		/* SRQ producer is 8 bytes. Need to update SGE producer index
3899		 * in first 4 bytes and need to update WQE producer in
3900		 * next 4 bytes.
3901		 */
3902		srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3903		/* Make sure sge producer is updated first */
3904		dma_wmb();
3905		srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3906
3907		wr = wr->next;
3908	}
3909
3910	DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3911		 qed_chain_get_elem_left(pbl));
3912	spin_unlock_irqrestore(&srq->lock, flags);
3913
3914	return status;
3915}
3916
3917int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3918		   const struct ib_recv_wr **bad_wr)
3919{
3920	struct qedr_qp *qp = get_qedr_qp(ibqp);
3921	struct qedr_dev *dev = qp->dev;
3922	unsigned long flags;
3923	int status = 0;
3924
3925	if (qp->qp_type == IB_QPT_GSI)
3926		return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3927
3928	spin_lock_irqsave(&qp->q_lock, flags);
3929
3930	while (wr) {
3931		int i;
3932
3933		if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3934		    QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3935		    wr->num_sge > qp->rq.max_sges) {
3936			DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3937			       qed_chain_get_elem_left_u32(&qp->rq.pbl),
3938			       QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3939			       qp->rq.max_sges);
3940			status = -ENOMEM;
3941			*bad_wr = wr;
3942			break;
3943		}
3944		for (i = 0; i < wr->num_sge; i++) {
3945			u32 flags = 0;
3946			struct rdma_rq_sge *rqe =
3947			    qed_chain_produce(&qp->rq.pbl);
3948
3949			/* First one must include the number
3950			 * of SGE in the list
3951			 */
3952			if (!i)
3953				SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3954					  wr->num_sge);
3955
3956			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3957				  wr->sg_list[i].lkey);
3958
3959			RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3960				   wr->sg_list[i].length, flags);
3961		}
3962
3963		/* Special case of no sges. FW requires between 1-4 sges...
3964		 * in this case we need to post 1 sge with length zero. this is
3965		 * because rdma write with immediate consumes an RQ.
3966		 */
3967		if (!wr->num_sge) {
3968			u32 flags = 0;
3969			struct rdma_rq_sge *rqe =
3970			    qed_chain_produce(&qp->rq.pbl);
3971
3972			/* First one must include the number
3973			 * of SGE in the list
3974			 */
3975			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3976			SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3977
3978			RQ_SGE_SET(rqe, 0, 0, flags);
3979			i = 1;
3980		}
3981
3982		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3983		qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3984
3985		qedr_inc_sw_prod(&qp->rq);
3986
3987		/* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3988		 * soon as we give the doorbell, we could get a completion
3989		 * for this wr, therefore we need to make sure that the
3990		 * memory is update before giving the doorbell.
3991		 * During qedr_poll_cq, rmb is called before accessing the
3992		 * cqe. This covers for the smp_rmb as well.
3993		 */
3994		smp_wmb();
3995
3996		qp->rq.db_data.data.value++;
3997
3998		writel(qp->rq.db_data.raw, qp->rq.db);
3999
4000		if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
4001			writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
4002		}
4003
4004		wr = wr->next;
4005	}
4006
4007	spin_unlock_irqrestore(&qp->q_lock, flags);
4008
4009	return status;
4010}
4011
4012static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
4013{
4014	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4015
4016	return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4017		cq->pbl_toggle;
4018}
4019
4020static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4021{
4022	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4023	struct qedr_qp *qp;
4024
4025	qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4026						   resp_cqe->qp_handle.lo,
4027						   u64);
4028	return qp;
4029}
4030
4031static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4032{
4033	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4034
4035	return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4036}
4037
4038/* Return latest CQE (needs processing) */
4039static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4040{
4041	return cq->latest_cqe;
4042}
4043
4044/* In fmr we need to increase the number of fmr completed counter for the fmr
4045 * algorithm determining whether we can free a pbl or not.
4046 * we need to perform this whether the work request was signaled or not. for
4047 * this purpose we call this function from the condition that checks if a wr
4048 * should be skipped, to make sure we don't miss it ( possibly this fmr
4049 * operation was not signalted)
4050 */
4051static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4052{
4053	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4054		qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4055}
4056
4057static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4058		       struct qedr_cq *cq, int num_entries,
4059		       struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4060		       int force)
4061{
4062	u16 cnt = 0;
4063
4064	while (num_entries && qp->sq.wqe_cons != hw_cons) {
4065		if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4066			qedr_chk_if_fmr(qp);
4067			/* skip WC */
4068			goto next_cqe;
4069		}
4070
4071		/* fill WC */
4072		wc->status = status;
4073		wc->vendor_err = 0;
4074		wc->wc_flags = 0;
4075		wc->src_qp = qp->id;
4076		wc->qp = &qp->ibqp;
4077
4078		wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4079		wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4080
4081		switch (wc->opcode) {
4082		case IB_WC_RDMA_WRITE:
4083			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4084			break;
4085		case IB_WC_COMP_SWAP:
4086		case IB_WC_FETCH_ADD:
4087			wc->byte_len = 8;
4088			break;
4089		case IB_WC_REG_MR:
4090			qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4091			break;
4092		case IB_WC_RDMA_READ:
4093		case IB_WC_SEND:
4094			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4095			break;
4096		default:
4097			break;
4098		}
4099
4100		num_entries--;
4101		wc++;
4102		cnt++;
4103next_cqe:
4104		while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4105			qed_chain_consume(&qp->sq.pbl);
4106		qedr_inc_sw_cons(&qp->sq);
4107	}
4108
4109	return cnt;
4110}
4111
4112static int qedr_poll_cq_req(struct qedr_dev *dev,
4113			    struct qedr_qp *qp, struct qedr_cq *cq,
4114			    int num_entries, struct ib_wc *wc,
4115			    struct rdma_cqe_requester *req)
4116{
4117	int cnt = 0;
4118
4119	switch (req->status) {
4120	case RDMA_CQE_REQ_STS_OK:
4121		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4122				  IB_WC_SUCCESS, 0);
4123		break;
4124	case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4125		if (qp->state != QED_ROCE_QP_STATE_ERR)
4126			DP_DEBUG(dev, QEDR_MSG_CQ,
4127				 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4128				 cq->icid, qp->icid);
4129		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4130				  IB_WC_WR_FLUSH_ERR, 1);
4131		break;
4132	default:
4133		/* process all WQE before the cosumer */
4134		qp->state = QED_ROCE_QP_STATE_ERR;
4135		cnt = process_req(dev, qp, cq, num_entries, wc,
4136				  req->sq_cons - 1, IB_WC_SUCCESS, 0);
4137		wc += cnt;
4138		/* if we have extra WC fill it with actual error info */
4139		if (cnt < num_entries) {
4140			enum ib_wc_status wc_status;
4141
4142			switch (req->status) {
4143			case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4144				DP_ERR(dev,
4145				       "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4146				       cq->icid, qp->icid);
4147				wc_status = IB_WC_BAD_RESP_ERR;
4148				break;
4149			case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4150				DP_ERR(dev,
4151				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4152				       cq->icid, qp->icid);
4153				wc_status = IB_WC_LOC_LEN_ERR;
4154				break;
4155			case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4156				DP_ERR(dev,
4157				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4158				       cq->icid, qp->icid);
4159				wc_status = IB_WC_LOC_QP_OP_ERR;
4160				break;
4161			case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4162				DP_ERR(dev,
4163				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4164				       cq->icid, qp->icid);
4165				wc_status = IB_WC_LOC_PROT_ERR;
4166				break;
4167			case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4168				DP_ERR(dev,
4169				       "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4170				       cq->icid, qp->icid);
4171				wc_status = IB_WC_MW_BIND_ERR;
4172				break;
4173			case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4174				DP_ERR(dev,
4175				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4176				       cq->icid, qp->icid);
4177				wc_status = IB_WC_REM_INV_REQ_ERR;
4178				break;
4179			case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4180				DP_ERR(dev,
4181				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4182				       cq->icid, qp->icid);
4183				wc_status = IB_WC_REM_ACCESS_ERR;
4184				break;
4185			case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4186				DP_ERR(dev,
4187				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4188				       cq->icid, qp->icid);
4189				wc_status = IB_WC_REM_OP_ERR;
4190				break;
4191			case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4192				DP_ERR(dev,
4193				       "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4194				       cq->icid, qp->icid);
4195				wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4196				break;
4197			case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4198				DP_ERR(dev,
4199				       "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4200				       cq->icid, qp->icid);
4201				wc_status = IB_WC_RETRY_EXC_ERR;
4202				break;
4203			default:
4204				DP_ERR(dev,
4205				       "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4206				       cq->icid, qp->icid);
4207				wc_status = IB_WC_GENERAL_ERR;
4208			}
4209			cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4210					   wc_status, 1);
4211		}
4212	}
4213
4214	return cnt;
4215}
4216
4217static inline int qedr_cqe_resp_status_to_ib(u8 status)
4218{
4219	switch (status) {
4220	case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4221		return IB_WC_LOC_ACCESS_ERR;
4222	case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4223		return IB_WC_LOC_LEN_ERR;
4224	case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4225		return IB_WC_LOC_QP_OP_ERR;
4226	case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4227		return IB_WC_LOC_PROT_ERR;
4228	case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4229		return IB_WC_MW_BIND_ERR;
4230	case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4231		return IB_WC_REM_INV_RD_REQ_ERR;
4232	case RDMA_CQE_RESP_STS_OK:
4233		return IB_WC_SUCCESS;
4234	default:
4235		return IB_WC_GENERAL_ERR;
4236	}
4237}
4238
4239static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4240					  struct ib_wc *wc)
4241{
4242	wc->status = IB_WC_SUCCESS;
4243	wc->byte_len = le32_to_cpu(resp->length);
4244
4245	if (resp->flags & QEDR_RESP_IMM) {
4246		wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4247		wc->wc_flags |= IB_WC_WITH_IMM;
4248
4249		if (resp->flags & QEDR_RESP_RDMA)
4250			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4251
4252		if (resp->flags & QEDR_RESP_INV)
4253			return -EINVAL;
4254
4255	} else if (resp->flags & QEDR_RESP_INV) {
4256		wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4257		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4258
4259		if (resp->flags & QEDR_RESP_RDMA)
4260			return -EINVAL;
4261
4262	} else if (resp->flags & QEDR_RESP_RDMA) {
4263		return -EINVAL;
4264	}
4265
4266	return 0;
4267}
4268
4269static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4270			       struct qedr_cq *cq, struct ib_wc *wc,
4271			       struct rdma_cqe_responder *resp, u64 wr_id)
4272{
4273	/* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4274	wc->opcode = IB_WC_RECV;
4275	wc->wc_flags = 0;
4276
4277	if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4278		if (qedr_set_ok_cqe_resp_wc(resp, wc))
4279			DP_ERR(dev,
4280			       "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4281			       cq, cq->icid, resp->flags);
4282
4283	} else {
4284		wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4285		if (wc->status == IB_WC_GENERAL_ERR)
4286			DP_ERR(dev,
4287			       "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4288			       cq, cq->icid, resp->status);
4289	}
4290
4291	/* Fill the rest of the WC */
4292	wc->vendor_err = 0;
4293	wc->src_qp = qp->id;
4294	wc->qp = &qp->ibqp;
4295	wc->wr_id = wr_id;
4296}
4297
4298static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4299				struct qedr_cq *cq, struct ib_wc *wc,
4300				struct rdma_cqe_responder *resp)
4301{
4302	struct qedr_srq *srq = qp->srq;
4303	u64 wr_id;
4304
4305	wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4306			 le32_to_cpu(resp->srq_wr_id.lo), u64);
4307
4308	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4309		wc->status = IB_WC_WR_FLUSH_ERR;
4310		wc->vendor_err = 0;
4311		wc->wr_id = wr_id;
4312		wc->byte_len = 0;
4313		wc->src_qp = qp->id;
4314		wc->qp = &qp->ibqp;
4315		wc->wr_id = wr_id;
4316	} else {
4317		__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4318	}
4319	atomic_inc(&srq->hw_srq.wr_cons_cnt);
4320
4321	return 1;
4322}
4323static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4324			    struct qedr_cq *cq, struct ib_wc *wc,
4325			    struct rdma_cqe_responder *resp)
4326{
4327	u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4328
4329	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4330
4331	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4332		qed_chain_consume(&qp->rq.pbl);
4333	qedr_inc_sw_cons(&qp->rq);
4334
4335	return 1;
4336}
4337
4338static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4339			      int num_entries, struct ib_wc *wc, u16 hw_cons)
4340{
4341	u16 cnt = 0;
4342
4343	while (num_entries && qp->rq.wqe_cons != hw_cons) {
4344		/* fill WC */
4345		wc->status = IB_WC_WR_FLUSH_ERR;
4346		wc->vendor_err = 0;
4347		wc->wc_flags = 0;
4348		wc->src_qp = qp->id;
4349		wc->byte_len = 0;
4350		wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4351		wc->qp = &qp->ibqp;
4352		num_entries--;
4353		wc++;
4354		cnt++;
4355		while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4356			qed_chain_consume(&qp->rq.pbl);
4357		qedr_inc_sw_cons(&qp->rq);
4358	}
4359
4360	return cnt;
4361}
4362
4363static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4364				 struct rdma_cqe_responder *resp, int *update)
4365{
4366	if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4367		consume_cqe(cq);
4368		*update |= 1;
4369	}
4370}
4371
4372static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4373				 struct qedr_cq *cq, int num_entries,
4374				 struct ib_wc *wc,
4375				 struct rdma_cqe_responder *resp)
4376{
4377	int cnt;
4378
4379	cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4380	consume_cqe(cq);
4381
4382	return cnt;
4383}
4384
4385static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4386			     struct qedr_cq *cq, int num_entries,
4387			     struct ib_wc *wc, struct rdma_cqe_responder *resp,
4388			     int *update)
4389{
4390	int cnt;
4391
4392	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4393		cnt = process_resp_flush(qp, cq, num_entries, wc,
4394					 resp->rq_cons_or_srq_id);
4395		try_consume_resp_cqe(cq, qp, resp, update);
4396	} else {
4397		cnt = process_resp_one(dev, qp, cq, wc, resp);
4398		consume_cqe(cq);
4399		*update |= 1;
4400	}
4401
4402	return cnt;
4403}
4404
4405static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4406				struct rdma_cqe_requester *req, int *update)
4407{
4408	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4409		consume_cqe(cq);
4410		*update |= 1;
4411	}
4412}
4413
4414int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4415{
4416	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4417	struct qedr_cq *cq = get_qedr_cq(ibcq);
4418	union rdma_cqe *cqe;
4419	u32 old_cons, new_cons;
4420	unsigned long flags;
4421	int update = 0;
4422	int done = 0;
4423
4424	if (cq->destroyed) {
4425		DP_ERR(dev,
4426		       "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4427		       cq, cq->icid);
4428		return 0;
4429	}
4430
4431	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4432		return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4433
4434	spin_lock_irqsave(&cq->cq_lock, flags);
4435	cqe = cq->latest_cqe;
4436	old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4437	while (num_entries && is_valid_cqe(cq, cqe)) {
4438		struct qedr_qp *qp;
4439		int cnt = 0;
4440
4441		/* prevent speculative reads of any field of CQE */
4442		rmb();
4443
4444		qp = cqe_get_qp(cqe);
4445		if (!qp) {
4446			WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4447			break;
4448		}
4449
4450		wc->qp = &qp->ibqp;
4451
4452		switch (cqe_get_type(cqe)) {
4453		case RDMA_CQE_TYPE_REQUESTER:
4454			cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4455					       &cqe->req);
4456			try_consume_req_cqe(cq, qp, &cqe->req, &update);
4457			break;
4458		case RDMA_CQE_TYPE_RESPONDER_RQ:
4459			cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4460						&cqe->resp, &update);
4461			break;
4462		case RDMA_CQE_TYPE_RESPONDER_SRQ:
4463			cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4464						    wc, &cqe->resp);
4465			update = 1;
4466			break;
4467		case RDMA_CQE_TYPE_INVALID:
4468		default:
4469			DP_ERR(dev, "Error: invalid CQE type = %d\n",
4470			       cqe_get_type(cqe));
4471		}
4472		num_entries -= cnt;
4473		wc += cnt;
4474		done += cnt;
4475
4476		cqe = get_cqe(cq);
4477	}
4478	new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4479
4480	cq->cq_cons += new_cons - old_cons;
4481
4482	if (update)
4483		/* doorbell notifies abount latest VALID entry,
4484		 * but chain already point to the next INVALID one
4485		 */
4486		doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4487
4488	spin_unlock_irqrestore(&cq->cq_lock, flags);
4489	return done;
4490}
4491
4492int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4493		     u32 port_num, const struct ib_wc *in_wc,
4494		     const struct ib_grh *in_grh, const struct ib_mad *in,
4495		     struct ib_mad *out_mad, size_t *out_mad_size,
4496		     u16 *out_mad_pkey_index)
4497{
4498	return IB_MAD_RESULT_SUCCESS;
4499}