ocrdma_verbs.c

   1/* This file is part of the Emulex RoCE Device Driver for
   2 * RoCE (RDMA over Converged Ethernet) adapters.
   3 * Copyright (C) 2012-2015 Emulex. All rights reserved.
   4 * EMULEX and SLI are trademarks of Emulex.
   5 * www.emulex.com
   6 *
   7 * This software is available to you under a choice of one of two licenses.
   8 * You may choose to be licensed under the terms of the GNU General Public
   9 * License (GPL) Version 2, available from the file COPYING in the main
  10 * directory of this source tree, or the BSD license below:
  11 *
  12 * Redistribution and use in source and binary forms, with or without
  13 * modification, are permitted provided that the following conditions
  14 * are met:
  15 *
  16 * - Redistributions of source code must retain the above copyright notice,
  17 *   this list of conditions and the following disclaimer.
  18 *
  19 * - Redistributions in binary form must reproduce the above copyright
  20 *   notice, this list of conditions and the following disclaimer in
  21 *   the documentation and/or other materials provided with the distribution.
  22 *
  23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE
  25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34 *
  35 * Contact Information:
  36 * linux-drivers@emulex.com
  37 *
  38 * Emulex
  39 * 3333 Susan Street
  40 * Costa Mesa, CA 92626
  41 */
  42
  43#include <linux/dma-mapping.h>
  44#include <net/addrconf.h>
  45#include <rdma/ib_verbs.h>
  46#include <rdma/ib_user_verbs.h>
  47#include <rdma/iw_cm.h>
  48#include <rdma/ib_umem.h>
  49#include <rdma/ib_addr.h>
  50#include <rdma/ib_cache.h>
  51#include <rdma/uverbs_ioctl.h>
  52
  53#include "ocrdma.h"
  54#include "ocrdma_hw.h"
  55#include "ocrdma_verbs.h"
  56#include <rdma/ocrdma-abi.h>
  57
  58int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
  59{
  60	if (index > 0)
  61		return -EINVAL;
  62
  63	*pkey = 0xffff;
  64	return 0;
  65}
  66
  67int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
  68			struct ib_udata *uhw)
  69{
  70	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
  71
  72	if (uhw->inlen || uhw->outlen)
  73		return -EINVAL;
  74
  75	memset(attr, 0, sizeof *attr);
  76	memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
  77	       min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
  78	addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
  79			    dev->nic_info.mac_addr);
  80	attr->max_mr_size = dev->attr.max_mr_size;
  81	attr->page_size_cap = 0xffff000;
  82	attr->vendor_id = dev->nic_info.pdev->vendor;
  83	attr->vendor_part_id = dev->nic_info.pdev->device;
  84	attr->hw_ver = dev->asic_id;
  85	attr->max_qp = dev->attr.max_qp;
  86	attr->max_ah = OCRDMA_MAX_AH;
  87	attr->max_qp_wr = dev->attr.max_wqe;
  88
  89	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
  90					IB_DEVICE_RC_RNR_NAK_GEN |
  91					IB_DEVICE_SHUTDOWN_PORT |
  92					IB_DEVICE_SYS_IMAGE_GUID |
  93					IB_DEVICE_MEM_MGT_EXTENSIONS;
  94	attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
  95	attr->max_send_sge = dev->attr.max_send_sge;
  96	attr->max_recv_sge = dev->attr.max_recv_sge;
  97	attr->max_sge_rd = dev->attr.max_rdma_sge;
  98	attr->max_cq = dev->attr.max_cq;
  99	attr->max_cqe = dev->attr.max_cqe;
 100	attr->max_mr = dev->attr.max_mr;
 101	attr->max_mw = dev->attr.max_mw;
 102	attr->max_pd = dev->attr.max_pd;
 103	attr->atomic_cap = 0;
 104	attr->max_qp_rd_atom =
 105	    min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
 106	attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
 107	attr->max_srq = dev->attr.max_srq;
 108	attr->max_srq_sge = dev->attr.max_srq_sge;
 109	attr->max_srq_wr = dev->attr.max_rqe;
 110	attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;
 111	attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr;
 112	attr->max_pkeys = 1;
 113	return 0;
 114}
 115
 116static inline void get_link_speed_and_width(struct ocrdma_dev *dev,
 117					    u16 *ib_speed, u8 *ib_width)
 118{
 119	int status;
 120	u8 speed;
 121
 122	status = ocrdma_mbx_get_link_speed(dev, &speed, NULL);
 123	if (status)
 124		speed = OCRDMA_PHYS_LINK_SPEED_ZERO;
 125
 126	switch (speed) {
 127	case OCRDMA_PHYS_LINK_SPEED_1GBPS:
 128		*ib_speed = IB_SPEED_SDR;
 129		*ib_width = IB_WIDTH_1X;
 130		break;
 131
 132	case OCRDMA_PHYS_LINK_SPEED_10GBPS:
 133		*ib_speed = IB_SPEED_QDR;
 134		*ib_width = IB_WIDTH_1X;
 135		break;
 136
 137	case OCRDMA_PHYS_LINK_SPEED_20GBPS:
 138		*ib_speed = IB_SPEED_DDR;
 139		*ib_width = IB_WIDTH_4X;
 140		break;
 141
 142	case OCRDMA_PHYS_LINK_SPEED_40GBPS:
 143		*ib_speed = IB_SPEED_QDR;
 144		*ib_width = IB_WIDTH_4X;
 145		break;
 146
 147	default:
 148		/* Unsupported */
 149		*ib_speed = IB_SPEED_SDR;
 150		*ib_width = IB_WIDTH_1X;
 151	}
 152}
 153
 154int ocrdma_query_port(struct ib_device *ibdev,
 155		      u32 port, struct ib_port_attr *props)
 156{
 157	enum ib_port_state port_state;
 158	struct ocrdma_dev *dev;
 159	struct net_device *netdev;
 160
 161	/* props being zeroed by the caller, avoid zeroing it here */
 162	dev = get_ocrdma_dev(ibdev);
 163	netdev = dev->nic_info.netdev;
 164	if (netif_running(netdev) && netif_oper_up(netdev)) {
 165		port_state = IB_PORT_ACTIVE;
 166		props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 167	} else {
 168		port_state = IB_PORT_DOWN;
 169		props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 170	}
 171	props->max_mtu = IB_MTU_4096;
 172	props->active_mtu = iboe_get_mtu(netdev->mtu);
 173	props->lid = 0;
 174	props->lmc = 0;
 175	props->sm_lid = 0;
 176	props->sm_sl = 0;
 177	props->state = port_state;
 178	props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 179				IB_PORT_DEVICE_MGMT_SUP |
 180				IB_PORT_VENDOR_CLASS_SUP;
 181	props->ip_gids = true;
 182	props->gid_tbl_len = OCRDMA_MAX_SGID;
 183	props->pkey_tbl_len = 1;
 184	props->bad_pkey_cntr = 0;
 185	props->qkey_viol_cntr = 0;
 186	get_link_speed_and_width(dev, &props->active_speed,
 187				 &props->active_width);
 188	props->max_msg_sz = 0x80000000;
 189	props->max_vl_num = 4;
 190	return 0;
 191}
 192
 193static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 194			   unsigned long len)
 195{
 196	struct ocrdma_mm *mm;
 197
 198	mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 199	if (mm == NULL)
 200		return -ENOMEM;
 201	mm->key.phy_addr = phy_addr;
 202	mm->key.len = len;
 203	INIT_LIST_HEAD(&mm->entry);
 204
 205	mutex_lock(&uctx->mm_list_lock);
 206	list_add_tail(&mm->entry, &uctx->mm_head);
 207	mutex_unlock(&uctx->mm_list_lock);
 208	return 0;
 209}
 210
 211static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 212			    unsigned long len)
 213{
 214	struct ocrdma_mm *mm, *tmp;
 215
 216	mutex_lock(&uctx->mm_list_lock);
 217	list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 218		if (len != mm->key.len && phy_addr != mm->key.phy_addr)
 219			continue;
 220
 221		list_del(&mm->entry);
 222		kfree(mm);
 223		break;
 224	}
 225	mutex_unlock(&uctx->mm_list_lock);
 226}
 227
 228static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 229			      unsigned long len)
 230{
 231	bool found = false;
 232	struct ocrdma_mm *mm;
 233
 234	mutex_lock(&uctx->mm_list_lock);
 235	list_for_each_entry(mm, &uctx->mm_head, entry) {
 236		if (len != mm->key.len && phy_addr != mm->key.phy_addr)
 237			continue;
 238
 239		found = true;
 240		break;
 241	}
 242	mutex_unlock(&uctx->mm_list_lock);
 243	return found;
 244}
 245
 246
 247static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool)
 248{
 249	u16 pd_bitmap_idx = 0;
 250	unsigned long *pd_bitmap;
 251
 252	if (dpp_pool) {
 253		pd_bitmap = dev->pd_mgr->pd_dpp_bitmap;
 254		pd_bitmap_idx = find_first_zero_bit(pd_bitmap,
 255						    dev->pd_mgr->max_dpp_pd);
 256		__set_bit(pd_bitmap_idx, pd_bitmap);
 257		dev->pd_mgr->pd_dpp_count++;
 258		if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh)
 259			dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count;
 260	} else {
 261		pd_bitmap = dev->pd_mgr->pd_norm_bitmap;
 262		pd_bitmap_idx = find_first_zero_bit(pd_bitmap,
 263						    dev->pd_mgr->max_normal_pd);
 264		__set_bit(pd_bitmap_idx, pd_bitmap);
 265		dev->pd_mgr->pd_norm_count++;
 266		if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh)
 267			dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count;
 268	}
 269	return pd_bitmap_idx;
 270}
 271
 272static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id,
 273					bool dpp_pool)
 274{
 275	u16 pd_count;
 276	u16 pd_bit_index;
 277
 278	pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count :
 279			      dev->pd_mgr->pd_norm_count;
 280	if (pd_count == 0)
 281		return -EINVAL;
 282
 283	if (dpp_pool) {
 284		pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start;
 285		if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) {
 286			return -EINVAL;
 287		} else {
 288			__clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap);
 289			dev->pd_mgr->pd_dpp_count--;
 290		}
 291	} else {
 292		pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start;
 293		if (pd_bit_index >= dev->pd_mgr->max_normal_pd) {
 294			return -EINVAL;
 295		} else {
 296			__clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap);
 297			dev->pd_mgr->pd_norm_count--;
 298		}
 299	}
 300
 301	return 0;
 302}
 303
 304static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id,
 305				   bool dpp_pool)
 306{
 307	int status;
 308
 309	mutex_lock(&dev->dev_lock);
 310	status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool);
 311	mutex_unlock(&dev->dev_lock);
 312	return status;
 313}
 314
 315static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd)
 316{
 317	u16 pd_idx = 0;
 318	int status = 0;
 319
 320	mutex_lock(&dev->dev_lock);
 321	if (pd->dpp_enabled) {
 322		/* try allocating DPP PD, if not available then normal PD */
 323		if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) {
 324			pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true);
 325			pd->id = dev->pd_mgr->pd_dpp_start + pd_idx;
 326			pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx;
 327		} else if (dev->pd_mgr->pd_norm_count <
 328			   dev->pd_mgr->max_normal_pd) {
 329			pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false);
 330			pd->id = dev->pd_mgr->pd_norm_start + pd_idx;
 331			pd->dpp_enabled = false;
 332		} else {
 333			status = -EINVAL;
 334		}
 335	} else {
 336		if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) {
 337			pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false);
 338			pd->id = dev->pd_mgr->pd_norm_start + pd_idx;
 339		} else {
 340			status = -EINVAL;
 341		}
 342	}
 343	mutex_unlock(&dev->dev_lock);
 344	return status;
 345}
 346
 347/*
 348 * NOTE:
 349 *
 350 * ocrdma_ucontext must be used here because this function is also
 351 * called from ocrdma_alloc_ucontext where ib_udata does not have
 352 * valid ib_ucontext pointer. ib_uverbs_get_context does not call
 353 * uobj_{alloc|get_xxx} helpers which are used to store the
 354 * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so
 355 * ib_udata does NOT imply valid ib_ucontext here!
 356 */
 357static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 358			    struct ocrdma_ucontext *uctx,
 359			    struct ib_udata *udata)
 360{
 361	int status;
 362
 363	if (udata && uctx && dev->attr.max_dpp_pds) {
 364		pd->dpp_enabled =
 365			ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R;
 366		pd->num_dpp_qp =
 367			pd->dpp_enabled ? (dev->nic_info.db_page_size /
 368					   dev->attr.wqe_size) : 0;
 369	}
 370
 371	if (dev->pd_mgr->pd_prealloc_valid)
 372		return ocrdma_get_pd_num(dev, pd);
 373
 374retry:
 375	status = ocrdma_mbx_alloc_pd(dev, pd);
 376	if (status) {
 377		if (pd->dpp_enabled) {
 378			pd->dpp_enabled = false;
 379			pd->num_dpp_qp = 0;
 380			goto retry;
 381		}
 382		return status;
 383	}
 384
 385	return 0;
 386}
 387
 388static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx,
 389				 struct ocrdma_pd *pd)
 390{
 391	return (uctx->cntxt_pd == pd);
 392}
 393
 394static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev,
 395			      struct ocrdma_pd *pd)
 396{
 397	if (dev->pd_mgr->pd_prealloc_valid)
 398		ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled);
 399	else
 400		ocrdma_mbx_dealloc_pd(dev, pd);
 401}
 402
 403static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev,
 404				    struct ocrdma_ucontext *uctx,
 405				    struct ib_udata *udata)
 406{
 407	struct ib_device *ibdev = &dev->ibdev;
 408	struct ib_pd *pd;
 409	int status;
 410
 411	pd = rdma_zalloc_drv_obj(ibdev, ib_pd);
 412	if (!pd)
 413		return -ENOMEM;
 414
 415	pd->device  = ibdev;
 416	uctx->cntxt_pd = get_ocrdma_pd(pd);
 417
 418	status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata);
 419	if (status) {
 420		kfree(uctx->cntxt_pd);
 421		goto err;
 422	}
 423
 424	uctx->cntxt_pd->uctx = uctx;
 425	uctx->cntxt_pd->ibpd.device = &dev->ibdev;
 426err:
 427	return status;
 428}
 429
 430static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx)
 431{
 432	struct ocrdma_pd *pd = uctx->cntxt_pd;
 433	struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
 434
 435	if (uctx->pd_in_use) {
 436		pr_err("%s(%d) Freeing in use pdid=0x%x.\n",
 437		       __func__, dev->id, pd->id);
 438	}
 439	uctx->cntxt_pd = NULL;
 440	_ocrdma_dealloc_pd(dev, pd);
 441	kfree(pd);
 442}
 443
 444static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx)
 445{
 446	struct ocrdma_pd *pd = NULL;
 447
 448	mutex_lock(&uctx->mm_list_lock);
 449	if (!uctx->pd_in_use) {
 450		uctx->pd_in_use = true;
 451		pd = uctx->cntxt_pd;
 452	}
 453	mutex_unlock(&uctx->mm_list_lock);
 454
 455	return pd;
 456}
 457
 458static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx)
 459{
 460	mutex_lock(&uctx->mm_list_lock);
 461	uctx->pd_in_use = false;
 462	mutex_unlock(&uctx->mm_list_lock);
 463}
 464
 465int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 466{
 467	struct ib_device *ibdev = uctx->device;
 468	int status;
 469	struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx);
 470	struct ocrdma_alloc_ucontext_resp resp = {};
 471	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 472	struct pci_dev *pdev = dev->nic_info.pdev;
 473	u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE);
 474
 475	if (!udata)
 476		return -EFAULT;
 477	INIT_LIST_HEAD(&ctx->mm_head);
 478	mutex_init(&ctx->mm_list_lock);
 479
 480	ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len,
 481					    &ctx->ah_tbl.pa, GFP_KERNEL);
 482	if (!ctx->ah_tbl.va)
 483		return -ENOMEM;
 484
 485	ctx->ah_tbl.len = map_len;
 486
 487	resp.ah_tbl_len = ctx->ah_tbl.len;
 488	resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va);
 489
 490	status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);
 491	if (status)
 492		goto map_err;
 493
 494	status = ocrdma_alloc_ucontext_pd(dev, ctx, udata);
 495	if (status)
 496		goto pd_err;
 497
 498	resp.dev_id = dev->id;
 499	resp.max_inline_data = dev->attr.max_inline_data;
 500	resp.wqe_size = dev->attr.wqe_size;
 501	resp.rqe_size = dev->attr.rqe_size;
 502	resp.dpp_wqe_size = dev->attr.wqe_size;
 503
 504	memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));
 505	status = ib_copy_to_udata(udata, &resp, sizeof(resp));
 506	if (status)
 507		goto cpy_err;
 508	return 0;
 509
 510cpy_err:
 511	ocrdma_dealloc_ucontext_pd(ctx);
 512pd_err:
 513	ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len);
 514map_err:
 515	dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va,
 516			  ctx->ah_tbl.pa);
 517	return status;
 518}
 519
 520void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
 521{
 522	struct ocrdma_mm *mm, *tmp;
 523	struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx);
 524	struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device);
 525	struct pci_dev *pdev = dev->nic_info.pdev;
 526
 527	ocrdma_dealloc_ucontext_pd(uctx);
 528
 529	ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len);
 530	dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va,
 531			  uctx->ah_tbl.pa);
 532
 533	list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 534		list_del(&mm->entry);
 535		kfree(mm);
 536	}
 537}
 538
 539int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 540{
 541	struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context);
 542	struct ocrdma_dev *dev = get_ocrdma_dev(context->device);
 543	unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
 544	u64 unmapped_db = (u64) dev->nic_info.unmapped_db;
 545	unsigned long len = (vma->vm_end - vma->vm_start);
 546	int status;
 547	bool found;
 548
 549	if (vma->vm_start & (PAGE_SIZE - 1))
 550		return -EINVAL;
 551	found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len);
 552	if (!found)
 553		return -EINVAL;
 554
 555	if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
 556		dev->nic_info.db_total_size)) &&
 557		(len <=	dev->nic_info.db_page_size)) {
 558		if (vma->vm_flags & VM_READ)
 559			return -EPERM;
 560
 561		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 562		status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 563					    len, vma->vm_page_prot);
 564	} else if (dev->nic_info.dpp_unmapped_len &&
 565		(vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) &&
 566		(vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +
 567			dev->nic_info.dpp_unmapped_len)) &&
 568		(len <= dev->nic_info.dpp_unmapped_len)) {
 569		if (vma->vm_flags & VM_READ)
 570			return -EPERM;
 571
 572		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 573		status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 574					    len, vma->vm_page_prot);
 575	} else {
 576		status = remap_pfn_range(vma, vma->vm_start,
 577					 vma->vm_pgoff, len, vma->vm_page_prot);
 578	}
 579	return status;
 580}
 581
 582static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 583				struct ib_udata *udata)
 584{
 585	int status;
 586	u64 db_page_addr;
 587	u64 dpp_page_addr = 0;
 588	u32 db_page_size;
 589	struct ocrdma_alloc_pd_uresp rsp;
 590	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 591		udata, struct ocrdma_ucontext, ibucontext);
 592
 593	memset(&rsp, 0, sizeof(rsp));
 594	rsp.id = pd->id;
 595	rsp.dpp_enabled = pd->dpp_enabled;
 596	db_page_addr = ocrdma_get_db_addr(dev, pd->id);
 597	db_page_size = dev->nic_info.db_page_size;
 598
 599	status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size);
 600	if (status)
 601		return status;
 602
 603	if (pd->dpp_enabled) {
 604		dpp_page_addr = dev->nic_info.dpp_unmapped_addr +
 605				(pd->id * PAGE_SIZE);
 606		status = ocrdma_add_mmap(uctx, dpp_page_addr,
 607				 PAGE_SIZE);
 608		if (status)
 609			goto dpp_map_err;
 610		rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);
 611		rsp.dpp_page_addr_lo = dpp_page_addr;
 612	}
 613
 614	status = ib_copy_to_udata(udata, &rsp, sizeof(rsp));
 615	if (status)
 616		goto ucopy_err;
 617
 618	pd->uctx = uctx;
 619	return 0;
 620
 621ucopy_err:
 622	if (pd->dpp_enabled)
 623		ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE);
 624dpp_map_err:
 625	ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);
 626	return status;
 627}
 628
 629int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 630{
 631	struct ib_device *ibdev = ibpd->device;
 632	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 633	struct ocrdma_pd *pd;
 634	int status;
 635	u8 is_uctx_pd = false;
 636	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 637		udata, struct ocrdma_ucontext, ibucontext);
 638
 639	if (udata) {
 640		pd = ocrdma_get_ucontext_pd(uctx);
 641		if (pd) {
 642			is_uctx_pd = true;
 643			goto pd_mapping;
 644		}
 645	}
 646
 647	pd = get_ocrdma_pd(ibpd);
 648	status = _ocrdma_alloc_pd(dev, pd, uctx, udata);
 649	if (status)
 650		goto exit;
 651
 652pd_mapping:
 653	if (udata) {
 654		status = ocrdma_copy_pd_uresp(dev, pd, udata);
 655		if (status)
 656			goto err;
 657	}
 658	return 0;
 659
 660err:
 661	if (is_uctx_pd)
 662		ocrdma_release_ucontext_pd(uctx);
 663	else
 664		_ocrdma_dealloc_pd(dev, pd);
 665exit:
 666	return status;
 667}
 668
 669int ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 670{
 671	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 672	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 673	struct ocrdma_ucontext *uctx = NULL;
 674	u64 usr_db;
 675
 676	uctx = pd->uctx;
 677	if (uctx) {
 678		u64 dpp_db = dev->nic_info.dpp_unmapped_addr +
 679			(pd->id * PAGE_SIZE);
 680		if (pd->dpp_enabled)
 681			ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE);
 682		usr_db = ocrdma_get_db_addr(dev, pd->id);
 683		ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size);
 684
 685		if (is_ucontext_pd(uctx, pd)) {
 686			ocrdma_release_ucontext_pd(uctx);
 687			return 0;
 688		}
 689	}
 690	_ocrdma_dealloc_pd(dev, pd);
 691	return 0;
 692}
 693
 694static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 695			    u32 pdid, int acc, u32 num_pbls, u32 addr_check)
 696{
 697	int status;
 698
 699	mr->hwmr.fr_mr = 0;
 700	mr->hwmr.local_rd = 1;
 701	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
 702	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 703	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
 704	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
 705	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
 706	mr->hwmr.num_pbls = num_pbls;
 707
 708	status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);
 709	if (status)
 710		return status;
 711
 712	mr->ibmr.lkey = mr->hwmr.lkey;
 713	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
 714		mr->ibmr.rkey = mr->hwmr.lkey;
 715	return 0;
 716}
 717
 718struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)
 719{
 720	int status;
 721	struct ocrdma_mr *mr;
 722	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 723	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 724
 725	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
 726		pr_err("%s err, invalid access rights\n", __func__);
 727		return ERR_PTR(-EINVAL);
 728	}
 729
 730	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 731	if (!mr)
 732		return ERR_PTR(-ENOMEM);
 733
 734	status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,
 735				   OCRDMA_ADDR_CHECK_DISABLE);
 736	if (status) {
 737		kfree(mr);
 738		return ERR_PTR(status);
 739	}
 740
 741	return &mr->ibmr;
 742}
 743
 744static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,
 745				   struct ocrdma_hw_mr *mr)
 746{
 747	struct pci_dev *pdev = dev->nic_info.pdev;
 748	int i = 0;
 749
 750	if (mr->pbl_table) {
 751		for (i = 0; i < mr->num_pbls; i++) {
 752			if (!mr->pbl_table[i].va)
 753				continue;
 754			dma_free_coherent(&pdev->dev, mr->pbl_size,
 755					  mr->pbl_table[i].va,
 756					  mr->pbl_table[i].pa);
 757		}
 758		kfree(mr->pbl_table);
 759		mr->pbl_table = NULL;
 760	}
 761}
 762
 763static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 764			      u32 num_pbes)
 765{
 766	u32 num_pbls = 0;
 767	u32 idx = 0;
 768	int status = 0;
 769	u32 pbl_size;
 770
 771	do {
 772		pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx);
 773		if (pbl_size > MAX_OCRDMA_PBL_SIZE) {
 774			status = -EFAULT;
 775			break;
 776		}
 777		num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));
 778		num_pbls = num_pbls / (pbl_size / sizeof(u64));
 779		idx++;
 780	} while (num_pbls >= dev->attr.max_num_mr_pbl);
 781
 782	mr->hwmr.num_pbes = num_pbes;
 783	mr->hwmr.num_pbls = num_pbls;
 784	mr->hwmr.pbl_size = pbl_size;
 785	return status;
 786}
 787
 788static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
 789{
 790	int status = 0;
 791	int i;
 792	u32 dma_len = mr->pbl_size;
 793	struct pci_dev *pdev = dev->nic_info.pdev;
 794	void *va;
 795	dma_addr_t pa;
 796
 797	mr->pbl_table = kcalloc(mr->num_pbls, sizeof(struct ocrdma_pbl),
 798				GFP_KERNEL);
 799
 800	if (!mr->pbl_table)
 801		return -ENOMEM;
 802
 803	for (i = 0; i < mr->num_pbls; i++) {
 804		va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);
 805		if (!va) {
 806			ocrdma_free_mr_pbl_tbl(dev, mr);
 807			status = -ENOMEM;
 808			break;
 809		}
 810		mr->pbl_table[i].va = va;
 811		mr->pbl_table[i].pa = pa;
 812	}
 813	return status;
 814}
 815
 816static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr)
 817{
 818	struct ocrdma_pbe *pbe;
 819	struct ib_block_iter biter;
 820	struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
 821	int pbe_cnt;
 822	u64 pg_addr;
 823
 824	if (!mr->hwmr.num_pbes)
 825		return;
 826
 827	pbe = (struct ocrdma_pbe *)pbl_tbl->va;
 828	pbe_cnt = 0;
 829
 830	rdma_umem_for_each_dma_block (mr->umem, &biter, PAGE_SIZE) {
 831		/* store the page address in pbe */
 832		pg_addr = rdma_block_iter_dma_address(&biter);
 833		pbe->pa_lo = cpu_to_le32(pg_addr);
 834		pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr));
 835		pbe_cnt += 1;
 836		pbe++;
 837
 838		/* if the given pbl is full storing the pbes,
 839		 * move to next pbl.
 840		 */
 841		if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) {
 842			pbl_tbl++;
 843			pbe = (struct ocrdma_pbe *)pbl_tbl->va;
 844			pbe_cnt = 0;
 845		}
 846	}
 847}
 848
 849struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 850				 u64 usr_addr, int acc, struct ib_dmah *dmah,
 851				 struct ib_udata *udata)
 852{
 853	int status = -ENOMEM;
 854	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 855	struct ocrdma_mr *mr;
 856	struct ocrdma_pd *pd;
 857
 858	if (dmah)
 859		return ERR_PTR(-EOPNOTSUPP);
 860
 861	pd = get_ocrdma_pd(ibpd);
 862
 863	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
 864		return ERR_PTR(-EINVAL);
 865
 866	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 867	if (!mr)
 868		return ERR_PTR(status);
 869	mr->umem = ib_umem_get(ibpd->device, start, len, acc);
 870	if (IS_ERR(mr->umem)) {
 871		status = -EFAULT;
 872		goto umem_err;
 873	}
 874	status = ocrdma_get_pbl_info(
 875		dev, mr, ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE));
 876	if (status)
 877		goto umem_err;
 878
 879	mr->hwmr.pbe_size = PAGE_SIZE;
 880	mr->hwmr.va = usr_addr;
 881	mr->hwmr.len = len;
 882	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 883	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
 884	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
 885	mr->hwmr.local_rd = 1;
 886	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
 887	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
 888	if (status)
 889		goto umem_err;
 890	build_user_pbes(dev, mr);
 891	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
 892	if (status)
 893		goto mbx_err;
 894	mr->ibmr.lkey = mr->hwmr.lkey;
 895	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
 896		mr->ibmr.rkey = mr->hwmr.lkey;
 897
 898	return &mr->ibmr;
 899
 900mbx_err:
 901	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 902umem_err:
 903	kfree(mr);
 904	return ERR_PTR(status);
 905}
 906
 907int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 908{
 909	struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);
 910	struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device);
 911
 912	(void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);
 913
 914	kfree(mr->pages);
 915	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 916
 917	/* it could be user registered memory. */
 918	ib_umem_release(mr->umem);
 919	kfree(mr);
 920
 921	/* Don't stop cleanup, in case FW is unresponsive */
 922	if (dev->mqe_ctx.fw_error_state) {
 923		pr_err("%s(%d) fw not responding.\n",
 924		       __func__, dev->id);
 925	}
 926	return 0;
 927}
 928
 929static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
 930				struct ib_udata *udata)
 931{
 932	int status;
 933	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 934		udata, struct ocrdma_ucontext, ibucontext);
 935	struct ocrdma_create_cq_uresp uresp;
 936
 937	/* this must be user flow! */
 938	if (!udata)
 939		return -EINVAL;
 940
 941	memset(&uresp, 0, sizeof(uresp));
 942	uresp.cq_id = cq->id;
 943	uresp.page_size = PAGE_ALIGN(cq->len);
 944	uresp.num_pages = 1;
 945	uresp.max_hw_cqe = cq->max_hw_cqe;
 946	uresp.page_addr[0] = virt_to_phys(cq->va);
 947	uresp.db_page_addr =  ocrdma_get_db_addr(dev, uctx->cntxt_pd->id);
 948	uresp.db_page_size = dev->nic_info.db_page_size;
 949	uresp.phase_change = cq->phase_change ? 1 : 0;
 950	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 951	if (status) {
 952		pr_err("%s(%d) copy error cqid=0x%x.\n",
 953		       __func__, dev->id, cq->id);
 954		goto err;
 955	}
 956	status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 957	if (status)
 958		goto err;
 959	status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size);
 960	if (status) {
 961		ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 962		goto err;
 963	}
 964	cq->ucontext = uctx;
 965err:
 966	return status;
 967}
 968
 969int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 970		     struct uverbs_attr_bundle *attrs)
 971{
 972	struct ib_udata *udata = &attrs->driver_udata;
 973	struct ib_device *ibdev = ibcq->device;
 974	int entries = attr->cqe;
 975	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
 976	struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 977	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 978		udata, struct ocrdma_ucontext, ibucontext);
 979	u16 pd_id = 0;
 980	int status;
 981	struct ocrdma_create_cq_ureq ureq;
 982
 983	if (attr->flags)
 984		return -EOPNOTSUPP;
 985
 986	if (udata) {
 987		if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 988			return -EFAULT;
 989	} else
 990		ureq.dpp_cq = 0;
 991
 992	spin_lock_init(&cq->cq_lock);
 993	spin_lock_init(&cq->comp_handler_lock);
 994	INIT_LIST_HEAD(&cq->sq_head);
 995	INIT_LIST_HEAD(&cq->rq_head);
 996
 997	if (udata)
 998		pd_id = uctx->cntxt_pd->id;
 999
1000	status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);
1001	if (status)
1002		return status;
1003
1004	if (udata) {
1005		status = ocrdma_copy_cq_uresp(dev, cq, udata);
1006		if (status)
1007			goto ctx_err;
1008	}
1009	cq->phase = OCRDMA_CQE_VALID;
1010	dev->cq_tbl[cq->id] = cq;
1011	return 0;
1012
1013ctx_err:
1014	ocrdma_mbx_destroy_cq(dev, cq);
1015	return status;
1016}
1017
1018int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,
1019		     struct ib_udata *udata)
1020{
1021	int status = 0;
1022	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1023
1024	if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {
1025		status = -EINVAL;
1026		return status;
1027	}
1028	ibcq->cqe = new_cnt;
1029	return status;
1030}
1031
1032static void ocrdma_flush_cq(struct ocrdma_cq *cq)
1033{
1034	int cqe_cnt;
1035	int valid_count = 0;
1036	unsigned long flags;
1037
1038	struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
1039	struct ocrdma_cqe *cqe = NULL;
1040
1041	cqe = cq->va;
1042	cqe_cnt = cq->cqe_cnt;
1043
1044	/* Last irq might have scheduled a polling thread
1045	 * sync-up with it before hard flushing.
1046	 */
1047	spin_lock_irqsave(&cq->cq_lock, flags);
1048	while (cqe_cnt) {
1049		if (is_cqe_valid(cq, cqe))
1050			valid_count++;
1051		cqe++;
1052		cqe_cnt--;
1053	}
1054	ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count);
1055	spin_unlock_irqrestore(&cq->cq_lock, flags);
1056}
1057
1058int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1059{
1060	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1061	struct ocrdma_eq *eq = NULL;
1062	struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
1063	int pdid = 0;
1064	u32 irq, indx;
1065
1066	dev->cq_tbl[cq->id] = NULL;
1067	indx = ocrdma_get_eq_table_index(dev, cq->eqn);
1068
1069	eq = &dev->eq_tbl[indx];
1070	irq = ocrdma_get_irq(dev, eq);
1071	synchronize_irq(irq);
1072	ocrdma_flush_cq(cq);
1073
1074	ocrdma_mbx_destroy_cq(dev, cq);
1075	if (cq->ucontext) {
1076		pdid = cq->ucontext->cntxt_pd->id;
1077		ocrdma_del_mmap(cq->ucontext, (u64) cq->pa,
1078				PAGE_ALIGN(cq->len));
1079		ocrdma_del_mmap(cq->ucontext,
1080				ocrdma_get_db_addr(dev, pdid),
1081				dev->nic_info.db_page_size);
1082	}
1083	return 0;
1084}
1085
1086static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1087{
1088	int status = -EINVAL;
1089
1090	if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {
1091		dev->qp_tbl[qp->id] = qp;
1092		status = 0;
1093	}
1094	return status;
1095}
1096
1097static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1098{
1099	dev->qp_tbl[qp->id] = NULL;
1100}
1101
1102static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
1103				  struct ib_qp_init_attr *attrs,
1104				  struct ib_udata *udata)
1105{
1106	if ((attrs->qp_type != IB_QPT_GSI) &&
1107	    (attrs->qp_type != IB_QPT_RC) &&
1108	    (attrs->qp_type != IB_QPT_UC) &&
1109	    (attrs->qp_type != IB_QPT_UD)) {
1110		pr_err("%s(%d) unsupported qp type=0x%x requested\n",
1111		       __func__, dev->id, attrs->qp_type);
1112		return -EOPNOTSUPP;
1113	}
1114	/* Skip the check for QP1 to support CM size of 128 */
1115	if ((attrs->qp_type != IB_QPT_GSI) &&
1116	    (attrs->cap.max_send_wr > dev->attr.max_wqe)) {
1117		pr_err("%s(%d) unsupported send_wr=0x%x requested\n",
1118		       __func__, dev->id, attrs->cap.max_send_wr);
1119		pr_err("%s(%d) supported send_wr=0x%x\n",
1120		       __func__, dev->id, dev->attr.max_wqe);
1121		return -EINVAL;
1122	}
1123	if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) {
1124		pr_err("%s(%d) unsupported recv_wr=0x%x requested\n",
1125		       __func__, dev->id, attrs->cap.max_recv_wr);
1126		pr_err("%s(%d) supported recv_wr=0x%x\n",
1127		       __func__, dev->id, dev->attr.max_rqe);
1128		return -EINVAL;
1129	}
1130	if (attrs->cap.max_inline_data > dev->attr.max_inline_data) {
1131		pr_err("%s(%d) unsupported inline data size=0x%x requested\n",
1132		       __func__, dev->id, attrs->cap.max_inline_data);
1133		pr_err("%s(%d) supported inline data size=0x%x\n",
1134		       __func__, dev->id, dev->attr.max_inline_data);
1135		return -EINVAL;
1136	}
1137	if (attrs->cap.max_send_sge > dev->attr.max_send_sge) {
1138		pr_err("%s(%d) unsupported send_sge=0x%x requested\n",
1139		       __func__, dev->id, attrs->cap.max_send_sge);
1140		pr_err("%s(%d) supported send_sge=0x%x\n",
1141		       __func__, dev->id, dev->attr.max_send_sge);
1142		return -EINVAL;
1143	}
1144	if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) {
1145		pr_err("%s(%d) unsupported recv_sge=0x%x requested\n",
1146		       __func__, dev->id, attrs->cap.max_recv_sge);
1147		pr_err("%s(%d) supported recv_sge=0x%x\n",
1148		       __func__, dev->id, dev->attr.max_recv_sge);
1149		return -EINVAL;
1150	}
1151	/* unprivileged user space cannot create special QP */
1152	if (udata && attrs->qp_type == IB_QPT_GSI) {
1153		pr_err
1154		    ("%s(%d) Userspace can't create special QPs of type=0x%x\n",
1155		     __func__, dev->id, attrs->qp_type);
1156		return -EINVAL;
1157	}
1158	/* allow creating only one GSI type of QP */
1159	if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
1160		pr_err("%s(%d) GSI special QPs already created.\n",
1161		       __func__, dev->id);
1162		return -EINVAL;
1163	}
1164	/* verify consumer QPs are not trying to use GSI QP's CQ */
1165	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
1166		if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) ||
1167			(dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {
1168			pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
1169				__func__, dev->id);
1170			return -EINVAL;
1171		}
1172	}
1173	return 0;
1174}
1175
1176static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
1177				struct ib_udata *udata, int dpp_offset,
1178				int dpp_credit_lmt, int srq)
1179{
1180	int status;
1181	u64 usr_db;
1182	struct ocrdma_create_qp_uresp uresp;
1183	struct ocrdma_pd *pd = qp->pd;
1184	struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
1185
1186	memset(&uresp, 0, sizeof(uresp));
1187	usr_db = dev->nic_info.unmapped_db +
1188			(pd->id * dev->nic_info.db_page_size);
1189	uresp.qp_id = qp->id;
1190	uresp.sq_dbid = qp->sq.dbid;
1191	uresp.num_sq_pages = 1;
1192	uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);
1193	uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va);
1194	uresp.num_wqe_allocated = qp->sq.max_cnt;
1195	if (!srq) {
1196		uresp.rq_dbid = qp->rq.dbid;
1197		uresp.num_rq_pages = 1;
1198		uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);
1199		uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va);
1200		uresp.num_rqe_allocated = qp->rq.max_cnt;
1201	}
1202	uresp.db_page_addr = usr_db;
1203	uresp.db_page_size = dev->nic_info.db_page_size;
1204	uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;
1205	uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1206	uresp.db_shift = OCRDMA_DB_RQ_SHIFT;
1207
1208	if (qp->dpp_enabled) {
1209		uresp.dpp_credit = dpp_credit_lmt;
1210		uresp.dpp_offset = dpp_offset;
1211	}
1212	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1213	if (status) {
1214		pr_err("%s(%d) user copy error.\n", __func__, dev->id);
1215		goto err;
1216	}
1217	status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0],
1218				 uresp.sq_page_size);
1219	if (status)
1220		goto err;
1221
1222	if (!srq) {
1223		status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0],
1224					 uresp.rq_page_size);
1225		if (status)
1226			goto rq_map_err;
1227	}
1228	return status;
1229rq_map_err:
1230	ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size);
1231err:
1232	return status;
1233}
1234
1235static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
1236			     struct ocrdma_pd *pd)
1237{
1238	if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1239		qp->sq_db = dev->nic_info.db +
1240			(pd->id * dev->nic_info.db_page_size) +
1241			OCRDMA_DB_GEN2_SQ_OFFSET;
1242		qp->rq_db = dev->nic_info.db +
1243			(pd->id * dev->nic_info.db_page_size) +
1244			OCRDMA_DB_GEN2_RQ_OFFSET;
1245	} else {
1246		qp->sq_db = dev->nic_info.db +
1247			(pd->id * dev->nic_info.db_page_size) +
1248			OCRDMA_DB_SQ_OFFSET;
1249		qp->rq_db = dev->nic_info.db +
1250			(pd->id * dev->nic_info.db_page_size) +
1251			OCRDMA_DB_RQ_OFFSET;
1252	}
1253}
1254
1255static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
1256{
1257	qp->wqe_wr_id_tbl =
1258	    kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)),
1259		    GFP_KERNEL);
1260	if (qp->wqe_wr_id_tbl == NULL)
1261		return -ENOMEM;
1262	qp->rqe_wr_id_tbl =
1263	    kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL);
1264	if (qp->rqe_wr_id_tbl == NULL)
1265		return -ENOMEM;
1266
1267	return 0;
1268}
1269
1270static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
1271				      struct ocrdma_pd *pd,
1272				      struct ib_qp_init_attr *attrs)
1273{
1274	qp->pd = pd;
1275	spin_lock_init(&qp->q_lock);
1276	INIT_LIST_HEAD(&qp->sq_entry);
1277	INIT_LIST_HEAD(&qp->rq_entry);
1278
1279	qp->qp_type = attrs->qp_type;
1280	qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;
1281	qp->max_inline_data = attrs->cap.max_inline_data;
1282	qp->sq.max_sges = attrs->cap.max_send_sge;
1283	qp->rq.max_sges = attrs->cap.max_recv_sge;
1284	qp->state = OCRDMA_QPS_RST;
1285	qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1286}
1287
1288static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev,
1289				   struct ib_qp_init_attr *attrs)
1290{
1291	if (attrs->qp_type == IB_QPT_GSI) {
1292		dev->gsi_qp_created = 1;
1293		dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq);
1294		dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq);
1295	}
1296}
1297
1298int ocrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
1299		     struct ib_udata *udata)
1300{
1301	int status;
1302	struct ib_pd *ibpd = ibqp->pd;
1303	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1304	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1305	struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device);
1306	struct ocrdma_create_qp_ureq ureq;
1307	u16 dpp_credit_lmt, dpp_offset;
1308
1309	if (attrs->create_flags)
1310		return -EOPNOTSUPP;
1311
1312	status = ocrdma_check_qp_params(ibpd, dev, attrs, udata);
1313	if (status)
1314		goto gen_err;
1315
1316	memset(&ureq, 0, sizeof(ureq));
1317	if (udata) {
1318		if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1319			return -EFAULT;
1320	}
1321	ocrdma_set_qp_init_params(qp, pd, attrs);
1322	if (udata == NULL)
1323		qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1324					OCRDMA_QP_FAST_REG);
1325
1326	mutex_lock(&dev->dev_lock);
1327	status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,
1328					ureq.dpp_cq_id,
1329					&dpp_offset, &dpp_credit_lmt);
1330	if (status)
1331		goto mbx_err;
1332
1333	/* user space QP's wr_id table are managed in library */
1334	if (udata == NULL) {
1335		status = ocrdma_alloc_wr_id_tbl(qp);
1336		if (status)
1337			goto map_err;
1338	}
1339
1340	status = ocrdma_add_qpn_map(dev, qp);
1341	if (status)
1342		goto map_err;
1343	ocrdma_set_qp_db(dev, qp, pd);
1344	if (udata) {
1345		status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,
1346					      dpp_credit_lmt,
1347					      (attrs->srq != NULL));
1348		if (status)
1349			goto cpy_err;
1350	}
1351	ocrdma_store_gsi_qp_cq(dev, attrs);
1352	qp->ibqp.qp_num = qp->id;
1353	mutex_unlock(&dev->dev_lock);
1354	return 0;
1355
1356cpy_err:
1357	ocrdma_del_qpn_map(dev, qp);
1358map_err:
1359	ocrdma_mbx_destroy_qp(dev, qp);
1360mbx_err:
1361	mutex_unlock(&dev->dev_lock);
1362	kfree(qp->wqe_wr_id_tbl);
1363	kfree(qp->rqe_wr_id_tbl);
1364	pr_err("%s(%d) error=%d\n", __func__, dev->id, status);
1365gen_err:
1366	return status;
1367}
1368
1369int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1370		      int attr_mask)
1371{
1372	int status = 0;
1373	struct ocrdma_qp *qp;
1374	struct ocrdma_dev *dev;
1375	enum ib_qp_state old_qps;
1376
1377	qp = get_ocrdma_qp(ibqp);
1378	dev = get_ocrdma_dev(ibqp->device);
1379	if (attr_mask & IB_QP_STATE)
1380		status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps);
1381	/* if new and previous states are same hw doesn't need to
1382	 * know about it.
1383	 */
1384	if (status < 0)
1385		return status;
1386	return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask);
1387}
1388
1389int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1390		     int attr_mask, struct ib_udata *udata)
1391{
1392	unsigned long flags;
1393	int status = -EINVAL;
1394	struct ocrdma_qp *qp;
1395	struct ocrdma_dev *dev;
1396	enum ib_qp_state old_qps, new_qps;
1397
1398	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1399		return -EOPNOTSUPP;
1400
1401	qp = get_ocrdma_qp(ibqp);
1402	dev = get_ocrdma_dev(ibqp->device);
1403
1404	/* syncronize with multiple context trying to change, retrive qps */
1405	mutex_lock(&dev->dev_lock);
1406	/* syncronize with wqe, rqe posting and cqe processing contexts */
1407	spin_lock_irqsave(&qp->q_lock, flags);
1408	old_qps = get_ibqp_state(qp->state);
1409	if (attr_mask & IB_QP_STATE)
1410		new_qps = attr->qp_state;
1411	else
1412		new_qps = old_qps;
1413	spin_unlock_irqrestore(&qp->q_lock, flags);
1414
1415	if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) {
1416		pr_err("%s(%d) invalid attribute mask=0x%x specified for\n"
1417		       "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",
1418		       __func__, dev->id, attr_mask, qp->id, ibqp->qp_type,
1419		       old_qps, new_qps);
1420		goto param_err;
1421	}
1422
1423	status = _ocrdma_modify_qp(ibqp, attr, attr_mask);
1424	if (status > 0)
1425		status = 0;
1426param_err:
1427	mutex_unlock(&dev->dev_lock);
1428	return status;
1429}
1430
1431static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu)
1432{
1433	switch (mtu) {
1434	case 256:
1435		return IB_MTU_256;
1436	case 512:
1437		return IB_MTU_512;
1438	case 1024:
1439		return IB_MTU_1024;
1440	case 2048:
1441		return IB_MTU_2048;
1442	case 4096:
1443		return IB_MTU_4096;
1444	default:
1445		return IB_MTU_1024;
1446	}
1447}
1448
1449static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags)
1450{
1451	int ib_qp_acc_flags = 0;
1452
1453	if (qp_cap_flags & OCRDMA_QP_INB_WR)
1454		ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1455	if (qp_cap_flags & OCRDMA_QP_INB_RD)
1456		ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1457	return ib_qp_acc_flags;
1458}
1459
1460int ocrdma_query_qp(struct ib_qp *ibqp,
1461		    struct ib_qp_attr *qp_attr,
1462		    int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1463{
1464	int status;
1465	u32 qp_state;
1466	struct ocrdma_qp_params params;
1467	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1468	struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device);
1469
1470	memset(&params, 0, sizeof(params));
1471	mutex_lock(&dev->dev_lock);
1472	status = ocrdma_mbx_query_qp(dev, qp, &params);
1473	mutex_unlock(&dev->dev_lock);
1474	if (status)
1475		goto mbx_err;
1476	if (qp->qp_type == IB_QPT_UD)
1477		qp_attr->qkey = params.qkey;
1478	qp_attr->path_mtu =
1479		ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx &
1480				OCRDMA_QP_PARAMS_PATH_MTU_MASK) >>
1481				OCRDMA_QP_PARAMS_PATH_MTU_SHIFT;
1482	qp_attr->path_mig_state = IB_MIG_MIGRATED;
1483	qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK;
1484	qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK;
1485	qp_attr->dest_qp_num =
1486	    params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK;
1487
1488	qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags);
1489	qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;
1490	qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1491	qp_attr->cap.max_send_sge = qp->sq.max_sges;
1492	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1493	qp_attr->cap.max_inline_data = qp->max_inline_data;
1494	qp_init_attr->cap = qp_attr->cap;
1495	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
1496
1497	rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
1498			params.rnt_rc_sl_fl &
1499			  OCRDMA_QP_PARAMS_FLOW_LABEL_MASK,
1500			qp->sgid_idx,
1501			(params.hop_lmt_rq_psn &
1502			 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >>
1503			 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT,
1504			(params.tclass_sq_psn &
1505			 OCRDMA_QP_PARAMS_TCLASS_MASK) >>
1506			 OCRDMA_QP_PARAMS_TCLASS_SHIFT);
1507	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid[0]);
1508
1509	rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
1510	rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl &
1511					   OCRDMA_QP_PARAMS_SL_MASK) >>
1512					   OCRDMA_QP_PARAMS_SL_SHIFT);
1513	qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn &
1514			    OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >>
1515				OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;
1516	qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn &
1517			      OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >>
1518				OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT;
1519	qp_attr->retry_cnt =
1520	    (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >>
1521		OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT;
1522	qp_attr->min_rnr_timer = 0;
1523	qp_attr->pkey_index = 0;
1524	qp_attr->port_num = 1;
1525	rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
1526	rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
1527	qp_attr->alt_pkey_index = 0;
1528	qp_attr->alt_port_num = 0;
1529	qp_attr->alt_timeout = 0;
1530	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1531	qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >>
1532		    OCRDMA_QP_PARAMS_STATE_SHIFT;
1533	qp_attr->qp_state = get_ibqp_state(qp_state);
1534	qp_attr->cur_qp_state = qp_attr->qp_state;
1535	qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0;
1536	qp_attr->max_dest_rd_atomic =
1537	    params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT;
1538	qp_attr->max_rd_atomic =
1539	    params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK;
1540	qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags &
1541				OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0;
1542	/* Sync driver QP state with FW */
1543	ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL);
1544mbx_err:
1545	return status;
1546}
1547
1548static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx)
1549{
1550	unsigned int i = idx / 32;
1551	u32 mask = (1U << (idx % 32));
1552
1553	srq->idx_bit_fields[i] ^= mask;
1554}
1555
1556static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
1557{
1558	return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt;
1559}
1560
1561static int is_hw_sq_empty(struct ocrdma_qp *qp)
1562{
1563	return (qp->sq.tail == qp->sq.head);
1564}
1565
1566static int is_hw_rq_empty(struct ocrdma_qp *qp)
1567{
1568	return (qp->rq.tail == qp->rq.head);
1569}
1570
1571static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
1572{
1573	return q->va + (q->head * q->entry_size);
1574}
1575
1576static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,
1577				      u32 idx)
1578{
1579	return q->va + (idx * q->entry_size);
1580}
1581
1582static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)
1583{
1584	q->head = (q->head + 1) & q->max_wqe_idx;
1585}
1586
1587static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)
1588{
1589	q->tail = (q->tail + 1) & q->max_wqe_idx;
1590}
1591
1592/* discard the cqe for a given QP */
1593static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1594{
1595	unsigned long cq_flags;
1596	unsigned long flags;
1597	u32 cur_getp, stop_getp;
1598	struct ocrdma_cqe *cqe;
1599	u32 qpn = 0, wqe_idx = 0;
1600
1601	spin_lock_irqsave(&cq->cq_lock, cq_flags);
1602
1603	/* traverse through the CQEs in the hw CQ,
1604	 * find the matching CQE for a given qp,
1605	 * mark the matching one discarded by clearing qpn.
1606	 * ring the doorbell in the poll_cq() as
1607	 * we don't complete out of order cqe.
1608	 */
1609
1610	cur_getp = cq->getp;
1611	/* find upto when do we reap the cq. */
1612	stop_getp = cur_getp;
1613	do {
1614		if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
1615			break;
1616
1617		cqe = cq->va + cur_getp;
1618		/* if (a) done reaping whole hw cq, or
1619		 *    (b) qp_xq becomes empty.
1620		 * then exit
1621		 */
1622		qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK;
1623		/* if previously discarded cqe found, skip that too. */
1624		/* check for matching qp */
1625		if (qpn == 0 || qpn != qp->id)
1626			goto skip_cqe;
1627
1628		if (is_cqe_for_sq(cqe)) {
1629			ocrdma_hwq_inc_tail(&qp->sq);
1630		} else {
1631			if (qp->srq) {
1632				wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
1633					OCRDMA_CQE_BUFTAG_SHIFT) &
1634					qp->srq->rq.max_wqe_idx;
1635				BUG_ON(wqe_idx < 1);
1636				spin_lock_irqsave(&qp->srq->q_lock, flags);
1637				ocrdma_hwq_inc_tail(&qp->srq->rq);
1638				ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1);
1639				spin_unlock_irqrestore(&qp->srq->q_lock, flags);
1640
1641			} else {
1642				ocrdma_hwq_inc_tail(&qp->rq);
1643			}
1644		}
1645		/* mark cqe discarded so that it is not picked up later
1646		 * in the poll_cq().
1647		 */
1648		cqe->cmn.qpn = 0;
1649skip_cqe:
1650		cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
1651	} while (cur_getp != stop_getp);
1652	spin_unlock_irqrestore(&cq->cq_lock, cq_flags);
1653}
1654
1655void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1656{
1657	int found = false;
1658	unsigned long flags;
1659	struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
1660	/* sync with any active CQ poll */
1661
1662	spin_lock_irqsave(&dev->flush_q_lock, flags);
1663	found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
1664	if (found)
1665		list_del(&qp->sq_entry);
1666	if (!qp->srq) {
1667		found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
1668		if (found)
1669			list_del(&qp->rq_entry);
1670	}
1671	spin_unlock_irqrestore(&dev->flush_q_lock, flags);
1672}
1673
1674int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1675{
1676	struct ocrdma_pd *pd;
1677	struct ocrdma_qp *qp;
1678	struct ocrdma_dev *dev;
1679	struct ib_qp_attr attrs;
1680	int attr_mask;
1681	unsigned long flags;
1682
1683	qp = get_ocrdma_qp(ibqp);
1684	dev = get_ocrdma_dev(ibqp->device);
1685
1686	pd = qp->pd;
1687
1688	/* change the QP state to ERROR */
1689	if (qp->state != OCRDMA_QPS_RST) {
1690		attrs.qp_state = IB_QPS_ERR;
1691		attr_mask = IB_QP_STATE;
1692		_ocrdma_modify_qp(ibqp, &attrs, attr_mask);
1693	}
1694	/* ensure that CQEs for newly created QP (whose id may be same with
1695	 * one which just getting destroyed are same), dont get
1696	 * discarded until the old CQEs are discarded.
1697	 */
1698	mutex_lock(&dev->dev_lock);
1699	(void) ocrdma_mbx_destroy_qp(dev, qp);
1700
1701	/*
1702	 * acquire CQ lock while destroy is in progress, in order to
1703	 * protect against proessing in-flight CQEs for this QP.
1704	 */
1705	spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);
1706	if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) {
1707		spin_lock(&qp->rq_cq->cq_lock);
1708		ocrdma_del_qpn_map(dev, qp);
1709		spin_unlock(&qp->rq_cq->cq_lock);
1710	} else {
1711		ocrdma_del_qpn_map(dev, qp);
1712	}
1713	spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags);
1714
1715	if (!pd->uctx) {
1716		ocrdma_discard_cqes(qp, qp->sq_cq);
1717		ocrdma_discard_cqes(qp, qp->rq_cq);
1718	}
1719	mutex_unlock(&dev->dev_lock);
1720
1721	if (pd->uctx) {
1722		ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa,
1723				PAGE_ALIGN(qp->sq.len));
1724		if (!qp->srq)
1725			ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa,
1726					PAGE_ALIGN(qp->rq.len));
1727	}
1728
1729	ocrdma_del_flush_qp(qp);
1730
1731	kfree(qp->wqe_wr_id_tbl);
1732	kfree(qp->rqe_wr_id_tbl);
1733	return 0;
1734}
1735
1736static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
1737				struct ib_udata *udata)
1738{
1739	int status;
1740	struct ocrdma_create_srq_uresp uresp;
1741
1742	memset(&uresp, 0, sizeof(uresp));
1743	uresp.rq_dbid = srq->rq.dbid;
1744	uresp.num_rq_pages = 1;
1745	uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va);
1746	uresp.rq_page_size = srq->rq.len;
1747	uresp.db_page_addr = dev->nic_info.unmapped_db +
1748	    (srq->pd->id * dev->nic_info.db_page_size);
1749	uresp.db_page_size = dev->nic_info.db_page_size;
1750	uresp.num_rqe_allocated = srq->rq.max_cnt;
1751	if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1752		uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1753		uresp.db_shift = 24;
1754	} else {
1755		uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
1756		uresp.db_shift = 16;
1757	}
1758
1759	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1760	if (status)
1761		return status;
1762	status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0],
1763				 uresp.rq_page_size);
1764	if (status)
1765		return status;
1766	return status;
1767}
1768
1769int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1770		      struct ib_udata *udata)
1771{
1772	int status;
1773	struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd);
1774	struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
1775	struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq);
1776
1777	if (init_attr->srq_type != IB_SRQT_BASIC)
1778		return -EOPNOTSUPP;
1779
1780	if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
1781		return -EINVAL;
1782	if (init_attr->attr.max_wr > dev->attr.max_rqe)
1783		return -EINVAL;
1784
1785	spin_lock_init(&srq->q_lock);
1786	srq->pd = pd;
1787	srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
1788	status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);
1789	if (status)
1790		return status;
1791
1792	if (!udata) {
1793		srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64),
1794					     GFP_KERNEL);
1795		if (!srq->rqe_wr_id_tbl) {
1796			status = -ENOMEM;
1797			goto arm_err;
1798		}
1799
1800		srq->bit_fields_len = (srq->rq.max_cnt / 32) +
1801		    (srq->rq.max_cnt % 32 ? 1 : 0);
1802		srq->idx_bit_fields =
1803		    kmalloc_array(srq->bit_fields_len, sizeof(u32),
1804				  GFP_KERNEL);
1805		if (!srq->idx_bit_fields) {
1806			status = -ENOMEM;
1807			goto arm_err;
1808		}
1809		memset(srq->idx_bit_fields, 0xff,
1810		       srq->bit_fields_len * sizeof(u32));
1811	}
1812
1813	if (init_attr->attr.srq_limit) {
1814		status = ocrdma_mbx_modify_srq(srq, &init_attr->attr);
1815		if (status)
1816			goto arm_err;
1817	}
1818
1819	if (udata) {
1820		status = ocrdma_copy_srq_uresp(dev, srq, udata);
1821		if (status)
1822			goto arm_err;
1823	}
1824
1825	return 0;
1826
1827arm_err:
1828	ocrdma_mbx_destroy_srq(dev, srq);
1829	kfree(srq->rqe_wr_id_tbl);
1830	kfree(srq->idx_bit_fields);
1831	return status;
1832}
1833
1834int ocrdma_modify_srq(struct ib_srq *ibsrq,
1835		      struct ib_srq_attr *srq_attr,
1836		      enum ib_srq_attr_mask srq_attr_mask,
1837		      struct ib_udata *udata)
1838{
1839	int status;
1840	struct ocrdma_srq *srq;
1841
1842	srq = get_ocrdma_srq(ibsrq);
1843	if (srq_attr_mask & IB_SRQ_MAX_WR)
1844		status = -EINVAL;
1845	else
1846		status = ocrdma_mbx_modify_srq(srq, srq_attr);
1847	return status;
1848}
1849
1850int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
1851{
1852	struct ocrdma_srq *srq;
1853
1854	srq = get_ocrdma_srq(ibsrq);
1855	return ocrdma_mbx_query_srq(srq, srq_attr);
1856}
1857
1858int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1859{
1860	struct ocrdma_srq *srq;
1861	struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
1862
1863	srq = get_ocrdma_srq(ibsrq);
1864
1865	ocrdma_mbx_destroy_srq(dev, srq);
1866
1867	if (srq->pd->uctx)
1868		ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa,
1869				PAGE_ALIGN(srq->rq.len));
1870
1871	kfree(srq->idx_bit_fields);
1872	kfree(srq->rqe_wr_id_tbl);
1873	return 0;
1874}
1875
1876/* unprivileged verbs and their support functions. */
1877static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
1878				struct ocrdma_hdr_wqe *hdr,
1879				const struct ib_send_wr *wr)
1880{
1881	struct ocrdma_ewqe_ud_hdr *ud_hdr =
1882		(struct ocrdma_ewqe_ud_hdr *)(hdr + 1);
1883	struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah);
1884
1885	ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn;
1886	if (qp->qp_type == IB_QPT_GSI)
1887		ud_hdr->qkey = qp->qkey;
1888	else
1889		ud_hdr->qkey = ud_wr(wr)->remote_qkey;
1890	ud_hdr->rsvd_ahid = ah->id;
1891	ud_hdr->hdr_type = ah->hdr_type;
1892	if (ah->av->valid & OCRDMA_AV_VLAN_VALID)
1893		hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT);
1894}
1895
1896static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
1897			      struct ocrdma_sge *sge, int num_sge,
1898			      struct ib_sge *sg_list)
1899{
1900	int i;
1901
1902	for (i = 0; i < num_sge; i++) {
1903		sge[i].lrkey = sg_list[i].lkey;
1904		sge[i].addr_lo = sg_list[i].addr;
1905		sge[i].addr_hi = upper_32_bits(sg_list[i].addr);
1906		sge[i].len = sg_list[i].length;
1907		hdr->total_len += sg_list[i].length;
1908	}
1909	if (num_sge == 0)
1910		memset(sge, 0, sizeof(*sge));
1911}
1912
1913static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge)
1914{
1915	uint32_t total_len = 0, i;
1916
1917	for (i = 0; i < num_sge; i++)
1918		total_len += sg_list[i].length;
1919	return total_len;
1920}
1921
1922
1923static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1924				    struct ocrdma_hdr_wqe *hdr,
1925				    struct ocrdma_sge *sge,
1926				    const struct ib_send_wr *wr, u32 wqe_size)
1927{
1928	int i;
1929	char *dpp_addr;
1930
1931	if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {
1932		hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge);
1933		if (unlikely(hdr->total_len > qp->max_inline_data)) {
1934			pr_err("%s() supported_len=0x%x,\n"
1935			       " unsupported len req=0x%x\n", __func__,
1936				qp->max_inline_data, hdr->total_len);
1937			return -EINVAL;
1938		}
1939		dpp_addr = (char *)sge;
1940		for (i = 0; i < wr->num_sge; i++) {
1941			memcpy(dpp_addr,
1942			       (void *)(unsigned long)wr->sg_list[i].addr,
1943			       wr->sg_list[i].length);
1944			dpp_addr += wr->sg_list[i].length;
1945		}
1946
1947		wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
1948		if (0 == hdr->total_len)
1949			wqe_size += sizeof(struct ocrdma_sge);
1950		hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
1951	} else {
1952		ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1953		if (wr->num_sge)
1954			wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));
1955		else
1956			wqe_size += sizeof(struct ocrdma_sge);
1957		hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1958	}
1959	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1960	return 0;
1961}
1962
1963static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1964			     const struct ib_send_wr *wr)
1965{
1966	struct ocrdma_sge *sge;
1967	u32 wqe_size = sizeof(*hdr);
1968
1969	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
1970		ocrdma_build_ud_hdr(qp, hdr, wr);
1971		sge = (struct ocrdma_sge *)(hdr + 2);
1972		wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
1973	} else {
1974		sge = (struct ocrdma_sge *)(hdr + 1);
1975	}
1976
1977	return ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1978}
1979
1980static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1981			      const struct ib_send_wr *wr)
1982{
1983	int status;
1984	struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1985	struct ocrdma_sge *sge = ext_rw + 1;
1986	u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw);
1987
1988	status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1989	if (status)
1990		return status;
1991	ext_rw->addr_lo = rdma_wr(wr)->remote_addr;
1992	ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
1993	ext_rw->lrkey = rdma_wr(wr)->rkey;
1994	ext_rw->len = hdr->total_len;
1995	return 0;
1996}
1997
1998static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1999			      const struct ib_send_wr *wr)
2000{
2001	struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
2002	struct ocrdma_sge *sge = ext_rw + 1;
2003	u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +
2004	    sizeof(struct ocrdma_hdr_wqe);
2005
2006	ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
2007	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2008	hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);
2009	hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2010
2011	ext_rw->addr_lo = rdma_wr(wr)->remote_addr;
2012	ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
2013	ext_rw->lrkey = rdma_wr(wr)->rkey;
2014	ext_rw->len = hdr->total_len;
2015}
2016
2017static int get_encoded_page_size(int pg_sz)
2018{
2019	/* Max size is 256M 4096 << 16 */
2020	int i = 0;
2021	for (; i < 17; i++)
2022		if (pg_sz == (4096 << i))
2023			break;
2024	return i;
2025}
2026
2027static int ocrdma_build_reg(struct ocrdma_qp *qp,
2028			    struct ocrdma_hdr_wqe *hdr,
2029			    const struct ib_reg_wr *wr)
2030{
2031	u64 fbo;
2032	struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1);
2033	struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr);
2034	struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
2035	struct ocrdma_pbe *pbe;
2036	u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr);
2037	int num_pbes = 0, i;
2038
2039	wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES);
2040
2041	hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT);
2042	hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2043
2044	if (wr->access & IB_ACCESS_LOCAL_WRITE)
2045		hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR;
2046	if (wr->access & IB_ACCESS_REMOTE_WRITE)
2047		hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR;
2048	if (wr->access & IB_ACCESS_REMOTE_READ)
2049		hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD;
2050	hdr->lkey = wr->key;
2051	hdr->total_len = mr->ibmr.length;
2052
2053	fbo = mr->ibmr.iova - mr->pages[0];
2054
2055	fast_reg->va_hi = upper_32_bits(mr->ibmr.iova);
2056	fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff);
2057	fast_reg->fbo_hi = upper_32_bits(fbo);
2058	fast_reg->fbo_lo = (u32) fbo & 0xffffffff;
2059	fast_reg->num_sges = mr->npages;
2060	fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size);
2061
2062	pbe = pbl_tbl->va;
2063	for (i = 0; i < mr->npages; i++) {
2064		u64 buf_addr = mr->pages[i];
2065
2066		pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK));
2067		pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr));
2068		num_pbes += 1;
2069		pbe++;
2070
2071		/* if the pbl is full storing the pbes,
2072		 * move to next pbl.
2073		*/
2074		if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) {
2075			pbl_tbl++;
2076			pbe = (struct ocrdma_pbe *)pbl_tbl->va;
2077		}
2078	}
2079
2080	return 0;
2081}
2082
2083static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
2084{
2085	u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT);
2086
2087	iowrite32(val, qp->sq_db);
2088}
2089
2090int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
2091		     const struct ib_send_wr **bad_wr)
2092{
2093	int status = 0;
2094	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2095	struct ocrdma_hdr_wqe *hdr;
2096	unsigned long flags;
2097
2098	spin_lock_irqsave(&qp->q_lock, flags);
2099	if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
2100		spin_unlock_irqrestore(&qp->q_lock, flags);
2101		*bad_wr = wr;
2102		return -EINVAL;
2103	}
2104
2105	while (wr) {
2106		if (qp->qp_type == IB_QPT_UD &&
2107		    (wr->opcode != IB_WR_SEND &&
2108		     wr->opcode != IB_WR_SEND_WITH_IMM)) {
2109			*bad_wr = wr;
2110			status = -EINVAL;
2111			break;
2112		}
2113		if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||
2114		    wr->num_sge > qp->sq.max_sges) {
2115			*bad_wr = wr;
2116			status = -ENOMEM;
2117			break;
2118		}
2119		hdr = ocrdma_hwq_head(&qp->sq);
2120		hdr->cw = 0;
2121		if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2122			hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2123		if (wr->send_flags & IB_SEND_FENCE)
2124			hdr->cw |=
2125			    (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);
2126		if (wr->send_flags & IB_SEND_SOLICITED)
2127			hdr->cw |=
2128			    (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);
2129		hdr->total_len = 0;
2130		switch (wr->opcode) {
2131		case IB_WR_SEND_WITH_IMM:
2132			hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2133			hdr->immdt = ntohl(wr->ex.imm_data);
2134			fallthrough;
2135		case IB_WR_SEND:
2136			hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2137			ocrdma_build_send(qp, hdr, wr);
2138			break;
2139		case IB_WR_SEND_WITH_INV:
2140			hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
2141			hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2142			hdr->lkey = wr->ex.invalidate_rkey;
2143			status = ocrdma_build_send(qp, hdr, wr);
2144			break;
2145		case IB_WR_RDMA_WRITE_WITH_IMM:
2146			hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2147			hdr->immdt = ntohl(wr->ex.imm_data);
2148			fallthrough;
2149		case IB_WR_RDMA_WRITE:
2150			hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);
2151			status = ocrdma_build_write(qp, hdr, wr);
2152			break;
2153		case IB_WR_RDMA_READ:
2154			ocrdma_build_read(qp, hdr, wr);
2155			break;
2156		case IB_WR_LOCAL_INV:
2157			hdr->cw |=
2158			    (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);
2159			hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) +
2160					sizeof(struct ocrdma_sge)) /
2161				OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;
2162			hdr->lkey = wr->ex.invalidate_rkey;
2163			break;
2164		case IB_WR_REG_MR:
2165			status = ocrdma_build_reg(qp, hdr, reg_wr(wr));
2166			break;
2167		default:
2168			status = -EINVAL;
2169			break;
2170		}
2171		if (status) {
2172			*bad_wr = wr;
2173			break;
2174		}
2175		if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2176			qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
2177		else
2178			qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
2179		qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
2180		ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &
2181				   OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE);
2182		/* make sure wqe is written before adapter can access it */
2183		wmb();
2184		/* inform hw to start processing it */
2185		ocrdma_ring_sq_db(qp);
2186
2187		/* update pointer, counter for next wr */
2188		ocrdma_hwq_inc_head(&qp->sq);
2189		wr = wr->next;
2190	}
2191	spin_unlock_irqrestore(&qp->q_lock, flags);
2192	return status;
2193}
2194
2195static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
2196{
2197	u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT);
2198
2199	iowrite32(val, qp->rq_db);
2200}
2201
2202static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe,
2203			     const struct ib_recv_wr *wr, u16 tag)
2204{
2205	u32 wqe_size = 0;
2206	struct ocrdma_sge *sge;
2207	if (wr->num_sge)
2208		wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);
2209	else
2210		wqe_size = sizeof(*sge) + sizeof(*rqe);
2211
2212	rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) <<
2213				OCRDMA_WQE_SIZE_SHIFT);
2214	rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2215	rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2216	rqe->total_len = 0;
2217	rqe->rsvd_tag = tag;
2218	sge = (struct ocrdma_sge *)(rqe + 1);
2219	ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);
2220	ocrdma_cpu_to_le32(rqe, wqe_size);
2221}
2222
2223int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
2224		     const struct ib_recv_wr **bad_wr)
2225{
2226	int status = 0;
2227	unsigned long flags;
2228	struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2229	struct ocrdma_hdr_wqe *rqe;
2230
2231	spin_lock_irqsave(&qp->q_lock, flags);
2232	if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
2233		spin_unlock_irqrestore(&qp->q_lock, flags);
2234		*bad_wr = wr;
2235		return -EINVAL;
2236	}
2237	while (wr) {
2238		if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||
2239		    wr->num_sge > qp->rq.max_sges) {
2240			*bad_wr = wr;
2241			status = -ENOMEM;
2242			break;
2243		}
2244		rqe = ocrdma_hwq_head(&qp->rq);
2245		ocrdma_build_rqe(rqe, wr, 0);
2246
2247		qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
2248		/* make sure rqe is written before adapter can access it */
2249		wmb();
2250
2251		/* inform hw to start processing it */
2252		ocrdma_ring_rq_db(qp);
2253
2254		/* update pointer, counter for next wr */
2255		ocrdma_hwq_inc_head(&qp->rq);
2256		wr = wr->next;
2257	}
2258	spin_unlock_irqrestore(&qp->q_lock, flags);
2259	return status;
2260}
2261
2262/* cqe for srq's rqe can potentially arrive out of order.
2263 * index gives the entry in the shadow table where to store
2264 * the wr_id. tag/index is returned in cqe to reference back
2265 * for a given rqe.
2266 */
2267static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)
2268{
2269	int row = 0;
2270	int indx = 0;
2271
2272	for (row = 0; row < srq->bit_fields_len; row++) {
2273		if (srq->idx_bit_fields[row]) {
2274			indx = ffs(srq->idx_bit_fields[row]);
2275			indx = (row * 32) + (indx - 1);
2276			BUG_ON(indx >= srq->rq.max_cnt);
2277			ocrdma_srq_toggle_bit(srq, indx);
2278			break;
2279		}
2280	}
2281
2282	BUG_ON(row == srq->bit_fields_len);
2283	return indx + 1; /* Use from index 1 */
2284}
2285
2286static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)
2287{
2288	u32 val = srq->rq.dbid | (1 << 16);
2289
2290	iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET);
2291}
2292
2293int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
2294			 const struct ib_recv_wr **bad_wr)
2295{
2296	int status = 0;
2297	unsigned long flags;
2298	struct ocrdma_srq *srq;
2299	struct ocrdma_hdr_wqe *rqe;
2300	u16 tag;
2301
2302	srq = get_ocrdma_srq(ibsrq);
2303
2304	spin_lock_irqsave(&srq->q_lock, flags);
2305	while (wr) {
2306		if (ocrdma_hwq_free_cnt(&srq->rq) == 0 ||
2307		    wr->num_sge > srq->rq.max_sges) {
2308			status = -ENOMEM;
2309			*bad_wr = wr;
2310			break;
2311		}
2312		tag = ocrdma_srq_get_idx(srq);
2313		rqe = ocrdma_hwq_head(&srq->rq);
2314		ocrdma_build_rqe(rqe, wr, tag);
2315
2316		srq->rqe_wr_id_tbl[tag] = wr->wr_id;
2317		/* make sure rqe is written before adapter can perform DMA */
2318		wmb();
2319		/* inform hw to start processing it */
2320		ocrdma_ring_srq_db(srq);
2321		/* update pointer, counter for next wr */
2322		ocrdma_hwq_inc_head(&srq->rq);
2323		wr = wr->next;
2324	}
2325	spin_unlock_irqrestore(&srq->q_lock, flags);
2326	return status;
2327}
2328
2329static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)
2330{
2331	enum ib_wc_status ibwc_status;
2332
2333	switch (status) {
2334	case OCRDMA_CQE_GENERAL_ERR:
2335		ibwc_status = IB_WC_GENERAL_ERR;
2336		break;
2337	case OCRDMA_CQE_LOC_LEN_ERR:
2338		ibwc_status = IB_WC_LOC_LEN_ERR;
2339		break;
2340	case OCRDMA_CQE_LOC_QP_OP_ERR:
2341		ibwc_status = IB_WC_LOC_QP_OP_ERR;
2342		break;
2343	case OCRDMA_CQE_LOC_EEC_OP_ERR:
2344		ibwc_status = IB_WC_LOC_EEC_OP_ERR;
2345		break;
2346	case OCRDMA_CQE_LOC_PROT_ERR:
2347		ibwc_status = IB_WC_LOC_PROT_ERR;
2348		break;
2349	case OCRDMA_CQE_WR_FLUSH_ERR:
2350		ibwc_status = IB_WC_WR_FLUSH_ERR;
2351		break;
2352	case OCRDMA_CQE_MW_BIND_ERR:
2353		ibwc_status = IB_WC_MW_BIND_ERR;
2354		break;
2355	case OCRDMA_CQE_BAD_RESP_ERR:
2356		ibwc_status = IB_WC_BAD_RESP_ERR;
2357		break;
2358	case OCRDMA_CQE_LOC_ACCESS_ERR:
2359		ibwc_status = IB_WC_LOC_ACCESS_ERR;
2360		break;
2361	case OCRDMA_CQE_REM_INV_REQ_ERR:
2362		ibwc_status = IB_WC_REM_INV_REQ_ERR;
2363		break;
2364	case OCRDMA_CQE_REM_ACCESS_ERR:
2365		ibwc_status = IB_WC_REM_ACCESS_ERR;
2366		break;
2367	case OCRDMA_CQE_REM_OP_ERR:
2368		ibwc_status = IB_WC_REM_OP_ERR;
2369		break;
2370	case OCRDMA_CQE_RETRY_EXC_ERR:
2371		ibwc_status = IB_WC_RETRY_EXC_ERR;
2372		break;
2373	case OCRDMA_CQE_RNR_RETRY_EXC_ERR:
2374		ibwc_status = IB_WC_RNR_RETRY_EXC_ERR;
2375		break;
2376	case OCRDMA_CQE_LOC_RDD_VIOL_ERR:
2377		ibwc_status = IB_WC_LOC_RDD_VIOL_ERR;
2378		break;
2379	case OCRDMA_CQE_REM_INV_RD_REQ_ERR:
2380		ibwc_status = IB_WC_REM_INV_RD_REQ_ERR;
2381		break;
2382	case OCRDMA_CQE_REM_ABORT_ERR:
2383		ibwc_status = IB_WC_REM_ABORT_ERR;
2384		break;
2385	case OCRDMA_CQE_INV_EECN_ERR:
2386		ibwc_status = IB_WC_INV_EECN_ERR;
2387		break;
2388	case OCRDMA_CQE_INV_EEC_STATE_ERR:
2389		ibwc_status = IB_WC_INV_EEC_STATE_ERR;
2390		break;
2391	case OCRDMA_CQE_FATAL_ERR:
2392		ibwc_status = IB_WC_FATAL_ERR;
2393		break;
2394	case OCRDMA_CQE_RESP_TIMEOUT_ERR:
2395		ibwc_status = IB_WC_RESP_TIMEOUT_ERR;
2396		break;
2397	default:
2398		ibwc_status = IB_WC_GENERAL_ERR;
2399		break;
2400	}
2401	return ibwc_status;
2402}
2403
2404static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2405		      u32 wqe_idx)
2406{
2407	struct ocrdma_hdr_wqe *hdr;
2408	struct ocrdma_sge *rw;
2409	int opcode;
2410
2411	hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);
2412
2413	ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;
2414	/* Undo the hdr->cw swap */
2415	opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;
2416	switch (opcode) {
2417	case OCRDMA_WRITE:
2418		ibwc->opcode = IB_WC_RDMA_WRITE;
2419		break;
2420	case OCRDMA_READ:
2421		rw = (struct ocrdma_sge *)(hdr + 1);
2422		ibwc->opcode = IB_WC_RDMA_READ;
2423		ibwc->byte_len = rw->len;
2424		break;
2425	case OCRDMA_SEND:
2426		ibwc->opcode = IB_WC_SEND;
2427		break;
2428	case OCRDMA_FR_MR:
2429		ibwc->opcode = IB_WC_REG_MR;
2430		break;
2431	case OCRDMA_LKEY_INV:
2432		ibwc->opcode = IB_WC_LOCAL_INV;
2433		break;
2434	default:
2435		ibwc->status = IB_WC_GENERAL_ERR;
2436		pr_err("%s() invalid opcode received = 0x%x\n",
2437		       __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK);
2438		break;
2439	}
2440}
2441
2442static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
2443						struct ocrdma_cqe *cqe)
2444{
2445	if (is_cqe_for_sq(cqe)) {
2446		cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2447				cqe->flags_status_srcqpn) &
2448					~OCRDMA_CQE_STATUS_MASK);
2449		cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2450				cqe->flags_status_srcqpn) |
2451				(OCRDMA_CQE_WR_FLUSH_ERR <<
2452					OCRDMA_CQE_STATUS_SHIFT));
2453	} else {
2454		if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2455			cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2456					cqe->flags_status_srcqpn) &
2457						~OCRDMA_CQE_UD_STATUS_MASK);
2458			cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2459					cqe->flags_status_srcqpn) |
2460					(OCRDMA_CQE_WR_FLUSH_ERR <<
2461						OCRDMA_CQE_UD_STATUS_SHIFT));
2462		} else {
2463			cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2464					cqe->flags_status_srcqpn) &
2465						~OCRDMA_CQE_STATUS_MASK);
2466			cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2467					cqe->flags_status_srcqpn) |
2468					(OCRDMA_CQE_WR_FLUSH_ERR <<
2469						OCRDMA_CQE_STATUS_SHIFT));
2470		}
2471	}
2472}
2473
2474static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2475				  struct ocrdma_qp *qp, int status)
2476{
2477	bool expand = false;
2478
2479	ibwc->byte_len = 0;
2480	ibwc->qp = &qp->ibqp;
2481	ibwc->status = ocrdma_to_ibwc_err(status);
2482
2483	ocrdma_flush_qp(qp);
2484	ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL);
2485
2486	/* if wqe/rqe pending for which cqe needs to be returned,
2487	 * trigger inflating it.
2488	 */
2489	if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
2490		expand = true;
2491		ocrdma_set_cqe_status_flushed(qp, cqe);
2492	}
2493	return expand;
2494}
2495
2496static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2497				  struct ocrdma_qp *qp, int status)
2498{
2499	ibwc->opcode = IB_WC_RECV;
2500	ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2501	ocrdma_hwq_inc_tail(&qp->rq);
2502
2503	return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2504}
2505
2506static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2507				  struct ocrdma_qp *qp, int status)
2508{
2509	ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2510	ocrdma_hwq_inc_tail(&qp->sq);
2511
2512	return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2513}
2514
2515
2516static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
2517				 struct ocrdma_cqe *cqe, struct ib_wc *ibwc,
2518				 bool *polled, bool *stop)
2519{
2520	bool expand;
2521	struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
2522	int status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2523		OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2524	if (status < OCRDMA_MAX_CQE_ERR)
2525		atomic_inc(&dev->cqe_err_stats[status]);
2526
2527	/* when hw sq is empty, but rq is not empty, so we continue
2528	 * to keep the cqe in order to get the cq event again.
2529	 */
2530	if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
2531		/* when cq for rq and sq is same, it is safe to return
2532		 * flush cqe for RQEs.
2533		 */
2534		if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2535			*polled = true;
2536			status = OCRDMA_CQE_WR_FLUSH_ERR;
2537			expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2538		} else {
2539			/* stop processing further cqe as this cqe is used for
2540			 * triggering cq event on buddy cq of RQ.
2541			 * When QP is destroyed, this cqe will be removed
2542			 * from the cq's hardware q.
2543			 */
2544			*polled = false;
2545			*stop = true;
2546			expand = false;
2547		}
2548	} else if (is_hw_sq_empty(qp)) {
2549		/* Do nothing */
2550		expand = false;
2551		*polled = false;
2552		*stop = false;
2553	} else {
2554		*polled = true;
2555		expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2556	}
2557	return expand;
2558}
2559
2560static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2561				     struct ocrdma_cqe *cqe,
2562				     struct ib_wc *ibwc, bool *polled)
2563{
2564	bool expand = false;
2565	int tail = qp->sq.tail;
2566	u32 wqe_idx;
2567
2568	if (!qp->wqe_wr_id_tbl[tail].signaled) {
2569		*polled = false;    /* WC cannot be consumed yet */
2570	} else {
2571		ibwc->status = IB_WC_SUCCESS;
2572		ibwc->wc_flags = 0;
2573		ibwc->qp = &qp->ibqp;
2574		ocrdma_update_wc(qp, ibwc, tail);
2575		*polled = true;
2576	}
2577	wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) &
2578			OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
2579	if (tail != wqe_idx)
2580		expand = true; /* Coalesced CQE can't be consumed yet */
2581
2582	ocrdma_hwq_inc_tail(&qp->sq);
2583	return expand;
2584}
2585
2586static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2587			     struct ib_wc *ibwc, bool *polled, bool *stop)
2588{
2589	int status;
2590	bool expand;
2591
2592	status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2593		OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2594
2595	if (status == OCRDMA_CQE_SUCCESS)
2596		expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
2597	else
2598		expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
2599	return expand;
2600}
2601
2602static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc,
2603				 struct ocrdma_cqe *cqe)
2604{
2605	int status;
2606	u16 hdr_type = 0;
2607
2608	status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2609		OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;
2610	ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) &
2611						OCRDMA_CQE_SRCQP_MASK;
2612	ibwc->pkey_index = 0;
2613	ibwc->wc_flags = IB_WC_GRH;
2614	ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2615			  OCRDMA_CQE_UD_XFER_LEN_SHIFT) &
2616			  OCRDMA_CQE_UD_XFER_LEN_MASK;
2617
2618	if (ocrdma_is_udp_encap_supported(dev)) {
2619		hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2620			    OCRDMA_CQE_UD_L3TYPE_SHIFT) &
2621			    OCRDMA_CQE_UD_L3TYPE_MASK;
2622		ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
2623		ibwc->network_hdr_type = hdr_type;
2624	}
2625
2626	return status;
2627}
2628
2629static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,
2630				       struct ocrdma_cqe *cqe,
2631				       struct ocrdma_qp *qp)
2632{
2633	unsigned long flags;
2634	struct ocrdma_srq *srq;
2635	u32 wqe_idx;
2636
2637	srq = get_ocrdma_srq(qp->ibqp.srq);
2638	wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
2639		OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx;
2640	BUG_ON(wqe_idx < 1);
2641
2642	ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];
2643	spin_lock_irqsave(&srq->q_lock, flags);
2644	ocrdma_srq_toggle_bit(srq, wqe_idx - 1);
2645	spin_unlock_irqrestore(&srq->q_lock, flags);
2646	ocrdma_hwq_inc_tail(&srq->rq);
2647}
2648
2649static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2650				struct ib_wc *ibwc, bool *polled, bool *stop,
2651				int status)
2652{
2653	bool expand;
2654	struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
2655
2656	if (status < OCRDMA_MAX_CQE_ERR)
2657		atomic_inc(&dev->cqe_err_stats[status]);
2658
2659	/* when hw_rq is empty, but wq is not empty, so continue
2660	 * to keep the cqe to get the cq event again.
2661	 */
2662	if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
2663		if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2664			*polled = true;
2665			status = OCRDMA_CQE_WR_FLUSH_ERR;
2666			expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2667		} else {
2668			*polled = false;
2669			*stop = true;
2670			expand = false;
2671		}
2672	} else if (is_hw_rq_empty(qp)) {
2673		/* Do nothing */
2674		expand = false;
2675		*polled = false;
2676		*stop = false;
2677	} else {
2678		*polled = true;
2679		expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2680	}
2681	return expand;
2682}
2683
2684static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2685				     struct ocrdma_cqe *cqe, struct ib_wc *ibwc)
2686{
2687	struct ocrdma_dev *dev;
2688
2689	dev = get_ocrdma_dev(qp->ibqp.device);
2690	ibwc->opcode = IB_WC_RECV;
2691	ibwc->qp = &qp->ibqp;
2692	ibwc->status = IB_WC_SUCCESS;
2693
2694	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2695		ocrdma_update_ud_rcqe(dev, ibwc, cqe);
2696	else
2697		ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);
2698
2699	if (is_cqe_imm(cqe)) {
2700		ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2701		ibwc->wc_flags |= IB_WC_WITH_IMM;
2702	} else if (is_cqe_wr_imm(cqe)) {
2703		ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2704		ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2705		ibwc->wc_flags |= IB_WC_WITH_IMM;
2706	} else if (is_cqe_invalidated(cqe)) {
2707		ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);
2708		ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;
2709	}
2710	if (qp->ibqp.srq) {
2711		ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
2712	} else {
2713		ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2714		ocrdma_hwq_inc_tail(&qp->rq);
2715	}
2716}
2717
2718static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2719			     struct ib_wc *ibwc, bool *polled, bool *stop)
2720{
2721	int status;
2722	bool expand = false;
2723
2724	ibwc->wc_flags = 0;
2725	if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2726		status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2727					OCRDMA_CQE_UD_STATUS_MASK) >>
2728					OCRDMA_CQE_UD_STATUS_SHIFT;
2729	} else {
2730		status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2731			     OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2732	}
2733
2734	if (status == OCRDMA_CQE_SUCCESS) {
2735		*polled = true;
2736		ocrdma_poll_success_rcqe(qp, cqe, ibwc);
2737	} else {
2738		expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
2739					      status);
2740	}
2741	return expand;
2742}
2743
2744static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,
2745				   u16 cur_getp)
2746{
2747	if (cq->phase_change) {
2748		if (cur_getp == 0)
2749			cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
2750	} else {
2751		/* clear valid bit */
2752		cqe->flags_status_srcqpn = 0;
2753	}
2754}
2755
2756static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
2757			    struct ib_wc *ibwc)
2758{
2759	u16 qpn = 0;
2760	int i = 0;
2761	bool expand = false;
2762	int polled_hw_cqes = 0;
2763	struct ocrdma_qp *qp = NULL;
2764	struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
2765	struct ocrdma_cqe *cqe;
2766	u16 cur_getp; bool polled = false; bool stop = false;
2767
2768	cur_getp = cq->getp;
2769	while (num_entries) {
2770		cqe = cq->va + cur_getp;
2771		/* check whether valid cqe or not */
2772		if (!is_cqe_valid(cq, cqe))
2773			break;
2774		qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);
2775		/* ignore discarded cqe */
2776		if (qpn == 0)
2777			goto skip_cqe;
2778		qp = dev->qp_tbl[qpn];
2779		BUG_ON(qp == NULL);
2780
2781		if (is_cqe_for_sq(cqe)) {
2782			expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,
2783						  &stop);
2784		} else {
2785			expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,
2786						  &stop);
2787		}
2788		if (expand)
2789			goto expand_cqe;
2790		if (stop)
2791			goto stop_cqe;
2792		/* clear qpn to avoid duplicate processing by discard_cqe() */
2793		cqe->cmn.qpn = 0;
2794skip_cqe:
2795		polled_hw_cqes += 1;
2796		cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
2797		ocrdma_change_cq_phase(cq, cqe, cur_getp);
2798expand_cqe:
2799		if (polled) {
2800			num_entries -= 1;
2801			i += 1;
2802			ibwc = ibwc + 1;
2803			polled = false;
2804		}
2805	}
2806stop_cqe:
2807	cq->getp = cur_getp;
2808
2809	if (polled_hw_cqes)
2810		ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes);
2811
2812	return i;
2813}
2814
2815/* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */
2816static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2817			      struct ocrdma_qp *qp, struct ib_wc *ibwc)
2818{
2819	int err_cqes = 0;
2820
2821	while (num_entries) {
2822		if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
2823			break;
2824		if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
2825			ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2826			ocrdma_hwq_inc_tail(&qp->sq);
2827		} else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2828			ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2829			ocrdma_hwq_inc_tail(&qp->rq);
2830		} else {
2831			return err_cqes;
2832		}
2833		ibwc->byte_len = 0;
2834		ibwc->status = IB_WC_WR_FLUSH_ERR;
2835		ibwc = ibwc + 1;
2836		err_cqes += 1;
2837		num_entries -= 1;
2838	}
2839	return err_cqes;
2840}
2841
2842int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2843{
2844	int cqes_to_poll = num_entries;
2845	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2846	struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2847	int num_os_cqe = 0, err_cqes = 0;
2848	struct ocrdma_qp *qp;
2849	unsigned long flags;
2850
2851	/* poll cqes from adapter CQ */
2852	spin_lock_irqsave(&cq->cq_lock, flags);
2853	num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc);
2854	spin_unlock_irqrestore(&cq->cq_lock, flags);
2855	cqes_to_poll -= num_os_cqe;
2856
2857	if (cqes_to_poll) {
2858		wc = wc + num_os_cqe;
2859		/* adapter returns single error cqe when qp moves to
2860		 * error state. So insert error cqes with wc_status as
2861		 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ
2862		 * respectively which uses this CQ.
2863		 */
2864		spin_lock_irqsave(&dev->flush_q_lock, flags);
2865		list_for_each_entry(qp, &cq->sq_head, sq_entry) {
2866			if (cqes_to_poll == 0)
2867				break;
2868			err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);
2869			cqes_to_poll -= err_cqes;
2870			num_os_cqe += err_cqes;
2871			wc = wc + err_cqes;
2872		}
2873		spin_unlock_irqrestore(&dev->flush_q_lock, flags);
2874	}
2875	return num_os_cqe;
2876}
2877
2878int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2879{
2880	struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2881	struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2882	u16 cq_id;
2883	unsigned long flags;
2884	bool arm_needed = false, sol_needed = false;
2885
2886	cq_id = cq->id;
2887
2888	spin_lock_irqsave(&cq->cq_lock, flags);
2889	if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)
2890		arm_needed = true;
2891	if (cq_flags & IB_CQ_SOLICITED)
2892		sol_needed = true;
2893
2894	ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0);
2895	spin_unlock_irqrestore(&cq->cq_lock, flags);
2896
2897	return 0;
2898}
2899
2900struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2901			      u32 max_num_sg)
2902{
2903	int status;
2904	struct ocrdma_mr *mr;
2905	struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
2906	struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
2907
2908	if (mr_type != IB_MR_TYPE_MEM_REG)
2909		return ERR_PTR(-EINVAL);
2910
2911	if (max_num_sg > dev->attr.max_pages_per_frmr)
2912		return ERR_PTR(-EINVAL);
2913
2914	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2915	if (!mr)
2916		return ERR_PTR(-ENOMEM);
2917
2918	mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
2919	if (!mr->pages) {
2920		status = -ENOMEM;
2921		goto pl_err;
2922	}
2923
2924	status = ocrdma_get_pbl_info(dev, mr, max_num_sg);
2925	if (status)
2926		goto pbl_err;
2927	mr->hwmr.fr_mr = 1;
2928	mr->hwmr.remote_rd = 0;
2929	mr->hwmr.remote_wr = 0;
2930	mr->hwmr.local_rd = 0;
2931	mr->hwmr.local_wr = 0;
2932	mr->hwmr.mw_bind = 0;
2933	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
2934	if (status)
2935		goto pbl_err;
2936	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);
2937	if (status)
2938		goto mbx_err;
2939	mr->ibmr.rkey = mr->hwmr.lkey;
2940	mr->ibmr.lkey = mr->hwmr.lkey;
2941	dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] =
2942		(unsigned long) mr;
2943	return &mr->ibmr;
2944mbx_err:
2945	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
2946pbl_err:
2947	kfree(mr->pages);
2948pl_err:
2949	kfree(mr);
2950	return ERR_PTR(-ENOMEM);
2951}
2952
2953static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr)
2954{
2955	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
2956
2957	if (unlikely(mr->npages == mr->hwmr.num_pbes))
2958		return -ENOMEM;
2959
2960	mr->pages[mr->npages++] = addr;
2961
2962	return 0;
2963}
2964
2965int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
2966		     unsigned int *sg_offset)
2967{
2968	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
2969
2970	mr->npages = 0;
2971
2972	return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page);
2973}