ib_verbs.c

   1/*
   2 * Broadcom NetXtreme-E RoCE driver.
   3 *
   4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
   5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
   6 *
   7 * This software is available to you under a choice of one of two
   8 * licenses.  You may choose to be licensed under the terms of the GNU
   9 * General Public License (GPL) Version 2, available from the file
  10 * COPYING in the main directory of this source tree, or the
  11 * BSD license below:
  12 *
  13 * Redistribution and use in source and binary forms, with or without
  14 * modification, are permitted provided that the following conditions
  15 * are met:
  16 *
  17 * 1. Redistributions of source code must retain the above copyright
  18 *    notice, this list of conditions and the following disclaimer.
  19 * 2. Redistributions in binary form must reproduce the above copyright
  20 *    notice, this list of conditions and the following disclaimer in
  21 *    the documentation and/or other materials provided with the
  22 *    distribution.
  23 *
  24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
  25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
  28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35 *
  36 * Description: IB Verbs interpreter
  37 */
  38
  39#include <linux/interrupt.h>
  40#include <linux/types.h>
  41#include <linux/pci.h>
  42#include <linux/netdevice.h>
  43#include <linux/if_ether.h>
  44#include <net/addrconf.h>
  45
  46#include <rdma/ib_verbs.h>
  47#include <rdma/ib_user_verbs.h>
  48#include <rdma/ib_umem.h>
  49#include <rdma/ib_addr.h>
  50#include <rdma/ib_mad.h>
  51#include <rdma/ib_cache.h>
  52#include <rdma/ib_pma.h>
  53#include <rdma/uverbs_ioctl.h>
  54#include <linux/hashtable.h>
  55
  56#include "roce_hsi.h"
  57#include "qplib_res.h"
  58#include "qplib_sp.h"
  59#include "qplib_fp.h"
  60#include "qplib_rcfw.h"
  61
  62#include "bnxt_re.h"
  63#include "ib_verbs.h"
  64#include "debugfs.h"
  65
  66#include <rdma/uverbs_types.h>
  67#include <rdma/uverbs_std_types.h>
  68
  69#include <rdma/ib_user_ioctl_cmds.h>
  70
  71#define UVERBS_MODULE_NAME bnxt_re
  72#include <rdma/uverbs_named_ioctl.h>
  73
  74#include <rdma/bnxt_re-abi.h>
  75
  76static int __from_ib_access_flags(int iflags)
  77{
  78	int qflags = 0;
  79
  80	if (iflags & IB_ACCESS_LOCAL_WRITE)
  81		qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
  82	if (iflags & IB_ACCESS_REMOTE_READ)
  83		qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
  84	if (iflags & IB_ACCESS_REMOTE_WRITE)
  85		qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
  86	if (iflags & IB_ACCESS_REMOTE_ATOMIC)
  87		qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
  88	if (iflags & IB_ACCESS_MW_BIND)
  89		qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
  90	if (iflags & IB_ZERO_BASED)
  91		qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
  92	if (iflags & IB_ACCESS_ON_DEMAND)
  93		qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
  94	return qflags;
  95};
  96
  97static int __to_ib_access_flags(int qflags)
  98{
  99	int iflags = 0;
 100
 101	if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
 102		iflags |= IB_ACCESS_LOCAL_WRITE;
 103	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
 104		iflags |= IB_ACCESS_REMOTE_WRITE;
 105	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
 106		iflags |= IB_ACCESS_REMOTE_READ;
 107	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
 108		iflags |= IB_ACCESS_REMOTE_ATOMIC;
 109	if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
 110		iflags |= IB_ACCESS_MW_BIND;
 111	if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
 112		iflags |= IB_ZERO_BASED;
 113	if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
 114		iflags |= IB_ACCESS_ON_DEMAND;
 115	return iflags;
 116}
 117
 118static u8 __qp_access_flags_from_ib(struct bnxt_qplib_chip_ctx *cctx, int iflags)
 119{
 120	u8 qflags = 0;
 121
 122	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))
 123		/* For Wh+ */
 124		return (u8)__from_ib_access_flags(iflags);
 125
 126	/* For P5, P7 and later chips */
 127	if (iflags & IB_ACCESS_LOCAL_WRITE)
 128		qflags |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
 129	if (iflags & IB_ACCESS_REMOTE_WRITE)
 130		qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
 131	if (iflags & IB_ACCESS_REMOTE_READ)
 132		qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
 133	if (iflags & IB_ACCESS_REMOTE_ATOMIC)
 134		qflags |= CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC;
 135
 136	return qflags;
 137}
 138
 139static int __qp_access_flags_to_ib(struct bnxt_qplib_chip_ctx *cctx, u8 qflags)
 140{
 141	int iflags = 0;
 142
 143	if (!bnxt_qplib_is_chip_gen_p5_p7(cctx))
 144		/* For Wh+ */
 145		return __to_ib_access_flags(qflags);
 146
 147	/* For P5, P7 and later chips */
 148	if (qflags & CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE)
 149		iflags |= IB_ACCESS_LOCAL_WRITE;
 150	if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE)
 151		iflags |= IB_ACCESS_REMOTE_WRITE;
 152	if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_READ)
 153		iflags |= IB_ACCESS_REMOTE_READ;
 154	if (qflags & CMDQ_MODIFY_QP_ACCESS_REMOTE_ATOMIC)
 155		iflags |= IB_ACCESS_REMOTE_ATOMIC;
 156
 157	return iflags;
 158}
 159
 160static void bnxt_re_check_and_set_relaxed_ordering(struct bnxt_re_dev *rdev,
 161						   struct bnxt_qplib_mrw *qplib_mr)
 162{
 163	if (_is_relaxed_ordering_supported(rdev->dev_attr->dev_cap_flags2) &&
 164	    pcie_relaxed_ordering_enabled(rdev->en_dev->pdev))
 165		qplib_mr->flags |= CMDQ_REGISTER_MR_FLAGS_ENABLE_RO;
 166}
 167
 168static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
 169			     struct bnxt_qplib_sge *sg_list, int num)
 170{
 171	int i, total = 0;
 172
 173	for (i = 0; i < num; i++) {
 174		sg_list[i].addr = ib_sg_list[i].addr;
 175		sg_list[i].lkey = ib_sg_list[i].lkey;
 176		sg_list[i].size = ib_sg_list[i].length;
 177		total += sg_list[i].size;
 178	}
 179	return total;
 180}
 181
 182/* Device */
 183int bnxt_re_query_device(struct ib_device *ibdev,
 184			 struct ib_device_attr *ib_attr,
 185			 struct ib_udata *udata)
 186{
 187	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 188	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
 189
 190	memset(ib_attr, 0, sizeof(*ib_attr));
 191	memcpy(&ib_attr->fw_ver, dev_attr->fw_ver,
 192	       min(sizeof(dev_attr->fw_ver),
 193		   sizeof(ib_attr->fw_ver)));
 194	addrconf_addr_eui48((u8 *)&ib_attr->sys_image_guid,
 195			    rdev->netdev->dev_addr);
 196	ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
 197	ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_SUPPORTED;
 198
 199	ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
 200	ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
 201	ib_attr->hw_ver = rdev->en_dev->pdev->revision;
 202	ib_attr->max_qp = dev_attr->max_qp;
 203	ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
 204	ib_attr->device_cap_flags =
 205				    IB_DEVICE_CURR_QP_STATE_MOD
 206				    | IB_DEVICE_RC_RNR_NAK_GEN
 207				    | IB_DEVICE_SHUTDOWN_PORT
 208				    | IB_DEVICE_SYS_IMAGE_GUID
 209				    | IB_DEVICE_RESIZE_MAX_WR
 210				    | IB_DEVICE_PORT_ACTIVE_EVENT
 211				    | IB_DEVICE_N_NOTIFY_CQ
 212				    | IB_DEVICE_MEM_WINDOW
 213				    | IB_DEVICE_MEM_WINDOW_TYPE_2B
 214				    | IB_DEVICE_MEM_MGT_EXTENSIONS;
 215	ib_attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
 216	ib_attr->max_send_sge = dev_attr->max_qp_sges;
 217	ib_attr->max_recv_sge = dev_attr->max_qp_sges;
 218	ib_attr->max_sge_rd = dev_attr->max_qp_sges;
 219	ib_attr->max_cq = dev_attr->max_cq;
 220	ib_attr->max_cqe = dev_attr->max_cq_wqes;
 221	ib_attr->max_mr = dev_attr->max_mr;
 222	ib_attr->max_pd = dev_attr->max_pd;
 223	ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
 224	ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
 225	ib_attr->atomic_cap = IB_ATOMIC_NONE;
 226	ib_attr->masked_atomic_cap = IB_ATOMIC_NONE;
 227	if (dev_attr->is_atomic) {
 228		ib_attr->atomic_cap = IB_ATOMIC_GLOB;
 229		ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
 230	}
 231
 232	ib_attr->max_ee_rd_atom = 0;
 233	ib_attr->max_res_rd_atom = 0;
 234	ib_attr->max_ee_init_rd_atom = 0;
 235	ib_attr->max_ee = 0;
 236	ib_attr->max_rdd = 0;
 237	ib_attr->max_mw = dev_attr->max_mw;
 238	ib_attr->max_raw_ipv6_qp = 0;
 239	ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
 240	ib_attr->max_mcast_grp = 0;
 241	ib_attr->max_mcast_qp_attach = 0;
 242	ib_attr->max_total_mcast_qp_attach = 0;
 243	ib_attr->max_ah = dev_attr->max_ah;
 244
 245	ib_attr->max_srq = dev_attr->max_srq;
 246	ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
 247	ib_attr->max_srq_sge = dev_attr->max_srq_sges;
 248
 249	ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
 250
 251	ib_attr->max_pkeys = 1;
 252	ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
 253	return 0;
 254}
 255
 256int bnxt_re_modify_device(struct ib_device *ibdev,
 257			  int device_modify_mask,
 258			  struct ib_device_modify *device_modify)
 259{
 260	ibdev_dbg(ibdev, "Modify device with mask 0x%x", device_modify_mask);
 261
 262	if (device_modify_mask & ~IB_DEVICE_MODIFY_NODE_DESC)
 263		return -EOPNOTSUPP;
 264
 265	if (!(device_modify_mask & IB_DEVICE_MODIFY_NODE_DESC))
 266		return 0;
 267
 268	memcpy(ibdev->node_desc, device_modify->node_desc, IB_DEVICE_NODE_DESC_MAX);
 269	return 0;
 270}
 271
 272/* Port */
 273int bnxt_re_query_port(struct ib_device *ibdev, u32 port_num,
 274		       struct ib_port_attr *port_attr)
 275{
 276	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 277	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
 278	int rc;
 279
 280	memset(port_attr, 0, sizeof(*port_attr));
 281
 282	if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
 283		port_attr->state = IB_PORT_ACTIVE;
 284		port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 285	} else {
 286		port_attr->state = IB_PORT_DOWN;
 287		port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 288	}
 289	port_attr->max_mtu = IB_MTU_4096;
 290	port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
 291	/* One GID is reserved for RawEth QP. Report one less */
 292	port_attr->gid_tbl_len = (rdev->rcfw.roce_mirror ? (dev_attr->max_sgid - 1) :
 293				  dev_attr->max_sgid);
 294	port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 295				    IB_PORT_DEVICE_MGMT_SUP |
 296				    IB_PORT_VENDOR_CLASS_SUP;
 297	port_attr->ip_gids = true;
 298
 299	port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
 300	port_attr->bad_pkey_cntr = 0;
 301	port_attr->qkey_viol_cntr = 0;
 302	port_attr->pkey_tbl_len = dev_attr->max_pkey;
 303	port_attr->lid = 0;
 304	port_attr->sm_lid = 0;
 305	port_attr->lmc = 0;
 306	port_attr->max_vl_num = 4;
 307	port_attr->sm_sl = 0;
 308	port_attr->subnet_timeout = 0;
 309	port_attr->init_type_reply = 0;
 310	rc = ib_get_eth_speed(&rdev->ibdev, port_num, &port_attr->active_speed,
 311			      &port_attr->active_width);
 312
 313	return rc;
 314}
 315
 316int bnxt_re_get_port_immutable(struct ib_device *ibdev, u32 port_num,
 317			       struct ib_port_immutable *immutable)
 318{
 319	struct ib_port_attr port_attr;
 320
 321	if (bnxt_re_query_port(ibdev, port_num, &port_attr))
 322		return -EINVAL;
 323
 324	immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
 325	immutable->gid_tbl_len = port_attr.gid_tbl_len;
 326	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
 327	immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
 328	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 329	return 0;
 330}
 331
 332void bnxt_re_query_fw_str(struct ib_device *ibdev, char *str)
 333{
 334	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 335
 336	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d.%d",
 337		 rdev->dev_attr->fw_ver[0], rdev->dev_attr->fw_ver[1],
 338		 rdev->dev_attr->fw_ver[2], rdev->dev_attr->fw_ver[3]);
 339}
 340
 341int bnxt_re_query_pkey(struct ib_device *ibdev, u32 port_num,
 342		       u16 index, u16 *pkey)
 343{
 344	if (index > 0)
 345		return -EINVAL;
 346
 347	*pkey = IB_DEFAULT_PKEY_FULL;
 348
 349	return 0;
 350}
 351
 352int bnxt_re_query_gid(struct ib_device *ibdev, u32 port_num,
 353		      int index, union ib_gid *gid)
 354{
 355	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 356	int rc;
 357
 358	/* Ignore port_num */
 359	memset(gid, 0, sizeof(*gid));
 360	rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
 361				 &rdev->qplib_res.sgid_tbl, index,
 362				 (struct bnxt_qplib_gid *)gid);
 363	return rc;
 364}
 365
 366int bnxt_re_del_gid(const struct ib_gid_attr *attr, void **context)
 367{
 368	int rc = 0;
 369	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 370	struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 371	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 372	struct bnxt_qplib_gid *gid_to_del;
 373	u16 vlan_id = 0xFFFF;
 374
 375	/* Delete the entry from the hardware */
 376	ctx = *context;
 377	if (!ctx)
 378		return -EINVAL;
 379
 380	if (sgid_tbl->active) {
 381		if (ctx->idx >= sgid_tbl->max)
 382			return -EINVAL;
 383		gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
 384		vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
 385		/* DEL_GID is called in WQ context(netdevice_event_work_handler)
 386		 * or via the ib_unregister_device path. In the former case QP1
 387		 * may not be destroyed yet, in which case just return as FW
 388		 * needs that entry to be present and will fail it's deletion.
 389		 * We could get invoked again after QP1 is destroyed OR get an
 390		 * ADD_GID call with a different GID value for the same index
 391		 * where we issue MODIFY_GID cmd to update the GID entry -- TBD
 392		 */
 393		if (ctx->idx == 0 &&
 394		    rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
 395		    ctx->refcnt == 1 && rdev->gsi_ctx.gsi_sqp) {
 396			ibdev_dbg(&rdev->ibdev,
 397				  "Trying to delete GID0 while QP1 is alive\n");
 398			return -EFAULT;
 399		}
 400		ctx->refcnt--;
 401		if (!ctx->refcnt) {
 402			rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
 403						 vlan_id,  true);
 404			if (rc) {
 405				ibdev_err(&rdev->ibdev,
 406					  "Failed to remove GID: %#x", rc);
 407			} else {
 408				ctx_tbl = sgid_tbl->ctx;
 409				ctx_tbl[ctx->idx] = NULL;
 410				kfree(ctx);
 411			}
 412		}
 413	} else {
 414		return -EINVAL;
 415	}
 416	return rc;
 417}
 418
 419int bnxt_re_add_gid(const struct ib_gid_attr *attr, void **context)
 420{
 421	int rc;
 422	u32 tbl_idx = 0;
 423	u16 vlan_id = 0xFFFF;
 424	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
 425	struct bnxt_re_dev *rdev = to_bnxt_re_dev(attr->device, ibdev);
 426	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
 427
 428	rc = rdma_read_gid_l2_fields(attr, &vlan_id, NULL);
 429	if (rc)
 430		return rc;
 431
 432	rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)&attr->gid,
 433				 rdev->qplib_res.netdev->dev_addr,
 434				 vlan_id, true, &tbl_idx, false, 0);
 435	if (rc == -EALREADY) {
 436		ctx_tbl = sgid_tbl->ctx;
 437		ctx_tbl[tbl_idx]->refcnt++;
 438		*context = ctx_tbl[tbl_idx];
 439		return 0;
 440	}
 441
 442	if (rc < 0) {
 443		ibdev_err(&rdev->ibdev, "Failed to add GID: %#x", rc);
 444		return rc;
 445	}
 446
 447	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 448	if (!ctx)
 449		return -ENOMEM;
 450	ctx_tbl = sgid_tbl->ctx;
 451	ctx->idx = tbl_idx;
 452	ctx->refcnt = 1;
 453	ctx_tbl[tbl_idx] = ctx;
 454	*context = ctx;
 455
 456	return rc;
 457}
 458
 459enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
 460					    u32 port_num)
 461{
 462	return IB_LINK_LAYER_ETHERNET;
 463}
 464
 465#define	BNXT_RE_FENCE_PBL_SIZE	DIV_ROUND_UP(BNXT_RE_FENCE_BYTES, PAGE_SIZE)
 466
 467static void bnxt_re_create_fence_wqe(struct bnxt_re_pd *pd)
 468{
 469	struct bnxt_re_fence_data *fence = &pd->fence;
 470	struct ib_mr *ib_mr = &fence->mr->ib_mr;
 471	struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
 472	struct bnxt_re_dev *rdev = pd->rdev;
 473
 474	if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
 475		return;
 476
 477	memset(wqe, 0, sizeof(*wqe));
 478	wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
 479	wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
 480	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
 481	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
 482	wqe->bind.zero_based = false;
 483	wqe->bind.parent_l_key = ib_mr->lkey;
 484	wqe->bind.va = (u64)(unsigned long)fence->va;
 485	wqe->bind.length = fence->size;
 486	wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
 487	wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
 488
 489	/* Save the initial rkey in fence structure for now;
 490	 * wqe->bind.r_key will be set at (re)bind time.
 491	 */
 492	fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
 493}
 494
 495static int bnxt_re_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
 496{
 497	struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
 498					     qplib_qp);
 499	struct ib_pd *ib_pd = qp->ib_qp.pd;
 500	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 501	struct bnxt_re_fence_data *fence = &pd->fence;
 502	struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
 503	struct bnxt_qplib_swqe wqe;
 504	int rc;
 505
 506	memcpy(&wqe, fence_wqe, sizeof(wqe));
 507	wqe.bind.r_key = fence->bind_rkey;
 508	fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
 509
 510	ibdev_dbg(&qp->rdev->ibdev,
 511		  "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
 512		wqe.bind.r_key, qp->qplib_qp.id, pd);
 513	rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
 514	if (rc) {
 515		ibdev_err(&qp->rdev->ibdev, "Failed to bind fence-WQE\n");
 516		return rc;
 517	}
 518	bnxt_qplib_post_send_db(&qp->qplib_qp);
 519
 520	return rc;
 521}
 522
 523static void bnxt_re_destroy_fence_mr(struct bnxt_re_pd *pd)
 524{
 525	struct bnxt_re_fence_data *fence = &pd->fence;
 526	struct bnxt_re_dev *rdev = pd->rdev;
 527	struct device *dev = &rdev->en_dev->pdev->dev;
 528	struct bnxt_re_mr *mr = fence->mr;
 529
 530	if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
 531		return;
 532
 533	if (fence->mw) {
 534		bnxt_re_dealloc_mw(fence->mw);
 535		fence->mw = NULL;
 536	}
 537	if (mr) {
 538		if (mr->ib_mr.rkey)
 539			bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
 540					     true);
 541		if (mr->ib_mr.lkey)
 542			bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
 543		kfree(mr);
 544		fence->mr = NULL;
 545	}
 546	if (fence->dma_addr) {
 547		dma_unmap_single(dev, fence->dma_addr, BNXT_RE_FENCE_BYTES,
 548				 DMA_BIDIRECTIONAL);
 549		fence->dma_addr = 0;
 550	}
 551}
 552
 553static int bnxt_re_create_fence_mr(struct bnxt_re_pd *pd)
 554{
 555	int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
 556	struct bnxt_re_fence_data *fence = &pd->fence;
 557	struct bnxt_re_dev *rdev = pd->rdev;
 558	struct device *dev = &rdev->en_dev->pdev->dev;
 559	struct bnxt_re_mr *mr = NULL;
 560	dma_addr_t dma_addr = 0;
 561	struct ib_mw *mw;
 562	int rc;
 563
 564	if (bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))
 565		return 0;
 566
 567	dma_addr = dma_map_single(dev, fence->va, BNXT_RE_FENCE_BYTES,
 568				  DMA_BIDIRECTIONAL);
 569	rc = dma_mapping_error(dev, dma_addr);
 570	if (rc) {
 571		ibdev_err(&rdev->ibdev, "Failed to dma-map fence-MR-mem\n");
 572		rc = -EIO;
 573		fence->dma_addr = 0;
 574		goto fail;
 575	}
 576	fence->dma_addr = dma_addr;
 577
 578	/* Allocate a MR */
 579	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 580	if (!mr) {
 581		rc = -ENOMEM;
 582		goto fail;
 583	}
 584	fence->mr = mr;
 585	mr->rdev = rdev;
 586	mr->qplib_mr.pd = &pd->qplib_pd;
 587	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
 588	mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
 589	if (!_is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags)) {
 590		rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
 591		if (rc) {
 592			ibdev_err(&rdev->ibdev, "Failed to alloc fence-HW-MR\n");
 593			goto fail;
 594		}
 595
 596		/* Register MR */
 597		mr->ib_mr.lkey = mr->qplib_mr.lkey;
 598	} else {
 599		mr->qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
 600	}
 601	mr->qplib_mr.va = (u64)(unsigned long)fence->va;
 602	mr->qplib_mr.total_size = BNXT_RE_FENCE_BYTES;
 603	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL,
 604			       BNXT_RE_FENCE_PBL_SIZE, PAGE_SIZE,
 605			       _is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags));
 606	if (rc) {
 607		ibdev_err(&rdev->ibdev, "Failed to register fence-MR\n");
 608		goto fail;
 609	}
 610	mr->ib_mr.rkey = mr->qplib_mr.rkey;
 611
 612	/* Create a fence MW only for kernel consumers */
 613	mw = bnxt_re_alloc_mw(&pd->ib_pd, IB_MW_TYPE_1, NULL);
 614	if (IS_ERR(mw)) {
 615		ibdev_err(&rdev->ibdev,
 616			  "Failed to create fence-MW for PD: %p\n", pd);
 617		rc = PTR_ERR(mw);
 618		goto fail;
 619	}
 620	fence->mw = mw;
 621
 622	bnxt_re_create_fence_wqe(pd);
 623	return 0;
 624
 625fail:
 626	bnxt_re_destroy_fence_mr(pd);
 627	return rc;
 628}
 629
 630static struct bnxt_re_user_mmap_entry*
 631bnxt_re_mmap_entry_insert(struct bnxt_re_ucontext *uctx, u64 mem_offset,
 632			  enum bnxt_re_mmap_flag mmap_flag, u64 *offset)
 633{
 634	struct bnxt_re_user_mmap_entry *entry;
 635	int ret;
 636
 637	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 638	if (!entry)
 639		return NULL;
 640
 641	entry->mem_offset = mem_offset;
 642	entry->mmap_flag = mmap_flag;
 643	entry->uctx = uctx;
 644
 645	switch (mmap_flag) {
 646	case BNXT_RE_MMAP_SH_PAGE:
 647		ret = rdma_user_mmap_entry_insert_exact(&uctx->ib_uctx,
 648							&entry->rdma_entry, PAGE_SIZE, 0);
 649		break;
 650	case BNXT_RE_MMAP_UC_DB:
 651	case BNXT_RE_MMAP_WC_DB:
 652	case BNXT_RE_MMAP_DBR_BAR:
 653	case BNXT_RE_MMAP_DBR_PAGE:
 654	case BNXT_RE_MMAP_TOGGLE_PAGE:
 655		ret = rdma_user_mmap_entry_insert(&uctx->ib_uctx,
 656						  &entry->rdma_entry, PAGE_SIZE);
 657		break;
 658	default:
 659		ret = -EINVAL;
 660		break;
 661	}
 662
 663	if (ret) {
 664		kfree(entry);
 665		return NULL;
 666	}
 667	if (offset)
 668		*offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
 669
 670	return entry;
 671}
 672
 673/* Protection Domains */
 674int bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
 675{
 676	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 677	struct bnxt_re_dev *rdev = pd->rdev;
 678
 679	if (udata) {
 680		rdma_user_mmap_entry_remove(pd->pd_db_mmap);
 681		pd->pd_db_mmap = NULL;
 682	}
 683
 684	bnxt_re_destroy_fence_mr(pd);
 685
 686	if (pd->qplib_pd.id) {
 687		if (!bnxt_qplib_dealloc_pd(&rdev->qplib_res,
 688					   &rdev->qplib_res.pd_tbl,
 689					   &pd->qplib_pd))
 690			atomic_dec(&rdev->stats.res.pd_count);
 691	}
 692	return 0;
 693}
 694
 695int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 696{
 697	struct ib_device *ibdev = ibpd->device;
 698	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
 699	struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(
 700		udata, struct bnxt_re_ucontext, ib_uctx);
 701	struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);
 702	struct bnxt_re_user_mmap_entry *entry = NULL;
 703	u32 active_pds;
 704	int rc = 0;
 705
 706	pd->rdev = rdev;
 707	if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) {
 708		ibdev_err(&rdev->ibdev, "Failed to allocate HW PD");
 709		rc = -ENOMEM;
 710		goto fail;
 711	}
 712
 713	if (udata) {
 714		struct bnxt_re_pd_resp resp = {};
 715
 716		if (!ucntx->dpi.dbr) {
 717			/* Allocate DPI in alloc_pd to avoid failing of
 718			 * ibv_devinfo and family of application when DPIs
 719			 * are depleted.
 720			 */
 721			if (bnxt_qplib_alloc_dpi(&rdev->qplib_res,
 722						 &ucntx->dpi, ucntx, BNXT_QPLIB_DPI_TYPE_UC)) {
 723				rc = -ENOMEM;
 724				goto dbfail;
 725			}
 726		}
 727
 728		resp.pdid = pd->qplib_pd.id;
 729		/* Still allow mapping this DBR to the new user PD. */
 730		resp.dpi = ucntx->dpi.dpi;
 731
 732		entry = bnxt_re_mmap_entry_insert(ucntx, (u64)ucntx->dpi.umdbr,
 733						  BNXT_RE_MMAP_UC_DB, &resp.dbr);
 734
 735		if (!entry) {
 736			rc = -ENOMEM;
 737			goto dbfail;
 738		}
 739
 740		pd->pd_db_mmap = &entry->rdma_entry;
 741
 742		rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
 743		if (rc) {
 744			rdma_user_mmap_entry_remove(pd->pd_db_mmap);
 745			rc = -EFAULT;
 746			goto dbfail;
 747		}
 748	}
 749
 750	if (!udata)
 751		if (bnxt_re_create_fence_mr(pd))
 752			ibdev_warn(&rdev->ibdev,
 753				   "Failed to create Fence-MR\n");
 754	active_pds = atomic_inc_return(&rdev->stats.res.pd_count);
 755	if (active_pds > rdev->stats.res.pd_watermark)
 756		rdev->stats.res.pd_watermark = active_pds;
 757
 758	return 0;
 759dbfail:
 760	bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
 761			      &pd->qplib_pd);
 762fail:
 763	return rc;
 764}
 765
 766/* Address Handles */
 767int bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
 768{
 769	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 770	struct bnxt_re_dev *rdev = ah->rdev;
 771	bool block = true;
 772	int rc;
 773
 774	block = !(flags & RDMA_DESTROY_AH_SLEEPABLE);
 775	rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block);
 776	if (BNXT_RE_CHECK_RC(rc)) {
 777		if (rc == -ETIMEDOUT)
 778			rc = 0;
 779		else
 780			goto fail;
 781	}
 782	atomic_dec(&rdev->stats.res.ah_count);
 783fail:
 784	return rc;
 785}
 786
 787static u8 bnxt_re_stack_to_dev_nw_type(enum rdma_network_type ntype)
 788{
 789	u8 nw_type;
 790
 791	switch (ntype) {
 792	case RDMA_NETWORK_IPV4:
 793		nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
 794		break;
 795	case RDMA_NETWORK_IPV6:
 796		nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
 797		break;
 798	default:
 799		nw_type = CMDQ_CREATE_AH_TYPE_V1;
 800		break;
 801	}
 802	return nw_type;
 803}
 804
 805int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr,
 806		      struct ib_udata *udata)
 807{
 808	struct ib_pd *ib_pd = ib_ah->pd;
 809	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
 810	struct rdma_ah_attr *ah_attr = init_attr->ah_attr;
 811	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
 812	struct bnxt_re_dev *rdev = pd->rdev;
 813	const struct ib_gid_attr *sgid_attr;
 814	struct bnxt_re_gid_ctx *ctx;
 815	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 816	u32 active_ahs;
 817	u8 nw_type;
 818	int rc;
 819
 820	if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
 821		ibdev_err(&rdev->ibdev, "Failed to alloc AH: GRH not set");
 822		return -EINVAL;
 823	}
 824
 825	ah->rdev = rdev;
 826	ah->qplib_ah.pd = &pd->qplib_pd;
 827
 828	/* Supply the configuration for the HW */
 829	memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
 830	       sizeof(union ib_gid));
 831	sgid_attr = grh->sgid_attr;
 832	/* Get the HW context of the GID. The reference
 833	 * of GID table entry is already taken by the caller.
 834	 */
 835	ctx = rdma_read_gid_hw_context(sgid_attr);
 836	ah->qplib_ah.sgid_index = ctx->idx;
 837	ah->qplib_ah.host_sgid_index = grh->sgid_index;
 838	ah->qplib_ah.traffic_class = grh->traffic_class;
 839	ah->qplib_ah.flow_label = grh->flow_label;
 840	ah->qplib_ah.hop_limit = grh->hop_limit;
 841	ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
 842
 843	/* Get network header type for this GID */
 844	nw_type = rdma_gid_attr_network_type(sgid_attr);
 845	ah->qplib_ah.nw_type = bnxt_re_stack_to_dev_nw_type(nw_type);
 846
 847	memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
 848	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah,
 849				  !(init_attr->flags &
 850				    RDMA_CREATE_AH_SLEEPABLE));
 851	if (rc) {
 852		ibdev_err(&rdev->ibdev, "Failed to allocate HW AH");
 853		return rc;
 854	}
 855
 856	/* Write AVID to shared page. */
 857	if (udata) {
 858		struct bnxt_re_ucontext *uctx = rdma_udata_to_drv_context(
 859			udata, struct bnxt_re_ucontext, ib_uctx);
 860		unsigned long flag;
 861		u32 *wrptr;
 862
 863		spin_lock_irqsave(&uctx->sh_lock, flag);
 864		wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
 865		*wrptr = ah->qplib_ah.id;
 866		wmb(); /* make sure cache is updated. */
 867		spin_unlock_irqrestore(&uctx->sh_lock, flag);
 868	}
 869	active_ahs = atomic_inc_return(&rdev->stats.res.ah_count);
 870	if (active_ahs > rdev->stats.res.ah_watermark)
 871		rdev->stats.res.ah_watermark = active_ahs;
 872
 873	return 0;
 874}
 875
 876int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
 877{
 878	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
 879
 880	ah_attr->type = ib_ah->type;
 881	rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
 882	memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
 883	rdma_ah_set_grh(ah_attr, NULL, 0,
 884			ah->qplib_ah.host_sgid_index,
 885			0, ah->qplib_ah.traffic_class);
 886	rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
 887	rdma_ah_set_port_num(ah_attr, 1);
 888	rdma_ah_set_static_rate(ah_attr, 0);
 889	return 0;
 890}
 891
 892unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
 893	__acquires(&qp->scq->cq_lock) __acquires(&qp->rcq->cq_lock)
 894{
 895	unsigned long flags;
 896
 897	spin_lock_irqsave(&qp->scq->cq_lock, flags);
 898	if (qp->rcq != qp->scq)
 899		spin_lock(&qp->rcq->cq_lock);
 900	else
 901		__acquire(&qp->rcq->cq_lock);
 902
 903	return flags;
 904}
 905
 906void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
 907			unsigned long flags)
 908	__releases(&qp->scq->cq_lock) __releases(&qp->rcq->cq_lock)
 909{
 910	if (qp->rcq != qp->scq)
 911		spin_unlock(&qp->rcq->cq_lock);
 912	else
 913		__release(&qp->rcq->cq_lock);
 914	spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
 915}
 916
 917static void bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
 918{
 919	struct bnxt_re_qp *gsi_sqp;
 920	struct bnxt_re_ah *gsi_sah;
 921	struct bnxt_re_dev *rdev;
 922	int rc;
 923
 924	rdev = qp->rdev;
 925	gsi_sqp = rdev->gsi_ctx.gsi_sqp;
 926	gsi_sah = rdev->gsi_ctx.gsi_sah;
 927
 928	ibdev_dbg(&rdev->ibdev, "Destroy the shadow AH\n");
 929	bnxt_qplib_destroy_ah(&rdev->qplib_res,
 930			      &gsi_sah->qplib_ah,
 931			      true);
 932	atomic_dec(&rdev->stats.res.ah_count);
 933	bnxt_qplib_clean_qp(&qp->qplib_qp);
 934
 935	ibdev_dbg(&rdev->ibdev, "Destroy the shadow QP\n");
 936	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 937	if (rc)
 938		ibdev_err(&rdev->ibdev, "Destroy Shadow QP failed");
 939
 940	bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
 941
 942	/* remove from active qp list */
 943	mutex_lock(&rdev->qp_lock);
 944	list_del(&gsi_sqp->list);
 945	mutex_unlock(&rdev->qp_lock);
 946	atomic_dec(&rdev->stats.res.qp_count);
 947
 948	kfree(rdev->gsi_ctx.sqp_tbl);
 949	kfree(gsi_sah);
 950	kfree(gsi_sqp);
 951	rdev->gsi_ctx.gsi_sqp = NULL;
 952	rdev->gsi_ctx.gsi_sah = NULL;
 953	rdev->gsi_ctx.sqp_tbl = NULL;
 954}
 955
 956static void bnxt_re_del_unique_gid(struct bnxt_re_dev *rdev)
 957{
 958	int rc;
 959
 960	if (!rdev->rcfw.roce_mirror)
 961		return;
 962
 963	rc = bnxt_qplib_del_sgid(&rdev->qplib_res.sgid_tbl,
 964				 (struct bnxt_qplib_gid *)&rdev->ugid,
 965				 0xFFFF, true);
 966	if (rc)
 967		dev_err(rdev_to_dev(rdev), "Failed to delete unique GID, rc: %d\n", rc);
 968}
 969
 970/* Queue Pairs */
 971int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
 972{
 973	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
 974	struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
 975	struct bnxt_re_dev *rdev = qp->rdev;
 976	struct bnxt_qplib_nq *scq_nq = NULL;
 977	struct bnxt_qplib_nq *rcq_nq = NULL;
 978	unsigned int flags;
 979	int rc;
 980
 981	bnxt_re_debug_rem_qpinfo(rdev, qp);
 982
 983	bnxt_qplib_flush_cqn_wq(&qp->qplib_qp);
 984
 985	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
 986	if (rc)
 987		ibdev_err(&rdev->ibdev, "Failed to destroy HW QP");
 988
 989	if (rdma_is_kernel_res(&qp->ib_qp.res)) {
 990		flags = bnxt_re_lock_cqs(qp);
 991		bnxt_qplib_clean_qp(&qp->qplib_qp);
 992		bnxt_re_unlock_cqs(qp, flags);
 993	}
 994
 995	bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
 996
 997	if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
 998		bnxt_re_destroy_gsi_sqp(qp);
 999
1000	mutex_lock(&rdev->qp_lock);
1001	list_del(&qp->list);
1002	mutex_unlock(&rdev->qp_lock);
1003	atomic_dec(&rdev->stats.res.qp_count);
1004	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)
1005		atomic_dec(&rdev->stats.res.rc_qp_count);
1006	else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)
1007		atomic_dec(&rdev->stats.res.ud_qp_count);
1008
1009	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE)
1010		bnxt_re_del_unique_gid(rdev);
1011
1012	ib_umem_release(qp->rumem);
1013	ib_umem_release(qp->sumem);
1014
1015	/* Flush all the entries of notification queue associated with
1016	 * given qp.
1017	 */
1018	scq_nq = qplib_qp->scq->nq;
1019	rcq_nq = qplib_qp->rcq->nq;
1020	bnxt_re_synchronize_nq(scq_nq);
1021	if (scq_nq != rcq_nq)
1022		bnxt_re_synchronize_nq(rcq_nq);
1023
1024	return 0;
1025}
1026
1027static u8 __from_ib_qp_type(enum ib_qp_type type)
1028{
1029	switch (type) {
1030	case IB_QPT_GSI:
1031		return CMDQ_CREATE_QP1_TYPE_GSI;
1032	case IB_QPT_RC:
1033		return CMDQ_CREATE_QP_TYPE_RC;
1034	case IB_QPT_UD:
1035		return CMDQ_CREATE_QP_TYPE_UD;
1036	case IB_QPT_RAW_PACKET:
1037		return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE;
1038	default:
1039		return IB_QPT_MAX;
1040	}
1041}
1042
1043static u16 bnxt_re_setup_rwqe_size(struct bnxt_qplib_qp *qplqp,
1044				   int rsge, int max)
1045{
1046	if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
1047		rsge = max;
1048	return bnxt_re_get_rwqe_size(rsge);
1049}
1050
1051static u16 bnxt_re_get_wqe_size(int ilsize, int nsge)
1052{
1053	u16 wqe_size, calc_ils;
1054
1055	wqe_size = bnxt_re_get_swqe_size(nsge);
1056	if (ilsize) {
1057		calc_ils = sizeof(struct sq_send_hdr) + ilsize;
1058		wqe_size = max_t(u16, calc_ils, wqe_size);
1059		wqe_size = ALIGN(wqe_size, sizeof(struct sq_send_hdr));
1060	}
1061	return wqe_size;
1062}
1063
1064static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
1065				   struct ib_qp_init_attr *init_attr)
1066{
1067	struct bnxt_qplib_dev_attr *dev_attr;
1068	struct bnxt_qplib_qp *qplqp;
1069	struct bnxt_re_dev *rdev;
1070	struct bnxt_qplib_q *sq;
1071	int align, ilsize;
1072
1073	rdev = qp->rdev;
1074	qplqp = &qp->qplib_qp;
1075	sq = &qplqp->sq;
1076	dev_attr = rdev->dev_attr;
1077
1078	align = sizeof(struct sq_send_hdr);
1079	ilsize = ALIGN(init_attr->cap.max_inline_data, align);
1080
1081	/* For gen p4 and gen p5 fixed wqe compatibility mode
1082	 * wqe size is fixed to 128 bytes - ie 6 SGEs
1083	 */
1084	if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) {
1085		sq->wqe_size = bnxt_re_get_swqe_size(BNXT_STATIC_MAX_SGE);
1086		sq->max_sge = BNXT_STATIC_MAX_SGE;
1087	} else {
1088		sq->wqe_size = bnxt_re_get_wqe_size(ilsize, sq->max_sge);
1089		if (sq->wqe_size > bnxt_re_get_swqe_size(dev_attr->max_qp_sges))
1090			return -EINVAL;
1091	}
1092
1093	if (init_attr->cap.max_inline_data) {
1094		qplqp->max_inline_data = sq->wqe_size -
1095			sizeof(struct sq_send_hdr);
1096		init_attr->cap.max_inline_data = qplqp->max_inline_data;
1097	}
1098
1099	return 0;
1100}
1101
1102static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
1103				struct bnxt_re_qp *qp, struct bnxt_re_ucontext *cntx,
1104				struct bnxt_re_qp_req *ureq)
1105{
1106	struct bnxt_qplib_qp *qplib_qp;
1107	int bytes = 0, psn_sz;
1108	struct ib_umem *umem;
1109	int psn_nume;
1110
1111	qplib_qp = &qp->qplib_qp;
1112
1113	bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
1114	/* Consider mapping PSN search memory only for RC QPs. */
1115	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
1116		psn_sz = bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx) ?
1117						   sizeof(struct sq_psn_search_ext) :
1118						   sizeof(struct sq_psn_search);
1119		if (cntx && bnxt_re_is_var_size_supported(rdev, cntx)) {
1120			psn_nume = ureq->sq_slots;
1121		} else {
1122			psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
1123			qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
1124				 sizeof(struct bnxt_qplib_sge));
1125		}
1126		if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2))
1127			psn_nume = roundup_pow_of_two(psn_nume);
1128		bytes += (psn_nume * psn_sz);
1129	}
1130
1131	bytes = PAGE_ALIGN(bytes);
1132	umem = ib_umem_get(&rdev->ibdev, ureq->qpsva, bytes,
1133			   IB_ACCESS_LOCAL_WRITE);
1134	if (IS_ERR(umem))
1135		return PTR_ERR(umem);
1136
1137	qp->sumem = umem;
1138	qplib_qp->sq.sg_info.umem = umem;
1139	qplib_qp->sq.sg_info.pgsize = PAGE_SIZE;
1140	qplib_qp->sq.sg_info.pgshft = PAGE_SHIFT;
1141	qplib_qp->qp_handle = ureq->qp_handle;
1142
1143	if (!qp->qplib_qp.srq) {
1144		bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
1145		bytes = PAGE_ALIGN(bytes);
1146		umem = ib_umem_get(&rdev->ibdev, ureq->qprva, bytes,
1147				   IB_ACCESS_LOCAL_WRITE);
1148		if (IS_ERR(umem))
1149			goto rqfail;
1150		qp->rumem = umem;
1151		qplib_qp->rq.sg_info.umem = umem;
1152		qplib_qp->rq.sg_info.pgsize = PAGE_SIZE;
1153		qplib_qp->rq.sg_info.pgshft = PAGE_SHIFT;
1154	}
1155
1156	qplib_qp->dpi = &cntx->dpi;
1157	return 0;
1158rqfail:
1159	ib_umem_release(qp->sumem);
1160	qp->sumem = NULL;
1161	memset(&qplib_qp->sq.sg_info, 0, sizeof(qplib_qp->sq.sg_info));
1162
1163	return PTR_ERR(umem);
1164}
1165
1166static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
1167				(struct bnxt_re_pd *pd,
1168				 struct bnxt_qplib_res *qp1_res,
1169				 struct bnxt_qplib_qp *qp1_qp)
1170{
1171	struct bnxt_re_dev *rdev = pd->rdev;
1172	struct bnxt_re_ah *ah;
1173	union ib_gid sgid;
1174	int rc;
1175
1176	ah = kzalloc(sizeof(*ah), GFP_KERNEL);
1177	if (!ah)
1178		return NULL;
1179
1180	ah->rdev = rdev;
1181	ah->qplib_ah.pd = &pd->qplib_pd;
1182
1183	rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
1184	if (rc)
1185		goto fail;
1186
1187	/* supply the dgid data same as sgid */
1188	memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
1189	       sizeof(union ib_gid));
1190	ah->qplib_ah.sgid_index = 0;
1191
1192	ah->qplib_ah.traffic_class = 0;
1193	ah->qplib_ah.flow_label = 0;
1194	ah->qplib_ah.hop_limit = 1;
1195	ah->qplib_ah.sl = 0;
1196	/* Have DMAC same as SMAC */
1197	ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
1198
1199	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, false);
1200	if (rc) {
1201		ibdev_err(&rdev->ibdev,
1202			  "Failed to allocate HW AH for Shadow QP");
1203		goto fail;
1204	}
1205	atomic_inc(&rdev->stats.res.ah_count);
1206
1207	return ah;
1208
1209fail:
1210	kfree(ah);
1211	return NULL;
1212}
1213
1214static struct bnxt_re_qp *bnxt_re_create_shadow_qp
1215				(struct bnxt_re_pd *pd,
1216				 struct bnxt_qplib_res *qp1_res,
1217				 struct bnxt_qplib_qp *qp1_qp)
1218{
1219	struct bnxt_re_dev *rdev = pd->rdev;
1220	struct bnxt_re_qp *qp;
1221	int rc;
1222
1223	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1224	if (!qp)
1225		return NULL;
1226
1227	qp->rdev = rdev;
1228
1229	/* Initialize the shadow QP structure from the QP1 values */
1230	ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
1231
1232	qp->qplib_qp.pd = &pd->qplib_pd;
1233	qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
1234	qp->qplib_qp.type = IB_QPT_UD;
1235
1236	qp->qplib_qp.max_inline_data = 0;
1237	qp->qplib_qp.sig_type = true;
1238
1239	/* Shadow QP SQ depth should be same as QP1 RQ depth */
1240	qp->qplib_qp.sq.wqe_size = bnxt_re_get_wqe_size(0, 6);
1241	qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
1242	qp->qplib_qp.sq.max_sw_wqe = qp1_qp->rq.max_wqe;
1243	qp->qplib_qp.sq.max_sge = 2;
1244	/* Q full delta can be 1 since it is internal QP */
1245	qp->qplib_qp.sq.q_full_delta = 1;
1246	qp->qplib_qp.sq.sg_info.pgsize = PAGE_SIZE;
1247	qp->qplib_qp.sq.sg_info.pgshft = PAGE_SHIFT;
1248
1249	qp->qplib_qp.scq = qp1_qp->scq;
1250	qp->qplib_qp.rcq = qp1_qp->rcq;
1251
1252	qp->qplib_qp.rq.wqe_size = bnxt_re_get_rwqe_size(6);
1253	qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
1254	qp->qplib_qp.rq.max_sw_wqe = qp1_qp->rq.max_wqe;
1255	qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
1256	/* Q full delta can be 1 since it is internal QP */
1257	qp->qplib_qp.rq.q_full_delta = 1;
1258	qp->qplib_qp.rq.sg_info.pgsize = PAGE_SIZE;
1259	qp->qplib_qp.rq.sg_info.pgshft = PAGE_SHIFT;
1260
1261	qp->qplib_qp.mtu = qp1_qp->mtu;
1262
1263	qp->qplib_qp.sq_hdr_buf_size = 0;
1264	qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
1265	qp->qplib_qp.dpi = &rdev->dpi_privileged;
1266
1267	rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
1268	if (rc)
1269		goto fail;
1270
1271	spin_lock_init(&qp->sq_lock);
1272	INIT_LIST_HEAD(&qp->list);
1273	mutex_lock(&rdev->qp_lock);
1274	list_add_tail(&qp->list, &rdev->qp_list);
1275	atomic_inc(&rdev->stats.res.qp_count);
1276	mutex_unlock(&rdev->qp_lock);
1277	return qp;
1278fail:
1279	kfree(qp);
1280	return NULL;
1281}
1282
1283static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
1284				struct ib_qp_init_attr *init_attr,
1285				struct bnxt_re_ucontext *uctx)
1286{
1287	struct bnxt_qplib_dev_attr *dev_attr;
1288	struct bnxt_qplib_qp *qplqp;
1289	struct bnxt_re_dev *rdev;
1290	struct bnxt_qplib_q *rq;
1291	int entries;
1292
1293	rdev = qp->rdev;
1294	qplqp = &qp->qplib_qp;
1295	rq = &qplqp->rq;
1296	dev_attr = rdev->dev_attr;
1297
1298	if (init_attr->srq) {
1299		struct bnxt_re_srq *srq;
1300
1301		srq = container_of(init_attr->srq, struct bnxt_re_srq, ib_srq);
1302		qplqp->srq = &srq->qplib_srq;
1303		rq->max_wqe = 0;
1304	} else {
1305		rq->max_sge = init_attr->cap.max_recv_sge;
1306		if (rq->max_sge > dev_attr->max_qp_sges)
1307			rq->max_sge = dev_attr->max_qp_sges;
1308		init_attr->cap.max_recv_sge = rq->max_sge;
1309		rq->wqe_size = bnxt_re_setup_rwqe_size(qplqp, rq->max_sge,
1310						       dev_attr->max_qp_sges);
1311		/* Allocate 1 more than what's provided so posting max doesn't
1312		 * mean empty.
1313		 */
1314		entries = bnxt_re_init_depth(init_attr->cap.max_recv_wr + 1, uctx);
1315		rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1316		rq->max_sw_wqe = rq->max_wqe;
1317		rq->q_full_delta = 0;
1318		rq->sg_info.pgsize = PAGE_SIZE;
1319		rq->sg_info.pgshft = PAGE_SHIFT;
1320	}
1321
1322	return 0;
1323}
1324
1325static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
1326{
1327	struct bnxt_qplib_dev_attr *dev_attr;
1328	struct bnxt_qplib_qp *qplqp;
1329	struct bnxt_re_dev *rdev;
1330
1331	rdev = qp->rdev;
1332	qplqp = &qp->qplib_qp;
1333	dev_attr = rdev->dev_attr;
1334
1335	if (!bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) {
1336		qplqp->rq.max_sge = dev_attr->max_qp_sges;
1337		if (qplqp->rq.max_sge > dev_attr->max_qp_sges)
1338			qplqp->rq.max_sge = dev_attr->max_qp_sges;
1339		qplqp->rq.max_sge = 6;
1340	}
1341}
1342
1343static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
1344				struct ib_qp_init_attr *init_attr,
1345				struct bnxt_re_ucontext *uctx,
1346				struct bnxt_re_qp_req *ureq)
1347{
1348	struct bnxt_qplib_dev_attr *dev_attr;
1349	struct bnxt_qplib_qp *qplqp;
1350	struct bnxt_re_dev *rdev;
1351	struct bnxt_qplib_q *sq;
1352	int diff = 0;
1353	int entries;
1354	int rc;
1355
1356	rdev = qp->rdev;
1357	qplqp = &qp->qplib_qp;
1358	sq = &qplqp->sq;
1359	dev_attr = rdev->dev_attr;
1360
1361	sq->max_sge = init_attr->cap.max_send_sge;
1362	entries = init_attr->cap.max_send_wr;
1363	if (uctx && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
1364		sq->max_wqe = ureq->sq_slots;
1365		sq->max_sw_wqe = ureq->sq_slots;
1366		sq->wqe_size = sizeof(struct sq_sge);
1367	} else {
1368		if (sq->max_sge > dev_attr->max_qp_sges) {
1369			sq->max_sge = dev_attr->max_qp_sges;
1370			init_attr->cap.max_send_sge = sq->max_sge;
1371		}
1372
1373		rc = bnxt_re_setup_swqe_size(qp, init_attr);
1374		if (rc)
1375			return rc;
1376
1377		/* Allocate 128 + 1 more than what's provided */
1378		diff = (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) ?
1379			0 : BNXT_QPLIB_RESERVED_QP_WRS;
1380		entries = bnxt_re_init_depth(entries + diff + 1, uctx);
1381		sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
1382		if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
1383			sq->max_sw_wqe = bnxt_qplib_get_depth(sq, qplqp->wqe_mode, true);
1384		else
1385			sq->max_sw_wqe = sq->max_wqe;
1386
1387	}
1388	sq->q_full_delta = diff + 1;
1389	/*
1390	 * Reserving one slot for Phantom WQE. Application can
1391	 * post one extra entry in this case. But allowing this to avoid
1392	 * unexpected Queue full condition
1393	 */
1394	qplqp->sq.q_full_delta -= 1;
1395	qplqp->sq.sg_info.pgsize = PAGE_SIZE;
1396	qplqp->sq.sg_info.pgshft = PAGE_SHIFT;
1397
1398	return 0;
1399}
1400
1401static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
1402				       struct ib_qp_init_attr *init_attr,
1403				       struct bnxt_re_ucontext *uctx)
1404{
1405	struct bnxt_qplib_dev_attr *dev_attr;
1406	struct bnxt_qplib_qp *qplqp;
1407	struct bnxt_re_dev *rdev;
1408	int entries;
1409
1410	rdev = qp->rdev;
1411	qplqp = &qp->qplib_qp;
1412	dev_attr = rdev->dev_attr;
1413
1414	if (!bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx)) {
1415		entries = bnxt_re_init_depth(init_attr->cap.max_send_wr + 1, uctx);
1416		qplqp->sq.max_wqe = min_t(u32, entries,
1417					  dev_attr->max_qp_wqes + 1);
1418		qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
1419			init_attr->cap.max_send_wr;
1420		qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
1421		if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
1422			qplqp->sq.max_sge = dev_attr->max_qp_sges;
1423	}
1424}
1425
1426static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
1427				struct ib_qp_init_attr *init_attr)
1428{
1429	struct bnxt_qplib_chip_ctx *chip_ctx;
1430	int qptype;
1431
1432	chip_ctx = rdev->chip_ctx;
1433
1434	qptype = __from_ib_qp_type(init_attr->qp_type);
1435	if (qptype == IB_QPT_MAX) {
1436		ibdev_err(&rdev->ibdev, "QP type 0x%x not supported", qptype);
1437		qptype = -EOPNOTSUPP;
1438		goto out;
1439	}
1440
1441	if (bnxt_qplib_is_chip_gen_p5_p7(chip_ctx) &&
1442	    init_attr->qp_type == IB_QPT_GSI)
1443		qptype = CMDQ_CREATE_QP_TYPE_GSI;
1444out:
1445	return qptype;
1446}
1447
1448static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1449				struct ib_qp_init_attr *init_attr,
1450				struct bnxt_re_ucontext *uctx,
1451				struct bnxt_re_qp_req *ureq)
1452{
1453	struct bnxt_qplib_dev_attr *dev_attr;
1454	struct bnxt_qplib_qp *qplqp;
1455	struct bnxt_re_dev *rdev;
1456	struct bnxt_re_cq *cq;
1457	int rc = 0, qptype;
1458
1459	rdev = qp->rdev;
1460	qplqp = &qp->qplib_qp;
1461	dev_attr = rdev->dev_attr;
1462
1463	/* Setup misc params */
1464	ether_addr_copy(qplqp->smac, rdev->netdev->dev_addr);
1465	qplqp->pd = &pd->qplib_pd;
1466	qplqp->qp_handle = (u64)qplqp;
1467	qplqp->max_inline_data = init_attr->cap.max_inline_data;
1468	qplqp->sig_type = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR;
1469	qptype = bnxt_re_init_qp_type(rdev, init_attr);
1470	if (qptype < 0) {
1471		rc = qptype;
1472		goto out;
1473	}
1474	qplqp->type = (u8)qptype;
1475	qplqp->wqe_mode = bnxt_re_is_var_size_supported(rdev, uctx);
1476	if (init_attr->qp_type == IB_QPT_RC) {
1477		qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
1478		qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1479	}
1480	qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
1481	qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
1482	if (init_attr->create_flags) {
1483		ibdev_dbg(&rdev->ibdev,
1484			  "QP create flags 0x%x not supported",
1485			  init_attr->create_flags);
1486		return -EOPNOTSUPP;
1487	}
1488
1489	/* Setup CQs */
1490	if (init_attr->send_cq) {
1491		cq = container_of(init_attr->send_cq, struct bnxt_re_cq, ib_cq);
1492		qplqp->scq = &cq->qplib_cq;
1493		qp->scq = cq;
1494	}
1495
1496	if (init_attr->recv_cq) {
1497		cq = container_of(init_attr->recv_cq, struct bnxt_re_cq, ib_cq);
1498		qplqp->rcq = &cq->qplib_cq;
1499		qp->rcq = cq;
1500	}
1501
1502	/* Setup RQ/SRQ */
1503	rc = bnxt_re_init_rq_attr(qp, init_attr, uctx);
1504	if (rc)
1505		goto out;
1506	if (init_attr->qp_type == IB_QPT_GSI)
1507		bnxt_re_adjust_gsi_rq_attr(qp);
1508
1509	/* Setup SQ */
1510	rc = bnxt_re_init_sq_attr(qp, init_attr, uctx, ureq);
1511	if (rc)
1512		goto out;
1513	if (init_attr->qp_type == IB_QPT_GSI)
1514		bnxt_re_adjust_gsi_sq_attr(qp, init_attr, uctx);
1515
1516	if (uctx) /* This will update DPI and qp_handle */
1517		rc = bnxt_re_init_user_qp(rdev, pd, qp, uctx, ureq);
1518out:
1519	return rc;
1520}
1521
1522static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
1523				     struct bnxt_re_pd *pd)
1524{
1525	struct bnxt_re_sqp_entries *sqp_tbl;
1526	struct bnxt_re_dev *rdev;
1527	struct bnxt_re_qp *sqp;
1528	struct bnxt_re_ah *sah;
1529	int rc = 0;
1530
1531	rdev = qp->rdev;
1532	/* Create a shadow QP to handle the QP1 traffic */
1533	sqp_tbl = kcalloc(BNXT_RE_MAX_GSI_SQP_ENTRIES, sizeof(*sqp_tbl),
1534			  GFP_KERNEL);
1535	if (!sqp_tbl)
1536		return -ENOMEM;
1537	rdev->gsi_ctx.sqp_tbl = sqp_tbl;
1538
1539	sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
1540	if (!sqp) {
1541		rc = -ENODEV;
1542		ibdev_err(&rdev->ibdev, "Failed to create Shadow QP for QP1");
1543		goto out;
1544	}
1545	rdev->gsi_ctx.gsi_sqp = sqp;
1546
1547	sqp->rcq = qp->rcq;
1548	sqp->scq = qp->scq;
1549	sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
1550					  &qp->qplib_qp);
1551	if (!sah) {
1552		bnxt_qplib_destroy_qp(&rdev->qplib_res,
1553				      &sqp->qplib_qp);
1554		rc = -ENODEV;
1555		ibdev_err(&rdev->ibdev,
1556			  "Failed to create AH entry for ShadowQP");
1557		goto out;
1558	}
1559	rdev->gsi_ctx.gsi_sah = sah;
1560
1561	return 0;
1562out:
1563	kfree(sqp_tbl);
1564	return rc;
1565}
1566
1567static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
1568				 struct ib_qp_init_attr *init_attr)
1569{
1570	struct bnxt_re_dev *rdev;
1571	struct bnxt_qplib_qp *qplqp;
1572	int rc;
1573
1574	rdev = qp->rdev;
1575	qplqp = &qp->qplib_qp;
1576
1577	qplqp->rq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1578	qplqp->sq_hdr_buf_size = BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
1579
1580	rc = bnxt_qplib_create_qp1(&rdev->qplib_res, qplqp);
1581	if (rc) {
1582		ibdev_err(&rdev->ibdev, "create HW QP1 failed!");
1583		goto out;
1584	}
1585
1586	rc = bnxt_re_create_shadow_gsi(qp, pd);
1587out:
1588	return rc;
1589}
1590
1591static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
1592				   struct ib_qp_init_attr *init_attr,
1593				   struct bnxt_qplib_dev_attr *dev_attr)
1594{
1595	bool rc = true;
1596
1597	if (init_attr->cap.max_send_wr > dev_attr->max_qp_wqes ||
1598	    init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes ||
1599	    init_attr->cap.max_send_sge > dev_attr->max_qp_sges ||
1600	    init_attr->cap.max_recv_sge > dev_attr->max_qp_sges ||
1601	    init_attr->cap.max_inline_data > dev_attr->max_inline_data) {
1602		ibdev_err(&rdev->ibdev,
1603			  "Create QP failed - max exceeded! 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x 0x%x/0x%x",
1604			  init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
1605			  init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
1606			  init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
1607			  init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
1608			  init_attr->cap.max_inline_data,
1609			  dev_attr->max_inline_data);
1610		rc = false;
1611	}
1612	return rc;
1613}
1614
1615static int bnxt_re_add_unique_gid(struct bnxt_re_dev *rdev)
1616{
1617	struct bnxt_qplib_ctx *hctx = &rdev->qplib_ctx;
1618	struct bnxt_qplib_res *res = &rdev->qplib_res;
1619	int rc;
1620
1621	if (!rdev->rcfw.roce_mirror)
1622		return 0;
1623
1624	rdev->ugid.global.subnet_prefix = cpu_to_be64(0xfe8000000000abcdLL);
1625	addrconf_ifid_eui48(&rdev->ugid.raw[8], rdev->netdev);
1626
1627	rc = bnxt_qplib_add_sgid(&res->sgid_tbl,
1628				 (struct bnxt_qplib_gid *)&rdev->ugid,
1629				 rdev->qplib_res.netdev->dev_addr,
1630				 0xFFFF, true, &rdev->ugid_index, true,
1631				 hctx->stats3.fw_id);
1632	if (rc)
1633		dev_err(rdev_to_dev(rdev), "Failed to add unique GID. rc = %d\n", rc);
1634
1635	return rc;
1636}
1637
1638int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
1639		      struct ib_udata *udata)
1640{
1641	struct bnxt_qplib_dev_attr *dev_attr;
1642	struct bnxt_re_ucontext *uctx;
1643	struct bnxt_re_qp_req ureq;
1644	struct bnxt_re_dev *rdev;
1645	struct bnxt_re_pd *pd;
1646	struct bnxt_re_qp *qp;
1647	struct ib_pd *ib_pd;
1648	u32 active_qps;
1649	int rc;
1650
1651	ib_pd = ib_qp->pd;
1652	pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1653	rdev = pd->rdev;
1654	dev_attr = rdev->dev_attr;
1655	qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1656
1657	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
1658	if (udata)
1659		if (ib_copy_from_udata(&ureq, udata,  min(udata->inlen, sizeof(ureq))))
1660			return -EFAULT;
1661
1662	rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
1663	if (!rc) {
1664		rc = -EINVAL;
1665		goto fail;
1666	}
1667
1668	qp->rdev = rdev;
1669	rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, uctx, &ureq);
1670	if (rc)
1671		goto fail;
1672
1673	if (qp_init_attr->qp_type == IB_QPT_GSI &&
1674	    !(bnxt_qplib_is_chip_gen_p5_p7(rdev->chip_ctx))) {
1675		rc = bnxt_re_create_gsi_qp(qp, pd, qp_init_attr);
1676		if (rc == -ENODEV)
1677			goto qp_destroy;
1678		if (rc)
1679			goto fail;
1680	} else {
1681		rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1682		if (rc) {
1683			ibdev_err(&rdev->ibdev, "Failed to create HW QP");
1684			goto free_umem;
1685		}
1686		if (udata) {
1687			struct bnxt_re_qp_resp resp;
1688
1689			resp.qpid = qp->qplib_qp.id;
1690			resp.rsvd = 0;
1691			rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1692			if (rc) {
1693				ibdev_err(&rdev->ibdev, "Failed to copy QP udata");
1694				goto qp_destroy;
1695			}
1696		}
1697	}
1698
1699	/* Support for RawEth QP is added to capture TCP pkt dump.
1700	 * So unique SGID is used to avoid incorrect statistics on per
1701	 * function stats_ctx
1702	 */
1703	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE) {
1704		rc = bnxt_re_add_unique_gid(rdev);
1705		if (rc)
1706			goto qp_destroy;
1707		qp->qplib_qp.ugid_index = rdev->ugid_index;
1708	}
1709
1710	qp->ib_qp.qp_num = qp->qplib_qp.id;
1711	if (qp_init_attr->qp_type == IB_QPT_GSI)
1712		rdev->gsi_ctx.gsi_qp = qp;
1713	spin_lock_init(&qp->sq_lock);
1714	spin_lock_init(&qp->rq_lock);
1715	INIT_LIST_HEAD(&qp->list);
1716	mutex_lock(&rdev->qp_lock);
1717	list_add_tail(&qp->list, &rdev->qp_list);
1718	mutex_unlock(&rdev->qp_lock);
1719	active_qps = atomic_inc_return(&rdev->stats.res.qp_count);
1720	if (active_qps > rdev->stats.res.qp_watermark)
1721		rdev->stats.res.qp_watermark = active_qps;
1722	if (qp_init_attr->qp_type == IB_QPT_RC) {
1723		active_qps = atomic_inc_return(&rdev->stats.res.rc_qp_count);
1724		if (active_qps > rdev->stats.res.rc_qp_watermark)
1725			rdev->stats.res.rc_qp_watermark = active_qps;
1726	} else if (qp_init_attr->qp_type == IB_QPT_UD) {
1727		active_qps = atomic_inc_return(&rdev->stats.res.ud_qp_count);
1728		if (active_qps > rdev->stats.res.ud_qp_watermark)
1729			rdev->stats.res.ud_qp_watermark = active_qps;
1730	}
1731	bnxt_re_debug_add_qpinfo(rdev, qp);
1732
1733	return 0;
1734qp_destroy:
1735	bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1736free_umem:
1737	ib_umem_release(qp->rumem);
1738	ib_umem_release(qp->sumem);
1739fail:
1740	return rc;
1741}
1742
1743static u8 __from_ib_qp_state(enum ib_qp_state state)
1744{
1745	switch (state) {
1746	case IB_QPS_RESET:
1747		return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1748	case IB_QPS_INIT:
1749		return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1750	case IB_QPS_RTR:
1751		return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1752	case IB_QPS_RTS:
1753		return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1754	case IB_QPS_SQD:
1755		return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1756	case IB_QPS_SQE:
1757		return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1758	case IB_QPS_ERR:
1759	default:
1760		return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1761	}
1762}
1763
1764static enum ib_qp_state __to_ib_qp_state(u8 state)
1765{
1766	switch (state) {
1767	case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1768		return IB_QPS_RESET;
1769	case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1770		return IB_QPS_INIT;
1771	case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1772		return IB_QPS_RTR;
1773	case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1774		return IB_QPS_RTS;
1775	case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1776		return IB_QPS_SQD;
1777	case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1778		return IB_QPS_SQE;
1779	case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1780	default:
1781		return IB_QPS_ERR;
1782	}
1783}
1784
1785static u32 __from_ib_mtu(enum ib_mtu mtu)
1786{
1787	switch (mtu) {
1788	case IB_MTU_256:
1789		return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1790	case IB_MTU_512:
1791		return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1792	case IB_MTU_1024:
1793		return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1794	case IB_MTU_2048:
1795		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1796	case IB_MTU_4096:
1797		return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1798	default:
1799		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1800	}
1801}
1802
1803static enum ib_mtu __to_ib_mtu(u32 mtu)
1804{
1805	switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1806	case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1807		return IB_MTU_256;
1808	case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1809		return IB_MTU_512;
1810	case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1811		return IB_MTU_1024;
1812	case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1813		return IB_MTU_2048;
1814	case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1815		return IB_MTU_4096;
1816	default:
1817		return IB_MTU_2048;
1818	}
1819}
1820
1821/* Shared Receive Queues */
1822int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
1823{
1824	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1825					       ib_srq);
1826	struct bnxt_re_dev *rdev = srq->rdev;
1827	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1828
1829	if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
1830		free_page((unsigned long)srq->uctx_srq_page);
1831		hash_del(&srq->hash_entry);
1832	}
1833	bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
1834	ib_umem_release(srq->umem);
1835	atomic_dec(&rdev->stats.res.srq_count);
1836	return 0;
1837}
1838
1839static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
1840				 struct bnxt_re_pd *pd,
1841				 struct bnxt_re_srq *srq,
1842				 struct ib_udata *udata)
1843{
1844	struct bnxt_re_srq_req ureq;
1845	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
1846	struct ib_umem *umem;
1847	int bytes = 0;
1848	struct bnxt_re_ucontext *cntx = rdma_udata_to_drv_context(
1849		udata, struct bnxt_re_ucontext, ib_uctx);
1850
1851	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1852		return -EFAULT;
1853
1854	bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
1855	bytes = PAGE_ALIGN(bytes);
1856	umem = ib_umem_get(&rdev->ibdev, ureq.srqva, bytes,
1857			   IB_ACCESS_LOCAL_WRITE);
1858	if (IS_ERR(umem))
1859		return PTR_ERR(umem);
1860
1861	srq->umem = umem;
1862	qplib_srq->sg_info.umem = umem;
1863	qplib_srq->sg_info.pgsize = PAGE_SIZE;
1864	qplib_srq->sg_info.pgshft = PAGE_SHIFT;
1865	qplib_srq->srq_handle = ureq.srq_handle;
1866	qplib_srq->dpi = &cntx->dpi;
1867
1868	return 0;
1869}
1870
1871int bnxt_re_create_srq(struct ib_srq *ib_srq,
1872		       struct ib_srq_init_attr *srq_init_attr,
1873		       struct ib_udata *udata)
1874{
1875	struct bnxt_qplib_dev_attr *dev_attr;
1876	struct bnxt_re_ucontext *uctx;
1877	struct bnxt_re_dev *rdev;
1878	struct bnxt_re_srq *srq;
1879	struct bnxt_re_pd *pd;
1880	struct ib_pd *ib_pd;
1881	u32 active_srqs;
1882	int rc, entries;
1883
1884	ib_pd = ib_srq->pd;
1885	pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
1886	rdev = pd->rdev;
1887	dev_attr = rdev->dev_attr;
1888	srq = container_of(ib_srq, struct bnxt_re_srq, ib_srq);
1889
1890	if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
1891		ibdev_err(&rdev->ibdev, "Create CQ failed - max exceeded");
1892		rc = -EINVAL;
1893		goto exit;
1894	}
1895
1896	if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
1897		rc = -EOPNOTSUPP;
1898		goto exit;
1899	}
1900
1901	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
1902	srq->rdev = rdev;
1903	srq->qplib_srq.pd = &pd->qplib_pd;
1904	srq->qplib_srq.dpi = &rdev->dpi_privileged;
1905	/* Allocate 1 more than what's provided so posting max doesn't
1906	 * mean empty
1907	 */
1908	entries = bnxt_re_init_depth(srq_init_attr->attr.max_wr + 1, uctx);
1909	if (entries > dev_attr->max_srq_wqes + 1)
1910		entries = dev_attr->max_srq_wqes + 1;
1911	srq->qplib_srq.max_wqe = entries;
1912
1913	srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
1914	 /* 128 byte wqe size for SRQ . So use max sges */
1915	srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
1916	srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
1917	srq->srq_limit = srq_init_attr->attr.srq_limit;
1918	srq->qplib_srq.eventq_hw_ring_id = rdev->nqr->nq[0].ring_id;
1919	srq->qplib_srq.sg_info.pgsize = PAGE_SIZE;
1920	srq->qplib_srq.sg_info.pgshft = PAGE_SHIFT;
1921
1922	if (udata) {
1923		rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
1924		if (rc)
1925			goto fail;
1926	}
1927
1928	rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
1929	if (rc) {
1930		ibdev_err(&rdev->ibdev, "Create HW SRQ failed!");
1931		goto fail;
1932	}
1933
1934	if (udata) {
1935		struct bnxt_re_srq_resp resp = {};
1936
1937		resp.srqid = srq->qplib_srq.id;
1938		if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
1939			hash_add(rdev->srq_hash, &srq->hash_entry, srq->qplib_srq.id);
1940			srq->uctx_srq_page = (void *)get_zeroed_page(GFP_KERNEL);
1941			if (!srq->uctx_srq_page) {
1942				rc = -ENOMEM;
1943				goto fail;
1944			}
1945			resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT;
1946		}
1947		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1948		if (rc) {
1949			ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
1950			bnxt_qplib_destroy_srq(&rdev->qplib_res,
1951					       &srq->qplib_srq);
1952			goto fail;
1953		}
1954	}
1955	active_srqs = atomic_inc_return(&rdev->stats.res.srq_count);
1956	if (active_srqs > rdev->stats.res.srq_watermark)
1957		rdev->stats.res.srq_watermark = active_srqs;
1958	spin_lock_init(&srq->lock);
1959
1960	return 0;
1961
1962fail:
1963	ib_umem_release(srq->umem);
1964exit:
1965	return rc;
1966}
1967
1968int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
1969		       enum ib_srq_attr_mask srq_attr_mask,
1970		       struct ib_udata *udata)
1971{
1972	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
1973					       ib_srq);
1974	struct bnxt_re_dev *rdev = srq->rdev;
1975
1976	switch (srq_attr_mask) {
1977	case IB_SRQ_MAX_WR:
1978		/* SRQ resize is not supported */
1979		return -EINVAL;
1980	case IB_SRQ_LIMIT:
1981		/* Change the SRQ threshold */
1982		if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
1983			return -EINVAL;
1984
1985		srq->qplib_srq.threshold = srq_attr->srq_limit;
1986		bnxt_qplib_srq_arm_db(&srq->qplib_srq.dbinfo, srq->qplib_srq.threshold);
1987
1988		/* On success, update the shadow */
1989		srq->srq_limit = srq_attr->srq_limit;
1990		/* No need to Build and send response back to udata */
1991		return 0;
1992	default:
1993		ibdev_err(&rdev->ibdev,
1994			  "Unsupported srq_attr_mask 0x%x", srq_attr_mask);
1995		return -EINVAL;
1996	}
1997}
1998
1999int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
2000{
2001	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
2002					       ib_srq);
2003	struct bnxt_re_srq tsrq;
2004	struct bnxt_re_dev *rdev = srq->rdev;
2005	int rc;
2006
2007	/* Get live SRQ attr */
2008	tsrq.qplib_srq.id = srq->qplib_srq.id;
2009	rc = bnxt_qplib_query_srq(&rdev->qplib_res, &tsrq.qplib_srq);
2010	if (rc) {
2011		ibdev_err(&rdev->ibdev, "Query HW SRQ failed!");
2012		return rc;
2013	}
2014	srq_attr->max_wr = srq->qplib_srq.max_wqe;
2015	srq_attr->max_sge = srq->qplib_srq.max_sge;
2016	srq_attr->srq_limit = tsrq.qplib_srq.threshold;
2017
2018	return 0;
2019}
2020
2021int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
2022			  const struct ib_recv_wr **bad_wr)
2023{
2024	struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq,
2025					       ib_srq);
2026	struct bnxt_qplib_swqe wqe;
2027	unsigned long flags;
2028	int rc = 0;
2029
2030	spin_lock_irqsave(&srq->lock, flags);
2031	while (wr) {
2032		/* Transcribe each ib_recv_wr to qplib_swqe */
2033		wqe.num_sge = wr->num_sge;
2034		bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
2035		wqe.wr_id = wr->wr_id;
2036		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2037
2038		rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
2039		if (rc) {
2040			*bad_wr = wr;
2041			break;
2042		}
2043		wr = wr->next;
2044	}
2045	spin_unlock_irqrestore(&srq->lock, flags);
2046
2047	return rc;
2048}
2049static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
2050				    struct bnxt_re_qp *qp1_qp,
2051				    int qp_attr_mask)
2052{
2053	struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
2054	int rc;
2055
2056	if (qp_attr_mask & IB_QP_STATE) {
2057		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2058		qp->qplib_qp.state = qp1_qp->qplib_qp.state;
2059	}
2060	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2061		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2062		qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
2063	}
2064
2065	if (qp_attr_mask & IB_QP_QKEY) {
2066		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2067		/* Using a Random  QKEY */
2068		qp->qplib_qp.qkey = 0x81818181;
2069	}
2070	if (qp_attr_mask & IB_QP_SQ_PSN) {
2071		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2072		qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
2073	}
2074
2075	rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2076	if (rc)
2077		ibdev_err(&rdev->ibdev, "Failed to modify Shadow QP for QP1");
2078	return rc;
2079}
2080
2081int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2082		      int qp_attr_mask, struct ib_udata *udata)
2083{
2084	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2085	struct bnxt_re_dev *rdev = qp->rdev;
2086	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
2087	enum ib_qp_state curr_qp_state, new_qp_state;
2088	int rc, entries;
2089	unsigned int flags;
2090	u8 nw_type;
2091
2092	if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2093		return -EOPNOTSUPP;
2094
2095	qp->qplib_qp.modify_flags = 0;
2096	if (qp_attr_mask & IB_QP_STATE) {
2097		curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
2098		new_qp_state = qp_attr->qp_state;
2099		if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
2100					ib_qp->qp_type, qp_attr_mask)) {
2101			ibdev_err(&rdev->ibdev,
2102				  "Invalid attribute mask: %#x specified ",
2103				  qp_attr_mask);
2104			ibdev_err(&rdev->ibdev,
2105				  "for qpn: %#x type: %#x",
2106				  ib_qp->qp_num, ib_qp->qp_type);
2107			ibdev_err(&rdev->ibdev,
2108				  "curr_qp_state=0x%x, new_qp_state=0x%x\n",
2109				  curr_qp_state, new_qp_state);
2110			return -EINVAL;
2111		}
2112		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
2113		qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
2114
2115		if (!qp->sumem &&
2116		    qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
2117			ibdev_dbg(&rdev->ibdev,
2118				  "Move QP = %p to flush list\n", qp);
2119			flags = bnxt_re_lock_cqs(qp);
2120			bnxt_qplib_add_flush_qp(&qp->qplib_qp);
2121			bnxt_re_unlock_cqs(qp, flags);
2122		}
2123		if (!qp->sumem &&
2124		    qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
2125			ibdev_dbg(&rdev->ibdev,
2126				  "Move QP = %p out of flush list\n", qp);
2127			flags = bnxt_re_lock_cqs(qp);
2128			bnxt_qplib_clean_qp(&qp->qplib_qp);
2129			bnxt_re_unlock_cqs(qp, flags);
2130		}
2131	}
2132	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
2133		qp->qplib_qp.modify_flags |=
2134				CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
2135		qp->qplib_qp.en_sqd_async_notify = true;
2136	}
2137	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
2138		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
2139		qp->qplib_qp.access =
2140			__qp_access_flags_from_ib(qp->qplib_qp.cctx,
2141						  qp_attr->qp_access_flags);
2142		/* LOCAL_WRITE access must be set to allow RC receive */
2143		qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_LOCAL_WRITE;
2144	}
2145	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
2146		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
2147		qp->qplib_qp.pkey_index = qp_attr->pkey_index;
2148	}
2149	if (qp_attr_mask & IB_QP_QKEY) {
2150		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
2151		qp->qplib_qp.qkey = qp_attr->qkey;
2152	}
2153	if (qp_attr_mask & IB_QP_AV) {
2154		const struct ib_global_route *grh =
2155			rdma_ah_read_grh(&qp_attr->ah_attr);
2156		const struct ib_gid_attr *sgid_attr;
2157		struct bnxt_re_gid_ctx *ctx;
2158
2159		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
2160				     CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
2161				     CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
2162				     CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
2163				     CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
2164				     CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
2165				     CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
2166		memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
2167		       sizeof(qp->qplib_qp.ah.dgid.data));
2168		qp->qplib_qp.ah.flow_label = grh->flow_label;
2169		sgid_attr = grh->sgid_attr;
2170		/* Get the HW context of the GID. The reference
2171		 * of GID table entry is already taken by the caller.
2172		 */
2173		ctx = rdma_read_gid_hw_context(sgid_attr);
2174		qp->qplib_qp.ah.sgid_index = ctx->idx;
2175		qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
2176		qp->qplib_qp.ah.hop_limit = grh->hop_limit;
2177		qp->qplib_qp.ah.traffic_class = grh->traffic_class >> 2;
2178		qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
2179		ether_addr_copy(qp->qplib_qp.ah.dmac,
2180				qp_attr->ah_attr.roce.dmac);
2181
2182		rc = rdma_read_gid_l2_fields(sgid_attr, NULL,
2183					     &qp->qplib_qp.smac[0]);
2184		if (rc)
2185			return rc;
2186
2187		nw_type = rdma_gid_attr_network_type(sgid_attr);
2188		switch (nw_type) {
2189		case RDMA_NETWORK_IPV4:
2190			qp->qplib_qp.nw_type =
2191				CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
2192			break;
2193		case RDMA_NETWORK_IPV6:
2194			qp->qplib_qp.nw_type =
2195				CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
2196			break;
2197		default:
2198			qp->qplib_qp.nw_type =
2199				CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
2200			break;
2201		}
2202	}
2203
2204	if (qp_attr->qp_state == IB_QPS_RTR) {
2205		enum ib_mtu qpmtu;
2206
2207		qpmtu = iboe_get_mtu(rdev->netdev->mtu);
2208		if (qp_attr_mask & IB_QP_PATH_MTU) {
2209			if (ib_mtu_enum_to_int(qp_attr->path_mtu) >
2210			    ib_mtu_enum_to_int(qpmtu))
2211				return -EINVAL;
2212			qpmtu = qp_attr->path_mtu;
2213		}
2214
2215		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
2216		qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu);
2217		qp->qplib_qp.mtu = ib_mtu_enum_to_int(qpmtu);
2218	}
2219
2220	if (qp_attr_mask & IB_QP_TIMEOUT) {
2221		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
2222		qp->qplib_qp.timeout = qp_attr->timeout;
2223	}
2224	if (qp_attr_mask & IB_QP_RETRY_CNT) {
2225		qp->qplib_qp.modify_flags |=
2226				CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
2227		qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
2228	}
2229	if (qp_attr_mask & IB_QP_RNR_RETRY) {
2230		qp->qplib_qp.modify_flags |=
2231				CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
2232		qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
2233	}
2234	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
2235		qp->qplib_qp.modify_flags |=
2236				CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
2237		qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
2238	}
2239	if (qp_attr_mask & IB_QP_RQ_PSN) {
2240		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
2241		qp->qplib_qp.rq.psn = qp_attr->rq_psn;
2242	}
2243	if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2244		qp->qplib_qp.modify_flags |=
2245				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
2246		/* Cap the max_rd_atomic to device max */
2247		qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
2248						   dev_attr->max_qp_rd_atom);
2249	}
2250	if (qp_attr_mask & IB_QP_SQ_PSN) {
2251		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
2252		qp->qplib_qp.sq.psn = qp_attr->sq_psn;
2253	}
2254	if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2255		if (qp_attr->max_dest_rd_atomic >
2256		    dev_attr->max_qp_init_rd_atom) {
2257			ibdev_err(&rdev->ibdev,
2258				  "max_dest_rd_atomic requested%d is > dev_max%d",
2259				  qp_attr->max_dest_rd_atomic,
2260				  dev_attr->max_qp_init_rd_atom);
2261			return -EINVAL;
2262		}
2263
2264		qp->qplib_qp.modify_flags |=
2265				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
2266		qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
2267	}
2268	if (qp_attr_mask & IB_QP_CAP) {
2269		struct bnxt_re_ucontext *uctx =
2270			rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
2271
2272		qp->qplib_qp.modify_flags |=
2273				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
2274				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
2275				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
2276				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
2277				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
2278		if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
2279		    (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
2280		    (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
2281		    (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
2282		    (qp_attr->cap.max_inline_data >=
2283						dev_attr->max_inline_data)) {
2284			ibdev_err(&rdev->ibdev,
2285				  "Create QP failed - max exceeded");
2286			return -EINVAL;
2287		}
2288		entries = bnxt_re_init_depth(qp_attr->cap.max_send_wr, uctx);
2289		qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
2290						dev_attr->max_qp_wqes + 1);
2291		qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
2292						qp_attr->cap.max_send_wr;
2293		/*
2294		 * Reserving one slot for Phantom WQE. Some application can
2295		 * post one extra entry in this case. Allowing this to avoid
2296		 * unexpected Queue full condition
2297		 */
2298		qp->qplib_qp.sq.q_full_delta -= 1;
2299		qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
2300		if (qp->qplib_qp.rq.max_wqe) {
2301			entries = bnxt_re_init_depth(qp_attr->cap.max_recv_wr, uctx);
2302			qp->qplib_qp.rq.max_wqe =
2303				min_t(u32, entries, dev_attr->max_qp_wqes + 1);
2304			qp->qplib_qp.rq.max_sw_wqe = qp->qplib_qp.rq.max_wqe;
2305			qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
2306						       qp_attr->cap.max_recv_wr;
2307			qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
2308		} else {
2309			/* SRQ was used prior, just ignore the RQ caps */
2310		}
2311	}
2312	if (qp_attr_mask & IB_QP_DEST_QPN) {
2313		qp->qplib_qp.modify_flags |=
2314				CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
2315		qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
2316	}
2317	rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
2318	if (rc) {
2319		ibdev_err(&rdev->ibdev, "Failed to modify HW QP");
2320		return rc;
2321	}
2322	if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_sqp)
2323		rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
2324	return rc;
2325}
2326
2327int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
2328		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2329{
2330	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2331	struct bnxt_re_dev *rdev = qp->rdev;
2332	struct bnxt_qplib_qp *qplib_qp;
2333	int rc;
2334
2335	qplib_qp = kzalloc(sizeof(*qplib_qp), GFP_KERNEL);
2336	if (!qplib_qp)
2337		return -ENOMEM;
2338
2339	qplib_qp->id = qp->qplib_qp.id;
2340	qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
2341
2342	rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
2343	if (rc) {
2344		ibdev_err(&rdev->ibdev, "Failed to query HW QP");
2345		goto out;
2346	}
2347	qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
2348	qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
2349	qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
2350	qp_attr->qp_access_flags = __qp_access_flags_to_ib(qp->qplib_qp.cctx,
2351							   qplib_qp->access);
2352	qp_attr->pkey_index = qplib_qp->pkey_index;
2353	qp_attr->qkey = qplib_qp->qkey;
2354	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2355	rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp->udp_sport,
2356			qplib_qp->ah.host_sgid_index,
2357			qplib_qp->ah.hop_limit,
2358			qplib_qp->ah.traffic_class);
2359	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp->ah.dgid.data);
2360	rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp->ah.sl);
2361	ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp->ah.dmac);
2362	qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
2363	qp_attr->timeout = qplib_qp->timeout;
2364	qp_attr->retry_cnt = qplib_qp->retry_cnt;
2365	qp_attr->rnr_retry = qplib_qp->rnr_retry;
2366	qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
2367	qp_attr->port_num = __to_ib_port_num(qplib_qp->port_id);
2368	qp_attr->rq_psn = qplib_qp->rq.psn;
2369	qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
2370	qp_attr->sq_psn = qplib_qp->sq.psn;
2371	qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
2372	qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
2373							 IB_SIGNAL_REQ_WR;
2374	qp_attr->dest_qp_num = qplib_qp->dest_qpn;
2375
2376	qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
2377	qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
2378	qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
2379	qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
2380	qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
2381	qp_init_attr->cap = qp_attr->cap;
2382
2383out:
2384	kfree(qplib_qp);
2385	return rc;
2386}
2387
2388/* Routine for sending QP1 packets for RoCE V1 an V2
2389 */
2390static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
2391				     const struct ib_send_wr *wr,
2392				     struct bnxt_qplib_swqe *wqe,
2393				     int payload_size)
2394{
2395	struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
2396					     ib_ah);
2397	struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
2398	const struct ib_gid_attr *sgid_attr = ah->ib_ah.sgid_attr;
2399	struct bnxt_qplib_sge sge;
2400	u8 nw_type;
2401	u16 ether_type;
2402	union ib_gid dgid;
2403	bool is_eth = false;
2404	bool is_vlan = false;
2405	bool is_grh = false;
2406	bool is_udp = false;
2407	u8 ip_version = 0;
2408	u16 vlan_id = 0xFFFF;
2409	void *buf;
2410	int i, rc;
2411
2412	memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
2413
2414	rc = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, NULL);
2415	if (rc)
2416		return rc;
2417
2418	/* Get network header type for this GID */
2419	nw_type = rdma_gid_attr_network_type(sgid_attr);
2420	switch (nw_type) {
2421	case RDMA_NETWORK_IPV4:
2422		nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
2423		break;
2424	case RDMA_NETWORK_IPV6:
2425		nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
2426		break;
2427	default:
2428		nw_type = BNXT_RE_ROCE_V1_PACKET;
2429		break;
2430	}
2431	memcpy(&dgid.raw, &qplib_ah->dgid, 16);
2432	is_udp = sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
2433	if (is_udp) {
2434		if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) {
2435			ip_version = 4;
2436			ether_type = ETH_P_IP;
2437		} else {
2438			ip_version = 6;
2439			ether_type = ETH_P_IPV6;
2440		}
2441		is_grh = false;
2442	} else {
2443		ether_type = ETH_P_IBOE;
2444		is_grh = true;
2445	}
2446
2447	is_eth = true;
2448	is_vlan = vlan_id && (vlan_id < 0x1000);
2449
2450	ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
2451			  ip_version, is_udp, 0, &qp->qp1_hdr);
2452
2453	/* ETH */
2454	ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
2455	ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
2456
2457	/* For vlan, check the sgid for vlan existence */
2458
2459	if (!is_vlan) {
2460		qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
2461	} else {
2462		qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
2463		qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
2464	}
2465
2466	if (is_grh || (ip_version == 6)) {
2467		memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid_attr->gid.raw,
2468		       sizeof(sgid_attr->gid));
2469		memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
2470		       sizeof(sgid_attr->gid));
2471		qp->qp1_hdr.grh.hop_limit     = qplib_ah->hop_limit;
2472	}
2473
2474	if (ip_version == 4) {
2475		qp->qp1_hdr.ip4.tos = 0;
2476		qp->qp1_hdr.ip4.id = 0;
2477		qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
2478		qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
2479
2480		memcpy(&qp->qp1_hdr.ip4.saddr, sgid_attr->gid.raw + 12, 4);
2481		memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
2482		qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
2483	}
2484
2485	if (is_udp) {
2486		qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
2487		qp->qp1_hdr.udp.sport = htons(0x8CD1);
2488		qp->qp1_hdr.udp.csum = 0;
2489	}
2490
2491	/* BTH */
2492	if (wr->opcode == IB_WR_SEND_WITH_IMM) {
2493		qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
2494		qp->qp1_hdr.immediate_present = 1;
2495	} else {
2496		qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
2497	}
2498	if (wr->send_flags & IB_SEND_SOLICITED)
2499		qp->qp1_hdr.bth.solicited_event = 1;
2500	/* pad_count */
2501	qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
2502
2503	/* P_key for QP1 is for all members */
2504	qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
2505	qp->qp1_hdr.bth.destination_qpn = IB_QP1;
2506	qp->qp1_hdr.bth.ack_req = 0;
2507	qp->send_psn++;
2508	qp->send_psn &= BTH_PSN_MASK;
2509	qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
2510	/* DETH */
2511	/* Use the priviledged Q_Key for QP1 */
2512	qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
2513	qp->qp1_hdr.deth.source_qpn = IB_QP1;
2514
2515	/* Pack the QP1 to the transmit buffer */
2516	buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
2517	if (buf) {
2518		ib_ud_header_pack(&qp->qp1_hdr, buf);
2519		for (i = wqe->num_sge; i; i--) {
2520			wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
2521			wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
2522			wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
2523		}
2524
2525		/*
2526		 * Max Header buf size for IPV6 RoCE V2 is 86,
2527		 * which is same as the QP1 SQ header buffer.
2528		 * Header buf size for IPV4 RoCE V2 can be 66.
2529		 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
2530		 * Subtract 20 bytes from QP1 SQ header buf size
2531		 */
2532		if (is_udp && ip_version == 4)
2533			sge.size -= 20;
2534		/*
2535		 * Max Header buf size for RoCE V1 is 78.
2536		 * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
2537		 * Subtract 8 bytes from QP1 SQ header buf size
2538		 */
2539		if (!is_udp)
2540			sge.size -= 8;
2541
2542		/* Subtract 4 bytes for non vlan packets */
2543		if (!is_vlan)
2544			sge.size -= 4;
2545
2546		wqe->sg_list[0].addr = sge.addr;
2547		wqe->sg_list[0].lkey = sge.lkey;
2548		wqe->sg_list[0].size = sge.size;
2549		wqe->num_sge++;
2550
2551	} else {
2552		ibdev_err(&qp->rdev->ibdev, "QP1 buffer is empty!");
2553		rc = -ENOMEM;
2554	}
2555	return rc;
2556}
2557
2558/* For the MAD layer, it only provides the recv SGE the size of
2559 * ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
2560 * nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
2561 * receive packet (334 bytes) with no VLAN and then copy the GRH
2562 * and the MAD datagram out to the provided SGE.
2563 */
2564static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
2565					    const struct ib_recv_wr *wr,
2566					    struct bnxt_qplib_swqe *wqe,
2567					    int payload_size)
2568{
2569	struct bnxt_re_sqp_entries *sqp_entry;
2570	struct bnxt_qplib_sge ref, sge;
2571	struct bnxt_re_dev *rdev;
2572	u32 rq_prod_index;
2573
2574	rdev = qp->rdev;
2575
2576	rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
2577
2578	if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
2579		return -ENOMEM;
2580
2581	/* Create 1 SGE to receive the entire
2582	 * ethernet packet
2583	 */
2584	/* Save the reference from ULP */
2585	ref.addr = wqe->sg_list[0].addr;
2586	ref.lkey = wqe->sg_list[0].lkey;
2587	ref.size = wqe->sg_list[0].size;
2588
2589	sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
2590
2591	/* SGE 1 */
2592	wqe->sg_list[0].addr = sge.addr;
2593	wqe->sg_list[0].lkey = sge.lkey;
2594	wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
2595	sge.size -= wqe->sg_list[0].size;
2596
2597	sqp_entry->sge.addr = ref.addr;
2598	sqp_entry->sge.lkey = ref.lkey;
2599	sqp_entry->sge.size = ref.size;
2600	/* Store the wrid for reporting completion */
2601	sqp_entry->wrid = wqe->wr_id;
2602	/* change the wqe->wrid to table index */
2603	wqe->wr_id = rq_prod_index;
2604	return 0;
2605}
2606
2607static int is_ud_qp(struct bnxt_re_qp *qp)
2608{
2609	return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
2610		qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
2611}
2612
2613static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
2614				  const struct ib_send_wr *wr,
2615				  struct bnxt_qplib_swqe *wqe)
2616{
2617	struct bnxt_re_ah *ah = NULL;
2618
2619	if (is_ud_qp(qp)) {
2620		ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
2621		wqe->send.q_key = ud_wr(wr)->remote_qkey;
2622		wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
2623		wqe->send.avid = ah->qplib_ah.id;
2624	}
2625	switch (wr->opcode) {
2626	case IB_WR_SEND:
2627		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
2628		break;
2629	case IB_WR_SEND_WITH_IMM:
2630		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
2631		wqe->send.imm_data = be32_to_cpu(wr->ex.imm_data);
2632		break;
2633	case IB_WR_SEND_WITH_INV:
2634		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
2635		wqe->send.inv_key = wr->ex.invalidate_rkey;
2636		break;
2637	default:
2638		return -EINVAL;
2639	}
2640	if (wr->send_flags & IB_SEND_SIGNALED)
2641		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2642	if (wr->send_flags & IB_SEND_FENCE)
2643		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2644	if (wr->send_flags & IB_SEND_SOLICITED)
2645		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2646	if (wr->send_flags & IB_SEND_INLINE)
2647		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2648
2649	return 0;
2650}
2651
2652static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
2653				  struct bnxt_qplib_swqe *wqe)
2654{
2655	switch (wr->opcode) {
2656	case IB_WR_RDMA_WRITE:
2657		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
2658		break;
2659	case IB_WR_RDMA_WRITE_WITH_IMM:
2660		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
2661		wqe->rdma.imm_data = be32_to_cpu(wr->ex.imm_data);
2662		break;
2663	case IB_WR_RDMA_READ:
2664		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
2665		wqe->rdma.inv_key = wr->ex.invalidate_rkey;
2666		break;
2667	default:
2668		return -EINVAL;
2669	}
2670	wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
2671	wqe->rdma.r_key = rdma_wr(wr)->rkey;
2672	if (wr->send_flags & IB_SEND_SIGNALED)
2673		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2674	if (wr->send_flags & IB_SEND_FENCE)
2675		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2676	if (wr->send_flags & IB_SEND_SOLICITED)
2677		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2678	if (wr->send_flags & IB_SEND_INLINE)
2679		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
2680
2681	return 0;
2682}
2683
2684static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
2685				    struct bnxt_qplib_swqe *wqe)
2686{
2687	switch (wr->opcode) {
2688	case IB_WR_ATOMIC_CMP_AND_SWP:
2689		wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
2690		wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2691		wqe->atomic.swap_data = atomic_wr(wr)->swap;
2692		break;
2693	case IB_WR_ATOMIC_FETCH_AND_ADD:
2694		wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
2695		wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
2696		break;
2697	default:
2698		return -EINVAL;
2699	}
2700	wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
2701	wqe->atomic.r_key = atomic_wr(wr)->rkey;
2702	if (wr->send_flags & IB_SEND_SIGNALED)
2703		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2704	if (wr->send_flags & IB_SEND_FENCE)
2705		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2706	if (wr->send_flags & IB_SEND_SOLICITED)
2707		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2708	return 0;
2709}
2710
2711static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
2712				 struct bnxt_qplib_swqe *wqe)
2713{
2714	wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
2715	wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
2716
2717	if (wr->send_flags & IB_SEND_SIGNALED)
2718		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2719	if (wr->send_flags & IB_SEND_SOLICITED)
2720		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
2721
2722	return 0;
2723}
2724
2725static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
2726				 struct bnxt_qplib_swqe *wqe)
2727{
2728	struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
2729	struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
2730	int access = wr->access;
2731
2732	wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
2733	wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
2734	wqe->frmr.page_list = mr->pages;
2735	wqe->frmr.page_list_len = mr->npages;
2736	wqe->frmr.levels = qplib_frpl->hwq.level;
2737	wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
2738
2739	if (wr->wr.send_flags & IB_SEND_SIGNALED)
2740		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
2741
2742	if (access & IB_ACCESS_LOCAL_WRITE)
2743		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
2744	if (access & IB_ACCESS_REMOTE_READ)
2745		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
2746	if (access & IB_ACCESS_REMOTE_WRITE)
2747		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
2748	if (access & IB_ACCESS_REMOTE_ATOMIC)
2749		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
2750	if (access & IB_ACCESS_MW_BIND)
2751		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
2752
2753	wqe->frmr.l_key = wr->key;
2754	wqe->frmr.length = wr->mr->length;
2755	wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K);
2756	wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K);
2757	wqe->frmr.va = wr->mr->iova;
2758	return 0;
2759}
2760
2761static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
2762				    const struct ib_send_wr *wr,
2763				    struct bnxt_qplib_swqe *wqe)
2764{
2765	/*  Copy the inline data to the data  field */
2766	u8 *in_data;
2767	u32 i, sge_len;
2768	void *sge_addr;
2769
2770	in_data = wqe->inline_data;
2771	for (i = 0; i < wr->num_sge; i++) {
2772		sge_addr = (void *)(unsigned long)
2773				wr->sg_list[i].addr;
2774		sge_len = wr->sg_list[i].length;
2775
2776		if ((sge_len + wqe->inline_len) >
2777		    BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
2778			ibdev_err(&rdev->ibdev,
2779				  "Inline data size requested > supported value");
2780			return -EINVAL;
2781		}
2782		sge_len = wr->sg_list[i].length;
2783
2784		memcpy(in_data, sge_addr, sge_len);
2785		in_data += wr->sg_list[i].length;
2786		wqe->inline_len += wr->sg_list[i].length;
2787	}
2788	return wqe->inline_len;
2789}
2790
2791static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
2792				   const struct ib_send_wr *wr,
2793				   struct bnxt_qplib_swqe *wqe)
2794{
2795	int payload_sz = 0;
2796
2797	if (wr->send_flags & IB_SEND_INLINE)
2798		payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
2799	else
2800		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
2801					       wqe->num_sge);
2802
2803	return payload_sz;
2804}
2805
2806static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
2807{
2808	if ((qp->ib_qp.qp_type == IB_QPT_UD ||
2809	     qp->ib_qp.qp_type == IB_QPT_GSI ||
2810	     qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
2811	     qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
2812		int qp_attr_mask;
2813		struct ib_qp_attr qp_attr;
2814
2815		qp_attr_mask = IB_QP_STATE;
2816		qp_attr.qp_state = IB_QPS_RTS;
2817		bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
2818		qp->qplib_qp.wqe_cnt = 0;
2819	}
2820}
2821
2822static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
2823				       struct bnxt_re_qp *qp,
2824				       const struct ib_send_wr *wr)
2825{
2826	int rc = 0, payload_sz = 0;
2827	unsigned long flags;
2828
2829	spin_lock_irqsave(&qp->sq_lock, flags);
2830	while (wr) {
2831		struct bnxt_qplib_swqe wqe = {};
2832
2833		/* Common */
2834		wqe.num_sge = wr->num_sge;
2835		if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2836			ibdev_err(&rdev->ibdev,
2837				  "Limit exceeded for Send SGEs");
2838			rc = -EINVAL;
2839			goto bad;
2840		}
2841
2842		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2843		if (payload_sz < 0) {
2844			rc = -EINVAL;
2845			goto bad;
2846		}
2847		wqe.wr_id = wr->wr_id;
2848
2849		wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
2850
2851		rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2852		if (!rc)
2853			rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2854bad:
2855		if (rc) {
2856			ibdev_err(&rdev->ibdev,
2857				  "Post send failed opcode = %#x rc = %d",
2858				  wr->opcode, rc);
2859			break;
2860		}
2861		wr = wr->next;
2862	}
2863	bnxt_qplib_post_send_db(&qp->qplib_qp);
2864	if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
2865		bnxt_ud_qp_hw_stall_workaround(qp);
2866	spin_unlock_irqrestore(&qp->sq_lock, flags);
2867	return rc;
2868}
2869
2870static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe)
2871{
2872	/* Need unconditional fence for non-wire memory opcode
2873	 * to work as expected.
2874	 */
2875	if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV ||
2876	    wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR ||
2877	    wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR ||
2878	    wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW)
2879		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
2880}
2881
2882int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
2883		      const struct ib_send_wr **bad_wr)
2884{
2885	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2886	struct bnxt_qplib_swqe wqe;
2887	int rc = 0, payload_sz = 0;
2888	unsigned long flags;
2889
2890	spin_lock_irqsave(&qp->sq_lock, flags);
2891	while (wr) {
2892		/* House keeping */
2893		memset(&wqe, 0, sizeof(wqe));
2894
2895		/* Common */
2896		wqe.num_sge = wr->num_sge;
2897		if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
2898			ibdev_err(&qp->rdev->ibdev,
2899				  "Limit exceeded for Send SGEs");
2900			rc = -EINVAL;
2901			goto bad;
2902		}
2903
2904		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
2905		if (payload_sz < 0) {
2906			rc = -EINVAL;
2907			goto bad;
2908		}
2909		wqe.wr_id = wr->wr_id;
2910
2911		switch (wr->opcode) {
2912		case IB_WR_SEND:
2913		case IB_WR_SEND_WITH_IMM:
2914			if (qp->qplib_qp.type == CMDQ_CREATE_QP1_TYPE_GSI) {
2915				rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
2916							       payload_sz);
2917				if (rc)
2918					goto bad;
2919				wqe.rawqp1.lflags |=
2920					SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
2921			}
2922			if (wr->send_flags & IB_SEND_IP_CSUM)
2923				wqe.rawqp1.lflags |=
2924					SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
2925			fallthrough;
2926		case IB_WR_SEND_WITH_INV:
2927			rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
2928			break;
2929		case IB_WR_RDMA_WRITE:
2930		case IB_WR_RDMA_WRITE_WITH_IMM:
2931		case IB_WR_RDMA_READ:
2932			rc = bnxt_re_build_rdma_wqe(wr, &wqe);
2933			break;
2934		case IB_WR_ATOMIC_CMP_AND_SWP:
2935		case IB_WR_ATOMIC_FETCH_AND_ADD:
2936			rc = bnxt_re_build_atomic_wqe(wr, &wqe);
2937			break;
2938		case IB_WR_RDMA_READ_WITH_INV:
2939			ibdev_err(&qp->rdev->ibdev,
2940				  "RDMA Read with Invalidate is not supported");
2941			rc = -EINVAL;
2942			goto bad;
2943		case IB_WR_LOCAL_INV:
2944			rc = bnxt_re_build_inv_wqe(wr, &wqe);
2945			break;
2946		case IB_WR_REG_MR:
2947			rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2948			break;
2949		default:
2950			/* Unsupported WRs */
2951			ibdev_err(&qp->rdev->ibdev,
2952				  "WR (%#x) is not supported", wr->opcode);
2953			rc = -EINVAL;
2954			goto bad;
2955		}
2956		if (!rc) {
2957			if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
2958				bnxt_re_legacy_set_uc_fence(&wqe);
2959			rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2960		}
2961bad:
2962		if (rc) {
2963			ibdev_err(&qp->rdev->ibdev,
2964				  "post_send failed op:%#x qps = %#x rc = %d\n",
2965				  wr->opcode, qp->qplib_qp.state, rc);
2966			*bad_wr = wr;
2967			break;
2968		}
2969		wr = wr->next;
2970	}
2971	bnxt_qplib_post_send_db(&qp->qplib_qp);
2972	if (!bnxt_qplib_is_chip_gen_p5_p7(qp->rdev->chip_ctx))
2973		bnxt_ud_qp_hw_stall_workaround(qp);
2974	spin_unlock_irqrestore(&qp->sq_lock, flags);
2975
2976	return rc;
2977}
2978
2979static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2980				       struct bnxt_re_qp *qp,
2981				       const struct ib_recv_wr *wr)
2982{
2983	struct bnxt_qplib_swqe wqe;
2984	int rc = 0;
2985
2986	while (wr) {
2987		/* House keeping */
2988		memset(&wqe, 0, sizeof(wqe));
2989
2990		/* Common */
2991		wqe.num_sge = wr->num_sge;
2992		if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2993			ibdev_err(&rdev->ibdev,
2994				  "Limit exceeded for Receive SGEs");
2995			rc = -EINVAL;
2996			break;
2997		}
2998		bnxt_re_build_sgl(wr->sg_list, wqe.sg_list, wr->num_sge);
2999		wqe.wr_id = wr->wr_id;
3000		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3001
3002		rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
3003		if (rc)
3004			break;
3005
3006		wr = wr->next;
3007	}
3008	if (!rc)
3009		bnxt_qplib_post_recv_db(&qp->qplib_qp);
3010	return rc;
3011}
3012
3013int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
3014		      const struct ib_recv_wr **bad_wr)
3015{
3016	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
3017	struct bnxt_qplib_swqe wqe;
3018	int rc = 0, payload_sz = 0;
3019	unsigned long flags;
3020	u32 count = 0;
3021
3022	spin_lock_irqsave(&qp->rq_lock, flags);
3023	while (wr) {
3024		/* House keeping */
3025		memset(&wqe, 0, sizeof(wqe));
3026
3027		/* Common */
3028		wqe.num_sge = wr->num_sge;
3029		if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
3030			ibdev_err(&qp->rdev->ibdev,
3031				  "Limit exceeded for Receive SGEs");
3032			rc = -EINVAL;
3033			*bad_wr = wr;
3034			break;
3035		}
3036
3037		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
3038					       wr->num_sge);
3039		wqe.wr_id = wr->wr_id;
3040		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
3041
3042		if (ib_qp->qp_type == IB_QPT_GSI &&
3043		    qp->qplib_qp.type != CMDQ_CREATE_QP_TYPE_GSI)
3044			rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
3045							      payload_sz);
3046		if (!rc)
3047			rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
3048		if (rc) {
3049			*bad_wr = wr;
3050			break;
3051		}
3052
3053		/* Ring DB if the RQEs posted reaches a threshold value */
3054		if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
3055			bnxt_qplib_post_recv_db(&qp->qplib_qp);
3056			count = 0;
3057		}
3058
3059		wr = wr->next;
3060	}
3061
3062	if (count)
3063		bnxt_qplib_post_recv_db(&qp->qplib_qp);
3064
3065	spin_unlock_irqrestore(&qp->rq_lock, flags);
3066
3067	return rc;
3068}
3069
3070static struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev)
3071{
3072	int min, indx;
3073
3074	mutex_lock(&rdev->nqr->load_lock);
3075	for (indx = 0, min = 0; indx < (rdev->nqr->num_msix - 1); indx++) {
3076		if (rdev->nqr->nq[min].load > rdev->nqr->nq[indx].load)
3077			min = indx;
3078	}
3079	rdev->nqr->nq[min].load++;
3080	mutex_unlock(&rdev->nqr->load_lock);
3081
3082	return &rdev->nqr->nq[min];
3083}
3084
3085static void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq)
3086{
3087	mutex_lock(&rdev->nqr->load_lock);
3088	nq->load--;
3089	mutex_unlock(&rdev->nqr->load_lock);
3090}
3091
3092/* Completion Queues */
3093int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
3094{
3095	struct bnxt_qplib_chip_ctx *cctx;
3096	struct bnxt_qplib_nq *nq;
3097	struct bnxt_re_dev *rdev;
3098	struct bnxt_re_cq *cq;
3099
3100	cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3101	rdev = cq->rdev;
3102	nq = cq->qplib_cq.nq;
3103	cctx = rdev->chip_ctx;
3104
3105	if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
3106		free_page((unsigned long)cq->uctx_cq_page);
3107		hash_del(&cq->hash_entry);
3108	}
3109	bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
3110
3111	bnxt_re_put_nq(rdev, nq);
3112	ib_umem_release(cq->umem);
3113
3114	atomic_dec(&rdev->stats.res.cq_count);
3115	kfree(cq->cql);
3116	return 0;
3117}
3118
3119int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
3120		      struct uverbs_attr_bundle *attrs)
3121{
3122	struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq);
3123	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev);
3124	struct ib_udata *udata = &attrs->driver_udata;
3125	struct bnxt_re_ucontext *uctx =
3126		rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
3127	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
3128	struct bnxt_qplib_chip_ctx *cctx;
3129	int cqe = attr->cqe;
3130	int rc, entries;
3131	u32 active_cqs;
3132
3133	if (attr->flags)
3134		return -EOPNOTSUPP;
3135
3136	/* Validate CQ fields */
3137	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
3138		ibdev_err(&rdev->ibdev, "Failed to create CQ -max exceeded");
3139		return -EINVAL;
3140	}
3141
3142	cq->rdev = rdev;
3143	cctx = rdev->chip_ctx;
3144	cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
3145
3146	entries = bnxt_re_init_depth(cqe + 1, uctx);
3147	if (entries > dev_attr->max_cq_wqes + 1)
3148		entries = dev_attr->max_cq_wqes + 1;
3149
3150	cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
3151	cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
3152	if (udata) {
3153		struct bnxt_re_cq_req req;
3154		if (ib_copy_from_udata(&req, udata, sizeof(req))) {
3155			rc = -EFAULT;
3156			goto fail;
3157		}
3158
3159		cq->umem = ib_umem_get(&rdev->ibdev, req.cq_va,
3160				       entries * sizeof(struct cq_base),
3161				       IB_ACCESS_LOCAL_WRITE);
3162		if (IS_ERR(cq->umem)) {
3163			rc = PTR_ERR(cq->umem);
3164			goto fail;
3165		}
3166		cq->qplib_cq.sg_info.umem = cq->umem;
3167		cq->qplib_cq.dpi = &uctx->dpi;
3168	} else {
3169		cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
3170		cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
3171				  GFP_KERNEL);
3172		if (!cq->cql) {
3173			rc = -ENOMEM;
3174			goto fail;
3175		}
3176
3177		cq->qplib_cq.dpi = &rdev->dpi_privileged;
3178	}
3179	cq->qplib_cq.max_wqe = entries;
3180	cq->qplib_cq.coalescing = &rdev->cq_coalescing;
3181	cq->qplib_cq.nq = bnxt_re_get_nq(rdev);
3182	cq->qplib_cq.cnq_hw_ring_id = cq->qplib_cq.nq->ring_id;
3183
3184	rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
3185	if (rc) {
3186		ibdev_err(&rdev->ibdev, "Failed to create HW CQ");
3187		goto fail;
3188	}
3189
3190	cq->ib_cq.cqe = entries;
3191	cq->cq_period = cq->qplib_cq.period;
3192
3193	active_cqs = atomic_inc_return(&rdev->stats.res.cq_count);
3194	if (active_cqs > rdev->stats.res.cq_watermark)
3195		rdev->stats.res.cq_watermark = active_cqs;
3196	spin_lock_init(&cq->cq_lock);
3197
3198	if (udata) {
3199		struct bnxt_re_cq_resp resp = {};
3200
3201		if (cctx->modes.toggle_bits & BNXT_QPLIB_CQ_TOGGLE_BIT) {
3202			hash_add(rdev->cq_hash, &cq->hash_entry, cq->qplib_cq.id);
3203			/* Allocate a page */
3204			cq->uctx_cq_page = (void *)get_zeroed_page(GFP_KERNEL);
3205			if (!cq->uctx_cq_page) {
3206				rc = -ENOMEM;
3207				goto c2fail;
3208			}
3209			resp.comp_mask |= BNXT_RE_CQ_TOGGLE_PAGE_SUPPORT;
3210		}
3211		resp.cqid = cq->qplib_cq.id;
3212		resp.tail = cq->qplib_cq.hwq.cons;
3213		resp.phase = cq->qplib_cq.period;
3214		resp.rsvd = 0;
3215		rc = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
3216		if (rc) {
3217			ibdev_err(&rdev->ibdev, "Failed to copy CQ udata");
3218			bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
3219			goto free_mem;
3220		}
3221	}
3222
3223	return 0;
3224
3225free_mem:
3226	free_page((unsigned long)cq->uctx_cq_page);
3227c2fail:
3228	ib_umem_release(cq->umem);
3229fail:
3230	kfree(cq->cql);
3231	return rc;
3232}
3233
3234static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
3235{
3236	struct bnxt_re_dev *rdev = cq->rdev;
3237
3238	bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
3239
3240	cq->qplib_cq.max_wqe = cq->resize_cqe;
3241	if (cq->resize_umem) {
3242		ib_umem_release(cq->umem);
3243		cq->umem = cq->resize_umem;
3244		cq->resize_umem = NULL;
3245		cq->resize_cqe = 0;
3246	}
3247}
3248
3249int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
3250{
3251	struct bnxt_qplib_sg_info sg_info = {};
3252	struct bnxt_qplib_dpi *orig_dpi = NULL;
3253	struct bnxt_qplib_dev_attr *dev_attr;
3254	struct bnxt_re_ucontext *uctx = NULL;
3255	struct bnxt_re_resize_cq_req req;
3256	struct bnxt_re_dev *rdev;
3257	struct bnxt_re_cq *cq;
3258	int rc, entries;
3259
3260	cq =  container_of(ibcq, struct bnxt_re_cq, ib_cq);
3261	rdev = cq->rdev;
3262	dev_attr = rdev->dev_attr;
3263	if (!ibcq->uobject) {
3264		ibdev_err(&rdev->ibdev, "Kernel CQ Resize not supported");
3265		return -EOPNOTSUPP;
3266	}
3267
3268	if (cq->resize_umem) {
3269		ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - Busy",
3270			  cq->qplib_cq.id);
3271		return -EBUSY;
3272	}
3273
3274	/* Check the requested cq depth out of supported depth */
3275	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
3276		ibdev_err(&rdev->ibdev, "Resize CQ %#x failed - out of range cqe %d",
3277			  cq->qplib_cq.id, cqe);
3278		return -EINVAL;
3279	}
3280
3281	uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ib_uctx);
3282	entries = bnxt_re_init_depth(cqe + 1, uctx);
3283	if (entries > dev_attr->max_cq_wqes + 1)
3284		entries = dev_attr->max_cq_wqes + 1;
3285
3286	/* uverbs consumer */
3287	if (ib_copy_from_udata(&req, udata, sizeof(req))) {
3288		rc = -EFAULT;
3289		goto fail;
3290	}
3291
3292	cq->resize_umem = ib_umem_get(&rdev->ibdev, req.cq_va,
3293				      entries * sizeof(struct cq_base),
3294				      IB_ACCESS_LOCAL_WRITE);
3295	if (IS_ERR(cq->resize_umem)) {
3296		rc = PTR_ERR(cq->resize_umem);
3297		ibdev_err(&rdev->ibdev, "%s: ib_umem_get failed! rc = %pe\n",
3298			  __func__, cq->resize_umem);
3299		cq->resize_umem = NULL;
3300		goto fail;
3301	}
3302	cq->resize_cqe = entries;
3303	memcpy(&sg_info, &cq->qplib_cq.sg_info, sizeof(sg_info));
3304	orig_dpi = cq->qplib_cq.dpi;
3305
3306	cq->qplib_cq.sg_info.umem = cq->resize_umem;
3307	cq->qplib_cq.sg_info.pgsize = PAGE_SIZE;
3308	cq->qplib_cq.sg_info.pgshft = PAGE_SHIFT;
3309	cq->qplib_cq.dpi = &uctx->dpi;
3310
3311	rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries);
3312	if (rc) {
3313		ibdev_err(&rdev->ibdev, "Resize HW CQ %#x failed!",
3314			  cq->qplib_cq.id);
3315		goto fail;
3316	}
3317
3318	cq->ib_cq.cqe = cq->resize_cqe;
3319	atomic_inc(&rdev->stats.res.resize_count);
3320
3321	return 0;
3322
3323fail:
3324	if (cq->resize_umem) {
3325		ib_umem_release(cq->resize_umem);
3326		cq->resize_umem = NULL;
3327		cq->resize_cqe = 0;
3328		memcpy(&cq->qplib_cq.sg_info, &sg_info, sizeof(sg_info));
3329		cq->qplib_cq.dpi = orig_dpi;
3330	}
3331	return rc;
3332}
3333
3334static u8 __req_to_ib_wc_status(u8 qstatus)
3335{
3336	switch (qstatus) {
3337	case CQ_REQ_STATUS_OK:
3338		return IB_WC_SUCCESS;
3339	case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
3340		return IB_WC_BAD_RESP_ERR;
3341	case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
3342		return IB_WC_LOC_LEN_ERR;
3343	case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
3344		return IB_WC_LOC_QP_OP_ERR;
3345	case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
3346		return IB_WC_LOC_PROT_ERR;
3347	case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
3348		return IB_WC_GENERAL_ERR;
3349	case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
3350		return IB_WC_REM_INV_REQ_ERR;
3351	case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
3352		return IB_WC_REM_ACCESS_ERR;
3353	case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
3354		return IB_WC_REM_OP_ERR;
3355	case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
3356		return IB_WC_RNR_RETRY_EXC_ERR;
3357	case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
3358		return IB_WC_RETRY_EXC_ERR;
3359	case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
3360		return IB_WC_WR_FLUSH_ERR;
3361	default:
3362		return IB_WC_GENERAL_ERR;
3363	}
3364	return 0;
3365}
3366
3367static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
3368{
3369	switch (qstatus) {
3370	case CQ_RES_RAWETH_QP1_STATUS_OK:
3371		return IB_WC_SUCCESS;
3372	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
3373		return IB_WC_LOC_ACCESS_ERR;
3374	case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
3375		return IB_WC_LOC_LEN_ERR;
3376	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
3377		return IB_WC_LOC_PROT_ERR;
3378	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
3379		return IB_WC_LOC_QP_OP_ERR;
3380	case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
3381		return IB_WC_GENERAL_ERR;
3382	case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
3383		return IB_WC_WR_FLUSH_ERR;
3384	case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
3385		return IB_WC_WR_FLUSH_ERR;
3386	default:
3387		return IB_WC_GENERAL_ERR;
3388	}
3389}
3390
3391static u8 __rc_to_ib_wc_status(u8 qstatus)
3392{
3393	switch (qstatus) {
3394	case CQ_RES_RC_STATUS_OK:
3395		return IB_WC_SUCCESS;
3396	case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
3397		return IB_WC_LOC_ACCESS_ERR;
3398	case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
3399		return IB_WC_LOC_LEN_ERR;
3400	case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
3401		return IB_WC_LOC_PROT_ERR;
3402	case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
3403		return IB_WC_LOC_QP_OP_ERR;
3404	case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
3405		return IB_WC_GENERAL_ERR;
3406	case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
3407		return IB_WC_REM_INV_REQ_ERR;
3408	case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
3409		return IB_WC_WR_FLUSH_ERR;
3410	case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
3411		return IB_WC_WR_FLUSH_ERR;
3412	default:
3413		return IB_WC_GENERAL_ERR;
3414	}
3415}
3416
3417static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
3418{
3419	switch (cqe->type) {
3420	case BNXT_QPLIB_SWQE_TYPE_SEND:
3421		wc->opcode = IB_WC_SEND;
3422		break;
3423	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
3424		wc->opcode = IB_WC_SEND;
3425		wc->wc_flags |= IB_WC_WITH_IMM;
3426		break;
3427	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
3428		wc->opcode = IB_WC_SEND;
3429		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3430		break;
3431	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
3432		wc->opcode = IB_WC_RDMA_WRITE;
3433		break;
3434	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
3435		wc->opcode = IB_WC_RDMA_WRITE;
3436		wc->wc_flags |= IB_WC_WITH_IMM;
3437		break;
3438	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
3439		wc->opcode = IB_WC_RDMA_READ;
3440		break;
3441	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
3442		wc->opcode = IB_WC_COMP_SWAP;
3443		break;
3444	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
3445		wc->opcode = IB_WC_FETCH_ADD;
3446		break;
3447	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
3448		wc->opcode = IB_WC_LOCAL_INV;
3449		break;
3450	case BNXT_QPLIB_SWQE_TYPE_REG_MR:
3451		wc->opcode = IB_WC_REG_MR;
3452		break;
3453	default:
3454		wc->opcode = IB_WC_SEND;
3455		break;
3456	}
3457
3458	wc->status = __req_to_ib_wc_status(cqe->status);
3459}
3460
3461static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
3462				     u16 raweth_qp1_flags2)
3463{
3464	bool is_ipv6 = false, is_ipv4 = false;
3465
3466	/* raweth_qp1_flags Bit 9-6 indicates itype */
3467	if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3468	    != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
3469		return -1;
3470
3471	if (raweth_qp1_flags2 &
3472	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
3473	    raweth_qp1_flags2 &
3474	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
3475		/* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
3476		(raweth_qp1_flags2 &
3477		 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
3478			(is_ipv6 = true) : (is_ipv4 = true);
3479		return ((is_ipv6) ?
3480			 BNXT_RE_ROCEV2_IPV6_PACKET :
3481			 BNXT_RE_ROCEV2_IPV4_PACKET);
3482	} else {
3483		return BNXT_RE_ROCE_V1_PACKET;
3484	}
3485}
3486
3487static int bnxt_re_to_ib_nw_type(int nw_type)
3488{
3489	u8 nw_hdr_type = 0xFF;
3490
3491	switch (nw_type) {
3492	case BNXT_RE_ROCE_V1_PACKET:
3493		nw_hdr_type = RDMA_NETWORK_ROCE_V1;
3494		break;
3495	case BNXT_RE_ROCEV2_IPV4_PACKET:
3496		nw_hdr_type = RDMA_NETWORK_IPV4;
3497		break;
3498	case BNXT_RE_ROCEV2_IPV6_PACKET:
3499		nw_hdr_type = RDMA_NETWORK_IPV6;
3500		break;
3501	}
3502	return nw_hdr_type;
3503}
3504
3505static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
3506				       void *rq_hdr_buf)
3507{
3508	u8 *tmp_buf = NULL;
3509	struct ethhdr *eth_hdr;
3510	u16 eth_type;
3511	bool rc = false;
3512
3513	tmp_buf = (u8 *)rq_hdr_buf;
3514	/*
3515	 * If dest mac is not same as I/F mac, this could be a
3516	 * loopback address or multicast address, check whether
3517	 * it is a loopback packet
3518	 */
3519	if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
3520		tmp_buf += 4;
3521		/* Check the  ether type */
3522		eth_hdr = (struct ethhdr *)tmp_buf;
3523		eth_type = ntohs(eth_hdr->h_proto);
3524		switch (eth_type) {
3525		case ETH_P_IBOE:
3526			rc = true;
3527			break;
3528		case ETH_P_IP:
3529		case ETH_P_IPV6: {
3530			u32 len;
3531			struct udphdr *udp_hdr;
3532
3533			len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
3534						      sizeof(struct ipv6hdr));
3535			tmp_buf += sizeof(struct ethhdr) + len;
3536			udp_hdr = (struct udphdr *)tmp_buf;
3537			if (ntohs(udp_hdr->dest) ==
3538				    ROCE_V2_UDP_DPORT)
3539				rc = true;
3540			break;
3541			}
3542		default:
3543			break;
3544		}
3545	}
3546
3547	return rc;
3548}
3549
3550static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *gsi_qp,
3551					 struct bnxt_qplib_cqe *cqe)
3552{
3553	struct bnxt_re_dev *rdev = gsi_qp->rdev;
3554	struct bnxt_re_sqp_entries *sqp_entry = NULL;
3555	struct bnxt_re_qp *gsi_sqp = rdev->gsi_ctx.gsi_sqp;
3556	dma_addr_t shrq_hdr_buf_map;
3557	struct ib_sge s_sge[2] = {};
3558	struct ib_sge r_sge[2] = {};
3559	struct bnxt_re_ah *gsi_sah;
3560	struct ib_recv_wr rwr = {};
3561	dma_addr_t rq_hdr_buf_map;
3562	struct ib_ud_wr udwr = {};
3563	struct ib_send_wr *swr;
3564	u32 skip_bytes = 0;
3565	int pkt_type = 0;
3566	void *rq_hdr_buf;
3567	u32 offset = 0;
3568	u32 tbl_idx;
3569	int rc;
3570
3571	swr = &udwr.wr;
3572	tbl_idx = cqe->wr_id;
3573
3574	rq_hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf +
3575			(tbl_idx * gsi_qp->qplib_qp.rq_hdr_buf_size);
3576	rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3577							  tbl_idx);
3578
3579	/* Shadow QP header buffer */
3580	shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
3581							    tbl_idx);
3582	sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3583
3584	/* Store this cqe */
3585	memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
3586	sqp_entry->qp1_qp = gsi_qp;
3587
3588	/* Find packet type from the cqe */
3589
3590	pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
3591					     cqe->raweth_qp1_flags2);
3592	if (pkt_type < 0) {
3593		ibdev_err(&rdev->ibdev, "Invalid packet\n");
3594		return -EINVAL;
3595	}
3596
3597	/* Adjust the offset for the user buffer and post in the rq */
3598
3599	if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
3600		offset = 20;
3601
3602	/*
3603	 * QP1 loopback packet has 4 bytes of internal header before
3604	 * ether header. Skip these four bytes.
3605	 */
3606	if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
3607		skip_bytes = 4;
3608
3609	/* First send SGE . Skip the ether header*/
3610	s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
3611			+ skip_bytes;
3612	s_sge[0].lkey = 0xFFFFFFFF;
3613	s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
3614				BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
3615
3616	/* Second Send SGE */
3617	s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
3618			BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
3619	if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
3620		s_sge[1].addr += 8;
3621	s_sge[1].lkey = 0xFFFFFFFF;
3622	s_sge[1].length = 256;
3623
3624	/* First recv SGE */
3625
3626	r_sge[0].addr = shrq_hdr_buf_map;
3627	r_sge[0].lkey = 0xFFFFFFFF;
3628	r_sge[0].length = 40;
3629
3630	r_sge[1].addr = sqp_entry->sge.addr + offset;
3631	r_sge[1].lkey = sqp_entry->sge.lkey;
3632	r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
3633
3634	/* Create receive work request */
3635	rwr.num_sge = 2;
3636	rwr.sg_list = r_sge;
3637	rwr.wr_id = tbl_idx;
3638	rwr.next = NULL;
3639
3640	rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
3641	if (rc) {
3642		ibdev_err(&rdev->ibdev,
3643			  "Failed to post Rx buffers to shadow QP");
3644		return -ENOMEM;
3645	}
3646
3647	swr->num_sge = 2;
3648	swr->sg_list = s_sge;
3649	swr->wr_id = tbl_idx;
3650	swr->opcode = IB_WR_SEND;
3651	swr->next = NULL;
3652	gsi_sah = rdev->gsi_ctx.gsi_sah;
3653	udwr.ah = &gsi_sah->ib_ah;
3654	udwr.remote_qpn = gsi_sqp->qplib_qp.id;
3655	udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
3656
3657	/* post data received  in the send queue */
3658	return bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
3659}
3660
3661static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
3662					  struct bnxt_qplib_cqe *cqe)
3663{
3664	wc->opcode = IB_WC_RECV;
3665	wc->status = __rawqp1_to_ib_wc_status(cqe->status);
3666	wc->wc_flags |= IB_WC_GRH;
3667}
3668
3669static bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
3670					u16 vlan_id)
3671{
3672	/*
3673	 * Check if the vlan is configured in the host.  If not configured, it
3674	 * can be a transparent VLAN. So dont report the vlan id.
3675	 */
3676	if (!__vlan_find_dev_deep_rcu(rdev->netdev,
3677				      htons(ETH_P_8021Q), vlan_id))
3678		return false;
3679	return true;
3680}
3681
3682static bool bnxt_re_is_vlan_pkt(struct bnxt_qplib_cqe *orig_cqe,
3683				u16 *vid, u8 *sl)
3684{
3685	bool ret = false;
3686	u32 metadata;
3687	u16 tpid;
3688
3689	metadata = orig_cqe->raweth_qp1_metadata;
3690	if (orig_cqe->raweth_qp1_flags2 &
3691		CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
3692		tpid = ((metadata &
3693			 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
3694			 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
3695		if (tpid == ETH_P_8021Q) {
3696			*vid = metadata &
3697			       CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
3698			*sl = (metadata &
3699			       CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
3700			       CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
3701			ret = true;
3702		}
3703	}
3704
3705	return ret;
3706}
3707
3708static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
3709				      struct bnxt_qplib_cqe *cqe)
3710{
3711	wc->opcode = IB_WC_RECV;
3712	wc->status = __rc_to_ib_wc_status(cqe->status);
3713
3714	if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3715		wc->wc_flags |= IB_WC_WITH_IMM;
3716	if (cqe->flags & CQ_RES_RC_FLAGS_INV)
3717		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3718	if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
3719	    (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
3720		wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3721}
3722
3723static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
3724					     struct ib_wc *wc,
3725					     struct bnxt_qplib_cqe *cqe)
3726{
3727	struct bnxt_re_dev *rdev = gsi_sqp->rdev;
3728	struct bnxt_re_qp *gsi_qp = NULL;
3729	struct bnxt_qplib_cqe *orig_cqe = NULL;
3730	struct bnxt_re_sqp_entries *sqp_entry = NULL;
3731	int nw_type;
3732	u32 tbl_idx;
3733	u16 vlan_id;
3734	u8 sl;
3735
3736	tbl_idx = cqe->wr_id;
3737
3738	sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
3739	gsi_qp = sqp_entry->qp1_qp;
3740	orig_cqe = &sqp_entry->cqe;
3741
3742	wc->wr_id = sqp_entry->wrid;
3743	wc->byte_len = orig_cqe->length;
3744	wc->qp = &gsi_qp->ib_qp;
3745
3746	wc->ex.imm_data = cpu_to_be32(orig_cqe->immdata);
3747	wc->src_qp = orig_cqe->src_qp;
3748	memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
3749	if (bnxt_re_is_vlan_pkt(orig_cqe, &vlan_id, &sl)) {
3750		if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3751			wc->vlan_id = vlan_id;
3752			wc->sl = sl;
3753			wc->wc_flags |= IB_WC_WITH_VLAN;
3754		}
3755	}
3756	wc->port_num = 1;
3757	wc->vendor_err = orig_cqe->status;
3758
3759	wc->opcode = IB_WC_RECV;
3760	wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
3761	wc->wc_flags |= IB_WC_GRH;
3762
3763	nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
3764					    orig_cqe->raweth_qp1_flags2);
3765	if (nw_type >= 0) {
3766		wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3767		wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3768	}
3769}
3770
3771static void bnxt_re_process_res_ud_wc(struct bnxt_re_qp *qp,
3772				      struct ib_wc *wc,
3773				      struct bnxt_qplib_cqe *cqe)
3774{
3775	struct bnxt_re_dev *rdev;
3776	u16 vlan_id = 0;
3777	u8 nw_type;
3778
3779	rdev = qp->rdev;
3780	wc->opcode = IB_WC_RECV;
3781	wc->status = __rc_to_ib_wc_status(cqe->status);
3782
3783	if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
3784		wc->wc_flags |= IB_WC_WITH_IMM;
3785	/* report only on GSI QP for Thor */
3786	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI) {
3787		wc->wc_flags |= IB_WC_GRH;
3788		memcpy(wc->smac, cqe->smac, ETH_ALEN);
3789		wc->wc_flags |= IB_WC_WITH_SMAC;
3790		if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
3791			vlan_id = (cqe->cfa_meta & 0xFFF);
3792		}
3793		/* Mark only if vlan_id is non zero */
3794		if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
3795			wc->vlan_id = vlan_id;
3796			wc->wc_flags |= IB_WC_WITH_VLAN;
3797		}
3798		nw_type = (cqe->flags & CQ_RES_UD_FLAGS_ROCE_IP_VER_MASK) >>
3799			   CQ_RES_UD_FLAGS_ROCE_IP_VER_SFT;
3800		wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
3801		wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
3802	}
3803
3804}
3805
3806static int send_phantom_wqe(struct bnxt_re_qp *qp)
3807{
3808	struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
3809	unsigned long flags;
3810	int rc;
3811
3812	spin_lock_irqsave(&qp->sq_lock, flags);
3813
3814	rc = bnxt_re_bind_fence_mw(lib_qp);
3815	if (!rc) {
3816		lib_qp->sq.phantom_wqe_cnt++;
3817		ibdev_dbg(&qp->rdev->ibdev,
3818			  "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
3819			  lib_qp->id, lib_qp->sq.hwq.prod,
3820			  HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
3821			  lib_qp->sq.phantom_wqe_cnt);
3822	}
3823
3824	spin_unlock_irqrestore(&qp->sq_lock, flags);
3825	return rc;
3826}
3827
3828int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
3829{
3830	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3831	struct bnxt_re_qp *qp, *sh_qp;
3832	struct bnxt_qplib_cqe *cqe;
3833	int i, ncqe, budget;
3834	struct bnxt_qplib_q *sq;
3835	struct bnxt_qplib_qp *lib_qp;
3836	u32 tbl_idx;
3837	struct bnxt_re_sqp_entries *sqp_entry = NULL;
3838	unsigned long flags;
3839
3840	/* User CQ; the only processing we do is to
3841	 * complete any pending CQ resize operation.
3842	 */
3843	if (cq->umem) {
3844		if (cq->resize_umem)
3845			bnxt_re_resize_cq_complete(cq);
3846		return 0;
3847	}
3848
3849	spin_lock_irqsave(&cq->cq_lock, flags);
3850	budget = min_t(u32, num_entries, cq->max_cql);
3851	num_entries = budget;
3852	if (!cq->cql) {
3853		ibdev_err(&cq->rdev->ibdev, "POLL CQ : no CQL to use");
3854		goto exit;
3855	}
3856	cqe = &cq->cql[0];
3857	while (budget) {
3858		lib_qp = NULL;
3859		ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
3860		if (lib_qp) {
3861			sq = &lib_qp->sq;
3862			if (sq->send_phantom) {
3863				qp = container_of(lib_qp,
3864						  struct bnxt_re_qp, qplib_qp);
3865				if (send_phantom_wqe(qp) == -ENOMEM)
3866					ibdev_err(&cq->rdev->ibdev,
3867						  "Phantom failed! Scheduled to send again\n");
3868				else
3869					sq->send_phantom = false;
3870			}
3871		}
3872		if (ncqe < budget)
3873			ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
3874							      cqe + ncqe,
3875							      budget - ncqe);
3876
3877		if (!ncqe)
3878			break;
3879
3880		for (i = 0; i < ncqe; i++, cqe++) {
3881			/* Transcribe each qplib_wqe back to ib_wc */
3882			memset(wc, 0, sizeof(*wc));
3883
3884			wc->wr_id = cqe->wr_id;
3885			wc->byte_len = cqe->length;
3886			qp = container_of
3887				((struct bnxt_qplib_qp *)
3888				 (unsigned long)(cqe->qp_handle),
3889				 struct bnxt_re_qp, qplib_qp);
3890			wc->qp = &qp->ib_qp;
3891			if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
3892				wc->ex.imm_data = cpu_to_be32(cqe->immdata);
3893			else
3894				wc->ex.invalidate_rkey = cqe->invrkey;
3895			wc->src_qp = cqe->src_qp;
3896			memcpy(wc->smac, cqe->smac, ETH_ALEN);
3897			wc->port_num = 1;
3898			wc->vendor_err = cqe->status;
3899
3900			switch (cqe->opcode) {
3901			case CQ_BASE_CQE_TYPE_REQ:
3902				sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3903				if (sh_qp &&
3904				    qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3905					/* Handle this completion with
3906					 * the stored completion
3907					 */
3908					memset(wc, 0, sizeof(*wc));
3909					continue;
3910				}
3911				bnxt_re_process_req_wc(wc, cqe);
3912				break;
3913			case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
3914				if (!cqe->status) {
3915					int rc = 0;
3916
3917					rc = bnxt_re_process_raw_qp_pkt_rx
3918								(qp, cqe);
3919					if (!rc) {
3920						memset(wc, 0, sizeof(*wc));
3921						continue;
3922					}
3923					cqe->status = -1;
3924				}
3925				/* Errors need not be looped back.
3926				 * But change the wr_id to the one
3927				 * stored in the table
3928				 */
3929				tbl_idx = cqe->wr_id;
3930				sqp_entry = &cq->rdev->gsi_ctx.sqp_tbl[tbl_idx];
3931				wc->wr_id = sqp_entry->wrid;
3932				bnxt_re_process_res_rawqp1_wc(wc, cqe);
3933				break;
3934			case CQ_BASE_CQE_TYPE_RES_RC:
3935				bnxt_re_process_res_rc_wc(wc, cqe);
3936				break;
3937			case CQ_BASE_CQE_TYPE_RES_UD:
3938				sh_qp = qp->rdev->gsi_ctx.gsi_sqp;
3939				if (sh_qp &&
3940				    qp->qplib_qp.id == sh_qp->qplib_qp.id) {
3941					/* Handle this completion with
3942					 * the stored completion
3943					 */
3944					if (cqe->status) {
3945						continue;
3946					} else {
3947						bnxt_re_process_res_shadow_qp_wc
3948								(qp, wc, cqe);
3949						break;
3950					}
3951				}
3952				bnxt_re_process_res_ud_wc(qp, wc, cqe);
3953				break;
3954			default:
3955				ibdev_err(&cq->rdev->ibdev,
3956					  "POLL CQ : type 0x%x not handled",
3957					  cqe->opcode);
3958				continue;
3959			}
3960			wc++;
3961			budget--;
3962		}
3963	}
3964exit:
3965	spin_unlock_irqrestore(&cq->cq_lock, flags);
3966	return num_entries - budget;
3967}
3968
3969int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
3970			  enum ib_cq_notify_flags ib_cqn_flags)
3971{
3972	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
3973	int type = 0, rc = 0;
3974	unsigned long flags;
3975
3976	spin_lock_irqsave(&cq->cq_lock, flags);
3977	/* Trigger on the very next completion */
3978	if (ib_cqn_flags & IB_CQ_NEXT_COMP)
3979		type = DBC_DBC_TYPE_CQ_ARMALL;
3980	/* Trigger on the next solicited completion */
3981	else if (ib_cqn_flags & IB_CQ_SOLICITED)
3982		type = DBC_DBC_TYPE_CQ_ARMSE;
3983
3984	/* Poll to see if there are missed events */
3985	if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
3986	    !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) {
3987		rc = 1;
3988		goto exit;
3989	}
3990	bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
3991
3992exit:
3993	spin_unlock_irqrestore(&cq->cq_lock, flags);
3994	return rc;
3995}
3996
3997/* Memory Regions */
3998struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
3999{
4000	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4001	struct bnxt_re_dev *rdev = pd->rdev;
4002	struct bnxt_re_mr *mr;
4003	u32 active_mrs;
4004	int rc;
4005
4006	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4007	if (!mr)
4008		return ERR_PTR(-ENOMEM);
4009
4010	mr->rdev = rdev;
4011	mr->qplib_mr.pd = &pd->qplib_pd;
4012	mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
4013	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4014
4015	if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
4016		bnxt_re_check_and_set_relaxed_ordering(rdev, &mr->qplib_mr);
4017
4018	/* Allocate and register 0 as the address */
4019	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4020	if (rc)
4021		goto fail;
4022
4023	mr->qplib_mr.hwq.level = PBL_LVL_MAX;
4024	mr->qplib_mr.total_size = -1; /* Infinte length */
4025	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, NULL, 0,
4026			       PAGE_SIZE, false);
4027	if (rc)
4028		goto fail_mr;
4029
4030	mr->ib_mr.lkey = mr->qplib_mr.lkey;
4031	if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
4032			       IB_ACCESS_REMOTE_ATOMIC))
4033		mr->ib_mr.rkey = mr->ib_mr.lkey;
4034	active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4035	if (active_mrs > rdev->stats.res.mr_watermark)
4036		rdev->stats.res.mr_watermark = active_mrs;
4037
4038	return &mr->ib_mr;
4039
4040fail_mr:
4041	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4042fail:
4043	kfree(mr);
4044	return ERR_PTR(rc);
4045}
4046
4047int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
4048{
4049	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4050	struct bnxt_re_dev *rdev = mr->rdev;
4051	int rc;
4052
4053	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4054	if (rc) {
4055		ibdev_err(&rdev->ibdev, "Dereg MR failed: %#x\n", rc);
4056		return rc;
4057	}
4058
4059	if (mr->pages) {
4060		rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
4061							&mr->qplib_frpl);
4062		kfree(mr->pages);
4063		mr->npages = 0;
4064		mr->pages = NULL;
4065	}
4066	ib_umem_release(mr->ib_umem);
4067
4068	kfree(mr);
4069	atomic_dec(&rdev->stats.res.mr_count);
4070	return rc;
4071}
4072
4073static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
4074{
4075	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4076
4077	if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
4078		return -ENOMEM;
4079
4080	mr->pages[mr->npages++] = addr;
4081	return 0;
4082}
4083
4084int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
4085		      unsigned int *sg_offset)
4086{
4087	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
4088
4089	mr->npages = 0;
4090	return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
4091}
4092
4093struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
4094			       u32 max_num_sg)
4095{
4096	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4097	struct bnxt_re_dev *rdev = pd->rdev;
4098	struct bnxt_re_mr *mr = NULL;
4099	u32 active_mrs;
4100	int rc;
4101
4102	if (type != IB_MR_TYPE_MEM_REG) {
4103		ibdev_dbg(&rdev->ibdev, "MR type 0x%x not supported", type);
4104		return ERR_PTR(-EINVAL);
4105	}
4106	if (max_num_sg > MAX_PBL_LVL_1_PGS)
4107		return ERR_PTR(-EINVAL);
4108
4109	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4110	if (!mr)
4111		return ERR_PTR(-ENOMEM);
4112
4113	mr->rdev = rdev;
4114	mr->qplib_mr.pd = &pd->qplib_pd;
4115	mr->qplib_mr.access_flags = BNXT_QPLIB_FR_PMR;
4116	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
4117
4118	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4119	if (rc)
4120		goto bail;
4121
4122	mr->ib_mr.lkey = mr->qplib_mr.lkey;
4123	mr->ib_mr.rkey = mr->ib_mr.lkey;
4124
4125	mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
4126	if (!mr->pages) {
4127		rc = -ENOMEM;
4128		goto fail;
4129	}
4130	rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
4131						 &mr->qplib_frpl, max_num_sg);
4132	if (rc) {
4133		ibdev_err(&rdev->ibdev,
4134			  "Failed to allocate HW FR page list");
4135		goto fail_mr;
4136	}
4137
4138	active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4139	if (active_mrs > rdev->stats.res.mr_watermark)
4140		rdev->stats.res.mr_watermark = active_mrs;
4141	return &mr->ib_mr;
4142
4143fail_mr:
4144	kfree(mr->pages);
4145fail:
4146	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4147bail:
4148	kfree(mr);
4149	return ERR_PTR(rc);
4150}
4151
4152struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
4153			       struct ib_udata *udata)
4154{
4155	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4156	struct bnxt_re_dev *rdev = pd->rdev;
4157	struct bnxt_re_mw *mw;
4158	u32 active_mws;
4159	int rc;
4160
4161	mw = kzalloc(sizeof(*mw), GFP_KERNEL);
4162	if (!mw)
4163		return ERR_PTR(-ENOMEM);
4164	mw->rdev = rdev;
4165	mw->qplib_mw.pd = &pd->qplib_pd;
4166
4167	mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
4168			       CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
4169			       CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
4170	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
4171	if (rc) {
4172		ibdev_err(&rdev->ibdev, "Allocate MW failed!");
4173		goto fail;
4174	}
4175	mw->ib_mw.rkey = mw->qplib_mw.rkey;
4176
4177	active_mws = atomic_inc_return(&rdev->stats.res.mw_count);
4178	if (active_mws > rdev->stats.res.mw_watermark)
4179		rdev->stats.res.mw_watermark = active_mws;
4180	return &mw->ib_mw;
4181
4182fail:
4183	kfree(mw);
4184	return ERR_PTR(rc);
4185}
4186
4187int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
4188{
4189	struct bnxt_re_mw *mw = container_of(ib_mw, struct bnxt_re_mw, ib_mw);
4190	struct bnxt_re_dev *rdev = mw->rdev;
4191	int rc;
4192
4193	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
4194	if (rc) {
4195		ibdev_err(&rdev->ibdev, "Free MW failed: %#x\n", rc);
4196		return rc;
4197	}
4198
4199	kfree(mw);
4200	atomic_dec(&rdev->stats.res.mw_count);
4201	return rc;
4202}
4203
4204static struct ib_mr *__bnxt_re_user_reg_mr(struct ib_pd *ib_pd, u64 length, u64 virt_addr,
4205					   int mr_access_flags, struct ib_umem *umem)
4206{
4207	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4208	struct bnxt_re_dev *rdev = pd->rdev;
4209	unsigned long page_size;
4210	struct bnxt_re_mr *mr;
4211	int umem_pgs, rc;
4212	u32 active_mrs;
4213
4214	if (length > BNXT_RE_MAX_MR_SIZE) {
4215		ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n",
4216			  length, BNXT_RE_MAX_MR_SIZE);
4217		return ERR_PTR(-ENOMEM);
4218	}
4219
4220	page_size = ib_umem_find_best_pgsz(umem, BNXT_RE_PAGE_SIZE_SUPPORTED, virt_addr);
4221	if (!page_size) {
4222		ibdev_err(&rdev->ibdev, "umem page size unsupported!");
4223		return ERR_PTR(-EINVAL);
4224	}
4225
4226	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
4227	if (!mr)
4228		return ERR_PTR(-ENOMEM);
4229
4230	mr->rdev = rdev;
4231	mr->qplib_mr.pd = &pd->qplib_pd;
4232	mr->qplib_mr.access_flags = __from_ib_access_flags(mr_access_flags);
4233	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
4234
4235	if (!_is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags)) {
4236		rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
4237		if (rc) {
4238			ibdev_err(&rdev->ibdev, "Failed to allocate MR rc = %d", rc);
4239			rc = -EIO;
4240			goto free_mr;
4241		}
4242		/* The fixed portion of the rkey is the same as the lkey */
4243		mr->ib_mr.rkey = mr->qplib_mr.rkey;
4244	} else {
4245		mr->qplib_mr.flags = CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
4246	}
4247	mr->ib_umem = umem;
4248	mr->qplib_mr.va = virt_addr;
4249	mr->qplib_mr.total_size = length;
4250
4251	if (mr_access_flags & IB_ACCESS_RELAXED_ORDERING)
4252		bnxt_re_check_and_set_relaxed_ordering(rdev, &mr->qplib_mr);
4253
4254	umem_pgs = ib_umem_num_dma_blocks(umem, page_size);
4255	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, umem,
4256			       umem_pgs, page_size,
4257			       _is_alloc_mr_unified(rdev->dev_attr->dev_cap_flags));
4258	if (rc) {
4259		ibdev_err(&rdev->ibdev, "Failed to register user MR - rc = %d\n", rc);
4260		rc = -EIO;
4261		goto free_mrw;
4262	}
4263
4264	mr->ib_mr.lkey = mr->qplib_mr.lkey;
4265	mr->ib_mr.rkey = mr->qplib_mr.lkey;
4266	active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);
4267	if (active_mrs > rdev->stats.res.mr_watermark)
4268		rdev->stats.res.mr_watermark = active_mrs;
4269
4270	return &mr->ib_mr;
4271
4272free_mrw:
4273	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
4274free_mr:
4275	kfree(mr);
4276	return ERR_PTR(rc);
4277}
4278
4279struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
4280				  u64 virt_addr, int mr_access_flags,
4281				  struct ib_dmah *dmah,
4282				  struct ib_udata *udata)
4283{
4284	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4285	struct bnxt_re_dev *rdev = pd->rdev;
4286	struct ib_umem *umem;
4287	struct ib_mr *ib_mr;
4288
4289	if (dmah)
4290		return ERR_PTR(-EOPNOTSUPP);
4291
4292	umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags);
4293	if (IS_ERR(umem))
4294		return ERR_CAST(umem);
4295
4296	ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4297	if (IS_ERR(ib_mr))
4298		ib_umem_release(umem);
4299	return ib_mr;
4300}
4301
4302struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
4303					 u64 length, u64 virt_addr, int fd,
4304					 int mr_access_flags,
4305					 struct ib_dmah *dmah,
4306					 struct uverbs_attr_bundle *attrs)
4307{
4308	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
4309	struct bnxt_re_dev *rdev = pd->rdev;
4310	struct ib_umem_dmabuf *umem_dmabuf;
4311	struct ib_umem *umem;
4312	struct ib_mr *ib_mr;
4313
4314	if (dmah)
4315		return ERR_PTR(-EOPNOTSUPP);
4316
4317	umem_dmabuf = ib_umem_dmabuf_get_pinned(&rdev->ibdev, start, length,
4318						fd, mr_access_flags);
4319	if (IS_ERR(umem_dmabuf))
4320		return ERR_CAST(umem_dmabuf);
4321
4322	umem = &umem_dmabuf->umem;
4323
4324	ib_mr = __bnxt_re_user_reg_mr(ib_pd, length, virt_addr, mr_access_flags, umem);
4325	if (IS_ERR(ib_mr))
4326		ib_umem_release(umem);
4327	return ib_mr;
4328}
4329
4330int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
4331{
4332	struct ib_device *ibdev = ctx->device;
4333	struct bnxt_re_ucontext *uctx =
4334		container_of(ctx, struct bnxt_re_ucontext, ib_uctx);
4335	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
4336	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
4337	struct bnxt_re_user_mmap_entry *entry;
4338	struct bnxt_re_uctx_resp resp = {};
4339	struct bnxt_re_uctx_req ureq = {};
4340	u32 chip_met_rev_num = 0;
4341	int rc;
4342
4343	ibdev_dbg(ibdev, "ABI version requested %u", ibdev->ops.uverbs_abi_ver);
4344
4345	if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
4346		ibdev_dbg(ibdev, " is different from the device %d ",
4347			  BNXT_RE_ABI_VERSION);
4348		return -EPERM;
4349	}
4350
4351	uctx->rdev = rdev;
4352
4353	uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
4354	if (!uctx->shpg) {
4355		rc = -ENOMEM;
4356		goto fail;
4357	}
4358	spin_lock_init(&uctx->sh_lock);
4359
4360	resp.comp_mask = BNXT_RE_UCNTX_CMASK_HAVE_CCTX;
4361	chip_met_rev_num = rdev->chip_ctx->chip_num;
4362	chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_rev & 0xFF) <<
4363			     BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
4364	chip_met_rev_num |= ((u32)rdev->chip_ctx->chip_metal & 0xFF) <<
4365			     BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
4366	resp.chip_id0 = chip_met_rev_num;
4367	/*Temp, Use xa_alloc instead */
4368	resp.dev_id = rdev->en_dev->pdev->devfn;
4369	resp.max_qp = rdev->qplib_ctx.qpc_count;
4370	resp.pg_size = PAGE_SIZE;
4371	resp.cqe_sz = sizeof(struct cq_base);
4372	resp.max_cqd = dev_attr->max_cq_wqes;
4373
4374	if (rdev->chip_ctx->modes.db_push)
4375		resp.comp_mask |= BNXT_RE_UCNTX_CMASK_WC_DPI_ENABLED;
4376
4377	entry = bnxt_re_mmap_entry_insert(uctx, 0, BNXT_RE_MMAP_SH_PAGE, NULL);
4378	if (!entry) {
4379		rc = -ENOMEM;
4380		goto cfail;
4381	}
4382	uctx->shpage_mmap = &entry->rdma_entry;
4383	if (rdev->pacing.dbr_pacing)
4384		resp.comp_mask |= BNXT_RE_UCNTX_CMASK_DBR_PACING_ENABLED;
4385
4386	if (_is_host_msn_table(rdev->qplib_res.dattr->dev_cap_flags2))
4387		resp.comp_mask |= BNXT_RE_UCNTX_CMASK_MSN_TABLE_ENABLED;
4388
4389	if (udata->inlen >= sizeof(ureq)) {
4390		rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq)));
4391		if (rc)
4392			goto cfail;
4393		if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT) {
4394			resp.comp_mask |= BNXT_RE_UCNTX_CMASK_POW2_DISABLED;
4395			uctx->cmask |= BNXT_RE_UCNTX_CAP_POW2_DISABLED;
4396		}
4397		if (ureq.comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_VAR_WQE_SUPPORT) {
4398			resp.comp_mask |= BNXT_RE_UCNTX_CMASK_HAVE_MODE;
4399			resp.mode = rdev->chip_ctx->modes.wqe_mode;
4400			if (resp.mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
4401				uctx->cmask |= BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;
4402		}
4403	}
4404
4405	rc = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
4406	if (rc) {
4407		ibdev_err(ibdev, "Failed to copy user context");
4408		rc = -EFAULT;
4409		goto cfail;
4410	}
4411
4412	return 0;
4413cfail:
4414	free_page((unsigned long)uctx->shpg);
4415	uctx->shpg = NULL;
4416fail:
4417	return rc;
4418}
4419
4420void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
4421{
4422	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4423						   struct bnxt_re_ucontext,
4424						   ib_uctx);
4425
4426	struct bnxt_re_dev *rdev = uctx->rdev;
4427
4428	rdma_user_mmap_entry_remove(uctx->shpage_mmap);
4429	uctx->shpage_mmap = NULL;
4430	if (uctx->shpg)
4431		free_page((unsigned long)uctx->shpg);
4432
4433	if (uctx->dpi.dbr) {
4434		/* Free DPI only if this is the first PD allocated by the
4435		 * application and mark the context dpi as NULL
4436		 */
4437		bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->dpi);
4438		uctx->dpi.dbr = NULL;
4439	}
4440}
4441
4442static int bnxt_re_setup_vnic(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)
4443{
4444	int rc;
4445
4446	rc = bnxt_re_hwrm_alloc_vnic(rdev);
4447	if (rc)
4448		return rc;
4449
4450	rc = bnxt_re_hwrm_cfg_vnic(rdev, qp->qplib_qp.id);
4451	if (rc)
4452		goto out_free_vnic;
4453
4454	return 0;
4455out_free_vnic:
4456	bnxt_re_hwrm_free_vnic(rdev);
4457	return rc;
4458}
4459
4460struct ib_flow *bnxt_re_create_flow(struct ib_qp *ib_qp,
4461				    struct ib_flow_attr *attr,
4462				    struct ib_udata *udata)
4463{
4464	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
4465	struct bnxt_re_dev *rdev = qp->rdev;
4466	struct bnxt_re_flow *flow;
4467	int rc;
4468
4469	if (attr->type != IB_FLOW_ATTR_SNIFFER ||
4470	    !rdev->rcfw.roce_mirror)
4471		return ERR_PTR(-EOPNOTSUPP);
4472
4473	mutex_lock(&rdev->qp_lock);
4474	if (rdev->sniffer_flow_created) {
4475		ibdev_err(&rdev->ibdev, "RoCE Mirroring is already Configured\n");
4476		mutex_unlock(&rdev->qp_lock);
4477		return ERR_PTR(-EBUSY);
4478	}
4479
4480	flow = kzalloc(sizeof(*flow), GFP_KERNEL);
4481	if (!flow) {
4482		mutex_unlock(&rdev->qp_lock);
4483		return ERR_PTR(-ENOMEM);
4484	}
4485
4486	flow->rdev = rdev;
4487
4488	rc = bnxt_re_setup_vnic(rdev, qp);
4489	if (rc)
4490		goto out_free_flow;
4491
4492	rc = bnxt_qplib_create_flow(&rdev->qplib_res);
4493	if (rc)
4494		goto out_free_vnic;
4495
4496	rdev->sniffer_flow_created = 1;
4497	mutex_unlock(&rdev->qp_lock);
4498
4499	return &flow->ib_flow;
4500
4501out_free_vnic:
4502	bnxt_re_hwrm_free_vnic(rdev);
4503out_free_flow:
4504	mutex_unlock(&rdev->qp_lock);
4505	kfree(flow);
4506	return ERR_PTR(rc);
4507}
4508
4509int bnxt_re_destroy_flow(struct ib_flow *flow_id)
4510{
4511	struct bnxt_re_flow *flow =
4512		container_of(flow_id, struct bnxt_re_flow, ib_flow);
4513	struct bnxt_re_dev *rdev = flow->rdev;
4514	int rc;
4515
4516	mutex_lock(&rdev->qp_lock);
4517	rc = bnxt_qplib_destroy_flow(&rdev->qplib_res);
4518	if (rc)
4519		ibdev_dbg(&rdev->ibdev, "failed to destroy_flow rc = %d\n", rc);
4520	rdev->sniffer_flow_created = 0;
4521
4522	bnxt_re_hwrm_free_vnic(rdev);
4523	mutex_unlock(&rdev->qp_lock);
4524	kfree(flow);
4525
4526	return rc;
4527}
4528
4529static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq_id)
4530{
4531	struct bnxt_re_cq *cq = NULL, *tmp_cq;
4532
4533	hash_for_each_possible(rdev->cq_hash, tmp_cq, hash_entry, cq_id) {
4534		if (tmp_cq->qplib_cq.id == cq_id) {
4535			cq = tmp_cq;
4536			break;
4537		}
4538	}
4539	return cq;
4540}
4541
4542static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id)
4543{
4544	struct bnxt_re_srq *srq = NULL, *tmp_srq;
4545
4546	hash_for_each_possible(rdev->srq_hash, tmp_srq, hash_entry, srq_id) {
4547		if (tmp_srq->qplib_srq.id == srq_id) {
4548			srq = tmp_srq;
4549			break;
4550		}
4551	}
4552	return srq;
4553}
4554
4555/* Helper function to mmap the virtual memory from user app */
4556int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
4557{
4558	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
4559						   struct bnxt_re_ucontext,
4560						   ib_uctx);
4561	struct bnxt_re_user_mmap_entry *bnxt_entry;
4562	struct rdma_user_mmap_entry *rdma_entry;
4563	int ret = 0;
4564	u64 pfn;
4565
4566	rdma_entry = rdma_user_mmap_entry_get(&uctx->ib_uctx, vma);
4567	if (!rdma_entry)
4568		return -EINVAL;
4569
4570	bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4571				  rdma_entry);
4572
4573	switch (bnxt_entry->mmap_flag) {
4574	case BNXT_RE_MMAP_WC_DB:
4575		pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4576		ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4577					pgprot_writecombine(vma->vm_page_prot),
4578					rdma_entry);
4579		break;
4580	case BNXT_RE_MMAP_UC_DB:
4581		pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4582		ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4583					pgprot_noncached(vma->vm_page_prot),
4584				rdma_entry);
4585		break;
4586	case BNXT_RE_MMAP_SH_PAGE:
4587		ret = vm_insert_page(vma, vma->vm_start, virt_to_page(uctx->shpg));
4588		break;
4589	case BNXT_RE_MMAP_DBR_BAR:
4590		pfn = bnxt_entry->mem_offset >> PAGE_SHIFT;
4591		ret = rdma_user_mmap_io(ib_uctx, vma, pfn, PAGE_SIZE,
4592					pgprot_noncached(vma->vm_page_prot),
4593					rdma_entry);
4594		break;
4595	case BNXT_RE_MMAP_DBR_PAGE:
4596	case BNXT_RE_MMAP_TOGGLE_PAGE:
4597		/* Driver doesn't expect write access for user space */
4598		if (vma->vm_flags & VM_WRITE)
4599			ret = -EFAULT;
4600		else
4601			ret = vm_insert_page(vma, vma->vm_start,
4602					     virt_to_page((void *)bnxt_entry->mem_offset));
4603		break;
4604	default:
4605		ret = -EINVAL;
4606		break;
4607	}
4608
4609	rdma_user_mmap_entry_put(rdma_entry);
4610	return ret;
4611}
4612
4613void bnxt_re_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
4614{
4615	struct bnxt_re_user_mmap_entry *bnxt_entry;
4616
4617	bnxt_entry = container_of(rdma_entry, struct bnxt_re_user_mmap_entry,
4618				  rdma_entry);
4619
4620	kfree(bnxt_entry);
4621}
4622
4623int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags,
4624			u32 port_num, const struct ib_wc *in_wc,
4625			const struct ib_grh *in_grh,
4626			const struct ib_mad *in_mad, struct ib_mad *out_mad,
4627			size_t *out_mad_size, u16 *out_mad_pkey_index)
4628{
4629	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
4630	struct ib_class_port_info cpi = {};
4631	int ret = IB_MAD_RESULT_SUCCESS;
4632	int rc = 0;
4633
4634	if (in_mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_PERF_MGMT)
4635		return ret;
4636
4637	switch (in_mad->mad_hdr.attr_id) {
4638	case IB_PMA_CLASS_PORT_INFO:
4639		cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;
4640		memcpy((out_mad->data + 40), &cpi, sizeof(cpi));
4641		break;
4642	case IB_PMA_PORT_COUNTERS_EXT:
4643		rc = bnxt_re_assign_pma_port_ext_counters(rdev, out_mad);
4644		break;
4645	case IB_PMA_PORT_COUNTERS:
4646		rc = bnxt_re_assign_pma_port_counters(rdev, out_mad);
4647		break;
4648	default:
4649		rc = -EINVAL;
4650		break;
4651	}
4652	if (rc)
4653		return IB_MAD_RESULT_FAILURE;
4654	ret |= IB_MAD_RESULT_REPLY;
4655	return ret;
4656}
4657
4658static int UVERBS_HANDLER(BNXT_RE_METHOD_NOTIFY_DRV)(struct uverbs_attr_bundle *attrs)
4659{
4660	struct bnxt_re_ucontext *uctx;
4661
4662	uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
4663	bnxt_re_pacing_alert(uctx->rdev);
4664	return 0;
4665}
4666
4667static int UVERBS_HANDLER(BNXT_RE_METHOD_ALLOC_PAGE)(struct uverbs_attr_bundle *attrs)
4668{
4669	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
4670	enum bnxt_re_alloc_page_type alloc_type;
4671	struct bnxt_re_user_mmap_entry *entry;
4672	enum bnxt_re_mmap_flag mmap_flag;
4673	struct bnxt_qplib_chip_ctx *cctx;
4674	struct bnxt_re_ucontext *uctx;
4675	struct bnxt_re_dev *rdev;
4676	u64 mmap_offset;
4677	u32 length;
4678	u32 dpi;
4679	u64 addr;
4680	int err;
4681
4682	uctx = container_of(ib_uverbs_get_ucontext(attrs), struct bnxt_re_ucontext, ib_uctx);
4683	if (IS_ERR(uctx))
4684		return PTR_ERR(uctx);
4685
4686	err = uverbs_get_const(&alloc_type, attrs, BNXT_RE_ALLOC_PAGE_TYPE);
4687	if (err)
4688		return err;
4689
4690	rdev = uctx->rdev;
4691	cctx = rdev->chip_ctx;
4692
4693	switch (alloc_type) {
4694	case BNXT_RE_ALLOC_WC_PAGE:
4695		if (cctx->modes.db_push)  {
4696			if (bnxt_qplib_alloc_dpi(&rdev->qplib_res, &uctx->wcdpi,
4697						 uctx, BNXT_QPLIB_DPI_TYPE_WC))
4698				return -ENOMEM;
4699			length = PAGE_SIZE;
4700			dpi = uctx->wcdpi.dpi;
4701			addr = (u64)uctx->wcdpi.umdbr;
4702			mmap_flag = BNXT_RE_MMAP_WC_DB;
4703		} else {
4704			return -EINVAL;
4705		}
4706
4707		break;
4708	case BNXT_RE_ALLOC_DBR_BAR_PAGE:
4709		length = PAGE_SIZE;
4710		addr = (u64)rdev->pacing.dbr_bar_addr;
4711		mmap_flag = BNXT_RE_MMAP_DBR_BAR;
4712		break;
4713
4714	case BNXT_RE_ALLOC_DBR_PAGE:
4715		length = PAGE_SIZE;
4716		addr = (u64)rdev->pacing.dbr_page;
4717		mmap_flag = BNXT_RE_MMAP_DBR_PAGE;
4718		break;
4719
4720	default:
4721		return -EOPNOTSUPP;
4722	}
4723
4724	entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mmap_offset);
4725	if (!entry)
4726		return -ENOMEM;
4727
4728	uobj->object = entry;
4729	uverbs_finalize_uobj_create(attrs, BNXT_RE_ALLOC_PAGE_HANDLE);
4730	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
4731			     &mmap_offset, sizeof(mmap_offset));
4732	if (err)
4733		return err;
4734
4735	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
4736			     &length, sizeof(length));
4737	if (err)
4738		return err;
4739
4740	err = uverbs_copy_to(attrs, BNXT_RE_ALLOC_PAGE_DPI,
4741			     &dpi, sizeof(dpi));
4742	if (err)
4743		return err;
4744
4745	return 0;
4746}
4747
4748static int alloc_page_obj_cleanup(struct ib_uobject *uobject,
4749				  enum rdma_remove_reason why,
4750			    struct uverbs_attr_bundle *attrs)
4751{
4752	struct  bnxt_re_user_mmap_entry *entry = uobject->object;
4753	struct bnxt_re_ucontext *uctx = entry->uctx;
4754
4755	switch (entry->mmap_flag) {
4756	case BNXT_RE_MMAP_WC_DB:
4757		if (uctx && uctx->wcdpi.dbr) {
4758			struct bnxt_re_dev *rdev = uctx->rdev;
4759
4760			bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi);
4761			uctx->wcdpi.dbr = NULL;
4762		}
4763		break;
4764	case BNXT_RE_MMAP_DBR_BAR:
4765	case BNXT_RE_MMAP_DBR_PAGE:
4766		break;
4767	default:
4768		goto exit;
4769	}
4770	rdma_user_mmap_entry_remove(&entry->rdma_entry);
4771exit:
4772	return 0;
4773}
4774
4775DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_ALLOC_PAGE,
4776			    UVERBS_ATTR_IDR(BNXT_RE_ALLOC_PAGE_HANDLE,
4777					    BNXT_RE_OBJECT_ALLOC_PAGE,
4778					    UVERBS_ACCESS_NEW,
4779					    UA_MANDATORY),
4780			    UVERBS_ATTR_CONST_IN(BNXT_RE_ALLOC_PAGE_TYPE,
4781						 enum bnxt_re_alloc_page_type,
4782						 UA_MANDATORY),
4783			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_OFFSET,
4784						UVERBS_ATTR_TYPE(u64),
4785						UA_MANDATORY),
4786			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_MMAP_LENGTH,
4787						UVERBS_ATTR_TYPE(u32),
4788						UA_MANDATORY),
4789			    UVERBS_ATTR_PTR_OUT(BNXT_RE_ALLOC_PAGE_DPI,
4790						UVERBS_ATTR_TYPE(u32),
4791						UA_MANDATORY));
4792
4793DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_DESTROY_PAGE,
4794				    UVERBS_ATTR_IDR(BNXT_RE_DESTROY_PAGE_HANDLE,
4795						    BNXT_RE_OBJECT_ALLOC_PAGE,
4796						    UVERBS_ACCESS_DESTROY,
4797						    UA_MANDATORY));
4798
4799DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_ALLOC_PAGE,
4800			    UVERBS_TYPE_ALLOC_IDR(alloc_page_obj_cleanup),
4801			    &UVERBS_METHOD(BNXT_RE_METHOD_ALLOC_PAGE),
4802			    &UVERBS_METHOD(BNXT_RE_METHOD_DESTROY_PAGE));
4803
4804DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_NOTIFY_DRV);
4805
4806DECLARE_UVERBS_GLOBAL_METHODS(BNXT_RE_OBJECT_NOTIFY_DRV,
4807			      &UVERBS_METHOD(BNXT_RE_METHOD_NOTIFY_DRV));
4808
4809/* Toggle MEM */
4810static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bundle *attrs)
4811{
4812	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);
4813	enum bnxt_re_mmap_flag mmap_flag = BNXT_RE_MMAP_TOGGLE_PAGE;
4814	enum bnxt_re_get_toggle_mem_type res_type;
4815	struct bnxt_re_user_mmap_entry *entry;
4816	struct bnxt_re_ucontext *uctx;
4817	struct ib_ucontext *ib_uctx;
4818	struct bnxt_re_dev *rdev;
4819	struct bnxt_re_srq *srq;
4820	u32 length = PAGE_SIZE;
4821	struct bnxt_re_cq *cq;
4822	u64 mem_offset;
4823	u32 offset = 0;
4824	u64 addr = 0;
4825	u32 res_id;
4826	int err;
4827
4828	ib_uctx = ib_uverbs_get_ucontext(attrs);
4829	if (IS_ERR(ib_uctx))
4830		return PTR_ERR(ib_uctx);
4831
4832	err = uverbs_get_const(&res_type, attrs, BNXT_RE_TOGGLE_MEM_TYPE);
4833	if (err)
4834		return err;
4835
4836	uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
4837	rdev = uctx->rdev;
4838	err = uverbs_copy_from(&res_id, attrs, BNXT_RE_TOGGLE_MEM_RES_ID);
4839	if (err)
4840		return err;
4841
4842	switch (res_type) {
4843	case BNXT_RE_CQ_TOGGLE_MEM:
4844		cq = bnxt_re_search_for_cq(rdev, res_id);
4845		if (!cq)
4846			return -EINVAL;
4847
4848		addr = (u64)cq->uctx_cq_page;
4849		break;
4850	case BNXT_RE_SRQ_TOGGLE_MEM:
4851		srq = bnxt_re_search_for_srq(rdev, res_id);
4852		if (!srq)
4853			return -EINVAL;
4854
4855		addr = (u64)srq->uctx_srq_page;
4856		break;
4857
4858	default:
4859		return -EOPNOTSUPP;
4860	}
4861
4862	entry = bnxt_re_mmap_entry_insert(uctx, addr, mmap_flag, &mem_offset);
4863	if (!entry)
4864		return -ENOMEM;
4865
4866	uobj->object = entry;
4867	uverbs_finalize_uobj_create(attrs, BNXT_RE_TOGGLE_MEM_HANDLE);
4868	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
4869			     &mem_offset, sizeof(mem_offset));
4870	if (err)
4871		return err;
4872
4873	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
4874			     &length, sizeof(length));
4875	if (err)
4876		return err;
4877
4878	err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
4879			     &offset, sizeof(offset));
4880	if (err)
4881		return err;
4882
4883	return 0;
4884}
4885
4886static int get_toggle_mem_obj_cleanup(struct ib_uobject *uobject,
4887				      enum rdma_remove_reason why,
4888				      struct uverbs_attr_bundle *attrs)
4889{
4890	struct  bnxt_re_user_mmap_entry *entry = uobject->object;
4891
4892	rdma_user_mmap_entry_remove(&entry->rdma_entry);
4893	return 0;
4894}
4895
4896DECLARE_UVERBS_NAMED_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM,
4897			    UVERBS_ATTR_IDR(BNXT_RE_TOGGLE_MEM_HANDLE,
4898					    BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4899					    UVERBS_ACCESS_NEW,
4900					    UA_MANDATORY),
4901			    UVERBS_ATTR_CONST_IN(BNXT_RE_TOGGLE_MEM_TYPE,
4902						 enum bnxt_re_get_toggle_mem_type,
4903						 UA_MANDATORY),
4904			    UVERBS_ATTR_PTR_IN(BNXT_RE_TOGGLE_MEM_RES_ID,
4905					       UVERBS_ATTR_TYPE(u32),
4906					       UA_MANDATORY),
4907			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_PAGE,
4908						UVERBS_ATTR_TYPE(u64),
4909						UA_MANDATORY),
4910			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
4911						UVERBS_ATTR_TYPE(u32),
4912						UA_MANDATORY),
4913			    UVERBS_ATTR_PTR_OUT(BNXT_RE_TOGGLE_MEM_MMAP_LENGTH,
4914						UVERBS_ATTR_TYPE(u32),
4915						UA_MANDATORY));
4916
4917DECLARE_UVERBS_NAMED_METHOD_DESTROY(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM,
4918				    UVERBS_ATTR_IDR(BNXT_RE_RELEASE_TOGGLE_MEM_HANDLE,
4919						    BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4920						    UVERBS_ACCESS_DESTROY,
4921						    UA_MANDATORY));
4922
4923DECLARE_UVERBS_NAMED_OBJECT(BNXT_RE_OBJECT_GET_TOGGLE_MEM,
4924			    UVERBS_TYPE_ALLOC_IDR(get_toggle_mem_obj_cleanup),
4925			    &UVERBS_METHOD(BNXT_RE_METHOD_GET_TOGGLE_MEM),
4926			    &UVERBS_METHOD(BNXT_RE_METHOD_RELEASE_TOGGLE_MEM));
4927
4928const struct uapi_definition bnxt_re_uapi_defs[] = {
4929	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_ALLOC_PAGE),
4930	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_NOTIFY_DRV),
4931	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(BNXT_RE_OBJECT_GET_TOGGLE_MEM),
4932	{}
4933};