siw_verbs.c

   1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2
   3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   4/* Copyright (c) 2008-2019, IBM Corporation */
   5
   6#include <linux/errno.h>
   7#include <linux/types.h>
   8#include <linux/uaccess.h>
   9#include <linux/vmalloc.h>
  10#include <linux/xarray.h>
  11#include <net/addrconf.h>
  12
  13#include <rdma/iw_cm.h>
  14#include <rdma/ib_verbs.h>
  15#include <rdma/ib_user_verbs.h>
  16#include <rdma/uverbs_ioctl.h>
  17
  18#include "siw.h"
  19#include "siw_verbs.h"
  20#include "siw_mem.h"
  21
  22static int siw_qp_state_to_ib_qp_state[SIW_QP_STATE_COUNT] = {
  23	[SIW_QP_STATE_IDLE] = IB_QPS_INIT,
  24	[SIW_QP_STATE_RTR] = IB_QPS_RTR,
  25	[SIW_QP_STATE_RTS] = IB_QPS_RTS,
  26	[SIW_QP_STATE_CLOSING] = IB_QPS_SQD,
  27	[SIW_QP_STATE_TERMINATE] = IB_QPS_SQE,
  28	[SIW_QP_STATE_ERROR] = IB_QPS_ERR
  29};
  30
  31static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
  32	[IB_QPS_RESET] = SIW_QP_STATE_IDLE,
  33	[IB_QPS_INIT] = SIW_QP_STATE_IDLE,
  34	[IB_QPS_RTR] = SIW_QP_STATE_RTR,
  35	[IB_QPS_RTS] = SIW_QP_STATE_RTS,
  36	[IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
  37	[IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
  38	[IB_QPS_ERR] = SIW_QP_STATE_ERROR
  39};
  40
  41static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
  42	[IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
  43	[IB_QPS_RTS] = "RTS",     [IB_QPS_SQD] = "SQD",   [IB_QPS_SQE] = "SQE",
  44	[IB_QPS_ERR] = "ERR"
  45};
  46
  47void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
  48{
  49	struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_entry);
  50
  51	kfree(entry);
  52}
  53
  54int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
  55{
  56	struct siw_ucontext *uctx = to_siw_ctx(ctx);
  57	size_t size = vma->vm_end - vma->vm_start;
  58	struct rdma_user_mmap_entry *rdma_entry;
  59	struct siw_user_mmap_entry *entry;
  60	int rv = -EINVAL;
  61
  62	/*
  63	 * Must be page aligned
  64	 */
  65	if (vma->vm_start & (PAGE_SIZE - 1)) {
  66		pr_warn("siw: mmap not page aligned\n");
  67		return -EINVAL;
  68	}
  69	rdma_entry = rdma_user_mmap_entry_get(&uctx->base_ucontext, vma);
  70	if (!rdma_entry) {
  71		siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
  72			vma->vm_pgoff, size);
  73		return -EINVAL;
  74	}
  75	entry = to_siw_mmap_entry(rdma_entry);
  76
  77	rv = remap_vmalloc_range(vma, entry->address, 0);
  78	if (rv)
  79		pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
  80			size);
  81	rdma_user_mmap_entry_put(rdma_entry);
  82
  83	return rv;
  84}
  85
  86int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
  87{
  88	struct siw_device *sdev = to_siw_dev(base_ctx->device);
  89	struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
  90	struct siw_uresp_alloc_ctx uresp = {};
  91	int rv;
  92
  93	if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
  94		rv = -ENOMEM;
  95		goto err_out;
  96	}
  97	ctx->sdev = sdev;
  98
  99	uresp.dev_id = sdev->vendor_part_id;
 100
 101	if (udata->outlen < sizeof(uresp)) {
 102		rv = -EINVAL;
 103		goto err_out;
 104	}
 105	rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 106	if (rv)
 107		goto err_out;
 108
 109	siw_dbg(base_ctx->device, "success. now %d context(s)\n",
 110		atomic_read(&sdev->num_ctx));
 111
 112	return 0;
 113
 114err_out:
 115	atomic_dec(&sdev->num_ctx);
 116	siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
 117		atomic_read(&sdev->num_ctx));
 118
 119	return rv;
 120}
 121
 122void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
 123{
 124	struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
 125
 126	atomic_dec(&uctx->sdev->num_ctx);
 127}
 128
 129int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
 130		     struct ib_udata *udata)
 131{
 132	struct siw_device *sdev = to_siw_dev(base_dev);
 133
 134	if (udata->inlen || udata->outlen)
 135		return -EINVAL;
 136
 137	memset(attr, 0, sizeof(*attr));
 138
 139	/* Revisit atomic caps if RFC 7306 gets supported */
 140	attr->atomic_cap = 0;
 141	attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
 142	attr->kernel_cap_flags = IBK_ALLOW_USER_UNREG;
 143	attr->max_cq = sdev->attrs.max_cq;
 144	attr->max_cqe = sdev->attrs.max_cqe;
 145	attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
 146	attr->max_mr = sdev->attrs.max_mr;
 147	attr->max_mw = sdev->attrs.max_mw;
 148	attr->max_mr_size = ~0ull;
 149	attr->max_pd = sdev->attrs.max_pd;
 150	attr->max_qp = sdev->attrs.max_qp;
 151	attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
 152	attr->max_qp_rd_atom = sdev->attrs.max_ord;
 153	attr->max_qp_wr = sdev->attrs.max_qp_wr;
 154	attr->max_recv_sge = sdev->attrs.max_sge;
 155	attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
 156	attr->max_send_sge = sdev->attrs.max_sge;
 157	attr->max_sge_rd = sdev->attrs.max_sge_rd;
 158	attr->max_srq = sdev->attrs.max_srq;
 159	attr->max_srq_sge = sdev->attrs.max_srq_sge;
 160	attr->max_srq_wr = sdev->attrs.max_srq_wr;
 161	attr->page_size_cap = PAGE_SIZE;
 162	attr->vendor_id = SIW_VENDOR_ID;
 163	attr->vendor_part_id = sdev->vendor_part_id;
 164
 165	addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
 166			    sdev->raw_gid);
 167
 168	return 0;
 169}
 170
 171int siw_query_port(struct ib_device *base_dev, u32 port,
 172		   struct ib_port_attr *attr)
 173{
 174	struct net_device *ndev;
 175	int rv;
 176
 177	memset(attr, 0, sizeof(*attr));
 178
 179	rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
 180			 &attr->active_width);
 181	if (rv)
 182		return rv;
 183
 184	ndev = ib_device_get_netdev(base_dev, SIW_PORT);
 185	if (!ndev)
 186		return -ENODEV;
 187
 188	attr->gid_tbl_len = 1;
 189	attr->max_msg_sz = -1;
 190	attr->max_mtu = ib_mtu_int_to_enum(ndev->max_mtu);
 191	attr->active_mtu = ib_mtu_int_to_enum(READ_ONCE(ndev->mtu));
 192	attr->state = ib_get_curr_port_state(ndev);
 193	attr->phys_state = attr->state == IB_PORT_ACTIVE ?
 194		IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
 195	attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
 196	/*
 197	 * All zero
 198	 *
 199	 * attr->lid = 0;
 200	 * attr->bad_pkey_cntr = 0;
 201	 * attr->qkey_viol_cntr = 0;
 202	 * attr->sm_lid = 0;
 203	 * attr->lmc = 0;
 204	 * attr->max_vl_num = 0;
 205	 * attr->sm_sl = 0;
 206	 * attr->subnet_timeout = 0;
 207	 * attr->init_type_repy = 0;
 208	 */
 209	dev_put(ndev);
 210	return rv;
 211}
 212
 213int siw_get_port_immutable(struct ib_device *base_dev, u32 port,
 214			   struct ib_port_immutable *port_immutable)
 215{
 216	struct ib_port_attr attr;
 217	int rv = siw_query_port(base_dev, port, &attr);
 218
 219	if (rv)
 220		return rv;
 221
 222	port_immutable->gid_tbl_len = attr.gid_tbl_len;
 223	port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 224
 225	return 0;
 226}
 227
 228int siw_query_gid(struct ib_device *base_dev, u32 port, int idx,
 229		  union ib_gid *gid)
 230{
 231	struct siw_device *sdev = to_siw_dev(base_dev);
 232
 233	/* subnet_prefix == interface_id == 0; */
 234	memset(gid, 0, sizeof(*gid));
 235	memcpy(gid->raw, sdev->raw_gid, ETH_ALEN);
 236
 237	return 0;
 238}
 239
 240int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 241{
 242	struct siw_device *sdev = to_siw_dev(pd->device);
 243
 244	if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
 245		atomic_dec(&sdev->num_pd);
 246		return -ENOMEM;
 247	}
 248	siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
 249
 250	return 0;
 251}
 252
 253int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 254{
 255	struct siw_device *sdev = to_siw_dev(pd->device);
 256
 257	siw_dbg_pd(pd, "free PD\n");
 258	atomic_dec(&sdev->num_pd);
 259	return 0;
 260}
 261
 262void siw_qp_get_ref(struct ib_qp *base_qp)
 263{
 264	siw_qp_get(to_siw_qp(base_qp));
 265}
 266
 267void siw_qp_put_ref(struct ib_qp *base_qp)
 268{
 269	siw_qp_put(to_siw_qp(base_qp));
 270}
 271
 272static struct rdma_user_mmap_entry *
 273siw_mmap_entry_insert(struct siw_ucontext *uctx,
 274		      void *address, size_t length,
 275		      u64 *offset)
 276{
 277	struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 278	int rv;
 279
 280	*offset = SIW_INVAL_UOBJ_KEY;
 281	if (!entry)
 282		return NULL;
 283
 284	entry->address = address;
 285
 286	rv = rdma_user_mmap_entry_insert(&uctx->base_ucontext,
 287					 &entry->rdma_entry,
 288					 length);
 289	if (rv) {
 290		kfree(entry);
 291		return NULL;
 292	}
 293
 294	*offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
 295
 296	return &entry->rdma_entry;
 297}
 298
 299/*
 300 * siw_create_qp()
 301 *
 302 * Create QP of requested size on given device.
 303 *
 304 * @qp:		Queue pait
 305 * @attrs:	Initial QP attributes.
 306 * @udata:	used to provide QP ID, SQ and RQ size back to user.
 307 */
 308
 309int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 310		  struct ib_udata *udata)
 311{
 312	struct ib_pd *pd = ibqp->pd;
 313	struct siw_qp *qp = to_siw_qp(ibqp);
 314	struct ib_device *base_dev = pd->device;
 315	struct siw_device *sdev = to_siw_dev(base_dev);
 316	struct siw_ucontext *uctx =
 317		rdma_udata_to_drv_context(udata, struct siw_ucontext,
 318					  base_ucontext);
 319	unsigned long flags;
 320	int num_sqe, num_rqe, rv = 0;
 321	size_t length;
 322
 323	siw_dbg(base_dev, "create new QP\n");
 324
 325	if (attrs->create_flags)
 326		return -EOPNOTSUPP;
 327
 328	if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
 329		siw_dbg(base_dev, "too many QP's\n");
 330		rv = -ENOMEM;
 331		goto err_atomic;
 332	}
 333	if (attrs->qp_type != IB_QPT_RC) {
 334		siw_dbg(base_dev, "only RC QP's supported\n");
 335		rv = -EOPNOTSUPP;
 336		goto err_atomic;
 337	}
 338	if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
 339	    (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
 340	    (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
 341	    (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
 342		siw_dbg(base_dev, "QP size error\n");
 343		rv = -EINVAL;
 344		goto err_atomic;
 345	}
 346	if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
 347		siw_dbg(base_dev, "max inline send: %d > %d\n",
 348			attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
 349		rv = -EINVAL;
 350		goto err_atomic;
 351	}
 352	/*
 353	 * NOTE: we don't allow for a QP unable to hold any SQ WQE
 354	 */
 355	if (attrs->cap.max_send_wr == 0) {
 356		siw_dbg(base_dev, "QP must have send queue\n");
 357		rv = -EINVAL;
 358		goto err_atomic;
 359	}
 360
 361	if (!attrs->send_cq || (!attrs->recv_cq && !attrs->srq)) {
 362		siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
 363		rv = -EINVAL;
 364		goto err_atomic;
 365	}
 366
 367	init_rwsem(&qp->state_lock);
 368	spin_lock_init(&qp->sq_lock);
 369	spin_lock_init(&qp->rq_lock);
 370	spin_lock_init(&qp->orq_lock);
 371
 372	rv = siw_qp_add(sdev, qp);
 373	if (rv)
 374		goto err_atomic;
 375
 376
 377	/* All queue indices are derived from modulo operations
 378	 * on a free running 'get' (consumer) and 'put' (producer)
 379	 * unsigned counter. Having queue sizes at power of two
 380	 * avoids handling counter wrap around.
 381	 */
 382	num_sqe = roundup_pow_of_two(attrs->cap.max_send_wr);
 383	num_rqe = attrs->cap.max_recv_wr;
 384	if (num_rqe)
 385		num_rqe = roundup_pow_of_two(num_rqe);
 386
 387	if (udata)
 388		qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
 389	else
 390		qp->sendq = vcalloc(num_sqe, sizeof(struct siw_sqe));
 391
 392	if (qp->sendq == NULL) {
 393		rv = -ENOMEM;
 394		goto err_out_xa;
 395	}
 396	if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
 397		if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
 398			qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
 399		else {
 400			rv = -EINVAL;
 401			goto err_out_xa;
 402		}
 403	}
 404	qp->pd = pd;
 405	qp->scq = to_siw_cq(attrs->send_cq);
 406	qp->rcq = to_siw_cq(attrs->recv_cq);
 407
 408	if (attrs->srq) {
 409		/*
 410		 * SRQ support.
 411		 * Verbs 6.3.7: ignore RQ size, if SRQ present
 412		 * Verbs 6.3.5: do not check PD of SRQ against PD of QP
 413		 */
 414		qp->srq = to_siw_srq(attrs->srq);
 415		qp->attrs.rq_size = 0;
 416		siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
 417			qp->base_qp.qp_num);
 418	} else if (num_rqe) {
 419		if (udata)
 420			qp->recvq =
 421				vmalloc_user(num_rqe * sizeof(struct siw_rqe));
 422		else
 423			qp->recvq = vcalloc(num_rqe, sizeof(struct siw_rqe));
 424
 425		if (qp->recvq == NULL) {
 426			rv = -ENOMEM;
 427			goto err_out_xa;
 428		}
 429		qp->attrs.rq_size = num_rqe;
 430	}
 431	qp->attrs.sq_size = num_sqe;
 432	qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
 433	qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
 434
 435	/* Make those two tunables fixed for now. */
 436	qp->tx_ctx.gso_seg_limit = 1;
 437	qp->tx_ctx.zcopy_tx = zcopy_tx;
 438
 439	qp->attrs.state = SIW_QP_STATE_IDLE;
 440
 441	if (udata) {
 442		struct siw_uresp_create_qp uresp = {};
 443
 444		uresp.num_sqe = num_sqe;
 445		uresp.num_rqe = num_rqe;
 446		uresp.qp_id = qp_id(qp);
 447
 448		if (qp->sendq) {
 449			length = num_sqe * sizeof(struct siw_sqe);
 450			qp->sq_entry =
 451				siw_mmap_entry_insert(uctx, qp->sendq,
 452						      length, &uresp.sq_key);
 453			if (!qp->sq_entry) {
 454				rv = -ENOMEM;
 455				goto err_out_xa;
 456			}
 457		}
 458
 459		if (qp->recvq) {
 460			length = num_rqe * sizeof(struct siw_rqe);
 461			qp->rq_entry =
 462				siw_mmap_entry_insert(uctx, qp->recvq,
 463						      length, &uresp.rq_key);
 464			if (!qp->rq_entry) {
 465				uresp.sq_key = SIW_INVAL_UOBJ_KEY;
 466				rv = -ENOMEM;
 467				goto err_out_xa;
 468			}
 469		}
 470
 471		if (udata->outlen < sizeof(uresp)) {
 472			rv = -EINVAL;
 473			goto err_out_xa;
 474		}
 475		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 476		if (rv)
 477			goto err_out_xa;
 478	}
 479	qp->tx_cpu = siw_get_tx_cpu(sdev);
 480	if (qp->tx_cpu < 0) {
 481		rv = -EINVAL;
 482		goto err_out_xa;
 483	}
 484	INIT_LIST_HEAD(&qp->devq);
 485	spin_lock_irqsave(&sdev->lock, flags);
 486	list_add_tail(&qp->devq, &sdev->qp_list);
 487	spin_unlock_irqrestore(&sdev->lock, flags);
 488
 489	init_completion(&qp->qp_free);
 490
 491	return 0;
 492
 493err_out_xa:
 494	xa_erase(&sdev->qp_xa, qp_id(qp));
 495	if (uctx) {
 496		rdma_user_mmap_entry_remove(qp->sq_entry);
 497		rdma_user_mmap_entry_remove(qp->rq_entry);
 498	}
 499	vfree(qp->sendq);
 500	vfree(qp->recvq);
 501
 502err_atomic:
 503	atomic_dec(&sdev->num_qp);
 504	return rv;
 505}
 506
 507/*
 508 * Minimum siw_query_qp() verb interface.
 509 *
 510 * @qp_attr_mask is not used but all available information is provided
 511 */
 512int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
 513		 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
 514{
 515	struct siw_qp *qp;
 516	struct net_device *ndev;
 517
 518	if (base_qp && qp_attr && qp_init_attr)
 519		qp = to_siw_qp(base_qp);
 520	else
 521		return -EINVAL;
 522
 523	ndev = ib_device_get_netdev(base_qp->device, SIW_PORT);
 524	if (!ndev)
 525		return -ENODEV;
 526
 527	qp_attr->qp_state = siw_qp_state_to_ib_qp_state[qp->attrs.state];
 528	qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
 529	qp_attr->cap.max_send_wr = qp->attrs.sq_size;
 530	qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
 531	qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
 532	qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
 533	qp_attr->path_mtu = ib_mtu_int_to_enum(READ_ONCE(ndev->mtu));
 534	qp_attr->max_rd_atomic = qp->attrs.irq_size;
 535	qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
 536
 537	qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
 538				   IB_ACCESS_REMOTE_WRITE |
 539				   IB_ACCESS_REMOTE_READ;
 540
 541	qp_init_attr->qp_type = base_qp->qp_type;
 542	qp_init_attr->send_cq = base_qp->send_cq;
 543	qp_init_attr->recv_cq = base_qp->recv_cq;
 544	qp_init_attr->srq = base_qp->srq;
 545
 546	qp_init_attr->cap = qp_attr->cap;
 547
 548	dev_put(ndev);
 549	return 0;
 550}
 551
 552int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
 553			int attr_mask, struct ib_udata *udata)
 554{
 555	struct siw_qp_attrs new_attrs;
 556	enum siw_qp_attr_mask siw_attr_mask = 0;
 557	struct siw_qp *qp = to_siw_qp(base_qp);
 558	int rv = 0;
 559
 560	if (!attr_mask)
 561		return 0;
 562
 563	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
 564		return -EOPNOTSUPP;
 565
 566	memset(&new_attrs, 0, sizeof(new_attrs));
 567
 568	if (attr_mask & IB_QP_ACCESS_FLAGS) {
 569		siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
 570
 571		if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
 572			new_attrs.flags |= SIW_RDMA_READ_ENABLED;
 573		if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
 574			new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
 575		if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
 576			new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
 577	}
 578	if (attr_mask & IB_QP_STATE) {
 579		siw_dbg_qp(qp, "desired IB QP state: %s\n",
 580			   ib_qp_state_to_string[attr->qp_state]);
 581
 582		new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
 583
 584		if (new_attrs.state > SIW_QP_STATE_RTS)
 585			qp->tx_ctx.tx_suspend = 1;
 586
 587		siw_attr_mask |= SIW_QP_ATTR_STATE;
 588	}
 589	if (!siw_attr_mask)
 590		goto out;
 591
 592	down_write(&qp->state_lock);
 593
 594	rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
 595
 596	up_write(&qp->state_lock);
 597out:
 598	return rv;
 599}
 600
 601int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
 602{
 603	struct siw_qp *qp = to_siw_qp(base_qp);
 604	struct siw_ucontext *uctx =
 605		rdma_udata_to_drv_context(udata, struct siw_ucontext,
 606					  base_ucontext);
 607	struct siw_qp_attrs qp_attrs;
 608
 609	siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
 610
 611	/*
 612	 * Mark QP as in process of destruction to prevent from
 613	 * any async callbacks to RDMA core
 614	 */
 615	qp->attrs.flags |= SIW_QP_IN_DESTROY;
 616	qp->rx_stream.rx_suspend = 1;
 617
 618	if (uctx) {
 619		rdma_user_mmap_entry_remove(qp->sq_entry);
 620		rdma_user_mmap_entry_remove(qp->rq_entry);
 621	}
 622
 623	down_write(&qp->state_lock);
 624
 625	qp_attrs.state = SIW_QP_STATE_ERROR;
 626	siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
 627
 628	if (qp->cep) {
 629		siw_cep_put(qp->cep);
 630		qp->cep = NULL;
 631	}
 632	up_write(&qp->state_lock);
 633
 634	qp->scq = qp->rcq = NULL;
 635
 636	siw_qp_put(qp);
 637	wait_for_completion(&qp->qp_free);
 638
 639	return 0;
 640}
 641
 642/*
 643 * siw_copy_inline_sgl()
 644 *
 645 * Prepare sgl of inlined data for sending. For userland callers
 646 * function checks if given buffer addresses and len's are within
 647 * process context bounds.
 648 * Data from all provided sge's are copied together into the wqe,
 649 * referenced by a single sge.
 650 */
 651static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
 652			       struct siw_sqe *sqe)
 653{
 654	struct ib_sge *core_sge = core_wr->sg_list;
 655	void *kbuf = &sqe->sge[1];
 656	int num_sge = core_wr->num_sge, bytes = 0;
 657
 658	sqe->sge[0].laddr = (uintptr_t)kbuf;
 659	sqe->sge[0].lkey = 0;
 660
 661	while (num_sge--) {
 662		if (!core_sge->length) {
 663			core_sge++;
 664			continue;
 665		}
 666		bytes += core_sge->length;
 667		if (bytes > SIW_MAX_INLINE) {
 668			bytes = -EINVAL;
 669			break;
 670		}
 671		memcpy(kbuf, ib_virt_dma_to_ptr(core_sge->addr),
 672		       core_sge->length);
 673
 674		kbuf += core_sge->length;
 675		core_sge++;
 676	}
 677	sqe->sge[0].length = max(bytes, 0);
 678	sqe->num_sge = bytes > 0 ? 1 : 0;
 679
 680	return bytes;
 681}
 682
 683/* Complete SQ WR's without processing */
 684static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
 685			   const struct ib_send_wr **bad_wr)
 686{
 687	int rv = 0;
 688
 689	while (wr) {
 690		struct siw_sqe sqe = {};
 691
 692		switch (wr->opcode) {
 693		case IB_WR_RDMA_WRITE:
 694			sqe.opcode = SIW_OP_WRITE;
 695			break;
 696		case IB_WR_RDMA_READ:
 697			sqe.opcode = SIW_OP_READ;
 698			break;
 699		case IB_WR_RDMA_READ_WITH_INV:
 700			sqe.opcode = SIW_OP_READ_LOCAL_INV;
 701			break;
 702		case IB_WR_SEND:
 703			sqe.opcode = SIW_OP_SEND;
 704			break;
 705		case IB_WR_SEND_WITH_IMM:
 706			sqe.opcode = SIW_OP_SEND_WITH_IMM;
 707			break;
 708		case IB_WR_SEND_WITH_INV:
 709			sqe.opcode = SIW_OP_SEND_REMOTE_INV;
 710			break;
 711		case IB_WR_LOCAL_INV:
 712			sqe.opcode = SIW_OP_INVAL_STAG;
 713			break;
 714		case IB_WR_REG_MR:
 715			sqe.opcode = SIW_OP_REG_MR;
 716			break;
 717		default:
 718			rv = -EINVAL;
 719			break;
 720		}
 721		if (!rv) {
 722			sqe.id = wr->wr_id;
 723			rv = siw_sqe_complete(qp, &sqe, 0,
 724					      SIW_WC_WR_FLUSH_ERR);
 725		}
 726		if (rv) {
 727			if (bad_wr)
 728				*bad_wr = wr;
 729			break;
 730		}
 731		wr = wr->next;
 732	}
 733	return rv;
 734}
 735
 736/* Complete RQ WR's without processing */
 737static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
 738			   const struct ib_recv_wr **bad_wr)
 739{
 740	struct siw_rqe rqe = {};
 741	int rv = 0;
 742
 743	while (wr) {
 744		rqe.id = wr->wr_id;
 745		rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
 746		if (rv) {
 747			if (bad_wr)
 748				*bad_wr = wr;
 749			break;
 750		}
 751		wr = wr->next;
 752	}
 753	return rv;
 754}
 755
 756/*
 757 * siw_post_send()
 758 *
 759 * Post a list of S-WR's to a SQ.
 760 *
 761 * @base_qp:	Base QP contained in siw QP
 762 * @wr:		Null terminated list of user WR's
 763 * @bad_wr:	Points to failing WR in case of synchronous failure.
 764 */
 765int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
 766		  const struct ib_send_wr **bad_wr)
 767{
 768	struct siw_qp *qp = to_siw_qp(base_qp);
 769	struct siw_wqe *wqe = tx_wqe(qp);
 770
 771	unsigned long flags;
 772	int rv = 0, imm_err = 0;
 773
 774	if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) {
 775		siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
 776		*bad_wr = wr;
 777		return -EINVAL;
 778	}
 779
 780	/*
 781	 * Try to acquire QP state lock. Must be non-blocking
 782	 * to accommodate kernel clients needs.
 783	 */
 784	if (!down_read_trylock(&qp->state_lock)) {
 785		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 786			/*
 787			 * ERROR state is final, so we can be sure
 788			 * this state will not change as long as the QP
 789			 * exists.
 790			 *
 791			 * This handles an ib_drain_sq() call with
 792			 * a concurrent request to set the QP state
 793			 * to ERROR.
 794			 */
 795			rv = siw_sq_flush_wr(qp, wr, bad_wr);
 796		} else {
 797			siw_dbg_qp(qp, "QP locked, state %d\n",
 798				   qp->attrs.state);
 799			*bad_wr = wr;
 800			rv = -ENOTCONN;
 801		}
 802		return rv;
 803	}
 804	if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
 805		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 806			/*
 807			 * Immediately flush this WR to CQ, if QP
 808			 * is in ERROR state. SQ is guaranteed to
 809			 * be empty, so WR complets in-order.
 810			 *
 811			 * Typically triggered by ib_drain_sq().
 812			 */
 813			rv = siw_sq_flush_wr(qp, wr, bad_wr);
 814		} else {
 815			siw_dbg_qp(qp, "QP out of state %d\n",
 816				   qp->attrs.state);
 817			*bad_wr = wr;
 818			rv = -ENOTCONN;
 819		}
 820		up_read(&qp->state_lock);
 821		return rv;
 822	}
 823	spin_lock_irqsave(&qp->sq_lock, flags);
 824
 825	while (wr) {
 826		u32 idx = qp->sq_put % qp->attrs.sq_size;
 827		struct siw_sqe *sqe = &qp->sendq[idx];
 828
 829		if (sqe->flags) {
 830			siw_dbg_qp(qp, "sq full\n");
 831			rv = -ENOMEM;
 832			break;
 833		}
 834		if (wr->num_sge > qp->attrs.sq_max_sges) {
 835			siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
 836			rv = -EINVAL;
 837			break;
 838		}
 839		sqe->id = wr->wr_id;
 840
 841		if ((wr->send_flags & IB_SEND_SIGNALED) ||
 842		    (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
 843			sqe->flags |= SIW_WQE_SIGNALLED;
 844
 845		if (wr->send_flags & IB_SEND_FENCE)
 846			sqe->flags |= SIW_WQE_READ_FENCE;
 847
 848		switch (wr->opcode) {
 849		case IB_WR_SEND:
 850		case IB_WR_SEND_WITH_INV:
 851			if (wr->send_flags & IB_SEND_SOLICITED)
 852				sqe->flags |= SIW_WQE_SOLICITED;
 853
 854			if (!(wr->send_flags & IB_SEND_INLINE)) {
 855				siw_copy_sgl(wr->sg_list, sqe->sge,
 856					     wr->num_sge);
 857				sqe->num_sge = wr->num_sge;
 858			} else {
 859				rv = siw_copy_inline_sgl(wr, sqe);
 860				if (rv <= 0) {
 861					rv = -EINVAL;
 862					break;
 863				}
 864				sqe->flags |= SIW_WQE_INLINE;
 865				sqe->num_sge = 1;
 866			}
 867			if (wr->opcode == IB_WR_SEND)
 868				sqe->opcode = SIW_OP_SEND;
 869			else {
 870				sqe->opcode = SIW_OP_SEND_REMOTE_INV;
 871				sqe->rkey = wr->ex.invalidate_rkey;
 872			}
 873			break;
 874
 875		case IB_WR_RDMA_READ_WITH_INV:
 876		case IB_WR_RDMA_READ:
 877			/*
 878			 * iWarp restricts RREAD sink to SGL containing
 879			 * 1 SGE only. we could relax to SGL with multiple
 880			 * elements referring the SAME ltag or even sending
 881			 * a private per-rreq tag referring to a checked
 882			 * local sgl with MULTIPLE ltag's.
 883			 */
 884			if (unlikely(wr->num_sge != 1)) {
 885				rv = -EINVAL;
 886				break;
 887			}
 888			siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
 889			/*
 890			 * NOTE: zero length RREAD is allowed!
 891			 */
 892			sqe->raddr = rdma_wr(wr)->remote_addr;
 893			sqe->rkey = rdma_wr(wr)->rkey;
 894			sqe->num_sge = 1;
 895
 896			if (wr->opcode == IB_WR_RDMA_READ)
 897				sqe->opcode = SIW_OP_READ;
 898			else
 899				sqe->opcode = SIW_OP_READ_LOCAL_INV;
 900			break;
 901
 902		case IB_WR_RDMA_WRITE:
 903			if (!(wr->send_flags & IB_SEND_INLINE)) {
 904				siw_copy_sgl(wr->sg_list, &sqe->sge[0],
 905					     wr->num_sge);
 906				sqe->num_sge = wr->num_sge;
 907			} else {
 908				rv = siw_copy_inline_sgl(wr, sqe);
 909				if (unlikely(rv < 0)) {
 910					rv = -EINVAL;
 911					break;
 912				}
 913				sqe->flags |= SIW_WQE_INLINE;
 914				sqe->num_sge = 1;
 915			}
 916			sqe->raddr = rdma_wr(wr)->remote_addr;
 917			sqe->rkey = rdma_wr(wr)->rkey;
 918			sqe->opcode = SIW_OP_WRITE;
 919			break;
 920
 921		case IB_WR_REG_MR:
 922			sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
 923			sqe->rkey = reg_wr(wr)->key;
 924			sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
 925			sqe->opcode = SIW_OP_REG_MR;
 926			break;
 927
 928		case IB_WR_LOCAL_INV:
 929			sqe->rkey = wr->ex.invalidate_rkey;
 930			sqe->opcode = SIW_OP_INVAL_STAG;
 931			break;
 932
 933		default:
 934			siw_dbg_qp(qp, "ib wr type %d unsupported\n",
 935				   wr->opcode);
 936			rv = -EINVAL;
 937			break;
 938		}
 939		siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%p\n",
 940			   sqe->opcode, sqe->flags,
 941			   (void *)(uintptr_t)sqe->id);
 942
 943		if (unlikely(rv < 0))
 944			break;
 945
 946		/* make SQE only valid after completely written */
 947		smp_wmb();
 948		sqe->flags |= SIW_WQE_VALID;
 949
 950		qp->sq_put++;
 951		wr = wr->next;
 952	}
 953
 954	/*
 955	 * Send directly if SQ processing is not in progress.
 956	 * Eventual immediate errors (rv < 0) do not affect the involved
 957	 * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
 958	 * processing, if new work is already pending. But rv and pointer
 959	 * to failed work request must be passed to caller.
 960	 */
 961	if (unlikely(rv < 0)) {
 962		/*
 963		 * Immediate error
 964		 */
 965		siw_dbg_qp(qp, "Immediate error %d\n", rv);
 966		imm_err = rv;
 967		*bad_wr = wr;
 968	}
 969	if (wqe->wr_status != SIW_WR_IDLE) {
 970		spin_unlock_irqrestore(&qp->sq_lock, flags);
 971		goto skip_direct_sending;
 972	}
 973	rv = siw_activate_tx(qp);
 974	spin_unlock_irqrestore(&qp->sq_lock, flags);
 975
 976	if (rv <= 0)
 977		goto skip_direct_sending;
 978
 979	if (rdma_is_kernel_res(&qp->base_qp.res)) {
 980		rv = siw_sq_start(qp);
 981	} else {
 982		qp->tx_ctx.in_syscall = 1;
 983
 984		if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
 985			siw_qp_cm_drop(qp, 0);
 986
 987		qp->tx_ctx.in_syscall = 0;
 988	}
 989skip_direct_sending:
 990
 991	up_read(&qp->state_lock);
 992
 993	if (unlikely(imm_err))
 994		return imm_err;
 995
 996	return (rv >= 0) ? 0 : rv;
 997}
 998
 999/*
1000 * siw_post_receive()
1001 *
1002 * Post a list of R-WR's to a RQ.
1003 *
1004 * @base_qp:	Base QP contained in siw QP
1005 * @wr:		Null terminated list of user WR's
1006 * @bad_wr:	Points to failing WR in case of synchronous failure.
1007 */
1008int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
1009		     const struct ib_recv_wr **bad_wr)
1010{
1011	struct siw_qp *qp = to_siw_qp(base_qp);
1012	unsigned long flags;
1013	int rv = 0;
1014
1015	if (qp->srq || qp->attrs.rq_size == 0) {
1016		*bad_wr = wr;
1017		return -EINVAL;
1018	}
1019	if (!rdma_is_kernel_res(&qp->base_qp.res)) {
1020		siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
1021		*bad_wr = wr;
1022		return -EINVAL;
1023	}
1024
1025	/*
1026	 * Try to acquire QP state lock. Must be non-blocking
1027	 * to accommodate kernel clients needs.
1028	 */
1029	if (!down_read_trylock(&qp->state_lock)) {
1030		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1031			/*
1032			 * ERROR state is final, so we can be sure
1033			 * this state will not change as long as the QP
1034			 * exists.
1035			 *
1036			 * This handles an ib_drain_rq() call with
1037			 * a concurrent request to set the QP state
1038			 * to ERROR.
1039			 */
1040			rv = siw_rq_flush_wr(qp, wr, bad_wr);
1041		} else {
1042			siw_dbg_qp(qp, "QP locked, state %d\n",
1043				   qp->attrs.state);
1044			*bad_wr = wr;
1045			rv = -ENOTCONN;
1046		}
1047		return rv;
1048	}
1049	if (qp->attrs.state > SIW_QP_STATE_RTS) {
1050		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1051			/*
1052			 * Immediately flush this WR to CQ, if QP
1053			 * is in ERROR state. RQ is guaranteed to
1054			 * be empty, so WR complets in-order.
1055			 *
1056			 * Typically triggered by ib_drain_rq().
1057			 */
1058			rv = siw_rq_flush_wr(qp, wr, bad_wr);
1059		} else {
1060			siw_dbg_qp(qp, "QP out of state %d\n",
1061				   qp->attrs.state);
1062			*bad_wr = wr;
1063			rv = -ENOTCONN;
1064		}
1065		up_read(&qp->state_lock);
1066		return rv;
1067	}
1068	/*
1069	 * Serialize potentially multiple producers.
1070	 * Not needed for single threaded consumer side.
1071	 */
1072	spin_lock_irqsave(&qp->rq_lock, flags);
1073
1074	while (wr) {
1075		u32 idx = qp->rq_put % qp->attrs.rq_size;
1076		struct siw_rqe *rqe = &qp->recvq[idx];
1077
1078		if (rqe->flags) {
1079			siw_dbg_qp(qp, "RQ full\n");
1080			rv = -ENOMEM;
1081			break;
1082		}
1083		if (wr->num_sge > qp->attrs.rq_max_sges) {
1084			siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1085			rv = -EINVAL;
1086			break;
1087		}
1088		rqe->id = wr->wr_id;
1089		rqe->num_sge = wr->num_sge;
1090		siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1091
1092		/* make sure RQE is completely written before valid */
1093		smp_wmb();
1094
1095		rqe->flags = SIW_WQE_VALID;
1096
1097		qp->rq_put++;
1098		wr = wr->next;
1099	}
1100	spin_unlock_irqrestore(&qp->rq_lock, flags);
1101
1102	up_read(&qp->state_lock);
1103
1104	if (rv < 0) {
1105		siw_dbg_qp(qp, "error %d\n", rv);
1106		*bad_wr = wr;
1107	}
1108	return rv;
1109}
1110
1111int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1112{
1113	struct siw_cq *cq = to_siw_cq(base_cq);
1114	struct siw_device *sdev = to_siw_dev(base_cq->device);
1115	struct siw_ucontext *ctx =
1116		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1117					  base_ucontext);
1118
1119	siw_dbg_cq(cq, "free CQ resources\n");
1120
1121	siw_cq_flush(cq);
1122
1123	if (ctx)
1124		rdma_user_mmap_entry_remove(cq->cq_entry);
1125
1126	atomic_dec(&sdev->num_cq);
1127
1128	vfree(cq->queue);
1129	return 0;
1130}
1131
1132/*
1133 * siw_create_cq()
1134 *
1135 * Populate CQ of requested size
1136 *
1137 * @base_cq: CQ as allocated by RDMA midlayer
1138 * @attr: Initial CQ attributes
1139 * @attrs: uverbs bundle
1140 */
1141
1142int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1143		  struct uverbs_attr_bundle *attrs)
1144{
1145	struct ib_udata *udata = &attrs->driver_udata;
1146	struct siw_device *sdev = to_siw_dev(base_cq->device);
1147	struct siw_cq *cq = to_siw_cq(base_cq);
1148	int rv, size = attr->cqe;
1149
1150	if (attr->flags)
1151		return -EOPNOTSUPP;
1152
1153	if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1154		siw_dbg(base_cq->device, "too many CQ's\n");
1155		rv = -ENOMEM;
1156		goto err_out;
1157	}
1158	if (size < 1 || size > sdev->attrs.max_cqe) {
1159		siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1160		rv = -EINVAL;
1161		goto err_out;
1162	}
1163	size = roundup_pow_of_two(size);
1164	cq->base_cq.cqe = size;
1165	cq->num_cqe = size;
1166
1167	if (udata)
1168		cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1169					 sizeof(struct siw_cq_ctrl));
1170	else
1171		cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1172				    sizeof(struct siw_cq_ctrl));
1173
1174	if (cq->queue == NULL) {
1175		rv = -ENOMEM;
1176		goto err_out;
1177	}
1178	get_random_bytes(&cq->id, 4);
1179	siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1180
1181	spin_lock_init(&cq->lock);
1182
1183	cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1184
1185	if (udata) {
1186		struct siw_uresp_create_cq uresp = {};
1187		struct siw_ucontext *ctx =
1188			rdma_udata_to_drv_context(udata, struct siw_ucontext,
1189						  base_ucontext);
1190		size_t length = size * sizeof(struct siw_cqe) +
1191			sizeof(struct siw_cq_ctrl);
1192
1193		cq->cq_entry =
1194			siw_mmap_entry_insert(ctx, cq->queue,
1195					      length, &uresp.cq_key);
1196		if (!cq->cq_entry) {
1197			rv = -ENOMEM;
1198			goto err_out;
1199		}
1200
1201		uresp.cq_id = cq->id;
1202		uresp.num_cqe = size;
1203
1204		if (udata->outlen < sizeof(uresp)) {
1205			rv = -EINVAL;
1206			goto err_out;
1207		}
1208		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1209		if (rv)
1210			goto err_out;
1211	}
1212	return 0;
1213
1214err_out:
1215	siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1216
1217	if (cq->queue) {
1218		struct siw_ucontext *ctx =
1219			rdma_udata_to_drv_context(udata, struct siw_ucontext,
1220						  base_ucontext);
1221		if (ctx)
1222			rdma_user_mmap_entry_remove(cq->cq_entry);
1223		vfree(cq->queue);
1224	}
1225	atomic_dec(&sdev->num_cq);
1226
1227	return rv;
1228}
1229
1230/*
1231 * siw_poll_cq()
1232 *
1233 * Reap CQ entries if available and copy work completion status into
1234 * array of WC's provided by caller. Returns number of reaped CQE's.
1235 *
1236 * @base_cq:	Base CQ contained in siw CQ.
1237 * @num_cqe:	Maximum number of CQE's to reap.
1238 * @wc:		Array of work completions to be filled by siw.
1239 */
1240int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1241{
1242	struct siw_cq *cq = to_siw_cq(base_cq);
1243	int i;
1244
1245	for (i = 0; i < num_cqe; i++) {
1246		if (!siw_reap_cqe(cq, wc))
1247			break;
1248		wc++;
1249	}
1250	return i;
1251}
1252
1253/*
1254 * siw_req_notify_cq()
1255 *
1256 * Request notification for new CQE's added to that CQ.
1257 * Defined flags:
1258 * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1259 *   event if a WQE with notification flag set enters the CQ
1260 * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1261 *   event if a WQE enters the CQ.
1262 * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1263 *   number of not reaped CQE's regardless of its notification
1264 *   type and current or new CQ notification settings.
1265 *
1266 * @base_cq:	Base CQ contained in siw CQ.
1267 * @flags:	Requested notification flags.
1268 */
1269int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1270{
1271	struct siw_cq *cq = to_siw_cq(base_cq);
1272
1273	siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1274
1275	if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1276		/*
1277		 * Enable CQ event for next solicited completion.
1278		 * and make it visible to all associated producers.
1279		 */
1280		smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1281	else
1282		/*
1283		 * Enable CQ event for any signalled completion.
1284		 * and make it visible to all associated producers.
1285		 */
1286		smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1287
1288	if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1289		return cq->cq_put - cq->cq_get;
1290
1291	return 0;
1292}
1293
1294/*
1295 * siw_dereg_mr()
1296 *
1297 * Release Memory Region.
1298 *
1299 * @base_mr: Base MR contained in siw MR.
1300 * @udata: points to user context, unused.
1301 */
1302int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1303{
1304	struct siw_mr *mr = to_siw_mr(base_mr);
1305	struct siw_device *sdev = to_siw_dev(base_mr->device);
1306
1307	siw_dbg_mem(mr->mem, "deregister MR\n");
1308
1309	atomic_dec(&sdev->num_mr);
1310
1311	siw_mr_drop_mem(mr);
1312	kfree_rcu(mr, rcu);
1313
1314	return 0;
1315}
1316
1317/*
1318 * siw_reg_user_mr()
1319 *
1320 * Register Memory Region.
1321 *
1322 * @pd:		Protection Domain
1323 * @start:	starting address of MR (virtual address)
1324 * @len:	len of MR
1325 * @rnic_va:	not used by siw
1326 * @rights:	MR access rights
1327 * @dmah:	dma handle
1328 * @udata:	user buffer to communicate STag and Key.
1329 */
1330struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1331			      u64 rnic_va, int rights,  struct ib_dmah *dmah,
1332			      struct ib_udata *udata)
1333{
1334	struct siw_mr *mr = NULL;
1335	struct siw_umem *umem = NULL;
1336	struct siw_ureq_reg_mr ureq;
1337	struct siw_device *sdev = to_siw_dev(pd->device);
1338	int rv;
1339
1340	siw_dbg_pd(pd, "start: 0x%p, va: 0x%p, len: %llu\n",
1341		   (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1342		   (unsigned long long)len);
1343
1344	if (dmah)
1345		return ERR_PTR(-EOPNOTSUPP);
1346
1347	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1348		siw_dbg_pd(pd, "too many mr's\n");
1349		rv = -ENOMEM;
1350		goto err_out;
1351	}
1352	if (!len) {
1353		rv = -EINVAL;
1354		goto err_out;
1355	}
1356	umem = siw_umem_get(pd->device, start, len, rights);
1357	if (IS_ERR(umem)) {
1358		rv = PTR_ERR(umem);
1359		siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1360		umem = NULL;
1361		goto err_out;
1362	}
1363	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1364	if (!mr) {
1365		rv = -ENOMEM;
1366		goto err_out;
1367	}
1368	rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1369	if (rv)
1370		goto err_out;
1371
1372	if (udata) {
1373		struct siw_uresp_reg_mr uresp = {};
1374		struct siw_mem *mem = mr->mem;
1375
1376		if (udata->inlen < sizeof(ureq)) {
1377			rv = -EINVAL;
1378			goto err_out;
1379		}
1380		rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1381		if (rv)
1382			goto err_out;
1383
1384		mr->base_mr.lkey |= ureq.stag_key;
1385		mr->base_mr.rkey |= ureq.stag_key;
1386		mem->stag |= ureq.stag_key;
1387		uresp.stag = mem->stag;
1388
1389		if (udata->outlen < sizeof(uresp)) {
1390			rv = -EINVAL;
1391			goto err_out;
1392		}
1393		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1394		if (rv)
1395			goto err_out;
1396	}
1397	mr->mem->stag_valid = 1;
1398
1399	return &mr->base_mr;
1400
1401err_out:
1402	atomic_dec(&sdev->num_mr);
1403	if (mr) {
1404		if (mr->mem)
1405			siw_mr_drop_mem(mr);
1406		kfree_rcu(mr, rcu);
1407	} else {
1408		if (umem)
1409			siw_umem_release(umem);
1410	}
1411	return ERR_PTR(rv);
1412}
1413
1414struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1415			   u32 max_sge)
1416{
1417	struct siw_device *sdev = to_siw_dev(pd->device);
1418	struct siw_mr *mr = NULL;
1419	struct siw_pbl *pbl = NULL;
1420	int rv;
1421
1422	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1423		siw_dbg_pd(pd, "too many mr's\n");
1424		rv = -ENOMEM;
1425		goto err_out;
1426	}
1427	if (mr_type != IB_MR_TYPE_MEM_REG) {
1428		siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1429		rv = -EOPNOTSUPP;
1430		goto err_out;
1431	}
1432	if (max_sge > SIW_MAX_SGE_PBL) {
1433		siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1434		rv = -ENOMEM;
1435		goto err_out;
1436	}
1437	pbl = siw_pbl_alloc(max_sge);
1438	if (IS_ERR(pbl)) {
1439		rv = PTR_ERR(pbl);
1440		siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1441		pbl = NULL;
1442		goto err_out;
1443	}
1444	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1445	if (!mr) {
1446		rv = -ENOMEM;
1447		goto err_out;
1448	}
1449	rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1450	if (rv)
1451		goto err_out;
1452
1453	mr->mem->is_pbl = 1;
1454
1455	siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1456
1457	return &mr->base_mr;
1458
1459err_out:
1460	atomic_dec(&sdev->num_mr);
1461
1462	if (!mr) {
1463		kfree(pbl);
1464	} else {
1465		if (mr->mem)
1466			siw_mr_drop_mem(mr);
1467		kfree_rcu(mr, rcu);
1468	}
1469	siw_dbg_pd(pd, "failed: %d\n", rv);
1470
1471	return ERR_PTR(rv);
1472}
1473
1474/* Just used to count number of pages being mapped */
1475static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1476{
1477	return 0;
1478}
1479
1480int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1481		  unsigned int *sg_off)
1482{
1483	struct scatterlist *slp;
1484	struct siw_mr *mr = to_siw_mr(base_mr);
1485	struct siw_mem *mem = mr->mem;
1486	struct siw_pbl *pbl = mem->pbl;
1487	struct siw_pble *pble;
1488	unsigned long pbl_size;
1489	int i, rv;
1490
1491	if (!pbl) {
1492		siw_dbg_mem(mem, "no PBL allocated\n");
1493		return -EINVAL;
1494	}
1495	pble = pbl->pbe;
1496
1497	if (pbl->max_buf < num_sle) {
1498		siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1499			    num_sle, pbl->max_buf);
1500		return -ENOMEM;
1501	}
1502	for_each_sg(sl, slp, num_sle, i) {
1503		if (sg_dma_len(slp) == 0) {
1504			siw_dbg_mem(mem, "empty SGE\n");
1505			return -EINVAL;
1506		}
1507		if (i == 0) {
1508			pble->addr = sg_dma_address(slp);
1509			pble->size = sg_dma_len(slp);
1510			pble->pbl_off = 0;
1511			pbl_size = pble->size;
1512			pbl->num_buf = 1;
1513		} else {
1514			/* Merge PBL entries if adjacent */
1515			if (pble->addr + pble->size == sg_dma_address(slp)) {
1516				pble->size += sg_dma_len(slp);
1517			} else {
1518				pble++;
1519				pbl->num_buf++;
1520				pble->addr = sg_dma_address(slp);
1521				pble->size = sg_dma_len(slp);
1522				pble->pbl_off = pbl_size;
1523			}
1524			pbl_size += sg_dma_len(slp);
1525		}
1526		siw_dbg_mem(mem,
1527			"sge[%d], size %u, addr 0x%p, total %lu\n",
1528			i, pble->size, ib_virt_dma_to_ptr(pble->addr),
1529			pbl_size);
1530	}
1531	rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1532	if (rv > 0) {
1533		mem->len = base_mr->length;
1534		mem->va = base_mr->iova;
1535		siw_dbg_mem(mem,
1536			"%llu bytes, start 0x%p, %u SLE to %u entries\n",
1537			mem->len, (void *)(uintptr_t)mem->va, num_sle,
1538			pbl->num_buf);
1539	}
1540	return rv;
1541}
1542
1543/*
1544 * siw_get_dma_mr()
1545 *
1546 * Create a (empty) DMA memory region, where no umem is attached.
1547 */
1548struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1549{
1550	struct siw_device *sdev = to_siw_dev(pd->device);
1551	struct siw_mr *mr = NULL;
1552	int rv;
1553
1554	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1555		siw_dbg_pd(pd, "too many mr's\n");
1556		rv = -ENOMEM;
1557		goto err_out;
1558	}
1559	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1560	if (!mr) {
1561		rv = -ENOMEM;
1562		goto err_out;
1563	}
1564	rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1565	if (rv)
1566		goto err_out;
1567
1568	mr->mem->stag_valid = 1;
1569
1570	siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1571
1572	return &mr->base_mr;
1573
1574err_out:
1575	if (rv)
1576		kfree(mr);
1577
1578	atomic_dec(&sdev->num_mr);
1579
1580	return ERR_PTR(rv);
1581}
1582
1583/*
1584 * siw_create_srq()
1585 *
1586 * Create Shared Receive Queue of attributes @init_attrs
1587 * within protection domain given by @pd.
1588 *
1589 * @base_srq:	Base SRQ contained in siw SRQ.
1590 * @init_attrs:	SRQ init attributes.
1591 * @udata:	points to user context
1592 */
1593int siw_create_srq(struct ib_srq *base_srq,
1594		   struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1595{
1596	struct siw_srq *srq = to_siw_srq(base_srq);
1597	struct ib_srq_attr *attrs = &init_attrs->attr;
1598	struct siw_device *sdev = to_siw_dev(base_srq->device);
1599	struct siw_ucontext *ctx =
1600		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1601					  base_ucontext);
1602	int rv;
1603
1604	if (init_attrs->srq_type != IB_SRQT_BASIC)
1605		return -EOPNOTSUPP;
1606
1607	if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1608		siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1609		rv = -ENOMEM;
1610		goto err_out;
1611	}
1612	if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1613	    attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1614		rv = -EINVAL;
1615		goto err_out;
1616	}
1617	srq->max_sge = attrs->max_sge;
1618	srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1619	srq->limit = attrs->srq_limit;
1620	if (srq->limit)
1621		srq->armed = true;
1622
1623	srq->is_kernel_res = !udata;
1624
1625	if (udata)
1626		srq->recvq =
1627			vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1628	else
1629		srq->recvq = vcalloc(srq->num_rqe, sizeof(struct siw_rqe));
1630
1631	if (srq->recvq == NULL) {
1632		rv = -ENOMEM;
1633		goto err_out;
1634	}
1635	if (udata) {
1636		struct siw_uresp_create_srq uresp = {};
1637		size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1638
1639		srq->srq_entry =
1640			siw_mmap_entry_insert(ctx, srq->recvq,
1641					      length, &uresp.srq_key);
1642		if (!srq->srq_entry) {
1643			rv = -ENOMEM;
1644			goto err_out;
1645		}
1646
1647		uresp.num_rqe = srq->num_rqe;
1648
1649		if (udata->outlen < sizeof(uresp)) {
1650			rv = -EINVAL;
1651			goto err_out;
1652		}
1653		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1654		if (rv)
1655			goto err_out;
1656	}
1657	spin_lock_init(&srq->lock);
1658
1659	siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1660
1661	return 0;
1662
1663err_out:
1664	if (srq->recvq) {
1665		if (ctx)
1666			rdma_user_mmap_entry_remove(srq->srq_entry);
1667		vfree(srq->recvq);
1668	}
1669	atomic_dec(&sdev->num_srq);
1670
1671	return rv;
1672}
1673
1674/*
1675 * siw_modify_srq()
1676 *
1677 * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1678 * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1679 *
1680 * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1681 * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1682 */
1683int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1684		   enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1685{
1686	struct siw_srq *srq = to_siw_srq(base_srq);
1687	unsigned long flags;
1688	int rv = 0;
1689
1690	spin_lock_irqsave(&srq->lock, flags);
1691
1692	if (attr_mask & IB_SRQ_MAX_WR) {
1693		/* resize request not yet supported */
1694		rv = -EOPNOTSUPP;
1695		goto out;
1696	}
1697	if (attr_mask & IB_SRQ_LIMIT) {
1698		if (attrs->srq_limit) {
1699			if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1700				rv = -EINVAL;
1701				goto out;
1702			}
1703			srq->armed = true;
1704		} else {
1705			srq->armed = false;
1706		}
1707		srq->limit = attrs->srq_limit;
1708	}
1709out:
1710	spin_unlock_irqrestore(&srq->lock, flags);
1711
1712	return rv;
1713}
1714
1715/*
1716 * siw_query_srq()
1717 *
1718 * Query SRQ attributes.
1719 */
1720int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1721{
1722	struct siw_srq *srq = to_siw_srq(base_srq);
1723	unsigned long flags;
1724
1725	spin_lock_irqsave(&srq->lock, flags);
1726
1727	attrs->max_wr = srq->num_rqe;
1728	attrs->max_sge = srq->max_sge;
1729	attrs->srq_limit = srq->limit;
1730
1731	spin_unlock_irqrestore(&srq->lock, flags);
1732
1733	return 0;
1734}
1735
1736/*
1737 * siw_destroy_srq()
1738 *
1739 * Destroy SRQ.
1740 * It is assumed that the SRQ is not referenced by any
1741 * QP anymore - the code trusts the RDMA core environment to keep track
1742 * of QP references.
1743 */
1744int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1745{
1746	struct siw_srq *srq = to_siw_srq(base_srq);
1747	struct siw_device *sdev = to_siw_dev(base_srq->device);
1748	struct siw_ucontext *ctx =
1749		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1750					  base_ucontext);
1751
1752	if (ctx)
1753		rdma_user_mmap_entry_remove(srq->srq_entry);
1754	vfree(srq->recvq);
1755	atomic_dec(&sdev->num_srq);
1756	return 0;
1757}
1758
1759/*
1760 * siw_post_srq_recv()
1761 *
1762 * Post a list of receive queue elements to SRQ.
1763 * NOTE: The function does not check or lock a certain SRQ state
1764 *       during the post operation. The code simply trusts the
1765 *       RDMA core environment.
1766 *
1767 * @base_srq:	Base SRQ contained in siw SRQ
1768 * @wr:		List of R-WR's
1769 * @bad_wr:	Updated to failing WR if posting fails.
1770 */
1771int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1772		      const struct ib_recv_wr **bad_wr)
1773{
1774	struct siw_srq *srq = to_siw_srq(base_srq);
1775	unsigned long flags;
1776	int rv = 0;
1777
1778	if (unlikely(!srq->is_kernel_res)) {
1779		siw_dbg_pd(base_srq->pd,
1780			   "[SRQ]: no kernel post_recv for mapped srq\n");
1781		rv = -EINVAL;
1782		goto out;
1783	}
1784	/*
1785	 * Serialize potentially multiple producers.
1786	 * Also needed to serialize potentially multiple
1787	 * consumers.
1788	 */
1789	spin_lock_irqsave(&srq->lock, flags);
1790
1791	while (wr) {
1792		u32 idx = srq->rq_put % srq->num_rqe;
1793		struct siw_rqe *rqe = &srq->recvq[idx];
1794
1795		if (rqe->flags) {
1796			siw_dbg_pd(base_srq->pd, "SRQ full\n");
1797			rv = -ENOMEM;
1798			break;
1799		}
1800		if (unlikely(wr->num_sge > srq->max_sge)) {
1801			siw_dbg_pd(base_srq->pd,
1802				   "[SRQ]: too many sge's: %d\n", wr->num_sge);
1803			rv = -EINVAL;
1804			break;
1805		}
1806		rqe->id = wr->wr_id;
1807		rqe->num_sge = wr->num_sge;
1808		siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1809
1810		/* Make sure S-RQE is completely written before valid */
1811		smp_wmb();
1812
1813		rqe->flags = SIW_WQE_VALID;
1814
1815		srq->rq_put++;
1816		wr = wr->next;
1817	}
1818	spin_unlock_irqrestore(&srq->lock, flags);
1819out:
1820	if (unlikely(rv < 0)) {
1821		siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1822		*bad_wr = wr;
1823	}
1824	return rv;
1825}
1826
1827void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1828{
1829	struct ib_event event;
1830	struct ib_qp *base_qp = &qp->base_qp;
1831
1832	/*
1833	 * Do not report asynchronous errors on QP which gets
1834	 * destroyed via verbs interface (siw_destroy_qp())
1835	 */
1836	if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1837		return;
1838
1839	event.event = etype;
1840	event.device = base_qp->device;
1841	event.element.qp = base_qp;
1842
1843	if (base_qp->event_handler) {
1844		siw_dbg_qp(qp, "reporting event %d\n", etype);
1845		base_qp->event_handler(&event, base_qp->qp_context);
1846	}
1847}
1848
1849void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1850{
1851	struct ib_event event;
1852	struct ib_cq *base_cq = &cq->base_cq;
1853
1854	event.event = etype;
1855	event.device = base_cq->device;
1856	event.element.cq = base_cq;
1857
1858	if (base_cq->event_handler) {
1859		siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1860		base_cq->event_handler(&event, base_cq->cq_context);
1861	}
1862}
1863
1864void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1865{
1866	struct ib_event event;
1867	struct ib_srq *base_srq = &srq->base_srq;
1868
1869	event.event = etype;
1870	event.device = base_srq->device;
1871	event.element.srq = base_srq;
1872
1873	if (base_srq->event_handler) {
1874		siw_dbg_pd(srq->base_srq.pd,
1875			   "reporting SRQ event %d\n", etype);
1876		base_srq->event_handler(&event, base_srq->srq_context);
1877	}
1878}
1879
1880void siw_port_event(struct siw_device *sdev, u32 port, enum ib_event_type etype)
1881{
1882	struct ib_event event;
1883
1884	event.event = etype;
1885	event.device = &sdev->base_dev;
1886	event.element.port_num = port;
1887
1888	siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1889
1890	ib_dispatch_event(&event);
1891}