rxe_verbs.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/*
   3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
   4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
   5 */
   6
   7#include <linux/dma-mapping.h>
   8#include <net/addrconf.h>
   9#include <rdma/uverbs_ioctl.h>
  10
  11#include "rxe.h"
  12#include "rxe_queue.h"
  13#include "rxe_hw_counters.h"
  14
  15static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr);
  16
  17/* dev */
  18static int rxe_query_device(struct ib_device *ibdev,
  19			    struct ib_device_attr *attr,
  20			    struct ib_udata *udata)
  21{
  22	struct rxe_dev *rxe = to_rdev(ibdev);
  23	int err;
  24
  25	if (udata->inlen || udata->outlen) {
  26		rxe_dbg_dev(rxe, "malformed udata\n");
  27		err = -EINVAL;
  28		goto err_out;
  29	}
  30
  31	memcpy(attr, &rxe->attr, sizeof(*attr));
  32
  33	return 0;
  34
  35err_out:
  36	rxe_err_dev(rxe, "returned err = %d\n", err);
  37	return err;
  38}
  39
  40static int rxe_query_port(struct ib_device *ibdev,
  41			  u32 port_num, struct ib_port_attr *attr)
  42{
  43	struct rxe_dev *rxe = to_rdev(ibdev);
  44	struct net_device *ndev;
  45	int err, ret;
  46
  47	if (port_num != 1) {
  48		err = -EINVAL;
  49		rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
  50		goto err_out;
  51	}
  52
  53	ndev = rxe_ib_device_get_netdev(ibdev);
  54	if (!ndev) {
  55		err = -ENODEV;
  56		goto err_out;
  57	}
  58
  59	memcpy(attr, &rxe->port.attr, sizeof(*attr));
  60
  61	mutex_lock(&rxe->usdev_lock);
  62	ret = ib_get_eth_speed(ibdev, port_num, &attr->active_speed,
  63			       &attr->active_width);
  64
  65	attr->state = ib_get_curr_port_state(ndev);
  66	if (attr->state == IB_PORT_ACTIVE)
  67		attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
  68	else if (netif_get_flags(ndev) & IFF_UP)
  69		attr->phys_state = IB_PORT_PHYS_STATE_POLLING;
  70	else
  71		attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
  72
  73	mutex_unlock(&rxe->usdev_lock);
  74
  75	dev_put(ndev);
  76	return ret;
  77
  78err_out:
  79	rxe_err_dev(rxe, "returned err = %d\n", err);
  80	return err;
  81}
  82
  83static int rxe_query_gid(struct ib_device *ibdev, u32 port, int idx,
  84			 union ib_gid *gid)
  85{
  86	struct rxe_dev *rxe = to_rdev(ibdev);
  87
  88	/* subnet_prefix == interface_id == 0; */
  89	memset(gid, 0, sizeof(*gid));
  90	memcpy(gid->raw, rxe->raw_gid, ETH_ALEN);
  91
  92	return 0;
  93}
  94
  95static int rxe_query_pkey(struct ib_device *ibdev,
  96			  u32 port_num, u16 index, u16 *pkey)
  97{
  98	struct rxe_dev *rxe = to_rdev(ibdev);
  99	int err;
 100
 101	if (index != 0) {
 102		err = -EINVAL;
 103		rxe_dbg_dev(rxe, "bad pkey index = %d\n", index);
 104		goto err_out;
 105	}
 106
 107	*pkey = IB_DEFAULT_PKEY_FULL;
 108	return 0;
 109
 110err_out:
 111	rxe_err_dev(rxe, "returned err = %d\n", err);
 112	return err;
 113}
 114
 115static int rxe_modify_device(struct ib_device *ibdev,
 116			     int mask, struct ib_device_modify *attr)
 117{
 118	struct rxe_dev *rxe = to_rdev(ibdev);
 119	int err;
 120
 121	if (mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID |
 122		     IB_DEVICE_MODIFY_NODE_DESC)) {
 123		err = -EOPNOTSUPP;
 124		rxe_dbg_dev(rxe, "unsupported mask = 0x%x\n", mask);
 125		goto err_out;
 126	}
 127
 128	if (mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID)
 129		rxe->attr.sys_image_guid = cpu_to_be64(attr->sys_image_guid);
 130
 131	if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
 132		memcpy(rxe->ib_dev.node_desc,
 133		       attr->node_desc, sizeof(rxe->ib_dev.node_desc));
 134	}
 135
 136	return 0;
 137
 138err_out:
 139	rxe_err_dev(rxe, "returned err = %d\n", err);
 140	return err;
 141}
 142
 143static int rxe_modify_port(struct ib_device *ibdev, u32 port_num,
 144			   int mask, struct ib_port_modify *attr)
 145{
 146	struct rxe_dev *rxe = to_rdev(ibdev);
 147	struct rxe_port *port;
 148	int err;
 149
 150	if (port_num != 1) {
 151		err = -EINVAL;
 152		rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
 153		goto err_out;
 154	}
 155
 156	//TODO is shutdown useful
 157	if (mask & ~(IB_PORT_RESET_QKEY_CNTR)) {
 158		err = -EOPNOTSUPP;
 159		rxe_dbg_dev(rxe, "unsupported mask = 0x%x\n", mask);
 160		goto err_out;
 161	}
 162
 163	port = &rxe->port;
 164	port->attr.port_cap_flags |= attr->set_port_cap_mask;
 165	port->attr.port_cap_flags &= ~attr->clr_port_cap_mask;
 166
 167	if (mask & IB_PORT_RESET_QKEY_CNTR)
 168		port->attr.qkey_viol_cntr = 0;
 169
 170	return 0;
 171
 172err_out:
 173	rxe_err_dev(rxe, "returned err = %d\n", err);
 174	return err;
 175}
 176
 177static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev,
 178					       u32 port_num)
 179{
 180	struct rxe_dev *rxe = to_rdev(ibdev);
 181	int err;
 182
 183	if (port_num != 1) {
 184		err = -EINVAL;
 185		rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
 186		goto err_out;
 187	}
 188
 189	return IB_LINK_LAYER_ETHERNET;
 190
 191err_out:
 192	rxe_err_dev(rxe, "returned err = %d\n", err);
 193	return err;
 194}
 195
 196static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num,
 197			      struct ib_port_immutable *immutable)
 198{
 199	struct rxe_dev *rxe = to_rdev(ibdev);
 200	struct ib_port_attr attr = {};
 201	int err;
 202
 203	if (port_num != 1) {
 204		err = -EINVAL;
 205		rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num);
 206		goto err_out;
 207	}
 208
 209	err = ib_query_port(ibdev, port_num, &attr);
 210	if (err)
 211		goto err_out;
 212
 213	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
 214	immutable->pkey_tbl_len = attr.pkey_tbl_len;
 215	immutable->gid_tbl_len = attr.gid_tbl_len;
 216	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
 217
 218	return 0;
 219
 220err_out:
 221	rxe_err_dev(rxe, "returned err = %d\n", err);
 222	return err;
 223}
 224
 225/* uc */
 226static int rxe_alloc_ucontext(struct ib_ucontext *ibuc, struct ib_udata *udata)
 227{
 228	struct rxe_dev *rxe = to_rdev(ibuc->device);
 229	struct rxe_ucontext *uc = to_ruc(ibuc);
 230	int err;
 231
 232	err = rxe_add_to_pool(&rxe->uc_pool, uc);
 233	if (err)
 234		rxe_err_dev(rxe, "unable to create uc\n");
 235
 236	return err;
 237}
 238
 239static void rxe_dealloc_ucontext(struct ib_ucontext *ibuc)
 240{
 241	struct rxe_ucontext *uc = to_ruc(ibuc);
 242	int err;
 243
 244	err = rxe_cleanup(uc);
 245	if (err)
 246		rxe_err_uc(uc, "cleanup failed, err = %d\n", err);
 247}
 248
 249/* pd */
 250static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 251{
 252	struct rxe_dev *rxe = to_rdev(ibpd->device);
 253	struct rxe_pd *pd = to_rpd(ibpd);
 254	int err;
 255
 256	err = rxe_add_to_pool(&rxe->pd_pool, pd);
 257	if (err) {
 258		rxe_dbg_dev(rxe, "unable to alloc pd\n");
 259		goto err_out;
 260	}
 261
 262	return 0;
 263
 264err_out:
 265	rxe_err_dev(rxe, "returned err = %d\n", err);
 266	return err;
 267}
 268
 269static int rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 270{
 271	struct rxe_pd *pd = to_rpd(ibpd);
 272	int err;
 273
 274	err = rxe_cleanup(pd);
 275	if (err)
 276		rxe_err_pd(pd, "cleanup failed, err = %d\n", err);
 277
 278	return 0;
 279}
 280
 281/* ah */
 282static int rxe_create_ah(struct ib_ah *ibah,
 283			 struct rdma_ah_init_attr *init_attr,
 284			 struct ib_udata *udata)
 285{
 286	struct rxe_dev *rxe = to_rdev(ibah->device);
 287	struct rxe_ah *ah = to_rah(ibah);
 288	struct rxe_create_ah_resp __user *uresp = NULL;
 289	int err, cleanup_err;
 290
 291	if (udata) {
 292		/* test if new user provider */
 293		if (udata->outlen >= sizeof(*uresp))
 294			uresp = udata->outbuf;
 295		ah->is_user = true;
 296	} else {
 297		ah->is_user = false;
 298	}
 299
 300	err = rxe_add_to_pool_ah(&rxe->ah_pool, ah,
 301			init_attr->flags & RDMA_CREATE_AH_SLEEPABLE);
 302	if (err) {
 303		rxe_dbg_dev(rxe, "unable to create ah\n");
 304		goto err_out;
 305	}
 306
 307	/* create index > 0 */
 308	ah->ah_num = ah->elem.index;
 309
 310	err = rxe_ah_chk_attr(ah, init_attr->ah_attr);
 311	if (err) {
 312		rxe_dbg_ah(ah, "bad attr\n");
 313		goto err_cleanup;
 314	}
 315
 316	if (uresp) {
 317		/* only if new user provider */
 318		err = copy_to_user(&uresp->ah_num, &ah->ah_num,
 319					 sizeof(uresp->ah_num));
 320		if (err) {
 321			err = -EFAULT;
 322			rxe_dbg_ah(ah, "unable to copy to user\n");
 323			goto err_cleanup;
 324		}
 325	} else if (ah->is_user) {
 326		/* only if old user provider */
 327		ah->ah_num = 0;
 328	}
 329
 330	rxe_init_av(init_attr->ah_attr, &ah->av);
 331	rxe_finalize(ah);
 332
 333	return 0;
 334
 335err_cleanup:
 336	cleanup_err = rxe_cleanup(ah);
 337	if (cleanup_err)
 338		rxe_err_ah(ah, "cleanup failed, err = %d\n", cleanup_err);
 339err_out:
 340	rxe_err_ah(ah, "returned err = %d\n", err);
 341	return err;
 342}
 343
 344static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
 345{
 346	struct rxe_ah *ah = to_rah(ibah);
 347	int err;
 348
 349	err = rxe_ah_chk_attr(ah, attr);
 350	if (err) {
 351		rxe_dbg_ah(ah, "bad attr\n");
 352		goto err_out;
 353	}
 354
 355	rxe_init_av(attr, &ah->av);
 356
 357	return 0;
 358
 359err_out:
 360	rxe_err_ah(ah, "returned err = %d\n", err);
 361	return err;
 362}
 363
 364static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
 365{
 366	struct rxe_ah *ah = to_rah(ibah);
 367
 368	memset(attr, 0, sizeof(*attr));
 369	attr->type = ibah->type;
 370	rxe_av_to_attr(&ah->av, attr);
 371
 372	return 0;
 373}
 374
 375static int rxe_destroy_ah(struct ib_ah *ibah, u32 flags)
 376{
 377	struct rxe_ah *ah = to_rah(ibah);
 378	int err;
 379
 380	err = rxe_cleanup_ah(ah, flags & RDMA_DESTROY_AH_SLEEPABLE);
 381	if (err)
 382		rxe_err_ah(ah, "cleanup failed, err = %d\n", err);
 383
 384	return 0;
 385}
 386
 387/* srq */
 388static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
 389			  struct ib_udata *udata)
 390{
 391	struct rxe_dev *rxe = to_rdev(ibsrq->device);
 392	struct rxe_pd *pd = to_rpd(ibsrq->pd);
 393	struct rxe_srq *srq = to_rsrq(ibsrq);
 394	struct rxe_create_srq_resp __user *uresp = NULL;
 395	int err, cleanup_err;
 396
 397	if (udata) {
 398		if (udata->outlen < sizeof(*uresp)) {
 399			err = -EINVAL;
 400			rxe_err_dev(rxe, "malformed udata\n");
 401			goto err_out;
 402		}
 403		uresp = udata->outbuf;
 404	}
 405
 406	if (init->srq_type != IB_SRQT_BASIC) {
 407		err = -EOPNOTSUPP;
 408		rxe_dbg_dev(rxe, "srq type = %d, not supported\n",
 409				init->srq_type);
 410		goto err_out;
 411	}
 412
 413	err = rxe_srq_chk_init(rxe, init);
 414	if (err) {
 415		rxe_dbg_dev(rxe, "invalid init attributes\n");
 416		goto err_out;
 417	}
 418
 419	err = rxe_add_to_pool(&rxe->srq_pool, srq);
 420	if (err) {
 421		rxe_dbg_dev(rxe, "unable to create srq, err = %d\n", err);
 422		goto err_out;
 423	}
 424
 425	rxe_get(pd);
 426	srq->pd = pd;
 427
 428	err = rxe_srq_from_init(rxe, srq, init, udata, uresp);
 429	if (err) {
 430		rxe_dbg_srq(srq, "create srq failed, err = %d\n", err);
 431		goto err_cleanup;
 432	}
 433
 434	return 0;
 435
 436err_cleanup:
 437	cleanup_err = rxe_cleanup(srq);
 438	if (cleanup_err)
 439		rxe_err_srq(srq, "cleanup failed, err = %d\n", cleanup_err);
 440err_out:
 441	rxe_err_dev(rxe, "returned err = %d\n", err);
 442	return err;
 443}
 444
 445static int rxe_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 446			  enum ib_srq_attr_mask mask,
 447			  struct ib_udata *udata)
 448{
 449	struct rxe_srq *srq = to_rsrq(ibsrq);
 450	struct rxe_dev *rxe = to_rdev(ibsrq->device);
 451	struct rxe_modify_srq_cmd cmd = {};
 452	int err;
 453
 454	if (udata) {
 455		if (udata->inlen < sizeof(cmd)) {
 456			err = -EINVAL;
 457			rxe_dbg_srq(srq, "malformed udata\n");
 458			goto err_out;
 459		}
 460
 461		err = ib_copy_from_udata(&cmd, udata, sizeof(cmd));
 462		if (err) {
 463			err = -EFAULT;
 464			rxe_dbg_srq(srq, "unable to read udata\n");
 465			goto err_out;
 466		}
 467	}
 468
 469	err = rxe_srq_chk_attr(rxe, srq, attr, mask);
 470	if (err) {
 471		rxe_dbg_srq(srq, "bad init attributes\n");
 472		goto err_out;
 473	}
 474
 475	err = rxe_srq_from_attr(rxe, srq, attr, mask, &cmd, udata);
 476	if (err) {
 477		rxe_dbg_srq(srq, "bad attr\n");
 478		goto err_out;
 479	}
 480
 481	return 0;
 482
 483err_out:
 484	rxe_err_srq(srq, "returned err = %d\n", err);
 485	return err;
 486}
 487
 488static int rxe_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 489{
 490	struct rxe_srq *srq = to_rsrq(ibsrq);
 491	int err;
 492
 493	if (srq->error) {
 494		err = -EINVAL;
 495		rxe_dbg_srq(srq, "srq in error state\n");
 496		goto err_out;
 497	}
 498
 499	attr->max_wr = srq->rq.queue->buf->index_mask;
 500	attr->max_sge = srq->rq.max_sge;
 501	attr->srq_limit = srq->limit;
 502	return 0;
 503
 504err_out:
 505	rxe_err_srq(srq, "returned err = %d\n", err);
 506	return err;
 507}
 508
 509static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
 510			     const struct ib_recv_wr **bad_wr)
 511{
 512	int err = 0;
 513	struct rxe_srq *srq = to_rsrq(ibsrq);
 514	unsigned long flags;
 515
 516	spin_lock_irqsave(&srq->rq.producer_lock, flags);
 517
 518	while (wr) {
 519		err = post_one_recv(&srq->rq, wr);
 520		if (unlikely(err))
 521			break;
 522		wr = wr->next;
 523	}
 524
 525	spin_unlock_irqrestore(&srq->rq.producer_lock, flags);
 526
 527	if (err) {
 528		*bad_wr = wr;
 529		rxe_err_srq(srq, "returned err = %d\n", err);
 530	}
 531
 532	return err;
 533}
 534
 535static int rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 536{
 537	struct rxe_srq *srq = to_rsrq(ibsrq);
 538	int err;
 539
 540	err = rxe_cleanup(srq);
 541	if (err)
 542		rxe_err_srq(srq, "cleanup failed, err = %d\n", err);
 543
 544	return 0;
 545}
 546
 547/* qp */
 548static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init,
 549			 struct ib_udata *udata)
 550{
 551	struct rxe_dev *rxe = to_rdev(ibqp->device);
 552	struct rxe_pd *pd = to_rpd(ibqp->pd);
 553	struct rxe_qp *qp = to_rqp(ibqp);
 554	struct rxe_create_qp_resp __user *uresp = NULL;
 555	int err, cleanup_err;
 556
 557	if (udata) {
 558		if (udata->inlen) {
 559			err = -EINVAL;
 560			rxe_dbg_dev(rxe, "malformed udata, err = %d\n", err);
 561			goto err_out;
 562		}
 563
 564		if (udata->outlen < sizeof(*uresp)) {
 565			err = -EINVAL;
 566			rxe_dbg_dev(rxe, "malformed udata, err = %d\n", err);
 567			goto err_out;
 568		}
 569
 570		qp->is_user = true;
 571		uresp = udata->outbuf;
 572	} else {
 573		qp->is_user = false;
 574	}
 575
 576	if (init->create_flags) {
 577		err = -EOPNOTSUPP;
 578		rxe_dbg_dev(rxe, "unsupported create_flags, err = %d\n", err);
 579		goto err_out;
 580	}
 581
 582	err = rxe_qp_chk_init(rxe, init);
 583	if (err) {
 584		rxe_dbg_dev(rxe, "bad init attr, err = %d\n", err);
 585		goto err_out;
 586	}
 587
 588	err = rxe_add_to_pool(&rxe->qp_pool, qp);
 589	if (err) {
 590		rxe_dbg_dev(rxe, "unable to create qp, err = %d\n", err);
 591		goto err_out;
 592	}
 593
 594	err = rxe_qp_from_init(rxe, qp, pd, init, uresp, ibqp->pd, udata);
 595	if (err) {
 596		rxe_dbg_qp(qp, "create qp failed, err = %d\n", err);
 597		goto err_cleanup;
 598	}
 599
 600	rxe_finalize(qp);
 601	return 0;
 602
 603err_cleanup:
 604	cleanup_err = rxe_cleanup(qp);
 605	if (cleanup_err)
 606		rxe_err_qp(qp, "cleanup failed, err = %d\n", cleanup_err);
 607err_out:
 608	rxe_err_dev(rxe, "returned err = %d\n", err);
 609	return err;
 610}
 611
 612static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 613			 int mask, struct ib_udata *udata)
 614{
 615	struct rxe_dev *rxe = to_rdev(ibqp->device);
 616	struct rxe_qp *qp = to_rqp(ibqp);
 617	int err;
 618
 619	if (mask & ~IB_QP_ATTR_STANDARD_BITS) {
 620		err = -EOPNOTSUPP;
 621		rxe_dbg_qp(qp, "unsupported mask = 0x%x, err = %d\n",
 622			   mask, err);
 623		goto err_out;
 624	}
 625
 626	err = rxe_qp_chk_attr(rxe, qp, attr, mask);
 627	if (err) {
 628		rxe_dbg_qp(qp, "bad mask/attr, err = %d\n", err);
 629		goto err_out;
 630	}
 631
 632	err = rxe_qp_from_attr(qp, attr, mask, udata);
 633	if (err) {
 634		rxe_dbg_qp(qp, "modify qp failed, err = %d\n", err);
 635		goto err_out;
 636	}
 637
 638	if ((mask & IB_QP_AV) && (attr->ah_attr.ah_flags & IB_AH_GRH))
 639		qp->src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
 640						  qp->ibqp.qp_num,
 641						  qp->attr.dest_qp_num);
 642
 643	return 0;
 644
 645err_out:
 646	rxe_err_qp(qp, "returned err = %d\n", err);
 647	return err;
 648}
 649
 650static int rxe_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 651			int mask, struct ib_qp_init_attr *init)
 652{
 653	struct rxe_qp *qp = to_rqp(ibqp);
 654
 655	rxe_qp_to_init(qp, init);
 656	rxe_qp_to_attr(qp, attr, mask);
 657
 658	return 0;
 659}
 660
 661static int rxe_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 662{
 663	struct rxe_qp *qp = to_rqp(ibqp);
 664	int err;
 665
 666	err = rxe_qp_chk_destroy(qp);
 667	if (err) {
 668		rxe_dbg_qp(qp, "unable to destroy qp, err = %d\n", err);
 669		goto err_out;
 670	}
 671
 672	err = rxe_cleanup(qp);
 673	if (err)
 674		rxe_err_qp(qp, "cleanup failed, err = %d\n", err);
 675
 676	return 0;
 677
 678err_out:
 679	rxe_err_qp(qp, "returned err = %d\n", err);
 680	return err;
 681}
 682
 683/* send wr */
 684
 685/* sanity check incoming send work request */
 686static int validate_send_wr(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
 687			    unsigned int *maskp, unsigned int *lengthp)
 688{
 689	int num_sge = ibwr->num_sge;
 690	struct rxe_sq *sq = &qp->sq;
 691	unsigned int mask = 0;
 692	unsigned long length = 0;
 693	int err = -EINVAL;
 694	int i;
 695
 696	do {
 697		mask = wr_opcode_mask(ibwr->opcode, qp);
 698		if (!mask) {
 699			rxe_err_qp(qp, "bad wr opcode for qp type\n");
 700			break;
 701		}
 702
 703		if (num_sge > sq->max_sge) {
 704			rxe_err_qp(qp, "num_sge > max_sge\n");
 705			break;
 706		}
 707
 708		length = 0;
 709		for (i = 0; i < ibwr->num_sge; i++)
 710			length += ibwr->sg_list[i].length;
 711
 712		if (length > RXE_PORT_MAX_MSG_SZ) {
 713			rxe_err_qp(qp, "message length too long\n");
 714			break;
 715		}
 716
 717		if (mask & WR_ATOMIC_MASK) {
 718			if (length != 8) {
 719				rxe_err_qp(qp, "atomic length != 8\n");
 720				break;
 721			}
 722			if (atomic_wr(ibwr)->remote_addr & 0x7) {
 723				rxe_err_qp(qp, "misaligned atomic address\n");
 724				break;
 725			}
 726		}
 727		if (ibwr->send_flags & IB_SEND_INLINE) {
 728			if (!(mask & WR_INLINE_MASK)) {
 729				rxe_err_qp(qp, "opcode doesn't support inline data\n");
 730				break;
 731			}
 732			if (length > sq->max_inline) {
 733				rxe_err_qp(qp, "inline length too big\n");
 734				break;
 735			}
 736		}
 737
 738		err = 0;
 739	} while (0);
 740
 741	*maskp = mask;
 742	*lengthp = (int)length;
 743
 744	return err;
 745}
 746
 747static int init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr,
 748			 const struct ib_send_wr *ibwr)
 749{
 750	wr->wr_id = ibwr->wr_id;
 751	wr->opcode = ibwr->opcode;
 752	wr->send_flags = ibwr->send_flags;
 753
 754	if (qp_type(qp) == IB_QPT_UD ||
 755	    qp_type(qp) == IB_QPT_GSI) {
 756		struct ib_ah *ibah = ud_wr(ibwr)->ah;
 757
 758		wr->wr.ud.remote_qpn = ud_wr(ibwr)->remote_qpn;
 759		wr->wr.ud.remote_qkey = ud_wr(ibwr)->remote_qkey;
 760		wr->wr.ud.ah_num = to_rah(ibah)->ah_num;
 761		if (qp_type(qp) == IB_QPT_GSI)
 762			wr->wr.ud.pkey_index = ud_wr(ibwr)->pkey_index;
 763
 764		switch (wr->opcode) {
 765		case IB_WR_SEND_WITH_IMM:
 766			wr->ex.imm_data = ibwr->ex.imm_data;
 767			break;
 768		case IB_WR_SEND:
 769			break;
 770		default:
 771			rxe_err_qp(qp, "bad wr opcode %d for UD/GSI QP\n",
 772					wr->opcode);
 773			return -EINVAL;
 774		}
 775	} else {
 776		switch (wr->opcode) {
 777		case IB_WR_RDMA_WRITE_WITH_IMM:
 778			wr->ex.imm_data = ibwr->ex.imm_data;
 779			fallthrough;
 780		case IB_WR_RDMA_READ:
 781		case IB_WR_RDMA_WRITE:
 782			wr->wr.rdma.remote_addr = rdma_wr(ibwr)->remote_addr;
 783			wr->wr.rdma.rkey	= rdma_wr(ibwr)->rkey;
 784			break;
 785		case IB_WR_SEND_WITH_IMM:
 786			wr->ex.imm_data = ibwr->ex.imm_data;
 787			break;
 788		case IB_WR_SEND_WITH_INV:
 789			wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey;
 790			break;
 791		case IB_WR_RDMA_READ_WITH_INV:
 792			wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey;
 793			wr->wr.rdma.remote_addr = rdma_wr(ibwr)->remote_addr;
 794			wr->wr.rdma.rkey	= rdma_wr(ibwr)->rkey;
 795			break;
 796		case IB_WR_ATOMIC_CMP_AND_SWP:
 797		case IB_WR_ATOMIC_FETCH_AND_ADD:
 798			wr->wr.atomic.remote_addr =
 799				atomic_wr(ibwr)->remote_addr;
 800			wr->wr.atomic.compare_add =
 801				atomic_wr(ibwr)->compare_add;
 802			wr->wr.atomic.swap = atomic_wr(ibwr)->swap;
 803			wr->wr.atomic.rkey = atomic_wr(ibwr)->rkey;
 804			break;
 805		case IB_WR_LOCAL_INV:
 806			wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey;
 807			break;
 808		case IB_WR_REG_MR:
 809			wr->wr.reg.mr = reg_wr(ibwr)->mr;
 810			wr->wr.reg.key = reg_wr(ibwr)->key;
 811			wr->wr.reg.access = reg_wr(ibwr)->access;
 812			break;
 813		case IB_WR_SEND:
 814		case IB_WR_BIND_MW:
 815		case IB_WR_FLUSH:
 816		case IB_WR_ATOMIC_WRITE:
 817			break;
 818		default:
 819			rxe_err_qp(qp, "unsupported wr opcode %d\n",
 820					wr->opcode);
 821			return -EINVAL;
 822		}
 823	}
 824
 825	return 0;
 826}
 827
 828static void copy_inline_data_to_wqe(struct rxe_send_wqe *wqe,
 829				    const struct ib_send_wr *ibwr)
 830{
 831	struct ib_sge *sge = ibwr->sg_list;
 832	u8 *p = wqe->dma.inline_data;
 833	int i;
 834
 835	for (i = 0; i < ibwr->num_sge; i++, sge++) {
 836		memcpy(p, ib_virt_dma_to_ptr(sge->addr), sge->length);
 837		p += sge->length;
 838	}
 839}
 840
 841static int init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr,
 842			 unsigned int mask, unsigned int length,
 843			 struct rxe_send_wqe *wqe)
 844{
 845	int num_sge = ibwr->num_sge;
 846	int err;
 847
 848	err = init_send_wr(qp, &wqe->wr, ibwr);
 849	if (err)
 850		return err;
 851
 852	/* local operation */
 853	if (unlikely(mask & WR_LOCAL_OP_MASK)) {
 854		wqe->mask = mask;
 855		wqe->state = wqe_state_posted;
 856		return 0;
 857	}
 858
 859	if (unlikely(ibwr->send_flags & IB_SEND_INLINE))
 860		copy_inline_data_to_wqe(wqe, ibwr);
 861	else
 862		memcpy(wqe->dma.sge, ibwr->sg_list,
 863		       num_sge * sizeof(struct ib_sge));
 864
 865	wqe->iova = mask & WR_ATOMIC_MASK ? atomic_wr(ibwr)->remote_addr :
 866		mask & WR_READ_OR_WRITE_MASK ? rdma_wr(ibwr)->remote_addr : 0;
 867	wqe->mask		= mask;
 868	wqe->dma.length		= length;
 869	wqe->dma.resid		= length;
 870	wqe->dma.num_sge	= num_sge;
 871	wqe->dma.cur_sge	= 0;
 872	wqe->dma.sge_offset	= 0;
 873	wqe->state		= wqe_state_posted;
 874	wqe->ssn		= atomic_add_return(1, &qp->ssn);
 875
 876	return 0;
 877}
 878
 879static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr)
 880{
 881	int err;
 882	struct rxe_sq *sq = &qp->sq;
 883	struct rxe_send_wqe *send_wqe;
 884	unsigned int mask;
 885	unsigned int length;
 886	int full;
 887
 888	err = validate_send_wr(qp, ibwr, &mask, &length);
 889	if (err)
 890		return err;
 891
 892	full = queue_full(sq->queue, QUEUE_TYPE_FROM_ULP);
 893	if (unlikely(full)) {
 894		rxe_err_qp(qp, "send queue full\n");
 895		return -ENOMEM;
 896	}
 897
 898	send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_FROM_ULP);
 899	err = init_send_wqe(qp, ibwr, mask, length, send_wqe);
 900	if (!err)
 901		queue_advance_producer(sq->queue, QUEUE_TYPE_FROM_ULP);
 902
 903	return err;
 904}
 905
 906static int rxe_post_send_kernel(struct rxe_qp *qp,
 907				const struct ib_send_wr *ibwr,
 908				const struct ib_send_wr **bad_wr)
 909{
 910	int err = 0;
 911	unsigned long flags;
 912	int good = 0;
 913
 914	spin_lock_irqsave(&qp->sq.sq_lock, flags);
 915	while (ibwr) {
 916		err = post_one_send(qp, ibwr);
 917		if (err) {
 918			*bad_wr = ibwr;
 919			break;
 920		} else {
 921			good++;
 922		}
 923		ibwr = ibwr->next;
 924	}
 925	spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
 926
 927	/* kickoff processing of any posted wqes */
 928	if (good)
 929		rxe_sched_task(&qp->send_task);
 930
 931	return err;
 932}
 933
 934static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 935			 const struct ib_send_wr **bad_wr)
 936{
 937	struct rxe_qp *qp = to_rqp(ibqp);
 938	int err;
 939	unsigned long flags;
 940
 941	spin_lock_irqsave(&qp->state_lock, flags);
 942	/* caller has already called destroy_qp */
 943	if (WARN_ON_ONCE(!qp->valid)) {
 944		spin_unlock_irqrestore(&qp->state_lock, flags);
 945		rxe_err_qp(qp, "qp has been destroyed\n");
 946		return -EINVAL;
 947	}
 948
 949	if (unlikely(qp_state(qp) < IB_QPS_RTS)) {
 950		spin_unlock_irqrestore(&qp->state_lock, flags);
 951		*bad_wr = wr;
 952		rxe_err_qp(qp, "qp not ready to send\n");
 953		return -EINVAL;
 954	}
 955	spin_unlock_irqrestore(&qp->state_lock, flags);
 956
 957	if (qp->is_user) {
 958		/* Utilize process context to do protocol processing */
 959		rxe_sched_task(&qp->send_task);
 960	} else {
 961		err = rxe_post_send_kernel(qp, wr, bad_wr);
 962		if (err)
 963			return err;
 964	}
 965
 966	return 0;
 967}
 968
 969/* recv wr */
 970static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr)
 971{
 972	int i;
 973	unsigned long length;
 974	struct rxe_recv_wqe *recv_wqe;
 975	int num_sge = ibwr->num_sge;
 976	int full;
 977	int err;
 978
 979	full = queue_full(rq->queue, QUEUE_TYPE_FROM_ULP);
 980	if (unlikely(full)) {
 981		err = -ENOMEM;
 982		rxe_dbg("queue full\n");
 983		goto err_out;
 984	}
 985
 986	if (unlikely(num_sge > rq->max_sge)) {
 987		err = -EINVAL;
 988		rxe_dbg("bad num_sge > max_sge\n");
 989		goto err_out;
 990	}
 991
 992	length = 0;
 993	for (i = 0; i < num_sge; i++)
 994		length += ibwr->sg_list[i].length;
 995
 996	if (length > RXE_PORT_MAX_MSG_SZ) {
 997		err = -EINVAL;
 998		rxe_dbg("message length too long\n");
 999		goto err_out;
1000	}
1001
1002	recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_FROM_ULP);
1003
1004	recv_wqe->wr_id = ibwr->wr_id;
1005	recv_wqe->dma.length = length;
1006	recv_wqe->dma.resid = length;
1007	recv_wqe->dma.num_sge = num_sge;
1008	recv_wqe->dma.cur_sge = 0;
1009	recv_wqe->dma.sge_offset = 0;
1010	memcpy(recv_wqe->dma.sge, ibwr->sg_list,
1011	       num_sge * sizeof(struct ib_sge));
1012
1013	queue_advance_producer(rq->queue, QUEUE_TYPE_FROM_ULP);
1014
1015	return 0;
1016
1017err_out:
1018	rxe_dbg("returned err = %d\n", err);
1019	return err;
1020}
1021
1022static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
1023			 const struct ib_recv_wr **bad_wr)
1024{
1025	int err = 0;
1026	struct rxe_qp *qp = to_rqp(ibqp);
1027	struct rxe_rq *rq = &qp->rq;
1028	unsigned long flags;
1029
1030	spin_lock_irqsave(&qp->state_lock, flags);
1031	/* caller has already called destroy_qp */
1032	if (WARN_ON_ONCE(!qp->valid)) {
1033		spin_unlock_irqrestore(&qp->state_lock, flags);
1034		rxe_err_qp(qp, "qp has been destroyed\n");
1035		return -EINVAL;
1036	}
1037
1038	/* see C10-97.2.1 */
1039	if (unlikely((qp_state(qp) < IB_QPS_INIT))) {
1040		spin_unlock_irqrestore(&qp->state_lock, flags);
1041		*bad_wr = wr;
1042		rxe_dbg_qp(qp, "qp not ready to post recv\n");
1043		return -EINVAL;
1044	}
1045	spin_unlock_irqrestore(&qp->state_lock, flags);
1046
1047	if (unlikely(qp->srq)) {
1048		*bad_wr = wr;
1049		rxe_dbg_qp(qp, "qp has srq, use post_srq_recv instead\n");
1050		return -EINVAL;
1051	}
1052
1053	spin_lock_irqsave(&rq->producer_lock, flags);
1054
1055	while (wr) {
1056		err = post_one_recv(rq, wr);
1057		if (unlikely(err)) {
1058			*bad_wr = wr;
1059			break;
1060		}
1061		wr = wr->next;
1062	}
1063
1064	spin_unlock_irqrestore(&rq->producer_lock, flags);
1065
1066	spin_lock_irqsave(&qp->state_lock, flags);
1067	if (qp_state(qp) == IB_QPS_ERR)
1068		rxe_sched_task(&qp->recv_task);
1069	spin_unlock_irqrestore(&qp->state_lock, flags);
1070
1071	return err;
1072}
1073
1074/* cq */
1075static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1076			 struct uverbs_attr_bundle *attrs)
1077{
1078	struct ib_udata *udata = &attrs->driver_udata;
1079	struct ib_device *dev = ibcq->device;
1080	struct rxe_dev *rxe = to_rdev(dev);
1081	struct rxe_cq *cq = to_rcq(ibcq);
1082	struct rxe_create_cq_resp __user *uresp = NULL;
1083	int err, cleanup_err;
1084
1085	if (udata) {
1086		if (udata->outlen < sizeof(*uresp)) {
1087			err = -EINVAL;
1088			rxe_dbg_dev(rxe, "malformed udata, err = %d\n", err);
1089			goto err_out;
1090		}
1091		uresp = udata->outbuf;
1092	}
1093
1094	if (attr->flags) {
1095		err = -EOPNOTSUPP;
1096		rxe_dbg_dev(rxe, "bad attr->flags, err = %d\n", err);
1097		goto err_out;
1098	}
1099
1100	err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector);
1101	if (err) {
1102		rxe_dbg_dev(rxe, "bad init attributes, err = %d\n", err);
1103		goto err_out;
1104	}
1105
1106	err = rxe_add_to_pool(&rxe->cq_pool, cq);
1107	if (err) {
1108		rxe_dbg_dev(rxe, "unable to create cq, err = %d\n", err);
1109		goto err_out;
1110	}
1111
1112	err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, udata,
1113			       uresp);
1114	if (err) {
1115		rxe_dbg_cq(cq, "create cq failed, err = %d\n", err);
1116		goto err_cleanup;
1117	}
1118
1119	return 0;
1120
1121err_cleanup:
1122	cleanup_err = rxe_cleanup(cq);
1123	if (cleanup_err)
1124		rxe_err_cq(cq, "cleanup failed, err = %d\n", cleanup_err);
1125err_out:
1126	rxe_err_dev(rxe, "returned err = %d\n", err);
1127	return err;
1128}
1129
1130static int rxe_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
1131{
1132	struct rxe_cq *cq = to_rcq(ibcq);
1133	struct rxe_dev *rxe = to_rdev(ibcq->device);
1134	struct rxe_resize_cq_resp __user *uresp = NULL;
1135	int err;
1136
1137	if (udata) {
1138		if (udata->outlen < sizeof(*uresp)) {
1139			err = -EINVAL;
1140			rxe_dbg_cq(cq, "malformed udata\n");
1141			goto err_out;
1142		}
1143		uresp = udata->outbuf;
1144	}
1145
1146	err = rxe_cq_chk_attr(rxe, cq, cqe, 0);
1147	if (err) {
1148		rxe_dbg_cq(cq, "bad attr, err = %d\n", err);
1149		goto err_out;
1150	}
1151
1152	err = rxe_cq_resize_queue(cq, cqe, uresp, udata);
1153	if (err) {
1154		rxe_dbg_cq(cq, "resize cq failed, err = %d\n", err);
1155		goto err_out;
1156	}
1157
1158	return 0;
1159
1160err_out:
1161	rxe_err_cq(cq, "returned err = %d\n", err);
1162	return err;
1163}
1164
1165static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
1166{
1167	int i;
1168	struct rxe_cq *cq = to_rcq(ibcq);
1169	struct rxe_cqe *cqe;
1170	unsigned long flags;
1171
1172	spin_lock_irqsave(&cq->cq_lock, flags);
1173	for (i = 0; i < num_entries; i++) {
1174		cqe = queue_head(cq->queue, QUEUE_TYPE_TO_ULP);
1175		if (!cqe)
1176			break;	/* queue empty */
1177
1178		memcpy(wc++, &cqe->ibwc, sizeof(*wc));
1179		queue_advance_consumer(cq->queue, QUEUE_TYPE_TO_ULP);
1180	}
1181	spin_unlock_irqrestore(&cq->cq_lock, flags);
1182
1183	return i;
1184}
1185
1186static int rxe_peek_cq(struct ib_cq *ibcq, int wc_cnt)
1187{
1188	struct rxe_cq *cq = to_rcq(ibcq);
1189	int count;
1190
1191	count = queue_count(cq->queue, QUEUE_TYPE_TO_ULP);
1192
1193	return (count > wc_cnt) ? wc_cnt : count;
1194}
1195
1196static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
1197{
1198	struct rxe_cq *cq = to_rcq(ibcq);
1199	int ret = 0;
1200	int empty;
1201	unsigned long irq_flags;
1202
1203	spin_lock_irqsave(&cq->cq_lock, irq_flags);
1204	cq->notify |= flags & IB_CQ_SOLICITED_MASK;
1205	empty = queue_empty(cq->queue, QUEUE_TYPE_TO_ULP);
1206
1207	if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty)
1208		ret = 1;
1209
1210	spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
1211
1212	return ret;
1213}
1214
1215static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1216{
1217	struct rxe_cq *cq = to_rcq(ibcq);
1218	int err;
1219
1220	/* See IBA C11-17: The CI shall return an error if this Verb is
1221	 * invoked while a Work Queue is still associated with the CQ.
1222	 */
1223	if (atomic_read(&cq->num_wq)) {
1224		err = -EINVAL;
1225		rxe_dbg_cq(cq, "still in use\n");
1226		goto err_out;
1227	}
1228
1229	err = rxe_cleanup(cq);
1230	if (err)
1231		rxe_err_cq(cq, "cleanup failed, err = %d\n", err);
1232
1233	return 0;
1234
1235err_out:
1236	rxe_err_cq(cq, "returned err = %d\n", err);
1237	return err;
1238}
1239
1240/* mr */
1241static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access)
1242{
1243	struct rxe_dev *rxe = to_rdev(ibpd->device);
1244	struct rxe_pd *pd = to_rpd(ibpd);
1245	struct rxe_mr *mr;
1246	int err;
1247
1248	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1249	if (!mr)
1250		return ERR_PTR(-ENOMEM);
1251
1252	err = rxe_add_to_pool(&rxe->mr_pool, mr);
1253	if (err) {
1254		rxe_dbg_dev(rxe, "unable to create mr\n");
1255		goto err_free;
1256	}
1257
1258	rxe_get(pd);
1259	mr->ibmr.pd = ibpd;
1260	mr->ibmr.device = ibpd->device;
1261
1262	rxe_mr_init_dma(access, mr);
1263	rxe_finalize(mr);
1264	return &mr->ibmr;
1265
1266err_free:
1267	kfree(mr);
1268	rxe_err_pd(pd, "returned err = %d\n", err);
1269	return ERR_PTR(err);
1270}
1271
1272static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, u64 start,
1273				     u64 length, u64 iova, int access,
1274				     struct ib_dmah *dmah,
1275				     struct ib_udata *udata)
1276{
1277	struct rxe_dev *rxe = to_rdev(ibpd->device);
1278	struct rxe_pd *pd = to_rpd(ibpd);
1279	struct rxe_mr *mr;
1280	int err, cleanup_err;
1281
1282	if (dmah)
1283		return ERR_PTR(-EOPNOTSUPP);
1284
1285	if (access & ~RXE_ACCESS_SUPPORTED_MR) {
1286		rxe_err_pd(pd, "access = %#x not supported (%#x)\n", access,
1287				RXE_ACCESS_SUPPORTED_MR);
1288		return ERR_PTR(-EOPNOTSUPP);
1289	}
1290
1291	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1292	if (!mr)
1293		return ERR_PTR(-ENOMEM);
1294
1295	err = rxe_add_to_pool(&rxe->mr_pool, mr);
1296	if (err) {
1297		rxe_dbg_pd(pd, "unable to create mr\n");
1298		goto err_free;
1299	}
1300
1301	rxe_get(pd);
1302	mr->ibmr.pd = ibpd;
1303	mr->ibmr.device = ibpd->device;
1304
1305	if (access & IB_ACCESS_ON_DEMAND)
1306		err = rxe_odp_mr_init_user(rxe, start, length, iova, access, mr);
1307	else
1308		err = rxe_mr_init_user(rxe, start, length, access, mr);
1309	if (err) {
1310		rxe_dbg_mr(mr, "reg_user_mr failed, err = %d\n", err);
1311		goto err_cleanup;
1312	}
1313
1314	rxe_finalize(mr);
1315	return &mr->ibmr;
1316
1317err_cleanup:
1318	cleanup_err = rxe_cleanup(mr);
1319	if (cleanup_err)
1320		rxe_err_mr(mr, "cleanup failed, err = %d\n", cleanup_err);
1321err_free:
1322	kfree(mr);
1323	rxe_err_pd(pd, "returned err = %d\n", err);
1324	return ERR_PTR(err);
1325}
1326
1327static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags,
1328				       u64 start, u64 length, u64 iova,
1329				       int access, struct ib_pd *ibpd,
1330				       struct ib_udata *udata)
1331{
1332	struct rxe_mr *mr = to_rmr(ibmr);
1333	struct rxe_pd *old_pd = to_rpd(ibmr->pd);
1334	struct rxe_pd *pd = to_rpd(ibpd);
1335
1336	/* for now only support the two easy cases:
1337	 * rereg_pd and rereg_access
1338	 */
1339	if (flags & ~RXE_MR_REREG_SUPPORTED) {
1340		rxe_err_mr(mr, "flags = %#x not supported\n", flags);
1341		return ERR_PTR(-EOPNOTSUPP);
1342	}
1343
1344	if (flags & IB_MR_REREG_PD) {
1345		rxe_put(old_pd);
1346		rxe_get(pd);
1347		mr->ibmr.pd = ibpd;
1348	}
1349
1350	if (flags & IB_MR_REREG_ACCESS) {
1351		if (access & ~RXE_ACCESS_SUPPORTED_MR) {
1352			rxe_err_mr(mr, "access = %#x not supported\n", access);
1353			return ERR_PTR(-EOPNOTSUPP);
1354		}
1355		mr->access = access;
1356	}
1357
1358	return NULL;
1359}
1360
1361static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
1362				  u32 max_num_sg)
1363{
1364	struct rxe_dev *rxe = to_rdev(ibpd->device);
1365	struct rxe_pd *pd = to_rpd(ibpd);
1366	struct rxe_mr *mr;
1367	int err, cleanup_err;
1368
1369	if (mr_type != IB_MR_TYPE_MEM_REG) {
1370		err = -EINVAL;
1371		rxe_dbg_pd(pd, "mr type %d not supported, err = %d\n",
1372			   mr_type, err);
1373		goto err_out;
1374	}
1375
1376	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1377	if (!mr)
1378		return ERR_PTR(-ENOMEM);
1379
1380	err = rxe_add_to_pool(&rxe->mr_pool, mr);
1381	if (err)
1382		goto err_free;
1383
1384	rxe_get(pd);
1385	mr->ibmr.pd = ibpd;
1386	mr->ibmr.device = ibpd->device;
1387
1388	err = rxe_mr_init_fast(max_num_sg, mr);
1389	if (err) {
1390		rxe_dbg_mr(mr, "alloc_mr failed, err = %d\n", err);
1391		goto err_cleanup;
1392	}
1393
1394	rxe_finalize(mr);
1395	return &mr->ibmr;
1396
1397err_cleanup:
1398	cleanup_err = rxe_cleanup(mr);
1399	if (cleanup_err)
1400		rxe_err_mr(mr, "cleanup failed, err = %d\n", err);
1401err_free:
1402	kfree(mr);
1403err_out:
1404	rxe_err_pd(pd, "returned err = %d\n", err);
1405	return ERR_PTR(err);
1406}
1407
1408static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1409{
1410	struct rxe_mr *mr = to_rmr(ibmr);
1411	int err, cleanup_err;
1412
1413	/* See IBA 10.6.7.2.6 */
1414	if (atomic_read(&mr->num_mw) > 0) {
1415		err = -EINVAL;
1416		rxe_dbg_mr(mr, "mr has mw's bound\n");
1417		goto err_out;
1418	}
1419
1420	cleanup_err = rxe_cleanup(mr);
1421	if (cleanup_err)
1422		rxe_err_mr(mr, "cleanup failed, err = %d\n", cleanup_err);
1423
1424	kfree_rcu_mightsleep(mr);
1425	return 0;
1426
1427err_out:
1428	rxe_err_mr(mr, "returned err = %d\n", err);
1429	return err;
1430}
1431
1432static ssize_t parent_show(struct device *device,
1433			   struct device_attribute *attr, char *buf)
1434{
1435	struct rxe_dev *rxe =
1436		rdma_device_to_drv_device(device, struct rxe_dev, ib_dev);
1437
1438	return sysfs_emit(buf, "%s\n", rxe_parent_name(rxe, 1));
1439}
1440
1441static DEVICE_ATTR_RO(parent);
1442
1443static struct attribute *rxe_dev_attributes[] = {
1444	&dev_attr_parent.attr,
1445	NULL
1446};
1447
1448static const struct attribute_group rxe_attr_group = {
1449	.attrs = rxe_dev_attributes,
1450};
1451
1452static int rxe_enable_driver(struct ib_device *ib_dev)
1453{
1454	struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev);
1455	struct net_device *ndev;
1456
1457	ndev = rxe_ib_device_get_netdev(ib_dev);
1458	if (!ndev)
1459		return -ENODEV;
1460
1461	rxe_set_port_state(rxe);
1462	dev_info(&rxe->ib_dev.dev, "added %s\n", netdev_name(ndev));
1463
1464	dev_put(ndev);
1465	return 0;
1466}
1467
1468static const struct ib_device_ops rxe_dev_ops = {
1469	.owner = THIS_MODULE,
1470	.driver_id = RDMA_DRIVER_RXE,
1471	.uverbs_abi_ver = RXE_UVERBS_ABI_VERSION,
1472
1473	.alloc_hw_port_stats = rxe_ib_alloc_hw_port_stats,
1474	.alloc_mr = rxe_alloc_mr,
1475	.alloc_mw = rxe_alloc_mw,
1476	.alloc_pd = rxe_alloc_pd,
1477	.alloc_ucontext = rxe_alloc_ucontext,
1478	.attach_mcast = rxe_attach_mcast,
1479	.create_ah = rxe_create_ah,
1480	.create_cq = rxe_create_cq,
1481	.create_qp = rxe_create_qp,
1482	.create_srq = rxe_create_srq,
1483	.create_user_ah = rxe_create_ah,
1484	.dealloc_driver = rxe_dealloc,
1485	.dealloc_mw = rxe_dealloc_mw,
1486	.dealloc_pd = rxe_dealloc_pd,
1487	.dealloc_ucontext = rxe_dealloc_ucontext,
1488	.dereg_mr = rxe_dereg_mr,
1489	.destroy_ah = rxe_destroy_ah,
1490	.destroy_cq = rxe_destroy_cq,
1491	.destroy_qp = rxe_destroy_qp,
1492	.destroy_srq = rxe_destroy_srq,
1493	.detach_mcast = rxe_detach_mcast,
1494	.device_group = &rxe_attr_group,
1495	.enable_driver = rxe_enable_driver,
1496	.get_dma_mr = rxe_get_dma_mr,
1497	.get_hw_stats = rxe_ib_get_hw_stats,
1498	.get_link_layer = rxe_get_link_layer,
1499	.get_port_immutable = rxe_port_immutable,
1500	.map_mr_sg = rxe_map_mr_sg,
1501	.mmap = rxe_mmap,
1502	.modify_ah = rxe_modify_ah,
1503	.modify_device = rxe_modify_device,
1504	.modify_port = rxe_modify_port,
1505	.modify_qp = rxe_modify_qp,
1506	.modify_srq = rxe_modify_srq,
1507	.peek_cq = rxe_peek_cq,
1508	.poll_cq = rxe_poll_cq,
1509	.post_recv = rxe_post_recv,
1510	.post_send = rxe_post_send,
1511	.post_srq_recv = rxe_post_srq_recv,
1512	.query_ah = rxe_query_ah,
1513	.query_device = rxe_query_device,
1514	.query_pkey = rxe_query_pkey,
1515	.query_gid = rxe_query_gid,
1516	.query_port = rxe_query_port,
1517	.query_qp = rxe_query_qp,
1518	.query_srq = rxe_query_srq,
1519	.reg_user_mr = rxe_reg_user_mr,
1520	.req_notify_cq = rxe_req_notify_cq,
1521	.rereg_user_mr = rxe_rereg_user_mr,
1522	.resize_cq = rxe_resize_cq,
1523
1524	INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
1525	INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
1526	INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
1527	INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp),
1528	INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
1529	INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
1530	INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
1531};
1532
1533int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name,
1534						struct net_device *ndev)
1535{
1536	int err;
1537	struct ib_device *dev = &rxe->ib_dev;
1538
1539	strscpy(dev->node_desc, "rxe", sizeof(dev->node_desc));
1540
1541	dev->node_type = RDMA_NODE_IB_CA;
1542	dev->phys_port_cnt = 1;
1543	dev->num_comp_vectors = num_possible_cpus();
1544	dev->local_dma_lkey = 0;
1545	addrconf_addr_eui48((unsigned char *)&dev->node_guid,
1546			    rxe->raw_gid);
1547
1548	dev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POST_SEND) |
1549				BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ);
1550
1551	ib_set_device_ops(dev, &rxe_dev_ops);
1552	err = ib_device_set_netdev(&rxe->ib_dev, ndev, 1);
1553	if (err)
1554		return err;
1555
1556	err = ib_register_device(dev, ibdev_name, NULL);
1557	if (err)
1558		rxe_dbg_dev(rxe, "failed with error %d\n", err);
1559
1560	/*
1561	 * Note that rxe may be invalid at this point if another thread
1562	 * unregistered it.
1563	 */
1564	return err;
1565}