mthca_provider.c

   1/*
   2 * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
   3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   4 * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   5 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   6 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   7 *
   8 * This software is available to you under a choice of one of two
   9 * licenses.  You may choose to be licensed under the terms of the GNU
  10 * General Public License (GPL) Version 2, available from the file
  11 * COPYING in the main directory of this source tree, or the
  12 * OpenIB.org BSD license below:
  13 *
  14 *     Redistribution and use in source and binary forms, with or
  15 *     without modification, are permitted provided that the following
  16 *     conditions are met:
  17 *
  18 *      - Redistributions of source code must retain the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer.
  21 *
  22 *      - Redistributions in binary form must reproduce the above
  23 *        copyright notice, this list of conditions and the following
  24 *        disclaimer in the documentation and/or other materials
  25 *        provided with the distribution.
  26 *
  27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34 * SOFTWARE.
  35 */
  36
  37#include <rdma/ib_smi.h>
  38#include <rdma/ib_umem.h>
  39#include <rdma/ib_user_verbs.h>
  40#include <rdma/uverbs_ioctl.h>
  41
  42#include <linux/sched.h>
  43#include <linux/slab.h>
  44#include <linux/stat.h>
  45#include <linux/mm.h>
  46#include <linux/export.h>
  47
  48#include "mthca_dev.h"
  49#include "mthca_cmd.h"
  50#include <rdma/mthca-abi.h>
  51#include "mthca_memfree.h"
  52
  53static int mthca_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
  54			      struct ib_udata *uhw)
  55{
  56	struct ib_smp *in_mad;
  57	struct ib_smp *out_mad;
  58	int err = -ENOMEM;
  59	struct mthca_dev *mdev = to_mdev(ibdev);
  60
  61	if (uhw->inlen || uhw->outlen)
  62		return -EINVAL;
  63
  64	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
  65	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
  66	if (!in_mad || !out_mad)
  67		goto out;
  68
  69	memset(props, 0, sizeof *props);
  70
  71	props->fw_ver              = mdev->fw_ver;
  72
  73	ib_init_query_mad(in_mad);
  74	in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
  75
  76	err = mthca_MAD_IFC(mdev, 1, 1,
  77			    1, NULL, NULL, in_mad, out_mad);
  78	if (err)
  79		goto out;
  80
  81	props->device_cap_flags    = mdev->device_cap_flags;
  82	props->vendor_id           = be32_to_cpup((__be32 *) (out_mad->data + 36)) &
  83		0xffffff;
  84	props->vendor_part_id      = be16_to_cpup((__be16 *) (out_mad->data + 30));
  85	props->hw_ver              = be32_to_cpup((__be32 *) (out_mad->data + 32));
  86	memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
  87
  88	props->max_mr_size         = ~0ull;
  89	props->page_size_cap       = mdev->limits.page_size_cap;
  90	props->max_qp              = mdev->limits.num_qps - mdev->limits.reserved_qps;
  91	props->max_qp_wr           = mdev->limits.max_wqes;
  92	props->max_send_sge        = mdev->limits.max_sg;
  93	props->max_recv_sge        = mdev->limits.max_sg;
  94	props->max_sge_rd          = mdev->limits.max_sg;
  95	props->max_cq              = mdev->limits.num_cqs - mdev->limits.reserved_cqs;
  96	props->max_cqe             = mdev->limits.max_cqes;
  97	props->max_mr              = mdev->limits.num_mpts - mdev->limits.reserved_mrws;
  98	props->max_pd              = mdev->limits.num_pds - mdev->limits.reserved_pds;
  99	props->max_qp_rd_atom      = 1 << mdev->qp_table.rdb_shift;
 100	props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma;
 101	props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
 102	props->max_srq             = mdev->limits.num_srqs - mdev->limits.reserved_srqs;
 103	props->max_srq_wr          = mdev->limits.max_srq_wqes;
 104	props->max_srq_sge         = mdev->limits.max_srq_sge;
 105	props->local_ca_ack_delay  = mdev->limits.local_ca_ack_delay;
 106	props->atomic_cap          = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ?
 107					IB_ATOMIC_HCA : IB_ATOMIC_NONE;
 108	props->max_pkeys           = mdev->limits.pkey_table_len;
 109	props->max_mcast_grp       = mdev->limits.num_mgms + mdev->limits.num_amgms;
 110	props->max_mcast_qp_attach = MTHCA_QP_PER_MGM;
 111	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 112					   props->max_mcast_grp;
 113
 114	err = 0;
 115 out:
 116	kfree(in_mad);
 117	kfree(out_mad);
 118	return err;
 119}
 120
 121static int mthca_query_port(struct ib_device *ibdev,
 122			    u32 port, struct ib_port_attr *props)
 123{
 124	struct ib_smp *in_mad;
 125	struct ib_smp *out_mad;
 126	int err = -ENOMEM;
 127
 128	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 129	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 130	if (!in_mad || !out_mad)
 131		goto out;
 132
 133	/* props being zeroed by the caller, avoid zeroing it here */
 134
 135	ib_init_query_mad(in_mad);
 136	in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
 137	in_mad->attr_mod = cpu_to_be32(port);
 138
 139	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
 140			    port, NULL, NULL, in_mad, out_mad);
 141	if (err)
 142		goto out;
 143
 144	props->lid               = be16_to_cpup((__be16 *) (out_mad->data + 16));
 145	props->lmc               = out_mad->data[34] & 0x7;
 146	props->sm_lid            = be16_to_cpup((__be16 *) (out_mad->data + 18));
 147	props->sm_sl             = out_mad->data[36] & 0xf;
 148	props->state             = out_mad->data[32] & 0xf;
 149	props->phys_state        = out_mad->data[33] >> 4;
 150	props->port_cap_flags    = be32_to_cpup((__be32 *) (out_mad->data + 20));
 151	props->gid_tbl_len       = to_mdev(ibdev)->limits.gid_table_len;
 152	props->max_msg_sz        = 0x80000000;
 153	props->pkey_tbl_len      = to_mdev(ibdev)->limits.pkey_table_len;
 154	props->bad_pkey_cntr     = be16_to_cpup((__be16 *) (out_mad->data + 46));
 155	props->qkey_viol_cntr    = be16_to_cpup((__be16 *) (out_mad->data + 48));
 156	props->active_width      = out_mad->data[31] & 0xf;
 157	props->active_speed      = out_mad->data[35] >> 4;
 158	props->max_mtu           = out_mad->data[41] & 0xf;
 159	props->active_mtu        = out_mad->data[36] >> 4;
 160	props->subnet_timeout    = out_mad->data[51] & 0x1f;
 161	props->max_vl_num        = out_mad->data[37] >> 4;
 162	props->init_type_reply   = out_mad->data[41] >> 4;
 163
 164 out:
 165	kfree(in_mad);
 166	kfree(out_mad);
 167	return err;
 168}
 169
 170static int mthca_modify_device(struct ib_device *ibdev,
 171			       int mask,
 172			       struct ib_device_modify *props)
 173{
 174	if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
 175		return -EOPNOTSUPP;
 176
 177	if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
 178		if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
 179			return -ERESTARTSYS;
 180		memcpy(ibdev->node_desc, props->node_desc,
 181		       IB_DEVICE_NODE_DESC_MAX);
 182		mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
 183	}
 184
 185	return 0;
 186}
 187
 188static int mthca_modify_port(struct ib_device *ibdev,
 189			     u32 port, int port_modify_mask,
 190			     struct ib_port_modify *props)
 191{
 192	struct mthca_set_ib_param set_ib;
 193	struct ib_port_attr attr;
 194	int err;
 195
 196	if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex))
 197		return -ERESTARTSYS;
 198
 199	err = ib_query_port(ibdev, port, &attr);
 200	if (err)
 201		goto out;
 202
 203	set_ib.set_si_guid     = 0;
 204	set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR);
 205
 206	set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
 207		~props->clr_port_cap_mask;
 208
 209	err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port);
 210	if (err)
 211		goto out;
 212out:
 213	mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
 214	return err;
 215}
 216
 217static int mthca_query_pkey(struct ib_device *ibdev,
 218			    u32 port, u16 index, u16 *pkey)
 219{
 220	struct ib_smp *in_mad;
 221	struct ib_smp *out_mad;
 222	int err = -ENOMEM;
 223
 224	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 225	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 226	if (!in_mad || !out_mad)
 227		goto out;
 228
 229	ib_init_query_mad(in_mad);
 230	in_mad->attr_id  = IB_SMP_ATTR_PKEY_TABLE;
 231	in_mad->attr_mod = cpu_to_be32(index / 32);
 232
 233	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
 234			    port, NULL, NULL, in_mad, out_mad);
 235	if (err)
 236		goto out;
 237
 238	*pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]);
 239
 240 out:
 241	kfree(in_mad);
 242	kfree(out_mad);
 243	return err;
 244}
 245
 246static int mthca_query_gid(struct ib_device *ibdev, u32 port,
 247			   int index, union ib_gid *gid)
 248{
 249	struct ib_smp *in_mad;
 250	struct ib_smp *out_mad;
 251	int err = -ENOMEM;
 252
 253	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 254	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 255	if (!in_mad || !out_mad)
 256		goto out;
 257
 258	ib_init_query_mad(in_mad);
 259	in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
 260	in_mad->attr_mod = cpu_to_be32(port);
 261
 262	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
 263			    port, NULL, NULL, in_mad, out_mad);
 264	if (err)
 265		goto out;
 266
 267	memcpy(gid->raw, out_mad->data + 8, 8);
 268
 269	ib_init_query_mad(in_mad);
 270	in_mad->attr_id  = IB_SMP_ATTR_GUID_INFO;
 271	in_mad->attr_mod = cpu_to_be32(index / 8);
 272
 273	err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1,
 274			    port, NULL, NULL, in_mad, out_mad);
 275	if (err)
 276		goto out;
 277
 278	memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8);
 279
 280 out:
 281	kfree(in_mad);
 282	kfree(out_mad);
 283	return err;
 284}
 285
 286static int mthca_alloc_ucontext(struct ib_ucontext *uctx,
 287				struct ib_udata *udata)
 288{
 289	struct ib_device *ibdev = uctx->device;
 290	struct mthca_alloc_ucontext_resp uresp = {};
 291	struct mthca_ucontext *context = to_mucontext(uctx);
 292	int                              err;
 293
 294	if (!(to_mdev(ibdev)->active))
 295		return -EAGAIN;
 296
 297	uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
 298	if (mthca_is_memfree(to_mdev(ibdev)))
 299		uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size;
 300	else
 301		uresp.uarc_size = 0;
 302
 303	err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
 304	if (err)
 305		return err;
 306
 307	context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
 308	if (IS_ERR(context->db_tab)) {
 309		err = PTR_ERR(context->db_tab);
 310		mthca_uar_free(to_mdev(ibdev), &context->uar);
 311		return err;
 312	}
 313
 314	if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
 315		mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab);
 316		mthca_uar_free(to_mdev(ibdev), &context->uar);
 317		return -EFAULT;
 318	}
 319
 320	context->reg_mr_warned = 0;
 321
 322	return 0;
 323}
 324
 325static void mthca_dealloc_ucontext(struct ib_ucontext *context)
 326{
 327	mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar,
 328				  to_mucontext(context)->db_tab);
 329	mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar);
 330}
 331
 332static int mthca_mmap_uar(struct ib_ucontext *context,
 333			  struct vm_area_struct *vma)
 334{
 335	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
 336		return -EINVAL;
 337
 338	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 339
 340	if (io_remap_pfn_range(vma, vma->vm_start,
 341			       to_mucontext(context)->uar.pfn,
 342			       PAGE_SIZE, vma->vm_page_prot))
 343		return -EAGAIN;
 344
 345	return 0;
 346}
 347
 348static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 349{
 350	struct ib_device *ibdev = ibpd->device;
 351	struct mthca_pd *pd = to_mpd(ibpd);
 352	int err;
 353
 354	err = mthca_pd_alloc(to_mdev(ibdev), !udata, pd);
 355	if (err)
 356		return err;
 357
 358	if (udata) {
 359		if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) {
 360			mthca_pd_free(to_mdev(ibdev), pd);
 361			return -EFAULT;
 362		}
 363	}
 364
 365	return 0;
 366}
 367
 368static int mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 369{
 370	mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
 371	return 0;
 372}
 373
 374static int mthca_ah_create(struct ib_ah *ibah,
 375			   struct rdma_ah_init_attr *init_attr,
 376			   struct ib_udata *udata)
 377
 378{
 379	struct mthca_ah *ah = to_mah(ibah);
 380
 381	return mthca_create_ah(to_mdev(ibah->device), to_mpd(ibah->pd),
 382			       init_attr->ah_attr, ah);
 383}
 384
 385static int mthca_ah_destroy(struct ib_ah *ah, u32 flags)
 386{
 387	mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
 388	return 0;
 389}
 390
 391static int mthca_create_srq(struct ib_srq *ibsrq,
 392			    struct ib_srq_init_attr *init_attr,
 393			    struct ib_udata *udata)
 394{
 395	struct mthca_create_srq ucmd;
 396	struct mthca_ucontext *context = rdma_udata_to_drv_context(
 397		udata, struct mthca_ucontext, ibucontext);
 398	struct mthca_srq *srq = to_msrq(ibsrq);
 399	int err;
 400
 401	if (init_attr->srq_type != IB_SRQT_BASIC)
 402		return -EOPNOTSUPP;
 403
 404	if (udata) {
 405		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
 406			return -EFAULT;
 407
 408		err = mthca_map_user_db(to_mdev(ibsrq->device), &context->uar,
 409					context->db_tab, ucmd.db_index,
 410					ucmd.db_page);
 411
 412		if (err)
 413			return err;
 414
 415		srq->mr.ibmr.lkey = ucmd.lkey;
 416		srq->db_index     = ucmd.db_index;
 417	}
 418
 419	err = mthca_alloc_srq(to_mdev(ibsrq->device), to_mpd(ibsrq->pd),
 420			      &init_attr->attr, srq, udata);
 421
 422	if (err && udata)
 423		mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar,
 424				    context->db_tab, ucmd.db_index);
 425
 426	if (err)
 427		return err;
 428
 429	if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) {
 430		mthca_free_srq(to_mdev(ibsrq->device), srq);
 431		mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar,
 432				    context->db_tab, ucmd.db_index);
 433		return -EFAULT;
 434	}
 435
 436	return 0;
 437}
 438
 439static int mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
 440{
 441	mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
 442	if (udata) {
 443		struct mthca_ucontext *context =
 444			rdma_udata_to_drv_context(
 445				udata,
 446				struct mthca_ucontext,
 447				ibucontext);
 448
 449		mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
 450				    context->db_tab, to_msrq(srq)->db_index);
 451	}
 452	return 0;
 453}
 454
 455static int mthca_create_qp(struct ib_qp *ibqp,
 456			   struct ib_qp_init_attr *init_attr,
 457			   struct ib_udata *udata)
 458{
 459	struct mthca_ucontext *context = rdma_udata_to_drv_context(
 460		udata, struct mthca_ucontext, ibucontext);
 461	struct mthca_create_qp ucmd;
 462	struct mthca_qp *qp = to_mqp(ibqp);
 463	struct mthca_dev *dev = to_mdev(ibqp->device);
 464	int err;
 465
 466	if (init_attr->create_flags)
 467		return -EOPNOTSUPP;
 468
 469	switch (init_attr->qp_type) {
 470	case IB_QPT_RC:
 471	case IB_QPT_UC:
 472	case IB_QPT_UD:
 473	{
 474		if (udata) {
 475			if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
 476				return -EFAULT;
 477
 478			err = mthca_map_user_db(dev, &context->uar,
 479						context->db_tab,
 480						ucmd.sq_db_index,
 481						ucmd.sq_db_page);
 482			if (err)
 483				return err;
 484
 485			err = mthca_map_user_db(dev, &context->uar,
 486						context->db_tab,
 487						ucmd.rq_db_index,
 488						ucmd.rq_db_page);
 489			if (err) {
 490				mthca_unmap_user_db(dev, &context->uar,
 491						    context->db_tab,
 492						    ucmd.sq_db_index);
 493				return err;
 494			}
 495
 496			qp->mr.ibmr.lkey = ucmd.lkey;
 497			qp->sq.db_index  = ucmd.sq_db_index;
 498			qp->rq.db_index  = ucmd.rq_db_index;
 499		}
 500
 501		err = mthca_alloc_qp(dev, to_mpd(ibqp->pd),
 502				     to_mcq(init_attr->send_cq),
 503				     to_mcq(init_attr->recv_cq),
 504				     init_attr->qp_type, init_attr->sq_sig_type,
 505				     &init_attr->cap, qp, udata);
 506
 507		if (err && udata) {
 508			mthca_unmap_user_db(dev, &context->uar, context->db_tab,
 509					    ucmd.sq_db_index);
 510			mthca_unmap_user_db(dev, &context->uar, context->db_tab,
 511					    ucmd.rq_db_index);
 512		}
 513
 514		qp->ibqp.qp_num = qp->qpn;
 515		break;
 516	}
 517	case IB_QPT_SMI:
 518	case IB_QPT_GSI:
 519	{
 520		qp->sqp = kzalloc(sizeof(struct mthca_sqp), GFP_KERNEL);
 521		if (!qp->sqp)
 522			return -ENOMEM;
 523
 524		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
 525
 526		err = mthca_alloc_sqp(dev, to_mpd(ibqp->pd),
 527				      to_mcq(init_attr->send_cq),
 528				      to_mcq(init_attr->recv_cq),
 529				      init_attr->sq_sig_type, &init_attr->cap,
 530				      qp->ibqp.qp_num, init_attr->port_num, qp,
 531				      udata);
 532		break;
 533	}
 534	default:
 535		/* Don't support raw QPs */
 536		return -EOPNOTSUPP;
 537	}
 538
 539	if (err) {
 540		kfree(qp->sqp);
 541		return err;
 542	}
 543
 544	init_attr->cap.max_send_wr     = qp->sq.max;
 545	init_attr->cap.max_recv_wr     = qp->rq.max;
 546	init_attr->cap.max_send_sge    = qp->sq.max_gs;
 547	init_attr->cap.max_recv_sge    = qp->rq.max_gs;
 548	init_attr->cap.max_inline_data = qp->max_inline_data;
 549
 550	return 0;
 551}
 552
 553static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
 554{
 555	if (udata) {
 556		struct mthca_ucontext *context =
 557			rdma_udata_to_drv_context(
 558				udata,
 559				struct mthca_ucontext,
 560				ibucontext);
 561
 562		mthca_unmap_user_db(to_mdev(qp->device),
 563				    &context->uar,
 564				    context->db_tab,
 565				    to_mqp(qp)->sq.db_index);
 566		mthca_unmap_user_db(to_mdev(qp->device),
 567				    &context->uar,
 568				    context->db_tab,
 569				    to_mqp(qp)->rq.db_index);
 570	}
 571	mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
 572	kfree(to_mqp(qp)->sqp);
 573	return 0;
 574}
 575
 576static int mthca_create_cq(struct ib_cq *ibcq,
 577			   const struct ib_cq_init_attr *attr,
 578			   struct uverbs_attr_bundle *attrs)
 579{
 580	struct ib_udata *udata = &attrs->driver_udata;
 581	struct ib_device *ibdev = ibcq->device;
 582	int entries = attr->cqe;
 583	struct mthca_create_cq ucmd;
 584	struct mthca_cq *cq;
 585	int nent;
 586	int err;
 587	struct mthca_ucontext *context = rdma_udata_to_drv_context(
 588		udata, struct mthca_ucontext, ibucontext);
 589
 590	if (attr->flags)
 591		return -EOPNOTSUPP;
 592
 593	if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
 594		return -EINVAL;
 595
 596	if (udata) {
 597		if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
 598			return -EFAULT;
 599
 600		err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
 601					context->db_tab, ucmd.set_db_index,
 602					ucmd.set_db_page);
 603		if (err)
 604			return err;
 605
 606		err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
 607					context->db_tab, ucmd.arm_db_index,
 608					ucmd.arm_db_page);
 609		if (err)
 610			goto err_unmap_set;
 611	}
 612
 613	cq = to_mcq(ibcq);
 614
 615	if (udata) {
 616		cq->buf.mr.ibmr.lkey = ucmd.lkey;
 617		cq->set_ci_db_index  = ucmd.set_db_index;
 618		cq->arm_db_index     = ucmd.arm_db_index;
 619	}
 620
 621	for (nent = 1; nent <= entries; nent <<= 1)
 622		; /* nothing */
 623
 624	err = mthca_init_cq(to_mdev(ibdev), nent, context,
 625			    udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
 626			    cq);
 627	if (err)
 628		goto err_unmap_arm;
 629
 630	if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) {
 631		mthca_free_cq(to_mdev(ibdev), cq);
 632		err = -EFAULT;
 633		goto err_unmap_arm;
 634	}
 635
 636	cq->resize_buf = NULL;
 637
 638	return 0;
 639
 640err_unmap_arm:
 641	if (udata)
 642		mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
 643				    context->db_tab, ucmd.arm_db_index);
 644
 645err_unmap_set:
 646	if (udata)
 647		mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
 648				    context->db_tab, ucmd.set_db_index);
 649
 650	return err;
 651}
 652
 653static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq,
 654				  int entries)
 655{
 656	int ret;
 657
 658	spin_lock_irq(&cq->lock);
 659	if (cq->resize_buf) {
 660		ret = -EBUSY;
 661		goto unlock;
 662	}
 663
 664	cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC);
 665	if (!cq->resize_buf) {
 666		ret = -ENOMEM;
 667		goto unlock;
 668	}
 669
 670	cq->resize_buf->state = CQ_RESIZE_ALLOC;
 671
 672	ret = 0;
 673
 674unlock:
 675	spin_unlock_irq(&cq->lock);
 676
 677	if (ret)
 678		return ret;
 679
 680	ret = mthca_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);
 681	if (ret) {
 682		spin_lock_irq(&cq->lock);
 683		kfree(cq->resize_buf);
 684		cq->resize_buf = NULL;
 685		spin_unlock_irq(&cq->lock);
 686		return ret;
 687	}
 688
 689	cq->resize_buf->cqe = entries - 1;
 690
 691	spin_lock_irq(&cq->lock);
 692	cq->resize_buf->state = CQ_RESIZE_READY;
 693	spin_unlock_irq(&cq->lock);
 694
 695	return 0;
 696}
 697
 698static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
 699{
 700	struct mthca_dev *dev = to_mdev(ibcq->device);
 701	struct mthca_cq *cq = to_mcq(ibcq);
 702	struct mthca_resize_cq ucmd;
 703	u32 lkey;
 704	int ret;
 705
 706	if (entries < 1 || entries > dev->limits.max_cqes)
 707		return -EINVAL;
 708
 709	mutex_lock(&cq->mutex);
 710
 711	entries = roundup_pow_of_two(entries + 1);
 712	if (entries == ibcq->cqe + 1) {
 713		ret = 0;
 714		goto out;
 715	}
 716
 717	if (cq->is_kernel) {
 718		ret = mthca_alloc_resize_buf(dev, cq, entries);
 719		if (ret)
 720			goto out;
 721		lkey = cq->resize_buf->buf.mr.ibmr.lkey;
 722	} else {
 723		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
 724			ret = -EFAULT;
 725			goto out;
 726		}
 727		lkey = ucmd.lkey;
 728	}
 729
 730	ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, ilog2(entries));
 731
 732	if (ret) {
 733		if (cq->resize_buf) {
 734			mthca_free_cq_buf(dev, &cq->resize_buf->buf,
 735					  cq->resize_buf->cqe);
 736			kfree(cq->resize_buf);
 737			spin_lock_irq(&cq->lock);
 738			cq->resize_buf = NULL;
 739			spin_unlock_irq(&cq->lock);
 740		}
 741		goto out;
 742	}
 743
 744	if (cq->is_kernel) {
 745		struct mthca_cq_buf tbuf;
 746		int tcqe;
 747
 748		spin_lock_irq(&cq->lock);
 749		if (cq->resize_buf->state == CQ_RESIZE_READY) {
 750			mthca_cq_resize_copy_cqes(cq);
 751			tbuf         = cq->buf;
 752			tcqe         = cq->ibcq.cqe;
 753			cq->buf      = cq->resize_buf->buf;
 754			cq->ibcq.cqe = cq->resize_buf->cqe;
 755		} else {
 756			tbuf = cq->resize_buf->buf;
 757			tcqe = cq->resize_buf->cqe;
 758		}
 759
 760		kfree(cq->resize_buf);
 761		cq->resize_buf = NULL;
 762		spin_unlock_irq(&cq->lock);
 763
 764		mthca_free_cq_buf(dev, &tbuf, tcqe);
 765	} else
 766		ibcq->cqe = entries - 1;
 767
 768out:
 769	mutex_unlock(&cq->mutex);
 770
 771	return ret;
 772}
 773
 774static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
 775{
 776	if (udata) {
 777		struct mthca_ucontext *context =
 778			rdma_udata_to_drv_context(
 779				udata,
 780				struct mthca_ucontext,
 781				ibucontext);
 782
 783		mthca_unmap_user_db(to_mdev(cq->device),
 784				    &context->uar,
 785				    context->db_tab,
 786				    to_mcq(cq)->arm_db_index);
 787		mthca_unmap_user_db(to_mdev(cq->device),
 788				    &context->uar,
 789				    context->db_tab,
 790				    to_mcq(cq)->set_ci_db_index);
 791	}
 792	mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
 793	return 0;
 794}
 795
 796static inline u32 convert_access(int acc)
 797{
 798	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC       : 0) |
 799	       (acc & IB_ACCESS_REMOTE_WRITE  ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) |
 800	       (acc & IB_ACCESS_REMOTE_READ   ? MTHCA_MPT_FLAG_REMOTE_READ  : 0) |
 801	       (acc & IB_ACCESS_LOCAL_WRITE   ? MTHCA_MPT_FLAG_LOCAL_WRITE  : 0) |
 802	       MTHCA_MPT_FLAG_LOCAL_READ;
 803}
 804
 805static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc)
 806{
 807	struct mthca_mr *mr;
 808	int err;
 809
 810	mr = kmalloc(sizeof *mr, GFP_KERNEL);
 811	if (!mr)
 812		return ERR_PTR(-ENOMEM);
 813
 814	err = mthca_mr_alloc_notrans(to_mdev(pd->device),
 815				     to_mpd(pd)->pd_num,
 816				     convert_access(acc), mr);
 817
 818	if (err) {
 819		kfree(mr);
 820		return ERR_PTR(err);
 821	}
 822
 823	mr->umem = NULL;
 824
 825	return &mr->ibmr;
 826}
 827
 828static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 829				       u64 virt, int acc, struct ib_dmah *dmah,
 830				       struct ib_udata *udata)
 831{
 832	struct mthca_dev *dev = to_mdev(pd->device);
 833	struct ib_block_iter biter;
 834	struct mthca_ucontext *context = rdma_udata_to_drv_context(
 835		udata, struct mthca_ucontext, ibucontext);
 836	struct mthca_mr *mr;
 837	struct mthca_reg_mr ucmd;
 838	u64 *pages;
 839	int n, i;
 840	int err = 0;
 841	int write_mtt_size;
 842
 843	if (dmah)
 844		return ERR_PTR(-EOPNOTSUPP);
 845
 846	if (udata->inlen < sizeof ucmd) {
 847		if (!context->reg_mr_warned) {
 848			mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n",
 849				   current->comm);
 850			mthca_warn(dev, "  Update libmthca to fix this.\n");
 851		}
 852		++context->reg_mr_warned;
 853		ucmd.mr_attrs = 0;
 854	} else if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
 855		return ERR_PTR(-EFAULT);
 856
 857	mr = kmalloc(sizeof *mr, GFP_KERNEL);
 858	if (!mr)
 859		return ERR_PTR(-ENOMEM);
 860
 861	mr->umem = ib_umem_get(pd->device, start, length, acc);
 862	if (IS_ERR(mr->umem)) {
 863		err = PTR_ERR(mr->umem);
 864		goto err;
 865	}
 866
 867	n = ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE);
 868
 869	mr->mtt = mthca_alloc_mtt(dev, n);
 870	if (IS_ERR(mr->mtt)) {
 871		err = PTR_ERR(mr->mtt);
 872		goto err_umem;
 873	}
 874
 875	pages = (u64 *) __get_free_page(GFP_KERNEL);
 876	if (!pages) {
 877		err = -ENOMEM;
 878		goto err_mtt;
 879	}
 880
 881	i = n = 0;
 882
 883	write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages));
 884
 885	rdma_umem_for_each_dma_block(mr->umem, &biter, PAGE_SIZE) {
 886		pages[i++] = rdma_block_iter_dma_address(&biter);
 887
 888		/*
 889		 * Be friendly to write_mtt and pass it chunks
 890		 * of appropriate size.
 891		 */
 892		if (i == write_mtt_size) {
 893			err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
 894			if (err)
 895				goto mtt_done;
 896			n += i;
 897			i = 0;
 898		}
 899	}
 900
 901	if (i)
 902		err = mthca_write_mtt(dev, mr->mtt, n, pages, i);
 903mtt_done:
 904	free_page((unsigned long) pages);
 905	if (err)
 906		goto err_mtt;
 907
 908	err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, PAGE_SHIFT, virt, length,
 909			     convert_access(acc), mr);
 910
 911	if (err)
 912		goto err_mtt;
 913
 914	return &mr->ibmr;
 915
 916err_mtt:
 917	mthca_free_mtt(dev, mr->mtt);
 918
 919err_umem:
 920	ib_umem_release(mr->umem);
 921
 922err:
 923	kfree(mr);
 924	return ERR_PTR(err);
 925}
 926
 927static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata)
 928{
 929	struct mthca_mr *mmr = to_mmr(mr);
 930
 931	mthca_free_mr(to_mdev(mr->device), mmr);
 932	ib_umem_release(mmr->umem);
 933	kfree(mmr);
 934
 935	return 0;
 936}
 937
 938static ssize_t hw_rev_show(struct device *device,
 939			   struct device_attribute *attr, char *buf)
 940{
 941	struct mthca_dev *dev =
 942		rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
 943
 944	return sysfs_emit(buf, "%x\n", dev->rev_id);
 945}
 946static DEVICE_ATTR_RO(hw_rev);
 947
 948static const char *hca_type_string(int hca_type)
 949{
 950	switch (hca_type) {
 951	case PCI_DEVICE_ID_MELLANOX_TAVOR:
 952		return "MT23108";
 953	case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT:
 954		return "MT25208 (MT23108 compat mode)";
 955	case PCI_DEVICE_ID_MELLANOX_ARBEL:
 956		return "MT25208";
 957	case PCI_DEVICE_ID_MELLANOX_SINAI:
 958	case PCI_DEVICE_ID_MELLANOX_SINAI_OLD:
 959		return "MT25204";
 960	}
 961
 962	return "unknown";
 963}
 964
 965static ssize_t hca_type_show(struct device *device,
 966			     struct device_attribute *attr, char *buf)
 967{
 968	struct mthca_dev *dev =
 969		rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
 970
 971	return sysfs_emit(buf, "%s\n", hca_type_string(dev->pdev->device));
 972}
 973static DEVICE_ATTR_RO(hca_type);
 974
 975static ssize_t board_id_show(struct device *device,
 976			     struct device_attribute *attr, char *buf)
 977{
 978	struct mthca_dev *dev =
 979		rdma_device_to_drv_device(device, struct mthca_dev, ib_dev);
 980
 981	return sysfs_emit(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id);
 982}
 983static DEVICE_ATTR_RO(board_id);
 984
 985static struct attribute *mthca_dev_attributes[] = {
 986	&dev_attr_hw_rev.attr,
 987	&dev_attr_hca_type.attr,
 988	&dev_attr_board_id.attr,
 989	NULL
 990};
 991
 992static const struct attribute_group mthca_attr_group = {
 993	.attrs = mthca_dev_attributes,
 994};
 995
 996static int mthca_init_node_data(struct mthca_dev *dev)
 997{
 998	struct ib_smp *in_mad;
 999	struct ib_smp *out_mad;
1000	int err = -ENOMEM;
1001
1002	in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
1003	out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
1004	if (!in_mad || !out_mad)
1005		goto out;
1006
1007	ib_init_query_mad(in_mad);
1008	in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
1009
1010	err = mthca_MAD_IFC(dev, 1, 1,
1011			    1, NULL, NULL, in_mad, out_mad);
1012	if (err)
1013		goto out;
1014
1015	memcpy(dev->ib_dev.node_desc, out_mad->data, IB_DEVICE_NODE_DESC_MAX);
1016
1017	in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
1018
1019	err = mthca_MAD_IFC(dev, 1, 1,
1020			    1, NULL, NULL, in_mad, out_mad);
1021	if (err)
1022		goto out;
1023
1024	if (mthca_is_memfree(dev))
1025		dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32));
1026	memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
1027
1028out:
1029	kfree(in_mad);
1030	kfree(out_mad);
1031	return err;
1032}
1033
1034static int mthca_port_immutable(struct ib_device *ibdev, u32 port_num,
1035			        struct ib_port_immutable *immutable)
1036{
1037	struct ib_port_attr attr;
1038	int err;
1039
1040	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB;
1041
1042	err = ib_query_port(ibdev, port_num, &attr);
1043	if (err)
1044		return err;
1045
1046	immutable->pkey_tbl_len = attr.pkey_tbl_len;
1047	immutable->gid_tbl_len = attr.gid_tbl_len;
1048	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
1049
1050	return 0;
1051}
1052
1053static void get_dev_fw_str(struct ib_device *device, char *str)
1054{
1055	struct mthca_dev *dev =
1056		container_of(device, struct mthca_dev, ib_dev);
1057	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%d",
1058		 (int) (dev->fw_ver >> 32),
1059		 (int) (dev->fw_ver >> 16) & 0xffff,
1060		 (int) dev->fw_ver & 0xffff);
1061}
1062
1063static const struct ib_device_ops mthca_dev_ops = {
1064	.owner = THIS_MODULE,
1065	.driver_id = RDMA_DRIVER_MTHCA,
1066	.uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION,
1067	.uverbs_no_driver_id_binding = 1,
1068
1069	.alloc_pd = mthca_alloc_pd,
1070	.alloc_ucontext = mthca_alloc_ucontext,
1071	.attach_mcast = mthca_multicast_attach,
1072	.create_ah = mthca_ah_create,
1073	.create_cq = mthca_create_cq,
1074	.create_qp = mthca_create_qp,
1075	.dealloc_pd = mthca_dealloc_pd,
1076	.dealloc_ucontext = mthca_dealloc_ucontext,
1077	.dereg_mr = mthca_dereg_mr,
1078	.destroy_ah = mthca_ah_destroy,
1079	.destroy_cq = mthca_destroy_cq,
1080	.destroy_qp = mthca_destroy_qp,
1081	.detach_mcast = mthca_multicast_detach,
1082	.device_group = &mthca_attr_group,
1083	.get_dev_fw_str = get_dev_fw_str,
1084	.get_dma_mr = mthca_get_dma_mr,
1085	.get_port_immutable = mthca_port_immutable,
1086	.mmap = mthca_mmap_uar,
1087	.modify_device = mthca_modify_device,
1088	.modify_port = mthca_modify_port,
1089	.modify_qp = mthca_modify_qp,
1090	.poll_cq = mthca_poll_cq,
1091	.process_mad = mthca_process_mad,
1092	.query_ah = mthca_ah_query,
1093	.query_device = mthca_query_device,
1094	.query_gid = mthca_query_gid,
1095	.query_pkey = mthca_query_pkey,
1096	.query_port = mthca_query_port,
1097	.query_qp = mthca_query_qp,
1098	.reg_user_mr = mthca_reg_user_mr,
1099	.resize_cq = mthca_resize_cq,
1100
1101	INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah),
1102	INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq),
1103	INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd),
1104	INIT_RDMA_OBJ_SIZE(ib_qp, mthca_qp, ibqp),
1105	INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext),
1106};
1107
1108static const struct ib_device_ops mthca_dev_arbel_srq_ops = {
1109	.create_srq = mthca_create_srq,
1110	.destroy_srq = mthca_destroy_srq,
1111	.modify_srq = mthca_modify_srq,
1112	.post_srq_recv = mthca_arbel_post_srq_recv,
1113	.query_srq = mthca_query_srq,
1114
1115	INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq),
1116};
1117
1118static const struct ib_device_ops mthca_dev_tavor_srq_ops = {
1119	.create_srq = mthca_create_srq,
1120	.destroy_srq = mthca_destroy_srq,
1121	.modify_srq = mthca_modify_srq,
1122	.post_srq_recv = mthca_tavor_post_srq_recv,
1123	.query_srq = mthca_query_srq,
1124
1125	INIT_RDMA_OBJ_SIZE(ib_srq, mthca_srq, ibsrq),
1126};
1127
1128static const struct ib_device_ops mthca_dev_arbel_ops = {
1129	.post_recv = mthca_arbel_post_receive,
1130	.post_send = mthca_arbel_post_send,
1131	.req_notify_cq = mthca_arbel_arm_cq,
1132};
1133
1134static const struct ib_device_ops mthca_dev_tavor_ops = {
1135	.post_recv = mthca_tavor_post_receive,
1136	.post_send = mthca_tavor_post_send,
1137	.req_notify_cq = mthca_tavor_arm_cq,
1138};
1139
1140int mthca_register_device(struct mthca_dev *dev)
1141{
1142	int ret;
1143
1144	ret = mthca_init_node_data(dev);
1145	if (ret)
1146		return ret;
1147
1148	dev->ib_dev.node_type            = RDMA_NODE_IB_CA;
1149	dev->ib_dev.phys_port_cnt        = dev->limits.num_ports;
1150	dev->ib_dev.num_comp_vectors     = 1;
1151	dev->ib_dev.dev.parent           = &dev->pdev->dev;
1152
1153	if (dev->mthca_flags & MTHCA_FLAG_SRQ) {
1154		if (mthca_is_memfree(dev))
1155			ib_set_device_ops(&dev->ib_dev,
1156					  &mthca_dev_arbel_srq_ops);
1157		else
1158			ib_set_device_ops(&dev->ib_dev,
1159					  &mthca_dev_tavor_srq_ops);
1160	}
1161
1162	ib_set_device_ops(&dev->ib_dev, &mthca_dev_ops);
1163
1164	if (mthca_is_memfree(dev))
1165		ib_set_device_ops(&dev->ib_dev, &mthca_dev_arbel_ops);
1166	else
1167		ib_set_device_ops(&dev->ib_dev, &mthca_dev_tavor_ops);
1168
1169	mutex_init(&dev->cap_mask_mutex);
1170
1171	ret = ib_register_device(&dev->ib_dev, "mthca%d", &dev->pdev->dev);
1172	if (ret)
1173		return ret;
1174
1175	mthca_start_catas_poll(dev);
1176
1177	return 0;
1178}
1179
1180void mthca_unregister_device(struct mthca_dev *dev)
1181{
1182	mthca_stop_catas_poll(dev);
1183	ib_unregister_device(&dev->ib_dev);
1184}