qp.c

   1/*
   2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/etherdevice.h>
  34#include <rdma/ib_umem.h>
  35#include <rdma/ib_cache.h>
  36#include <rdma/ib_user_verbs.h>
  37#include <rdma/rdma_counter.h>
  38#include <linux/mlx5/fs.h>
  39#include "mlx5_ib.h"
  40#include "ib_rep.h"
  41#include "counters.h"
  42#include "cmd.h"
  43#include "umr.h"
  44#include "qp.h"
  45#include "wr.h"
  46
  47enum {
  48	MLX5_IB_ACK_REQ_FREQ	= 8,
  49};
  50
  51enum {
  52	MLX5_IB_DEFAULT_SCHED_QUEUE	= 0x83,
  53	MLX5_IB_DEFAULT_QP0_SCHED_QUEUE	= 0x3f,
  54	MLX5_IB_LINK_TYPE_IB		= 0,
  55	MLX5_IB_LINK_TYPE_ETH		= 1
  56};
  57
  58enum raw_qp_set_mask_map {
  59	MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID		= 1UL << 0,
  60	MLX5_RAW_QP_RATE_LIMIT			= 1UL << 1,
  61};
  62
  63enum {
  64	MLX5_QP_RM_GO_BACK_N			= 0x1,
  65};
  66
  67struct mlx5_modify_raw_qp_param {
  68	u16 operation;
  69
  70	u32 set_mask; /* raw_qp_set_mask_map */
  71
  72	struct mlx5_rate_limit rl;
  73
  74	u8 rq_q_ctr_id;
  75	u32 port;
  76};
  77
  78struct mlx5_ib_qp_event_work {
  79	struct work_struct work;
  80	struct mlx5_core_qp *qp;
  81	int type;
  82};
  83
  84static struct workqueue_struct *mlx5_ib_qp_event_wq;
  85
  86static void get_cqs(enum ib_qp_type qp_type,
  87		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
  88		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
  89
  90static int is_qp0(enum ib_qp_type qp_type)
  91{
  92	return qp_type == IB_QPT_SMI;
  93}
  94
  95static int is_sqp(enum ib_qp_type qp_type)
  96{
  97	return is_qp0(qp_type) || is_qp1(qp_type);
  98}
  99
 100/**
 101 * mlx5_ib_read_user_wqe_common() - Copy a WQE (or part of) from user WQ
 102 * to kernel buffer
 103 *
 104 * @umem: User space memory where the WQ is
 105 * @buffer: buffer to copy to
 106 * @buflen: buffer length
 107 * @wqe_index: index of WQE to copy from
 108 * @wq_offset: offset to start of WQ
 109 * @wq_wqe_cnt: number of WQEs in WQ
 110 * @wq_wqe_shift: log2 of WQE size
 111 * @bcnt: number of bytes to copy
 112 * @bytes_copied: number of bytes to copy (return value)
 113 *
 114 * Copies from start of WQE bcnt or less bytes.
 115 * Does not gurantee to copy the entire WQE.
 116 *
 117 * Return: zero on success, or an error code.
 118 */
 119static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem, void *buffer,
 120					size_t buflen, int wqe_index,
 121					int wq_offset, int wq_wqe_cnt,
 122					int wq_wqe_shift, int bcnt,
 123					size_t *bytes_copied)
 124{
 125	size_t offset = wq_offset + ((wqe_index % wq_wqe_cnt) << wq_wqe_shift);
 126	size_t wq_end = wq_offset + (wq_wqe_cnt << wq_wqe_shift);
 127	size_t copy_length;
 128	int ret;
 129
 130	/* don't copy more than requested, more than buffer length or
 131	 * beyond WQ end
 132	 */
 133	copy_length = min_t(u32, buflen, wq_end - offset);
 134	copy_length = min_t(u32, copy_length, bcnt);
 135
 136	ret = ib_umem_copy_from(buffer, umem, offset, copy_length);
 137	if (ret)
 138		return ret;
 139
 140	if (!ret && bytes_copied)
 141		*bytes_copied = copy_length;
 142
 143	return 0;
 144}
 145
 146static int mlx5_ib_read_kernel_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
 147				      void *buffer, size_t buflen, size_t *bc)
 148{
 149	struct mlx5_wqe_ctrl_seg *ctrl;
 150	size_t bytes_copied = 0;
 151	size_t wqe_length;
 152	void *p;
 153	int ds;
 154
 155	wqe_index = wqe_index & qp->sq.fbc.sz_m1;
 156
 157	/* read the control segment first */
 158	p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
 159	ctrl = p;
 160	ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
 161	wqe_length = ds * MLX5_WQE_DS_UNITS;
 162
 163	/* read rest of WQE if it spreads over more than one stride */
 164	while (bytes_copied < wqe_length) {
 165		size_t copy_length =
 166			min_t(size_t, buflen - bytes_copied, MLX5_SEND_WQE_BB);
 167
 168		if (!copy_length)
 169			break;
 170
 171		memcpy(buffer + bytes_copied, p, copy_length);
 172		bytes_copied += copy_length;
 173
 174		wqe_index = (wqe_index + 1) & qp->sq.fbc.sz_m1;
 175		p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
 176	}
 177	*bc = bytes_copied;
 178	return 0;
 179}
 180
 181static int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
 182				    void *buffer, size_t buflen, size_t *bc)
 183{
 184	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 185	struct ib_umem *umem = base->ubuffer.umem;
 186	struct mlx5_ib_wq *wq = &qp->sq;
 187	struct mlx5_wqe_ctrl_seg *ctrl;
 188	size_t bytes_copied;
 189	size_t bytes_copied2;
 190	size_t wqe_length;
 191	int ret;
 192	int ds;
 193
 194	/* at first read as much as possible */
 195	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
 196					   wq->offset, wq->wqe_cnt,
 197					   wq->wqe_shift, buflen,
 198					   &bytes_copied);
 199	if (ret)
 200		return ret;
 201
 202	/* we need at least control segment size to proceed */
 203	if (bytes_copied < sizeof(*ctrl))
 204		return -EINVAL;
 205
 206	ctrl = buffer;
 207	ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
 208	wqe_length = ds * MLX5_WQE_DS_UNITS;
 209
 210	/* if we copied enough then we are done */
 211	if (bytes_copied >= wqe_length) {
 212		*bc = bytes_copied;
 213		return 0;
 214	}
 215
 216	/* otherwise this a wrapped around wqe
 217	 * so read the remaining bytes starting
 218	 * from  wqe_index 0
 219	 */
 220	ret = mlx5_ib_read_user_wqe_common(umem, buffer + bytes_copied,
 221					   buflen - bytes_copied, 0, wq->offset,
 222					   wq->wqe_cnt, wq->wqe_shift,
 223					   wqe_length - bytes_copied,
 224					   &bytes_copied2);
 225
 226	if (ret)
 227		return ret;
 228	*bc = bytes_copied + bytes_copied2;
 229	return 0;
 230}
 231
 232int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
 233			size_t buflen, size_t *bc)
 234{
 235	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 236	struct ib_umem *umem = base->ubuffer.umem;
 237
 238	if (buflen < sizeof(struct mlx5_wqe_ctrl_seg))
 239		return -EINVAL;
 240
 241	if (!umem)
 242		return mlx5_ib_read_kernel_wqe_sq(qp, wqe_index, buffer,
 243						  buflen, bc);
 244
 245	return mlx5_ib_read_user_wqe_sq(qp, wqe_index, buffer, buflen, bc);
 246}
 247
 248static int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index,
 249				    void *buffer, size_t buflen, size_t *bc)
 250{
 251	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 252	struct ib_umem *umem = base->ubuffer.umem;
 253	struct mlx5_ib_wq *wq = &qp->rq;
 254	size_t bytes_copied;
 255	int ret;
 256
 257	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
 258					   wq->offset, wq->wqe_cnt,
 259					   wq->wqe_shift, buflen,
 260					   &bytes_copied);
 261
 262	if (ret)
 263		return ret;
 264	*bc = bytes_copied;
 265	return 0;
 266}
 267
 268int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
 269			size_t buflen, size_t *bc)
 270{
 271	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 272	struct ib_umem *umem = base->ubuffer.umem;
 273	struct mlx5_ib_wq *wq = &qp->rq;
 274	size_t wqe_size = 1 << wq->wqe_shift;
 275
 276	if (buflen < wqe_size)
 277		return -EINVAL;
 278
 279	if (!umem)
 280		return -EOPNOTSUPP;
 281
 282	return mlx5_ib_read_user_wqe_rq(qp, wqe_index, buffer, buflen, bc);
 283}
 284
 285static int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
 286				     void *buffer, size_t buflen, size_t *bc)
 287{
 288	struct ib_umem *umem = srq->umem;
 289	size_t bytes_copied;
 290	int ret;
 291
 292	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index, 0,
 293					   srq->msrq.max, srq->msrq.wqe_shift,
 294					   buflen, &bytes_copied);
 295
 296	if (ret)
 297		return ret;
 298	*bc = bytes_copied;
 299	return 0;
 300}
 301
 302int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,
 303			 size_t buflen, size_t *bc)
 304{
 305	struct ib_umem *umem = srq->umem;
 306	size_t wqe_size = 1 << srq->msrq.wqe_shift;
 307
 308	if (buflen < wqe_size)
 309		return -EINVAL;
 310
 311	if (!umem)
 312		return -EOPNOTSUPP;
 313
 314	return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);
 315}
 316
 317static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp)
 318{
 319	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
 320	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
 321	struct mlx5_ib_qp *qp = to_mqp(ibqp);
 322	void *pas_ext_union, *err_syn;
 323	u32 *outb;
 324	int err;
 325
 326	if (!MLX5_CAP_GEN(dev->mdev, qpc_extension) ||
 327	    !MLX5_CAP_GEN(dev->mdev, qp_error_syndrome))
 328		return;
 329
 330	outb = kzalloc(outlen, GFP_KERNEL);
 331	if (!outb)
 332		return;
 333
 334	err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
 335				 true);
 336	if (err)
 337		goto out;
 338
 339	pas_ext_union =
 340		MLX5_ADDR_OF(query_qp_out, outb, qp_pas_or_qpc_ext_and_pas);
 341	err_syn = MLX5_ADDR_OF(qpc_extension_and_pas_list_in, pas_ext_union,
 342			       qpc_data_extension.error_syndrome);
 343
 344	pr_err("%s/%d: QP %d error: %s (0x%x 0x%x 0x%x)\n",
 345	       ibqp->device->name, ibqp->port, ibqp->qp_num,
 346	       ib_wc_status_msg(
 347		       MLX5_GET(cqe_error_syndrome, err_syn, syndrome)),
 348	       MLX5_GET(cqe_error_syndrome, err_syn, vendor_error_syndrome),
 349	       MLX5_GET(cqe_error_syndrome, err_syn, hw_syndrome_type),
 350	       MLX5_GET(cqe_error_syndrome, err_syn, hw_error_syndrome));
 351out:
 352	kfree(outb);
 353}
 354
 355static void mlx5_ib_handle_qp_event(struct work_struct *_work)
 356{
 357	struct mlx5_ib_qp_event_work *qpe_work =
 358		container_of(_work, struct mlx5_ib_qp_event_work, work);
 359	struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp;
 360	struct ib_event event = {};
 361
 362	event.device = ibqp->device;
 363	event.element.qp = ibqp;
 364	switch (qpe_work->type) {
 365	case MLX5_EVENT_TYPE_PATH_MIG:
 366		event.event = IB_EVENT_PATH_MIG;
 367		break;
 368	case MLX5_EVENT_TYPE_COMM_EST:
 369		event.event = IB_EVENT_COMM_EST;
 370		break;
 371	case MLX5_EVENT_TYPE_SQ_DRAINED:
 372		event.event = IB_EVENT_SQ_DRAINED;
 373		break;
 374	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 375		event.event = IB_EVENT_QP_LAST_WQE_REACHED;
 376		break;
 377	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 378		event.event = IB_EVENT_QP_FATAL;
 379		break;
 380	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 381		event.event = IB_EVENT_PATH_MIG_ERR;
 382		break;
 383	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 384		event.event = IB_EVENT_QP_REQ_ERR;
 385		break;
 386	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 387		event.event = IB_EVENT_QP_ACCESS_ERR;
 388		break;
 389	default:
 390		pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n",
 391			qpe_work->type, qpe_work->qp->qpn);
 392		goto out;
 393	}
 394
 395	if ((event.event == IB_EVENT_QP_FATAL) ||
 396	    (event.event == IB_EVENT_QP_ACCESS_ERR))
 397		mlx5_ib_qp_err_syndrome(ibqp);
 398
 399	ibqp->event_handler(&event, ibqp->qp_context);
 400
 401out:
 402	mlx5_core_res_put(&qpe_work->qp->common);
 403	kfree(qpe_work);
 404}
 405
 406static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
 407{
 408	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
 409	struct mlx5_ib_qp_event_work *qpe_work;
 410
 411	if (type == MLX5_EVENT_TYPE_PATH_MIG) {
 412		/* This event is only valid for trans_qps */
 413		to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
 414	}
 415
 416	if (!ibqp->event_handler)
 417		goto out_no_handler;
 418
 419	qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);
 420	if (!qpe_work)
 421		goto out_no_handler;
 422
 423	qpe_work->qp = qp;
 424	qpe_work->type = type;
 425	INIT_WORK(&qpe_work->work, mlx5_ib_handle_qp_event);
 426	queue_work(mlx5_ib_qp_event_wq, &qpe_work->work);
 427	return;
 428
 429out_no_handler:
 430	mlx5_core_res_put(&qp->common);
 431}
 432
 433static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
 434		       int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
 435{
 436	int wqe_size;
 437	int wq_size;
 438
 439	/* Sanity check RQ size before proceeding */
 440	if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)))
 441		return -EINVAL;
 442
 443	if (!has_rq) {
 444		qp->rq.max_gs = 0;
 445		qp->rq.wqe_cnt = 0;
 446		qp->rq.wqe_shift = 0;
 447		cap->max_recv_wr = 0;
 448		cap->max_recv_sge = 0;
 449	} else {
 450		int wq_sig = !!(qp->flags_en & MLX5_QP_FLAG_SIGNATURE);
 451
 452		if (ucmd) {
 453			qp->rq.wqe_cnt = ucmd->rq_wqe_count;
 454			if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))
 455				return -EINVAL;
 456			qp->rq.wqe_shift = ucmd->rq_wqe_shift;
 457			if ((1 << qp->rq.wqe_shift) /
 458				    sizeof(struct mlx5_wqe_data_seg) <
 459			    wq_sig)
 460				return -EINVAL;
 461			qp->rq.max_gs =
 462				(1 << qp->rq.wqe_shift) /
 463					sizeof(struct mlx5_wqe_data_seg) -
 464				wq_sig;
 465			qp->rq.max_post = qp->rq.wqe_cnt;
 466		} else {
 467			wqe_size =
 468				wq_sig ? sizeof(struct mlx5_wqe_signature_seg) :
 469					 0;
 470			wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
 471			wqe_size = roundup_pow_of_two(wqe_size);
 472			wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
 473			wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB);
 474			qp->rq.wqe_cnt = wq_size / wqe_size;
 475			if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) {
 476				mlx5_ib_dbg(dev, "wqe_size %d, max %d\n",
 477					    wqe_size,
 478					    MLX5_CAP_GEN(dev->mdev,
 479							 max_wqe_sz_rq));
 480				return -EINVAL;
 481			}
 482			qp->rq.wqe_shift = ilog2(wqe_size);
 483			qp->rq.max_gs =
 484				(1 << qp->rq.wqe_shift) /
 485					sizeof(struct mlx5_wqe_data_seg) -
 486				wq_sig;
 487			qp->rq.max_post = qp->rq.wqe_cnt;
 488		}
 489	}
 490
 491	return 0;
 492}
 493
 494static int sq_overhead(struct ib_qp_init_attr *attr)
 495{
 496	int size = 0;
 497
 498	switch (attr->qp_type) {
 499	case IB_QPT_XRC_INI:
 500		size += sizeof(struct mlx5_wqe_xrc_seg);
 501		fallthrough;
 502	case IB_QPT_RC:
 503		size += sizeof(struct mlx5_wqe_ctrl_seg) +
 504			max(sizeof(struct mlx5_wqe_atomic_seg) +
 505			    sizeof(struct mlx5_wqe_raddr_seg),
 506			    sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 507			    sizeof(struct mlx5_mkey_seg) +
 508			    MLX5_IB_SQ_UMR_INLINE_THRESHOLD /
 509			    MLX5_IB_UMR_OCTOWORD);
 510		break;
 511
 512	case IB_QPT_XRC_TGT:
 513		return 0;
 514
 515	case IB_QPT_UC:
 516		size += sizeof(struct mlx5_wqe_ctrl_seg) +
 517			max(sizeof(struct mlx5_wqe_raddr_seg),
 518			    sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 519			    sizeof(struct mlx5_mkey_seg));
 520		break;
 521
 522	case IB_QPT_UD:
 523		if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
 524			size += sizeof(struct mlx5_wqe_eth_pad) +
 525				sizeof(struct mlx5_wqe_eth_seg);
 526		fallthrough;
 527	case IB_QPT_SMI:
 528	case MLX5_IB_QPT_HW_GSI:
 529		size += sizeof(struct mlx5_wqe_ctrl_seg) +
 530			sizeof(struct mlx5_wqe_datagram_seg);
 531		break;
 532
 533	case MLX5_IB_QPT_REG_UMR:
 534		size += sizeof(struct mlx5_wqe_ctrl_seg) +
 535			sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 536			sizeof(struct mlx5_mkey_seg);
 537		break;
 538
 539	default:
 540		return -EINVAL;
 541	}
 542
 543	return size;
 544}
 545
 546static int calc_send_wqe(struct ib_qp_init_attr *attr)
 547{
 548	int inl_size = 0;
 549	int size;
 550
 551	size = sq_overhead(attr);
 552	if (size < 0)
 553		return size;
 554
 555	if (attr->cap.max_inline_data) {
 556		inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
 557			attr->cap.max_inline_data;
 558	}
 559
 560	size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
 561	if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN &&
 562	    ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
 563		return MLX5_SIG_WQE_SIZE;
 564	else
 565		return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
 566}
 567
 568static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
 569{
 570	int max_sge;
 571
 572	if (attr->qp_type == IB_QPT_RC)
 573		max_sge = (min_t(int, wqe_size, 512) -
 574			   sizeof(struct mlx5_wqe_ctrl_seg) -
 575			   sizeof(struct mlx5_wqe_raddr_seg)) /
 576			sizeof(struct mlx5_wqe_data_seg);
 577	else if (attr->qp_type == IB_QPT_XRC_INI)
 578		max_sge = (min_t(int, wqe_size, 512) -
 579			   sizeof(struct mlx5_wqe_ctrl_seg) -
 580			   sizeof(struct mlx5_wqe_xrc_seg) -
 581			   sizeof(struct mlx5_wqe_raddr_seg)) /
 582			sizeof(struct mlx5_wqe_data_seg);
 583	else
 584		max_sge = (wqe_size - sq_overhead(attr)) /
 585			sizeof(struct mlx5_wqe_data_seg);
 586
 587	return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
 588		     sizeof(struct mlx5_wqe_data_seg));
 589}
 590
 591static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
 592			struct mlx5_ib_qp *qp)
 593{
 594	int wqe_size;
 595	int wq_size;
 596
 597	if (!attr->cap.max_send_wr)
 598		return 0;
 599
 600	wqe_size = calc_send_wqe(attr);
 601	mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size);
 602	if (wqe_size < 0)
 603		return wqe_size;
 604
 605	if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
 606		mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
 607			    wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
 608		return -EINVAL;
 609	}
 610
 611	qp->max_inline_data = wqe_size - sq_overhead(attr) -
 612			      sizeof(struct mlx5_wqe_inline_seg);
 613	attr->cap.max_inline_data = qp->max_inline_data;
 614
 615	wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
 616	qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
 617	if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
 618		mlx5_ib_dbg(dev, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
 619			    attr->cap.max_send_wr, wqe_size, MLX5_SEND_WQE_BB,
 620			    qp->sq.wqe_cnt,
 621			    1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
 622		return -ENOMEM;
 623	}
 624	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
 625	qp->sq.max_gs = get_send_sge(attr, wqe_size);
 626	if (qp->sq.max_gs < attr->cap.max_send_sge)
 627		return -ENOMEM;
 628
 629	attr->cap.max_send_sge = qp->sq.max_gs;
 630	qp->sq.max_post = wq_size / wqe_size;
 631	attr->cap.max_send_wr = qp->sq.max_post;
 632
 633	return wq_size;
 634}
 635
 636static int set_user_buf_size(struct mlx5_ib_dev *dev,
 637			    struct mlx5_ib_qp *qp,
 638			    struct mlx5_ib_create_qp *ucmd,
 639			    struct mlx5_ib_qp_base *base,
 640			    struct ib_qp_init_attr *attr)
 641{
 642	int desc_sz = 1 << qp->sq.wqe_shift;
 643
 644	if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
 645		mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n",
 646			     desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
 647		return -EINVAL;
 648	}
 649
 650	if (ucmd->sq_wqe_count && !is_power_of_2(ucmd->sq_wqe_count)) {
 651		mlx5_ib_warn(dev, "sq_wqe_count %d is not a power of two\n",
 652			     ucmd->sq_wqe_count);
 653		return -EINVAL;
 654	}
 655
 656	qp->sq.wqe_cnt = ucmd->sq_wqe_count;
 657
 658	if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
 659		mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n",
 660			     qp->sq.wqe_cnt,
 661			     1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
 662		return -EINVAL;
 663	}
 664
 665	if (attr->qp_type == IB_QPT_RAW_PACKET ||
 666	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 667		base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 668		qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
 669	} else {
 670		base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
 671					 (qp->sq.wqe_cnt << 6);
 672	}
 673
 674	return 0;
 675}
 676
 677static int qp_has_rq(struct ib_qp_init_attr *attr)
 678{
 679	if (attr->qp_type == IB_QPT_XRC_INI ||
 680	    attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
 681	    attr->qp_type == MLX5_IB_QPT_REG_UMR ||
 682	    !attr->cap.max_recv_wr)
 683		return 0;
 684
 685	return 1;
 686}
 687
 688enum {
 689	/* this is the first blue flame register in the array of bfregs assigned
 690	 * to a processes. Since we do not use it for blue flame but rather
 691	 * regular 64 bit doorbells, we do not need a lock for maintaiing
 692	 * "odd/even" order
 693	 */
 694	NUM_NON_BLUE_FLAME_BFREGS = 1,
 695};
 696
 697static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
 698{
 699	return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
 700	       bfregi->num_static_sys_pages * MLX5_NON_FP_BFREGS_PER_UAR;
 701}
 702
 703static int num_med_bfreg(struct mlx5_ib_dev *dev,
 704			 struct mlx5_bfreg_info *bfregi)
 705{
 706	int n;
 707
 708	n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
 709	    NUM_NON_BLUE_FLAME_BFREGS;
 710
 711	return n >= 0 ? n : 0;
 712}
 713
 714static int first_med_bfreg(struct mlx5_ib_dev *dev,
 715			   struct mlx5_bfreg_info *bfregi)
 716{
 717	return num_med_bfreg(dev, bfregi) ? 1 : -ENOMEM;
 718}
 719
 720static int first_hi_bfreg(struct mlx5_ib_dev *dev,
 721			  struct mlx5_bfreg_info *bfregi)
 722{
 723	int med;
 724
 725	med = num_med_bfreg(dev, bfregi);
 726	return ++med;
 727}
 728
 729static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
 730				  struct mlx5_bfreg_info *bfregi)
 731{
 732	int i;
 733
 734	for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
 735		if (!bfregi->count[i]) {
 736			bfregi->count[i]++;
 737			return i;
 738		}
 739	}
 740
 741	return -ENOMEM;
 742}
 743
 744static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
 745				 struct mlx5_bfreg_info *bfregi)
 746{
 747	int minidx = first_med_bfreg(dev, bfregi);
 748	int i;
 749
 750	if (minidx < 0)
 751		return minidx;
 752
 753	for (i = minidx; i < first_hi_bfreg(dev, bfregi); i++) {
 754		if (bfregi->count[i] < bfregi->count[minidx])
 755			minidx = i;
 756		if (!bfregi->count[minidx])
 757			break;
 758	}
 759
 760	bfregi->count[minidx]++;
 761	return minidx;
 762}
 763
 764static int alloc_bfreg(struct mlx5_ib_dev *dev,
 765		       struct mlx5_bfreg_info *bfregi)
 766{
 767	int bfregn = -ENOMEM;
 768
 769	if (bfregi->lib_uar_dyn)
 770		return -EINVAL;
 771
 772	mutex_lock(&bfregi->lock);
 773	if (bfregi->ver >= 2) {
 774		bfregn = alloc_high_class_bfreg(dev, bfregi);
 775		if (bfregn < 0)
 776			bfregn = alloc_med_class_bfreg(dev, bfregi);
 777	}
 778
 779	if (bfregn < 0) {
 780		BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
 781		bfregn = 0;
 782		bfregi->count[bfregn]++;
 783	}
 784	mutex_unlock(&bfregi->lock);
 785
 786	return bfregn;
 787}
 788
 789void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
 790{
 791	mutex_lock(&bfregi->lock);
 792	bfregi->count[bfregn]--;
 793	mutex_unlock(&bfregi->lock);
 794}
 795
 796static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
 797{
 798	switch (state) {
 799	case IB_QPS_RESET:	return MLX5_QP_STATE_RST;
 800	case IB_QPS_INIT:	return MLX5_QP_STATE_INIT;
 801	case IB_QPS_RTR:	return MLX5_QP_STATE_RTR;
 802	case IB_QPS_RTS:	return MLX5_QP_STATE_RTS;
 803	case IB_QPS_SQD:	return MLX5_QP_STATE_SQD;
 804	case IB_QPS_SQE:	return MLX5_QP_STATE_SQER;
 805	case IB_QPS_ERR:	return MLX5_QP_STATE_ERR;
 806	default:		return -1;
 807	}
 808}
 809
 810static int to_mlx5_st(enum ib_qp_type type)
 811{
 812	switch (type) {
 813	case IB_QPT_RC:			return MLX5_QP_ST_RC;
 814	case IB_QPT_UC:			return MLX5_QP_ST_UC;
 815	case IB_QPT_UD:			return MLX5_QP_ST_UD;
 816	case MLX5_IB_QPT_REG_UMR:	return MLX5_QP_ST_REG_UMR;
 817	case IB_QPT_XRC_INI:
 818	case IB_QPT_XRC_TGT:		return MLX5_QP_ST_XRC;
 819	case IB_QPT_SMI:		return MLX5_QP_ST_QP0;
 820	case MLX5_IB_QPT_HW_GSI:	return MLX5_QP_ST_QP1;
 821	case MLX5_IB_QPT_DCI:		return MLX5_QP_ST_DCI;
 822	case IB_QPT_RAW_PACKET:		return MLX5_QP_ST_RAW_ETHERTYPE;
 823	default:		return -EINVAL;
 824	}
 825}
 826
 827static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
 828			     struct mlx5_ib_cq *recv_cq);
 829static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
 830			       struct mlx5_ib_cq *recv_cq);
 831
 832int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
 833			struct mlx5_bfreg_info *bfregi, u32 bfregn,
 834			bool dyn_bfreg)
 835{
 836	unsigned int bfregs_per_sys_page;
 837	u32 index_of_sys_page;
 838	u32 offset;
 839
 840	if (bfregi->lib_uar_dyn)
 841		return -EINVAL;
 842
 843	bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
 844				MLX5_NON_FP_BFREGS_PER_UAR;
 845	index_of_sys_page = bfregn / bfregs_per_sys_page;
 846
 847	if (dyn_bfreg) {
 848		index_of_sys_page += bfregi->num_static_sys_pages;
 849
 850		if (index_of_sys_page >= bfregi->num_sys_pages)
 851			return -EINVAL;
 852
 853		if (bfregn > bfregi->num_dyn_bfregs ||
 854		    bfregi->sys_pages[index_of_sys_page] == MLX5_IB_INVALID_UAR_INDEX) {
 855			mlx5_ib_dbg(dev, "Invalid dynamic uar index\n");
 856			return -EINVAL;
 857		}
 858	}
 859
 860	offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
 861	return bfregi->sys_pages[index_of_sys_page] + offset;
 862}
 863
 864static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 865			    struct mlx5_ib_rwq *rwq, struct ib_udata *udata)
 866{
 867	struct mlx5_ib_ucontext *context =
 868		rdma_udata_to_drv_context(
 869			udata,
 870			struct mlx5_ib_ucontext,
 871			ibucontext);
 872
 873	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
 874		atomic_dec(&dev->delay_drop.rqs_cnt);
 875
 876	mlx5_ib_db_unmap_user(context, &rwq->db);
 877	ib_umem_release(rwq->umem);
 878}
 879
 880static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 881			  struct ib_udata *udata, struct mlx5_ib_rwq *rwq,
 882			  struct mlx5_ib_create_wq *ucmd)
 883{
 884	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
 885		udata, struct mlx5_ib_ucontext, ibucontext);
 886	unsigned long page_size = 0;
 887	u32 offset = 0;
 888	int err;
 889
 890	if (!ucmd->buf_addr)
 891		return -EINVAL;
 892
 893	rwq->umem = ib_umem_get(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0);
 894	if (IS_ERR(rwq->umem)) {
 895		mlx5_ib_dbg(dev, "umem_get failed\n");
 896		err = PTR_ERR(rwq->umem);
 897		return err;
 898	}
 899
 900	page_size = mlx5_umem_find_best_quantized_pgoff(
 901		rwq->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
 902		page_offset, 64, &rwq->rq_page_offset);
 903	if (!page_size) {
 904		mlx5_ib_warn(dev, "bad offset\n");
 905		err = -EINVAL;
 906		goto err_umem;
 907	}
 908
 909	rwq->rq_num_pas = ib_umem_num_dma_blocks(rwq->umem, page_size);
 910	rwq->page_shift = order_base_2(page_size);
 911	rwq->log_page_size =  rwq->page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 912	rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
 913
 914	mlx5_ib_dbg(
 915		dev,
 916		"addr 0x%llx, size %zd, npages %zu, page_size %ld, ncont %d, offset %d\n",
 917		(unsigned long long)ucmd->buf_addr, rwq->buf_size,
 918		ib_umem_num_pages(rwq->umem), page_size, rwq->rq_num_pas,
 919		offset);
 920
 921	err = mlx5_ib_db_map_user(ucontext, ucmd->db_addr, &rwq->db);
 922	if (err) {
 923		mlx5_ib_dbg(dev, "map failed\n");
 924		goto err_umem;
 925	}
 926
 927	return 0;
 928
 929err_umem:
 930	ib_umem_release(rwq->umem);
 931	return err;
 932}
 933
 934static int adjust_bfregn(struct mlx5_ib_dev *dev,
 935			 struct mlx5_bfreg_info *bfregi, int bfregn)
 936{
 937	return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
 938				bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
 939}
 940
 941static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 942			   struct mlx5_ib_qp *qp, struct ib_udata *udata,
 943			   struct ib_qp_init_attr *attr, u32 **in,
 944			   struct mlx5_ib_create_qp_resp *resp, int *inlen,
 945			   struct mlx5_ib_qp_base *base,
 946			   struct mlx5_ib_create_qp *ucmd)
 947{
 948	struct mlx5_ib_ucontext *context;
 949	struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
 950	unsigned int page_offset_quantized = 0;
 951	unsigned long page_size = 0;
 952	int uar_index = 0;
 953	int bfregn;
 954	int ncont = 0;
 955	__be64 *pas;
 956	void *qpc;
 957	int err;
 958	u16 uid;
 959	u32 uar_flags;
 960
 961	context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,
 962					    ibucontext);
 963	uar_flags = qp->flags_en &
 964		    (MLX5_QP_FLAG_UAR_PAGE_INDEX | MLX5_QP_FLAG_BFREG_INDEX);
 965	switch (uar_flags) {
 966	case MLX5_QP_FLAG_UAR_PAGE_INDEX:
 967		uar_index = ucmd->bfreg_index;
 968		bfregn = MLX5_IB_INVALID_BFREG;
 969		break;
 970	case MLX5_QP_FLAG_BFREG_INDEX:
 971		uar_index = bfregn_to_uar_index(dev, &context->bfregi,
 972						ucmd->bfreg_index, true);
 973		if (uar_index < 0)
 974			return uar_index;
 975		bfregn = MLX5_IB_INVALID_BFREG;
 976		break;
 977	case 0:
 978		if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
 979			return -EINVAL;
 980		bfregn = alloc_bfreg(dev, &context->bfregi);
 981		if (bfregn < 0)
 982			return bfregn;
 983		break;
 984	default:
 985		return -EINVAL;
 986	}
 987
 988	mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
 989	if (bfregn != MLX5_IB_INVALID_BFREG)
 990		uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn,
 991						false);
 992
 993	qp->rq.offset = 0;
 994	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
 995	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 996
 997	err = set_user_buf_size(dev, qp, ucmd, base, attr);
 998	if (err)
 999		goto err_bfreg;
1000
1001	if (ucmd->buf_addr && ubuffer->buf_size) {
1002		ubuffer->buf_addr = ucmd->buf_addr;
1003		ubuffer->umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr,
1004					    ubuffer->buf_size, 0);
1005		if (IS_ERR(ubuffer->umem)) {
1006			err = PTR_ERR(ubuffer->umem);
1007			goto err_bfreg;
1008		}
1009		page_size = mlx5_umem_find_best_quantized_pgoff(
1010			ubuffer->umem, qpc, log_page_size,
1011			MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64,
1012			&page_offset_quantized);
1013		if (!page_size) {
1014			err = -EINVAL;
1015			goto err_umem;
1016		}
1017		ncont = ib_umem_num_dma_blocks(ubuffer->umem, page_size);
1018	} else {
1019		ubuffer->umem = NULL;
1020	}
1021
1022	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1023		 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
1024	*in = kvzalloc(*inlen, GFP_KERNEL);
1025	if (!*in) {
1026		err = -ENOMEM;
1027		goto err_umem;
1028	}
1029
1030	uid = (attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
1031	MLX5_SET(create_qp_in, *in, uid, uid);
1032	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
1033	pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
1034	if (ubuffer->umem) {
1035		mlx5_ib_populate_pas(ubuffer->umem, page_size, pas, 0);
1036		MLX5_SET(qpc, qpc, log_page_size,
1037			 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1038		MLX5_SET(qpc, qpc, page_offset, page_offset_quantized);
1039	}
1040	MLX5_SET(qpc, qpc, uar_page, uar_index);
1041	if (bfregn != MLX5_IB_INVALID_BFREG)
1042		resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
1043	else
1044		resp->bfreg_index = MLX5_IB_INVALID_BFREG;
1045	qp->bfregn = bfregn;
1046
1047	err = mlx5_ib_db_map_user(context, ucmd->db_addr, &qp->db);
1048	if (err) {
1049		mlx5_ib_dbg(dev, "map failed\n");
1050		goto err_free;
1051	}
1052
1053	return 0;
1054
1055err_free:
1056	kvfree(*in);
1057
1058err_umem:
1059	ib_umem_release(ubuffer->umem);
1060
1061err_bfreg:
1062	if (bfregn != MLX5_IB_INVALID_BFREG)
1063		mlx5_ib_free_bfreg(dev, &context->bfregi, bfregn);
1064	return err;
1065}
1066
1067static void destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1068		       struct mlx5_ib_qp_base *base, struct ib_udata *udata)
1069{
1070	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
1071		udata, struct mlx5_ib_ucontext, ibucontext);
1072
1073	if (udata) {
1074		/* User QP */
1075		mlx5_ib_db_unmap_user(context, &qp->db);
1076		ib_umem_release(base->ubuffer.umem);
1077
1078		/*
1079		 * Free only the BFREGs which are handled by the kernel.
1080		 * BFREGs of UARs allocated dynamically are handled by user.
1081		 */
1082		if (qp->bfregn != MLX5_IB_INVALID_BFREG)
1083			mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
1084		return;
1085	}
1086
1087	/* Kernel QP */
1088	kvfree(qp->sq.wqe_head);
1089	kvfree(qp->sq.w_list);
1090	kvfree(qp->sq.wrid);
1091	kvfree(qp->sq.wr_data);
1092	kvfree(qp->rq.wrid);
1093	if (qp->db.db)
1094		mlx5_db_free(dev->mdev, &qp->db);
1095	if (qp->buf.frags)
1096		mlx5_frag_buf_free(dev->mdev, &qp->buf);
1097}
1098
1099static int _create_kernel_qp(struct mlx5_ib_dev *dev,
1100			     struct ib_qp_init_attr *init_attr,
1101			     struct mlx5_ib_qp *qp, u32 **in, int *inlen,
1102			     struct mlx5_ib_qp_base *base)
1103{
1104	int uar_index;
1105	void *qpc;
1106	int err;
1107
1108	if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
1109		qp->bf.bfreg = &dev->fp_bfreg;
1110	else
1111		qp->bf.bfreg = &dev->bfreg;
1112
1113	/* We need to divide by two since each register is comprised of
1114	 * two buffers of identical size, namely odd and even
1115	 */
1116	qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2;
1117	uar_index = qp->bf.bfreg->index;
1118
1119	err = calc_sq_size(dev, init_attr, qp);
1120	if (err < 0) {
1121		mlx5_ib_dbg(dev, "err %d\n", err);
1122		return err;
1123	}
1124
1125	qp->rq.offset = 0;
1126	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
1127	base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
1128
1129	err = mlx5_frag_buf_alloc_node(dev->mdev, base->ubuffer.buf_size,
1130				       &qp->buf, dev->mdev->priv.numa_node);
1131	if (err) {
1132		mlx5_ib_dbg(dev, "err %d\n", err);
1133		return err;
1134	}
1135
1136	if (qp->rq.wqe_cnt)
1137		mlx5_init_fbc(qp->buf.frags, qp->rq.wqe_shift,
1138			      ilog2(qp->rq.wqe_cnt), &qp->rq.fbc);
1139
1140	if (qp->sq.wqe_cnt) {
1141		int sq_strides_offset = (qp->sq.offset  & (PAGE_SIZE - 1)) /
1142					MLX5_SEND_WQE_BB;
1143		mlx5_init_fbc_offset(qp->buf.frags +
1144				     (qp->sq.offset / PAGE_SIZE),
1145				     ilog2(MLX5_SEND_WQE_BB),
1146				     ilog2(qp->sq.wqe_cnt),
1147				     sq_strides_offset, &qp->sq.fbc);
1148
1149		qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
1150	}
1151
1152	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1153		 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
1154	*in = kvzalloc(*inlen, GFP_KERNEL);
1155	if (!*in) {
1156		err = -ENOMEM;
1157		goto err_buf;
1158	}
1159
1160	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
1161	MLX5_SET(qpc, qpc, uar_page, uar_index);
1162	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));
1163	MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1164
1165	/* Set "fast registration enabled" for all kernel QPs */
1166	MLX5_SET(qpc, qpc, fre, 1);
1167	MLX5_SET(qpc, qpc, rlky, 1);
1168
1169	if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
1170		MLX5_SET(qpc, qpc, deth_sqpn, 1);
1171
1172	mlx5_fill_page_frag_array(&qp->buf,
1173				  (__be64 *)MLX5_ADDR_OF(create_qp_in,
1174							 *in, pas));
1175
1176	err = mlx5_db_alloc(dev->mdev, &qp->db);
1177	if (err) {
1178		mlx5_ib_dbg(dev, "err %d\n", err);
1179		goto err_free;
1180	}
1181
1182	qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1183				     sizeof(*qp->sq.wrid), GFP_KERNEL);
1184	qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
1185					sizeof(*qp->sq.wr_data), GFP_KERNEL);
1186	qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1187				     sizeof(*qp->rq.wrid), GFP_KERNEL);
1188	qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
1189				       sizeof(*qp->sq.w_list), GFP_KERNEL);
1190	qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
1191					 sizeof(*qp->sq.wqe_head), GFP_KERNEL);
1192
1193	if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
1194	    !qp->sq.w_list || !qp->sq.wqe_head) {
1195		err = -ENOMEM;
1196		goto err_wrid;
1197	}
1198
1199	return 0;
1200
1201err_wrid:
1202	kvfree(qp->sq.wqe_head);
1203	kvfree(qp->sq.w_list);
1204	kvfree(qp->sq.wrid);
1205	kvfree(qp->sq.wr_data);
1206	kvfree(qp->rq.wrid);
1207	mlx5_db_free(dev->mdev, &qp->db);
1208
1209err_free:
1210	kvfree(*in);
1211
1212err_buf:
1213	mlx5_frag_buf_free(dev->mdev, &qp->buf);
1214	return err;
1215}
1216
1217static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
1218{
1219	if (attr->srq || (qp->type == IB_QPT_XRC_TGT) ||
1220	    (qp->type == MLX5_IB_QPT_DCI) || (qp->type == IB_QPT_XRC_INI))
1221		return MLX5_SRQ_RQ;
1222	else if (!qp->has_rq)
1223		return MLX5_ZERO_LEN_RQ;
1224
1225	return MLX5_NON_ZERO_RQ;
1226}
1227
1228static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1229				    struct mlx5_ib_qp *qp,
1230				    struct mlx5_ib_sq *sq, u32 tdn,
1231				    struct ib_pd *pd)
1232{
1233	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
1234	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1235
1236	MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
1237	MLX5_SET(tisc, tisc, transport_domain, tdn);
1238	if (!mlx5_ib_lag_should_assign_affinity(dev) &&
1239	    mlx5_lag_is_lacp_owner(dev->mdev))
1240		MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
1241	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
1242		MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
1243
1244	return mlx5_core_create_tis(dev->mdev, in, &sq->tisn);
1245}
1246
1247static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1248				      struct mlx5_ib_sq *sq, struct ib_pd *pd)
1249{
1250	mlx5_cmd_destroy_tis(dev->mdev, sq->tisn, to_mpd(pd)->uid);
1251}
1252
1253static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)
1254{
1255	if (sq->flow_rule)
1256		mlx5_del_flow_rules(sq->flow_rule);
1257	sq->flow_rule = NULL;
1258}
1259
1260static bool fr_supported(int ts_cap)
1261{
1262	return ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING ||
1263	       ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME;
1264}
1265
1266static int get_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1267			 bool fr_sup, bool rt_sup)
1268{
1269	if (cq->private_flags & MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS) {
1270		if (!rt_sup) {
1271			mlx5_ib_dbg(dev,
1272				    "Real time TS format is not supported\n");
1273			return -EOPNOTSUPP;
1274		}
1275		return MLX5_TIMESTAMP_FORMAT_REAL_TIME;
1276	}
1277	if (cq->create_flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
1278		if (!fr_sup) {
1279			mlx5_ib_dbg(dev,
1280				    "Free running TS format is not supported\n");
1281			return -EOPNOTSUPP;
1282		}
1283		return MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
1284	}
1285	return fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :
1286			MLX5_TIMESTAMP_FORMAT_DEFAULT;
1287}
1288
1289static int get_rq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *recv_cq)
1290{
1291	u8 ts_cap = MLX5_CAP_GEN(dev->mdev, rq_ts_format);
1292
1293	return get_ts_format(dev, recv_cq, fr_supported(ts_cap),
1294			     rt_supported(ts_cap));
1295}
1296
1297static int get_sq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq)
1298{
1299	u8 ts_cap = MLX5_CAP_GEN(dev->mdev, sq_ts_format);
1300
1301	return get_ts_format(dev, send_cq, fr_supported(ts_cap),
1302			     rt_supported(ts_cap));
1303}
1304
1305static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq,
1306			    struct mlx5_ib_cq *recv_cq)
1307{
1308	u8 ts_cap = MLX5_CAP_ROCE(dev->mdev, qp_ts_format);
1309	bool fr_sup = fr_supported(ts_cap);
1310	bool rt_sup = rt_supported(ts_cap);
1311	u8 default_ts = fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :
1312				 MLX5_TIMESTAMP_FORMAT_DEFAULT;
1313	int send_ts_format =
1314		send_cq ? get_ts_format(dev, send_cq, fr_sup, rt_sup) :
1315			  default_ts;
1316	int recv_ts_format =
1317		recv_cq ? get_ts_format(dev, recv_cq, fr_sup, rt_sup) :
1318			  default_ts;
1319
1320	if (send_ts_format < 0 || recv_ts_format < 0)
1321		return -EOPNOTSUPP;
1322
1323	if (send_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &&
1324	    recv_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &&
1325	    send_ts_format != recv_ts_format) {
1326		mlx5_ib_dbg(
1327			dev,
1328			"The send ts_format does not match the receive ts_format\n");
1329		return -EOPNOTSUPP;
1330	}
1331
1332	return send_ts_format == default_ts ? recv_ts_format : send_ts_format;
1333}
1334
1335static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1336				   struct ib_udata *udata,
1337				   struct mlx5_ib_sq *sq, void *qpin,
1338				   struct ib_pd *pd, struct mlx5_ib_cq *cq)
1339{
1340	struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
1341	__be64 *pas;
1342	void *in;
1343	void *sqc;
1344	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1345	void *wq;
1346	int inlen;
1347	int err;
1348	unsigned int page_offset_quantized;
1349	unsigned long page_size;
1350	int ts_format;
1351
1352	ts_format = get_sq_ts_format(dev, cq);
1353	if (ts_format < 0)
1354		return ts_format;
1355
1356	sq->ubuffer.umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr,
1357				       ubuffer->buf_size, 0);
1358	if (IS_ERR(sq->ubuffer.umem))
1359		return PTR_ERR(sq->ubuffer.umem);
1360	page_size = mlx5_umem_find_best_quantized_pgoff(
1361		ubuffer->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
1362		page_offset, 64, &page_offset_quantized);
1363	if (!page_size) {
1364		err = -EINVAL;
1365		goto err_umem;
1366	}
1367
1368	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1369		sizeof(u64) *
1370			ib_umem_num_dma_blocks(sq->ubuffer.umem, page_size);
1371	in = kvzalloc(inlen, GFP_KERNEL);
1372	if (!in) {
1373		err = -ENOMEM;
1374		goto err_umem;
1375	}
1376
1377	MLX5_SET(create_sq_in, in, uid, to_mpd(pd)->uid);
1378	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1379	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1380	if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
1381		MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);
1382	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1383	MLX5_SET(sqc, sqc, ts_format, ts_format);
1384	MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
1385	MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
1386	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1387	MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
1388	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1389	    MLX5_CAP_ETH(dev->mdev, swp))
1390		MLX5_SET(sqc, sqc, allow_swp, 1);
1391
1392	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1393	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1394	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1395	MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
1396	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1397	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1398	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
1399	MLX5_SET(wq, wq, log_wq_pg_sz,
1400		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1401	MLX5_SET(wq, wq, page_offset, page_offset_quantized);
1402
1403	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1404	mlx5_ib_populate_pas(sq->ubuffer.umem, page_size, pas, 0);
1405
1406	err = mlx5_core_create_sq_tracked(dev, in, inlen, &sq->base.mqp);
1407
1408	kvfree(in);
1409
1410	if (err)
1411		goto err_umem;
1412
1413	return 0;
1414
1415err_umem:
1416	ib_umem_release(sq->ubuffer.umem);
1417	sq->ubuffer.umem = NULL;
1418
1419	return err;
1420}
1421
1422static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1423				     struct mlx5_ib_sq *sq)
1424{
1425	destroy_flow_rule_vport_sq(sq);
1426	mlx5_core_destroy_sq_tracked(dev, &sq->base.mqp);
1427	ib_umem_release(sq->ubuffer.umem);
1428}
1429
1430static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1431				   struct mlx5_ib_rq *rq, void *qpin,
1432				   struct ib_pd *pd, struct mlx5_ib_cq *cq)
1433{
1434	struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
1435	__be64 *pas;
1436	void *in;
1437	void *rqc;
1438	void *wq;
1439	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1440	struct ib_umem *umem = rq->base.ubuffer.umem;
1441	unsigned int page_offset_quantized;
1442	unsigned long page_size = 0;
1443	int ts_format;
1444	size_t inlen;
1445	int err;
1446
1447	ts_format = get_rq_ts_format(dev, cq);
1448	if (ts_format < 0)
1449		return ts_format;
1450
1451	page_size = mlx5_umem_find_best_quantized_pgoff(umem, wq, log_wq_pg_sz,
1452							MLX5_ADAPTER_PAGE_SHIFT,
1453							page_offset, 64,
1454							&page_offset_quantized);
1455	if (!page_size)
1456		return -EINVAL;
1457
1458	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1459		sizeof(u64) * ib_umem_num_dma_blocks(umem, page_size);
1460	in = kvzalloc(inlen, GFP_KERNEL);
1461	if (!in)
1462		return -ENOMEM;
1463
1464	MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
1465	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1466	if (!(rq->flags & MLX5_IB_RQ_CVLAN_STRIPPING))
1467		MLX5_SET(rqc, rqc, vsd, 1);
1468	MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
1469	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1470	MLX5_SET(rqc, rqc, ts_format, ts_format);
1471	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1472	MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1473	MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1474
1475	if (mqp->flags & IB_QP_CREATE_SCATTER_FCS)
1476		MLX5_SET(rqc, rqc, scatter_fcs, 1);
1477
1478	wq = MLX5_ADDR_OF(rqc, rqc, wq);
1479	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1480	if (rq->flags & MLX5_IB_RQ_PCI_WRITE_END_PADDING)
1481		MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1482	MLX5_SET(wq, wq, page_offset, page_offset_quantized);
1483	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1484	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1485	MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
1486	MLX5_SET(wq, wq, log_wq_pg_sz,
1487		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1488	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
1489
1490	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1491	mlx5_ib_populate_pas(umem, page_size, pas, 0);
1492
1493	err = mlx5_core_create_rq_tracked(dev, in, inlen, &rq->base.mqp);
1494
1495	kvfree(in);
1496
1497	return err;
1498}
1499
1500static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1501				     struct mlx5_ib_rq *rq)
1502{
1503	mlx5_core_destroy_rq_tracked(dev, &rq->base.mqp);
1504}
1505
1506static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1507				      struct mlx5_ib_rq *rq,
1508				      u32 qp_flags_en,
1509				      struct ib_pd *pd)
1510{
1511	if (qp_flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1512			   MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1513		mlx5_ib_disable_lb(dev, false, true);
1514	mlx5_cmd_destroy_tir(dev->mdev, rq->tirn, to_mpd(pd)->uid);
1515}
1516
1517static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1518				    struct mlx5_ib_rq *rq, u32 tdn,
1519				    u32 *qp_flags_en, struct ib_pd *pd,
1520				    u32 *out)
1521{
1522	u8 lb_flag = 0;
1523	u32 *in;
1524	void *tirc;
1525	int inlen;
1526	int err;
1527
1528	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1529	in = kvzalloc(inlen, GFP_KERNEL);
1530	if (!in)
1531		return -ENOMEM;
1532
1533	MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
1534	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1535	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1536	MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1537	MLX5_SET(tirc, tirc, transport_domain, tdn);
1538	if (*qp_flags_en & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1539		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1540
1541	if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1542		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1543
1544	if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
1545		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1546
1547	if (dev->is_rep) {
1548		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1549		*qp_flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
1550	}
1551
1552	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
1553	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1554	err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1555	rq->tirn = MLX5_GET(create_tir_out, out, tirn);
1556	if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1557		err = mlx5_ib_enable_lb(dev, false, true);
1558
1559		if (err)
1560			destroy_raw_packet_qp_tir(dev, rq, 0, pd);
1561	}
1562	kvfree(in);
1563
1564	return err;
1565}
1566
1567static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1568				u32 *in, size_t inlen, struct ib_pd *pd,
1569				struct ib_udata *udata,
1570				struct mlx5_ib_create_qp_resp *resp,
1571				struct ib_qp_init_attr *init_attr)
1572{
1573	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1574	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1575	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1576	struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1577		udata, struct mlx5_ib_ucontext, ibucontext);
1578	int err;
1579	u32 tdn = mucontext->tdn;
1580	u16 uid = to_mpd(pd)->uid;
1581	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
1582
1583	if (!qp->sq.wqe_cnt && !qp->rq.wqe_cnt)
1584		return -EINVAL;
1585	if (qp->sq.wqe_cnt) {
1586		err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);
1587		if (err)
1588			return err;
1589
1590		err = create_raw_packet_qp_sq(dev, udata, sq, in, pd,
1591					      to_mcq(init_attr->send_cq));
1592		if (err)
1593			goto err_destroy_tis;
1594
1595		if (uid) {
1596			resp->tisn = sq->tisn;
1597			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TISN;
1598			resp->sqn = sq->base.mqp.qpn;
1599			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
1600		}
1601
1602		sq->base.container_mibqp = qp;
1603		sq->base.mqp.event = mlx5_ib_qp_event;
1604	}
1605
1606	if (qp->rq.wqe_cnt) {
1607		rq->base.container_mibqp = qp;
1608
1609		if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
1610			rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
1611		if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING)
1612			rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
1613		err = create_raw_packet_qp_rq(dev, rq, in, pd,
1614					      to_mcq(init_attr->recv_cq));
1615		if (err)
1616			goto err_destroy_sq;
1617
1618		err = create_raw_packet_qp_tir(dev, rq, tdn, &qp->flags_en, pd,
1619					       out);
1620		if (err)
1621			goto err_destroy_rq;
1622
1623		if (uid) {
1624			resp->rqn = rq->base.mqp.qpn;
1625			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_RQN;
1626			resp->tirn = rq->tirn;
1627			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1628			if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
1629			    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
1630				resp->tir_icm_addr = MLX5_GET(
1631					create_tir_out, out, icm_address_31_0);
1632				resp->tir_icm_addr |=
1633					(u64)MLX5_GET(create_tir_out, out,
1634						      icm_address_39_32)
1635					<< 32;
1636				resp->tir_icm_addr |=
1637					(u64)MLX5_GET(create_tir_out, out,
1638						      icm_address_63_40)
1639					<< 40;
1640				resp->comp_mask |=
1641					MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1642			}
1643		}
1644	}
1645
1646	qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1647						     rq->base.mqp.qpn;
1648	return 0;
1649
1650err_destroy_rq:
1651	destroy_raw_packet_qp_rq(dev, rq);
1652err_destroy_sq:
1653	if (!qp->sq.wqe_cnt)
1654		return err;
1655	destroy_raw_packet_qp_sq(dev, sq);
1656err_destroy_tis:
1657	destroy_raw_packet_qp_tis(dev, sq, pd);
1658
1659	return err;
1660}
1661
1662static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
1663				  struct mlx5_ib_qp *qp)
1664{
1665	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1666	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1667	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1668
1669	if (qp->rq.wqe_cnt) {
1670		destroy_raw_packet_qp_tir(dev, rq, qp->flags_en, qp->ibqp.pd);
1671		destroy_raw_packet_qp_rq(dev, rq);
1672	}
1673
1674	if (qp->sq.wqe_cnt) {
1675		destroy_raw_packet_qp_sq(dev, sq);
1676		destroy_raw_packet_qp_tis(dev, sq, qp->ibqp.pd);
1677	}
1678}
1679
1680static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
1681				    struct mlx5_ib_raw_packet_qp *raw_packet_qp)
1682{
1683	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1684	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1685
1686	sq->sq = &qp->sq;
1687	rq->rq = &qp->rq;
1688	sq->doorbell = &qp->db;
1689	rq->doorbell = &qp->db;
1690}
1691
1692static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1693{
1694	if (qp->flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1695			    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1696		mlx5_ib_disable_lb(dev, false, true);
1697	mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
1698			     to_mpd(qp->ibqp.pd)->uid);
1699}
1700
1701struct mlx5_create_qp_params {
1702	struct ib_udata *udata;
1703	size_t inlen;
1704	size_t outlen;
1705	size_t ucmd_size;
1706	void *ucmd;
1707	u8 is_rss_raw : 1;
1708	struct ib_qp_init_attr *attr;
1709	u32 uidx;
1710	struct mlx5_ib_create_qp_resp resp;
1711};
1712
1713static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1714				 struct mlx5_ib_qp *qp,
1715				 struct mlx5_create_qp_params *params)
1716{
1717	struct ib_qp_init_attr *init_attr = params->attr;
1718	struct mlx5_ib_create_qp_rss *ucmd = params->ucmd;
1719	struct ib_udata *udata = params->udata;
1720	struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1721		udata, struct mlx5_ib_ucontext, ibucontext);
1722	int inlen;
1723	int outlen;
1724	int err;
1725	u32 *in;
1726	u32 *out;
1727	void *tirc;
1728	void *hfso;
1729	u32 selected_fields = 0;
1730	u32 outer_l4;
1731	u32 tdn = mucontext->tdn;
1732	u8 lb_flag = 0;
1733
1734	if (ucmd->comp_mask) {
1735		mlx5_ib_dbg(dev, "invalid comp mask\n");
1736		return -EOPNOTSUPP;
1737	}
1738
1739	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
1740	    !(ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
1741		mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
1742		return -EOPNOTSUPP;
1743	}
1744
1745	if (dev->is_rep)
1746		qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
1747
1748	if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1749		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1750
1751	if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
1752		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1753
1754	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1755	outlen = MLX5_ST_SZ_BYTES(create_tir_out);
1756	in = kvzalloc(inlen + outlen, GFP_KERNEL);
1757	if (!in)
1758		return -ENOMEM;
1759
1760	out = in + MLX5_ST_SZ_DW(create_tir_in);
1761	MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
1762	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1763	MLX5_SET(tirc, tirc, disp_type,
1764		 MLX5_TIRC_DISP_TYPE_INDIRECT);
1765	MLX5_SET(tirc, tirc, indirect_table,
1766		 init_attr->rwq_ind_tbl->ind_tbl_num);
1767	MLX5_SET(tirc, tirc, transport_domain, tdn);
1768
1769	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1770
1771	if (ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1772		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1773
1774	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
1775
1776	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER)
1777		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
1778	else
1779		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1780
1781	switch (ucmd->rx_hash_function) {
1782	case MLX5_RX_HASH_FUNC_TOEPLITZ:
1783	{
1784		void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1785		size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1786
1787		if (len != ucmd->rx_key_len) {
1788			err = -EINVAL;
1789			goto err;
1790		}
1791
1792		MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1793		memcpy(rss_key, ucmd->rx_hash_key, len);
1794		break;
1795	}
1796	default:
1797		err = -EOPNOTSUPP;
1798		goto err;
1799	}
1800
1801	if (!ucmd->rx_hash_fields_mask) {
1802		/* special case when this TIR serves as steering entry without hashing */
1803		if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1804			goto create_tir;
1805		err = -EINVAL;
1806		goto err;
1807	}
1808
1809	if (((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1810	     (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1811	     ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1812	     (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
1813		err = -EINVAL;
1814		goto err;
1815	}
1816
1817	/* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
1818	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1819	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
1820		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1821			 MLX5_L3_PROT_TYPE_IPV4);
1822	else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1823		 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1824		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1825			 MLX5_L3_PROT_TYPE_IPV6);
1826
1827	outer_l4 = ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1828		    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1829			   << 0 |
1830		   ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1831		    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1832			   << 1 |
1833		   (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
1834
1835	/* Check that only one l4 protocol is set */
1836	if (outer_l4 & (outer_l4 - 1)) {
1837		err = -EINVAL;
1838		goto err;
1839	}
1840
1841	/* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
1842	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1843	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1844		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1845			 MLX5_L4_PROT_TYPE_TCP);
1846	else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1847		 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1848		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1849			 MLX5_L4_PROT_TYPE_UDP);
1850
1851	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1852	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
1853		selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
1854
1855	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
1856	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1857		selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
1858
1859	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1860	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
1861		selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
1862
1863	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
1864	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1865		selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
1866
1867	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
1868		selected_fields |= MLX5_HASH_FIELD_SEL_IPSEC_SPI;
1869
1870	MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1871
1872create_tir:
1873	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1874	err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1875
1876	qp->rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn);
1877	if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1878		err = mlx5_ib_enable_lb(dev, false, true);
1879
1880		if (err)
1881			mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
1882					     to_mpd(pd)->uid);
1883	}
1884
1885	if (err)
1886		goto err;
1887
1888	if (mucontext->devx_uid) {
1889		params->resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1890		params->resp.tirn = qp->rss_qp.tirn;
1891		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
1892		    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
1893			params->resp.tir_icm_addr =
1894				MLX5_GET(create_tir_out, out, icm_address_31_0);
1895			params->resp.tir_icm_addr |=
1896				(u64)MLX5_GET(create_tir_out, out,
1897					      icm_address_39_32)
1898				<< 32;
1899			params->resp.tir_icm_addr |=
1900				(u64)MLX5_GET(create_tir_out, out,
1901					      icm_address_63_40)
1902				<< 40;
1903			params->resp.comp_mask |=
1904				MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1905		}
1906	}
1907
1908	kvfree(in);
1909	/* qpn is reserved for that QP */
1910	qp->trans_qp.base.mqp.qpn = 0;
1911	qp->is_rss = true;
1912	return 0;
1913
1914err:
1915	kvfree(in);
1916	return err;
1917}
1918
1919static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,
1920					 struct mlx5_ib_qp *qp,
1921					 struct ib_qp_init_attr *init_attr,
1922					 void *qpc)
1923{
1924	int scqe_sz;
1925	bool allow_scat_cqe = false;
1926
1927	allow_scat_cqe = qp->flags_en & MLX5_QP_FLAG_ALLOW_SCATTER_CQE;
1928
1929	if (!allow_scat_cqe && init_attr->sq_sig_type != IB_SIGNAL_ALL_WR)
1930		return;
1931
1932	scqe_sz = mlx5_ib_get_cqe_size(init_attr->send_cq);
1933	if (scqe_sz == 128) {
1934		MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
1935		return;
1936	}
1937
1938	if (init_attr->qp_type != MLX5_IB_QPT_DCI ||
1939	    MLX5_CAP_GEN(dev->mdev, dc_req_scat_data_cqe))
1940		MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
1941}
1942
1943static int atomic_size_to_mode(int size_mask)
1944{
1945	/* driver does not support atomic_size > 256B
1946	 * and does not know how to translate bigger sizes
1947	 */
1948	int supported_size_mask = size_mask & 0x1ff;
1949	int log_max_size;
1950
1951	if (!supported_size_mask)
1952		return -EOPNOTSUPP;
1953
1954	log_max_size = __fls(supported_size_mask);
1955
1956	if (log_max_size > 3)
1957		return log_max_size;
1958
1959	return MLX5_ATOMIC_MODE_8B;
1960}
1961
1962static int get_atomic_mode(struct mlx5_ib_dev *dev,
1963			   struct mlx5_ib_qp *qp)
1964{
1965	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
1966	u8 atomic = MLX5_CAP_GEN(dev->mdev, atomic);
1967	int atomic_mode = -EOPNOTSUPP;
1968	int atomic_size_mask;
1969
1970	if (!atomic)
1971		return -EOPNOTSUPP;
1972
1973	if (qp->type == MLX5_IB_QPT_DCT)
1974		atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
1975	else
1976		atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
1977
1978	if ((atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_CMP_SWAP) ||
1979	    (atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_FETCH_ADD))
1980		atomic_mode = atomic_size_to_mode(atomic_size_mask);
1981
1982	if (atomic_mode <= 0 &&
1983	    (atomic_operations & MLX5_ATOMIC_OPS_CMP_SWAP &&
1984	     atomic_operations & MLX5_ATOMIC_OPS_FETCH_ADD))
1985		atomic_mode = MLX5_ATOMIC_MODE_IB_COMP;
1986
1987	/* OOO DP QPs do not support larger than 8-Bytes atomic operations */
1988	if (atomic_mode > MLX5_ATOMIC_MODE_8B && qp->is_ooo_rq)
1989		atomic_mode = MLX5_ATOMIC_MODE_8B;
1990
1991	return atomic_mode;
1992}
1993
1994static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1995			     struct mlx5_create_qp_params *params)
1996{
1997	struct ib_qp_init_attr *attr = params->attr;
1998	u32 uidx = params->uidx;
1999	struct mlx5_ib_resources *devr = &dev->devr;
2000	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2001	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2002	struct mlx5_core_dev *mdev = dev->mdev;
2003	struct mlx5_ib_qp_base *base;
2004	unsigned long flags;
2005	void *qpc;
2006	u32 *in;
2007	int err;
2008
2009	if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2010		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2011
2012	in = kvzalloc(inlen, GFP_KERNEL);
2013	if (!in)
2014		return -ENOMEM;
2015
2016	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2017
2018	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);
2019	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2020	MLX5_SET(qpc, qpc, pd, to_mpd(devr->p0)->pdn);
2021
2022	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2023		MLX5_SET(qpc, qpc, block_lb_mc, 1);
2024	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2025		MLX5_SET(qpc, qpc, cd_master, 1);
2026	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2027		MLX5_SET(qpc, qpc, cd_slave_send, 1);
2028	if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
2029		MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2030
2031	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));
2032	MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);
2033	MLX5_SET(qpc, qpc, no_sq, 1);
2034	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2035	MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
2036	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
2037	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr->xrcd)->xrcdn);
2038	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2039
2040	/* 0xffffff means we ask to work with cqe version 0 */
2041	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2042		MLX5_SET(qpc, qpc, user_index, uidx);
2043
2044	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
2045		MLX5_SET(qpc, qpc, end_padding_mode,
2046			 MLX5_WQ_END_PAD_MODE_ALIGN);
2047		/* Special case to clean flag */
2048		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2049	}
2050
2051	base = &qp->trans_qp.base;
2052	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2053	kvfree(in);
2054	if (err)
2055		return err;
2056
2057	base->container_mibqp = qp;
2058	base->mqp.event = mlx5_ib_qp_event;
2059	if (MLX5_CAP_GEN(mdev, ece_support))
2060		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2061
2062	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2063	list_add_tail(&qp->qps_list, &dev->qp_list);
2064	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2065
2066	qp->trans_qp.xrcdn = to_mxrcd(attr->xrcd)->xrcdn;
2067	return 0;
2068}
2069
2070static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2071		      struct mlx5_ib_qp *qp,
2072		      struct mlx5_create_qp_params *params)
2073{
2074	struct ib_qp_init_attr *init_attr = params->attr;
2075	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2076	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2077	struct ib_udata *udata = params->udata;
2078	u32 uidx = params->uidx;
2079	struct mlx5_ib_resources *devr = &dev->devr;
2080	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2081	struct mlx5_core_dev *mdev = dev->mdev;
2082	struct mlx5_ib_cq *send_cq;
2083	struct mlx5_ib_cq *recv_cq;
2084	unsigned long flags;
2085	struct mlx5_ib_qp_base *base;
2086	int ts_format;
2087	int mlx5_st;
2088	void *qpc;
2089	u32 *in;
2090	int err;
2091
2092	spin_lock_init(&qp->sq.lock);
2093	spin_lock_init(&qp->rq.lock);
2094
2095	mlx5_st = to_mlx5_st(qp->type);
2096	if (mlx5_st < 0)
2097		return -EINVAL;
2098
2099	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2100		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2101
2102	base = &qp->trans_qp.base;
2103
2104	qp->has_rq = qp_has_rq(init_attr);
2105	err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
2106	if (err) {
2107		mlx5_ib_dbg(dev, "err %d\n", err);
2108		return err;
2109	}
2110
2111	if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
2112	    ucmd->rq_wqe_count != qp->rq.wqe_cnt)
2113		return -EINVAL;
2114
2115	if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
2116		return -EINVAL;
2117
2118	ts_format = get_qp_ts_format(dev, to_mcq(init_attr->send_cq),
2119				     to_mcq(init_attr->recv_cq));
2120
2121	if (ts_format < 0)
2122		return ts_format;
2123
2124	err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
2125			      &inlen, base, ucmd);
2126	if (err)
2127		return err;
2128
2129	if (MLX5_CAP_GEN(mdev, ece_support))
2130		MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
2131	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2132
2133	MLX5_SET(qpc, qpc, st, mlx5_st);
2134	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2135	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
2136
2137	if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
2138		MLX5_SET(qpc, qpc, wq_signature, 1);
2139
2140	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2141		MLX5_SET(qpc, qpc, cd_master, 1);
2142	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2143		MLX5_SET(qpc, qpc, cd_slave_send, 1);
2144	if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE)
2145		configure_requester_scat_cqe(dev, qp, init_attr, qpc);
2146
2147	if (qp->rq.wqe_cnt) {
2148		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2149		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2150	}
2151
2152	if (qp->flags_en & MLX5_QP_FLAG_DCI_STREAM) {
2153		MLX5_SET(qpc, qpc, log_num_dci_stream_channels,
2154			 ucmd->dci_streams.log_num_concurent);
2155		MLX5_SET(qpc, qpc, log_num_dci_errored_streams,
2156			 ucmd->dci_streams.log_num_errored);
2157	}
2158
2159	MLX5_SET(qpc, qpc, ts_format, ts_format);
2160	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
2161
2162	MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2163
2164	/* Set default resources */
2165	if (init_attr->srq) {
2166		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2167		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2168			 to_msrq(init_attr->srq)->msrq.srqn);
2169	} else {
2170		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2171		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2172			 to_msrq(devr->s1)->msrq.srqn);
2173	}
2174
2175	if (init_attr->send_cq)
2176		MLX5_SET(qpc, qpc, cqn_snd,
2177			 to_mcq(init_attr->send_cq)->mcq.cqn);
2178
2179	if (init_attr->recv_cq)
2180		MLX5_SET(qpc, qpc, cqn_rcv,
2181			 to_mcq(init_attr->recv_cq)->mcq.cqn);
2182
2183	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2184
2185	/* 0xffffff means we ask to work with cqe version 0 */
2186	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2187		MLX5_SET(qpc, qpc, user_index, uidx);
2188
2189	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
2190		MLX5_SET(qpc, qpc, end_padding_mode,
2191			 MLX5_WQ_END_PAD_MODE_ALIGN);
2192		/* Special case to clean flag */
2193		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2194	}
2195
2196	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2197
2198	kvfree(in);
2199	if (err)
2200		goto err_create;
2201
2202	base->container_mibqp = qp;
2203	base->mqp.event = mlx5_ib_qp_event;
2204	if (MLX5_CAP_GEN(mdev, ece_support))
2205		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2206
2207	get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
2208		&send_cq, &recv_cq);
2209	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2210	mlx5_ib_lock_cqs(send_cq, recv_cq);
2211	/* Maintain device to QPs access, needed for further handling via reset
2212	 * flow
2213	 */
2214	list_add_tail(&qp->qps_list, &dev->qp_list);
2215	/* Maintain CQ to QPs access, needed for further handling via reset flow
2216	 */
2217	if (send_cq)
2218		list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2219	if (recv_cq)
2220		list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2221	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2222	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2223
2224	return 0;
2225
2226err_create:
2227	destroy_qp(dev, qp, base, udata);
2228	return err;
2229}
2230
2231static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2232			  struct mlx5_ib_qp *qp,
2233			  struct mlx5_create_qp_params *params)
2234{
2235	struct ib_qp_init_attr *init_attr = params->attr;
2236	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2237	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2238	struct ib_udata *udata = params->udata;
2239	u32 uidx = params->uidx;
2240	struct mlx5_ib_resources *devr = &dev->devr;
2241	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2242	struct mlx5_core_dev *mdev = dev->mdev;
2243	struct mlx5_ib_cq *send_cq;
2244	struct mlx5_ib_cq *recv_cq;
2245	unsigned long flags;
2246	struct mlx5_ib_qp_base *base;
2247	int ts_format;
2248	int mlx5_st;
2249	void *qpc;
2250	u32 *in;
2251	int err;
2252
2253	spin_lock_init(&qp->sq.lock);
2254	spin_lock_init(&qp->rq.lock);
2255
2256	mlx5_st = to_mlx5_st(qp->type);
2257	if (mlx5_st < 0)
2258		return -EINVAL;
2259
2260	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2261		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2262
2263	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
2264		qp->underlay_qpn = init_attr->source_qpn;
2265
2266	base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2267		qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
2268	       &qp->raw_packet_qp.rq.base :
2269	       &qp->trans_qp.base;
2270
2271	qp->has_rq = qp_has_rq(init_attr);
2272	err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
2273	if (err) {
2274		mlx5_ib_dbg(dev, "err %d\n", err);
2275		return err;
2276	}
2277
2278	if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
2279	    ucmd->rq_wqe_count != qp->rq.wqe_cnt)
2280		return -EINVAL;
2281
2282	if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
2283		return -EINVAL;
2284
2285	if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
2286		ts_format = get_qp_ts_format(dev, to_mcq(init_attr->send_cq),
2287					     to_mcq(init_attr->recv_cq));
2288		if (ts_format < 0)
2289			return ts_format;
2290	}
2291
2292	err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
2293			      &inlen, base, ucmd);
2294	if (err)
2295		return err;
2296
2297	if (is_sqp(init_attr->qp_type))
2298		qp->port = init_attr->port_num;
2299
2300	if (MLX5_CAP_GEN(mdev, ece_support))
2301		MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
2302	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2303
2304	MLX5_SET(qpc, qpc, st, mlx5_st);
2305	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2306	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
2307
2308	if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
2309		MLX5_SET(qpc, qpc, wq_signature, 1);
2310
2311	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2312		MLX5_SET(qpc, qpc, block_lb_mc, 1);
2313
2314	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2315		MLX5_SET(qpc, qpc, cd_master, 1);
2316	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2317		MLX5_SET(qpc, qpc, cd_slave_send, 1);
2318	if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
2319		MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2320	if (qp->flags_en & MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE)
2321		MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);
2322	if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
2323	    (init_attr->qp_type == IB_QPT_RC ||
2324	     init_attr->qp_type == IB_QPT_UC)) {
2325		int rcqe_sz = mlx5_ib_get_cqe_size(init_attr->recv_cq);
2326
2327		MLX5_SET(qpc, qpc, cs_res,
2328			 rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :
2329					  MLX5_RES_SCAT_DATA32_CQE);
2330	}
2331	if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
2332	    (qp->type == MLX5_IB_QPT_DCI || qp->type == IB_QPT_RC))
2333		configure_requester_scat_cqe(dev, qp, init_attr, qpc);
2334
2335	if (qp->rq.wqe_cnt) {
2336		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2337		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2338	}
2339
2340	if (init_attr->qp_type != IB_QPT_RAW_PACKET)
2341		MLX5_SET(qpc, qpc, ts_format, ts_format);
2342
2343	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
2344
2345	if (qp->sq.wqe_cnt) {
2346		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2347	} else {
2348		MLX5_SET(qpc, qpc, no_sq, 1);
2349		if (init_attr->srq &&
2350		    init_attr->srq->srq_type == IB_SRQT_TM)
2351			MLX5_SET(qpc, qpc, offload_type,
2352				 MLX5_QPC_OFFLOAD_TYPE_RNDV);
2353	}
2354
2355	/* Set default resources */
2356	switch (init_attr->qp_type) {
2357	case IB_QPT_XRC_INI:
2358		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2359		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2360		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
2361		break;
2362	default:
2363		if (init_attr->srq) {
2364			MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2365			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
2366		} else {
2367			MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2368			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
2369		}
2370	}
2371
2372	if (init_attr->send_cq)
2373		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
2374
2375	if (init_attr->recv_cq)
2376		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
2377
2378	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2379
2380	/* 0xffffff means we ask to work with cqe version 0 */
2381	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2382		MLX5_SET(qpc, qpc, user_index, uidx);
2383
2384	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING &&
2385	    init_attr->qp_type != IB_QPT_RAW_PACKET) {
2386		MLX5_SET(qpc, qpc, end_padding_mode,
2387			 MLX5_WQ_END_PAD_MODE_ALIGN);
2388		/* Special case to clean flag */
2389		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2390	}
2391
2392	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2393	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2394		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
2395		raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
2396		err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,
2397					   &params->resp, init_attr);
2398	} else
2399		err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2400
2401	kvfree(in);
2402	if (err)
2403		goto err_create;
2404
2405	base->container_mibqp = qp;
2406	base->mqp.event = mlx5_ib_qp_event;
2407	if (MLX5_CAP_GEN(mdev, ece_support))
2408		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2409
2410	get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
2411		&send_cq, &recv_cq);
2412	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2413	mlx5_ib_lock_cqs(send_cq, recv_cq);
2414	/* Maintain device to QPs access, needed for further handling via reset
2415	 * flow
2416	 */
2417	list_add_tail(&qp->qps_list, &dev->qp_list);
2418	/* Maintain CQ to QPs access, needed for further handling via reset flow
2419	 */
2420	if (send_cq)
2421		list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2422	if (recv_cq)
2423		list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2424	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2425	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2426
2427	return 0;
2428
2429err_create:
2430	destroy_qp(dev, qp, base, udata);
2431	return err;
2432}
2433
2434static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2435			    struct mlx5_ib_qp *qp,
2436			    struct mlx5_create_qp_params *params)
2437{
2438	struct ib_qp_init_attr *attr = params->attr;
2439	u32 uidx = params->uidx;
2440	struct mlx5_ib_resources *devr = &dev->devr;
2441	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2442	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2443	struct mlx5_core_dev *mdev = dev->mdev;
2444	struct mlx5_ib_cq *send_cq;
2445	struct mlx5_ib_cq *recv_cq;
2446	unsigned long flags;
2447	struct mlx5_ib_qp_base *base;
2448	int mlx5_st;
2449	void *qpc;
2450	u32 *in;
2451	int err;
2452
2453	spin_lock_init(&qp->sq.lock);
2454	spin_lock_init(&qp->rq.lock);
2455
2456	mlx5_st = to_mlx5_st(qp->type);
2457	if (mlx5_st < 0)
2458		return -EINVAL;
2459
2460	if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2461		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2462
2463	base = &qp->trans_qp.base;
2464
2465	qp->has_rq = qp_has_rq(attr);
2466	err = set_rq_size(dev, &attr->cap, qp->has_rq, qp, NULL);
2467	if (err) {
2468		mlx5_ib_dbg(dev, "err %d\n", err);
2469		return err;
2470	}
2471
2472	err = _create_kernel_qp(dev, attr, qp, &in, &inlen, base);
2473	if (err)
2474		return err;
2475
2476	if (is_sqp(attr->qp_type))
2477		qp->port = attr->port_num;
2478
2479	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2480
2481	MLX5_SET(qpc, qpc, st, mlx5_st);
2482	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2483
2484	if (attr->qp_type != MLX5_IB_QPT_REG_UMR)
2485		MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
2486	else
2487		MLX5_SET(qpc, qpc, latency_sensitive, 1);
2488
2489
2490	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2491		MLX5_SET(qpc, qpc, block_lb_mc, 1);
2492
2493	if (qp->rq.wqe_cnt) {
2494		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2495		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2496	}
2497
2498	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));
2499
2500	if (qp->sq.wqe_cnt)
2501		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2502	else
2503		MLX5_SET(qpc, qpc, no_sq, 1);
2504
2505	if (attr->srq) {
2506		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2507		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2508			 to_msrq(attr->srq)->msrq.srqn);
2509	} else {
2510		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2511		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2512			 to_msrq(devr->s1)->msrq.srqn);
2513	}
2514
2515	if (attr->send_cq)
2516		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr->send_cq)->mcq.cqn);
2517
2518	if (attr->recv_cq)
2519		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr->recv_cq)->mcq.cqn);
2520
2521	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2522
2523	/* 0xffffff means we ask to work with cqe version 0 */
2524	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2525		MLX5_SET(qpc, qpc, user_index, uidx);
2526
2527	/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
2528	if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO)
2529		MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
2530
2531	if (qp->flags & IB_QP_CREATE_INTEGRITY_EN &&
2532	    MLX5_CAP_GEN(mdev, go_back_n))
2533		MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);
2534
2535	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2536	kvfree(in);
2537	if (err)
2538		goto err_create;
2539
2540	base->container_mibqp = qp;
2541	base->mqp.event = mlx5_ib_qp_event;
2542
2543	get_cqs(qp->type, attr->send_cq, attr->recv_cq,
2544		&send_cq, &recv_cq);
2545	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2546	mlx5_ib_lock_cqs(send_cq, recv_cq);
2547	/* Maintain device to QPs access, needed for further handling via reset
2548	 * flow
2549	 */
2550	list_add_tail(&qp->qps_list, &dev->qp_list);
2551	/* Maintain CQ to QPs access, needed for further handling via reset flow
2552	 */
2553	if (send_cq)
2554		list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2555	if (recv_cq)
2556		list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2557	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2558	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2559
2560	return 0;
2561
2562err_create:
2563	destroy_qp(dev, qp, base, NULL);
2564	return err;
2565}
2566
2567static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2568	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
2569{
2570	if (send_cq) {
2571		if (recv_cq) {
2572			if (send_cq->mcq.cqn < recv_cq->mcq.cqn)  {
2573				spin_lock(&send_cq->lock);
2574				spin_lock_nested(&recv_cq->lock,
2575						 SINGLE_DEPTH_NESTING);
2576			} else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2577				spin_lock(&send_cq->lock);
2578				__acquire(&recv_cq->lock);
2579			} else {
2580				spin_lock(&recv_cq->lock);
2581				spin_lock_nested(&send_cq->lock,
2582						 SINGLE_DEPTH_NESTING);
2583			}
2584		} else {
2585			spin_lock(&send_cq->lock);
2586			__acquire(&recv_cq->lock);
2587		}
2588	} else if (recv_cq) {
2589		spin_lock(&recv_cq->lock);
2590		__acquire(&send_cq->lock);
2591	} else {
2592		__acquire(&send_cq->lock);
2593		__acquire(&recv_cq->lock);
2594	}
2595}
2596
2597static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2598	__releases(&send_cq->lock) __releases(&recv_cq->lock)
2599{
2600	if (send_cq) {
2601		if (recv_cq) {
2602			if (send_cq->mcq.cqn < recv_cq->mcq.cqn)  {
2603				spin_unlock(&recv_cq->lock);
2604				spin_unlock(&send_cq->lock);
2605			} else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2606				__release(&recv_cq->lock);
2607				spin_unlock(&send_cq->lock);
2608			} else {
2609				spin_unlock(&send_cq->lock);
2610				spin_unlock(&recv_cq->lock);
2611			}
2612		} else {
2613			__release(&recv_cq->lock);
2614			spin_unlock(&send_cq->lock);
2615		}
2616	} else if (recv_cq) {
2617		__release(&send_cq->lock);
2618		spin_unlock(&recv_cq->lock);
2619	} else {
2620		__release(&recv_cq->lock);
2621		__release(&send_cq->lock);
2622	}
2623}
2624
2625static void get_cqs(enum ib_qp_type qp_type,
2626		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
2627		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
2628{
2629	switch (qp_type) {
2630	case IB_QPT_XRC_TGT:
2631		*send_cq = NULL;
2632		*recv_cq = NULL;
2633		break;
2634	case MLX5_IB_QPT_REG_UMR:
2635	case IB_QPT_XRC_INI:
2636		*send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2637		*recv_cq = NULL;
2638		break;
2639
2640	case IB_QPT_SMI:
2641	case MLX5_IB_QPT_HW_GSI:
2642	case IB_QPT_RC:
2643	case IB_QPT_UC:
2644	case IB_QPT_UD:
2645	case IB_QPT_RAW_PACKET:
2646		*send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2647		*recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
2648		break;
2649	default:
2650		*send_cq = NULL;
2651		*recv_cq = NULL;
2652		break;
2653	}
2654}
2655
2656static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2657				const struct mlx5_modify_raw_qp_param *raw_qp_param,
2658				u8 lag_tx_affinity);
2659
2660static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2661			      struct ib_udata *udata)
2662{
2663	struct mlx5_ib_cq *send_cq, *recv_cq;
2664	struct mlx5_ib_qp_base *base;
2665	unsigned long flags;
2666	int err;
2667
2668	if (qp->is_rss) {
2669		destroy_rss_raw_qp_tir(dev, qp);
2670		return;
2671	}
2672
2673	base = (qp->type == IB_QPT_RAW_PACKET ||
2674		qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
2675		       &qp->raw_packet_qp.rq.base :
2676		       &qp->trans_qp.base;
2677
2678	if (qp->state != IB_QPS_RESET) {
2679		if (qp->type != IB_QPT_RAW_PACKET &&
2680		    !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
2681			err = mlx5_core_qp_modify(dev, MLX5_CMD_OP_2RST_QP, 0,
2682						  NULL, &base->mqp, NULL);
2683		} else {
2684			struct mlx5_modify_raw_qp_param raw_qp_param = {
2685				.operation = MLX5_CMD_OP_2RST_QP
2686			};
2687
2688			err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0);
2689		}
2690		if (err)
2691			mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
2692				     base->mqp.qpn);
2693	}
2694
2695	get_cqs(qp->type, qp->ibqp.send_cq, qp->ibqp.recv_cq, &send_cq,
2696		&recv_cq);
2697
2698	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2699	mlx5_ib_lock_cqs(send_cq, recv_cq);
2700	/* del from lists under both locks above to protect reset flow paths */
2701	list_del(&qp->qps_list);
2702	if (send_cq)
2703		list_del(&qp->cq_send_list);
2704
2705	if (recv_cq)
2706		list_del(&qp->cq_recv_list);
2707
2708	if (!udata) {
2709		__mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
2710				   qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
2711		if (send_cq != recv_cq)
2712			__mlx5_ib_cq_clean(send_cq, base->mqp.qpn,
2713					   NULL);
2714	}
2715	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2716	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2717
2718	if (qp->type == IB_QPT_RAW_PACKET ||
2719	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2720		destroy_raw_packet_qp(dev, qp);
2721	} else {
2722		err = mlx5_core_destroy_qp(dev, &base->mqp);
2723		if (err)
2724			mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
2725				     base->mqp.qpn);
2726	}
2727
2728	destroy_qp(dev, qp, base, udata);
2729}
2730
2731static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2732		      struct mlx5_ib_qp *qp,
2733		      struct mlx5_create_qp_params *params)
2734{
2735	struct ib_qp_init_attr *attr = params->attr;
2736	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2737	u32 uidx = params->uidx;
2738	void *dctc;
2739
2740	if (mlx5_lag_is_active(dev->mdev) && !MLX5_CAP_GEN(dev->mdev, lag_dct))
2741		return -EOPNOTSUPP;
2742
2743	qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
2744	if (!qp->dct.in)
2745		return -ENOMEM;
2746
2747	MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid);
2748	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
2749	MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
2750	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
2751	MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
2752	MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
2753	MLX5_SET(dctc, dctc, user_index, uidx);
2754	if (MLX5_CAP_GEN(dev->mdev, ece_support))
2755		MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
2756
2757	if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) {
2758		int rcqe_sz = mlx5_ib_get_cqe_size(attr->recv_cq);
2759
2760		if (rcqe_sz == 128)
2761			MLX5_SET(dctc, dctc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
2762	}
2763
2764	qp->state = IB_QPS_RESET;
2765	return 0;
2766}
2767
2768static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
2769			 enum ib_qp_type *type)
2770{
2771	if (attr->qp_type == IB_QPT_DRIVER && !MLX5_CAP_GEN(dev->mdev, dct))
2772		goto out;
2773
2774	switch (attr->qp_type) {
2775	case IB_QPT_XRC_TGT:
2776	case IB_QPT_XRC_INI:
2777		if (!MLX5_CAP_GEN(dev->mdev, xrc))
2778			goto out;
2779		fallthrough;
2780	case IB_QPT_RC:
2781	case IB_QPT_UC:
2782	case IB_QPT_SMI:
2783	case MLX5_IB_QPT_HW_GSI:
2784	case IB_QPT_DRIVER:
2785	case IB_QPT_GSI:
2786	case IB_QPT_RAW_PACKET:
2787	case IB_QPT_UD:
2788	case MLX5_IB_QPT_REG_UMR:
2789		break;
2790	default:
2791		goto out;
2792	}
2793
2794	*type = attr->qp_type;
2795	return 0;
2796
2797out:
2798	mlx5_ib_dbg(dev, "Unsupported QP type %d\n", attr->qp_type);
2799	return -EOPNOTSUPP;
2800}
2801
2802static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2803			    struct ib_qp_init_attr *attr,
2804			    struct ib_udata *udata)
2805{
2806	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
2807		udata, struct mlx5_ib_ucontext, ibucontext);
2808
2809	if (!udata) {
2810		/* Kernel create_qp callers */
2811		if (attr->rwq_ind_tbl)
2812			return -EOPNOTSUPP;
2813
2814		switch (attr->qp_type) {
2815		case IB_QPT_RAW_PACKET:
2816		case IB_QPT_DRIVER:
2817			return -EOPNOTSUPP;
2818		default:
2819			return 0;
2820		}
2821	}
2822
2823	/* Userspace create_qp callers */
2824	if (attr->qp_type == IB_QPT_RAW_PACKET && !ucontext->cqe_version) {
2825		mlx5_ib_dbg(dev,
2826			"Raw Packet QP is only supported for CQE version > 0\n");
2827		return -EINVAL;
2828	}
2829
2830	if (attr->qp_type != IB_QPT_RAW_PACKET && attr->rwq_ind_tbl) {
2831		mlx5_ib_dbg(dev,
2832			    "Wrong QP type %d for the RWQ indirect table\n",
2833			    attr->qp_type);
2834		return -EINVAL;
2835	}
2836
2837	/*
2838	 * We don't need to see this warning, it means that kernel code
2839	 * missing ib_pd. Placed here to catch developer's mistakes.
2840	 */
2841	WARN_ONCE(!pd && attr->qp_type != IB_QPT_XRC_TGT,
2842		  "There is a missing PD pointer assignment\n");
2843	return 0;
2844}
2845
2846static bool get_dp_ooo_cap(struct mlx5_core_dev *mdev, enum ib_qp_type qp_type)
2847{
2848	if (!MLX5_CAP_GEN_2(mdev, dp_ordering_force))
2849		return false;
2850
2851	switch (qp_type) {
2852	case IB_QPT_RC:
2853		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_rc);
2854	case IB_QPT_XRC_INI:
2855	case IB_QPT_XRC_TGT:
2856		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_xrc);
2857	case IB_QPT_UC:
2858		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_uc);
2859	case IB_QPT_UD:
2860		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_ud);
2861	case MLX5_IB_QPT_DCI:
2862	case MLX5_IB_QPT_DCT:
2863		return MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_dc);
2864	default:
2865		return false;
2866	}
2867}
2868
2869static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2870				bool cond, struct mlx5_ib_qp *qp)
2871{
2872	if (!(*flags & flag))
2873		return;
2874
2875	if (cond) {
2876		qp->flags_en |= flag;
2877		*flags &= ~flag;
2878		return;
2879	}
2880
2881	switch (flag) {
2882	case MLX5_QP_FLAG_SCATTER_CQE:
2883	case MLX5_QP_FLAG_ALLOW_SCATTER_CQE:
2884		/*
2885		 * We don't return error if these flags were provided,
2886		 * and mlx5 doesn't have right capability.
2887		 */
2888		*flags &= ~(MLX5_QP_FLAG_SCATTER_CQE |
2889			    MLX5_QP_FLAG_ALLOW_SCATTER_CQE);
2890		return;
2891	default:
2892		break;
2893	}
2894	mlx5_ib_dbg(dev, "Vendor create QP flag 0x%X is not supported\n", flag);
2895}
2896
2897static int process_vendor_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2898				void *ucmd, struct ib_qp_init_attr *attr)
2899{
2900	struct mlx5_core_dev *mdev = dev->mdev;
2901	bool cond;
2902	int flags;
2903
2904	if (attr->rwq_ind_tbl)
2905		flags = ((struct mlx5_ib_create_qp_rss *)ucmd)->flags;
2906	else
2907		flags = ((struct mlx5_ib_create_qp *)ucmd)->flags;
2908
2909	switch (flags & (MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI)) {
2910	case MLX5_QP_FLAG_TYPE_DCI:
2911		qp->type = MLX5_IB_QPT_DCI;
2912		break;
2913	case MLX5_QP_FLAG_TYPE_DCT:
2914		qp->type = MLX5_IB_QPT_DCT;
2915		break;
2916	default:
2917		if (qp->type != IB_QPT_DRIVER)
2918			break;
2919		/*
2920		 * It is IB_QPT_DRIVER and or no subtype or
2921		 * wrong subtype were provided.
2922		 */
2923		return -EINVAL;
2924	}
2925
2926	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCI, true, qp);
2927	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCT, true, qp);
2928	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_DCI_STREAM,
2929			    MLX5_CAP_GEN(mdev, log_max_dci_stream_channels),
2930			    qp);
2931
2932	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SIGNATURE, true, qp);
2933	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SCATTER_CQE,
2934			    MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
2935	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_ALLOW_SCATTER_CQE,
2936			    MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
2937
2938	if (qp->type == IB_QPT_RAW_PACKET) {
2939		cond = MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan) ||
2940		       MLX5_CAP_ETH(mdev, tunnel_stateless_gre) ||
2941		       MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx);
2942		process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TUNNEL_OFFLOADS,
2943				    cond, qp);
2944		process_vendor_flag(dev, &flags,
2945				    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC, true,
2946				    qp);
2947		process_vendor_flag(dev, &flags,
2948				    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC, true,
2949				    qp);
2950	}
2951
2952	if (qp->type == IB_QPT_RC)
2953		process_vendor_flag(dev, &flags,
2954				    MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE,
2955				    MLX5_CAP_GEN(mdev, qp_packet_based), qp);
2956
2957	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_BFREG_INDEX, true, qp);
2958	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_UAR_PAGE_INDEX, true, qp);
2959
2960	cond = qp->flags_en & ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |
2961				MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
2962				MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC);
2963	if (attr->rwq_ind_tbl && cond) {
2964		mlx5_ib_dbg(dev, "RSS RAW QP has unsupported flags 0x%X\n",
2965			    cond);
2966		return -EINVAL;
2967	}
2968
2969	if (flags)
2970		mlx5_ib_dbg(dev, "udata has unsupported flags 0x%X\n", flags);
2971
2972	return (flags) ? -EINVAL : 0;
2973	}
2974
2975static void process_create_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2976				bool cond, struct mlx5_ib_qp *qp)
2977{
2978	if (!(*flags & flag))
2979		return;
2980
2981	if (cond) {
2982		qp->flags |= flag;
2983		*flags &= ~flag;
2984		return;
2985	}
2986
2987	mlx5_ib_dbg(dev, "Verbs create QP flag 0x%X is not supported\n", flag);
2988}
2989
2990static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2991				struct ib_qp_init_attr *attr)
2992{
2993	enum ib_qp_type qp_type = qp->type;
2994	struct mlx5_core_dev *mdev = dev->mdev;
2995	int create_flags = attr->create_flags;
2996	bool cond;
2997
2998	if (qp_type == MLX5_IB_QPT_DCT)
2999		return (create_flags) ? -EINVAL : 0;
3000
3001	if (qp_type == IB_QPT_RAW_PACKET && attr->rwq_ind_tbl)
3002		return (create_flags) ? -EINVAL : 0;
3003
3004	process_create_flag(dev, &create_flags, IB_QP_CREATE_NETIF_QP,
3005			    mlx5_get_flow_namespace(dev->mdev,
3006						    MLX5_FLOW_NAMESPACE_BYPASS),
3007			    qp);
3008	process_create_flag(dev, &create_flags,
3009			    IB_QP_CREATE_INTEGRITY_EN,
3010			    MLX5_CAP_GEN(mdev, sho), qp);
3011	process_create_flag(dev, &create_flags,
3012			    IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
3013			    MLX5_CAP_GEN(mdev, block_lb_mc), qp);
3014	process_create_flag(dev, &create_flags, IB_QP_CREATE_CROSS_CHANNEL,
3015			    MLX5_CAP_GEN(mdev, cd), qp);
3016	process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_SEND,
3017			    MLX5_CAP_GEN(mdev, cd), qp);
3018	process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_RECV,
3019			    MLX5_CAP_GEN(mdev, cd), qp);
3020
3021	if (qp_type == IB_QPT_UD) {
3022		process_create_flag(dev, &create_flags,
3023				    IB_QP_CREATE_IPOIB_UD_LSO,
3024				    MLX5_CAP_GEN(mdev, ipoib_basic_offloads),
3025				    qp);
3026		cond = MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_IB;
3027		process_create_flag(dev, &create_flags, IB_QP_CREATE_SOURCE_QPN,
3028				    cond, qp);
3029	}
3030
3031	if (qp_type == IB_QPT_RAW_PACKET) {
3032		cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
3033		       MLX5_CAP_ETH(mdev, scatter_fcs);
3034		process_create_flag(dev, &create_flags,
3035				    IB_QP_CREATE_SCATTER_FCS, cond, qp);
3036
3037		cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
3038		       MLX5_CAP_ETH(mdev, vlan_cap);
3039		process_create_flag(dev, &create_flags,
3040				    IB_QP_CREATE_CVLAN_STRIPPING, cond, qp);
3041	}
3042
3043	process_create_flag(dev, &create_flags,
3044			    IB_QP_CREATE_PCI_WRITE_END_PADDING,
3045			    MLX5_CAP_GEN(mdev, end_pad), qp);
3046
3047	process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,
3048			    true, qp);
3049
3050	if (create_flags) {
3051		mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",
3052			    create_flags);
3053		return -EOPNOTSUPP;
3054	}
3055	return 0;
3056}
3057
3058static int process_udata_size(struct mlx5_ib_dev *dev,
3059			      struct mlx5_create_qp_params *params)
3060{
3061	size_t ucmd = sizeof(struct mlx5_ib_create_qp);
3062	struct ib_udata *udata = params->udata;
3063	size_t outlen = udata->outlen;
3064	size_t inlen = udata->inlen;
3065
3066	params->outlen = min(outlen, sizeof(struct mlx5_ib_create_qp_resp));
3067	params->ucmd_size = ucmd;
3068	if (!params->is_rss_raw) {
3069		/* User has old rdma-core, which doesn't support ECE */
3070		size_t min_inlen =
3071			offsetof(struct mlx5_ib_create_qp, ece_options);
3072
3073		/*
3074		 * We will check in check_ucmd_data() that user
3075		 * cleared everything after inlen.
3076		 */
3077		params->inlen = (inlen < min_inlen) ? 0 : min(inlen, ucmd);
3078		goto out;
3079	}
3080
3081	/* RSS RAW QP */
3082	if (inlen < offsetofend(struct mlx5_ib_create_qp_rss, flags))
3083		return -EINVAL;
3084
3085	if (outlen < offsetofend(struct mlx5_ib_create_qp_resp, bfreg_index))
3086		return -EINVAL;
3087
3088	ucmd = sizeof(struct mlx5_ib_create_qp_rss);
3089	params->ucmd_size = ucmd;
3090	if (inlen > ucmd && !ib_is_udata_cleared(udata, ucmd, inlen - ucmd))
3091		return -EINVAL;
3092
3093	params->inlen = min(ucmd, inlen);
3094out:
3095	if (!params->inlen)
3096		mlx5_ib_dbg(dev, "udata is too small\n");
3097
3098	return (params->inlen) ? 0 : -EINVAL;
3099}
3100
3101static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
3102		     struct mlx5_ib_qp *qp,
3103		     struct mlx5_create_qp_params *params)
3104{
3105	int err;
3106
3107	if (params->is_rss_raw) {
3108		err = create_rss_raw_qp_tir(dev, pd, qp, params);
3109		goto out;
3110	}
3111
3112	switch (qp->type) {
3113	case MLX5_IB_QPT_DCT:
3114		err = create_dct(dev, pd, qp, params);
3115		break;
3116	case MLX5_IB_QPT_DCI:
3117		err = create_dci(dev, pd, qp, params);
3118		break;
3119	case IB_QPT_XRC_TGT:
3120		err = create_xrc_tgt_qp(dev, qp, params);
3121		break;
3122	case IB_QPT_GSI:
3123		err = mlx5_ib_create_gsi(pd, qp, params->attr);
3124		break;
3125	case MLX5_IB_QPT_HW_GSI:
3126		rdma_restrack_no_track(&qp->ibqp.res);
3127		fallthrough;
3128	case MLX5_IB_QPT_REG_UMR:
3129	default:
3130		if (params->udata)
3131			err = create_user_qp(dev, pd, qp, params);
3132		else
3133			err = create_kernel_qp(dev, pd, qp, params);
3134	}
3135
3136out:
3137	if (err) {
3138		mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
3139		return err;
3140	}
3141
3142	if (is_qp0(qp->type))
3143		qp->ibqp.qp_num = 0;
3144	else if (is_qp1(qp->type))
3145		qp->ibqp.qp_num = 1;
3146	else
3147		qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
3148
3149	mlx5_ib_dbg(dev,
3150		"QP type %d, ib qpn 0x%X, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x, ece 0x%x\n",
3151		qp->type, qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
3152		params->attr->recv_cq ? to_mcq(params->attr->recv_cq)->mcq.cqn :
3153					-1,
3154		params->attr->send_cq ? to_mcq(params->attr->send_cq)->mcq.cqn :
3155					-1,
3156		params->resp.ece_options);
3157
3158	return 0;
3159}
3160
3161static int check_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3162			 struct ib_qp_init_attr *attr)
3163{
3164	int ret = 0;
3165
3166	switch (qp->type) {
3167	case MLX5_IB_QPT_DCT:
3168		ret = (!attr->srq || !attr->recv_cq) ? -EINVAL : 0;
3169		break;
3170	case MLX5_IB_QPT_DCI:
3171		ret = (attr->cap.max_recv_wr || attr->cap.max_recv_sge) ?
3172			      -EINVAL :
3173			      0;
3174		break;
3175	case IB_QPT_RAW_PACKET:
3176		ret = (attr->rwq_ind_tbl && attr->send_cq) ? -EINVAL : 0;
3177		break;
3178	default:
3179		break;
3180	}
3181
3182	if (ret)
3183		mlx5_ib_dbg(dev, "QP type %d has wrong attributes\n", qp->type);
3184
3185	return ret;
3186}
3187
3188static int get_qp_uidx(struct mlx5_ib_qp *qp,
3189		       struct mlx5_create_qp_params *params)
3190{
3191	struct mlx5_ib_create_qp *ucmd = params->ucmd;
3192	struct ib_udata *udata = params->udata;
3193	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
3194		udata, struct mlx5_ib_ucontext, ibucontext);
3195
3196	if (params->is_rss_raw)
3197		return 0;
3198
3199	return get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &params->uidx);
3200}
3201
3202static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
3203{
3204	struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device);
3205
3206	if (mqp->state == IB_QPS_RTR) {
3207		int err;
3208
3209		err = mlx5_core_destroy_dct(dev, &mqp->dct.mdct);
3210		if (err) {
3211			mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
3212			return err;
3213		}
3214	}
3215
3216	kfree(mqp->dct.in);
3217	return 0;
3218}
3219
3220static int check_ucmd_data(struct mlx5_ib_dev *dev,
3221			   struct mlx5_create_qp_params *params)
3222{
3223	struct ib_udata *udata = params->udata;
3224	size_t size, last;
3225	int ret;
3226
3227	if (params->is_rss_raw)
3228		/*
3229		 * These QPs don't have "reserved" field in their
3230		 * create_qp input struct, so their data is always valid.
3231		 */
3232		last = sizeof(struct mlx5_ib_create_qp_rss);
3233	else
3234		last = offsetof(struct mlx5_ib_create_qp, reserved);
3235
3236	if (udata->inlen <= last)
3237		return 0;
3238
3239	/*
3240	 * User provides different create_qp structures based on the
3241	 * flow and we need to know if he cleared memory after our
3242	 * struct create_qp ends.
3243	 */
3244	size = udata->inlen - last;
3245	ret = ib_is_udata_cleared(params->udata, last, size);
3246	if (!ret)
3247		mlx5_ib_dbg(
3248			dev,
3249			"udata is not cleared, inlen = %zu, ucmd = %zu, last = %zu, size = %zu\n",
3250			udata->inlen, params->ucmd_size, last, size);
3251	return ret ? 0 : -EINVAL;
3252}
3253
3254int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
3255		      struct ib_udata *udata)
3256{
3257	struct mlx5_create_qp_params params = {};
3258	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3259	struct mlx5_ib_qp *qp = to_mqp(ibqp);
3260	struct ib_pd *pd = ibqp->pd;
3261	enum ib_qp_type type;
3262	int err;
3263
3264	err = mlx5_ib_dev_res_srq_init(dev);
3265	if (err)
3266		return err;
3267
3268	err = check_qp_type(dev, attr, &type);
3269	if (err)
3270		return err;
3271
3272	err = check_valid_flow(dev, pd, attr, udata);
3273	if (err)
3274		return err;
3275
3276	params.udata = udata;
3277	params.uidx = MLX5_IB_DEFAULT_UIDX;
3278	params.attr = attr;
3279	params.is_rss_raw = !!attr->rwq_ind_tbl;
3280
3281	if (udata) {
3282		err = process_udata_size(dev, &params);
3283		if (err)
3284			return err;
3285
3286		err = check_ucmd_data(dev, &params);
3287		if (err)
3288			return err;
3289
3290		params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL);
3291		if (!params.ucmd)
3292			return -ENOMEM;
3293
3294		err = ib_copy_from_udata(params.ucmd, udata, params.inlen);
3295		if (err)
3296			goto free_ucmd;
3297	}
3298
3299	mutex_init(&qp->mutex);
3300	qp->type = type;
3301	if (udata) {
3302		err = process_vendor_flags(dev, qp, params.ucmd, attr);
3303		if (err)
3304			goto free_ucmd;
3305
3306		err = get_qp_uidx(qp, &params);
3307		if (err)
3308			goto free_ucmd;
3309	}
3310	err = process_create_flags(dev, qp, attr);
3311	if (err)
3312		goto free_ucmd;
3313
3314	err = check_qp_attr(dev, qp, attr);
3315	if (err)
3316		goto free_ucmd;
3317
3318	err = create_qp(dev, pd, qp, &params);
3319	if (err)
3320		goto free_ucmd;
3321
3322	kfree(params.ucmd);
3323	params.ucmd = NULL;
3324
3325	if (udata)
3326		/*
3327		 * It is safe to copy response for all user create QP flows,
3328		 * including MLX5_IB_QPT_DCT, which doesn't need it.
3329		 * In that case, resp will be filled with zeros.
3330		 */
3331		err = ib_copy_to_udata(udata, &params.resp, params.outlen);
3332	if (err)
3333		goto destroy_qp;
3334
3335	return 0;
3336
3337destroy_qp:
3338	switch (qp->type) {
3339	case MLX5_IB_QPT_DCT:
3340		mlx5_ib_destroy_dct(qp);
3341		break;
3342	case IB_QPT_GSI:
3343		mlx5_ib_destroy_gsi(qp);
3344		break;
3345	default:
3346		destroy_qp_common(dev, qp, udata);
3347	}
3348
3349free_ucmd:
3350	kfree(params.ucmd);
3351	return err;
3352}
3353
3354int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
3355{
3356	struct mlx5_ib_dev *dev = to_mdev(qp->device);
3357	struct mlx5_ib_qp *mqp = to_mqp(qp);
3358
3359	if (mqp->type == IB_QPT_GSI)
3360		return mlx5_ib_destroy_gsi(mqp);
3361
3362	if (mqp->type == MLX5_IB_QPT_DCT)
3363		return mlx5_ib_destroy_dct(mqp);
3364
3365	destroy_qp_common(dev, mqp, udata);
3366	return 0;
3367}
3368
3369static int set_qpc_atomic_flags(struct mlx5_ib_qp *qp,
3370				const struct ib_qp_attr *attr, int attr_mask,
3371				void *qpc)
3372{
3373	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
3374	u8 dest_rd_atomic;
3375	u32 access_flags;
3376
3377	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
3378		dest_rd_atomic = attr->max_dest_rd_atomic;
3379	else
3380		dest_rd_atomic = qp->trans_qp.resp_depth;
3381
3382	if (attr_mask & IB_QP_ACCESS_FLAGS)
3383		access_flags = attr->qp_access_flags;
3384	else
3385		access_flags = qp->trans_qp.atomic_rd_en;
3386
3387	if (!dest_rd_atomic)
3388		access_flags &= IB_ACCESS_REMOTE_WRITE;
3389
3390	MLX5_SET(qpc, qpc, rre, !!(access_flags & IB_ACCESS_REMOTE_READ));
3391
3392	if (access_flags & IB_ACCESS_REMOTE_ATOMIC) {
3393		int atomic_mode;
3394
3395		atomic_mode = get_atomic_mode(dev, qp);
3396		if (atomic_mode < 0)
3397			return -EOPNOTSUPP;
3398
3399		MLX5_SET(qpc, qpc, rae, 1);
3400		MLX5_SET(qpc, qpc, atomic_mode, atomic_mode);
3401	}
3402
3403	MLX5_SET(qpc, qpc, rwe, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3404	return 0;
3405}
3406
3407enum {
3408	MLX5_PATH_FLAG_FL	= 1 << 0,
3409	MLX5_PATH_FLAG_FREE_AR	= 1 << 1,
3410	MLX5_PATH_FLAG_COUNTER	= 1 << 2,
3411};
3412
3413static int mlx5_to_ib_rate_map(u8 rate)
3414{
3415	static const int rates[] = { IB_RATE_PORT_CURRENT, IB_RATE_56_GBPS,
3416				     IB_RATE_25_GBPS,	   IB_RATE_100_GBPS,
3417				     IB_RATE_200_GBPS,	   IB_RATE_50_GBPS,
3418				     IB_RATE_400_GBPS };
3419
3420	if (rate < ARRAY_SIZE(rates))
3421		return rates[rate];
3422
3423	return rate - MLX5_STAT_RATE_OFFSET;
3424}
3425
3426static int ib_to_mlx5_rate_map(u8 rate)
3427{
3428	switch (rate) {
3429	case IB_RATE_PORT_CURRENT:
3430		return 0;
3431	case IB_RATE_56_GBPS:
3432		return 1;
3433	case IB_RATE_25_GBPS:
3434		return 2;
3435	case IB_RATE_100_GBPS:
3436		return 3;
3437	case IB_RATE_200_GBPS:
3438		return 4;
3439	case IB_RATE_50_GBPS:
3440		return 5;
3441	case IB_RATE_400_GBPS:
3442		return 6;
3443	default:
3444		return rate + MLX5_STAT_RATE_OFFSET;
3445	}
3446
3447	return 0;
3448}
3449
3450int mlx5r_ib_rate(struct mlx5_ib_dev *dev, u8 rate)
3451{
3452	u32 stat_rate_support;
3453
3454	if (rate == IB_RATE_PORT_CURRENT || rate == IB_RATE_800_GBPS ||
3455	    rate == IB_RATE_1600_GBPS)
3456		return 0;
3457
3458	if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_1600_GBPS)
3459		return -EINVAL;
3460
3461	stat_rate_support = MLX5_CAP_GEN(dev->mdev, stat_rate_support);
3462	while (rate != IB_RATE_PORT_CURRENT &&
3463	       !(1 << ib_to_mlx5_rate_map(rate) & stat_rate_support))
3464		--rate;
3465
3466	return ib_to_mlx5_rate_map(rate);
3467}
3468
3469static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
3470				      struct mlx5_ib_sq *sq, u8 sl,
3471				      struct ib_pd *pd)
3472{
3473	void *in;
3474	void *tisc;
3475	int inlen;
3476	int err;
3477
3478	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
3479	in = kvzalloc(inlen, GFP_KERNEL);
3480	if (!in)
3481		return -ENOMEM;
3482
3483	MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
3484	MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
3485
3486	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3487	MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
3488
3489	err = mlx5_core_modify_tis(dev, sq->tisn, in);
3490
3491	kvfree(in);
3492
3493	return err;
3494}
3495
3496static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
3497					 struct mlx5_ib_sq *sq, u8 tx_affinity,
3498					 struct ib_pd *pd)
3499{
3500	void *in;
3501	void *tisc;
3502	int inlen;
3503	int err;
3504
3505	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
3506	in = kvzalloc(inlen, GFP_KERNEL);
3507	if (!in)
3508		return -ENOMEM;
3509
3510	MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
3511	MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
3512
3513	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3514	MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
3515
3516	err = mlx5_core_modify_tis(dev, sq->tisn, in);
3517
3518	kvfree(in);
3519
3520	return err;
3521}
3522
3523static void mlx5_set_path_udp_sport(void *path, const struct rdma_ah_attr *ah,
3524				    u32 lqpn, u32 rqpn)
3525
3526{
3527	u32 fl = ah->grh.flow_label;
3528
3529	if (!fl)
3530		fl = rdma_calc_flow_label(lqpn, rqpn);
3531
3532	MLX5_SET(ads, path, udp_sport, rdma_flow_label_to_udp_sport(fl));
3533}
3534
3535static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3536			 const struct rdma_ah_attr *ah, void *path, u8 port,
3537			 int attr_mask, u32 path_flags,
3538			 const struct ib_qp_attr *attr, bool alt)
3539{
3540	const struct ib_global_route *grh = rdma_ah_read_grh(ah);
3541	int err;
3542	enum ib_gid_type gid_type;
3543	u8 ah_flags = rdma_ah_get_ah_flags(ah);
3544	u8 sl = rdma_ah_get_sl(ah);
3545
3546	if (attr_mask & IB_QP_PKEY_INDEX)
3547		MLX5_SET(ads, path, pkey_index,
3548			 alt ? attr->alt_pkey_index : attr->pkey_index);
3549
3550	if (ah_flags & IB_AH_GRH) {
3551		const struct ib_port_immutable *immutable;
3552
3553		immutable = ib_port_immutable_read(&dev->ib_dev, port);
3554		if (grh->sgid_index >= immutable->gid_tbl_len) {
3555			pr_err("sgid_index (%u) too large. max is %d\n",
3556			       grh->sgid_index,
3557			       immutable->gid_tbl_len);
3558			return -EINVAL;
3559		}
3560	}
3561
3562	if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
3563		if (!(ah_flags & IB_AH_GRH))
3564			return -EINVAL;
3565
3566		ether_addr_copy(MLX5_ADDR_OF(ads, path, rmac_47_32),
3567				ah->roce.dmac);
3568		if ((qp->type == IB_QPT_RC ||
3569		     qp->type == IB_QPT_UC ||
3570		     qp->type == IB_QPT_XRC_INI ||
3571		     qp->type == IB_QPT_XRC_TGT) &&
3572		    (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &&
3573		    (attr_mask & IB_QP_DEST_QPN))
3574			mlx5_set_path_udp_sport(path, ah,
3575						qp->ibqp.qp_num,
3576						attr->dest_qp_num);
3577		MLX5_SET(ads, path, eth_prio, sl & 0x7);
3578		gid_type = ah->grh.sgid_attr->gid_type;
3579		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
3580			MLX5_SET(ads, path, dscp, grh->traffic_class >> 2);
3581	} else {
3582		MLX5_SET(ads, path, fl, !!(path_flags & MLX5_PATH_FLAG_FL));
3583		MLX5_SET(ads, path, free_ar,
3584			 !!(path_flags & MLX5_PATH_FLAG_FREE_AR));
3585		MLX5_SET(ads, path, rlid, rdma_ah_get_dlid(ah));
3586		MLX5_SET(ads, path, mlid, rdma_ah_get_path_bits(ah));
3587		MLX5_SET(ads, path, grh, !!(ah_flags & IB_AH_GRH));
3588		MLX5_SET(ads, path, sl, sl);
3589	}
3590
3591	if (ah_flags & IB_AH_GRH) {
3592		MLX5_SET(ads, path, src_addr_index, grh->sgid_index);
3593		MLX5_SET(ads, path, hop_limit, grh->hop_limit);
3594		MLX5_SET(ads, path, tclass, grh->traffic_class);
3595		MLX5_SET(ads, path, flow_label, grh->flow_label);
3596		memcpy(MLX5_ADDR_OF(ads, path, rgid_rip), grh->dgid.raw,
3597		       sizeof(grh->dgid.raw));
3598	}
3599
3600	err = mlx5r_ib_rate(dev, rdma_ah_get_static_rate(ah));
3601	if (err < 0)
3602		return err;
3603	MLX5_SET(ads, path, stat_rate, err);
3604	MLX5_SET(ads, path, vhca_port_num, port);
3605
3606	if (attr_mask & IB_QP_TIMEOUT)
3607		MLX5_SET(ads, path, ack_timeout,
3608			 alt ? attr->alt_timeout : attr->timeout);
3609
3610	if ((qp->type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
3611		return modify_raw_packet_eth_prio(dev->mdev,
3612						  &qp->raw_packet_qp.sq,
3613						  sl & 0xf, qp->ibqp.pd);
3614
3615	return 0;
3616}
3617
3618static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = {
3619	[MLX5_QP_STATE_INIT] = {
3620		[MLX5_QP_STATE_INIT] = {
3621			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE		|
3622					  MLX5_QP_OPTPAR_RAE		|
3623					  MLX5_QP_OPTPAR_RWE		|
3624					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3625					  MLX5_QP_OPTPAR_PRI_PORT	|
3626					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3627			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE		|
3628					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3629					  MLX5_QP_OPTPAR_PRI_PORT	|
3630					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3631			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX	|
3632					  MLX5_QP_OPTPAR_Q_KEY		|
3633					  MLX5_QP_OPTPAR_PRI_PORT,
3634			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE		|
3635					  MLX5_QP_OPTPAR_RAE		|
3636					  MLX5_QP_OPTPAR_RWE		|
3637					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3638					  MLX5_QP_OPTPAR_PRI_PORT	|
3639					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3640		},
3641		[MLX5_QP_STATE_RTR] = {
3642			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
3643					  MLX5_QP_OPTPAR_RRE            |
3644					  MLX5_QP_OPTPAR_RAE            |
3645					  MLX5_QP_OPTPAR_RWE            |
3646					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3647					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3648			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
3649					  MLX5_QP_OPTPAR_RWE            |
3650					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3651					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3652			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX     |
3653					  MLX5_QP_OPTPAR_Q_KEY,
3654			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX	|
3655					   MLX5_QP_OPTPAR_Q_KEY,
3656			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3657					  MLX5_QP_OPTPAR_RRE            |
3658					  MLX5_QP_OPTPAR_RAE            |
3659					  MLX5_QP_OPTPAR_RWE            |
3660					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3661					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3662		},
3663	},
3664	[MLX5_QP_STATE_RTR] = {
3665		[MLX5_QP_STATE_RTS] = {
3666			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH	|
3667					  MLX5_QP_OPTPAR_RRE		|
3668					  MLX5_QP_OPTPAR_RAE		|
3669					  MLX5_QP_OPTPAR_RWE		|
3670					  MLX5_QP_OPTPAR_PM_STATE	|
3671					  MLX5_QP_OPTPAR_RNR_TIMEOUT,
3672			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH	|
3673					  MLX5_QP_OPTPAR_RWE		|
3674					  MLX5_QP_OPTPAR_PM_STATE,
3675			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3676			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH	|
3677					  MLX5_QP_OPTPAR_RRE		|
3678					  MLX5_QP_OPTPAR_RAE		|
3679					  MLX5_QP_OPTPAR_RWE		|
3680					  MLX5_QP_OPTPAR_PM_STATE	|
3681					  MLX5_QP_OPTPAR_RNR_TIMEOUT,
3682		},
3683	},
3684	[MLX5_QP_STATE_RTS] = {
3685		[MLX5_QP_STATE_RTS] = {
3686			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE		|
3687					  MLX5_QP_OPTPAR_RAE		|
3688					  MLX5_QP_OPTPAR_RWE		|
3689					  MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3690					  MLX5_QP_OPTPAR_PM_STATE	|
3691					  MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3692			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE		|
3693					  MLX5_QP_OPTPAR_PM_STATE	|
3694					  MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3695			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY		|
3696					  MLX5_QP_OPTPAR_SRQN		|
3697					  MLX5_QP_OPTPAR_CQN_RCV,
3698			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE		|
3699					  MLX5_QP_OPTPAR_RAE		|
3700					  MLX5_QP_OPTPAR_RWE		|
3701					  MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3702					  MLX5_QP_OPTPAR_PM_STATE	|
3703					  MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3704		},
3705	},
3706	[MLX5_QP_STATE_SQER] = {
3707		[MLX5_QP_STATE_RTS] = {
3708			[MLX5_QP_ST_UD]	 = MLX5_QP_OPTPAR_Q_KEY,
3709			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
3710			[MLX5_QP_ST_UC]	 = MLX5_QP_OPTPAR_RWE,
3711			[MLX5_QP_ST_RC]	 = MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3712					   MLX5_QP_OPTPAR_RWE		|
3713					   MLX5_QP_OPTPAR_RAE		|
3714					   MLX5_QP_OPTPAR_RRE,
3715			[MLX5_QP_ST_XRC]  = MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3716					   MLX5_QP_OPTPAR_RWE		|
3717					   MLX5_QP_OPTPAR_RAE		|
3718					   MLX5_QP_OPTPAR_RRE,
3719		},
3720	},
3721	[MLX5_QP_STATE_SQD] = {
3722		[MLX5_QP_STATE_RTS] = {
3723			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3724			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
3725			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
3726			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3727					  MLX5_QP_OPTPAR_RWE		|
3728					  MLX5_QP_OPTPAR_RAE		|
3729					  MLX5_QP_OPTPAR_RRE,
3730		},
3731	},
3732};
3733
3734static int ib_nr_to_mlx5_nr(int ib_mask)
3735{
3736	switch (ib_mask) {
3737	case IB_QP_STATE:
3738		return 0;
3739	case IB_QP_CUR_STATE:
3740		return 0;
3741	case IB_QP_EN_SQD_ASYNC_NOTIFY:
3742		return 0;
3743	case IB_QP_ACCESS_FLAGS:
3744		return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE |
3745			MLX5_QP_OPTPAR_RAE;
3746	case IB_QP_PKEY_INDEX:
3747		return MLX5_QP_OPTPAR_PKEY_INDEX;
3748	case IB_QP_PORT:
3749		return MLX5_QP_OPTPAR_PRI_PORT;
3750	case IB_QP_QKEY:
3751		return MLX5_QP_OPTPAR_Q_KEY;
3752	case IB_QP_AV:
3753		return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH |
3754			MLX5_QP_OPTPAR_PRI_PORT;
3755	case IB_QP_PATH_MTU:
3756		return 0;
3757	case IB_QP_TIMEOUT:
3758		return MLX5_QP_OPTPAR_ACK_TIMEOUT;
3759	case IB_QP_RETRY_CNT:
3760		return MLX5_QP_OPTPAR_RETRY_COUNT;
3761	case IB_QP_RNR_RETRY:
3762		return MLX5_QP_OPTPAR_RNR_RETRY;
3763	case IB_QP_RQ_PSN:
3764		return 0;
3765	case IB_QP_MAX_QP_RD_ATOMIC:
3766		return MLX5_QP_OPTPAR_SRA_MAX;
3767	case IB_QP_ALT_PATH:
3768		return MLX5_QP_OPTPAR_ALT_ADDR_PATH;
3769	case IB_QP_MIN_RNR_TIMER:
3770		return MLX5_QP_OPTPAR_RNR_TIMEOUT;
3771	case IB_QP_SQ_PSN:
3772		return 0;
3773	case IB_QP_MAX_DEST_RD_ATOMIC:
3774		return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE |
3775			MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE;
3776	case IB_QP_PATH_MIG_STATE:
3777		return MLX5_QP_OPTPAR_PM_STATE;
3778	case IB_QP_CAP:
3779		return 0;
3780	case IB_QP_DEST_QPN:
3781		return 0;
3782	}
3783	return 0;
3784}
3785
3786static int ib_mask_to_mlx5_opt(int ib_mask)
3787{
3788	int result = 0;
3789	int i;
3790
3791	for (i = 0; i < 8 * sizeof(int); i++) {
3792		if ((1 << i) & ib_mask)
3793			result |= ib_nr_to_mlx5_nr(1 << i);
3794	}
3795
3796	return result;
3797}
3798
3799static int modify_raw_packet_qp_rq(
3800	struct mlx5_ib_dev *dev, struct mlx5_ib_rq *rq, int new_state,
3801	const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
3802{
3803	void *in;
3804	void *rqc;
3805	int inlen;
3806	int err;
3807
3808	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
3809	in = kvzalloc(inlen, GFP_KERNEL);
3810	if (!in)
3811		return -ENOMEM;
3812
3813	MLX5_SET(modify_rq_in, in, rq_state, rq->state);
3814	MLX5_SET(modify_rq_in, in, uid, to_mpd(pd)->uid);
3815
3816	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
3817	MLX5_SET(rqc, rqc, state, new_state);
3818
3819	if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
3820		if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
3821			MLX5_SET64(modify_rq_in, in, modify_bitmask,
3822				   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
3823			MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
3824		} else
3825			dev_info_once(
3826				&dev->ib_dev.dev,
3827				"RAW PACKET QP counters are not supported on current FW\n");
3828	}
3829
3830	err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in);
3831	if (err)
3832		goto out;
3833
3834	rq->state = new_state;
3835
3836out:
3837	kvfree(in);
3838	return err;
3839}
3840
3841static int modify_raw_packet_qp_sq(
3842	struct mlx5_core_dev *dev, struct mlx5_ib_sq *sq, int new_state,
3843	const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
3844{
3845	struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
3846	struct mlx5_rate_limit old_rl = ibqp->rl;
3847	struct mlx5_rate_limit new_rl = old_rl;
3848	bool new_rate_added = false;
3849	u16 rl_index = 0;
3850	void *in;
3851	void *sqc;
3852	int inlen;
3853	int err;
3854
3855	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
3856	in = kvzalloc(inlen, GFP_KERNEL);
3857	if (!in)
3858		return -ENOMEM;
3859
3860	MLX5_SET(modify_sq_in, in, uid, to_mpd(pd)->uid);
3861	MLX5_SET(modify_sq_in, in, sq_state, sq->state);
3862
3863	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
3864	MLX5_SET(sqc, sqc, state, new_state);
3865
3866	if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) {
3867		if (new_state != MLX5_SQC_STATE_RDY)
3868			pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
3869				__func__);
3870		else
3871			new_rl = raw_qp_param->rl;
3872	}
3873
3874	if (!mlx5_rl_are_equal(&old_rl, &new_rl)) {
3875		if (new_rl.rate) {
3876			err = mlx5_rl_add_rate(dev, &rl_index, &new_rl);
3877			if (err) {
3878				pr_err("Failed configuring rate limit(err %d): \
3879				       rate %u, max_burst_sz %u, typical_pkt_sz %u\n",
3880				       err, new_rl.rate, new_rl.max_burst_sz,
3881				       new_rl.typical_pkt_sz);
3882
3883				goto out;
3884			}
3885			new_rate_added = true;
3886		}
3887
3888		MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
3889		/* index 0 means no limit */
3890		MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
3891	}
3892
3893	err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in);
3894	if (err) {
3895		/* Remove new rate from table if failed */
3896		if (new_rate_added)
3897			mlx5_rl_remove_rate(dev, &new_rl);
3898		goto out;
3899	}
3900
3901	/* Only remove the old rate after new rate was set */
3902	if ((old_rl.rate && !mlx5_rl_are_equal(&old_rl, &new_rl)) ||
3903	    (new_state != MLX5_SQC_STATE_RDY)) {
3904		mlx5_rl_remove_rate(dev, &old_rl);
3905		if (new_state != MLX5_SQC_STATE_RDY)
3906			memset(&new_rl, 0, sizeof(new_rl));
3907	}
3908
3909	ibqp->rl = new_rl;
3910	sq->state = new_state;
3911
3912out:
3913	kvfree(in);
3914	return err;
3915}
3916
3917static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3918				const struct mlx5_modify_raw_qp_param *raw_qp_param,
3919				u8 tx_affinity)
3920{
3921	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
3922	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
3923	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
3924	int modify_rq = !!qp->rq.wqe_cnt;
3925	int modify_sq = !!qp->sq.wqe_cnt;
3926	int rq_state;
3927	int sq_state;
3928	int err;
3929
3930	switch (raw_qp_param->operation) {
3931	case MLX5_CMD_OP_RST2INIT_QP:
3932		rq_state = MLX5_RQC_STATE_RDY;
3933		sq_state = MLX5_SQC_STATE_RST;
3934		break;
3935	case MLX5_CMD_OP_2ERR_QP:
3936		rq_state = MLX5_RQC_STATE_ERR;
3937		sq_state = MLX5_SQC_STATE_ERR;
3938		break;
3939	case MLX5_CMD_OP_2RST_QP:
3940		rq_state = MLX5_RQC_STATE_RST;
3941		sq_state = MLX5_SQC_STATE_RST;
3942		break;
3943	case MLX5_CMD_OP_RTR2RTS_QP:
3944	case MLX5_CMD_OP_RTS2RTS_QP:
3945		if (raw_qp_param->set_mask & ~MLX5_RAW_QP_RATE_LIMIT)
3946			return -EINVAL;
3947
3948		modify_rq = 0;
3949		sq_state = MLX5_SQC_STATE_RDY;
3950		break;
3951	case MLX5_CMD_OP_INIT2INIT_QP:
3952	case MLX5_CMD_OP_INIT2RTR_QP:
3953		if (raw_qp_param->set_mask)
3954			return -EINVAL;
3955		else
3956			return 0;
3957	default:
3958		WARN_ON(1);
3959		return -EINVAL;
3960	}
3961
3962	if (modify_rq) {
3963		err =  modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param,
3964					       qp->ibqp.pd);
3965		if (err)
3966			return err;
3967	}
3968
3969	if (modify_sq) {
3970		struct mlx5_flow_handle *flow_rule;
3971
3972		if (tx_affinity) {
3973			err = modify_raw_packet_tx_affinity(dev->mdev, sq,
3974							    tx_affinity,
3975							    qp->ibqp.pd);
3976			if (err)
3977				return err;
3978		}
3979
3980		flow_rule = create_flow_rule_vport_sq(dev, sq,
3981						      raw_qp_param->port);
3982		if (IS_ERR(flow_rule))
3983			return PTR_ERR(flow_rule);
3984
3985		err = modify_raw_packet_qp_sq(dev->mdev, sq, sq_state,
3986					      raw_qp_param, qp->ibqp.pd);
3987		if (err) {
3988			if (flow_rule)
3989				mlx5_del_flow_rules(flow_rule);
3990			return err;
3991		}
3992
3993		if (flow_rule) {
3994			destroy_flow_rule_vport_sq(sq);
3995			sq->flow_rule = flow_rule;
3996		}
3997
3998		return err;
3999	}
4000
4001	return 0;
4002}
4003
4004static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,
4005				       struct ib_udata *udata)
4006{
4007	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
4008		udata, struct mlx5_ib_ucontext, ibucontext);
4009	u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4010	atomic_t *tx_port_affinity;
4011
4012	if (ucontext)
4013		tx_port_affinity = &ucontext->tx_port_affinity;
4014	else
4015		tx_port_affinity = &dev->port[port_num].roce.tx_port_affinity;
4016
4017	return (unsigned int)atomic_add_return(1, tx_port_affinity) %
4018		(dev->lag_active ? dev->lag_ports : MLX5_CAP_GEN(dev->mdev, num_lag_ports)) + 1;
4019}
4020
4021static bool qp_supports_affinity(struct mlx5_ib_qp *qp)
4022{
4023	if ((qp->type == IB_QPT_RC) || (qp->type == IB_QPT_UD) ||
4024	    (qp->type == IB_QPT_UC) || (qp->type == IB_QPT_RAW_PACKET) ||
4025	    (qp->type == IB_QPT_XRC_INI) || (qp->type == IB_QPT_XRC_TGT) ||
4026	    (qp->type == MLX5_IB_QPT_DCI))
4027		return true;
4028	return false;
4029}
4030
4031static unsigned int get_tx_affinity(struct ib_qp *qp,
4032				    const struct ib_qp_attr *attr,
4033				    int attr_mask, u8 init,
4034				    struct ib_udata *udata)
4035{
4036	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
4037		udata, struct mlx5_ib_ucontext, ibucontext);
4038	struct mlx5_ib_dev *dev = to_mdev(qp->device);
4039	struct mlx5_ib_qp *mqp = to_mqp(qp);
4040	struct mlx5_ib_qp_base *qp_base;
4041	unsigned int tx_affinity;
4042
4043	if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
4044	      qp_supports_affinity(mqp)))
4045		return 0;
4046
4047	if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
4048		tx_affinity = mqp->gsi_lag_port;
4049	else if (init)
4050		tx_affinity = get_tx_affinity_rr(dev, udata);
4051	else if ((attr_mask & IB_QP_AV) && attr->xmit_slave)
4052		tx_affinity =
4053			mlx5_lag_get_slave_port(dev->mdev, attr->xmit_slave);
4054	else
4055		return 0;
4056
4057	qp_base = &mqp->trans_qp.base;
4058	if (ucontext)
4059		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x ucontext %p\n",
4060			    tx_affinity, qp_base->mqp.qpn, ucontext);
4061	else
4062		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",
4063			    tx_affinity, qp_base->mqp.qpn);
4064	return tx_affinity;
4065}
4066
4067static int __mlx5_ib_qp_set_raw_qp_counter(struct mlx5_ib_qp *qp, u32 set_id,
4068					   struct mlx5_core_dev *mdev)
4069{
4070	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4071	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4072	u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};
4073	void *rqc;
4074
4075	if (!qp->rq.wqe_cnt)
4076		return 0;
4077
4078	MLX5_SET(modify_rq_in, in, rq_state, rq->state);
4079	MLX5_SET(modify_rq_in, in, uid, to_mpd(qp->ibqp.pd)->uid);
4080
4081	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
4082	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
4083
4084	MLX5_SET64(modify_rq_in, in, modify_bitmask,
4085		   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
4086	MLX5_SET(rqc, rqc, counter_set_id, set_id);
4087
4088	return mlx5_core_modify_rq(mdev, rq->base.mqp.qpn, in);
4089}
4090
4091static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
4092				    struct rdma_counter *counter)
4093{
4094	struct mlx5_ib_dev *dev = to_mdev(qp->device);
4095	u32 in[MLX5_ST_SZ_DW(rts2rts_qp_in)] = {};
4096	struct mlx5_ib_qp *mqp = to_mqp(qp);
4097	struct mlx5_ib_qp_base *base;
4098	u32 set_id;
4099	u32 *qpc;
4100
4101	if (counter)
4102		set_id = counter->id;
4103	else
4104		set_id = mlx5_ib_get_counters_id(dev, mqp->port - 1);
4105
4106	if (mqp->type == IB_QPT_RAW_PACKET)
4107		return __mlx5_ib_qp_set_raw_qp_counter(mqp, set_id, dev->mdev);
4108
4109	base = &mqp->trans_qp.base;
4110	MLX5_SET(rts2rts_qp_in, in, opcode, MLX5_CMD_OP_RTS2RTS_QP);
4111	MLX5_SET(rts2rts_qp_in, in, qpn, base->mqp.qpn);
4112	MLX5_SET(rts2rts_qp_in, in, uid, base->mqp.uid);
4113	MLX5_SET(rts2rts_qp_in, in, opt_param_mask,
4114		 MLX5_QP_OPTPAR_COUNTER_SET_ID);
4115
4116	qpc = MLX5_ADDR_OF(rts2rts_qp_in, in, qpc);
4117	MLX5_SET(qpc, qpc, counter_set_id, set_id);
4118	return mlx5_cmd_exec_in(dev->mdev, rts2rts_qp, in);
4119}
4120
4121static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
4122			       const struct ib_qp_attr *attr, int attr_mask,
4123			       enum ib_qp_state cur_state,
4124			       enum ib_qp_state new_state,
4125			       const struct mlx5_ib_modify_qp *ucmd,
4126			       struct mlx5_ib_modify_qp_resp *resp,
4127			       struct ib_udata *udata)
4128{
4129	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
4130		[MLX5_QP_STATE_RST] = {
4131			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4132			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4133			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_RST2INIT_QP,
4134		},
4135		[MLX5_QP_STATE_INIT]  = {
4136			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4137			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4138			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_INIT2INIT_QP,
4139			[MLX5_QP_STATE_RTR]	= MLX5_CMD_OP_INIT2RTR_QP,
4140		},
4141		[MLX5_QP_STATE_RTR]   = {
4142			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4143			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4144			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTR2RTS_QP,
4145		},
4146		[MLX5_QP_STATE_RTS]   = {
4147			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4148			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4149			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTS2RTS_QP,
4150		},
4151		[MLX5_QP_STATE_SQD] = {
4152			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4153			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4154			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQD_RTS_QP,
4155		},
4156		[MLX5_QP_STATE_SQER] = {
4157			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4158			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4159			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQERR2RTS_QP,
4160		},
4161		[MLX5_QP_STATE_ERR] = {
4162			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4163			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4164		}
4165	};
4166
4167	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4168	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4169	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
4170	struct mlx5_ib_cq *send_cq, *recv_cq;
4171	struct mlx5_ib_pd *pd;
4172	enum mlx5_qp_state mlx5_cur, mlx5_new;
4173	void *qpc, *pri_path, *alt_path;
4174	enum mlx5_qp_optpar optpar = 0;
4175	u32 set_id = 0;
4176	int mlx5_st;
4177	int err;
4178	u16 op;
4179	u8 tx_affinity = 0;
4180
4181	mlx5_st = to_mlx5_st(qp->type);
4182	if (mlx5_st < 0)
4183		return -EINVAL;
4184
4185	qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
4186	if (!qpc)
4187		return -ENOMEM;
4188
4189	pd = to_mpd(qp->ibqp.pd);
4190	MLX5_SET(qpc, qpc, st, mlx5_st);
4191
4192	if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
4193		MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
4194	} else {
4195		switch (attr->path_mig_state) {
4196		case IB_MIG_MIGRATED:
4197			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
4198			break;
4199		case IB_MIG_REARM:
4200			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_REARM);
4201			break;
4202		case IB_MIG_ARMED:
4203			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_ARMED);
4204			break;
4205		}
4206	}
4207
4208	tx_affinity = get_tx_affinity(ibqp, attr, attr_mask,
4209				      cur_state == IB_QPS_RESET &&
4210				      new_state == IB_QPS_INIT, udata);
4211
4212	MLX5_SET(qpc, qpc, lag_tx_port_affinity, tx_affinity);
4213	if (tx_affinity && new_state == IB_QPS_RTR &&
4214	    MLX5_CAP_GEN(dev->mdev, init2_lag_tx_port_affinity))
4215		optpar |= MLX5_QP_OPTPAR_LAG_TX_AFF;
4216
4217	if (is_sqp(qp->type)) {
4218		MLX5_SET(qpc, qpc, mtu, IB_MTU_256);
4219		MLX5_SET(qpc, qpc, log_msg_max, 8);
4220	} else if ((qp->type == IB_QPT_UD &&
4221		    !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) ||
4222		   qp->type == MLX5_IB_QPT_REG_UMR) {
4223		MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);
4224		MLX5_SET(qpc, qpc, log_msg_max, 12);
4225	} else if (attr_mask & IB_QP_PATH_MTU) {
4226		if (attr->path_mtu < IB_MTU_256 ||
4227		    attr->path_mtu > IB_MTU_4096) {
4228			mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu);
4229			err = -EINVAL;
4230			goto out;
4231		}
4232		MLX5_SET(qpc, qpc, mtu, attr->path_mtu);
4233		MLX5_SET(qpc, qpc, log_msg_max,
4234			 MLX5_CAP_GEN(dev->mdev, log_max_msg));
4235	}
4236
4237	if (attr_mask & IB_QP_DEST_QPN)
4238		MLX5_SET(qpc, qpc, remote_qpn, attr->dest_qp_num);
4239
4240	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
4241	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
4242
4243	if (attr_mask & IB_QP_PKEY_INDEX)
4244		MLX5_SET(ads, pri_path, pkey_index, attr->pkey_index);
4245
4246	/* todo implement counter_index functionality */
4247
4248	if (dev->ib_dev.type == RDMA_DEVICE_TYPE_SMI && is_qp0(qp->type)) {
4249		MLX5_SET(ads, pri_path, vhca_port_num,
4250			 smi_to_native_portnum(dev, qp->port));
4251		if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR)
4252			MLX5_SET(ads, pri_path, plane_index, qp->port);
4253	} else if (is_sqp(qp->type))
4254		MLX5_SET(ads, pri_path, vhca_port_num, qp->port);
4255
4256	if (attr_mask & IB_QP_PORT)
4257		MLX5_SET(ads, pri_path, vhca_port_num, attr->port_num);
4258
4259	if (attr_mask & IB_QP_AV) {
4260		err = mlx5_set_path(dev, qp, &attr->ah_attr, pri_path,
4261				    attr_mask & IB_QP_PORT ? attr->port_num :
4262							     qp->port,
4263				    attr_mask, 0, attr, false);
4264		if (err)
4265			goto out;
4266	}
4267
4268	if (attr_mask & IB_QP_TIMEOUT)
4269		MLX5_SET(ads, pri_path, ack_timeout, attr->timeout);
4270
4271	if (attr_mask & IB_QP_ALT_PATH) {
4272		err = mlx5_set_path(dev, qp, &attr->alt_ah_attr, alt_path,
4273				    attr->alt_port_num,
4274				    attr_mask | IB_QP_PKEY_INDEX |
4275					    IB_QP_TIMEOUT,
4276				    0, attr, true);
4277		if (err)
4278			goto out;
4279	}
4280
4281	get_cqs(qp->type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
4282		&send_cq, &recv_cq);
4283
4284	MLX5_SET(qpc, qpc, pd, pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
4285	if (send_cq)
4286		MLX5_SET(qpc, qpc, cqn_snd, send_cq->mcq.cqn);
4287	if (recv_cq)
4288		MLX5_SET(qpc, qpc, cqn_rcv, recv_cq->mcq.cqn);
4289
4290	MLX5_SET(qpc, qpc, log_ack_req_freq, MLX5_IB_ACK_REQ_FREQ);
4291
4292	if (attr_mask & IB_QP_RNR_RETRY)
4293		MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry);
4294
4295	if (attr_mask & IB_QP_RETRY_CNT)
4296		MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt);
4297
4298	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic)
4299		MLX5_SET(qpc, qpc, log_sra_max, fls(attr->max_rd_atomic - 1));
4300
4301	if (attr_mask & IB_QP_SQ_PSN)
4302		MLX5_SET(qpc, qpc, next_send_psn, attr->sq_psn);
4303
4304	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic)
4305		MLX5_SET(qpc, qpc, log_rra_max,
4306			 fls(attr->max_dest_rd_atomic - 1));
4307
4308	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
4309		err = set_qpc_atomic_flags(qp, attr, attr_mask, qpc);
4310		if (err)
4311			goto out;
4312	}
4313
4314	if (attr_mask & IB_QP_MIN_RNR_TIMER)
4315		MLX5_SET(qpc, qpc, min_rnr_nak, attr->min_rnr_timer);
4316
4317	if (attr_mask & IB_QP_RQ_PSN)
4318		MLX5_SET(qpc, qpc, next_rcv_psn, attr->rq_psn);
4319
4320	if (attr_mask & IB_QP_QKEY)
4321		MLX5_SET(qpc, qpc, q_key, attr->qkey);
4322
4323	if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
4324		MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
4325
4326	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4327		u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
4328			       qp->port) - 1;
4329
4330		/* Underlay port should be used - index 0 function per port */
4331		if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
4332			port_num = 0;
4333
4334		if (ibqp->counter)
4335			set_id = ibqp->counter->id;
4336		else
4337			set_id = mlx5_ib_get_counters_id(dev, port_num);
4338		MLX5_SET(qpc, qpc, counter_set_id, set_id);
4339	}
4340
4341	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
4342		MLX5_SET(qpc, qpc, rlky, 1);
4343
4344	if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
4345		MLX5_SET(qpc, qpc, deth_sqpn, 1);
4346
4347	if (qp->is_ooo_rq && cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4348		MLX5_SET(qpc, qpc, dp_ordering_1, 1);
4349		MLX5_SET(qpc, qpc, dp_ordering_force, 1);
4350	}
4351
4352	mlx5_cur = to_mlx5_state(cur_state);
4353	mlx5_new = to_mlx5_state(new_state);
4354
4355	if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
4356	    !optab[mlx5_cur][mlx5_new]) {
4357		err = -EINVAL;
4358		goto out;
4359	}
4360
4361	op = optab[mlx5_cur][mlx5_new];
4362	optpar |= ib_mask_to_mlx5_opt(attr_mask);
4363	optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
4364
4365	if (qp->type == IB_QPT_RAW_PACKET ||
4366	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
4367		struct mlx5_modify_raw_qp_param raw_qp_param = {};
4368
4369		raw_qp_param.operation = op;
4370		if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4371			raw_qp_param.rq_q_ctr_id = set_id;
4372			raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
4373		}
4374
4375		if (attr_mask & IB_QP_PORT)
4376			raw_qp_param.port = attr->port_num;
4377
4378		if (attr_mask & IB_QP_RATE_LIMIT) {
4379			raw_qp_param.rl.rate = attr->rate_limit;
4380
4381			if (ucmd->burst_info.max_burst_sz) {
4382				if (attr->rate_limit &&
4383				    MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) {
4384					raw_qp_param.rl.max_burst_sz =
4385						ucmd->burst_info.max_burst_sz;
4386				} else {
4387					err = -EINVAL;
4388					goto out;
4389				}
4390			}
4391
4392			if (ucmd->burst_info.typical_pkt_sz) {
4393				if (attr->rate_limit &&
4394				    MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) {
4395					raw_qp_param.rl.typical_pkt_sz =
4396						ucmd->burst_info.typical_pkt_sz;
4397				} else {
4398					err = -EINVAL;
4399					goto out;
4400				}
4401			}
4402
4403			raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
4404		}
4405
4406		err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
4407	} else {
4408		if (udata) {
4409			/* For the kernel flows, the resp will stay zero */
4410			resp->ece_options =
4411				MLX5_CAP_GEN(dev->mdev, ece_support) ?
4412					ucmd->ece_options : 0;
4413			resp->response_length = sizeof(*resp);
4414		}
4415		err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp,
4416					  &resp->ece_options);
4417	}
4418
4419	if (err)
4420		goto out;
4421
4422	qp->state = new_state;
4423
4424	if (attr_mask & IB_QP_ACCESS_FLAGS)
4425		qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
4426	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4427		qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
4428	if (attr_mask & IB_QP_PORT)
4429		qp->port = attr->port_num;
4430	if (attr_mask & IB_QP_ALT_PATH)
4431		qp->trans_qp.alt_port = attr->alt_port_num;
4432
4433	/*
4434	 * If we moved a kernel QP to RESET, clean up all old CQ
4435	 * entries and reinitialize the QP.
4436	 */
4437	if (new_state == IB_QPS_RESET &&
4438	    !ibqp->uobject && qp->type != IB_QPT_XRC_TGT) {
4439		mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
4440				 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
4441		if (send_cq != recv_cq)
4442			mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL);
4443
4444		qp->rq.head = 0;
4445		qp->rq.tail = 0;
4446		qp->sq.head = 0;
4447		qp->sq.tail = 0;
4448		qp->sq.cur_post = 0;
4449		if (qp->sq.wqe_cnt)
4450			qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
4451		qp->sq.last_poll = 0;
4452		qp->db.db[MLX5_RCV_DBR] = 0;
4453		qp->db.db[MLX5_SND_DBR] = 0;
4454	}
4455
4456	if ((new_state == IB_QPS_RTS) && qp->counter_pending) {
4457		err = __mlx5_ib_qp_set_counter(ibqp, ibqp->counter);
4458		if (!err)
4459			qp->counter_pending = 0;
4460	}
4461
4462out:
4463	kfree(qpc);
4464	return err;
4465}
4466
4467static inline bool is_valid_mask(int mask, int req, int opt)
4468{
4469	if ((mask & req) != req)
4470		return false;
4471
4472	if (mask & ~(req | opt))
4473		return false;
4474
4475	return true;
4476}
4477
4478/* check valid transition for driver QP types
4479 * for now the only QP type that this function supports is DCI
4480 */
4481static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
4482				enum ib_qp_attr_mask attr_mask)
4483{
4484	int req = IB_QP_STATE;
4485	int opt = 0;
4486
4487	if (new_state == IB_QPS_RESET) {
4488		return is_valid_mask(attr_mask, req, opt);
4489	} else if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4490		req |= IB_QP_PKEY_INDEX | IB_QP_PORT;
4491		return is_valid_mask(attr_mask, req, opt);
4492	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4493		opt = IB_QP_PKEY_INDEX | IB_QP_PORT;
4494		return is_valid_mask(attr_mask, req, opt);
4495	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4496		req |= IB_QP_PATH_MTU;
4497		opt = IB_QP_PKEY_INDEX | IB_QP_AV;
4498		return is_valid_mask(attr_mask, req, opt);
4499	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4500		req |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4501		       IB_QP_MAX_QP_RD_ATOMIC | IB_QP_SQ_PSN;
4502		opt = IB_QP_MIN_RNR_TIMER;
4503		return is_valid_mask(attr_mask, req, opt);
4504	} else if (cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) {
4505		opt = IB_QP_MIN_RNR_TIMER;
4506		return is_valid_mask(attr_mask, req, opt);
4507	} else if (cur_state != IB_QPS_RESET && new_state == IB_QPS_ERR) {
4508		return is_valid_mask(attr_mask, req, opt);
4509	}
4510	return false;
4511}
4512
4513/* mlx5_ib_modify_dct: modify a DCT QP
4514 * valid transitions are:
4515 * RESET to INIT: must set access_flags, pkey_index and port
4516 * INIT  to RTR : must set min_rnr_timer, tclass, flow_label,
4517 *			   mtu, gid_index and hop_limit
4518 * Other transitions and attributes are illegal
4519 */
4520static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4521			      int attr_mask, struct mlx5_ib_modify_qp *ucmd,
4522			      struct ib_udata *udata)
4523{
4524	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4525	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4526	enum ib_qp_state cur_state, new_state;
4527	int required = IB_QP_STATE;
4528	void *dctc;
4529	int err;
4530
4531	if (!(attr_mask & IB_QP_STATE))
4532		return -EINVAL;
4533
4534	cur_state = qp->state;
4535	new_state = attr->qp_state;
4536
4537	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
4538	if (MLX5_CAP_GEN(dev->mdev, ece_support) && ucmd->ece_options)
4539		/*
4540		 * DCT doesn't initialize QP till modify command is executed,
4541		 * so we need to overwrite previously set ECE field if user
4542		 * provided any value except zero, which means not set/not
4543		 * valid.
4544		 */
4545		MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
4546
4547	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4548		u16 set_id;
4549
4550		required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;
4551		if (!is_valid_mask(attr_mask, required, 0))
4552			return -EINVAL;
4553
4554		if (attr->port_num == 0 ||
4555		    attr->port_num > dev->num_ports) {
4556			mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4557				    attr->port_num, dev->num_ports);
4558			return -EINVAL;
4559		}
4560		if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
4561			MLX5_SET(dctc, dctc, rre, 1);
4562		if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
4563			MLX5_SET(dctc, dctc, rwe, 1);
4564		if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
4565			int atomic_mode;
4566
4567			atomic_mode = get_atomic_mode(dev, qp);
4568			if (atomic_mode < 0)
4569				return -EOPNOTSUPP;
4570
4571			MLX5_SET(dctc, dctc, atomic_mode, atomic_mode);
4572			MLX5_SET(dctc, dctc, rae, 1);
4573		}
4574		MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
4575		if (mlx5_lag_is_active(dev->mdev))
4576			MLX5_SET(dctc, dctc, port,
4577				 get_tx_affinity_rr(dev, udata));
4578		else
4579			MLX5_SET(dctc, dctc, port, attr->port_num);
4580
4581		set_id = mlx5_ib_get_counters_id(dev, attr->port_num - 1);
4582		MLX5_SET(dctc, dctc, counter_set_id, set_id);
4583
4584		qp->port = attr->port_num;
4585	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4586		struct mlx5_ib_modify_qp_resp resp = {};
4587		u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {};
4588		u32 min_resp_len = offsetofend(typeof(resp), dctn);
4589
4590		if (udata->outlen < min_resp_len)
4591			return -EINVAL;
4592		/*
4593		 * If we don't have enough space for the ECE options,
4594		 * simply indicate it with resp.response_length.
4595		 */
4596		resp.response_length = (udata->outlen < sizeof(resp)) ?
4597					       min_resp_len :
4598					       sizeof(resp);
4599
4600		required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
4601		if (!is_valid_mask(attr_mask, required, 0))
4602			return -EINVAL;
4603		MLX5_SET(dctc, dctc, min_rnr_nak, attr->min_rnr_timer);
4604		MLX5_SET(dctc, dctc, tclass, attr->ah_attr.grh.traffic_class);
4605		MLX5_SET(dctc, dctc, flow_label, attr->ah_attr.grh.flow_label);
4606		MLX5_SET(dctc, dctc, mtu, attr->path_mtu);
4607		MLX5_SET(dctc, dctc, my_addr_index, attr->ah_attr.grh.sgid_index);
4608		MLX5_SET(dctc, dctc, hop_limit, attr->ah_attr.grh.hop_limit);
4609		if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
4610			MLX5_SET(dctc, dctc, eth_prio, attr->ah_attr.sl & 0x7);
4611		if (qp->is_ooo_rq) {
4612			MLX5_SET(dctc, dctc, dp_ordering_1, 1);
4613			MLX5_SET(dctc, dctc, dp_ordering_force, 1);
4614		}
4615
4616		err = mlx5_core_create_dct(dev, &qp->dct.mdct, qp->dct.in,
4617					   MLX5_ST_SZ_BYTES(create_dct_in), out,
4618					   sizeof(out));
4619		err = mlx5_cmd_check(dev->mdev, err, qp->dct.in, out);
4620		if (err)
4621			return err;
4622		resp.dctn = qp->dct.mdct.mqp.qpn;
4623		if (MLX5_CAP_GEN(dev->mdev, ece_support))
4624			resp.ece_options = MLX5_GET(create_dct_out, out, ece);
4625		err = ib_copy_to_udata(udata, &resp, resp.response_length);
4626		if (err) {
4627			mlx5_core_destroy_dct(dev, &qp->dct.mdct);
4628			return err;
4629		}
4630	} else {
4631		mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
4632		return -EINVAL;
4633	}
4634
4635	qp->state = new_state;
4636	return 0;
4637}
4638
4639static bool mlx5_ib_modify_qp_allowed(struct mlx5_ib_dev *dev,
4640				      struct mlx5_ib_qp *qp)
4641{
4642	if (dev->profile != &raw_eth_profile)
4643		return true;
4644
4645	if (qp->type == IB_QPT_RAW_PACKET || qp->type == MLX5_IB_QPT_REG_UMR)
4646		return true;
4647
4648	return false;
4649}
4650
4651static int validate_rd_atomic(struct mlx5_ib_dev *dev, struct ib_qp_attr *attr,
4652			      int attr_mask, enum ib_qp_type qp_type)
4653{
4654	int log_max_ra_res;
4655	int log_max_ra_req;
4656
4657	if (qp_type == MLX5_IB_QPT_DCI) {
4658		log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
4659						   log_max_ra_res_dc);
4660		log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
4661						   log_max_ra_req_dc);
4662	} else {
4663		log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
4664						   log_max_ra_res_qp);
4665		log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
4666						   log_max_ra_req_qp);
4667	}
4668
4669	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
4670	    attr->max_rd_atomic > log_max_ra_res) {
4671		mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
4672			    attr->max_rd_atomic);
4673		return false;
4674	}
4675
4676	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
4677	    attr->max_dest_rd_atomic > log_max_ra_req) {
4678		mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
4679			    attr->max_dest_rd_atomic);
4680		return false;
4681	}
4682	return true;
4683}
4684
4685int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4686		      int attr_mask, struct ib_udata *udata)
4687{
4688	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4689	struct mlx5_ib_modify_qp_resp resp = {};
4690	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4691	struct mlx5_ib_modify_qp ucmd = {};
4692	enum ib_qp_type qp_type;
4693	enum ib_qp_state cur_state, new_state;
4694	int err = -EINVAL;
4695
4696	if (!mlx5_ib_modify_qp_allowed(dev, qp))
4697		return -EOPNOTSUPP;
4698
4699	if (attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
4700		return -EOPNOTSUPP;
4701
4702	if (ibqp->rwq_ind_tbl)
4703		return -ENOSYS;
4704
4705	if (udata && udata->inlen) {
4706		if (udata->inlen < offsetofend(typeof(ucmd), ece_options))
4707			return -EINVAL;
4708
4709		if (udata->inlen > sizeof(ucmd) &&
4710		    !ib_is_udata_cleared(udata, sizeof(ucmd),
4711					 udata->inlen - sizeof(ucmd)))
4712			return -EOPNOTSUPP;
4713
4714		if (ib_copy_from_udata(&ucmd, udata,
4715				       min(udata->inlen, sizeof(ucmd))))
4716			return -EFAULT;
4717
4718		if (ucmd.comp_mask & ~MLX5_IB_MODIFY_QP_OOO_DP ||
4719		    memchr_inv(&ucmd.burst_info.reserved, 0,
4720			       sizeof(ucmd.burst_info.reserved)))
4721			return -EOPNOTSUPP;
4722
4723		if (ucmd.comp_mask & MLX5_IB_MODIFY_QP_OOO_DP) {
4724			if (!get_dp_ooo_cap(dev->mdev, qp->type))
4725				return -EOPNOTSUPP;
4726			qp->is_ooo_rq = 1;
4727		}
4728	}
4729
4730	if (qp->type == IB_QPT_GSI)
4731		return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
4732
4733	qp_type = (qp->type == MLX5_IB_QPT_HW_GSI) ? IB_QPT_GSI : qp->type;
4734
4735	if (qp_type == MLX5_IB_QPT_DCT)
4736		return mlx5_ib_modify_dct(ibqp, attr, attr_mask, &ucmd, udata);
4737
4738	mutex_lock(&qp->mutex);
4739
4740	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
4741	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
4742
4743	if (qp->flags & IB_QP_CREATE_SOURCE_QPN) {
4744		if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
4745			mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
4746				    attr_mask);
4747			goto out;
4748		}
4749	} else if (qp_type != MLX5_IB_QPT_REG_UMR &&
4750		   qp_type != MLX5_IB_QPT_DCI &&
4751		   !ib_modify_qp_is_ok(cur_state, new_state, qp_type,
4752				       attr_mask)) {
4753		mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4754			    cur_state, new_state, qp->type, attr_mask);
4755		goto out;
4756	} else if (qp_type == MLX5_IB_QPT_DCI &&
4757		   !modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {
4758		mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4759			    cur_state, new_state, qp_type, attr_mask);
4760		goto out;
4761	}
4762
4763	if ((attr_mask & IB_QP_PORT) &&
4764	    (attr->port_num == 0 ||
4765	     attr->port_num > dev->num_ports)) {
4766		mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4767			    attr->port_num, dev->num_ports);
4768		goto out;
4769	}
4770
4771	if ((attr_mask & IB_QP_PKEY_INDEX) &&
4772	    attr->pkey_index >= dev->pkey_table_len) {
4773		mlx5_ib_dbg(dev, "invalid pkey index %d\n", attr->pkey_index);
4774		goto out;
4775	}
4776
4777	if (!validate_rd_atomic(dev, attr, attr_mask, qp_type))
4778		goto out;
4779
4780	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
4781		err = 0;
4782		goto out;
4783	}
4784
4785	err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,
4786				  new_state, &ucmd, &resp, udata);
4787
4788	/* resp.response_length is set in ECE supported flows only */
4789	if (!err && resp.response_length &&
4790	    udata->outlen >= resp.response_length)
4791		/* Return -EFAULT to the user and expect him to destroy QP. */
4792		err = ib_copy_to_udata(udata, &resp, resp.response_length);
4793
4794out:
4795	mutex_unlock(&qp->mutex);
4796	return err;
4797}
4798
4799static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
4800{
4801	switch (mlx5_state) {
4802	case MLX5_QP_STATE_RST:      return IB_QPS_RESET;
4803	case MLX5_QP_STATE_INIT:     return IB_QPS_INIT;
4804	case MLX5_QP_STATE_RTR:      return IB_QPS_RTR;
4805	case MLX5_QP_STATE_RTS:      return IB_QPS_RTS;
4806	case MLX5_QP_STATE_SQ_DRAINING:
4807	case MLX5_QP_STATE_SQD:      return IB_QPS_SQD;
4808	case MLX5_QP_STATE_SQER:     return IB_QPS_SQE;
4809	case MLX5_QP_STATE_ERR:      return IB_QPS_ERR;
4810	default:		     return -1;
4811	}
4812}
4813
4814static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
4815{
4816	switch (mlx5_mig_state) {
4817	case MLX5_QP_PM_ARMED:		return IB_MIG_ARMED;
4818	case MLX5_QP_PM_REARM:		return IB_MIG_REARM;
4819	case MLX5_QP_PM_MIGRATED:	return IB_MIG_MIGRATED;
4820	default: return -1;
4821	}
4822}
4823
4824static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
4825			    struct rdma_ah_attr *ah_attr, void *path)
4826{
4827	int port = MLX5_GET(ads, path, vhca_port_num);
4828	int static_rate;
4829
4830	memset(ah_attr, 0, sizeof(*ah_attr));
4831
4832	if (!port || port > ibdev->num_ports)
4833		return;
4834
4835	ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port);
4836
4837	rdma_ah_set_port_num(ah_attr, port);
4838	rdma_ah_set_sl(ah_attr, MLX5_GET(ads, path, sl));
4839
4840	rdma_ah_set_dlid(ah_attr, MLX5_GET(ads, path, rlid));
4841	rdma_ah_set_path_bits(ah_attr, MLX5_GET(ads, path, mlid));
4842
4843	static_rate = MLX5_GET(ads, path, stat_rate);
4844	rdma_ah_set_static_rate(ah_attr, mlx5_to_ib_rate_map(static_rate));
4845	if (MLX5_GET(ads, path, grh) ||
4846	    ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
4847		rdma_ah_set_grh(ah_attr, NULL, MLX5_GET(ads, path, flow_label),
4848				MLX5_GET(ads, path, src_addr_index),
4849				MLX5_GET(ads, path, hop_limit),
4850				MLX5_GET(ads, path, tclass));
4851		rdma_ah_set_dgid_raw(ah_attr, MLX5_ADDR_OF(ads, path, rgid_rip));
4852	}
4853}
4854
4855static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
4856					struct mlx5_ib_sq *sq,
4857					u8 *sq_state)
4858{
4859	int err;
4860
4861	err = mlx5_core_query_sq_state(dev->mdev, sq->base.mqp.qpn, sq_state);
4862	if (err)
4863		goto out;
4864	sq->state = *sq_state;
4865
4866out:
4867	return err;
4868}
4869
4870static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
4871					struct mlx5_ib_rq *rq,
4872					u8 *rq_state)
4873{
4874	void *out;
4875	void *rqc;
4876	int inlen;
4877	int err;
4878
4879	inlen = MLX5_ST_SZ_BYTES(query_rq_out);
4880	out = kvzalloc(inlen, GFP_KERNEL);
4881	if (!out)
4882		return -ENOMEM;
4883
4884	err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out);
4885	if (err)
4886		goto out;
4887
4888	rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
4889	*rq_state = MLX5_GET(rqc, rqc, state);
4890	rq->state = *rq_state;
4891
4892out:
4893	kvfree(out);
4894	return err;
4895}
4896
4897static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
4898				  struct mlx5_ib_qp *qp, u8 *qp_state)
4899{
4900	static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
4901		[MLX5_RQC_STATE_RST] = {
4902			[MLX5_SQC_STATE_RST]	= IB_QPS_RESET,
4903			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE_BAD,
4904			[MLX5_SQC_STATE_ERR]	= MLX5_QP_STATE_BAD,
4905			[MLX5_SQ_STATE_NA]	= IB_QPS_RESET,
4906		},
4907		[MLX5_RQC_STATE_RDY] = {
4908			[MLX5_SQC_STATE_RST]	= MLX5_QP_STATE,
4909			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE,
4910			[MLX5_SQC_STATE_ERR]	= IB_QPS_SQE,
4911			[MLX5_SQ_STATE_NA]	= MLX5_QP_STATE,
4912		},
4913		[MLX5_RQC_STATE_ERR] = {
4914			[MLX5_SQC_STATE_RST]    = MLX5_QP_STATE_BAD,
4915			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE_BAD,
4916			[MLX5_SQC_STATE_ERR]	= IB_QPS_ERR,
4917			[MLX5_SQ_STATE_NA]	= IB_QPS_ERR,
4918		},
4919		[MLX5_RQ_STATE_NA] = {
4920			[MLX5_SQC_STATE_RST]    = MLX5_QP_STATE,
4921			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE,
4922			[MLX5_SQC_STATE_ERR]	= MLX5_QP_STATE,
4923			[MLX5_SQ_STATE_NA]	= MLX5_QP_STATE_BAD,
4924		},
4925	};
4926
4927	*qp_state = sqrq_trans[rq_state][sq_state];
4928
4929	if (*qp_state == MLX5_QP_STATE_BAD) {
4930		WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4931		     qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
4932		     qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
4933		return -EINVAL;
4934	}
4935
4936	if (*qp_state == MLX5_QP_STATE)
4937		*qp_state = qp->state;
4938
4939	return 0;
4940}
4941
4942static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
4943				     struct mlx5_ib_qp *qp,
4944				     u8 *raw_packet_qp_state)
4945{
4946	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4947	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
4948	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4949	int err;
4950	u8 sq_state = MLX5_SQ_STATE_NA;
4951	u8 rq_state = MLX5_RQ_STATE_NA;
4952
4953	if (qp->sq.wqe_cnt) {
4954		err = query_raw_packet_qp_sq_state(dev, sq, &sq_state);
4955		if (err)
4956			return err;
4957	}
4958
4959	if (qp->rq.wqe_cnt) {
4960		err = query_raw_packet_qp_rq_state(dev, rq, &rq_state);
4961		if (err)
4962			return err;
4963	}
4964
4965	return sqrq_state_to_qp_state(sq_state, rq_state, qp,
4966				      raw_packet_qp_state);
4967}
4968
4969static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
4970			 struct ib_qp_attr *qp_attr)
4971{
4972	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
4973	void *qpc, *pri_path, *alt_path;
4974	u32 *outb;
4975	int err;
4976
4977	outb = kzalloc(outlen, GFP_KERNEL);
4978	if (!outb)
4979		return -ENOMEM;
4980
4981	err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
4982				 false);
4983	if (err)
4984		goto out;
4985
4986	qpc = MLX5_ADDR_OF(query_qp_out, outb, qpc);
4987
4988	qp->state = to_ib_qp_state(MLX5_GET(qpc, qpc, state));
4989	if (MLX5_GET(qpc, qpc, state) == MLX5_QP_STATE_SQ_DRAINING)
4990		qp_attr->sq_draining = 1;
4991
4992	qp_attr->path_mtu = MLX5_GET(qpc, qpc, mtu);
4993	qp_attr->path_mig_state = to_ib_mig_state(MLX5_GET(qpc, qpc, pm_state));
4994	qp_attr->qkey = MLX5_GET(qpc, qpc, q_key);
4995	qp_attr->rq_psn = MLX5_GET(qpc, qpc, next_rcv_psn);
4996	qp_attr->sq_psn = MLX5_GET(qpc, qpc, next_send_psn);
4997	qp_attr->dest_qp_num = MLX5_GET(qpc, qpc, remote_qpn);
4998
4999	if (MLX5_GET(qpc, qpc, rre))
5000		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ;
5001	if (MLX5_GET(qpc, qpc, rwe))
5002		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_WRITE;
5003	if (MLX5_GET(qpc, qpc, rae))
5004		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_ATOMIC;
5005
5006	qp_attr->max_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_sra_max);
5007	qp_attr->max_dest_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_rra_max);
5008	qp_attr->min_rnr_timer = MLX5_GET(qpc, qpc, min_rnr_nak);
5009	qp_attr->retry_cnt = MLX5_GET(qpc, qpc, retry_count);
5010	qp_attr->rnr_retry = MLX5_GET(qpc, qpc, rnr_retry);
5011
5012	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
5013	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
5014
5015	if (qp->type == IB_QPT_RC || qp->type == IB_QPT_UC ||
5016	    qp->type == IB_QPT_XRC_INI || qp->type == IB_QPT_XRC_TGT) {
5017		to_rdma_ah_attr(dev, &qp_attr->ah_attr, pri_path);
5018		to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, alt_path);
5019		qp_attr->alt_pkey_index = MLX5_GET(ads, alt_path, pkey_index);
5020		qp_attr->alt_port_num = MLX5_GET(ads, alt_path, vhca_port_num);
5021	}
5022
5023	qp_attr->pkey_index = MLX5_GET(ads, pri_path, pkey_index);
5024	qp_attr->port_num = MLX5_GET(ads, pri_path, vhca_port_num);
5025	qp_attr->timeout = MLX5_GET(ads, pri_path, ack_timeout);
5026	qp_attr->alt_timeout = MLX5_GET(ads, alt_path, ack_timeout);
5027
5028out:
5029	kfree(outb);
5030	return err;
5031}
5032
5033static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
5034				struct ib_qp_attr *qp_attr, int qp_attr_mask,
5035				struct ib_qp_init_attr *qp_init_attr)
5036{
5037	struct mlx5_core_dct	*dct = &mqp->dct.mdct;
5038	u32 *out;
5039	u32 access_flags = 0;
5040	int outlen = MLX5_ST_SZ_BYTES(query_dct_out);
5041	void *dctc;
5042	int err;
5043	int supported_mask = IB_QP_STATE |
5044			     IB_QP_ACCESS_FLAGS |
5045			     IB_QP_PORT |
5046			     IB_QP_MIN_RNR_TIMER |
5047			     IB_QP_AV |
5048			     IB_QP_PATH_MTU |
5049			     IB_QP_PKEY_INDEX;
5050
5051	if (qp_attr_mask & ~supported_mask)
5052		return -EINVAL;
5053	if (mqp->state != IB_QPS_RTR)
5054		return -EINVAL;
5055
5056	out = kzalloc(outlen, GFP_KERNEL);
5057	if (!out)
5058		return -ENOMEM;
5059
5060	err = mlx5_core_dct_query(dev, dct, out, outlen);
5061	if (err)
5062		goto out;
5063
5064	dctc = MLX5_ADDR_OF(query_dct_out, out, dct_context_entry);
5065
5066	if (qp_attr_mask & IB_QP_STATE)
5067		qp_attr->qp_state = IB_QPS_RTR;
5068
5069	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
5070		if (MLX5_GET(dctc, dctc, rre))
5071			access_flags |= IB_ACCESS_REMOTE_READ;
5072		if (MLX5_GET(dctc, dctc, rwe))
5073			access_flags |= IB_ACCESS_REMOTE_WRITE;
5074		if (MLX5_GET(dctc, dctc, rae))
5075			access_flags |= IB_ACCESS_REMOTE_ATOMIC;
5076		qp_attr->qp_access_flags = access_flags;
5077	}
5078
5079	if (qp_attr_mask & IB_QP_PORT)
5080		qp_attr->port_num = mqp->port;
5081	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER)
5082		qp_attr->min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);
5083	if (qp_attr_mask & IB_QP_AV) {
5084		qp_attr->ah_attr.grh.traffic_class = MLX5_GET(dctc, dctc, tclass);
5085		qp_attr->ah_attr.grh.flow_label = MLX5_GET(dctc, dctc, flow_label);
5086		qp_attr->ah_attr.grh.sgid_index = MLX5_GET(dctc, dctc, my_addr_index);
5087		qp_attr->ah_attr.grh.hop_limit = MLX5_GET(dctc, dctc, hop_limit);
5088	}
5089	if (qp_attr_mask & IB_QP_PATH_MTU)
5090		qp_attr->path_mtu = MLX5_GET(dctc, dctc, mtu);
5091	if (qp_attr_mask & IB_QP_PKEY_INDEX)
5092		qp_attr->pkey_index = MLX5_GET(dctc, dctc, pkey_index);
5093out:
5094	kfree(out);
5095	return err;
5096}
5097
5098int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
5099		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
5100{
5101	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
5102	struct mlx5_ib_qp *qp = to_mqp(ibqp);
5103	int err = 0;
5104	u8 raw_packet_qp_state;
5105
5106	if (ibqp->rwq_ind_tbl)
5107		return -ENOSYS;
5108
5109	if (qp->type == IB_QPT_GSI)
5110		return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
5111					    qp_init_attr);
5112
5113	/* Not all of output fields are applicable, make sure to zero them */
5114	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5115	memset(qp_attr, 0, sizeof(*qp_attr));
5116
5117	if (unlikely(qp->type == MLX5_IB_QPT_DCT))
5118		return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
5119					    qp_attr_mask, qp_init_attr);
5120
5121	mutex_lock(&qp->mutex);
5122
5123	if (qp->type == IB_QPT_RAW_PACKET ||
5124	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
5125		err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
5126		if (err)
5127			goto out;
5128		qp->state = raw_packet_qp_state;
5129		qp_attr->port_num = 1;
5130	} else {
5131		err = query_qp_attr(dev, qp, qp_attr);
5132		if (err)
5133			goto out;
5134	}
5135
5136	qp_attr->qp_state	     = qp->state;
5137	qp_attr->cur_qp_state	     = qp_attr->qp_state;
5138	qp_attr->cap.max_recv_wr     = qp->rq.wqe_cnt;
5139	qp_attr->cap.max_recv_sge    = qp->rq.max_gs;
5140
5141	if (!ibqp->uobject) {
5142		qp_attr->cap.max_send_wr  = qp->sq.max_post;
5143		qp_attr->cap.max_send_sge = qp->sq.max_gs;
5144		qp_init_attr->qp_context = ibqp->qp_context;
5145	} else {
5146		qp_attr->cap.max_send_wr  = 0;
5147		qp_attr->cap.max_send_sge = 0;
5148	}
5149
5150	qp_init_attr->qp_type = qp->type;
5151	qp_init_attr->recv_cq = ibqp->recv_cq;
5152	qp_init_attr->send_cq = ibqp->send_cq;
5153	qp_init_attr->srq = ibqp->srq;
5154	qp_attr->cap.max_inline_data = qp->max_inline_data;
5155
5156	qp_init_attr->cap	     = qp_attr->cap;
5157
5158	qp_init_attr->create_flags = qp->flags;
5159
5160	qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
5161		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
5162
5163out:
5164	mutex_unlock(&qp->mutex);
5165	return err;
5166}
5167
5168int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
5169{
5170	struct mlx5_ib_dev *dev = to_mdev(ibxrcd->device);
5171	struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
5172
5173	if (!MLX5_CAP_GEN(dev->mdev, xrc))
5174		return -EOPNOTSUPP;
5175
5176	return mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
5177}
5178
5179int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
5180{
5181	struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
5182	u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
5183
5184	return mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);
5185}
5186
5187static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
5188{
5189	struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
5190	struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);
5191	struct ib_event event;
5192
5193	if (rwq->ibwq.event_handler) {
5194		event.device     = rwq->ibwq.device;
5195		event.element.wq = &rwq->ibwq;
5196		switch (type) {
5197		case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
5198			event.event = IB_EVENT_WQ_FATAL;
5199			break;
5200		default:
5201			mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
5202			return;
5203		}
5204
5205		rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
5206	}
5207}
5208
5209static int set_delay_drop(struct mlx5_ib_dev *dev)
5210{
5211	int err = 0;
5212
5213	mutex_lock(&dev->delay_drop.lock);
5214	if (dev->delay_drop.activate)
5215		goto out;
5216
5217	err = mlx5_core_set_delay_drop(dev, dev->delay_drop.timeout);
5218	if (err)
5219		goto out;
5220
5221	dev->delay_drop.activate = true;
5222out:
5223	mutex_unlock(&dev->delay_drop.lock);
5224
5225	if (!err)
5226		atomic_inc(&dev->delay_drop.rqs_cnt);
5227	return err;
5228}
5229
5230static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
5231		      struct ib_wq_init_attr *init_attr)
5232{
5233	struct mlx5_ib_dev *dev;
5234	int has_net_offloads;
5235	__be64 *rq_pas0;
5236	int ts_format;
5237	void *in;
5238	void *rqc;
5239	void *wq;
5240	int inlen;
5241	int err;
5242
5243	dev = to_mdev(pd->device);
5244
5245	ts_format = get_rq_ts_format(dev, to_mcq(init_attr->cq));
5246	if (ts_format < 0)
5247		return ts_format;
5248
5249	inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;
5250	in = kvzalloc(inlen, GFP_KERNEL);
5251	if (!in)
5252		return -ENOMEM;
5253
5254	MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
5255	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
5256	MLX5_SET(rqc,  rqc, mem_rq_type,
5257		 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
5258	MLX5_SET(rqc, rqc, ts_format, ts_format);
5259	MLX5_SET(rqc, rqc, user_index, rwq->user_index);
5260	MLX5_SET(rqc,  rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn);
5261	MLX5_SET(rqc,  rqc, state, MLX5_RQC_STATE_RST);
5262	MLX5_SET(rqc,  rqc, flush_in_error_en, 1);
5263	wq = MLX5_ADDR_OF(rqc, rqc, wq);
5264	MLX5_SET(wq, wq, wq_type,
5265		 rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ ?
5266		 MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC);
5267	if (init_attr->create_flags & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5268		if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
5269			mlx5_ib_dbg(dev, "Scatter end padding is not supported\n");
5270			err = -EOPNOTSUPP;
5271			goto out;
5272		} else {
5273			MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
5274		}
5275	}
5276	MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
5277	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ) {
5278		/*
5279		 * In Firmware number of strides in each WQE is:
5280		 *   "512 * 2^single_wqe_log_num_of_strides"
5281		 * Values 3 to 8 are accepted as 10 to 15, 9 to 18 are
5282		 * accepted as 0 to 9
5283		 */
5284		static const u8 fw_map[] = { 10, 11, 12, 13, 14, 15, 0, 1,
5285					     2,  3,  4,  5,  6,  7,  8, 9 };
5286		MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);
5287		MLX5_SET(wq, wq, log_wqe_stride_size,
5288			 rwq->single_stride_log_num_of_bytes -
5289			 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);
5290		MLX5_SET(wq, wq, log_wqe_num_of_strides,
5291			 fw_map[rwq->log_num_strides -
5292				MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES]);
5293	}
5294	MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
5295	MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
5296	MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);
5297	MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);
5298	MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);
5299	MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
5300	has_net_offloads = MLX5_CAP_GEN(dev->mdev, eth_net_offloads);
5301	if (init_attr->create_flags & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5302		if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5303			mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5304			err = -EOPNOTSUPP;
5305			goto out;
5306		}
5307	} else {
5308		MLX5_SET(rqc, rqc, vsd, 1);
5309	}
5310	if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS) {
5311		if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, scatter_fcs))) {
5312			mlx5_ib_dbg(dev, "Scatter FCS is not supported\n");
5313			err = -EOPNOTSUPP;
5314			goto out;
5315		}
5316		MLX5_SET(rqc, rqc, scatter_fcs, 1);
5317	}
5318	if (init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5319		if (!(dev->ib_dev.attrs.raw_packet_caps &
5320		      IB_RAW_PACKET_CAP_DELAY_DROP)) {
5321			mlx5_ib_dbg(dev, "Delay drop is not supported\n");
5322			err = -EOPNOTSUPP;
5323			goto out;
5324		}
5325		MLX5_SET(rqc, rqc, delay_drop_en, 1);
5326	}
5327	rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
5328	mlx5_ib_populate_pas(rwq->umem, 1UL << rwq->page_shift, rq_pas0, 0);
5329	err = mlx5_core_create_rq_tracked(dev, in, inlen, &rwq->core_qp);
5330	if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5331		err = set_delay_drop(dev);
5332		if (err) {
5333			mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
5334				     err);
5335			mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5336		} else {
5337			rwq->create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;
5338		}
5339	}
5340out:
5341	kvfree(in);
5342	return err;
5343}
5344
5345static int set_user_rq_size(struct mlx5_ib_dev *dev,
5346			    struct ib_wq_init_attr *wq_init_attr,
5347			    struct mlx5_ib_create_wq *ucmd,
5348			    struct mlx5_ib_rwq *rwq)
5349{
5350	/* Sanity check RQ size before proceeding */
5351	if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
5352		return -EINVAL;
5353
5354	if (!ucmd->rq_wqe_count)
5355		return -EINVAL;
5356
5357	rwq->wqe_count = ucmd->rq_wqe_count;
5358	rwq->wqe_shift = ucmd->rq_wqe_shift;
5359	if (check_shl_overflow(rwq->wqe_count, rwq->wqe_shift, &rwq->buf_size))
5360		return -EINVAL;
5361
5362	rwq->log_rq_stride = rwq->wqe_shift;
5363	rwq->log_rq_size = ilog2(rwq->wqe_count);
5364	return 0;
5365}
5366
5367static bool log_of_strides_valid(struct mlx5_ib_dev *dev, u32 log_num_strides)
5368{
5369	if ((log_num_strides > MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||
5370	    (log_num_strides < MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
5371		return false;
5372
5373	if (!MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) &&
5374	    (log_num_strides < MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
5375		return false;
5376
5377	return true;
5378}
5379
5380static int prepare_user_rq(struct ib_pd *pd,
5381			   struct ib_wq_init_attr *init_attr,
5382			   struct ib_udata *udata,
5383			   struct mlx5_ib_rwq *rwq)
5384{
5385	struct mlx5_ib_dev *dev = to_mdev(pd->device);
5386	struct mlx5_ib_create_wq ucmd = {};
5387	int err;
5388	size_t required_cmd_sz;
5389
5390	required_cmd_sz = offsetofend(struct mlx5_ib_create_wq,
5391				      single_stride_log_num_of_bytes);
5392	if (udata->inlen < required_cmd_sz) {
5393		mlx5_ib_dbg(dev, "invalid inlen\n");
5394		return -EINVAL;
5395	}
5396
5397	if (udata->inlen > sizeof(ucmd) &&
5398	    !ib_is_udata_cleared(udata, sizeof(ucmd),
5399				 udata->inlen - sizeof(ucmd))) {
5400		mlx5_ib_dbg(dev, "inlen is not supported\n");
5401		return -EOPNOTSUPP;
5402	}
5403
5404	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
5405		mlx5_ib_dbg(dev, "copy failed\n");
5406		return -EFAULT;
5407	}
5408
5409	if (ucmd.comp_mask & (~MLX5_IB_CREATE_WQ_STRIDING_RQ)) {
5410		mlx5_ib_dbg(dev, "invalid comp mask\n");
5411		return -EOPNOTSUPP;
5412	} else if (ucmd.comp_mask & MLX5_IB_CREATE_WQ_STRIDING_RQ) {
5413		if (!MLX5_CAP_GEN(dev->mdev, striding_rq)) {
5414			mlx5_ib_dbg(dev, "Striding RQ is not supported\n");
5415			return -EOPNOTSUPP;
5416		}
5417		if ((ucmd.single_stride_log_num_of_bytes <
5418		    MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES) ||
5419		    (ucmd.single_stride_log_num_of_bytes >
5420		     MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES)) {
5421			mlx5_ib_dbg(dev, "Invalid log stride size (%u. Range is %u - %u)\n",
5422				    ucmd.single_stride_log_num_of_bytes,
5423				    MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES,
5424				    MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);
5425			return -EINVAL;
5426		}
5427		if (!log_of_strides_valid(dev,
5428					  ucmd.single_wqe_log_num_of_strides)) {
5429			mlx5_ib_dbg(
5430				dev,
5431				"Invalid log num strides (%u. Range is %u - %u)\n",
5432				ucmd.single_wqe_log_num_of_strides,
5433				MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) ?
5434					MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES :
5435					MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
5436				MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
5437			return -EINVAL;
5438		}
5439		rwq->single_stride_log_num_of_bytes =
5440			ucmd.single_stride_log_num_of_bytes;
5441		rwq->log_num_strides = ucmd.single_wqe_log_num_of_strides;
5442		rwq->two_byte_shift_en = !!ucmd.two_byte_shift_en;
5443		rwq->create_flags |= MLX5_IB_WQ_FLAGS_STRIDING_RQ;
5444	}
5445
5446	err = set_user_rq_size(dev, init_attr, &ucmd, rwq);
5447	if (err) {
5448		mlx5_ib_dbg(dev, "err %d\n", err);
5449		return err;
5450	}
5451
5452	err = create_user_rq(dev, pd, udata, rwq, &ucmd);
5453	if (err) {
5454		mlx5_ib_dbg(dev, "err %d\n", err);
5455		return err;
5456	}
5457
5458	rwq->user_index = ucmd.user_index;
5459	return 0;
5460}
5461
5462struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
5463				struct ib_wq_init_attr *init_attr,
5464				struct ib_udata *udata)
5465{
5466	struct mlx5_ib_dev *dev;
5467	struct mlx5_ib_rwq *rwq;
5468	struct mlx5_ib_create_wq_resp resp = {};
5469	size_t min_resp_len;
5470	int err;
5471
5472	if (!udata)
5473		return ERR_PTR(-ENOSYS);
5474
5475	min_resp_len = offsetofend(struct mlx5_ib_create_wq_resp, reserved);
5476	if (udata->outlen && udata->outlen < min_resp_len)
5477		return ERR_PTR(-EINVAL);
5478
5479	if (!capable(CAP_SYS_RAWIO) &&
5480	    init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP)
5481		return ERR_PTR(-EPERM);
5482
5483	dev = to_mdev(pd->device);
5484	switch (init_attr->wq_type) {
5485	case IB_WQT_RQ:
5486		rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);
5487		if (!rwq)
5488			return ERR_PTR(-ENOMEM);
5489		err = prepare_user_rq(pd, init_attr, udata, rwq);
5490		if (err)
5491			goto err;
5492		err = create_rq(rwq, pd, init_attr);
5493		if (err)
5494			goto err_user_rq;
5495		break;
5496	default:
5497		mlx5_ib_dbg(dev, "unsupported wq type %d\n",
5498			    init_attr->wq_type);
5499		return ERR_PTR(-EINVAL);
5500	}
5501
5502	rwq->ibwq.wq_num = rwq->core_qp.qpn;
5503	rwq->ibwq.state = IB_WQS_RESET;
5504	if (udata->outlen) {
5505		resp.response_length = offsetofend(
5506			struct mlx5_ib_create_wq_resp, response_length);
5507		err = ib_copy_to_udata(udata, &resp, resp.response_length);
5508		if (err)
5509			goto err_copy;
5510	}
5511
5512	rwq->core_qp.event = mlx5_ib_wq_event;
5513	rwq->ibwq.event_handler = init_attr->event_handler;
5514	return &rwq->ibwq;
5515
5516err_copy:
5517	mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5518err_user_rq:
5519	destroy_user_rq(dev, pd, rwq, udata);
5520err:
5521	kfree(rwq);
5522	return ERR_PTR(err);
5523}
5524
5525int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
5526{
5527	struct mlx5_ib_dev *dev = to_mdev(wq->device);
5528	struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5529	int ret;
5530
5531	ret = mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5532	if (ret)
5533		return ret;
5534	destroy_user_rq(dev, wq->pd, rwq, udata);
5535	kfree(rwq);
5536	return 0;
5537}
5538
5539int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
5540				 struct ib_rwq_ind_table_init_attr *init_attr,
5541				 struct ib_udata *udata)
5542{
5543	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl =
5544		to_mrwq_ind_table(ib_rwq_ind_table);
5545	struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_table->device);
5546	int sz = 1 << init_attr->log_ind_tbl_size;
5547	struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
5548	size_t min_resp_len;
5549	int inlen;
5550	int err;
5551	int i;
5552	u32 *in;
5553	void *rqtc;
5554
5555	if (udata->inlen > 0 &&
5556	    !ib_is_udata_cleared(udata, 0,
5557				 udata->inlen))
5558		return -EOPNOTSUPP;
5559
5560	if (init_attr->log_ind_tbl_size >
5561	    MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
5562		mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",
5563			    init_attr->log_ind_tbl_size,
5564			    MLX5_CAP_GEN(dev->mdev, log_max_rqt_size));
5565		return -EINVAL;
5566	}
5567
5568	min_resp_len =
5569		offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, reserved);
5570	if (udata->outlen && udata->outlen < min_resp_len)
5571		return -EINVAL;
5572
5573	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
5574	in = kvzalloc(inlen, GFP_KERNEL);
5575	if (!in)
5576		return -ENOMEM;
5577
5578	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
5579
5580	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5581	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
5582
5583	for (i = 0; i < sz; i++)
5584		MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
5585
5586	rwq_ind_tbl->uid = to_mpd(init_attr->ind_tbl[0]->pd)->uid;
5587	MLX5_SET(create_rqt_in, in, uid, rwq_ind_tbl->uid);
5588
5589	err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
5590	kvfree(in);
5591	if (err)
5592		return err;
5593
5594	rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
5595	if (udata->outlen) {
5596		resp.response_length =
5597			offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp,
5598				    response_length);
5599		err = ib_copy_to_udata(udata, &resp, resp.response_length);
5600		if (err)
5601			goto err_copy;
5602	}
5603
5604	return 0;
5605
5606err_copy:
5607	mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
5608	return err;
5609}
5610
5611int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
5612{
5613	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
5614	struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
5615
5616	return mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
5617}
5618
5619int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
5620		      u32 wq_attr_mask, struct ib_udata *udata)
5621{
5622	struct mlx5_ib_dev *dev = to_mdev(wq->device);
5623	struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5624	struct mlx5_ib_modify_wq ucmd = {};
5625	size_t required_cmd_sz;
5626	int curr_wq_state;
5627	int wq_state;
5628	int inlen;
5629	int err;
5630	void *rqc;
5631	void *in;
5632
5633	required_cmd_sz = offsetofend(struct mlx5_ib_modify_wq, reserved);
5634	if (udata->inlen < required_cmd_sz)
5635		return -EINVAL;
5636
5637	if (udata->inlen > sizeof(ucmd) &&
5638	    !ib_is_udata_cleared(udata, sizeof(ucmd),
5639				 udata->inlen - sizeof(ucmd)))
5640		return -EOPNOTSUPP;
5641
5642	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
5643		return -EFAULT;
5644
5645	if (ucmd.comp_mask || ucmd.reserved)
5646		return -EOPNOTSUPP;
5647
5648	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
5649	in = kvzalloc(inlen, GFP_KERNEL);
5650	if (!in)
5651		return -ENOMEM;
5652
5653	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
5654
5655	curr_wq_state = wq_attr->curr_wq_state;
5656	wq_state = wq_attr->wq_state;
5657	if (curr_wq_state == IB_WQS_ERR)
5658		curr_wq_state = MLX5_RQC_STATE_ERR;
5659	if (wq_state == IB_WQS_ERR)
5660		wq_state = MLX5_RQC_STATE_ERR;
5661	MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
5662	MLX5_SET(modify_rq_in, in, uid, to_mpd(wq->pd)->uid);
5663	MLX5_SET(rqc, rqc, state, wq_state);
5664
5665	if (wq_attr_mask & IB_WQ_FLAGS) {
5666		if (wq_attr->flags_mask & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5667			if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
5668			      MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5669				mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5670				err = -EOPNOTSUPP;
5671				goto out;
5672			}
5673			MLX5_SET64(modify_rq_in, in, modify_bitmask,
5674				   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
5675			MLX5_SET(rqc, rqc, vsd,
5676				 (wq_attr->flags & IB_WQ_FLAGS_CVLAN_STRIPPING) ? 0 : 1);
5677		}
5678
5679		if (wq_attr->flags_mask & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5680			mlx5_ib_dbg(dev, "Modifying scatter end padding is not supported\n");
5681			err = -EOPNOTSUPP;
5682			goto out;
5683		}
5684	}
5685
5686	if (curr_wq_state == IB_WQS_RESET && wq_state == IB_WQS_RDY) {
5687		u16 set_id;
5688
5689		set_id = mlx5_ib_get_counters_id(dev, 0);
5690		if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
5691			MLX5_SET64(modify_rq_in, in, modify_bitmask,
5692				   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
5693			MLX5_SET(rqc, rqc, counter_set_id, set_id);
5694		} else
5695			dev_info_once(
5696				&dev->ib_dev.dev,
5697				"Receive WQ counters are not supported on current FW\n");
5698	}
5699
5700	err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in);
5701	if (!err)
5702		rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
5703
5704out:
5705	kvfree(in);
5706	return err;
5707}
5708
5709struct mlx5_ib_drain_cqe {
5710	struct ib_cqe cqe;
5711	struct completion done;
5712};
5713
5714static void mlx5_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
5715{
5716	struct mlx5_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
5717						     struct mlx5_ib_drain_cqe,
5718						     cqe);
5719
5720	complete(&cqe->done);
5721}
5722
5723/* This function returns only once the drained WR was completed */
5724static void handle_drain_completion(struct ib_cq *cq,
5725				    struct mlx5_ib_drain_cqe *sdrain,
5726				    struct mlx5_ib_dev *dev)
5727{
5728	struct mlx5_core_dev *mdev = dev->mdev;
5729
5730	if (cq->poll_ctx == IB_POLL_DIRECT) {
5731		while (wait_for_completion_timeout(&sdrain->done, HZ / 10) <= 0)
5732			ib_process_cq_direct(cq, -1);
5733		return;
5734	}
5735
5736	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5737		struct mlx5_ib_cq *mcq = to_mcq(cq);
5738		bool triggered = false;
5739		unsigned long flags;
5740
5741		spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
5742		/* Make sure that the CQ handler won't run if wasn't run yet */
5743		if (!mcq->mcq.reset_notify_added)
5744			mcq->mcq.reset_notify_added = 1;
5745		else
5746			triggered = true;
5747		spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
5748
5749		if (triggered) {
5750			/* Wait for any scheduled/running task to be ended */
5751			switch (cq->poll_ctx) {
5752			case IB_POLL_SOFTIRQ:
5753				irq_poll_disable(&cq->iop);
5754				irq_poll_enable(&cq->iop);
5755				break;
5756			case IB_POLL_WORKQUEUE:
5757				cancel_work_sync(&cq->work);
5758				break;
5759			default:
5760				WARN_ON_ONCE(1);
5761			}
5762		}
5763
5764		/* Run the CQ handler - this makes sure that the drain WR will
5765		 * be processed if wasn't processed yet.
5766		 */
5767		mcq->mcq.comp(&mcq->mcq, NULL);
5768	}
5769
5770	wait_for_completion(&sdrain->done);
5771}
5772
5773void mlx5_ib_drain_sq(struct ib_qp *qp)
5774{
5775	struct ib_cq *cq = qp->send_cq;
5776	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5777	struct mlx5_ib_drain_cqe sdrain;
5778	const struct ib_send_wr *bad_swr;
5779	struct ib_rdma_wr swr = {
5780		.wr = {
5781			.next = NULL,
5782			{ .wr_cqe	= &sdrain.cqe, },
5783			.opcode	= IB_WR_RDMA_WRITE,
5784		},
5785	};
5786	int ret;
5787	struct mlx5_ib_dev *dev = to_mdev(qp->device);
5788	struct mlx5_core_dev *mdev = dev->mdev;
5789
5790	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
5791	if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5792		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5793		return;
5794	}
5795
5796	sdrain.cqe.done = mlx5_ib_drain_qp_done;
5797	init_completion(&sdrain.done);
5798
5799	ret = mlx5_ib_post_send_drain(qp, &swr.wr, &bad_swr);
5800	if (ret) {
5801		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5802		return;
5803	}
5804
5805	handle_drain_completion(cq, &sdrain, dev);
5806}
5807
5808void mlx5_ib_drain_rq(struct ib_qp *qp)
5809{
5810	struct ib_cq *cq = qp->recv_cq;
5811	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5812	struct mlx5_ib_drain_cqe rdrain;
5813	struct ib_recv_wr rwr = {};
5814	const struct ib_recv_wr *bad_rwr;
5815	int ret;
5816	struct mlx5_ib_dev *dev = to_mdev(qp->device);
5817	struct mlx5_core_dev *mdev = dev->mdev;
5818
5819	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
5820	if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5821		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5822		return;
5823	}
5824
5825	rwr.wr_cqe = &rdrain.cqe;
5826	rdrain.cqe.done = mlx5_ib_drain_qp_done;
5827	init_completion(&rdrain.done);
5828
5829	ret = mlx5_ib_post_recv_drain(qp, &rwr, &bad_rwr);
5830	if (ret) {
5831		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5832		return;
5833	}
5834
5835	handle_drain_completion(cq, &rdrain, dev);
5836}
5837
5838/*
5839 * Bind a qp to a counter. If @counter is NULL then bind the qp to
5840 * the default counter
5841 */
5842int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter)
5843{
5844	struct mlx5_ib_dev *dev = to_mdev(qp->device);
5845	struct mlx5_ib_qp *mqp = to_mqp(qp);
5846	int err = 0;
5847
5848	mutex_lock(&mqp->mutex);
5849	if (mqp->state == IB_QPS_RESET) {
5850		qp->counter = counter;
5851		goto out;
5852	}
5853
5854	if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id)) {
5855		err = -EOPNOTSUPP;
5856		goto out;
5857	}
5858
5859	if (mqp->state == IB_QPS_RTS) {
5860		err = __mlx5_ib_qp_set_counter(qp, counter);
5861		if (!err)
5862			qp->counter = counter;
5863
5864		goto out;
5865	}
5866
5867	mqp->counter_pending = 1;
5868	qp->counter = counter;
5869
5870out:
5871	mutex_unlock(&mqp->mutex);
5872	return err;
5873}
5874
5875int mlx5_ib_qp_event_init(void)
5876{
5877	mlx5_ib_qp_event_wq = alloc_ordered_workqueue("mlx5_ib_qp_event_wq", 0);
5878	if (!mlx5_ib_qp_event_wq)
5879		return -ENOMEM;
5880
5881	return 0;
5882}
5883
5884void mlx5_ib_qp_event_cleanup(void)
5885{
5886	destroy_workqueue(mlx5_ib_qp_event_wq);
5887}