qpc.c

  1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2/*
  3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
  4 */
  5
  6#include <linux/gfp.h>
  7#include <linux/mlx5/qp.h>
  8#include <linux/mlx5/driver.h>
  9#include "mlx5_ib.h"
 10#include "qp.h"
 11
 12static int mlx5_core_drain_dct(struct mlx5_ib_dev *dev,
 13			       struct mlx5_core_dct *dct);
 14
 15static struct mlx5_core_rsc_common *
 16mlx5_get_rsc(struct mlx5_qp_table *table, u32 rsn)
 17{
 18	struct mlx5_core_rsc_common *common;
 19	unsigned long flags;
 20
 21	spin_lock_irqsave(&table->lock, flags);
 22
 23	common = radix_tree_lookup(&table->tree, rsn);
 24	if (common && !common->invalid)
 25		refcount_inc(&common->refcount);
 26	else
 27		common = NULL;
 28
 29	spin_unlock_irqrestore(&table->lock, flags);
 30
 31	return common;
 32}
 33
 34void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common)
 35{
 36	if (refcount_dec_and_test(&common->refcount))
 37		complete(&common->free);
 38}
 39
 40static u64 qp_allowed_event_types(void)
 41{
 42	u64 mask;
 43
 44	mask = BIT(MLX5_EVENT_TYPE_PATH_MIG) |
 45	       BIT(MLX5_EVENT_TYPE_COMM_EST) |
 46	       BIT(MLX5_EVENT_TYPE_SQ_DRAINED) |
 47	       BIT(MLX5_EVENT_TYPE_SRQ_LAST_WQE) |
 48	       BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR) |
 49	       BIT(MLX5_EVENT_TYPE_PATH_MIG_FAILED) |
 50	       BIT(MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) |
 51	       BIT(MLX5_EVENT_TYPE_WQ_ACCESS_ERROR);
 52
 53	return mask;
 54}
 55
 56static u64 rq_allowed_event_types(void)
 57{
 58	u64 mask;
 59
 60	mask = BIT(MLX5_EVENT_TYPE_SRQ_LAST_WQE) |
 61	       BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);
 62
 63	return mask;
 64}
 65
 66static u64 sq_allowed_event_types(void)
 67{
 68	return BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);
 69}
 70
 71static u64 dct_allowed_event_types(void)
 72{
 73	return BIT(MLX5_EVENT_TYPE_DCT_DRAINED);
 74}
 75
 76static bool is_event_type_allowed(int rsc_type, int event_type)
 77{
 78	switch (rsc_type) {
 79	case MLX5_EVENT_QUEUE_TYPE_QP:
 80		return BIT(event_type) & qp_allowed_event_types();
 81	case MLX5_EVENT_QUEUE_TYPE_RQ:
 82		return BIT(event_type) & rq_allowed_event_types();
 83	case MLX5_EVENT_QUEUE_TYPE_SQ:
 84		return BIT(event_type) & sq_allowed_event_types();
 85	case MLX5_EVENT_QUEUE_TYPE_DCT:
 86		return BIT(event_type) & dct_allowed_event_types();
 87	default:
 88		WARN(1, "Event arrived for unknown resource type");
 89		return false;
 90	}
 91}
 92
 93static int dct_event_notifier(struct mlx5_ib_dev *dev, struct mlx5_eqe *eqe)
 94{
 95	struct mlx5_core_dct *dct;
 96	unsigned long flags;
 97	u32 qpn;
 98
 99	qpn = be32_to_cpu(eqe->data.dct.dctn) & 0xFFFFFF;
100	xa_lock_irqsave(&dev->qp_table.dct_xa, flags);
101	dct = xa_load(&dev->qp_table.dct_xa, qpn);
102	if (dct)
103		complete(&dct->drained);
104	xa_unlock_irqrestore(&dev->qp_table.dct_xa, flags);
105	return NOTIFY_OK;
106}
107
108static int rsc_event_notifier(struct notifier_block *nb,
109			      unsigned long type, void *data)
110{
111	struct mlx5_ib_dev *dev =
112		container_of(nb, struct mlx5_ib_dev, qp_table.nb);
113	struct mlx5_core_rsc_common *common;
114	struct mlx5_eqe *eqe = data;
115	u8 event_type = (u8)type;
116	struct mlx5_core_qp *qp;
117	u32 rsn;
118
119	switch (event_type) {
120	case MLX5_EVENT_TYPE_DCT_DRAINED:
121		return dct_event_notifier(dev, eqe);
122	case MLX5_EVENT_TYPE_PATH_MIG:
123	case MLX5_EVENT_TYPE_COMM_EST:
124	case MLX5_EVENT_TYPE_SQ_DRAINED:
125	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
126	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
127	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
128	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
129	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
130		rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
131		rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
132		break;
133	default:
134		return NOTIFY_DONE;
135	}
136
137	common = mlx5_get_rsc(&dev->qp_table, rsn);
138	if (!common)
139		return NOTIFY_OK;
140
141	if (!is_event_type_allowed((rsn >> MLX5_USER_INDEX_LEN), event_type))
142		goto out;
143
144	switch (common->res) {
145	case MLX5_RES_QP:
146	case MLX5_RES_RQ:
147	case MLX5_RES_SQ:
148		qp = (struct mlx5_core_qp *)common;
149		qp->event(qp, event_type);
150		/* Need to put resource in event handler */
151		return NOTIFY_OK;
152	default:
153		break;
154	}
155out:
156	mlx5_core_put_rsc(common);
157
158	return NOTIFY_OK;
159}
160
161static int create_resource_common(struct mlx5_ib_dev *dev,
162				  struct mlx5_core_qp *qp, int rsc_type)
163{
164	struct mlx5_qp_table *table = &dev->qp_table;
165	int err;
166
167	qp->common.res = rsc_type;
168	spin_lock_irq(&table->lock);
169	err = radix_tree_insert(&table->tree,
170				qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN),
171				qp);
172	spin_unlock_irq(&table->lock);
173	if (err)
174		return err;
175
176	refcount_set(&qp->common.refcount, 1);
177	init_completion(&qp->common.free);
178	qp->pid = current->pid;
179
180	return 0;
181}
182
183static void modify_resource_common_state(struct mlx5_ib_dev *dev,
184					 struct mlx5_core_qp *qp,
185					 bool invalid)
186{
187	struct mlx5_qp_table *table = &dev->qp_table;
188	unsigned long flags;
189
190	spin_lock_irqsave(&table->lock, flags);
191	qp->common.invalid = invalid;
192	spin_unlock_irqrestore(&table->lock, flags);
193}
194
195static void destroy_resource_common(struct mlx5_ib_dev *dev,
196				    struct mlx5_core_qp *qp)
197{
198	struct mlx5_qp_table *table = &dev->qp_table;
199	unsigned long flags;
200
201	spin_lock_irqsave(&table->lock, flags);
202	radix_tree_delete(&table->tree,
203			  qp->qpn | (qp->common.res << MLX5_USER_INDEX_LEN));
204	spin_unlock_irqrestore(&table->lock, flags);
205	mlx5_core_put_rsc((struct mlx5_core_rsc_common *)qp);
206	wait_for_completion(&qp->common.free);
207}
208
209static int _mlx5_core_destroy_dct(struct mlx5_ib_dev *dev,
210				  struct mlx5_core_dct *dct)
211{
212	u32 in[MLX5_ST_SZ_DW(destroy_dct_in)] = {};
213	struct mlx5_core_qp *qp = &dct->mqp;
214
215	MLX5_SET(destroy_dct_in, in, opcode, MLX5_CMD_OP_DESTROY_DCT);
216	MLX5_SET(destroy_dct_in, in, dctn, qp->qpn);
217	MLX5_SET(destroy_dct_in, in, uid, qp->uid);
218	return mlx5_cmd_exec_in(dev->mdev, destroy_dct, in);
219}
220
221int mlx5_core_create_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,
222			 u32 *in, int inlen, u32 *out, int outlen)
223{
224	struct mlx5_core_qp *qp = &dct->mqp;
225	int err;
226
227	init_completion(&dct->drained);
228	MLX5_SET(create_dct_in, in, opcode, MLX5_CMD_OP_CREATE_DCT);
229
230	err = mlx5_cmd_do(dev->mdev, in, inlen, out, outlen);
231	if (err)
232		return err;
233
234	qp->qpn = MLX5_GET(create_dct_out, out, dctn);
235	qp->uid = MLX5_GET(create_dct_in, in, uid);
236	err = xa_err(xa_store_irq(&dev->qp_table.dct_xa, qp->qpn, dct, GFP_KERNEL));
237	if (err)
238		goto err_cmd;
239
240	return 0;
241err_cmd:
242	_mlx5_core_destroy_dct(dev, dct);
243	return err;
244}
245
246int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
247		       u32 *in, int inlen, u32 *out)
248{
249	u32 din[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
250	int err;
251
252	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
253
254	err = mlx5_cmd_exec(dev->mdev, in, inlen, out,
255			    MLX5_ST_SZ_BYTES(create_qp_out));
256	if (err)
257		return err;
258
259	qp->uid = MLX5_GET(create_qp_in, in, uid);
260	qp->qpn = MLX5_GET(create_qp_out, out, qpn);
261
262	err = create_resource_common(dev, qp, MLX5_RES_QP);
263	if (err)
264		goto err_cmd;
265
266	if (dev->ib_dev.type != RDMA_DEVICE_TYPE_SMI)
267		mlx5_debug_qp_add(dev->mdev, qp);
268
269	return 0;
270
271err_cmd:
272	MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);
273	MLX5_SET(destroy_qp_in, din, qpn, qp->qpn);
274	MLX5_SET(destroy_qp_in, din, uid, qp->uid);
275	mlx5_cmd_exec_in(dev->mdev, destroy_qp, din);
276	return err;
277}
278
279static int mlx5_core_drain_dct(struct mlx5_ib_dev *dev,
280			       struct mlx5_core_dct *dct)
281{
282	u32 in[MLX5_ST_SZ_DW(drain_dct_in)] = {};
283	struct mlx5_core_qp *qp = &dct->mqp;
284
285	MLX5_SET(drain_dct_in, in, opcode, MLX5_CMD_OP_DRAIN_DCT);
286	MLX5_SET(drain_dct_in, in, dctn, qp->qpn);
287	MLX5_SET(drain_dct_in, in, uid, qp->uid);
288	return mlx5_cmd_exec_in(dev->mdev, drain_dct, in);
289}
290
291int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev,
292			  struct mlx5_core_dct *dct)
293{
294	struct mlx5_qp_table *table = &dev->qp_table;
295	struct mlx5_core_dct *tmp;
296	int err;
297
298	err = mlx5_core_drain_dct(dev, dct);
299	if (err) {
300		if (dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
301			goto destroy;
302
303		return err;
304	}
305	wait_for_completion(&dct->drained);
306
307destroy:
308	tmp = xa_cmpxchg_irq(&table->dct_xa, dct->mqp.qpn, dct, XA_ZERO_ENTRY, GFP_KERNEL);
309	if (WARN_ON(tmp != dct))
310		return xa_err(tmp) ?: -EINVAL;
311
312	err = _mlx5_core_destroy_dct(dev, dct);
313	if (err) {
314		xa_cmpxchg_irq(&table->dct_xa, dct->mqp.qpn, XA_ZERO_ENTRY, dct, 0);
315		return err;
316	}
317	xa_erase_irq(&table->dct_xa, dct->mqp.qpn);
318	return 0;
319}
320
321int mlx5_core_destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp)
322{
323	u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
324
325	if (dev->ib_dev.type != RDMA_DEVICE_TYPE_SMI)
326		mlx5_debug_qp_remove(dev->mdev, qp);
327
328	destroy_resource_common(dev, qp);
329
330	MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
331	MLX5_SET(destroy_qp_in, in, qpn, qp->qpn);
332	MLX5_SET(destroy_qp_in, in, uid, qp->uid);
333	return mlx5_cmd_exec_in(dev->mdev, destroy_qp, in);
334}
335
336int mlx5_core_set_delay_drop(struct mlx5_ib_dev *dev,
337			     u32 timeout_usec)
338{
339	u32 in[MLX5_ST_SZ_DW(set_delay_drop_params_in)] = {};
340
341	MLX5_SET(set_delay_drop_params_in, in, opcode,
342		 MLX5_CMD_OP_SET_DELAY_DROP_PARAMS);
343	MLX5_SET(set_delay_drop_params_in, in, delay_drop_timeout,
344		 timeout_usec / 100);
345	return mlx5_cmd_exec_in(dev->mdev, set_delay_drop_params, in);
346}
347
348struct mbox_info {
349	u32 *in;
350	u32 *out;
351	int inlen;
352	int outlen;
353};
354
355static int mbox_alloc(struct mbox_info *mbox, int inlen, int outlen)
356{
357	mbox->inlen  = inlen;
358	mbox->outlen = outlen;
359	mbox->in = kzalloc(mbox->inlen, GFP_KERNEL);
360	mbox->out = kzalloc(mbox->outlen, GFP_KERNEL);
361	if (!mbox->in || !mbox->out) {
362		kfree(mbox->in);
363		kfree(mbox->out);
364		return -ENOMEM;
365	}
366
367	return 0;
368}
369
370static void mbox_free(struct mbox_info *mbox)
371{
372	kfree(mbox->in);
373	kfree(mbox->out);
374}
375
376static int get_ece_from_mbox(void *out, u16 opcode)
377{
378	int ece = 0;
379
380	switch (opcode) {
381	case MLX5_CMD_OP_INIT2INIT_QP:
382		ece = MLX5_GET(init2init_qp_out, out, ece);
383		break;
384	case MLX5_CMD_OP_INIT2RTR_QP:
385		ece = MLX5_GET(init2rtr_qp_out, out, ece);
386		break;
387	case MLX5_CMD_OP_RTR2RTS_QP:
388		ece = MLX5_GET(rtr2rts_qp_out, out, ece);
389		break;
390	case MLX5_CMD_OP_RTS2RTS_QP:
391		ece = MLX5_GET(rts2rts_qp_out, out, ece);
392		break;
393	case MLX5_CMD_OP_RST2INIT_QP:
394		ece = MLX5_GET(rst2init_qp_out, out, ece);
395		break;
396	default:
397		break;
398	}
399
400	return ece;
401}
402
403static int modify_qp_mbox_alloc(struct mlx5_core_dev *dev, u16 opcode, int qpn,
404				u32 opt_param_mask, void *qpc,
405				struct mbox_info *mbox, u16 uid, u32 ece)
406{
407	mbox->out = NULL;
408	mbox->in = NULL;
409
410#define MBOX_ALLOC(mbox, typ)  \
411	mbox_alloc(mbox, MLX5_ST_SZ_BYTES(typ##_in), MLX5_ST_SZ_BYTES(typ##_out))
412
413#define MOD_QP_IN_SET(typ, in, _opcode, _qpn, _uid)                            \
414	do {                                                                   \
415		MLX5_SET(typ##_in, in, opcode, _opcode);                       \
416		MLX5_SET(typ##_in, in, qpn, _qpn);                             \
417		MLX5_SET(typ##_in, in, uid, _uid);                             \
418	} while (0)
419
420#define MOD_QP_IN_SET_QPC(typ, in, _opcode, _qpn, _opt_p, _qpc, _uid)          \
421	do {                                                                   \
422		MOD_QP_IN_SET(typ, in, _opcode, _qpn, _uid);                   \
423		MLX5_SET(typ##_in, in, opt_param_mask, _opt_p);                \
424		memcpy(MLX5_ADDR_OF(typ##_in, in, qpc), _qpc,                  \
425		       MLX5_ST_SZ_BYTES(qpc));                                 \
426	} while (0)
427
428	switch (opcode) {
429	/* 2RST & 2ERR */
430	case MLX5_CMD_OP_2RST_QP:
431		if (MBOX_ALLOC(mbox, qp_2rst))
432			return -ENOMEM;
433		MOD_QP_IN_SET(qp_2rst, mbox->in, opcode, qpn, uid);
434		break;
435	case MLX5_CMD_OP_2ERR_QP:
436		if (MBOX_ALLOC(mbox, qp_2err))
437			return -ENOMEM;
438		MOD_QP_IN_SET(qp_2err, mbox->in, opcode, qpn, uid);
439		break;
440
441	/* MODIFY with QPC */
442	case MLX5_CMD_OP_RST2INIT_QP:
443		if (MBOX_ALLOC(mbox, rst2init_qp))
444			return -ENOMEM;
445		MOD_QP_IN_SET_QPC(rst2init_qp, mbox->in, opcode, qpn,
446				  opt_param_mask, qpc, uid);
447		MLX5_SET(rst2init_qp_in, mbox->in, ece, ece);
448		break;
449	case MLX5_CMD_OP_INIT2RTR_QP:
450		if (MBOX_ALLOC(mbox, init2rtr_qp))
451			return -ENOMEM;
452		MOD_QP_IN_SET_QPC(init2rtr_qp, mbox->in, opcode, qpn,
453				  opt_param_mask, qpc, uid);
454		MLX5_SET(init2rtr_qp_in, mbox->in, ece, ece);
455		break;
456	case MLX5_CMD_OP_RTR2RTS_QP:
457		if (MBOX_ALLOC(mbox, rtr2rts_qp))
458			return -ENOMEM;
459		MOD_QP_IN_SET_QPC(rtr2rts_qp, mbox->in, opcode, qpn,
460				  opt_param_mask, qpc, uid);
461		MLX5_SET(rtr2rts_qp_in, mbox->in, ece, ece);
462		break;
463	case MLX5_CMD_OP_RTS2RTS_QP:
464		if (MBOX_ALLOC(mbox, rts2rts_qp))
465			return -ENOMEM;
466		MOD_QP_IN_SET_QPC(rts2rts_qp, mbox->in, opcode, qpn,
467				  opt_param_mask, qpc, uid);
468		MLX5_SET(rts2rts_qp_in, mbox->in, ece, ece);
469		break;
470	case MLX5_CMD_OP_SQERR2RTS_QP:
471		if (MBOX_ALLOC(mbox, sqerr2rts_qp))
472			return -ENOMEM;
473		MOD_QP_IN_SET_QPC(sqerr2rts_qp, mbox->in, opcode, qpn,
474				  opt_param_mask, qpc, uid);
475		break;
476	case MLX5_CMD_OP_SQD_RTS_QP:
477		if (MBOX_ALLOC(mbox, sqd2rts_qp))
478			return -ENOMEM;
479		MOD_QP_IN_SET_QPC(sqd2rts_qp, mbox->in, opcode, qpn,
480				  opt_param_mask, qpc, uid);
481		break;
482	case MLX5_CMD_OP_INIT2INIT_QP:
483		if (MBOX_ALLOC(mbox, init2init_qp))
484			return -ENOMEM;
485		MOD_QP_IN_SET_QPC(init2init_qp, mbox->in, opcode, qpn,
486				  opt_param_mask, qpc, uid);
487		MLX5_SET(init2init_qp_in, mbox->in, ece, ece);
488		break;
489	default:
490		return -EINVAL;
491	}
492	return 0;
493}
494
495int mlx5_core_qp_modify(struct mlx5_ib_dev *dev, u16 opcode, u32 opt_param_mask,
496			void *qpc, struct mlx5_core_qp *qp, u32 *ece)
497{
498	struct mbox_info mbox;
499	int err;
500
501	err = modify_qp_mbox_alloc(dev->mdev, opcode, qp->qpn, opt_param_mask,
502				   qpc, &mbox, qp->uid, (ece) ? *ece : 0);
503	if (err)
504		return err;
505
506	err = mlx5_cmd_exec(dev->mdev, mbox.in, mbox.inlen, mbox.out,
507			    mbox.outlen);
508
509	if (ece)
510		*ece = get_ece_from_mbox(mbox.out, opcode);
511
512	mbox_free(&mbox);
513	return err;
514}
515
516int mlx5_init_qp_table(struct mlx5_ib_dev *dev)
517{
518	struct mlx5_qp_table *table = &dev->qp_table;
519
520	spin_lock_init(&table->lock);
521	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
522	xa_init(&table->dct_xa);
523
524	if (dev->ib_dev.type != RDMA_DEVICE_TYPE_SMI)
525		mlx5_qp_debugfs_init(dev->mdev);
526
527	table->nb.notifier_call = rsc_event_notifier;
528	mlx5_notifier_register(dev->mdev, &table->nb);
529
530	return 0;
531}
532
533void mlx5_cleanup_qp_table(struct mlx5_ib_dev *dev)
534{
535	struct mlx5_qp_table *table = &dev->qp_table;
536
537	mlx5_notifier_unregister(dev->mdev, &table->nb);
538	if (dev->ib_dev.type != RDMA_DEVICE_TYPE_SMI)
539		mlx5_qp_debugfs_cleanup(dev->mdev);
540}
541
542int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
543		       u32 *out, int outlen, bool qpc_ext)
544{
545	u32 in[MLX5_ST_SZ_DW(query_qp_in)] = {};
546
547	MLX5_SET(query_qp_in, in, opcode, MLX5_CMD_OP_QUERY_QP);
548	MLX5_SET(query_qp_in, in, qpn, qp->qpn);
549	MLX5_SET(query_qp_in, in, qpc_ext, qpc_ext);
550
551	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, outlen);
552}
553
554int mlx5_core_dct_query(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,
555			u32 *out, int outlen)
556{
557	u32 in[MLX5_ST_SZ_DW(query_dct_in)] = {};
558	struct mlx5_core_qp *qp = &dct->mqp;
559
560	MLX5_SET(query_dct_in, in, opcode, MLX5_CMD_OP_QUERY_DCT);
561	MLX5_SET(query_dct_in, in, dctn, qp->qpn);
562
563	return mlx5_cmd_exec(dev->mdev, (void *)&in, sizeof(in), (void *)out,
564			     outlen);
565}
566
567int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn)
568{
569	u32 out[MLX5_ST_SZ_DW(alloc_xrcd_out)] = {};
570	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)] = {};
571	int err;
572
573	MLX5_SET(alloc_xrcd_in, in, opcode, MLX5_CMD_OP_ALLOC_XRCD);
574	err = mlx5_cmd_exec_inout(dev->mdev, alloc_xrcd, in, out);
575	if (!err)
576		*xrcdn = MLX5_GET(alloc_xrcd_out, out, xrcd);
577	return err;
578}
579
580int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn)
581{
582	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)] = {};
583
584	MLX5_SET(dealloc_xrcd_in, in, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
585	MLX5_SET(dealloc_xrcd_in, in, xrcd, xrcdn);
586	return mlx5_cmd_exec_in(dev->mdev, dealloc_xrcd, in);
587}
588
589static int destroy_rq_tracked(struct mlx5_ib_dev *dev, u32 rqn, u16 uid)
590{
591	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};
592
593	MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
594	MLX5_SET(destroy_rq_in, in, rqn, rqn);
595	MLX5_SET(destroy_rq_in, in, uid, uid);
596	return mlx5_cmd_exec_in(dev->mdev, destroy_rq, in);
597}
598
599int mlx5_core_create_rq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,
600				struct mlx5_core_qp *rq)
601{
602	int err;
603	u32 rqn;
604
605	err = mlx5_core_create_rq(dev->mdev, in, inlen, &rqn);
606	if (err)
607		return err;
608
609	rq->uid = MLX5_GET(create_rq_in, in, uid);
610	rq->qpn = rqn;
611	err = create_resource_common(dev, rq, MLX5_RES_RQ);
612	if (err)
613		goto err_destroy_rq;
614
615	return 0;
616
617err_destroy_rq:
618	destroy_rq_tracked(dev, rq->qpn, rq->uid);
619
620	return err;
621}
622
623int mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev,
624				 struct mlx5_core_qp *rq)
625{
626	int ret;
627
628	/* The rq destruction can be called again in case it fails, hence we
629	 * mark the common resource as invalid and only once FW destruction
630	 * is completed successfully we actually destroy the resources.
631	 */
632	modify_resource_common_state(dev, rq, true);
633	ret = destroy_rq_tracked(dev, rq->qpn, rq->uid);
634	if (ret) {
635		modify_resource_common_state(dev, rq, false);
636		return ret;
637	}
638	destroy_resource_common(dev, rq);
639	return 0;
640}
641
642static void destroy_sq_tracked(struct mlx5_ib_dev *dev, u32 sqn, u16 uid)
643{
644	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};
645
646	MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
647	MLX5_SET(destroy_sq_in, in, sqn, sqn);
648	MLX5_SET(destroy_sq_in, in, uid, uid);
649	mlx5_cmd_exec_in(dev->mdev, destroy_sq, in);
650}
651
652int mlx5_core_create_sq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,
653				struct mlx5_core_qp *sq)
654{
655	u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {};
656	int err;
657
658	MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
659	err = mlx5_cmd_exec(dev->mdev, in, inlen, out, sizeof(out));
660	if (err)
661		return err;
662
663	sq->qpn = MLX5_GET(create_sq_out, out, sqn);
664	sq->uid = MLX5_GET(create_sq_in, in, uid);
665	err = create_resource_common(dev, sq, MLX5_RES_SQ);
666	if (err)
667		goto err_destroy_sq;
668
669	return 0;
670
671err_destroy_sq:
672	destroy_sq_tracked(dev, sq->qpn, sq->uid);
673
674	return err;
675}
676
677void mlx5_core_destroy_sq_tracked(struct mlx5_ib_dev *dev,
678				  struct mlx5_core_qp *sq)
679{
680	destroy_resource_common(dev, sq);
681	destroy_sq_tracked(dev, sq->qpn, sq->uid);
682}
683
684struct mlx5_core_rsc_common *mlx5_core_res_hold(struct mlx5_ib_dev *dev,
685						int res_num,
686						enum mlx5_res_type res_type)
687{
688	u32 rsn = res_num | (res_type << MLX5_USER_INDEX_LEN);
689	struct mlx5_qp_table *table = &dev->qp_table;
690
691	return mlx5_get_rsc(table, rsn);
692}
693
694void mlx5_core_res_put(struct mlx5_core_rsc_common *res)
695{
696	mlx5_core_put_rsc(res);
697}