devx.c

   1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/*
   3 * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
   4 */
   5
   6#include <rdma/ib_user_verbs.h>
   7#include <rdma/ib_verbs.h>
   8#include <rdma/uverbs_types.h>
   9#include <rdma/uverbs_ioctl.h>
  10#include <rdma/mlx5_user_ioctl_cmds.h>
  11#include <rdma/mlx5_user_ioctl_verbs.h>
  12#include <rdma/ib_umem.h>
  13#include <rdma/uverbs_std_types.h>
  14#include <linux/mlx5/driver.h>
  15#include <linux/mlx5/fs.h>
  16#include <rdma/ib_ucaps.h>
  17#include "mlx5_ib.h"
  18#include "devx.h"
  19#include "qp.h"
  20#include <linux/xarray.h>
  21
  22#define UVERBS_MODULE_NAME mlx5_ib
  23#include <rdma/uverbs_named_ioctl.h>
  24
  25static void dispatch_event_fd(struct list_head *fd_list, const void *data);
  26
  27enum devx_obj_flags {
  28	DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
  29	DEVX_OBJ_FLAGS_DCT = 1 << 1,
  30	DEVX_OBJ_FLAGS_CQ = 1 << 2,
  31	DEVX_OBJ_FLAGS_HW_FREED = 1 << 3,
  32};
  33
  34#define MAX_ASYNC_CMDS 8
  35
  36struct mlx5_async_cmd {
  37	struct ib_uobject *uobject;
  38	void *in;
  39	int in_size;
  40	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
  41	int err;
  42	struct mlx5_async_work cb_work;
  43	struct completion comp;
  44};
  45
  46struct devx_async_data {
  47	struct mlx5_ib_dev *mdev;
  48	struct list_head list;
  49	struct devx_async_cmd_event_file *ev_file;
  50	struct mlx5_async_work cb_work;
  51	u16 cmd_out_len;
  52	/* must be last field in this structure */
  53	struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
  54};
  55
  56struct devx_async_event_data {
  57	struct list_head list; /* headed in ev_file->event_list */
  58	struct mlx5_ib_uapi_devx_async_event_hdr hdr;
  59};
  60
  61/* first level XA value data structure */
  62struct devx_event {
  63	struct xarray object_ids; /* second XA level, Key = object id */
  64	struct list_head unaffiliated_list;
  65};
  66
  67/* second level XA value data structure */
  68struct devx_obj_event {
  69	struct rcu_head rcu;
  70	struct list_head obj_sub_list;
  71};
  72
  73struct devx_event_subscription {
  74	struct list_head file_list; /* headed in ev_file->
  75				     * subscribed_events_list
  76				     */
  77	struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
  78				   * devx_obj_event->obj_sub_list
  79				   */
  80	struct list_head obj_list; /* headed in devx_object */
  81	struct list_head event_list; /* headed in ev_file->event_list or in
  82				      * temp list via subscription
  83				      */
  84
  85	u8 is_cleaned:1;
  86	u32 xa_key_level1;
  87	u32 xa_key_level2;
  88	struct rcu_head	rcu;
  89	u64 cookie;
  90	struct devx_async_event_file *ev_file;
  91	struct eventfd_ctx *eventfd;
  92};
  93
  94struct devx_async_event_file {
  95	struct ib_uobject uobj;
  96	/* Head of events that are subscribed to this FD */
  97	struct list_head subscribed_events_list;
  98	spinlock_t lock;
  99	wait_queue_head_t poll_wait;
 100	struct list_head event_list;
 101	struct mlx5_ib_dev *dev;
 102	u8 omit_data:1;
 103	u8 is_overflow_err:1;
 104	u8 is_destroyed:1;
 105};
 106
 107struct devx_umem {
 108	struct mlx5_core_dev		*mdev;
 109	struct ib_umem			*umem;
 110	u32				dinlen;
 111	u32				dinbox[MLX5_ST_SZ_DW(destroy_umem_in)];
 112};
 113
 114struct devx_umem_reg_cmd {
 115	void				*in;
 116	u32				inlen;
 117	u32				out[MLX5_ST_SZ_DW(create_umem_out)];
 118};
 119
 120static struct mlx5_ib_ucontext *
 121devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
 122{
 123	return to_mucontext(ib_uverbs_get_ucontext(attrs));
 124}
 125
 126static int set_uctx_ucaps(struct mlx5_ib_dev *dev, u64 req_ucaps, u32 *cap)
 127{
 128	if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_LOCAL)) {
 129		if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
 130			*cap |= MLX5_UCTX_CAP_RDMA_CTRL;
 131		else
 132			return -EOPNOTSUPP;
 133	}
 134
 135	if (UCAP_ENABLED(req_ucaps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA)) {
 136		if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
 137		    MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)
 138			*cap |= MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA;
 139		else
 140			return -EOPNOTSUPP;
 141	}
 142
 143	return 0;
 144}
 145
 146int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user, u64 req_ucaps)
 147{
 148	u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {};
 149	u32 out[MLX5_ST_SZ_DW(create_uctx_out)] = {};
 150	void *uctx;
 151	int err;
 152	u16 uid;
 153	u32 cap = 0;
 154
 155	/* 0 means not supported */
 156	if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
 157		return -EINVAL;
 158
 159	uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
 160	if (is_user &&
 161	    (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX) &&
 162	    rdma_dev_has_raw_cap(&dev->ib_dev))
 163		cap |= MLX5_UCTX_CAP_RAW_TX;
 164	if (is_user &&
 165	    (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
 166	     MLX5_UCTX_CAP_INTERNAL_DEV_RES) &&
 167	    capable(CAP_SYS_RAWIO))
 168		cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
 169
 170	if (req_ucaps) {
 171		err = set_uctx_ucaps(dev, req_ucaps, &cap);
 172		if (err)
 173			return err;
 174	}
 175
 176	MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
 177	MLX5_SET(uctx, uctx, cap, cap);
 178
 179	err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 180	if (err)
 181		return err;
 182
 183	uid = MLX5_GET(create_uctx_out, out, uid);
 184	return uid;
 185}
 186
 187void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
 188{
 189	u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {};
 190	u32 out[MLX5_ST_SZ_DW(destroy_uctx_out)] = {};
 191
 192	MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
 193	MLX5_SET(destroy_uctx_in, in, uid, uid);
 194
 195	mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 196}
 197
 198static bool is_legacy_unaffiliated_event_num(u16 event_num)
 199{
 200	switch (event_num) {
 201	case MLX5_EVENT_TYPE_PORT_CHANGE:
 202		return true;
 203	default:
 204		return false;
 205	}
 206}
 207
 208static bool is_legacy_obj_event_num(u16 event_num)
 209{
 210	switch (event_num) {
 211	case MLX5_EVENT_TYPE_PATH_MIG:
 212	case MLX5_EVENT_TYPE_COMM_EST:
 213	case MLX5_EVENT_TYPE_SQ_DRAINED:
 214	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 215	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
 216	case MLX5_EVENT_TYPE_CQ_ERROR:
 217	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 218	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 219	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 220	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 221	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
 222	case MLX5_EVENT_TYPE_DCT_DRAINED:
 223	case MLX5_EVENT_TYPE_COMP:
 224	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
 225	case MLX5_EVENT_TYPE_XRQ_ERROR:
 226		return true;
 227	default:
 228		return false;
 229	}
 230}
 231
 232static u16 get_legacy_obj_type(u16 opcode)
 233{
 234	switch (opcode) {
 235	case MLX5_CMD_OP_CREATE_RQ:
 236	case MLX5_CMD_OP_CREATE_RMP:
 237		return MLX5_EVENT_QUEUE_TYPE_RQ;
 238	case MLX5_CMD_OP_CREATE_QP:
 239		return MLX5_EVENT_QUEUE_TYPE_QP;
 240	case MLX5_CMD_OP_CREATE_SQ:
 241		return MLX5_EVENT_QUEUE_TYPE_SQ;
 242	case MLX5_CMD_OP_CREATE_DCT:
 243		return MLX5_EVENT_QUEUE_TYPE_DCT;
 244	default:
 245		return 0;
 246	}
 247}
 248
 249static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
 250{
 251	u16 opcode;
 252
 253	opcode = (obj->obj_id >> 32) & 0xffff;
 254
 255	if (is_legacy_obj_event_num(event_num))
 256		return get_legacy_obj_type(opcode);
 257
 258	switch (opcode) {
 259	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 260		return (obj->obj_id >> 48);
 261	case MLX5_CMD_OP_CREATE_RQ:
 262		return MLX5_OBJ_TYPE_RQ;
 263	case MLX5_CMD_OP_CREATE_QP:
 264		return MLX5_OBJ_TYPE_QP;
 265	case MLX5_CMD_OP_CREATE_SQ:
 266		return MLX5_OBJ_TYPE_SQ;
 267	case MLX5_CMD_OP_CREATE_DCT:
 268		return MLX5_OBJ_TYPE_DCT;
 269	case MLX5_CMD_OP_CREATE_TIR:
 270		return MLX5_OBJ_TYPE_TIR;
 271	case MLX5_CMD_OP_CREATE_TIS:
 272		return MLX5_OBJ_TYPE_TIS;
 273	case MLX5_CMD_OP_CREATE_PSV:
 274		return MLX5_OBJ_TYPE_PSV;
 275	case MLX5_OBJ_TYPE_MKEY:
 276		return MLX5_OBJ_TYPE_MKEY;
 277	case MLX5_CMD_OP_CREATE_RMP:
 278		return MLX5_OBJ_TYPE_RMP;
 279	case MLX5_CMD_OP_CREATE_XRC_SRQ:
 280		return MLX5_OBJ_TYPE_XRC_SRQ;
 281	case MLX5_CMD_OP_CREATE_XRQ:
 282		return MLX5_OBJ_TYPE_XRQ;
 283	case MLX5_CMD_OP_CREATE_RQT:
 284		return MLX5_OBJ_TYPE_RQT;
 285	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 286		return MLX5_OBJ_TYPE_FLOW_COUNTER;
 287	case MLX5_CMD_OP_CREATE_CQ:
 288		return MLX5_OBJ_TYPE_CQ;
 289	default:
 290		return 0;
 291	}
 292}
 293
 294static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
 295{
 296	switch (event_type) {
 297	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 298	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 299	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 300	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 301	case MLX5_EVENT_TYPE_PATH_MIG:
 302	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 303	case MLX5_EVENT_TYPE_COMM_EST:
 304	case MLX5_EVENT_TYPE_SQ_DRAINED:
 305	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
 306	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
 307		return eqe->data.qp_srq.type;
 308	case MLX5_EVENT_TYPE_CQ_ERROR:
 309	case MLX5_EVENT_TYPE_XRQ_ERROR:
 310		return 0;
 311	case MLX5_EVENT_TYPE_DCT_DRAINED:
 312	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
 313		return MLX5_EVENT_QUEUE_TYPE_DCT;
 314	default:
 315		return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
 316	}
 317}
 318
 319static u32 get_dec_obj_id(u64 obj_id)
 320{
 321	return (obj_id & 0xffffffff);
 322}
 323
 324/*
 325 * As the obj_id in the firmware is not globally unique the object type
 326 * must be considered upon checking for a valid object id.
 327 * For that the opcode of the creator command is encoded as part of the obj_id.
 328 */
 329static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
 330{
 331	return ((u64)opcode << 32) | obj_id;
 332}
 333
 334static u32 devx_get_created_obj_id(const void *in, const void *out, u16 opcode)
 335{
 336	switch (opcode) {
 337	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 338		return MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 339	case MLX5_CMD_OP_CREATE_UMEM:
 340		return MLX5_GET(create_umem_out, out, umem_id);
 341	case MLX5_CMD_OP_CREATE_MKEY:
 342		return MLX5_GET(create_mkey_out, out, mkey_index);
 343	case MLX5_CMD_OP_CREATE_CQ:
 344		return MLX5_GET(create_cq_out, out, cqn);
 345	case MLX5_CMD_OP_ALLOC_PD:
 346		return MLX5_GET(alloc_pd_out, out, pd);
 347	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 348		return MLX5_GET(alloc_transport_domain_out, out,
 349				transport_domain);
 350	case MLX5_CMD_OP_CREATE_RMP:
 351		return MLX5_GET(create_rmp_out, out, rmpn);
 352	case MLX5_CMD_OP_CREATE_SQ:
 353		return MLX5_GET(create_sq_out, out, sqn);
 354	case MLX5_CMD_OP_CREATE_RQ:
 355		return MLX5_GET(create_rq_out, out, rqn);
 356	case MLX5_CMD_OP_CREATE_RQT:
 357		return MLX5_GET(create_rqt_out, out, rqtn);
 358	case MLX5_CMD_OP_CREATE_TIR:
 359		return MLX5_GET(create_tir_out, out, tirn);
 360	case MLX5_CMD_OP_CREATE_TIS:
 361		return MLX5_GET(create_tis_out, out, tisn);
 362	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 363		return MLX5_GET(alloc_q_counter_out, out, counter_set_id);
 364	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 365		return MLX5_GET(create_flow_table_out, out, table_id);
 366	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 367		return MLX5_GET(create_flow_group_out, out, group_id);
 368	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 369		return MLX5_GET(set_fte_in, in, flow_index);
 370	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 371		return MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
 372	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 373		return MLX5_GET(alloc_packet_reformat_context_out, out,
 374				packet_reformat_id);
 375	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 376		return MLX5_GET(alloc_modify_header_context_out, out,
 377				modify_header_id);
 378	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 379		return MLX5_GET(create_scheduling_element_out, out,
 380				scheduling_element_id);
 381	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 382		return MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
 383	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 384		return MLX5_GET(set_l2_table_entry_in, in, table_index);
 385	case MLX5_CMD_OP_CREATE_QP:
 386		return MLX5_GET(create_qp_out, out, qpn);
 387	case MLX5_CMD_OP_CREATE_SRQ:
 388		return MLX5_GET(create_srq_out, out, srqn);
 389	case MLX5_CMD_OP_CREATE_XRC_SRQ:
 390		return MLX5_GET(create_xrc_srq_out, out, xrc_srqn);
 391	case MLX5_CMD_OP_CREATE_DCT:
 392		return MLX5_GET(create_dct_out, out, dctn);
 393	case MLX5_CMD_OP_CREATE_XRQ:
 394		return MLX5_GET(create_xrq_out, out, xrqn);
 395	case MLX5_CMD_OP_ATTACH_TO_MCG:
 396		return MLX5_GET(attach_to_mcg_in, in, qpn);
 397	case MLX5_CMD_OP_ALLOC_XRCD:
 398		return MLX5_GET(alloc_xrcd_out, out, xrcd);
 399	case MLX5_CMD_OP_CREATE_PSV:
 400		return MLX5_GET(create_psv_out, out, psv0_index);
 401	default:
 402		/* The entry must match to one of the devx_is_obj_create_cmd */
 403		WARN_ON(true);
 404		return 0;
 405	}
 406}
 407
 408static u64 devx_get_obj_id(const void *in)
 409{
 410	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 411	u64 obj_id;
 412
 413	switch (opcode) {
 414	case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 415	case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 416		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
 417					MLX5_GET(general_obj_in_cmd_hdr, in,
 418						 obj_type) << 16,
 419					MLX5_GET(general_obj_in_cmd_hdr, in,
 420						 obj_id));
 421		break;
 422	case MLX5_CMD_OP_QUERY_MKEY:
 423		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
 424					MLX5_GET(query_mkey_in, in,
 425						 mkey_index));
 426		break;
 427	case MLX5_CMD_OP_QUERY_CQ:
 428		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 429					MLX5_GET(query_cq_in, in, cqn));
 430		break;
 431	case MLX5_CMD_OP_MODIFY_CQ:
 432		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 433					MLX5_GET(modify_cq_in, in, cqn));
 434		break;
 435	case MLX5_CMD_OP_QUERY_SQ:
 436		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 437					MLX5_GET(query_sq_in, in, sqn));
 438		break;
 439	case MLX5_CMD_OP_MODIFY_SQ:
 440		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 441					MLX5_GET(modify_sq_in, in, sqn));
 442		break;
 443	case MLX5_CMD_OP_QUERY_RQ:
 444		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 445					MLX5_GET(query_rq_in, in, rqn));
 446		break;
 447	case MLX5_CMD_OP_MODIFY_RQ:
 448		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 449					MLX5_GET(modify_rq_in, in, rqn));
 450		break;
 451	case MLX5_CMD_OP_QUERY_RMP:
 452		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 453					MLX5_GET(query_rmp_in, in, rmpn));
 454		break;
 455	case MLX5_CMD_OP_MODIFY_RMP:
 456		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 457					MLX5_GET(modify_rmp_in, in, rmpn));
 458		break;
 459	case MLX5_CMD_OP_QUERY_RQT:
 460		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 461					MLX5_GET(query_rqt_in, in, rqtn));
 462		break;
 463	case MLX5_CMD_OP_MODIFY_RQT:
 464		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 465					MLX5_GET(modify_rqt_in, in, rqtn));
 466		break;
 467	case MLX5_CMD_OP_QUERY_TIR:
 468		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 469					MLX5_GET(query_tir_in, in, tirn));
 470		break;
 471	case MLX5_CMD_OP_MODIFY_TIR:
 472		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 473					MLX5_GET(modify_tir_in, in, tirn));
 474		break;
 475	case MLX5_CMD_OP_QUERY_TIS:
 476		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 477					MLX5_GET(query_tis_in, in, tisn));
 478		break;
 479	case MLX5_CMD_OP_MODIFY_TIS:
 480		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 481					MLX5_GET(modify_tis_in, in, tisn));
 482		break;
 483	case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 484		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 485					MLX5_GET(query_flow_table_in, in,
 486						 table_id));
 487		break;
 488	case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 489		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 490					MLX5_GET(modify_flow_table_in, in,
 491						 table_id));
 492		break;
 493	case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 494		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
 495					MLX5_GET(query_flow_group_in, in,
 496						 group_id));
 497		break;
 498	case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 499		obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 500					MLX5_GET(query_fte_in, in,
 501						 flow_index));
 502		break;
 503	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 504		obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 505					MLX5_GET(set_fte_in, in, flow_index));
 506		break;
 507	case MLX5_CMD_OP_QUERY_Q_COUNTER:
 508		obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
 509					MLX5_GET(query_q_counter_in, in,
 510						 counter_set_id));
 511		break;
 512	case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 513		obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
 514					MLX5_GET(query_flow_counter_in, in,
 515						 flow_counter_id));
 516		break;
 517	case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 518		obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
 519					MLX5_GET(query_modify_header_context_in,
 520						 in, modify_header_id));
 521		break;
 522	case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 523		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 524					MLX5_GET(query_scheduling_element_in,
 525						 in, scheduling_element_id));
 526		break;
 527	case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 528		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 529					MLX5_GET(modify_scheduling_element_in,
 530						 in, scheduling_element_id));
 531		break;
 532	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 533		obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
 534					MLX5_GET(add_vxlan_udp_dport_in, in,
 535						 vxlan_udp_port));
 536		break;
 537	case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 538		obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 539					MLX5_GET(query_l2_table_entry_in, in,
 540						 table_index));
 541		break;
 542	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 543		obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 544					MLX5_GET(set_l2_table_entry_in, in,
 545						 table_index));
 546		break;
 547	case MLX5_CMD_OP_QUERY_QP:
 548		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 549					MLX5_GET(query_qp_in, in, qpn));
 550		break;
 551	case MLX5_CMD_OP_RST2INIT_QP:
 552		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 553					MLX5_GET(rst2init_qp_in, in, qpn));
 554		break;
 555	case MLX5_CMD_OP_INIT2INIT_QP:
 556		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 557					MLX5_GET(init2init_qp_in, in, qpn));
 558		break;
 559	case MLX5_CMD_OP_INIT2RTR_QP:
 560		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 561					MLX5_GET(init2rtr_qp_in, in, qpn));
 562		break;
 563	case MLX5_CMD_OP_RTR2RTS_QP:
 564		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 565					MLX5_GET(rtr2rts_qp_in, in, qpn));
 566		break;
 567	case MLX5_CMD_OP_RTS2RTS_QP:
 568		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 569					MLX5_GET(rts2rts_qp_in, in, qpn));
 570		break;
 571	case MLX5_CMD_OP_SQERR2RTS_QP:
 572		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 573					MLX5_GET(sqerr2rts_qp_in, in, qpn));
 574		break;
 575	case MLX5_CMD_OP_2ERR_QP:
 576		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 577					MLX5_GET(qp_2err_in, in, qpn));
 578		break;
 579	case MLX5_CMD_OP_2RST_QP:
 580		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 581					MLX5_GET(qp_2rst_in, in, qpn));
 582		break;
 583	case MLX5_CMD_OP_QUERY_DCT:
 584		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 585					MLX5_GET(query_dct_in, in, dctn));
 586		break;
 587	case MLX5_CMD_OP_QUERY_XRQ:
 588	case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 589	case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 590		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 591					MLX5_GET(query_xrq_in, in, xrqn));
 592		break;
 593	case MLX5_CMD_OP_QUERY_XRC_SRQ:
 594		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 595					MLX5_GET(query_xrc_srq_in, in,
 596						 xrc_srqn));
 597		break;
 598	case MLX5_CMD_OP_ARM_XRC_SRQ:
 599		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 600					MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
 601		break;
 602	case MLX5_CMD_OP_QUERY_SRQ:
 603		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
 604					MLX5_GET(query_srq_in, in, srqn));
 605		break;
 606	case MLX5_CMD_OP_ARM_RQ:
 607		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 608					MLX5_GET(arm_rq_in, in, srq_number));
 609		break;
 610	case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 611		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 612					MLX5_GET(drain_dct_in, in, dctn));
 613		break;
 614	case MLX5_CMD_OP_ARM_XRQ:
 615	case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 616	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 617	case MLX5_CMD_OP_MODIFY_XRQ:
 618		obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 619					MLX5_GET(arm_xrq_in, in, xrqn));
 620		break;
 621	case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 622		obj_id = get_enc_obj_id
 623				(MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
 624				 MLX5_GET(query_packet_reformat_context_in,
 625					  in, packet_reformat_id));
 626		break;
 627	default:
 628		obj_id = 0;
 629	}
 630
 631	return obj_id;
 632}
 633
 634static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
 635				 struct ib_uobject *uobj, const void *in)
 636{
 637	struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
 638	u64 obj_id = devx_get_obj_id(in);
 639
 640	if (!obj_id)
 641		return false;
 642
 643	switch (uobj_get_object_id(uobj)) {
 644	case UVERBS_OBJECT_CQ:
 645		return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 646				      to_mcq(uobj->object)->mcq.cqn) ==
 647				      obj_id;
 648
 649	case UVERBS_OBJECT_SRQ:
 650	{
 651		struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
 652		u16 opcode;
 653
 654		switch (srq->common.res) {
 655		case MLX5_RES_XSRQ:
 656			opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
 657			break;
 658		case MLX5_RES_XRQ:
 659			opcode = MLX5_CMD_OP_CREATE_XRQ;
 660			break;
 661		default:
 662			if (!dev->mdev->issi)
 663				opcode = MLX5_CMD_OP_CREATE_SRQ;
 664			else
 665				opcode = MLX5_CMD_OP_CREATE_RMP;
 666		}
 667
 668		return get_enc_obj_id(opcode,
 669				      to_msrq(uobj->object)->msrq.srqn) ==
 670				      obj_id;
 671	}
 672
 673	case UVERBS_OBJECT_QP:
 674	{
 675		struct mlx5_ib_qp *qp = to_mqp(uobj->object);
 676
 677		if (qp->type == IB_QPT_RAW_PACKET ||
 678		    (qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
 679			struct mlx5_ib_raw_packet_qp *raw_packet_qp =
 680							 &qp->raw_packet_qp;
 681			struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
 682			struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
 683
 684			return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 685					       rq->base.mqp.qpn) == obj_id ||
 686				get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 687					       sq->base.mqp.qpn) == obj_id ||
 688				get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 689					       rq->tirn) == obj_id ||
 690				get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 691					       sq->tisn) == obj_id);
 692		}
 693
 694		if (qp->type == MLX5_IB_QPT_DCT)
 695			return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 696					      qp->dct.mdct.mqp.qpn) == obj_id;
 697		return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 698				      qp->ibqp.qp_num) == obj_id;
 699	}
 700
 701	case UVERBS_OBJECT_WQ:
 702		return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 703				      to_mrwq(uobj->object)->core_qp.qpn) ==
 704				      obj_id;
 705
 706	case UVERBS_OBJECT_RWQ_IND_TBL:
 707		return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 708				      to_mrwq_ind_table(uobj->object)->rqtn) ==
 709				      obj_id;
 710
 711	case MLX5_IB_OBJECT_DEVX_OBJ:
 712	{
 713		u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 714		struct devx_obj *devx_uobj = uobj->object;
 715
 716		if (opcode == MLX5_CMD_OP_QUERY_FLOW_COUNTER &&
 717		    devx_uobj->flow_counter_bulk_size) {
 718			u64 end;
 719
 720			end = devx_uobj->obj_id +
 721				devx_uobj->flow_counter_bulk_size;
 722			return devx_uobj->obj_id <= obj_id && end > obj_id;
 723		}
 724
 725		return devx_uobj->obj_id == obj_id;
 726	}
 727
 728	default:
 729		return false;
 730	}
 731}
 732
 733static void devx_set_umem_valid(const void *in)
 734{
 735	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 736
 737	switch (opcode) {
 738	case MLX5_CMD_OP_CREATE_MKEY:
 739		MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
 740		break;
 741	case MLX5_CMD_OP_CREATE_CQ:
 742	{
 743		void *cqc;
 744
 745		MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
 746		cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 747		MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
 748		break;
 749	}
 750	case MLX5_CMD_OP_CREATE_QP:
 751	{
 752		void *qpc;
 753
 754		qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 755		MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
 756		MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
 757		break;
 758	}
 759
 760	case MLX5_CMD_OP_CREATE_RQ:
 761	{
 762		void *rqc, *wq;
 763
 764		rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 765		wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 766		MLX5_SET(wq, wq, dbr_umem_valid, 1);
 767		MLX5_SET(wq, wq, wq_umem_valid, 1);
 768		break;
 769	}
 770
 771	case MLX5_CMD_OP_CREATE_SQ:
 772	{
 773		void *sqc, *wq;
 774
 775		sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 776		wq = MLX5_ADDR_OF(sqc, sqc, wq);
 777		MLX5_SET(wq, wq, dbr_umem_valid, 1);
 778		MLX5_SET(wq, wq, wq_umem_valid, 1);
 779		break;
 780	}
 781
 782	case MLX5_CMD_OP_MODIFY_CQ:
 783		MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
 784		break;
 785
 786	case MLX5_CMD_OP_CREATE_RMP:
 787	{
 788		void *rmpc, *wq;
 789
 790		rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
 791		wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
 792		MLX5_SET(wq, wq, dbr_umem_valid, 1);
 793		MLX5_SET(wq, wq, wq_umem_valid, 1);
 794		break;
 795	}
 796
 797	case MLX5_CMD_OP_CREATE_XRQ:
 798	{
 799		void *xrqc, *wq;
 800
 801		xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
 802		wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
 803		MLX5_SET(wq, wq, dbr_umem_valid, 1);
 804		MLX5_SET(wq, wq, wq_umem_valid, 1);
 805		break;
 806	}
 807
 808	case MLX5_CMD_OP_CREATE_XRC_SRQ:
 809	{
 810		void *xrc_srqc;
 811
 812		MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
 813		xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
 814					xrc_srq_context_entry);
 815		MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
 816		break;
 817	}
 818
 819	default:
 820		return;
 821	}
 822}
 823
 824static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
 825{
 826	*opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 827
 828	switch (*opcode) {
 829	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 830	case MLX5_CMD_OP_CREATE_MKEY:
 831	case MLX5_CMD_OP_CREATE_CQ:
 832	case MLX5_CMD_OP_ALLOC_PD:
 833	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 834	case MLX5_CMD_OP_CREATE_RMP:
 835	case MLX5_CMD_OP_CREATE_SQ:
 836	case MLX5_CMD_OP_CREATE_RQ:
 837	case MLX5_CMD_OP_CREATE_RQT:
 838	case MLX5_CMD_OP_CREATE_TIR:
 839	case MLX5_CMD_OP_CREATE_TIS:
 840	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 841	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 842	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 843	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 844	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 845	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 846	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 847	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 848	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 849	case MLX5_CMD_OP_CREATE_QP:
 850	case MLX5_CMD_OP_CREATE_SRQ:
 851	case MLX5_CMD_OP_CREATE_XRC_SRQ:
 852	case MLX5_CMD_OP_CREATE_DCT:
 853	case MLX5_CMD_OP_CREATE_XRQ:
 854	case MLX5_CMD_OP_ATTACH_TO_MCG:
 855	case MLX5_CMD_OP_ALLOC_XRCD:
 856		return true;
 857	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 858	{
 859		u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 860		if (op_mod == 0)
 861			return true;
 862		return false;
 863	}
 864	case MLX5_CMD_OP_CREATE_PSV:
 865	{
 866		u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
 867
 868		if (num_psv == 1)
 869			return true;
 870		return false;
 871	}
 872	default:
 873		return false;
 874	}
 875}
 876
 877static bool devx_is_obj_modify_cmd(const void *in)
 878{
 879	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 880
 881	switch (opcode) {
 882	case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 883	case MLX5_CMD_OP_MODIFY_CQ:
 884	case MLX5_CMD_OP_MODIFY_RMP:
 885	case MLX5_CMD_OP_MODIFY_SQ:
 886	case MLX5_CMD_OP_MODIFY_RQ:
 887	case MLX5_CMD_OP_MODIFY_RQT:
 888	case MLX5_CMD_OP_MODIFY_TIR:
 889	case MLX5_CMD_OP_MODIFY_TIS:
 890	case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 891	case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 892	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 893	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 894	case MLX5_CMD_OP_RST2INIT_QP:
 895	case MLX5_CMD_OP_INIT2RTR_QP:
 896	case MLX5_CMD_OP_INIT2INIT_QP:
 897	case MLX5_CMD_OP_RTR2RTS_QP:
 898	case MLX5_CMD_OP_RTS2RTS_QP:
 899	case MLX5_CMD_OP_SQERR2RTS_QP:
 900	case MLX5_CMD_OP_2ERR_QP:
 901	case MLX5_CMD_OP_2RST_QP:
 902	case MLX5_CMD_OP_ARM_XRC_SRQ:
 903	case MLX5_CMD_OP_ARM_RQ:
 904	case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 905	case MLX5_CMD_OP_ARM_XRQ:
 906	case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
 907	case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
 908	case MLX5_CMD_OP_MODIFY_XRQ:
 909		return true;
 910	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 911	{
 912		u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 913
 914		if (op_mod == 1)
 915			return true;
 916		return false;
 917	}
 918	default:
 919		return false;
 920	}
 921}
 922
 923static bool devx_is_obj_query_cmd(const void *in)
 924{
 925	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 926
 927	switch (opcode) {
 928	case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 929	case MLX5_CMD_OP_QUERY_MKEY:
 930	case MLX5_CMD_OP_QUERY_CQ:
 931	case MLX5_CMD_OP_QUERY_RMP:
 932	case MLX5_CMD_OP_QUERY_SQ:
 933	case MLX5_CMD_OP_QUERY_RQ:
 934	case MLX5_CMD_OP_QUERY_RQT:
 935	case MLX5_CMD_OP_QUERY_TIR:
 936	case MLX5_CMD_OP_QUERY_TIS:
 937	case MLX5_CMD_OP_QUERY_Q_COUNTER:
 938	case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 939	case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 940	case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 941	case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 942	case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 943	case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 944	case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 945	case MLX5_CMD_OP_QUERY_QP:
 946	case MLX5_CMD_OP_QUERY_SRQ:
 947	case MLX5_CMD_OP_QUERY_XRC_SRQ:
 948	case MLX5_CMD_OP_QUERY_DCT:
 949	case MLX5_CMD_OP_QUERY_XRQ:
 950	case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
 951	case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
 952	case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
 953		return true;
 954	default:
 955		return false;
 956	}
 957}
 958
 959static bool devx_is_whitelist_cmd(void *in)
 960{
 961	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 962
 963	switch (opcode) {
 964	case MLX5_CMD_OP_QUERY_HCA_CAP:
 965	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 966	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
 967	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
 968		return true;
 969	default:
 970		return false;
 971	}
 972}
 973
 974static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
 975{
 976	if (devx_is_whitelist_cmd(cmd_in)) {
 977		struct mlx5_ib_dev *dev;
 978
 979		if (c->devx_uid)
 980			return c->devx_uid;
 981
 982		dev = to_mdev(c->ibucontext.device);
 983		if (dev->devx_whitelist_uid)
 984			return dev->devx_whitelist_uid;
 985
 986		return -EOPNOTSUPP;
 987	}
 988
 989	if (!c->devx_uid)
 990		return -EINVAL;
 991
 992	return c->devx_uid;
 993}
 994
 995static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
 996{
 997	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 998
 999	/* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
1000	if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
1001	     MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
1002	    (opcode >= MLX5_CMD_OP_GENERAL_START &&
1003	     opcode < MLX5_CMD_OP_GENERAL_END))
1004		return true;
1005
1006	switch (opcode) {
1007	case MLX5_CMD_OP_QUERY_HCA_CAP:
1008	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
1009	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
1010	case MLX5_CMD_OP_QUERY_VPORT_STATE:
1011	case MLX5_CMD_OP_QUERY_ADAPTER:
1012	case MLX5_CMD_OP_QUERY_ISSI:
1013	case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
1014	case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
1015	case MLX5_CMD_OP_QUERY_VNIC_ENV:
1016	case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
1017	case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
1018	case MLX5_CMD_OP_NOP:
1019	case MLX5_CMD_OP_QUERY_CONG_STATUS:
1020	case MLX5_CMD_OP_QUERY_CONG_PARAMS:
1021	case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
1022	case MLX5_CMD_OP_QUERY_LAG:
1023	case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
1024		return true;
1025	default:
1026		return false;
1027	}
1028}
1029
1030static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
1031	struct uverbs_attr_bundle *attrs)
1032{
1033	struct mlx5_ib_ucontext *c;
1034	struct mlx5_ib_dev *dev;
1035	int user_vector;
1036	int dev_eqn;
1037	int err;
1038
1039	if (uverbs_copy_from(&user_vector, attrs,
1040			     MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
1041		return -EFAULT;
1042
1043	c = devx_ufile2uctx(attrs);
1044	if (IS_ERR(c))
1045		return PTR_ERR(c);
1046	dev = to_mdev(c->ibucontext.device);
1047
1048	err = mlx5_comp_eqn_get(dev->mdev, user_vector, &dev_eqn);
1049	if (err < 0)
1050		return err;
1051
1052	if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
1053			   &dev_eqn, sizeof(dev_eqn)))
1054		return -EFAULT;
1055
1056	return 0;
1057}
1058
1059/*
1060 *Security note:
1061 * The hardware protection mechanism works like this: Each device object that
1062 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
1063 * the device specification manual) upon its creation. Then upon doorbell,
1064 * hardware fetches the object context for which the doorbell was rang, and
1065 * validates that the UAR through which the DB was rang matches the UAR ID
1066 * of the object.
1067 * If no match the doorbell is silently ignored by the hardware. Of course,
1068 * the user cannot ring a doorbell on a UAR that was not mapped to it.
1069 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
1070 * mailboxes (except tagging them with UID), we expose to the user its UAR
1071 * ID, so it can embed it in these objects in the expected specification
1072 * format. So the only thing the user can do is hurt itself by creating a
1073 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
1074 * may ring a doorbell on its objects.
1075 * The consequence of that will be that another user can schedule a QP/SQ
1076 * of the buggy user for execution (just insert it to the hardware schedule
1077 * queue or arm its CQ for event generation), no further harm is expected.
1078 */
1079static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
1080	struct uverbs_attr_bundle *attrs)
1081{
1082	struct mlx5_ib_ucontext *c;
1083	struct mlx5_ib_dev *dev;
1084	u32 user_idx;
1085	s32 dev_idx;
1086
1087	c = devx_ufile2uctx(attrs);
1088	if (IS_ERR(c))
1089		return PTR_ERR(c);
1090	dev = to_mdev(c->ibucontext.device);
1091
1092	if (uverbs_copy_from(&user_idx, attrs,
1093			     MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1094		return -EFAULT;
1095
1096	dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
1097	if (dev_idx < 0)
1098		return dev_idx;
1099
1100	if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1101			   &dev_idx, sizeof(dev_idx)))
1102		return -EFAULT;
1103
1104	return 0;
1105}
1106
1107static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
1108	struct uverbs_attr_bundle *attrs)
1109{
1110	struct mlx5_ib_ucontext *c;
1111	struct mlx5_ib_dev *dev;
1112	void *cmd_in = uverbs_attr_get_alloced_ptr(
1113		attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
1114	int cmd_out_len = uverbs_attr_get_len(attrs,
1115					MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1116	void *cmd_out;
1117	int err, err2;
1118	int uid;
1119
1120	c = devx_ufile2uctx(attrs);
1121	if (IS_ERR(c))
1122		return PTR_ERR(c);
1123	dev = to_mdev(c->ibucontext.device);
1124
1125	uid = devx_get_uid(c, cmd_in);
1126	if (uid < 0)
1127		return uid;
1128
1129	/* Only white list of some general HCA commands are allowed for this method. */
1130	if (!devx_is_general_cmd(cmd_in, dev))
1131		return -EINVAL;
1132
1133	cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1134	if (IS_ERR(cmd_out))
1135		return PTR_ERR(cmd_out);
1136
1137	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1138	err = mlx5_cmd_do(dev->mdev, cmd_in,
1139			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1140			  cmd_out, cmd_out_len);
1141	if (err && err != -EREMOTEIO)
1142		return err;
1143
1144	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1145			      cmd_out_len);
1146
1147	return err2 ?: err;
1148}
1149
1150static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1151				       u32 *dinlen,
1152				       u32 *obj_id)
1153{
1154	u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
1155	u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1156
1157	*obj_id = devx_get_created_obj_id(in, out, opcode);
1158	*dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1159	MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1160
1161	switch (opcode) {
1162	case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1163		MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1164		MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1165		MLX5_SET(general_obj_in_cmd_hdr, din, obj_type,
1166			 MLX5_GET(general_obj_in_cmd_hdr, in, obj_type));
1167		break;
1168
1169	case MLX5_CMD_OP_CREATE_UMEM:
1170		MLX5_SET(destroy_umem_in, din, opcode,
1171			 MLX5_CMD_OP_DESTROY_UMEM);
1172		MLX5_SET(destroy_umem_in, din, umem_id, *obj_id);
1173		break;
1174	case MLX5_CMD_OP_CREATE_MKEY:
1175		MLX5_SET(destroy_mkey_in, din, opcode,
1176			 MLX5_CMD_OP_DESTROY_MKEY);
1177		MLX5_SET(destroy_mkey_in, din, mkey_index, *obj_id);
1178		break;
1179	case MLX5_CMD_OP_CREATE_CQ:
1180		MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1181		MLX5_SET(destroy_cq_in, din, cqn, *obj_id);
1182		break;
1183	case MLX5_CMD_OP_ALLOC_PD:
1184		MLX5_SET(dealloc_pd_in, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1185		MLX5_SET(dealloc_pd_in, din, pd, *obj_id);
1186		break;
1187	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1188		MLX5_SET(dealloc_transport_domain_in, din, opcode,
1189			 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1190		MLX5_SET(dealloc_transport_domain_in, din, transport_domain,
1191			 *obj_id);
1192		break;
1193	case MLX5_CMD_OP_CREATE_RMP:
1194		MLX5_SET(destroy_rmp_in, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1195		MLX5_SET(destroy_rmp_in, din, rmpn, *obj_id);
1196		break;
1197	case MLX5_CMD_OP_CREATE_SQ:
1198		MLX5_SET(destroy_sq_in, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1199		MLX5_SET(destroy_sq_in, din, sqn, *obj_id);
1200		break;
1201	case MLX5_CMD_OP_CREATE_RQ:
1202		MLX5_SET(destroy_rq_in, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1203		MLX5_SET(destroy_rq_in, din, rqn, *obj_id);
1204		break;
1205	case MLX5_CMD_OP_CREATE_RQT:
1206		MLX5_SET(destroy_rqt_in, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1207		MLX5_SET(destroy_rqt_in, din, rqtn, *obj_id);
1208		break;
1209	case MLX5_CMD_OP_CREATE_TIR:
1210		MLX5_SET(destroy_tir_in, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1211		MLX5_SET(destroy_tir_in, din, tirn, *obj_id);
1212		break;
1213	case MLX5_CMD_OP_CREATE_TIS:
1214		MLX5_SET(destroy_tis_in, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1215		MLX5_SET(destroy_tis_in, din, tisn, *obj_id);
1216		break;
1217	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1218		MLX5_SET(dealloc_q_counter_in, din, opcode,
1219			 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1220		MLX5_SET(dealloc_q_counter_in, din, counter_set_id, *obj_id);
1221		break;
1222	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1223		*dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1224		MLX5_SET(destroy_flow_table_in, din, other_vport,
1225			 MLX5_GET(create_flow_table_in,  in, other_vport));
1226		MLX5_SET(destroy_flow_table_in, din, vport_number,
1227			 MLX5_GET(create_flow_table_in,  in, vport_number));
1228		MLX5_SET(destroy_flow_table_in, din, other_eswitch,
1229			 MLX5_GET(create_flow_table_in,  in, other_eswitch));
1230		MLX5_SET(destroy_flow_table_in, din, eswitch_owner_vhca_id,
1231			 MLX5_GET(create_flow_table_in, in,
1232				  eswitch_owner_vhca_id));
1233		MLX5_SET(destroy_flow_table_in, din, table_type,
1234			 MLX5_GET(create_flow_table_in,  in, table_type));
1235		MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1236		MLX5_SET(destroy_flow_table_in, din, opcode,
1237			 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1238		break;
1239	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1240		*dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1241		MLX5_SET(destroy_flow_group_in, din, other_vport,
1242			 MLX5_GET(create_flow_group_in, in, other_vport));
1243		MLX5_SET(destroy_flow_group_in, din, vport_number,
1244			 MLX5_GET(create_flow_group_in, in, vport_number));
1245		MLX5_SET(destroy_flow_group_in, din, other_eswitch,
1246			 MLX5_GET(create_flow_group_in, in, other_eswitch));
1247		MLX5_SET(destroy_flow_group_in, din, eswitch_owner_vhca_id,
1248			 MLX5_GET(create_flow_group_in, in,
1249				  eswitch_owner_vhca_id));
1250		MLX5_SET(destroy_flow_group_in, din, table_type,
1251			 MLX5_GET(create_flow_group_in, in, table_type));
1252		MLX5_SET(destroy_flow_group_in, din, table_id,
1253			 MLX5_GET(create_flow_group_in, in, table_id));
1254		MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1255		MLX5_SET(destroy_flow_group_in, din, opcode,
1256			 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1257		break;
1258	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1259		*dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1260		MLX5_SET(delete_fte_in, din, other_vport,
1261			 MLX5_GET(set_fte_in,  in, other_vport));
1262		MLX5_SET(delete_fte_in, din, vport_number,
1263			 MLX5_GET(set_fte_in, in, vport_number));
1264		MLX5_SET(delete_fte_in, din, other_eswitch,
1265			 MLX5_GET(set_fte_in,  in, other_eswitch));
1266		MLX5_SET(delete_fte_in, din, eswitch_owner_vhca_id,
1267			 MLX5_GET(set_fte_in, in, eswitch_owner_vhca_id));
1268		MLX5_SET(delete_fte_in, din, table_type,
1269			 MLX5_GET(set_fte_in, in, table_type));
1270		MLX5_SET(delete_fte_in, din, table_id,
1271			 MLX5_GET(set_fte_in, in, table_id));
1272		MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1273		MLX5_SET(delete_fte_in, din, opcode,
1274			 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1275		break;
1276	case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1277		MLX5_SET(dealloc_flow_counter_in, din, opcode,
1278			 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1279		MLX5_SET(dealloc_flow_counter_in, din, flow_counter_id,
1280			 *obj_id);
1281		break;
1282	case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
1283		MLX5_SET(dealloc_packet_reformat_context_in, din, opcode,
1284			 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
1285		MLX5_SET(dealloc_packet_reformat_context_in, din,
1286			 packet_reformat_id, *obj_id);
1287		break;
1288	case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1289		MLX5_SET(dealloc_modify_header_context_in, din, opcode,
1290			 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1291		MLX5_SET(dealloc_modify_header_context_in, din,
1292			 modify_header_id, *obj_id);
1293		break;
1294	case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1295		*dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1296		MLX5_SET(destroy_scheduling_element_in, din,
1297			 scheduling_hierarchy,
1298			 MLX5_GET(create_scheduling_element_in, in,
1299				  scheduling_hierarchy));
1300		MLX5_SET(destroy_scheduling_element_in, din,
1301			 scheduling_element_id, *obj_id);
1302		MLX5_SET(destroy_scheduling_element_in, din, opcode,
1303			 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1304		break;
1305	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1306		*dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1307		MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1308		MLX5_SET(delete_vxlan_udp_dport_in, din, opcode,
1309			 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1310		break;
1311	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1312		*dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1313		MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1314		MLX5_SET(delete_l2_table_entry_in, din, opcode,
1315			 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1316		break;
1317	case MLX5_CMD_OP_CREATE_QP:
1318		MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1319		MLX5_SET(destroy_qp_in, din, qpn, *obj_id);
1320		break;
1321	case MLX5_CMD_OP_CREATE_SRQ:
1322		MLX5_SET(destroy_srq_in, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1323		MLX5_SET(destroy_srq_in, din, srqn, *obj_id);
1324		break;
1325	case MLX5_CMD_OP_CREATE_XRC_SRQ:
1326		MLX5_SET(destroy_xrc_srq_in, din, opcode,
1327			 MLX5_CMD_OP_DESTROY_XRC_SRQ);
1328		MLX5_SET(destroy_xrc_srq_in, din, xrc_srqn, *obj_id);
1329		break;
1330	case MLX5_CMD_OP_CREATE_DCT:
1331		MLX5_SET(destroy_dct_in, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1332		MLX5_SET(destroy_dct_in, din, dctn, *obj_id);
1333		break;
1334	case MLX5_CMD_OP_CREATE_XRQ:
1335		MLX5_SET(destroy_xrq_in, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1336		MLX5_SET(destroy_xrq_in, din, xrqn, *obj_id);
1337		break;
1338	case MLX5_CMD_OP_ATTACH_TO_MCG:
1339		*dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1340		MLX5_SET(detach_from_mcg_in, din, qpn,
1341			 MLX5_GET(attach_to_mcg_in, in, qpn));
1342		memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1343		       MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1344		       MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1345		MLX5_SET(detach_from_mcg_in, din, opcode,
1346			 MLX5_CMD_OP_DETACH_FROM_MCG);
1347		MLX5_SET(detach_from_mcg_in, din, qpn, *obj_id);
1348		break;
1349	case MLX5_CMD_OP_ALLOC_XRCD:
1350		MLX5_SET(dealloc_xrcd_in, din, opcode,
1351			 MLX5_CMD_OP_DEALLOC_XRCD);
1352		MLX5_SET(dealloc_xrcd_in, din, xrcd, *obj_id);
1353		break;
1354	case MLX5_CMD_OP_CREATE_PSV:
1355		MLX5_SET(destroy_psv_in, din, opcode,
1356			 MLX5_CMD_OP_DESTROY_PSV);
1357		MLX5_SET(destroy_psv_in, din, psvn, *obj_id);
1358		break;
1359	default:
1360		/* The entry must match to one of the devx_is_obj_create_cmd */
1361		WARN_ON(true);
1362		break;
1363	}
1364}
1365
1366static int devx_handle_mkey_indirect(struct devx_obj *obj,
1367				     struct mlx5_ib_dev *dev,
1368				     void *in, void *out)
1369{
1370	struct mlx5_ib_mkey *mkey = &obj->mkey;
1371	void *mkc;
1372	u8 key;
1373
1374	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1375	key = MLX5_GET(mkc, mkc, mkey_7_0);
1376	mkey->key = mlx5_idx_to_mkey(
1377			MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1378	mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1379	mkey->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1380	init_waitqueue_head(&mkey->wait);
1381
1382	return mlx5r_store_odp_mkey(dev, mkey);
1383}
1384
1385static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1386				   struct devx_obj *obj,
1387				   void *in, int in_len)
1388{
1389	int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1390			MLX5_FLD_SZ_BYTES(create_mkey_in,
1391			memory_key_mkey_entry);
1392	void *mkc;
1393	u8 access_mode;
1394
1395	if (in_len < min_len)
1396		return -EINVAL;
1397
1398	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1399
1400	access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1401	access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1402
1403	if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
1404		access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1405		if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1406			obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
1407		return 0;
1408	}
1409
1410	MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1411	/* TPH is not allowed to bypass the regular kernel's verbs flow */
1412	MLX5_SET(mkc, mkc, pcie_tph_en, 0);
1413	MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,
1414		 MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX);
1415	return 0;
1416}
1417
1418static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1419				      struct devx_event_subscription *sub)
1420{
1421	struct devx_event *event;
1422	struct devx_obj_event *xa_val_level2;
1423
1424	if (sub->is_cleaned)
1425		return;
1426
1427	sub->is_cleaned = 1;
1428	list_del_rcu(&sub->xa_list);
1429
1430	if (list_empty(&sub->obj_list))
1431		return;
1432
1433	list_del_rcu(&sub->obj_list);
1434	/* check whether key level 1 for this obj_sub_list is empty */
1435	event = xa_load(&dev->devx_event_table.event_xa,
1436			sub->xa_key_level1);
1437	WARN_ON(!event);
1438
1439	xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1440	if (list_empty(&xa_val_level2->obj_sub_list)) {
1441		xa_erase(&event->object_ids,
1442			 sub->xa_key_level2);
1443		kfree_rcu(xa_val_level2, rcu);
1444	}
1445}
1446
1447static int devx_obj_cleanup(struct ib_uobject *uobject,
1448			    enum rdma_remove_reason why,
1449			    struct uverbs_attr_bundle *attrs)
1450{
1451	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1452	struct mlx5_devx_event_table *devx_event_table;
1453	struct devx_obj *obj = uobject->object;
1454	struct devx_event_subscription *sub_entry, *tmp;
1455	struct mlx5_ib_dev *dev;
1456	int ret;
1457
1458	dev = mlx5_udata_to_mdev(&attrs->driver_udata);
1459	if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY &&
1460	    xa_erase(&obj->ib_dev->odp_mkeys,
1461		     mlx5_base_mkey(obj->mkey.key)))
1462		/*
1463		 * The pagefault_single_data_segment() does commands against
1464		 * the mmkey, we must wait for that to stop before freeing the
1465		 * mkey, as another allocation could get the same mkey #.
1466		 */
1467		mlx5r_deref_wait_odp_mkey(&obj->mkey);
1468
1469	if (obj->flags & DEVX_OBJ_FLAGS_HW_FREED)
1470		ret = 0;
1471	else if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1472		ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1473	else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1474		ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1475	else
1476		ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1477				    obj->dinlen, out, sizeof(out));
1478	if (ret)
1479		return ret;
1480
1481	devx_event_table = &dev->devx_event_table;
1482
1483	mutex_lock(&devx_event_table->event_xa_lock);
1484	list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1485		devx_cleanup_subscription(dev, sub_entry);
1486	mutex_unlock(&devx_event_table->event_xa_lock);
1487
1488	kfree(obj);
1489	return ret;
1490}
1491
1492static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1493{
1494	struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1495	struct mlx5_devx_event_table *table;
1496	struct devx_event *event;
1497	struct devx_obj_event *obj_event;
1498	u32 obj_id = mcq->cqn;
1499
1500	table = &obj->ib_dev->devx_event_table;
1501	rcu_read_lock();
1502	event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1503	if (!event)
1504		goto out;
1505
1506	obj_event = xa_load(&event->object_ids, obj_id);
1507	if (!obj_event)
1508		goto out;
1509
1510	dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1511out:
1512	rcu_read_unlock();
1513}
1514
1515static bool is_apu_cq(struct mlx5_ib_dev *dev, const void *in)
1516{
1517	if (!MLX5_CAP_GEN(dev->mdev, apu) ||
1518	    !MLX5_GET(cqc, MLX5_ADDR_OF(create_cq_in, in, cq_context), apu_cq))
1519		return false;
1520
1521	return true;
1522}
1523
1524static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
1525	struct uverbs_attr_bundle *attrs)
1526{
1527	void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1528	int cmd_out_len =  uverbs_attr_get_len(attrs,
1529					MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
1530	int cmd_in_len = uverbs_attr_get_len(attrs,
1531					MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1532	void *cmd_out;
1533	struct ib_uobject *uobj = uverbs_attr_get_uobject(
1534		attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
1535	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1536		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1537	struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1538	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1539	struct devx_obj *obj;
1540	u16 obj_type = 0;
1541	int err, err2 = 0;
1542	int uid;
1543	u32 obj_id;
1544	u16 opcode;
1545
1546	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1547		return -EINVAL;
1548
1549	uid = devx_get_uid(c, cmd_in);
1550	if (uid < 0)
1551		return uid;
1552
1553	if (!devx_is_obj_create_cmd(cmd_in, &opcode))
1554		return -EINVAL;
1555
1556	cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1557	if (IS_ERR(cmd_out))
1558		return PTR_ERR(cmd_out);
1559
1560	obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1561	if (!obj)
1562		return -ENOMEM;
1563
1564	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1565	if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1566		err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1567		if (err)
1568			goto obj_free;
1569	} else {
1570		devx_set_umem_valid(cmd_in);
1571	}
1572
1573	if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1574		obj->flags |= DEVX_OBJ_FLAGS_DCT;
1575		err = mlx5_core_create_dct(dev, &obj->core_dct, cmd_in,
1576					   cmd_in_len, cmd_out, cmd_out_len);
1577	} else if (opcode == MLX5_CMD_OP_CREATE_CQ &&
1578		   !is_apu_cq(dev, cmd_in)) {
1579		obj->flags |= DEVX_OBJ_FLAGS_CQ;
1580		obj->core_cq.comp = devx_cq_comp;
1581		err = mlx5_create_cq(dev->mdev, &obj->core_cq,
1582				     cmd_in, cmd_in_len, cmd_out,
1583				     cmd_out_len);
1584	} else {
1585		err = mlx5_cmd_do(dev->mdev, cmd_in, cmd_in_len,
1586				  cmd_out, cmd_out_len);
1587	}
1588
1589	if (err == -EREMOTEIO)
1590		err2 = uverbs_copy_to(attrs,
1591				      MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
1592				      cmd_out, cmd_out_len);
1593	if (err)
1594		goto obj_free;
1595
1596	if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1597		u32 bulk = MLX5_GET(alloc_flow_counter_in,
1598				    cmd_in,
1599				    flow_counter_bulk_log_size);
1600
1601		if (bulk)
1602			bulk = 1 << bulk;
1603		else
1604			bulk = 128UL * MLX5_GET(alloc_flow_counter_in,
1605						cmd_in,
1606						flow_counter_bulk);
1607		obj->flow_counter_bulk_size = bulk;
1608	}
1609
1610	uobj->object = obj;
1611	INIT_LIST_HEAD(&obj->event_sub);
1612	obj->ib_dev = dev;
1613	devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1614				   &obj_id);
1615	WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1616
1617	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1618	if (err)
1619		goto obj_destroy;
1620
1621	if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1622		obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1623	obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1624
1625	if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1626		err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1627		if (err)
1628			goto obj_destroy;
1629	}
1630	return 0;
1631
1632obj_destroy:
1633	if (obj->flags & DEVX_OBJ_FLAGS_DCT)
1634		mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
1635	else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
1636		mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
1637	else
1638		mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
1639			      sizeof(out));
1640obj_free:
1641	kfree(obj);
1642	return err2 ?: err;
1643}
1644
1645static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
1646	struct uverbs_attr_bundle *attrs)
1647{
1648	void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1649	int cmd_out_len = uverbs_attr_get_len(attrs,
1650					MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1651	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1652							  MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
1653	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1654		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1655	struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1656	void *cmd_out;
1657	int err, err2;
1658	int uid;
1659
1660	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1661		return -EINVAL;
1662
1663	uid = devx_get_uid(c, cmd_in);
1664	if (uid < 0)
1665		return uid;
1666
1667	if (!devx_is_obj_modify_cmd(cmd_in))
1668		return -EINVAL;
1669
1670	if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1671		return -EINVAL;
1672
1673	cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1674	if (IS_ERR(cmd_out))
1675		return PTR_ERR(cmd_out);
1676
1677	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1678	devx_set_umem_valid(cmd_in);
1679
1680	err = mlx5_cmd_do(mdev->mdev, cmd_in,
1681			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1682			  cmd_out, cmd_out_len);
1683	if (err && err != -EREMOTEIO)
1684		return err;
1685
1686	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1687			      cmd_out, cmd_out_len);
1688
1689	return err2 ?: err;
1690}
1691
1692static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1693	struct uverbs_attr_bundle *attrs)
1694{
1695	void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1696	int cmd_out_len = uverbs_attr_get_len(attrs,
1697					      MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1698	struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1699							  MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1700	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1701		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1702	void *cmd_out;
1703	int err, err2;
1704	int uid;
1705	struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1706
1707	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1708		return -EINVAL;
1709
1710	uid = devx_get_uid(c, cmd_in);
1711	if (uid < 0)
1712		return uid;
1713
1714	if (!devx_is_obj_query_cmd(cmd_in))
1715		return -EINVAL;
1716
1717	if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1718		return -EINVAL;
1719
1720	cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1721	if (IS_ERR(cmd_out))
1722		return PTR_ERR(cmd_out);
1723
1724	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1725	err = mlx5_cmd_do(mdev->mdev, cmd_in,
1726			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1727			  cmd_out, cmd_out_len);
1728	if (err && err != -EREMOTEIO)
1729		return err;
1730
1731	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1732			      cmd_out, cmd_out_len);
1733
1734	return err2 ?: err;
1735}
1736
1737struct devx_async_event_queue {
1738	spinlock_t		lock;
1739	wait_queue_head_t	poll_wait;
1740	struct list_head	event_list;
1741	atomic_t		bytes_in_use;
1742	u8			is_destroyed:1;
1743};
1744
1745struct devx_async_cmd_event_file {
1746	struct ib_uobject		uobj;
1747	struct devx_async_event_queue	ev_queue;
1748	struct mlx5_async_ctx		async_ctx;
1749};
1750
1751static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1752{
1753	spin_lock_init(&ev_queue->lock);
1754	INIT_LIST_HEAD(&ev_queue->event_list);
1755	init_waitqueue_head(&ev_queue->poll_wait);
1756	atomic_set(&ev_queue->bytes_in_use, 0);
1757	ev_queue->is_destroyed = 0;
1758}
1759
1760static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1761	struct uverbs_attr_bundle *attrs)
1762{
1763	struct devx_async_cmd_event_file *ev_file;
1764
1765	struct ib_uobject *uobj = uverbs_attr_get_uobject(
1766		attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
1767	struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
1768
1769	ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1770			       uobj);
1771	devx_init_event_queue(&ev_file->ev_queue);
1772	mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
1773	return 0;
1774}
1775
1776static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1777	struct uverbs_attr_bundle *attrs)
1778{
1779	struct ib_uobject *uobj = uverbs_attr_get_uobject(
1780		attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1781	struct devx_async_event_file *ev_file;
1782	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1783		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1784	struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1785	u32 flags;
1786	int err;
1787
1788	err = uverbs_get_flags32(&flags, attrs,
1789		MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1790		MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1791
1792	if (err)
1793		return err;
1794
1795	ev_file = container_of(uobj, struct devx_async_event_file,
1796			       uobj);
1797	spin_lock_init(&ev_file->lock);
1798	INIT_LIST_HEAD(&ev_file->event_list);
1799	init_waitqueue_head(&ev_file->poll_wait);
1800	if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1801		ev_file->omit_data = 1;
1802	INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1803	ev_file->dev = dev;
1804	get_device(&dev->ib_dev.dev);
1805	return 0;
1806}
1807
1808static void devx_query_callback(int status, struct mlx5_async_work *context)
1809{
1810	struct devx_async_data *async_data =
1811		container_of(context, struct devx_async_data, cb_work);
1812	struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1813	struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
1814	unsigned long flags;
1815
1816	/*
1817	 * Note that if the struct devx_async_cmd_event_file uobj begins to be
1818	 * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1819	 * routine returns, ensuring that it always remains valid here.
1820	 */
1821	spin_lock_irqsave(&ev_queue->lock, flags);
1822	list_add_tail(&async_data->list, &ev_queue->event_list);
1823	spin_unlock_irqrestore(&ev_queue->lock, flags);
1824
1825	wake_up_interruptible(&ev_queue->poll_wait);
1826}
1827
1828#define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1829
1830static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1831	struct uverbs_attr_bundle *attrs)
1832{
1833	void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1834				MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1835	struct ib_uobject *uobj = uverbs_attr_get_uobject(
1836				attrs,
1837				MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1838	u16 cmd_out_len;
1839	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1840		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1841	struct ib_uobject *fd_uobj;
1842	int err;
1843	int uid;
1844	struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
1845	struct devx_async_cmd_event_file *ev_file;
1846	struct devx_async_data *async_data;
1847
1848	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1849		return -EINVAL;
1850
1851	uid = devx_get_uid(c, cmd_in);
1852	if (uid < 0)
1853		return uid;
1854
1855	if (!devx_is_obj_query_cmd(cmd_in))
1856		return -EINVAL;
1857
1858	err = uverbs_get_const(&cmd_out_len, attrs,
1859			       MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1860	if (err)
1861		return err;
1862
1863	if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
1864		return -EINVAL;
1865
1866	fd_uobj = uverbs_attr_get_uobject(attrs,
1867				MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1868	if (IS_ERR(fd_uobj))
1869		return PTR_ERR(fd_uobj);
1870
1871	ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1872			       uobj);
1873
1874	if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1875			MAX_ASYNC_BYTES_IN_USE) {
1876		atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1877		return -EAGAIN;
1878	}
1879
1880	async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1881					  cmd_out_len), GFP_KERNEL);
1882	if (!async_data) {
1883		err = -ENOMEM;
1884		goto sub_bytes;
1885	}
1886
1887	err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1888			       MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1889	if (err)
1890		goto free_async;
1891
1892	async_data->cmd_out_len = cmd_out_len;
1893	async_data->mdev = mdev;
1894	async_data->ev_file = ev_file;
1895
1896	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1897	err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1898		    uverbs_attr_get_len(attrs,
1899				MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1900		    async_data->hdr.out_data,
1901		    async_data->cmd_out_len,
1902		    devx_query_callback, &async_data->cb_work);
1903
1904	if (err)
1905		goto free_async;
1906
1907	return 0;
1908
1909free_async:
1910	kvfree(async_data);
1911sub_bytes:
1912	atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1913	return err;
1914}
1915
1916static void
1917subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1918			   u32 key_level1,
1919			   bool is_level2,
1920			   u32 key_level2)
1921{
1922	struct devx_event *event;
1923	struct devx_obj_event *xa_val_level2;
1924
1925	/* Level 1 is valid for future use, no need to free */
1926	if (!is_level2)
1927		return;
1928
1929	event = xa_load(&devx_event_table->event_xa, key_level1);
1930	WARN_ON(!event);
1931
1932	xa_val_level2 = xa_load(&event->object_ids,
1933				key_level2);
1934	if (list_empty(&xa_val_level2->obj_sub_list)) {
1935		xa_erase(&event->object_ids,
1936			 key_level2);
1937		kfree_rcu(xa_val_level2, rcu);
1938	}
1939}
1940
1941static int
1942subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1943			 u32 key_level1,
1944			 bool is_level2,
1945			 u32 key_level2)
1946{
1947	struct devx_obj_event *obj_event;
1948	struct devx_event *event;
1949	int err;
1950
1951	event = xa_load(&devx_event_table->event_xa, key_level1);
1952	if (!event) {
1953		event = kzalloc(sizeof(*event), GFP_KERNEL);
1954		if (!event)
1955			return -ENOMEM;
1956
1957		INIT_LIST_HEAD(&event->unaffiliated_list);
1958		xa_init(&event->object_ids);
1959
1960		err = xa_insert(&devx_event_table->event_xa,
1961				key_level1,
1962				event,
1963				GFP_KERNEL);
1964		if (err) {
1965			kfree(event);
1966			return err;
1967		}
1968	}
1969
1970	if (!is_level2)
1971		return 0;
1972
1973	obj_event = xa_load(&event->object_ids, key_level2);
1974	if (!obj_event) {
1975		obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1976		if (!obj_event)
1977			/* Level1 is valid for future use, no need to free */
1978			return -ENOMEM;
1979
1980		INIT_LIST_HEAD(&obj_event->obj_sub_list);
1981		err = xa_insert(&event->object_ids,
1982				key_level2,
1983				obj_event,
1984				GFP_KERNEL);
1985		if (err) {
1986			kfree(obj_event);
1987			return err;
1988		}
1989	}
1990
1991	return 0;
1992}
1993
1994static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1995				   struct devx_obj *obj)
1996{
1997	int i;
1998
1999	for (i = 0; i < num_events; i++) {
2000		if (obj) {
2001			if (!is_legacy_obj_event_num(event_type_num_list[i]))
2002				return false;
2003		} else if (!is_legacy_unaffiliated_event_num(
2004				event_type_num_list[i])) {
2005			return false;
2006		}
2007	}
2008
2009	return true;
2010}
2011
2012#define MAX_SUPP_EVENT_NUM 255
2013static bool is_valid_events(struct mlx5_core_dev *dev,
2014			    int num_events, u16 *event_type_num_list,
2015			    struct devx_obj *obj)
2016{
2017	__be64 *aff_events;
2018	__be64 *unaff_events;
2019	int mask_entry;
2020	int mask_bit;
2021	int i;
2022
2023	if (MLX5_CAP_GEN(dev, event_cap)) {
2024		aff_events = MLX5_CAP_DEV_EVENT(dev,
2025						user_affiliated_events);
2026		unaff_events = MLX5_CAP_DEV_EVENT(dev,
2027						  user_unaffiliated_events);
2028	} else {
2029		return is_valid_events_legacy(num_events, event_type_num_list,
2030					      obj);
2031	}
2032
2033	for (i = 0; i < num_events; i++) {
2034		if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
2035			return false;
2036
2037		mask_entry = event_type_num_list[i] / 64;
2038		mask_bit = event_type_num_list[i] % 64;
2039
2040		if (obj) {
2041			/* CQ completion */
2042			if (event_type_num_list[i] == 0)
2043				continue;
2044
2045			if (!(be64_to_cpu(aff_events[mask_entry]) &
2046					(1ull << mask_bit)))
2047				return false;
2048
2049			continue;
2050		}
2051
2052		if (!(be64_to_cpu(unaff_events[mask_entry]) &
2053				(1ull << mask_bit)))
2054			return false;
2055	}
2056
2057	return true;
2058}
2059
2060#define MAX_NUM_EVENTS 16
2061static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
2062	struct uverbs_attr_bundle *attrs)
2063{
2064	struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
2065				attrs,
2066				MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
2067	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2068		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2069	struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2070	struct ib_uobject *fd_uobj;
2071	struct devx_obj *obj = NULL;
2072	struct devx_async_event_file *ev_file;
2073	struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
2074	u16 *event_type_num_list;
2075	struct devx_event_subscription *event_sub, *tmp_sub;
2076	struct list_head sub_list;
2077	int redirect_fd;
2078	bool use_eventfd = false;
2079	int num_events;
2080	u16 obj_type = 0;
2081	u64 cookie = 0;
2082	u32 obj_id = 0;
2083	int err;
2084	int i;
2085
2086	if (!c->devx_uid)
2087		return -EINVAL;
2088
2089	if (!IS_ERR(devx_uobj)) {
2090		obj = (struct devx_obj *)devx_uobj->object;
2091		if (obj)
2092			obj_id = get_dec_obj_id(obj->obj_id);
2093	}
2094
2095	fd_uobj = uverbs_attr_get_uobject(attrs,
2096				MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
2097	if (IS_ERR(fd_uobj))
2098		return PTR_ERR(fd_uobj);
2099
2100	ev_file = container_of(fd_uobj, struct devx_async_event_file,
2101			       uobj);
2102
2103	if (uverbs_attr_is_valid(attrs,
2104				 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
2105		err = uverbs_copy_from(&redirect_fd, attrs,
2106			       MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
2107		if (err)
2108			return err;
2109
2110		use_eventfd = true;
2111	}
2112
2113	if (uverbs_attr_is_valid(attrs,
2114				 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
2115		if (use_eventfd)
2116			return -EINVAL;
2117
2118		err = uverbs_copy_from(&cookie, attrs,
2119				MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
2120		if (err)
2121			return err;
2122	}
2123
2124	num_events = uverbs_attr_ptr_get_array_size(
2125		attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2126		sizeof(u16));
2127
2128	if (num_events < 0)
2129		return num_events;
2130
2131	if (num_events > MAX_NUM_EVENTS)
2132		return -EINVAL;
2133
2134	event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
2135			MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
2136
2137	if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
2138		return -EINVAL;
2139
2140	INIT_LIST_HEAD(&sub_list);
2141
2142	/* Protect from concurrent subscriptions to same XA entries to allow
2143	 * both to succeed
2144	 */
2145	mutex_lock(&devx_event_table->event_xa_lock);
2146	for (i = 0; i < num_events; i++) {
2147		u32 key_level1;
2148
2149		if (obj)
2150			obj_type = get_dec_obj_type(obj,
2151						    event_type_num_list[i]);
2152		key_level1 = event_type_num_list[i] | obj_type << 16;
2153
2154		err = subscribe_event_xa_alloc(devx_event_table,
2155					       key_level1,
2156					       obj,
2157					       obj_id);
2158		if (err)
2159			goto err;
2160
2161		event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2162		if (!event_sub) {
2163			err = -ENOMEM;
2164			goto err;
2165		}
2166
2167		list_add_tail(&event_sub->event_list, &sub_list);
2168		uverbs_uobject_get(&ev_file->uobj);
2169		if (use_eventfd) {
2170			event_sub->eventfd =
2171				eventfd_ctx_fdget(redirect_fd);
2172
2173			if (IS_ERR(event_sub->eventfd)) {
2174				err = PTR_ERR(event_sub->eventfd);
2175				event_sub->eventfd = NULL;
2176				goto err;
2177			}
2178		}
2179
2180		event_sub->cookie = cookie;
2181		event_sub->ev_file = ev_file;
2182		/* May be needed upon cleanup the devx object/subscription */
2183		event_sub->xa_key_level1 = key_level1;
2184		event_sub->xa_key_level2 = obj_id;
2185		INIT_LIST_HEAD(&event_sub->obj_list);
2186	}
2187
2188	/* Once all the allocations and the XA data insertions were done we
2189	 * can go ahead and add all the subscriptions to the relevant lists
2190	 * without concern of a failure.
2191	 */
2192	list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2193		struct devx_event *event;
2194		struct devx_obj_event *obj_event;
2195
2196		list_del_init(&event_sub->event_list);
2197
2198		spin_lock_irq(&ev_file->lock);
2199		list_add_tail_rcu(&event_sub->file_list,
2200				  &ev_file->subscribed_events_list);
2201		spin_unlock_irq(&ev_file->lock);
2202
2203		event = xa_load(&devx_event_table->event_xa,
2204				event_sub->xa_key_level1);
2205		WARN_ON(!event);
2206
2207		if (!obj) {
2208			list_add_tail_rcu(&event_sub->xa_list,
2209					  &event->unaffiliated_list);
2210			continue;
2211		}
2212
2213		obj_event = xa_load(&event->object_ids, obj_id);
2214		WARN_ON(!obj_event);
2215		list_add_tail_rcu(&event_sub->xa_list,
2216				  &obj_event->obj_sub_list);
2217		list_add_tail_rcu(&event_sub->obj_list,
2218				  &obj->event_sub);
2219	}
2220
2221	mutex_unlock(&devx_event_table->event_xa_lock);
2222	return 0;
2223
2224err:
2225	list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2226		list_del(&event_sub->event_list);
2227
2228		subscribe_event_xa_dealloc(devx_event_table,
2229					   event_sub->xa_key_level1,
2230					   obj,
2231					   obj_id);
2232
2233		if (event_sub->eventfd)
2234			eventfd_ctx_put(event_sub->eventfd);
2235		uverbs_uobject_put(&event_sub->ev_file->uobj);
2236		kfree(event_sub);
2237	}
2238
2239	mutex_unlock(&devx_event_table->event_xa_lock);
2240	return err;
2241}
2242
2243static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2244			 struct uverbs_attr_bundle *attrs,
2245			 struct devx_umem *obj, u32 access_flags)
2246{
2247	u64 addr;
2248	size_t size;
2249	int err;
2250
2251	if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
2252	    uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
2253		return -EFAULT;
2254
2255	err = ib_check_mr_access(&dev->ib_dev, access_flags);
2256	if (err)
2257		return err;
2258
2259	if (uverbs_attr_is_valid(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD)) {
2260		struct ib_umem_dmabuf *umem_dmabuf;
2261		int dmabuf_fd;
2262
2263		err = uverbs_get_raw_fd(&dmabuf_fd, attrs,
2264					MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD);
2265		if (err)
2266			return -EFAULT;
2267
2268		umem_dmabuf = ib_umem_dmabuf_get_pinned(
2269			&dev->ib_dev, addr, size, dmabuf_fd, access_flags);
2270		if (IS_ERR(umem_dmabuf))
2271			return PTR_ERR(umem_dmabuf);
2272		obj->umem = &umem_dmabuf->umem;
2273	} else {
2274		obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access_flags);
2275		if (IS_ERR(obj->umem))
2276			return PTR_ERR(obj->umem);
2277	}
2278	return 0;
2279}
2280
2281static unsigned int devx_umem_find_best_pgsize(struct ib_umem *umem,
2282					       unsigned long pgsz_bitmap)
2283{
2284	unsigned long page_size;
2285
2286	/* Don't bother checking larger page sizes as offset must be zero and
2287	 * total DEVX umem length must be equal to total umem length.
2288	 */
2289	pgsz_bitmap &= GENMASK_ULL(max_t(u64, order_base_2(umem->length),
2290					 PAGE_SHIFT),
2291				   MLX5_ADAPTER_PAGE_SHIFT);
2292	if (!pgsz_bitmap)
2293		return 0;
2294
2295	page_size = ib_umem_find_best_pgoff(umem, pgsz_bitmap, U64_MAX);
2296	if (!page_size)
2297		return 0;
2298
2299	/* If the page_size is less than the CPU page size then we can use the
2300	 * offset and create a umem which is a subset of the page list.
2301	 * For larger page sizes we can't be sure the DMA  list reflects the
2302	 * VA so we must ensure that the umem extent is exactly equal to the
2303	 * page list. Reduce the page size until one of these cases is true.
2304	 */
2305	while ((ib_umem_dma_offset(umem, page_size) != 0 ||
2306		(umem->length % page_size) != 0) &&
2307		page_size > PAGE_SIZE)
2308		page_size /= 2;
2309
2310	return page_size;
2311}
2312
2313static int devx_umem_reg_cmd_alloc(struct mlx5_ib_dev *dev,
2314				   struct uverbs_attr_bundle *attrs,
2315				   struct devx_umem *obj,
2316				   struct devx_umem_reg_cmd *cmd,
2317				   int access)
2318{
2319	unsigned long pgsz_bitmap;
2320	unsigned int page_size;
2321	__be64 *mtt;
2322	void *umem;
2323	int ret;
2324
2325	/*
2326	 * If the user does not pass in pgsz_bitmap then the user promises not
2327	 * to use umem_offset!=0 in any commands that allocate on top of the
2328	 * umem.
2329	 *
2330	 * If the user wants to use a umem_offset then it must pass in
2331	 * pgsz_bitmap which guides the maximum page size and thus maximum
2332	 * object alignment inside the umem. See the PRM.
2333	 *
2334	 * Users are not allowed to use IOVA here, mkeys are not supported on
2335	 * umem.
2336	 */
2337	ret = uverbs_get_const_default(&pgsz_bitmap, attrs,
2338			MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
2339			GENMASK_ULL(63,
2340				    min(PAGE_SHIFT, MLX5_ADAPTER_PAGE_SHIFT)));
2341	if (ret)
2342		return ret;
2343
2344	page_size = devx_umem_find_best_pgsize(obj->umem, pgsz_bitmap);
2345	if (!page_size)
2346		return -EINVAL;
2347
2348	cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2349		     (MLX5_ST_SZ_BYTES(mtt) *
2350		      ib_umem_num_dma_blocks(obj->umem, page_size));
2351	cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2352	if (IS_ERR(cmd->in))
2353		return PTR_ERR(cmd->in);
2354
2355	umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2356	mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2357
2358	MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
2359	MLX5_SET64(umem, umem, num_of_mtt,
2360		   ib_umem_num_dma_blocks(obj->umem, page_size));
2361	MLX5_SET(umem, umem, log_page_size,
2362		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
2363	MLX5_SET(umem, umem, page_offset,
2364		 ib_umem_dma_offset(obj->umem, page_size));
2365
2366	if (mlx5_umem_needs_ats(dev, obj->umem, access))
2367		MLX5_SET(umem, umem, ats, 1);
2368
2369	mlx5_ib_populate_pas(obj->umem, page_size, mtt,
2370			     (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2371				     MLX5_IB_MTT_READ);
2372	return 0;
2373}
2374
2375static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
2376	struct uverbs_attr_bundle *attrs)
2377{
2378	struct devx_umem_reg_cmd cmd;
2379	struct devx_umem *obj;
2380	struct ib_uobject *uobj = uverbs_attr_get_uobject(
2381		attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2382	u32 obj_id;
2383	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2384		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
2385	struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
2386	int access_flags;
2387	int err;
2388
2389	if (!c->devx_uid)
2390		return -EINVAL;
2391
2392	err = uverbs_get_flags32(&access_flags, attrs,
2393				 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2394				 IB_ACCESS_LOCAL_WRITE |
2395				 IB_ACCESS_REMOTE_WRITE |
2396				 IB_ACCESS_REMOTE_READ |
2397				 IB_ACCESS_RELAXED_ORDERING);
2398	if (err)
2399		return err;
2400
2401	obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2402	if (!obj)
2403		return -ENOMEM;
2404
2405	err = devx_umem_get(dev, &c->ibucontext, attrs, obj, access_flags);
2406	if (err)
2407		goto err_obj_free;
2408
2409	err = devx_umem_reg_cmd_alloc(dev, attrs, obj, &cmd, access_flags);
2410	if (err)
2411		goto err_umem_release;
2412
2413	MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
2414	err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2415			    sizeof(cmd.out));
2416	if (err)
2417		goto err_umem_release;
2418
2419	obj->mdev = dev->mdev;
2420	uobj->object = obj;
2421	devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
2422	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
2423
2424	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id,
2425			     sizeof(obj_id));
2426	return err;
2427
2428err_umem_release:
2429	ib_umem_release(obj->umem);
2430err_obj_free:
2431	kfree(obj);
2432	return err;
2433}
2434
2435static int devx_umem_cleanup(struct ib_uobject *uobject,
2436			     enum rdma_remove_reason why,
2437			     struct uverbs_attr_bundle *attrs)
2438{
2439	struct devx_umem *obj = uobject->object;
2440	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2441	int err;
2442
2443	err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
2444	if (err)
2445		return err;
2446
2447	ib_umem_release(obj->umem);
2448	kfree(obj);
2449	return 0;
2450}
2451
2452static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2453				  unsigned long event_type)
2454{
2455	__be64 *unaff_events;
2456	int mask_entry;
2457	int mask_bit;
2458
2459	if (!MLX5_CAP_GEN(dev, event_cap))
2460		return is_legacy_unaffiliated_event_num(event_type);
2461
2462	unaff_events = MLX5_CAP_DEV_EVENT(dev,
2463					  user_unaffiliated_events);
2464	WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2465
2466	mask_entry = event_type / 64;
2467	mask_bit = event_type % 64;
2468
2469	if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2470		return false;
2471
2472	return true;
2473}
2474
2475static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2476{
2477	struct mlx5_eqe *eqe = data;
2478	u32 obj_id = 0;
2479
2480	switch (event_type) {
2481	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2482	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2483	case MLX5_EVENT_TYPE_PATH_MIG:
2484	case MLX5_EVENT_TYPE_COMM_EST:
2485	case MLX5_EVENT_TYPE_SQ_DRAINED:
2486	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2487	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2488	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2489	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2490	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2491		obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2492		break;
2493	case MLX5_EVENT_TYPE_XRQ_ERROR:
2494		obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2495		break;
2496	case MLX5_EVENT_TYPE_DCT_DRAINED:
2497	case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
2498		obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2499		break;
2500	case MLX5_EVENT_TYPE_CQ_ERROR:
2501		obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2502		break;
2503	default:
2504		obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2505		break;
2506	}
2507
2508	return obj_id;
2509}
2510
2511static int deliver_event(struct devx_event_subscription *event_sub,
2512			 const void *data)
2513{
2514	struct devx_async_event_file *ev_file;
2515	struct devx_async_event_data *event_data;
2516	unsigned long flags;
2517
2518	ev_file = event_sub->ev_file;
2519
2520	if (ev_file->omit_data) {
2521		spin_lock_irqsave(&ev_file->lock, flags);
2522		if (!list_empty(&event_sub->event_list) ||
2523		    ev_file->is_destroyed) {
2524			spin_unlock_irqrestore(&ev_file->lock, flags);
2525			return 0;
2526		}
2527
2528		list_add_tail(&event_sub->event_list, &ev_file->event_list);
2529		spin_unlock_irqrestore(&ev_file->lock, flags);
2530		wake_up_interruptible(&ev_file->poll_wait);
2531		return 0;
2532	}
2533
2534	event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2535			     GFP_ATOMIC);
2536	if (!event_data) {
2537		spin_lock_irqsave(&ev_file->lock, flags);
2538		ev_file->is_overflow_err = 1;
2539		spin_unlock_irqrestore(&ev_file->lock, flags);
2540		return -ENOMEM;
2541	}
2542
2543	event_data->hdr.cookie = event_sub->cookie;
2544	memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2545
2546	spin_lock_irqsave(&ev_file->lock, flags);
2547	if (!ev_file->is_destroyed)
2548		list_add_tail(&event_data->list, &ev_file->event_list);
2549	else
2550		kfree(event_data);
2551	spin_unlock_irqrestore(&ev_file->lock, flags);
2552	wake_up_interruptible(&ev_file->poll_wait);
2553
2554	return 0;
2555}
2556
2557static void dispatch_event_fd(struct list_head *fd_list,
2558			      const void *data)
2559{
2560	struct devx_event_subscription *item;
2561
2562	list_for_each_entry_rcu(item, fd_list, xa_list) {
2563		if (item->eventfd)
2564			eventfd_signal(item->eventfd);
2565		else
2566			deliver_event(item, data);
2567	}
2568}
2569
2570static int devx_event_notifier(struct notifier_block *nb,
2571			       unsigned long event_type, void *data)
2572{
2573	struct mlx5_devx_event_table *table;
2574	struct mlx5_ib_dev *dev;
2575	struct devx_event *event;
2576	struct devx_obj_event *obj_event;
2577	u16 obj_type = 0;
2578	bool is_unaffiliated;
2579	u32 obj_id;
2580
2581	/* Explicit filtering to kernel events which may occur frequently */
2582	if (event_type == MLX5_EVENT_TYPE_CMD ||
2583	    event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2584		return NOTIFY_OK;
2585
2586	table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2587	dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2588	is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2589
2590	if (!is_unaffiliated)
2591		obj_type = get_event_obj_type(event_type, data);
2592
2593	rcu_read_lock();
2594	event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2595	if (!event) {
2596		rcu_read_unlock();
2597		return NOTIFY_DONE;
2598	}
2599
2600	if (is_unaffiliated) {
2601		dispatch_event_fd(&event->unaffiliated_list, data);
2602		rcu_read_unlock();
2603		return NOTIFY_OK;
2604	}
2605
2606	obj_id = devx_get_obj_id_from_event(event_type, data);
2607	obj_event = xa_load(&event->object_ids, obj_id);
2608	if (!obj_event) {
2609		rcu_read_unlock();
2610		return NOTIFY_DONE;
2611	}
2612
2613	dispatch_event_fd(&obj_event->obj_sub_list, data);
2614
2615	rcu_read_unlock();
2616	return NOTIFY_OK;
2617}
2618
2619int mlx5_ib_devx_init(struct mlx5_ib_dev *dev)
2620{
2621	struct mlx5_devx_event_table *table = &dev->devx_event_table;
2622	int uid;
2623
2624	uid = mlx5_ib_devx_create(dev, false, 0);
2625	if (uid > 0) {
2626		dev->devx_whitelist_uid = uid;
2627		xa_init(&table->event_xa);
2628		mutex_init(&table->event_xa_lock);
2629		MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2630		mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2631	}
2632
2633	return 0;
2634}
2635
2636void mlx5_ib_devx_cleanup(struct mlx5_ib_dev *dev)
2637{
2638	struct mlx5_devx_event_table *table = &dev->devx_event_table;
2639	struct devx_event_subscription *sub, *tmp;
2640	struct devx_event *event;
2641	void *entry;
2642	unsigned long id;
2643
2644	if (dev->devx_whitelist_uid) {
2645		mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
2646		mutex_lock(&dev->devx_event_table.event_xa_lock);
2647		xa_for_each(&table->event_xa, id, entry) {
2648			event = entry;
2649			list_for_each_entry_safe(
2650				sub, tmp, &event->unaffiliated_list, xa_list)
2651				devx_cleanup_subscription(dev, sub);
2652			kfree(entry);
2653		}
2654		mutex_unlock(&dev->devx_event_table.event_xa_lock);
2655		xa_destroy(&table->event_xa);
2656
2657		mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid);
2658	}
2659}
2660
2661static void devx_async_destroy_cb(int status, struct mlx5_async_work *context)
2662{
2663	struct mlx5_async_cmd *devx_out = container_of(context,
2664					  struct mlx5_async_cmd, cb_work);
2665	struct devx_obj *obj = devx_out->uobject->object;
2666
2667	if (!status)
2668		obj->flags |= DEVX_OBJ_FLAGS_HW_FREED;
2669
2670	complete(&devx_out->comp);
2671}
2672
2673static void devx_async_destroy(struct mlx5_ib_dev *dev,
2674			       struct mlx5_async_cmd *cmd)
2675{
2676	init_completion(&cmd->comp);
2677	cmd->err = mlx5_cmd_exec_cb(&dev->async_ctx, cmd->in, cmd->in_size,
2678				    &cmd->out, sizeof(cmd->out),
2679				    devx_async_destroy_cb, &cmd->cb_work);
2680}
2681
2682static void devx_wait_async_destroy(struct mlx5_async_cmd *cmd)
2683{
2684	if (!cmd->err)
2685		wait_for_completion(&cmd->comp);
2686	atomic_set(&cmd->uobject->usecnt, 0);
2687}
2688
2689void mlx5_ib_ufile_hw_cleanup(struct ib_uverbs_file *ufile)
2690{
2691	struct mlx5_async_cmd *async_cmd;
2692	struct ib_ucontext *ucontext = ufile->ucontext;
2693	struct ib_device *device = ucontext->device;
2694	struct mlx5_ib_dev *dev = to_mdev(device);
2695	struct ib_uobject *uobject;
2696	struct devx_obj *obj;
2697	int head = 0;
2698	int tail = 0;
2699
2700	async_cmd = kcalloc(MAX_ASYNC_CMDS, sizeof(*async_cmd), GFP_KERNEL);
2701	if (!async_cmd)
2702		return;
2703
2704	list_for_each_entry(uobject, &ufile->uobjects, list) {
2705		WARN_ON(uverbs_try_lock_object(uobject, UVERBS_LOOKUP_WRITE));
2706
2707		/*
2708		 * Currently we only support QP destruction, if other objects
2709		 * are to be destroyed need to add type synchronization to the
2710		 * cleanup algorithm and handle pre/post FW cleanup for the
2711		 * new types if needed.
2712		 */
2713		if (uobj_get_object_id(uobject) != MLX5_IB_OBJECT_DEVX_OBJ ||
2714		    (get_dec_obj_type(uobject->object, MLX5_EVENT_TYPE_MAX) !=
2715		     MLX5_OBJ_TYPE_QP)) {
2716			atomic_set(&uobject->usecnt, 0);
2717			continue;
2718		}
2719
2720		obj = uobject->object;
2721
2722		async_cmd[tail % MAX_ASYNC_CMDS].in = obj->dinbox;
2723		async_cmd[tail % MAX_ASYNC_CMDS].in_size = obj->dinlen;
2724		async_cmd[tail % MAX_ASYNC_CMDS].uobject = uobject;
2725
2726		devx_async_destroy(dev, &async_cmd[tail % MAX_ASYNC_CMDS]);
2727		tail++;
2728
2729		if (tail - head == MAX_ASYNC_CMDS) {
2730			devx_wait_async_destroy(&async_cmd[head % MAX_ASYNC_CMDS]);
2731			head++;
2732		}
2733	}
2734
2735	while (head != tail) {
2736		devx_wait_async_destroy(&async_cmd[head % MAX_ASYNC_CMDS]);
2737		head++;
2738	}
2739
2740	kfree(async_cmd);
2741}
2742
2743static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2744					 size_t count, loff_t *pos)
2745{
2746	struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2747	struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2748	struct devx_async_data *event;
2749	int ret = 0;
2750	size_t eventsz;
2751
2752	spin_lock_irq(&ev_queue->lock);
2753
2754	while (list_empty(&ev_queue->event_list)) {
2755		spin_unlock_irq(&ev_queue->lock);
2756
2757		if (filp->f_flags & O_NONBLOCK)
2758			return -EAGAIN;
2759
2760		if (wait_event_interruptible(
2761			    ev_queue->poll_wait,
2762			    (!list_empty(&ev_queue->event_list) ||
2763			     ev_queue->is_destroyed))) {
2764			return -ERESTARTSYS;
2765		}
2766
2767		spin_lock_irq(&ev_queue->lock);
2768		if (ev_queue->is_destroyed) {
2769			spin_unlock_irq(&ev_queue->lock);
2770			return -EIO;
2771		}
2772	}
2773
2774	event = list_entry(ev_queue->event_list.next,
2775			   struct devx_async_data, list);
2776	eventsz = event->cmd_out_len +
2777			sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2778
2779	if (eventsz > count) {
2780		spin_unlock_irq(&ev_queue->lock);
2781		return -ENOSPC;
2782	}
2783
2784	list_del(ev_queue->event_list.next);
2785	spin_unlock_irq(&ev_queue->lock);
2786
2787	if (copy_to_user(buf, &event->hdr, eventsz))
2788		ret = -EFAULT;
2789	else
2790		ret = eventsz;
2791
2792	atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2793	kvfree(event);
2794	return ret;
2795}
2796
2797static __poll_t devx_async_cmd_event_poll(struct file *filp,
2798					      struct poll_table_struct *wait)
2799{
2800	struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2801	struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2802	__poll_t pollflags = 0;
2803
2804	poll_wait(filp, &ev_queue->poll_wait, wait);
2805
2806	spin_lock_irq(&ev_queue->lock);
2807	if (ev_queue->is_destroyed)
2808		pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2809	else if (!list_empty(&ev_queue->event_list))
2810		pollflags = EPOLLIN | EPOLLRDNORM;
2811	spin_unlock_irq(&ev_queue->lock);
2812
2813	return pollflags;
2814}
2815
2816static const struct file_operations devx_async_cmd_event_fops = {
2817	.owner	 = THIS_MODULE,
2818	.read	 = devx_async_cmd_event_read,
2819	.poll    = devx_async_cmd_event_poll,
2820	.release = uverbs_uobject_fd_release,
2821};
2822
2823static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2824				     size_t count, loff_t *pos)
2825{
2826	struct devx_async_event_file *ev_file = filp->private_data;
2827	struct devx_event_subscription *event_sub;
2828	struct devx_async_event_data *event;
2829	int ret = 0;
2830	size_t eventsz;
2831	bool omit_data;
2832	void *event_data;
2833
2834	omit_data = ev_file->omit_data;
2835
2836	spin_lock_irq(&ev_file->lock);
2837
2838	if (ev_file->is_overflow_err) {
2839		ev_file->is_overflow_err = 0;
2840		spin_unlock_irq(&ev_file->lock);
2841		return -EOVERFLOW;
2842	}
2843
2844
2845	while (list_empty(&ev_file->event_list)) {
2846		spin_unlock_irq(&ev_file->lock);
2847
2848		if (filp->f_flags & O_NONBLOCK)
2849			return -EAGAIN;
2850
2851		if (wait_event_interruptible(ev_file->poll_wait,
2852			    (!list_empty(&ev_file->event_list) ||
2853			     ev_file->is_destroyed))) {
2854			return -ERESTARTSYS;
2855		}
2856
2857		spin_lock_irq(&ev_file->lock);
2858		if (ev_file->is_destroyed) {
2859			spin_unlock_irq(&ev_file->lock);
2860			return -EIO;
2861		}
2862	}
2863
2864	if (omit_data) {
2865		event_sub = list_first_entry(&ev_file->event_list,
2866					struct devx_event_subscription,
2867					event_list);
2868		eventsz = sizeof(event_sub->cookie);
2869		event_data = &event_sub->cookie;
2870	} else {
2871		event = list_first_entry(&ev_file->event_list,
2872				      struct devx_async_event_data, list);
2873		eventsz = sizeof(struct mlx5_eqe) +
2874			sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2875		event_data = &event->hdr;
2876	}
2877
2878	if (eventsz > count) {
2879		spin_unlock_irq(&ev_file->lock);
2880		return -EINVAL;
2881	}
2882
2883	if (omit_data)
2884		list_del_init(&event_sub->event_list);
2885	else
2886		list_del(&event->list);
2887
2888	spin_unlock_irq(&ev_file->lock);
2889
2890	if (copy_to_user(buf, event_data, eventsz))
2891		/* This points to an application issue, not a kernel concern */
2892		ret = -EFAULT;
2893	else
2894		ret = eventsz;
2895
2896	if (!omit_data)
2897		kfree(event);
2898	return ret;
2899}
2900
2901static __poll_t devx_async_event_poll(struct file *filp,
2902				      struct poll_table_struct *wait)
2903{
2904	struct devx_async_event_file *ev_file = filp->private_data;
2905	__poll_t pollflags = 0;
2906
2907	poll_wait(filp, &ev_file->poll_wait, wait);
2908
2909	spin_lock_irq(&ev_file->lock);
2910	if (ev_file->is_destroyed)
2911		pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2912	else if (!list_empty(&ev_file->event_list))
2913		pollflags = EPOLLIN | EPOLLRDNORM;
2914	spin_unlock_irq(&ev_file->lock);
2915
2916	return pollflags;
2917}
2918
2919static void devx_free_subscription(struct rcu_head *rcu)
2920{
2921	struct devx_event_subscription *event_sub =
2922		container_of(rcu, struct devx_event_subscription, rcu);
2923
2924	if (event_sub->eventfd)
2925		eventfd_ctx_put(event_sub->eventfd);
2926	uverbs_uobject_put(&event_sub->ev_file->uobj);
2927	kfree(event_sub);
2928}
2929
2930static const struct file_operations devx_async_event_fops = {
2931	.owner	 = THIS_MODULE,
2932	.read	 = devx_async_event_read,
2933	.poll    = devx_async_event_poll,
2934	.release = uverbs_uobject_fd_release,
2935};
2936
2937static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2938					      enum rdma_remove_reason why)
2939{
2940	struct devx_async_cmd_event_file *comp_ev_file =
2941		container_of(uobj, struct devx_async_cmd_event_file,
2942			     uobj);
2943	struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2944	struct devx_async_data *entry, *tmp;
2945
2946	spin_lock_irq(&ev_queue->lock);
2947	ev_queue->is_destroyed = 1;
2948	spin_unlock_irq(&ev_queue->lock);
2949	wake_up_interruptible(&ev_queue->poll_wait);
2950
2951	mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2952
2953	spin_lock_irq(&comp_ev_file->ev_queue.lock);
2954	list_for_each_entry_safe(entry, tmp,
2955				 &comp_ev_file->ev_queue.event_list, list) {
2956		list_del(&entry->list);
2957		kvfree(entry);
2958	}
2959	spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2960};
2961
2962static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2963					  enum rdma_remove_reason why)
2964{
2965	struct devx_async_event_file *ev_file =
2966		container_of(uobj, struct devx_async_event_file,
2967			     uobj);
2968	struct devx_event_subscription *event_sub, *event_sub_tmp;
2969	struct mlx5_ib_dev *dev = ev_file->dev;
2970
2971	spin_lock_irq(&ev_file->lock);
2972	ev_file->is_destroyed = 1;
2973
2974	/* free the pending events allocation */
2975	if (ev_file->omit_data) {
2976		struct devx_event_subscription *event_sub, *tmp;
2977
2978		list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2979					 event_list)
2980			list_del_init(&event_sub->event_list);
2981
2982	} else {
2983		struct devx_async_event_data *entry, *tmp;
2984
2985		list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2986					 list) {
2987			list_del(&entry->list);
2988			kfree(entry);
2989		}
2990	}
2991
2992	spin_unlock_irq(&ev_file->lock);
2993	wake_up_interruptible(&ev_file->poll_wait);
2994
2995	mutex_lock(&dev->devx_event_table.event_xa_lock);
2996	/* delete the subscriptions which are related to this FD */
2997	list_for_each_entry_safe(event_sub, event_sub_tmp,
2998				 &ev_file->subscribed_events_list, file_list) {
2999		devx_cleanup_subscription(dev, event_sub);
3000		list_del_rcu(&event_sub->file_list);
3001		/* subscription may not be used by the read API any more */
3002		call_rcu(&event_sub->rcu, devx_free_subscription);
3003	}
3004	mutex_unlock(&dev->devx_event_table.event_xa_lock);
3005
3006	put_device(&dev->ib_dev.dev);
3007};
3008
3009DECLARE_UVERBS_NAMED_METHOD(
3010	MLX5_IB_METHOD_DEVX_UMEM_REG,
3011	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
3012			MLX5_IB_OBJECT_DEVX_UMEM,
3013			UVERBS_ACCESS_NEW,
3014			UA_MANDATORY),
3015	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
3016			   UVERBS_ATTR_TYPE(u64),
3017			   UA_MANDATORY),
3018	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
3019			   UVERBS_ATTR_TYPE(u64),
3020			   UA_MANDATORY),
3021	UVERBS_ATTR_RAW_FD(MLX5_IB_ATTR_DEVX_UMEM_REG_DMABUF_FD,
3022			   UA_OPTIONAL),
3023	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
3024			     enum ib_access_flags),
3025	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_PGSZ_BITMAP,
3026			     u64),
3027	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
3028			    UVERBS_ATTR_TYPE(u32),
3029			    UA_MANDATORY));
3030
3031DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3032	MLX5_IB_METHOD_DEVX_UMEM_DEREG,
3033	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
3034			MLX5_IB_OBJECT_DEVX_UMEM,
3035			UVERBS_ACCESS_DESTROY,
3036			UA_MANDATORY));
3037
3038DECLARE_UVERBS_NAMED_METHOD(
3039	MLX5_IB_METHOD_DEVX_QUERY_EQN,
3040	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
3041			   UVERBS_ATTR_TYPE(u32),
3042			   UA_MANDATORY),
3043	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
3044			    UVERBS_ATTR_TYPE(u32),
3045			    UA_MANDATORY));
3046
3047DECLARE_UVERBS_NAMED_METHOD(
3048	MLX5_IB_METHOD_DEVX_QUERY_UAR,
3049	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
3050			   UVERBS_ATTR_TYPE(u32),
3051			   UA_MANDATORY),
3052	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
3053			    UVERBS_ATTR_TYPE(u32),
3054			    UA_MANDATORY));
3055
3056DECLARE_UVERBS_NAMED_METHOD(
3057	MLX5_IB_METHOD_DEVX_OTHER,
3058	UVERBS_ATTR_PTR_IN(
3059		MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
3060		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3061		UA_MANDATORY,
3062		UA_ALLOC_AND_COPY),
3063	UVERBS_ATTR_PTR_OUT(
3064		MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
3065		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3066		UA_MANDATORY));
3067
3068DECLARE_UVERBS_NAMED_METHOD(
3069	MLX5_IB_METHOD_DEVX_OBJ_CREATE,
3070	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
3071			MLX5_IB_OBJECT_DEVX_OBJ,
3072			UVERBS_ACCESS_NEW,
3073			UA_MANDATORY),
3074	UVERBS_ATTR_PTR_IN(
3075		MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
3076		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3077		UA_MANDATORY,
3078		UA_ALLOC_AND_COPY),
3079	UVERBS_ATTR_PTR_OUT(
3080		MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
3081		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3082		UA_MANDATORY));
3083
3084DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3085	MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
3086	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
3087			MLX5_IB_OBJECT_DEVX_OBJ,
3088			UVERBS_ACCESS_DESTROY,
3089			UA_MANDATORY));
3090
3091DECLARE_UVERBS_NAMED_METHOD(
3092	MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
3093	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
3094			UVERBS_IDR_ANY_OBJECT,
3095			UVERBS_ACCESS_READ,
3096			UA_MANDATORY),
3097	UVERBS_ATTR_PTR_IN(
3098		MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
3099		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3100		UA_MANDATORY,
3101		UA_ALLOC_AND_COPY),
3102	UVERBS_ATTR_PTR_OUT(
3103		MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
3104		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3105		UA_MANDATORY));
3106
3107DECLARE_UVERBS_NAMED_METHOD(
3108	MLX5_IB_METHOD_DEVX_OBJ_QUERY,
3109	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
3110			UVERBS_IDR_ANY_OBJECT,
3111			UVERBS_ACCESS_READ,
3112			UA_MANDATORY),
3113	UVERBS_ATTR_PTR_IN(
3114		MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
3115		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3116		UA_MANDATORY,
3117		UA_ALLOC_AND_COPY),
3118	UVERBS_ATTR_PTR_OUT(
3119		MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
3120		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
3121		UA_MANDATORY));
3122
3123DECLARE_UVERBS_NAMED_METHOD(
3124	MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
3125	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
3126			UVERBS_IDR_ANY_OBJECT,
3127			UVERBS_ACCESS_READ,
3128			UA_MANDATORY),
3129	UVERBS_ATTR_PTR_IN(
3130		MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
3131		UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
3132		UA_MANDATORY,
3133		UA_ALLOC_AND_COPY),
3134	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
3135		u16, UA_MANDATORY),
3136	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
3137		MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3138		UVERBS_ACCESS_READ,
3139		UA_MANDATORY),
3140	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
3141		UVERBS_ATTR_TYPE(u64),
3142		UA_MANDATORY));
3143
3144DECLARE_UVERBS_NAMED_METHOD(
3145	MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
3146	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
3147		MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3148		UVERBS_ACCESS_READ,
3149		UA_MANDATORY),
3150	UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
3151		MLX5_IB_OBJECT_DEVX_OBJ,
3152		UVERBS_ACCESS_READ,
3153		UA_OPTIONAL),
3154	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
3155		UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
3156		UA_MANDATORY,
3157		UA_ALLOC_AND_COPY),
3158	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
3159		UVERBS_ATTR_TYPE(u64),
3160		UA_OPTIONAL),
3161	UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
3162		UVERBS_ATTR_TYPE(u32),
3163		UA_OPTIONAL));
3164
3165DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
3166			      &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
3167			      &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
3168			      &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
3169			      &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
3170
3171DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
3172			    UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
3173			    &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
3174			    &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
3175			    &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
3176			    &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
3177			    &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
3178
3179DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
3180			    UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
3181			    &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
3182			    &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
3183
3184
3185DECLARE_UVERBS_NAMED_METHOD(
3186	MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
3187	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
3188			MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3189			UVERBS_ACCESS_NEW,
3190			UA_MANDATORY));
3191
3192DECLARE_UVERBS_NAMED_OBJECT(
3193	MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3194	UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
3195			     devx_async_cmd_event_destroy_uobj,
3196			     &devx_async_cmd_event_fops, "[devx_async_cmd]",
3197			     O_RDONLY),
3198	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
3199
3200DECLARE_UVERBS_NAMED_METHOD(
3201	MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
3202	UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
3203			MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3204			UVERBS_ACCESS_NEW,
3205			UA_MANDATORY),
3206	UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
3207			enum mlx5_ib_uapi_devx_create_event_channel_flags,
3208			UA_MANDATORY));
3209
3210DECLARE_UVERBS_NAMED_OBJECT(
3211	MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3212	UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
3213			     devx_async_event_destroy_uobj,
3214			     &devx_async_event_fops, "[devx_async_event]",
3215			     O_RDONLY),
3216	&UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
3217
3218static bool devx_is_supported(struct ib_device *device)
3219{
3220	struct mlx5_ib_dev *dev = to_mdev(device);
3221
3222	return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
3223}
3224
3225const struct uapi_definition mlx5_ib_devx_defs[] = {
3226	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3227		MLX5_IB_OBJECT_DEVX,
3228		UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3229	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3230		MLX5_IB_OBJECT_DEVX_OBJ,
3231		UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3232	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3233		MLX5_IB_OBJECT_DEVX_UMEM,
3234		UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3235	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3236		MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
3237		UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3238	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
3239		MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
3240		UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
3241	{},
3242};